bodyshop/server/notifications/queues/appQueue.js

const { Queue, Worker } = require("bullmq");

let addQueue;
let consolidateQueue;

const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
  if (!addQueue || !consolidateQueue) {
    logger.logger.info("Initializing Notifications Queues");

    addQueue = new Queue("notificationsAdd", {
      connection: pubClient,
      prefix: "{BULLMQ}",
      defaultJobOptions: { removeOnComplete: true, removeOnFail: true }
    });

    consolidateQueue = new Queue("notificationsConsolidate", {
      connection: pubClient,
      prefix: "{BULLMQ}",
      defaultJobOptions: { removeOnComplete: true, removeOnFail: true }
    });

    const addWorker = new Worker(
      "notificationsAdd",
      async (job) => {
        const { jobId, key, variables, recipients } = job.data;
        logger.logger.info(`Adding notifications for jobId ${jobId}`);

        const redisKeyPrefix = `app:notifications:${jobId}`;
        const notification = { key, variables, timestamp: Date.now() };

        for (const recipient of recipients) {
          const { user } = recipient;
          const userKey = `${redisKeyPrefix}:${user}`;
          const existingNotifications = await pubClient.get(userKey);
          const notifications = existingNotifications ? JSON.parse(existingNotifications) : [];
          notifications.push(notification);
          await pubClient.set(userKey, JSON.stringify(notifications), "EX", 40);
          logger.logger.debug(`Stored notification for ${user} under ${userKey}: ${JSON.stringify(notifications)}`);
        }

        const consolidateKey = `app:consolidate:${jobId}`;
        const flagSet = await pubClient.setnx(consolidateKey, "pending");
        logger.logger.debug(`Consolidation flag set for jobId ${jobId}: ${flagSet}`);

        if (flagSet) {
          await consolidateQueue.add(
            "consolidate-notifications",
            { jobId, recipients },
            { jobId: `consolidate:${jobId}`, delay: 5000 }
          );
          logger.logger.info(`Scheduled consolidation for jobId ${jobId}`);
          await pubClient.expire(consolidateKey, 300);
        } else {
          logger.logger.debug(`Consolidation already scheduled for jobId ${jobId}`);
        }
      },
      {
        connection: pubClient,
        prefix: "{BULLMQ}",
        concurrency: 5
      }
    );

    const consolidateWorker = new Worker(
      "notificationsConsolidate",
      async (job) => {
        const { jobId, recipients } = job.data;
        logger.logger.info(`Consolidating notifications for jobId ${jobId}`);

        const redisKeyPrefix = `app:notifications:${jobId}`;
        const lockKey = `lock:consolidate:${jobId}`;
        const lockAcquired = await pubClient.set(lockKey, "locked", "NX", "EX", 10);
        logger.logger.debug(`Lock acquisition for jobId ${jobId}: ${lockAcquired}`);

        if (lockAcquired) {
          try {
            const allNotifications = {};
            const uniqueUsers = [...new Set(recipients.map((r) => r.user))];
            logger.logger.debug(`Unique users for jobId ${jobId}: ${uniqueUsers}`);

            for (const user of uniqueUsers) {
              const userKey = `${redisKeyPrefix}:${user}`;
              const notifications = await pubClient.get(userKey);
              logger.logger.debug(`Retrieved notifications for ${user}: ${notifications}`);

              if (notifications) {
                const parsedNotifications = JSON.parse(notifications);
                const userRecipients = recipients.filter((r) => r.user === user);
                for (const { bodyShopId } of userRecipients) {
                  allNotifications[user] = allNotifications[user] || {};
                  allNotifications[user][bodyShopId] = parsedNotifications;
                }
                await pubClient.del(userKey);
                logger.logger.debug(`Deleted Redis key ${userKey}`);
              } else {
                logger.logger.warn(`No notifications found for ${user} under ${userKey}`);
              }
            }

            logger.logger.debug(`Consolidated notifications: ${JSON.stringify(allNotifications)}`);

            for (const [user, bodyShopData] of Object.entries(allNotifications)) {
              const userMapping = await redisHelpers.getUserSocketMapping(user);
              logger.logger.debug(`User socket mapping for ${user}: ${JSON.stringify(userMapping)}`);

              for (const [bodyShopId, notifications] of Object.entries(bodyShopData)) {
                if (userMapping && userMapping[bodyShopId]?.socketIds) {
                  userMapping[bodyShopId].socketIds.forEach((socketId) => {
                    logger.logger.debug(
                      `Emitting to socket ${socketId}: ${JSON.stringify({ jobId, bodyShopId, notifications })}`
                    );
                    ioRedis.to(socketId).emit("notification", {
                      jobId,
                      bodyShopId,
                      notifications
                    });
                  });
                  logger.logger.info(
                    `Sent ${notifications.length} consolidated notifications to ${user} for jobId ${jobId}`
                  );
                } else {
                  logger.logger.warn(`No socket IDs found for ${user} in bodyShopId ${bodyShopId}`);
                }
              }
            }

            await pubClient.del(`app:consolidate:${jobId}`);
          } catch (err) {
            logger.logger.error(`Consolidation error for jobId ${jobId}: ${err.message}`, { error: err });
            throw err; // Re-throw to trigger failed event
          } finally {
            await pubClient.del(lockKey);
          }
        } else {
          logger.logger.info(`Skipped consolidation for jobId ${jobId} - lock held by another worker`);
        }
      },
      {
        connection: pubClient,
        prefix: "{BULLMQ}",
        concurrency: 1,
        limiter: { max: 1, duration: 5000 }
      }
    );

    addWorker.on("completed", (job) => logger.logger.info(`Add job ${job.id} completed`));
    consolidateWorker.on("completed", (job) => logger.logger.info(`Consolidate job ${job.id} completed`));
    addWorker.on("failed", (job, err) =>
      logger.logger.error(`Add job ${job.id} failed: ${err.message}`, { error: err })
    );
    consolidateWorker.on("failed", (job, err) =>
      logger.logger.error(`Consolidate job ${job.id} failed: ${err.message}`, { error: err })
    );

    const shutdown = async () => {
      logger.logger.info("Closing app queue workers...");
      await Promise.all([addWorker.close(), consolidateWorker.close()]);
      logger.logger.info("App queue workers closed");
    };
    process.on("SIGTERM", shutdown);
    process.on("SIGINT", shutdown);
  }

  return addQueue; // Return the add queue for dispatching
};

const getQueue = () => {
  if (!addQueue) throw new Error("Add queue not initialized. Ensure loadAppQueue is called during bootstrap.");
  return addQueue;
};

const dispatchAppsToQueue = async ({ appsToDispatch, logger }) => {
  const appQueue = getQueue();

  for (const app of appsToDispatch) {
    const { jobId, bodyShopId, key, variables, recipients } = app;
    await appQueue.add(
      "add-notification",
      { jobId, bodyShopId, key, variables, recipients },
      { jobId: `${jobId}:${Date.now()}` }
    );
    logger.logger.info(`Added notification to queue for jobId ${jobId} with ${recipients.length} recipients`);
  }
};

module.exports = { loadAppQueue, getQueue, dispatchAppsToQueue };