feature/IO-3096-GlobalNotifications - Checkpoint, App Queue

server/notifications/queues/appQueue.js

@@ -18,6 +18,8 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
 
         const redisKey = `app:notifications:${jobId}`;
         const lastSentKey = `${redisKey}:lastSent`;
+        const lockKey = `lock:send-notifications:${jobId}`;
+        const recurringFlagKey = `app:recurring:${jobId}`;
 
         if (job.name === "add-notification") {
           const notification = { key, variables, timestamp: Date.now() };
@@ -63,22 +65,31 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
           if (hasNewNotifications) {
             await pubClient.set(lastSentKey, Date.now(), "EX", 300);
           } else {
-            // Only remove if no active "add-notification" jobs are pending
             const activeJobs = await appQueue.getActive();
             const hasPendingAdds = activeJobs.some((j) => j.name === "add-notification" && j.data.jobId === jobId);
             if (!hasPendingAdds) {
-              const recurringJobKey = `send-notifications:${jobId}`;
+              const lockAcquired = await pubClient.set(lockKey, "locked", "NX", "EX", 10);
-              const removed = await appQueue.removeRepeatable("send-notifications", {
+              if (lockAcquired) {
-                every: 30 * 1000,
+                const recurringJobKey = `send-notifications:${jobId}`;
-                jobId: recurringJobKey
+                const repeatableJobs = await appQueue.getRepeatableJobs();
-              });
+                const jobExists = repeatableJobs.some((j) => j.key === recurringJobKey);
-              if (removed) {
+                if (jobExists) {
-                logger.logger.info(
+                  await appQueue.removeRepeatableByKey(recurringJobKey);
-                  `Successfully removed recurring send-notifications job for jobId ${jobId} due to no new notifications`
+                  // Drain all remaining send-notifications jobs for this jobId
-                );
+                  await appQueue.drain(false); // false to not force removal of active jobs
+                  logger.logger.info(
+                    `Successfully removed recurring send-notifications job and drained queue for jobId ${jobId} with key ${recurringJobKey}`
+                  );
+                } else {
+                  logger.logger.info(
+                    `No recurring send-notifications job found for jobId ${jobId} with key ${recurringJobKey} - processing leftover scheduled instance`
+                  );
+                }
+                await pubClient.del(lockKey);
+                await pubClient.del(recurringFlagKey);
               } else {
-                logger.logger.warn(
+                logger.logger.info(
-                  `Failed to remove recurring send-notifications job for jobId ${jobId} - may already be removed`
+                  `Skipped removal of send-notifications for jobId ${jobId} - lock held by another worker`
                 );
               }
             } else {
@@ -100,20 +111,25 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
       if (job.name === "add-notification") {
         const { jobId } = job.data;
         const recurringJobKey = `send-notifications:${jobId}`;
-        const existingJobs = await appQueue.getRepeatableJobs();
+        const recurringFlagKey = `app:recurring:${jobId}`;
-        if (!existingJobs.some((j) => j.key === recurringJobKey)) {
+        const flagSet = await pubClient.setnx(recurringFlagKey, "active");
-          await appQueue.add(
+        if (flagSet) {
-            "send-notifications",
+          const existingJobs = await appQueue.getRepeatableJobs();
-            { jobId, bodyShopId: job.data.bodyShopId, recipients: job.data.recipients },
+          if (!existingJobs.some((j) => j.key === recurringJobKey)) {
-            {
+            await appQueue.add(
-              repeat: {
+              "send-notifications",
-                every: 30 * 1000, // Every 30 seconds
+              { jobId, bodyShopId: job.data.bodyShopId, recipients: job.data.recipients },
-                limit: 10 // 5 minutes
+              {
-              },
+                repeat: {
-              jobId: recurringJobKey
+                  every: 30 * 1000,
-            }
+                  limit: 10
-          );
+                },
-          logger.logger.info(`Scheduled 30s notification send for jobId ${jobId}`);
+                jobId: recurringJobKey
+              }
+            );
+            logger.logger.info(`Scheduled 30s notification send for jobId ${jobId} with key ${recurringJobKey}`);
+            await pubClient.expire(recurringFlagKey, 300);
+          }
         }
       }
       logger.logger.info(`Job ${job.id} completed`);