admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/IO-3096-GlobalNotifications - Checkpoint, App Queue

Browse Source
This commit is contained in:
Dave Richer
2025-02-18 14:29:07 -05:00
parent c1ea8e8a3d
commit 00005c881e
1 changed files with 42 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
server/notifications/queues/appQueue.js
View File

@@ -18,6 +18,8 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
const redisKey = `app:notifications:${jobId}`; const redisKey = `app:notifications:${jobId}`;
const lastSentKey = `${redisKey}:lastSent`; const lastSentKey = `${redisKey}:lastSent`;
const lockKey = `lock:send-notifications:${jobId}`;
const recurringFlagKey = `app:recurring:${jobId}`;
if (job.name === "add-notification") { if (job.name === "add-notification") {
const notification = { key, variables, timestamp: Date.now() }; const notification = { key, variables, timestamp: Date.now() };
@@ -63,22 +65,31 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
if (hasNewNotifications) { if (hasNewNotifications) {
await pubClient.set(lastSentKey, Date.now(), "EX", 300); await pubClient.set(lastSentKey, Date.now(), "EX", 300);
} else { } else {
// Only remove if no active "add-notification" jobs are pending
const activeJobs = await appQueue.getActive(); const activeJobs = await appQueue.getActive();
const hasPendingAdds = activeJobs.some((j) => j.name === "add-notification" && j.data.jobId === jobId); const hasPendingAdds = activeJobs.some((j) => j.name === "add-notification" && j.data.jobId === jobId);
if (!hasPendingAdds) { if (!hasPendingAdds) {
const recurringJobKey = `send-notifications:${jobId}`; const lockAcquired = await pubClient.set(lockKey, "locked", "NX", "EX", 10);
const removed = await appQueue.removeRepeatable("send-notifications", { if (lockAcquired) {
every: 30 * 1000, const recurringJobKey = `send-notifications:${jobId}`;
jobId: recurringJobKey const repeatableJobs = await appQueue.getRepeatableJobs();
}); const jobExists = repeatableJobs.some((j) => j.key === recurringJobKey);
if (removed) { if (jobExists) {
logger.logger.info( await appQueue.removeRepeatableByKey(recurringJobKey);
`Successfully removed recurring send-notifications job for jobId ${jobId} due to no new notifications` // Drain all remaining send-notifications jobs for this jobId
); await appQueue.drain(false); // false to not force removal of active jobs
logger.logger.info(
`Successfully removed recurring send-notifications job and drained queue for jobId ${jobId} with key ${recurringJobKey}`
);
} else {
logger.logger.info(
`No recurring send-notifications job found for jobId ${jobId} with key ${recurringJobKey} - processing leftover scheduled instance`
);
}
await pubClient.del(lockKey);
await pubClient.del(recurringFlagKey);
} else { } else {
logger.logger.warn( logger.logger.info(
`Failed to remove recurring send-notifications job for jobId ${jobId} - may already be removed` `Skipped removal of send-notifications for jobId ${jobId} - lock held by another worker`
); );
} }
} else { } else {
@@ -100,20 +111,25 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
if (job.name === "add-notification") { if (job.name === "add-notification") {
const { jobId } = job.data; const { jobId } = job.data;
const recurringJobKey = `send-notifications:${jobId}`; const recurringJobKey = `send-notifications:${jobId}`;
const existingJobs = await appQueue.getRepeatableJobs(); const recurringFlagKey = `app:recurring:${jobId}`;
if (!existingJobs.some((j) => j.key === recurringJobKey)) { const flagSet = await pubClient.setnx(recurringFlagKey, "active");
await appQueue.add( if (flagSet) {
"send-notifications", const existingJobs = await appQueue.getRepeatableJobs();
{ jobId, bodyShopId: job.data.bodyShopId, recipients: job.data.recipients }, if (!existingJobs.some((j) => j.key === recurringJobKey)) {
{ await appQueue.add(
repeat: { "send-notifications",
every: 30 * 1000, // Every 30 seconds { jobId, bodyShopId: job.data.bodyShopId, recipients: job.data.recipients },
limit: 10 // 5 minutes {
}, repeat: {
jobId: recurringJobKey every: 30 * 1000,
} limit: 10
); },
logger.logger.info(`Scheduled 30s notification send for jobId ${jobId}`); jobId: recurringJobKey
}
);
logger.logger.info(`Scheduled 30s notification send for jobId ${jobId} with key ${recurringJobKey}`);
await pubClient.expire(recurringFlagKey, 300);
}
} }
} }
logger.logger.info(`Job ${job.id} completed`); logger.logger.info(`Job ${job.id} completed`);