admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/IO-3096-GlobalNotifications - Checkpoint, App Queue

Browse Source
This commit is contained in:
Dave Richer
2025-02-18 17:37:24 -05:00
parent 00005c881e
commit 2a81517104
4 changed files with 190 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server.js
View File

@@ -195,7 +195,15 @@ const connectToRedisCluster = async () => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
redisCluster.on("ready", () => { redisCluster.on("ready", () => {
logger.log(`Redis cluster connection established.`, "INFO", "redis", "api"); logger.log(`Redis cluster connection established.`, "INFO", "redis", "api");
resolve(redisCluster); if (process.env.NODE_ENV === "development" && process.env?.CLEAR_REDIS_ON_START === "true") {
logger.log("[Development] Flushing Redis Cluster on Service start...", "INFO", "redis", "api");
const master = redisCluster.nodes("master");
Promise.all(master.map((node) => node.flushall())).then(() => {
resolve(redisCluster);
});
} else {
resolve(redisCluster);
}
}); });
redisCluster.on("error", (err) => { redisCluster.on("error", (err) => {

263
server/notifications/queues/appQueue.js
View File

@@ -1,103 +1,57 @@
const { Queue, Worker } = require("bullmq"); const { Queue, Worker } = require("bullmq");
let appQueue; let addQueue;
let consolidateQueue;
const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => { const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
if (!appQueue) { if (!addQueue || !consolidateQueue) {
logger.logger.info("Initializing Notifications App Queue"); logger.logger.info("Initializing Notifications Queues");
appQueue = new Queue("notificationsApp", {
addQueue = new Queue("notificationsAdd", {
connection: pubClient, connection: pubClient,
prefix: "{BULLMQ}" prefix: "{BULLMQ}",
defaultJobOptions: { removeOnComplete: true, removeOnFail: true }
}); });
const worker = new Worker( consolidateQueue = new Queue("notificationsConsolidate", {
"notificationsApp", connection: pubClient,
prefix: "{BULLMQ}",
defaultJobOptions: { removeOnComplete: true, removeOnFail: true }
});
const addWorker = new Worker(
"notificationsAdd",
async (job) => { async (job) => {
const { jobId, bodyShopId, key, variables, recipients } = job.data; const { jobId, key, variables, recipients } = job.data;
logger.logger.info(`Processing app job ${job.id} for jobId ${jobId}`); logger.logger.info(`Adding notifications for jobId ${jobId}`);
const redisKey = `app:notifications:${jobId}`; const redisKeyPrefix = `app:notifications:${jobId}`;
const lastSentKey = `${redisKey}:lastSent`; const notification = { key, variables, timestamp: Date.now() };
const lockKey = `lock:send-notifications:${jobId}`;
const recurringFlagKey = `app:recurring:${jobId}`;
if (job.name === "add-notification") { for (const recipient of recipients) {
const notification = { key, variables, timestamp: Date.now() }; const { user } = recipient;
for (const recipient of recipients) { const userKey = `${redisKeyPrefix}:${user}`;
const { user } = recipient; const existingNotifications = await pubClient.get(userKey);
const userKey = `${redisKey}:${user}`; const notifications = existingNotifications ? JSON.parse(existingNotifications) : [];
const existingNotifications = await pubClient.get(userKey); notifications.push(notification);
const notifications = existingNotifications ? JSON.parse(existingNotifications) : []; await pubClient.set(userKey, JSON.stringify(notifications), "EX", 40);
notifications.push(notification); logger.logger.debug(`Stored notification for ${user} under ${userKey}: ${JSON.stringify(notifications)}`);
await pubClient.set(userKey, JSON.stringify(notifications), "EX", 40); }
}
} else if (job.name === "send-notifications") {
let hasNewNotifications = false;
const lastSent = parseInt((await pubClient.get(lastSentKey)) || "0", 10);
for (const recipient of recipients) { const consolidateKey = `app:consolidate:${jobId}`;
const { user, bodyShopId: recipientBodyShopId } = recipient; const flagSet = await pubClient.setnx(consolidateKey, "pending");
const userKey = `${redisKey}:${user}`; logger.logger.debug(`Consolidation flag set for jobId ${jobId}: ${flagSet}`);
const notifications = await pubClient.get(userKey);
if (notifications) {
const parsedNotifications = JSON.parse(notifications);
const newNotifications = parsedNotifications.filter((n) => n.timestamp > lastSent);
if (newNotifications.length > 0) {
hasNewNotifications = true;
const socketIds = await redisHelpers.getUserSocketMapping(user);
if (socketIds && socketIds[bodyShopId]?.socketIds) {
socketIds[bodyShopId].socketIds.forEach((socketId) => {
ioRedis.to(socketId).emit("notification", {
jobId,
bodyShopId: recipientBodyShopId,
notifications: newNotifications
});
});
logger.logger.info(`Sent ${newNotifications.length} new notifications to ${user} for jobId ${jobId}`);
} else {
logger.logger.warn(`No socket IDs found for ${user} in bodyShopId ${bodyShopId}`);
}
await pubClient.del(userKey);
}
}
}
if (hasNewNotifications) { if (flagSet) {
await pubClient.set(lastSentKey, Date.now(), "EX", 300); await consolidateQueue.add(
} else { "consolidate-notifications",
const activeJobs = await appQueue.getActive(); { jobId, recipients },
const hasPendingAdds = activeJobs.some((j) => j.name === "add-notification" && j.data.jobId === jobId); { jobId: `consolidate:${jobId}`, delay: 5000 }
if (!hasPendingAdds) { );
const lockAcquired = await pubClient.set(lockKey, "locked", "NX", "EX", 10); logger.logger.info(`Scheduled consolidation for jobId ${jobId}`);
if (lockAcquired) { await pubClient.expire(consolidateKey, 300);
const recurringJobKey = `send-notifications:${jobId}`; } else {
const repeatableJobs = await appQueue.getRepeatableJobs(); logger.logger.debug(`Consolidation already scheduled for jobId ${jobId}`);
const jobExists = repeatableJobs.some((j) => j.key === recurringJobKey);
if (jobExists) {
await appQueue.removeRepeatableByKey(recurringJobKey);
// Drain all remaining send-notifications jobs for this jobId
await appQueue.drain(false); // false to not force removal of active jobs
logger.logger.info(
`Successfully removed recurring send-notifications job and drained queue for jobId ${jobId} with key ${recurringJobKey}`
);
} else {
logger.logger.info(
`No recurring send-notifications job found for jobId ${jobId} with key ${recurringJobKey} - processing leftover scheduled instance`
);
}
await pubClient.del(lockKey);
await pubClient.del(recurringFlagKey);
} else {
logger.logger.info(
`Skipped removal of send-notifications for jobId ${jobId} - lock held by another worker`
);
}
} else {
logger.logger.info(
`Skipping removal of send-notifications for jobId ${jobId} - pending add-notification jobs exist`
);
}
}
} }
}, },
{ {
@@ -107,61 +61,112 @@ const loadAppQueue = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
} }
); );
worker.on("completed", async (job) => { const consolidateWorker = new Worker(
if (job.name === "add-notification") { "notificationsConsolidate",
const { jobId } = job.data; async (job) => {
const recurringJobKey = `send-notifications:${jobId}`; const { jobId, recipients } = job.data;
const recurringFlagKey = `app:recurring:${jobId}`; logger.logger.info(`Consolidating notifications for jobId ${jobId}`);
const flagSet = await pubClient.setnx(recurringFlagKey, "active");
if (flagSet) { const redisKeyPrefix = `app:notifications:${jobId}`;
const existingJobs = await appQueue.getRepeatableJobs(); const lockKey = `lock:consolidate:${jobId}`;
if (!existingJobs.some((j) => j.key === recurringJobKey)) { const lockAcquired = await pubClient.set(lockKey, "locked", "NX", "EX", 10);
await appQueue.add( logger.logger.debug(`Lock acquisition for jobId ${jobId}: ${lockAcquired}`);
"send-notifications",
{ jobId, bodyShopId: job.data.bodyShopId, recipients: job.data.recipients }, if (lockAcquired) {
{ try {
repeat: { const allNotifications = {};
every: 30 * 1000, const uniqueUsers = [...new Set(recipients.map((r) => r.user))];
limit: 10 logger.logger.debug(`Unique users for jobId ${jobId}: ${uniqueUsers}`);
},
jobId: recurringJobKey for (const user of uniqueUsers) {
const userKey = `${redisKeyPrefix}:${user}`;
const notifications = await pubClient.get(userKey);
logger.logger.debug(`Retrieved notifications for ${user}: ${notifications}`);
if (notifications) {
const parsedNotifications = JSON.parse(notifications);
const userRecipients = recipients.filter((r) => r.user === user);
for (const { bodyShopId } of userRecipients) {
allNotifications[user] = allNotifications[user] || {};
allNotifications[user][bodyShopId] = parsedNotifications;
}
await pubClient.del(userKey);
logger.logger.debug(`Deleted Redis key ${userKey}`);
} else {
logger.logger.warn(`No notifications found for ${user} under ${userKey}`);
} }
); }
logger.logger.info(`Scheduled 30s notification send for jobId ${jobId} with key ${recurringJobKey}`);
await pubClient.expire(recurringFlagKey, 300); logger.logger.debug(`Consolidated notifications: ${JSON.stringify(allNotifications)}`);
for (const [user, bodyShopData] of Object.entries(allNotifications)) {
const userMapping = await redisHelpers.getUserSocketMapping(user);
logger.logger.debug(`User socket mapping for ${user}: ${JSON.stringify(userMapping)}`);
for (const [bodyShopId, notifications] of Object.entries(bodyShopData)) {
if (userMapping && userMapping[bodyShopId]?.socketIds) {
userMapping[bodyShopId].socketIds.forEach((socketId) => {
logger.logger.debug(
`Emitting to socket ${socketId}: ${JSON.stringify({ jobId, bodyShopId, notifications })}`
);
ioRedis.to(socketId).emit("notification", {
jobId,
bodyShopId,
notifications
});
});
logger.logger.info(
`Sent ${notifications.length} consolidated notifications to ${user} for jobId ${jobId}`
);
} else {
logger.logger.warn(`No socket IDs found for ${user} in bodyShopId ${bodyShopId}`);
}
}
}
await pubClient.del(`app:consolidate:${jobId}`);
} catch (err) {
logger.logger.error(`Consolidation error for jobId ${jobId}: ${err.message}`, { error: err });
throw err; // Re-throw to trigger failed event
} finally {
await pubClient.del(lockKey);
} }
} else {
logger.logger.info(`Skipped consolidation for jobId ${jobId} - lock held by another worker`);
} }
},
{
connection: pubClient,
prefix: "{BULLMQ}",
concurrency: 1,
limiter: { max: 1, duration: 5000 }
} }
logger.logger.info(`Job ${job.id} completed`); );
});
worker.on("failed", (job, err) => { addWorker.on("completed", (job) => logger.logger.info(`Add job ${job.id} completed`));
logger.logger.error(`Job ${job.id} failed: ${err.message}`, { error: err }); consolidateWorker.on("completed", (job) => logger.logger.info(`Consolidate job ${job.id} completed`));
}); addWorker.on("failed", (job, err) =>
logger.logger.error(`Add job ${job.id} failed: ${err.message}`, { error: err })
worker.on("error", (err) => { );
logger.logger.error("Worker error:", { error: err }); consolidateWorker.on("failed", (job, err) =>
}); logger.logger.error(`Consolidate job ${job.id} failed: ${err.message}`, { error: err })
);
const shutdown = async () => { const shutdown = async () => {
if (worker) { logger.logger.info("Closing app queue workers...");
logger.logger.info("Closing app queue worker..."); await Promise.all([addWorker.close(), consolidateWorker.close()]);
await worker.close(); logger.logger.info("App queue workers closed");
logger.logger.info("App queue worker closed");
}
}; };
process.on("SIGTERM", shutdown); process.on("SIGTERM", shutdown);
process.on("SIGINT", shutdown); process.on("SIGINT", shutdown);
} }
return appQueue; return addQueue; // Return the add queue for dispatching
}; };
const getQueue = () => { const getQueue = () => {
if (!appQueue) { if (!addQueue) throw new Error("Add queue not initialized. Ensure loadAppQueue is called during bootstrap.");
throw new Error("App queue not initialized. Ensure loadAppQueue is called during bootstrap."); return addQueue;
}
return appQueue;
}; };
const dispatchAppsToQueue = async ({ appsToDispatch, logger }) => { const dispatchAppsToQueue = async ({ appsToDispatch, logger }) => {
@@ -174,7 +179,7 @@ const dispatchAppsToQueue = async ({ appsToDispatch, logger }) => {
{ jobId, bodyShopId, key, variables, recipients }, { jobId, bodyShopId, key, variables, recipients },
{ jobId: `${jobId}:${Date.now()}` } { jobId: `${jobId}:${Date.now()}` }
); );
logger.logger.info(`Added app notification to queue for jobId ${jobId} with ${recipients.length} recipients`); logger.logger.info(`Added notification to queue for jobId ${jobId} with ${recipients.length} recipients`);
} }
}; };

86
server/utils/redisHelpers.js
View File

@@ -136,75 +136,75 @@ const applyRedisHelpers = ({ pubClient, app, logger }) => {
const addUserSocketMapping = async (email, socketId, bodyshopId) => { const addUserSocketMapping = async (email, socketId, bodyshopId) => {
const userKey = `user:${email}`; const userKey = `user:${email}`;
const bodyshopKey = `${userKey}:bodyshops:${bodyshopId}`; const socketMappingKey = `${userKey}:socketMapping`;
try { try {
logger.log(`Adding socket ${socketId} to user ${email} for bodyshop ${bodyshopId}`, "debug", "redis"); logger.log(`Adding socket ${socketId} to user ${email} for bodyshop ${bodyshopId}`, "debug", "redis");
// Mark the bodyshop as associated with the user in the hash // Save the mapping: socketId -> bodyshopId
await pubClient.hset(userKey, `bodyshops:${bodyshopId}`, "1"); await pubClient.hset(socketMappingKey, socketId, bodyshopId);
// Add the socket ID to the bodyshop-specific set // Set TTL (24 hours) for the mapping hash
await pubClient.sadd(bodyshopKey, socketId); await pubClient.expire(socketMappingKey, 86400);
// Set TTL to 24 hours for both keys
await pubClient.expire(userKey, 86400);
await pubClient.expire(bodyshopKey, 86400);
} catch (error) { } catch (error) {
logger.log(`Error adding socket mapping for ${email} (bodyshop ${bodyshopId}): ${error}`, "ERROR", "redis"); logger.log(`Error adding socket mapping for ${email} (bodyshop ${bodyshopId}): ${error}`, "ERROR", "redis");
} }
}; };
const refreshUserSocketTTL = async (email, bodyshopId) => { const refreshUserSocketTTL = async (email) => {
const userKey = `user:${email}`; const userKey = `user:${email}`;
const bodyshopKey = `${userKey}:bodyshops:${bodyshopId}`; const socketMappingKey = `${userKey}:socketMapping`;
try { try {
const userExists = await pubClient.exists(userKey); const exists = await pubClient.exists(socketMappingKey);
if (userExists) { if (exists) {
await pubClient.expire(userKey, 86400); await pubClient.expire(socketMappingKey, 86400);
} logger.log(`Refreshed TTL for ${email} socket mapping`, "debug", "redis");
const bodyshopExists = await pubClient.exists(bodyshopKey);
if (bodyshopExists) {
await pubClient.expire(bodyshopKey, 86400);
logger.log(`Refreshed TTL for ${email} bodyshop ${bodyshopId} socket mapping`, "debug", "redis");
} }
} catch (error) { } catch (error) {
logger.log(`Error refreshing TTL for ${email} (bodyshop ${bodyshopId}): ${error}`, "ERROR", "redis"); logger.log(`Error refreshing TTL for ${email}: ${error}`, "ERROR", "redis");
} }
}; };
const removeUserSocketMapping = async (email, socketId, bodyshopId) => { const removeUserSocketMapping = async (email, socketId) => {
const userKey = `user:${email}`; const userKey = `user:${email}`;
const bodyshopKey = `${userKey}:bodyshops:${bodyshopId}`; const socketMappingKey = `${userKey}:socketMapping`;
try { try {
logger.log(`Removing socket ${socketId} from user ${email} for bodyshop ${bodyshopId}`, "DEBUG", "redis"); logger.log(`Removing socket ${socketId} mapping for user ${email}`, "DEBUG", "redis");
await pubClient.srem(bodyshopKey, socketId); // Look up the bodyshopId associated with this socket
// Refresh TTL if there are still sockets, or let it expire const bodyshopId = await pubClient.hget(socketMappingKey, socketId);
const remainingSockets = await pubClient.scard(bodyshopKey); if (!bodyshopId) {
if (remainingSockets > 0) { logger.log(`Socket ${socketId} not found for user ${email}`, "DEBUG", "redis");
await pubClient.expire(bodyshopKey, 86400); return;
} else {
// Optionally remove the bodyshop field from the hash if no sockets remain
await pubClient.hdel(userKey, `bodyshops:${bodyshopId}`);
} }
// Refresh user key TTL if there are still bodyshops // Remove the socket mapping
const remainingBodyshops = await pubClient.hlen(userKey); await pubClient.hdel(socketMappingKey, socketId);
if (remainingBodyshops > 0) { logger.log(
await pubClient.expire(userKey, 86400); `Removed socket ${socketId} (associated with bodyshop ${bodyshopId}) for user ${email}`,
"DEBUG",
"redis"
);
// Refresh TTL if any socket mappings remain
const remainingSockets = await pubClient.hlen(socketMappingKey);
if (remainingSockets > 0) {
await pubClient.expire(socketMappingKey, 86400);
} }
} catch (error) { } catch (error) {
logger.log(`Error removing socket mapping for ${email} (bodyshop ${bodyshopId}): ${error}`, "ERROR", "redis"); logger.log(`Error removing socket mapping for ${email}: ${error}`, "ERROR", "redis");
} }
}; };
const getUserSocketMapping = async (email) => { const getUserSocketMapping = async (email) => {
const userKey = `user:${email}`; const userKey = `user:${email}`;
const socketMappingKey = `${userKey}:socketMapping`;
try { try {
// Get all bodyshop fields from the hash // Retrieve all socket mappings for the user
const bodyshops = await pubClient.hkeys(userKey); const mapping = await pubClient.hgetall(socketMappingKey);
const ttl = await pubClient.ttl(socketMappingKey);
// Group socket IDs by bodyshopId
const result = {}; const result = {};
for (const bodyshopField of bodyshops) { for (const [socketId, bodyshopId] of Object.entries(mapping)) {
const bodyshopId = bodyshopField.split("bodyshops:")[1]; if (!result[bodyshopId]) {
const bodyshopKey = `${userKey}:bodyshops:${bodyshopId}`; result[bodyshopId] = { socketIds: [], ttl };
const socketIds = await pubClient.smembers(bodyshopKey); }
const ttl = await pubClient.ttl(bodyshopKey); result[bodyshopId].socketIds.push(socketId);
result[bodyshopId] = { socketIds, ttl };
} }
return result; return result;
} catch (error) { } catch (error) {

20
server/web-sockets/redisSocketEvents.js
View File

@@ -2,14 +2,7 @@ const { admin } = require("../firebase/firebase-handler");
const redisSocketEvents = ({ const redisSocketEvents = ({
io, io,
redisHelpers: { redisHelpers: { addUserSocketMapping, removeUserSocketMapping, refreshUserSocketTTL },
addUserSocketMapping,
removeUserSocketMapping,
setSessionData,
getSessionData,
clearSessionData,
refreshUserSocketTTL
},
ioHelpers: { getBodyshopRoom, getBodyshopConversationRoom }, ioHelpers: { getBodyshopRoom, getBodyshopConversationRoom },
logger logger
}) => { }) => {
@@ -30,7 +23,6 @@ const redisSocketEvents = ({
try { try {
const user = await admin.auth().verifyIdToken(token); const user = await admin.auth().verifyIdToken(token);
socket.user = user; socket.user = user;
await setSessionData(socket.id, "user", { ...user, bodyshopId });
await addUserSocketMapping(user.email, socket.id, bodyshopId); await addUserSocketMapping(user.email, socket.id, bodyshopId);
next(); next();
} catch (error) { } catch (error) {
@@ -60,7 +52,6 @@ const redisSocketEvents = ({
return; return;
} }
socket.user = user; socket.user = user;
await setSessionData(socket.id, "user", { ...user, bodyshopId });
await refreshUserSocketTTL(user.email, bodyshopId); await refreshUserSocketTTL(user.email, bodyshopId);
createLogEvent( createLogEvent(
socket, socket,
@@ -124,18 +115,15 @@ const redisSocketEvents = ({
const registerDisconnectEvents = (socket) => { const registerDisconnectEvents = (socket) => {
const disconnect = async () => { const disconnect = async () => {
if (socket.user?.email) { if (socket.user?.email) {
const userData = await getSessionData(socket.id, "user"); await removeUserSocketMapping(socket.user.email, socket.id);
const bodyshopId = userData?.bodyshopId;
if (bodyshopId) {
await removeUserSocketMapping(socket.user.email, socket.id, bodyshopId);
}
await clearSessionData(socket.id);
} }
// Leave all rooms except the default room (socket.id)
const rooms = Array.from(socket.rooms).filter((room) => room !== socket.id); const rooms = Array.from(socket.rooms).filter((room) => room !== socket.id);
for (const room of rooms) { for (const room of rooms) {
socket.leave(room); socket.leave(room);
} }
}; };
socket.on("disconnect", disconnect); socket.on("disconnect", disconnect);
}; };