Move queue to redis to prevent multiple firings.
This commit is contained in:
Patrick Fic
@@ -1,60 +1,172 @@
|
||||
import * as cron from "node-cron";
|
||||
import { Queue, Worker, QueueEvents } from "bullmq";
|
||||
import { logger } from "../server.js";
|
||||
import { S3Sync, createS3SyncFromEnv } from "./s3Sync.js";
|
||||
import { S3Sync, createS3SyncFromEnv, analyzeJobsDirectory } from "./s3Sync.js";
|
||||
|
||||
const SCHEDULER_QUEUE_NAME = "scheduledTasksQueue";
|
||||
|
||||
const connectionOpts = {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
enableReadyCheck: true,
|
||||
reconnectOnError: (err: Error) => err.message.includes("READONLY")
|
||||
};
|
||||
|
||||
export interface ScheduledTaskInfo {
|
||||
name: string;
|
||||
nextRun: Date | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export class DailyS3Scheduler {
|
||||
private s3Sync: S3Sync | null = null;
|
||||
private cronJob: cron.ScheduledTask | null = null;
|
||||
private schedulerQueue: Queue | null = null;
|
||||
private schedulerWorker: Worker | null = null;
|
||||
private queueEvents: QueueEvents | null = null;
|
||||
private isInitialized: boolean = false;
|
||||
|
||||
constructor() {
|
||||
this.s3Sync = createS3SyncFromEnv();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the daily S3 sync scheduler
|
||||
* Runs at midnight PST (00:00 PST = 08:00 UTC during standard time, 07:00 UTC during daylight time)
|
||||
* Start the daily S3 sync scheduler using BullMQ
|
||||
* This ensures only one worker executes the job across all cluster instances
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (!this.s3Sync) {
|
||||
logger.warn("S3 sync not configured. Skipping scheduler setup.");
|
||||
if (this.isInitialized) {
|
||||
logger.warn("Scheduler already initialized");
|
||||
return;
|
||||
}
|
||||
|
||||
// Test S3 connection before starting scheduler
|
||||
// const connectionTest = await this.s3Sync.testConnection();
|
||||
// if (!connectionTest) {
|
||||
// logger.error("S3 connection test failed. S3 sync scheduler will not be started.");
|
||||
// return;
|
||||
// }
|
||||
try {
|
||||
// Initialize the queue
|
||||
this.schedulerQueue = new Queue(SCHEDULER_QUEUE_NAME, {
|
||||
connection: connectionOpts,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10, // Keep last 10 completed jobs
|
||||
removeOnFail: 50, // Keep last 50 failed jobs
|
||||
attempts: 3,
|
||||
backoff: { type: "exponential", delay: 2000 }
|
||||
}
|
||||
});
|
||||
|
||||
// Cron expression for midnight PST
|
||||
// Note: This uses PST timezone. During PDT (daylight time), it will still run at midnight local time
|
||||
const cronExpression = "0 6 * * *"; // Every day at midnight
|
||||
const timezone = "America/Los_Angeles"; // PST/PDT timezone
|
||||
// Initialize queue events for monitoring
|
||||
this.queueEvents = new QueueEvents(SCHEDULER_QUEUE_NAME, {
|
||||
connection: connectionOpts
|
||||
});
|
||||
|
||||
this.cronJob = cron.schedule(
|
||||
cronExpression,
|
||||
async () => {
|
||||
//await this.performDailySync();
|
||||
await this.triggerJobAnalysis();
|
||||
},
|
||||
{
|
||||
timezone: timezone
|
||||
}
|
||||
);
|
||||
// Create worker to process scheduled jobs
|
||||
this.schedulerWorker = new Worker(
|
||||
SCHEDULER_QUEUE_NAME,
|
||||
async (job) => {
|
||||
logger.info(`[Scheduler] Processing job: ${job.name} (ID: ${job.id})`);
|
||||
|
||||
try {
|
||||
if (job.name === "daily-s3-sync") {
|
||||
await this.performDailySync();
|
||||
} else if (job.name === "daily-analytics") {
|
||||
await this.triggerJobAnalysis();
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[Scheduler] Job ${job.name} failed:`, error);
|
||||
throw error; // Re-throw to trigger retry
|
||||
}
|
||||
},
|
||||
{
|
||||
connection: connectionOpts,
|
||||
concurrency: 1, // Process one job at a time
|
||||
autorun: true
|
||||
}
|
||||
);
|
||||
|
||||
logger.info(`Daily scheduler started. Will run at midnight PST/PDT.`);
|
||||
logger.info(`Next sync scheduled for: ${this.getNextRunTime()}`);
|
||||
// Set up worker event handlers
|
||||
this.schedulerWorker.on("completed", (job) => {
|
||||
logger.info(`[Scheduler] Job ${job.name} completed successfully (ID: ${job.id})`);
|
||||
});
|
||||
|
||||
this.schedulerWorker.on("failed", (job, err) => {
|
||||
logger.error(`[Scheduler] Job ${job?.name} failed (ID: ${job?.id}):`, err);
|
||||
});
|
||||
|
||||
this.schedulerWorker.on("error", (err) => {
|
||||
logger.error("[Scheduler] Worker error:", err);
|
||||
});
|
||||
|
||||
// Add repeatable jobs (cron-like scheduling)
|
||||
// Only one instance across all workers will execute these at the scheduled time
|
||||
|
||||
// Daily S3 sync at 1 AM PST
|
||||
await this.schedulerQueue.add(
|
||||
"daily-s3-sync",
|
||||
{},
|
||||
{
|
||||
repeat: {
|
||||
pattern: "0 1 * * *", // Every day at 1 AM
|
||||
tz: "America/Los_Angeles" // PST/PDT timezone
|
||||
},
|
||||
jobId: "daily-s3-sync" // Use consistent ID to prevent duplicates
|
||||
}
|
||||
);
|
||||
|
||||
// Analytics job - every minute (adjust as needed)
|
||||
await this.schedulerQueue.add(
|
||||
"daily-analytics",
|
||||
{},
|
||||
{
|
||||
repeat: {
|
||||
pattern: "21 * * * *", //9PM
|
||||
tz: "America/Los_Angeles"
|
||||
},
|
||||
jobId: "daily-analytics"
|
||||
}
|
||||
);
|
||||
|
||||
this.isInitialized = true;
|
||||
logger.info(`[Scheduler] Daily scheduler started using BullMQ (Worker ${process.env.NODE_APP_INSTANCE || "N/A"})`);
|
||||
|
||||
// Log next scheduled runs
|
||||
const status = await this.getStatus();
|
||||
status.nextRun.forEach(task => {
|
||||
logger.info(`[Scheduler] ${task.name} - Next run: ${task.nextRun}`);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error("[Scheduler] Failed to start scheduler:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler
|
||||
* Stop the scheduler and clean up resources
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.cronJob) {
|
||||
this.cronJob.stop();
|
||||
this.cronJob = null;
|
||||
logger.info("Daily S3 sync scheduler stopped.");
|
||||
async stop(): Promise<void> {
|
||||
try {
|
||||
if (this.schedulerWorker) {
|
||||
await this.schedulerWorker.close();
|
||||
this.schedulerWorker = null;
|
||||
logger.info("[Scheduler] Worker stopped");
|
||||
}
|
||||
|
||||
if (this.queueEvents) {
|
||||
await this.queueEvents.close();
|
||||
this.queueEvents = null;
|
||||
}
|
||||
|
||||
if (this.schedulerQueue) {
|
||||
// Remove all repeatable jobs
|
||||
const repeatableJobs = await this.schedulerQueue.getRepeatableJobs();
|
||||
for (const job of repeatableJobs) {
|
||||
await this.schedulerQueue.removeRepeatableByKey(job.key);
|
||||
}
|
||||
await this.schedulerQueue.close();
|
||||
this.schedulerQueue = null;
|
||||
logger.info("[Scheduler] Queue stopped and repeatable jobs removed");
|
||||
}
|
||||
|
||||
this.isInitialized = false;
|
||||
} catch (error) {
|
||||
logger.error("[Scheduler] Error stopping scheduler:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,49 +206,37 @@ export class DailyS3Scheduler {
|
||||
}
|
||||
|
||||
async triggerJobAnalysis(): Promise<void> {
|
||||
if (!this.s3Sync) {
|
||||
logger.error("S3 sync not configured");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info("Triggering jobs directory analysis...");
|
||||
try {
|
||||
const analysis = await this.s3Sync.analyzeJobsDirectory();
|
||||
const analysis = await analyzeJobsDirectory();
|
||||
logger.info("Jobs directory analysis completed:", analysis);
|
||||
} catch (error) {
|
||||
logger.error("Jobs directory analysis failed:", error);
|
||||
|
||||
//TODO: implement a logger here to send an email or something.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the next scheduled run time
|
||||
* Get the next scheduled run time for all repeatable jobs
|
||||
*/
|
||||
private getNextRunTime(): string {
|
||||
if (!this.cronJob) {
|
||||
return "Not scheduled";
|
||||
private async getNextRunTime(): Promise<ScheduledTaskInfo[]> {
|
||||
if (!this.schedulerQueue) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Create a date object for midnight PST today
|
||||
const now = new Date();
|
||||
const pstNow = new Date(now.toLocaleString("en-US", { timeZone: "America/Los_Angeles" }));
|
||||
|
||||
// If it's past midnight today, next run is tomorrow at midnight
|
||||
const nextRun = new Date(pstNow);
|
||||
if (pstNow.getHours() > 0 || pstNow.getMinutes() > 0 || pstNow.getSeconds() > 0) {
|
||||
nextRun.setDate(nextRun.getDate() + 1);
|
||||
try {
|
||||
const repeatableJobs = await this.schedulerQueue.getRepeatableJobs();
|
||||
|
||||
return repeatableJobs.map(job => ({
|
||||
name: job.name || job.id || "unknown",
|
||||
nextRun: job.next ? new Date(job.next) : null,
|
||||
status: "scheduled"
|
||||
}));
|
||||
} catch (error) {
|
||||
logger.error("[Scheduler] Failed to get next run times:", error);
|
||||
return [];
|
||||
}
|
||||
nextRun.setHours(0, 0, 0, 0);
|
||||
|
||||
return nextRun.toLocaleString("en-US", {
|
||||
timeZone: "America/Los_Angeles",
|
||||
weekday: "long",
|
||||
year: "numeric",
|
||||
month: "long",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
timeZoneName: "short"
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,7 +245,7 @@ export class DailyS3Scheduler {
|
||||
async getStatus(): Promise<{
|
||||
isConfigured: boolean;
|
||||
isRunning: boolean;
|
||||
nextRun: string;
|
||||
nextRun: ScheduledTaskInfo[];
|
||||
syncStats?: { bucketName: string; region: string; keyPrefix: string; available: boolean };
|
||||
}> {
|
||||
let syncStats;
|
||||
@@ -157,10 +257,12 @@ export class DailyS3Scheduler {
|
||||
}
|
||||
}
|
||||
|
||||
const nextRun = await this.getNextRunTime();
|
||||
|
||||
return {
|
||||
isConfigured: this.s3Sync !== null,
|
||||
isRunning: this.cronJob !== null,
|
||||
nextRun: this.getNextRunTime(),
|
||||
isRunning: this.isInitialized && this.schedulerWorker !== null,
|
||||
nextRun,
|
||||
syncStats
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user