import { Job, Queue, QueueEvents, Worker } from "bullmq"; import { fileTypeFromFile } from "file-type"; import { FileTypeResult } from "file-type/core"; import fs from "fs-extra"; import { access, FileHandle, open as fsOpen } from "fs/promises"; import gm from "gm"; import imageThumbnail from "image-thumbnail"; import path from "path"; import { logger } from "../server.js"; import { AssetPaths, FolderPaths } from "./serverInit.js"; //@ts-ignore import simpleThumb from "simple-thumbnail"; const QUEUE_NAME = "thumbnailQueue"; const connectionOpts = { host: "localhost", port: 6379, enableReadyCheck: true, reconnectOnError: (err: Error) => err.message.includes("READONLY") }; const thumbnailQueue = new Queue(QUEUE_NAME, { connection: connectionOpts, defaultJobOptions: { removeOnComplete: true, removeOnFail: true, attempts: 3, backoff: { type: "exponential", delay: 1000 } } }); const thumbnailQueueEvents = new QueueEvents(QUEUE_NAME, { connection: connectionOpts }); const thumbnailWorker = new Worker( QUEUE_NAME, async (job: Job<{ file: string }>) => { const { file } = job.data; logger.debug(`[ThumbnailWorker] Starting thumbnail generation for ${file}`); try { await job.updateProgress(10); const result = await processThumbnail(file, job); await job.updateProgress(100); logger.debug(`[ThumbnailWorker] Completed thumbnail generation for ${file}`); return result; } catch (error) { logger.error(`[ThumbnailWorker] Error generating thumbnail for ${file}:`, error); throw error; } }, { connection: connectionOpts, concurrency: 1 } ); // Worker event listeners for logging thumbnailWorker.on("ready", () => { logger.debug("[ThumbnailWorker] Worker is ready"); }); thumbnailWorker.on("active", (job, prev) => { logger.debug(`[ThumbnailWorker] Job ${job.id} active (previous: ${prev})`); }); thumbnailWorker.on("completed", async (job) => { logger.debug(`[ThumbnailWorker] Job ${job.id} completed`); await job.remove(); logger.debug(`[ThumbnailWorker] Job ${job.id} removed from Redis`); }); thumbnailWorker.on("failed", (job, err) => { logger.error(`[ThumbnailWorker] Job ${job?.id} failed:`, err); }); thumbnailWorker.on("stalled", (jobId) => { logger.error(`[ThumbnailWorker] Job stalled: ${jobId}`); }); thumbnailWorker.on("error", (err) => { logger.error("[ThumbnailWorker] Worker error:", err); }); thumbnailWorker.on("ioredis:close", () => { logger.error("[ThumbnailWorker] Redis connection closed"); }); // Queue event listeners thumbnailQueue.on("waiting", (job) => { logger.debug(`[ThumbnailQueue] Job waiting in queue: ${job.data.file}`); }); thumbnailQueue.on("error", (err) => { logger.error("[ThumbnailQueue] Queue error:", err); }); async function processThumbnail(file: string, job?: Job): Promise { let fileHandle: FileHandle | null = null; let thumbPath = "unknown"; try { await access(file, fs.constants.R_OK); // Wait for stable file size to avoid incomplete thumbnails const size1 = (await fs.stat(file)).size; await new Promise((r) => setTimeout(r, 100)); const size2 = (await fs.stat(file)).size; if (size1 !== size2) { throw new Error("File is still being written to, skipping thumbnail generation"); } fileHandle = await fsOpen(file, "r"); const type: FileTypeResult | undefined = await fileTypeFromFile(file); const baseName = path.parse(path.basename(file)).name; thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`); await fs.ensureDir(path.dirname(thumbPath)); if (await fs.pathExists(thumbPath)) { logger.debug(`[ThumbnailWorker] Using existing thumbnail: ${thumbPath}`); return path.relative(path.dirname(file), thumbPath); } if (!type?.mime) throw new Error("Unknown file type"); await fileHandle.close(); fileHandle = null; if (["application/pdf", "image/heic", "image/heif"].includes(type.mime)) { logger.debug(`[ThumbnailWorker] Generating PDF/HEIC thumbnail for: ${file}`); await generatePdfThumbnail(file, thumbPath); } else if (type.mime.startsWith("video")) { logger.debug(`[ThumbnailWorker] Generating video thumbnail for: ${file}`); await simpleThumb(file, thumbPath, "250x?"); } else { logger.debug(`[ThumbnailWorker] Generating image thumbnail for: ${file}`); const thumbnailBuffer = await imageThumbnail(file, { responseType: "buffer", height: 250, width: 250 }); await fs.writeFile(thumbPath, thumbnailBuffer); } return path.relative(path.dirname(file), thumbPath); } catch (error) { logger.error("[ThumbnailWorker] Error generating thumbnail:", { thumbPath, error, message: (error as Error).message }); return path.relative(path.dirname(file), AssetPaths.File); } finally { if (fileHandle) { try { await fileHandle.close(); } catch (closeError) { logger.error("[ThumbnailWorker] Error closing file handle:", closeError); } } } } async function generatePdfThumbnail(file: string, thumbPath: string): Promise { return new Promise((resolve, reject) => { gm(file + "[0]") // first page only .setFormat("jpg") .resize(250, 250, "!") .quality(75) .write(thumbPath, (err) => (err ? reject(err) : resolve(thumbPath))); }); } /** * Add a thumbnail generation job and wait for result. * Returns the relative path to the thumbnail or default file icon on error. */ export default async function GenerateThumbnail(file: string): Promise { const baseName = path.parse(path.basename(file)).name; const thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`); if (await fs.pathExists(thumbPath)) { logger.debug(`[GenerateThumbnail] Returning existing thumbnail immediately for ${file}`); return path.relative(path.dirname(file), thumbPath); } try { logger.debug(`[GenerateThumbnail] Adding job to queue for ${file}`); const job = await thumbnailQueue.add("generate", { file }); const result = await job.waitUntilFinished(thumbnailQueueEvents); logger.debug(`[GenerateThumbnail] Job completed for ${file}`); return result as string; } catch (error) { logger.error(`[GenerateThumbnail] Job failed for ${file}:`, error); return path.relative(path.dirname(file), AssetPaths.File); } }