From 614d62fc9c0d9c9cd99c922454bbb09db23513f9 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Fri, 6 Sep 2024 16:16:47 -0700 Subject: [PATCH] Add redis to dockerfile. --- Dockerfile | 7 +++++ jobs/jobsUploadMedia.ts | 2 +- server.ts | 2 +- util/generateThumbnail.ts | 2 ++ util/heicConverter.ts | 57 +++++++++++++++++++++----------------- util/heicQueueProcessor.ts | 2 +- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8aae3ec..09eeba8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,13 @@ COPY . . RUN apt -y update RUN apt install -y wget +RUN apt install -y lsb-release curl gpg +RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg +RUN chmod 644 /usr/share/keyrings/redis-archive-keyring.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/redis.list +RUN apt update +RUN apt install -y redis + # PNG ,JPG ,Tiff & WebP support # Consider adding more support with testing https://gist.github.com/hurricup/e14ae5bc47705fca6b1680e7a1fb6580 RUN apt install -y libjpeg-dev diff --git a/jobs/jobsUploadMedia.ts b/jobs/jobsUploadMedia.ts index 5d930c3..60166d8 100644 --- a/jobs/jobsUploadMedia.ts +++ b/jobs/jobsUploadMedia.ts @@ -45,7 +45,7 @@ export async function jobsUploadMedia(req: Request, res: Response) { //Check if there's a heic in the file set. If so, modify the file set. await ConvertHeicFiles(req.files as Express.Multer.File[]); -return; + logger.debug( "Creating thumbnails for newly uploaded media", (req.files as Express.Multer.File[]).map((f) => f.filename) diff --git a/server.ts b/server.ts index 3a790b3..2c670ed 100644 --- a/server.ts +++ b/server.ts @@ -19,7 +19,7 @@ import { JobsListMedia } from "./jobs/jobsListMedia.js"; import { JobsMoveMedia } from "./jobs/jobsMoveMedia.js"; import { JobMediaUploadMulter, jobsUploadMedia } from "./jobs/jobsUploadMedia.js"; import InitServer, { FolderPaths } from "./util/serverInit.js"; -import ValidateImsToken from "./util/validateToken.js"; +import ValidateImsToken from "./util/validateToken.js"; dotenv.config({ path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) diff --git a/util/generateThumbnail.ts b/util/generateThumbnail.ts index 6439fa9..5964543 100644 --- a/util/generateThumbnail.ts +++ b/util/generateThumbnail.ts @@ -43,6 +43,8 @@ export default async function GenerateThumbnail( await simpleThumb(file, thumbPath, "250x?", { // path: ffmpeg, }); + } else if (type?.mime === "image/heic" || type?.mime === "image/heif") { + await GeneratePdfThumbnail(file, thumbPath); } else { logger.debug("Thumbnail being created for : " + thumbPath); //Ignoring typescript as the interface is lacking a parameter. diff --git a/util/heicConverter.ts b/util/heicConverter.ts index 89ec0dd..d494d40 100644 --- a/util/heicConverter.ts +++ b/util/heicConverter.ts @@ -12,7 +12,7 @@ import { FolderPaths } from "./serverInit.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const HeicQueue = new Queue("HEIC Queue", { connection: { host: "localhost", port: 6379 , } }); +const HeicQueue = new Queue("HEIC Queue", { connection: { host: "localhost", port: 6379 } }); dotenv.config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) @@ -23,8 +23,12 @@ const imageMagick = gm.subClass({ imageMagick: true }); export async function ConvertHeicFiles(files: Express.Multer.File[]) { const validFiles = await filterValidHeicFiles(files); -const jobs = await HeicQueue.addBulk(validFiles.map(file => ({name: file.filename, data: {convertedFileName:generateUniqueHeicFilename(file) , file} }))) -logger.log("debug", `The Jobs Object ${(JSON.stringify(jobs,null,2))}`) + const jobs = await HeicQueue.addBulk( + validFiles.map((file) => ({ + name: file.filename, + data: { convertedFileName: generateUniqueHeicFilename(file), file } + })) + ); // await Promise.all( // validFiles.map(async (file) => { // const convertedFileName = generateUniqueHeicFilename(file); @@ -96,43 +100,46 @@ const HeicWorker = new Worker( file.path = `${file.destination}/${convertedFileName}`; return true; } catch (error) { - logger.log("error", `QUEUE ERROR: Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}`); + logger.log( + "error", + `QUEUE ERROR: Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}` + ); return false; } }, { - connection: { host: "localhost", port: 6379 }, + connection: { host: "localhost", port: 6379 } } ); -HeicQueue.on('waiting', job => { - logger.log("debug", `Job is waiting in queue! ${job.data.convertedFileName}`); -}) -HeicQueue.on('error', error => { - logger.log("error", `Queue Error! ${error}`); -}) +HeicQueue.on("waiting", (job) => { + logger.log("debug", `[BULLMQ] Job is waiting in queue! ${job.data.convertedFileName}`); +}); +HeicQueue.on("error", (error) => { + logger.log("error", `[BULLMQ] Queue Error! ${error}`); +}); HeicWorker.on("ready", () => { - logger.log('debug',`Worker Ready`); + logger.log("debug", `[BULLMQ] Worker Ready`); }); HeicWorker.on("active", (job, prev) => { - logger.log('debug',`Job ${job.id} is now active; previous status was ${prev}`); + logger.log("debug", `[BULLMQ] Job ${job.id} is now active; previous status was ${prev}`); }); HeicWorker.on("completed", (jobId, returnvalue) => { - logger.log('debug',`${jobId.id} has completed and returned ${returnvalue}`); + logger.log("debug", `[BULLMQ] ${jobId.id} has completed and returned ${returnvalue}`); }); HeicWorker.on("failed", (jobId, failedReason) => { - logger.log('error',`${jobId} has failed with reason ${failedReason}`); + logger.log("error", `[BULLMQ] ${jobId} has failed with reason ${failedReason}`); +}); +HeicWorker.on("error", (error) => { + logger.log("error", `[BULLMQ] There was a queue error! ${error}`); +}); +HeicWorker.on("stalled", (error) => { + logger.log("error", `[BULLMQ] There was a worker stall! ${error}`); +}); +HeicWorker.on("ioredis:close", () => { + logger.log("error", `[BULLMQ] Redis connection closed!`); }); -HeicWorker.on('error', error => { - logger.log('error', `There was a queue error! ${error}`) -}) -HeicWorker.on('stalled', error => { - logger.log('error', `There was a worker stall! ${error}`) -}) -HeicWorker.on('ioredis:close', () => { - logger.log('error', `Redis connection closed!`) -}) // const queueEvents = new QueueEvents( "HEIC Queue"); @@ -146,4 +153,4 @@ HeicWorker.on('ioredis:close', () => { // queueEvents.on('progress', (jobId, data) => { // // jobId received a progress event -// }); \ No newline at end of file +// }); diff --git a/util/heicQueueProcessor.ts b/util/heicQueueProcessor.ts index a0685f8..d160f4a 100644 --- a/util/heicQueueProcessor.ts +++ b/util/heicQueueProcessor.ts @@ -27,7 +27,7 @@ // await fs.ensureDir(path.join(file.destination, FolderPaths.ConvertedOriginalSubDir)); // await fs.move(file.path, `${path.join(file.destination, FolderPaths.ConvertedOriginalSubDir)}/${file.filename}`); // } else { -// await fs.unlink(file.path); +// await fs.unlink(file.path); // } // }