From b23e446ed3c3ab3dfed0bd1ec5d2b91788719752 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 23 Jul 2025 17:59:13 -0700 Subject: [PATCH] 1.0.14 Package Updates and Assets fix --- Dockerfile | 1 + jobs/jobsDeleteMedia.ts | 6 +- jobs/jobsDownloadMedia.ts | 2 +- jobs/jobsListMedia.ts | 2 +- jobs/jobsMoveMedia.ts | 8 +- package-lock.json | 18 ++-- package.json | 2 +- server.ts | 2 +- util/generateThumbnail.ts | 62 +++++++++++--- util/heicConverter.ts | 167 ++++++++++++++++++++++++++++++++------ util/serverInit.ts | 1 + util/validateToken.ts | 2 +- 12 files changed, 216 insertions(+), 57 deletions(-) diff --git a/Dockerfile b/Dockerfile index a50104c..aae6cf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ WORKDIR /usr/src/app # Copy built application from builder COPY --from=builder /usr/src/app/dist ./dist +COPY ./assets /assets COPY --from=builder /usr/src/app/node_modules ./node_modules COPY --from=builder /usr/src/app/.env.production ./.env.production COPY --from=builder /usr/src/app/ecosystem.config.cjs ./ecosystem.config.cjs diff --git a/jobs/jobsDeleteMedia.ts b/jobs/jobsDeleteMedia.ts index 8888bed..5259c71 100644 --- a/jobs/jobsDeleteMedia.ts +++ b/jobs/jobsDeleteMedia.ts @@ -126,7 +126,7 @@ async function processDeleteOperation( } } } catch (err) { - logger.warn(`${logPrefix}Failed to delete ${filePath}: ${err}`); + logger.warning(`${logPrefix}Failed to delete ${filePath}: ${err}`); } }; @@ -173,7 +173,7 @@ async function processDeleteOperation( } } } catch (error) { - logger.warn(`Error checking/deleting ConvertedOriginal files for ${mediaFile.path}:`, error); + logger.warning(`Error checking/deleting ConvertedOriginal files for ${mediaFile.path}:`, error); } } catch (error) { logger.error(`${logPrefix}Error in deleteFileWithThumbs for ${mediaFile.path}:`, error); @@ -254,7 +254,7 @@ async function processDeleteOperation( } } } catch (error) { - logger.warn(`[DeleteWorker] Failed to delete vendor copies for ${billFile}:`, error); + logger.warning(`[DeleteWorker] Failed to delete vendor copies for ${billFile}:`, error); } })() ); diff --git a/jobs/jobsDownloadMedia.ts b/jobs/jobsDownloadMedia.ts index 9e2ea52..db541eb 100644 --- a/jobs/jobsDownloadMedia.ts +++ b/jobs/jobsDownloadMedia.ts @@ -38,7 +38,7 @@ export async function jobsDownloadMedia(req: Request, res: Response) { const fileOnDisk: Buffer = await fs.readFile(relativePathFn(jobid, file.name)); zip.file(baseName, fileOnDisk); } catch (err) { - logger.warn(`Could not add file to zip: ${file.name}`, err); + logger.warning(`Could not add file to zip: ${file.name}`, err); } } }) diff --git a/jobs/jobsListMedia.ts b/jobs/jobsListMedia.ts index 4dca971..a5fc4d7 100644 --- a/jobs/jobsListMedia.ts +++ b/jobs/jobsListMedia.ts @@ -88,7 +88,7 @@ async function processExistingFiles(jobid: string): Promise { try { if (!(await fs.pathExists(relativeFilePath))) { - logger.warn(`File no longer exists: ${relativeFilePath}`); + logger.warning(`File no longer exists: ${relativeFilePath}`); continue; } diff --git a/jobs/jobsMoveMedia.ts b/jobs/jobsMoveMedia.ts index 68630cb..ee82891 100644 --- a/jobs/jobsMoveMedia.ts +++ b/jobs/jobsMoveMedia.ts @@ -121,7 +121,7 @@ async function processMoveOperation( overwrite: true }) .then(() => logger.debug(`[MoveWorker] Moved main file: ${file}`)) - .catch((err) => logger.warn(`[MoveWorker] Failed to move main file ${file}:`, err)) + .catch((err) => logger.warning(`[MoveWorker] Failed to move main file ${file}:`, err)) ); // Move thumbnails @@ -163,7 +163,7 @@ async function processMoveOperation( } } } catch (error) { - logger.warn(`[MoveWorker] Failed to move ConvertedOriginal for ${file}:`, error); + logger.warning(`[MoveWorker] Failed to move ConvertedOriginal for ${file}:`, error); } })() ); @@ -179,7 +179,7 @@ async function processMoveOperation( { overwrite: true } ) .then(() => logger.debug(`[MoveWorker] Moved bill file: ${file}`)) - .catch((err) => logger.warn(`[MoveWorker] Failed to move bill file ${file}:`, err)) + .catch((err) => logger.warning(`[MoveWorker] Failed to move bill file ${file}:`, err)) ); // Move bill thumbnails @@ -233,7 +233,7 @@ async function processMoveOperation( } } } catch (error) { - logger.warn(`[MoveWorker] Failed to move bill ConvertedOriginal for ${file}:`, error); + logger.warning(`[MoveWorker] Failed to move bill ConvertedOriginal for ${file}:`, error); } })() ); diff --git a/package-lock.json b/package-lock.json index f0d1e9e..838dd4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@types/compression": "^1.8.1", - "axios": "^1.10.0", + "axios": "^1.11.0", "body-parser": "^2.2.0", "bullmq": "^5.56.5", "compression": "^1.8.1", @@ -908,13 +908,13 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.10.0.tgz", - "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", + "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", + "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, @@ -4218,12 +4218,12 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.10.0.tgz", - "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", + "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", "requires": { "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", + "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, diff --git a/package.json b/package.json index dd849ed..b2a45f5 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@types/compression": "^1.8.1", - "axios": "^1.10.0", + "axios": "^1.11.0", "body-parser": "^2.2.0", "bullmq": "^5.56.5", "compression": "^1.8.1", diff --git a/server.ts b/server.ts index 2c4c933..f049b44 100644 --- a/server.ts +++ b/server.ts @@ -135,7 +135,7 @@ app.get("/health", (req, res) => { // Static files InitServer(); app.use(FolderPaths.StaticPath, express.static(FolderPaths.Root, { etag: false, maxAge: 30 * 1000 })); -app.use("/assets", express.static("./assets", { etag: false, maxAge: 30 * 1000 })); +app.use("/assets", express.static("/assets", { etag: false, maxAge: 30 * 1000 })); app.listen(port, () => { logger.info(`ImEX Media Server is running at http://localhost:${port}`); diff --git a/util/generateThumbnail.ts b/util/generateThumbnail.ts index 8bfab50..9387a4f 100644 --- a/util/generateThumbnail.ts +++ b/util/generateThumbnail.ts @@ -48,8 +48,21 @@ const thumbnailWorker = new Worker( logger.debug(`[ThumbnailWorker] Completed thumbnail generation for ${file}`); return result; } catch (error) { + // Handle VipsJpeg and other thumbnail generation errors gracefully + const errorMessage = (error as Error).message; + if ( + errorMessage.includes("VipsJpeg") || + errorMessage.includes("Premature end") || + errorMessage.includes("JPEG") || + errorMessage.includes("SOS") + ) { + logger.warning(`[ThumbnailWorker] Image processing error for ${file}, returning default icon:`, errorMessage); + return path.relative(path.dirname(file), AssetPaths.File); + } + logger.error(`[ThumbnailWorker] Error generating thumbnail for ${file}:`, error); - throw error; + // For other errors, still return default icon instead of throwing + return path.relative(path.dirname(file), AssetPaths.File); } }, { @@ -126,23 +139,50 @@ async function processThumbnail(file: string, job?: Job): Promise { if (["application/pdf", "image/heic", "image/heif"].includes(type.mime)) { logger.debug(`[ThumbnailWorker] Generating PDF/HEIC thumbnail for: ${file}`); - await generatePdfThumbnail(file, thumbPath); + try { + await generatePdfThumbnail(file, thumbPath); + } catch (pdfError) { + logger.warning(`[ThumbnailWorker] Failed to generate PDF/HEIC thumbnail for ${file}:`, { + error: pdfError, + message: (pdfError as Error).message + }); + // Don't throw, just return the default file icon + return path.relative(path.dirname(file), AssetPaths.File); + } } else if (type.mime.startsWith("video")) { logger.debug(`[ThumbnailWorker] Generating video thumbnail for: ${file}`); - await simpleThumb(file, thumbPath, "250x?"); + try { + await simpleThumb(file, thumbPath, "250x?"); + } catch (videoError) { + logger.warning(`[ThumbnailWorker] Failed to generate video thumbnail for ${file}:`, { + error: videoError, + message: (videoError as Error).message + }); + // Don't throw, just return the default file icon + return path.relative(path.dirname(file), AssetPaths.File); + } } else { logger.debug(`[ThumbnailWorker] Generating image thumbnail for: ${file}`); - const thumbnailBuffer = await imageThumbnail(file, { - responseType: "buffer", - height: 250, - width: 250 - }); - await fs.writeFile(thumbPath, thumbnailBuffer); + try { + const thumbnailBuffer = await imageThumbnail(file, { + responseType: "buffer", + height: 250, + width: 250 + }); + await fs.writeFile(thumbPath, thumbnailBuffer); + } catch (thumbnailError) { + logger.warning(`[ThumbnailWorker] Failed to generate image thumbnail for ${file}:`, { + error: thumbnailError, + message: (thumbnailError as Error).message + }); + // Don't throw, just return the default file icon + return path.relative(path.dirname(file), AssetPaths.File); + } } return path.relative(path.dirname(file), thumbPath); } catch (error) { - logger.error("[ThumbnailWorker] Error generating thumbnail:", { + logger.warning("[ThumbnailWorker] Could not generate thumbnail, returning default file icon:", { thumbPath, error, message: (error as Error).message @@ -189,7 +229,7 @@ export default async function GenerateThumbnail(file: string): Promise { logger.debug(`[GenerateThumbnail] Job completed for ${file}`); return result as string; } catch (error) { - logger.error(`[GenerateThumbnail] Job failed for ${file}:`, error); + logger.warning(`[GenerateThumbnail] Job failed for ${file}, returning default file icon:`, error); return path.relative(path.dirname(file), AssetPaths.File); } } diff --git a/util/heicConverter.ts b/util/heicConverter.ts index f59c1dc..e846a3e 100644 --- a/util/heicConverter.ts +++ b/util/heicConverter.ts @@ -5,7 +5,6 @@ import { FileTypeResult } from "file-type/core"; import fs from "fs-extra"; import gm from "gm"; import path from "path"; -import { fileURLToPath } from "url"; import { logger } from "../server.js"; import { generateUniqueHeicFilename } from "./generateUniqueFilename.js"; import { FolderPaths } from "./serverInit.js"; @@ -81,8 +80,12 @@ async function handleOriginalFile(fileInfo: { path: string; destination: string; await fs.unlink(fileInfo.path); } } catch (error) { - logger.error("Error handling original file:", error); - throw error; + // Don't throw error for file handling issues - log as warning and continue + logger.warning("Error handling original HEIC file (continuing with conversion):", { + file: fileInfo.originalFilename, + path: fileInfo.path, + error: (error as Error).message + }); } } @@ -91,15 +94,38 @@ async function handleOriginalFile(fileInfo: { path: string; destination: string; */ async function convertToJpeg(inputPath: string, outputPath: string): Promise { return new Promise((resolve, reject) => { - const readStream = fs.createReadStream(inputPath); - const writeStream = fs.createWriteStream(outputPath); + try { + const readStream = fs.createReadStream(inputPath); + const writeStream = fs.createWriteStream(outputPath); - gm(readStream) - .setFormat("jpg") - .stream() - .pipe(writeStream) - .on("finish", () => resolve(outputPath)) - .on("error", reject); + // Add error handling for stream creation + readStream.on("error", (err) => { + logger.warning(`Error reading HEIC file ${inputPath}:`, err); + reject(new Error(`Failed to read HEIC file: ${err.message}`)); + }); + + writeStream.on("error", (err) => { + logger.warning(`Error writing converted file ${outputPath}:`, err); + reject(new Error(`Failed to write converted file: ${err.message}`)); + }); + + gm(readStream) + .setFormat("jpg") + .stream() + .on("error", (err) => { + logger.warning(`GraphicsMagick conversion error for ${inputPath}:`, err); + reject(new Error(`GraphicsMagick conversion failed: ${err.message}`)); + }) + .pipe(writeStream) + .on("finish", () => resolve(outputPath)) + .on("error", (err) => { + logger.warning(`Stream pipe error for ${inputPath}:`, err); + reject(new Error(`Stream processing failed: ${err.message}`)); + }); + } catch (error) { + logger.warning(`Unexpected error in convertToJpeg for ${inputPath}:`, error); + reject(new Error(`Conversion setup failed: ${(error as Error).message}`)); + } }); } @@ -127,33 +153,73 @@ export async function convertHeicFiles(files: Express.Multer.File[]) { })); // Add jobs and wait for completion of each before proceeding + const successfulConversions: Array<{ originalFilename: string; convertedFileName: string }> = []; + const failedConversions: Array<{ originalFilename: string; error: string }> = []; + for (const jobData of jobs) { try { const job = await heicQueue.add(jobData.name, jobData.data); - await job.waitUntilFinished(heicQueueEvents); - logger.debug(`Job ${job.id} finished successfully.`); + const result = await job.waitUntilFinished(heicQueueEvents); + + if (result && result.success) { + logger.debug(`Job ${job.id} finished successfully.`); + successfulConversions.push({ + originalFilename: jobData.data.fileInfo.originalFilename, + convertedFileName: jobData.data.convertedFileName + }); + } else { + logger.warning(`Job ${job.id} completed but conversion failed for ${jobData.data.fileInfo.originalFilename}`); + failedConversions.push({ + originalFilename: jobData.data.fileInfo.originalFilename, + error: result?.error || "Unknown conversion error" + }); + } } catch (error) { - logger.error(`Job for ${jobData.data.fileInfo.originalFilename} failed:`, error); - // Depending on your error handling strategy you might rethrow or continue + logger.warning(`Job for ${jobData.data.fileInfo.originalFilename} failed:`, error); + failedConversions.push({ + originalFilename: jobData.data.fileInfo.originalFilename, + error: (error as Error).message + }); + // Continue with next job instead of stopping } } - // Update original files list with new names, mimetype, and path + // Log summary + if (failedConversions.length > 0) { + logger.warning( + `HEIC conversion summary: ${successfulConversions.length} successful, ${failedConversions.length} failed:`, + { + failed: failedConversions.map((f) => f.originalFilename) + } + ); + } else { + logger.debug(`HEIC conversion summary: All ${successfulConversions.length} files converted successfully`); + } + + // Update original files list with new names, mimetype, and path - only for successful conversions const filenameToIndex = new Map(files.map((f, i) => [f.filename, i])); - for (const { data } of jobs) { - const idx = filenameToIndex.get(data.fileInfo.originalFilename); + for (const conversion of successfulConversions) { + const idx = filenameToIndex.get(conversion.originalFilename); if (idx !== undefined) { const oldPath = files[idx].path; - files[idx].filename = data.convertedFileName; + files[idx].filename = conversion.convertedFileName; files[idx].mimetype = "image/jpeg"; - files[idx].path = path.join(data.fileInfo.destination, data.convertedFileName); - logger.debug(`Updated file entry: ${data.fileInfo.originalFilename} -> ${data.convertedFileName}`, { + files[idx].path = path.join(files[idx].destination, conversion.convertedFileName); + logger.debug(`Updated file entry: ${conversion.originalFilename} -> ${conversion.convertedFileName}`, { oldPath, newPath: files[idx].path, newMimetype: files[idx].mimetype }); } } + + // Remove failed conversions from the files array to prevent further processing + for (let i = files.length - 1; i >= 0; i--) { + if (failedConversions.some((f) => f.originalFilename === files[i].filename)) { + logger.debug(`Removing failed HEIC file from processing: ${files[i].filename}`); + files.splice(i, 1); + } + } } // Worker processing HEIC conversion jobs @@ -166,11 +232,12 @@ const heicWorker = new Worker( }> ) => { const { fileInfo, convertedFileName } = job.data; + const outputPath = path.join(fileInfo.destination, convertedFileName); + try { logger.debug(`Converting ${fileInfo.originalFilename} from HEIC to JPEG.`); await job.updateProgress(10); - const outputPath = path.join(fileInfo.destination, convertedFileName); await convertToJpeg(fileInfo.path, outputPath); await job.updateProgress(50); @@ -179,10 +246,60 @@ const heicWorker = new Worker( logger.debug(`Successfully converted ${fileInfo.originalFilename} to JPEG.`); await job.updateProgress(100); - return true; + return { success: true, convertedFileName }; } catch (error) { - logger.error(`Error converting ${fileInfo.originalFilename}:`, error); - throw error; + const errorMessage = (error as Error).message; + // Handle GraphicsMagick and other conversion errors gracefully + if ( + errorMessage.includes("Magick") || + errorMessage.includes("HEIC") || + errorMessage.includes("corrupt") || + errorMessage.includes("invalid") + ) { + logger.warning( + `[heicWorker] HEIC conversion error for ${fileInfo.originalFilename}, moving to damaged folder:`, + errorMessage + ); + + // Clean up both the original HEIC file and any partially created JPEG + try { + // Move original HEIC file to damaged subfolder instead of deleting + const damagedDir = path.join(fileInfo.destination, FolderPaths.DamagedSubDir); + await fs.ensureDir(damagedDir); + const damagedFilePath = path.join(damagedDir, fileInfo.originalFilename); + + if (await fs.pathExists(fileInfo.path)) { + await fs.move(fileInfo.path, damagedFilePath, { overwrite: true }); + logger.debug(`Moved damaged HEIC file to: ${damagedFilePath}`); + } + } catch (cleanupError) { + logger.warning(`Failed to move damaged file for ${fileInfo.originalFilename}:`, cleanupError); + } + + return { success: false, error: errorMessage, originalFilename: fileInfo.originalFilename }; + } + + logger.error(`[heicWorker] Unexpected error converting ${fileInfo.originalFilename}:`, error); + + // For other errors, also clean up any partial files + try { + // Move original HEIC file to damaged subfolder instead of deleting + const damagedDir = path.join(fileInfo.destination, FolderPaths.DamagedSubDir); + await fs.ensureDir(damagedDir); + const damagedFilePath = path.join(damagedDir, fileInfo.originalFilename); + + if (await fs.pathExists(fileInfo.path)) { + await fs.move(fileInfo.path, damagedFilePath, { overwrite: true }); + logger.debug(`Moved damaged HEIC file after unexpected error to: ${damagedFilePath}`); + } + } catch (cleanupError) { + logger.warning( + `Failed to move damaged file after unexpected error for ${fileInfo.originalFilename}:`, + cleanupError + ); + } + + return { success: false, error: errorMessage, originalFilename: fileInfo.originalFilename }; } }, { diff --git a/util/serverInit.ts b/util/serverInit.ts index bb1e0c0..c4366a1 100644 --- a/util/serverInit.ts +++ b/util/serverInit.ts @@ -23,6 +23,7 @@ export const FolderPaths = { ThumbsSubDir: "thumbs", BillsSubDir: "bills", ConvertedOriginalSubDir: "ConvertedOriginal", + DamagedSubDir: "DamagedOriginal", StaticPath: "/static", JobsFolder, VendorsFolder diff --git a/util/validateToken.ts b/util/validateToken.ts index 0613dcb..3712a84 100644 --- a/util/validateToken.ts +++ b/util/validateToken.ts @@ -19,7 +19,7 @@ export default function ValidateImsToken(req: Request, res: Response, next: Next const token = req.headers.ims_token || req.headers["ims-token"] || req.headers["x-ims-token"]; if (token !== IMS_TOKEN) { - logger.warn("Invalid IMS token provided.", { provided: token }); + logger.warning("Invalid IMS token provided.", { provided: token }); res.sendStatus(401); return; }