admin/bodyshop-media-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.0.14

Browse Source
This commit is contained in:
Allan Carr
2025-07-13 00:17:08 -07:00
parent 231130267f
commit 7f782d5a64
19 changed files with 2564 additions and 1416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
Dockerfile
View File

@@ -3,40 +3,7 @@
FROM node:22-alpine AS builder
# Install build dependencies
RUN apk add --no-cache \
bash wget build-base autoconf automake cmake libtool pkgconf \
libjpeg-turbo-dev libpng-dev libwebp-dev tiff-dev libde265-dev \
ruby ruby-dev
# Replace source built libde265 and libheif with installed libraries in next release
# libheif-dev libde265-dev x265-dev
# Build libde265
WORKDIR /build/libde265
RUN wget https://github.com/strukturag/libde265/archive/v1.0.15.tar.gz \
&& tar -xvf v1.0.15.tar.gz \
&& cd libde265-1.0.15 \
&& cmake . \
&& make \
&& make install
# Build libheif
WORKDIR /build/libheif
RUN wget https://github.com/strukturag/libheif/archive/v1.19.7.tar.gz \
&& tar -xvf v1.19.7.tar.gz \
&& cd libheif-1.19.7 \
&& cmake --preset=release . \
&& make \
&& make install
# Build ImageMagick
WORKDIR /build/imagemagick
RUN wget https://download.imagemagick.org/archive/releases/ImageMagick-7.1.1-44.tar.xz \
&& tar -xvf ImageMagick-7.1.1-44.tar.xz \
&& cd ImageMagick-7.1.1-44 \
&& ./configure --with-heic=yes --with-webp=yes \
&& make \
&& make install
RUN apk add --no-cache bash wget
# Node.js application build stage
WORKDIR /usr/src/app
@@ -50,18 +17,12 @@ RUN npm run build
# Final stage
FROM node:22-alpine
# Enable community repository for additional packages
RUN echo "https://dl-cdn.alpinelinux.org/alpine/v$(grep -oE '[0-9]+\.[0-9]+' /etc/alpine-release)/community" >> /etc/apk/repositories
RUN apk update
# Install runtime dependencies only
RUN apk add --no-cache \
bash redis ghostscript graphicsmagick imagemagick \
libjpeg-turbo libpng libwebp tiff
# Copy built libraries from builder
COPY --from=builder /usr/local/lib/ /usr/local/lib/
COPY --from=builder /usr/local/bin/ /usr/local/bin/
COPY --from=builder /usr/local/include/ /usr/local/include/
# Update library cache
RUN ldconfig /usr/local/lib
RUN apk add --no-cache bash redis ghostscript graphicsmagick imagemagick libjpeg-turbo libpng libwebp tiff libheif libde265 x265 ffmpeg
RUN npm install -g pm2

19
bills/billRequestValidator.ts
View File

@@ -1,11 +1,16 @@
import { NextFunction, Request, Response } from "express";
export default function BillRequestValidator(req: Request, res: Response, next: NextFunction) {
const jobid: string = (req.body.jobid || "").trim();
if (jobid === "") {
res.status(400).json({ error: "No RO Number has been specified." });
return;
} else {
const validateBillRequest = (req: Request, res: Response, next: NextFunction) => {
try {
const jobid: string = (req.body.jobid || "").trim();
if (!jobid) {
res.status(400).json({ error: "No RO Number has been specified." });
return;
}
next();
} catch (error) {
res.status(500).json({ error: "Error validating job request.", details: (error as Error).message });
}
}
};
export default validateBillRequest;

203
bills/billsListMedia.ts
View File

@@ -13,82 +13,151 @@ import { FolderPaths } from "../util/serverInit.js";
/** @description Bills will use the hierarchy of PDFs stored under the Job first, and then the Bills folder. */
export async function BillsListMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
//const vendorid: string = (req.body.vendorid || "").trim();
const invoice_number: string = (req.body.invoice_number || "").trim();
let ret: MediaFile[];
//Ensure all directories exist.
await fs.ensureDir(PathToRoBillsFolder(jobid));
if (req.files) {
ret = await Promise.all(
(req.files as Express.Multer.File[]).map(async (file) => {
const relativeFilePath: string = path.join(PathToRoBillsFolder(jobid), file.filename);
try {
if (req.files) {
const uploadedFiles = await processUploadedBillFiles(req.files as Express.Multer.File[], jobid);
if (!res.headersSent) res.json(uploadedFiles);
} else {
const existingFiles = await processExistingBillFiles(jobid, invoice_number);
if (!res.headersSent) res.json(existingFiles);
}
} catch (error) {
// Optionally add logger here if you use one
if (!res.headersSent) res.status(500).json(error);
}
}
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await fileTypeFromFile(relativeFilePath);
async function processUploadedBillFiles(files: Express.Multer.File[], jobid: string): Promise<MediaFile[]> {
const processFile = async (file: Express.Multer.File): Promise<MediaFile> => {
const relativeFilePath: string = path.join(PathToRoBillsFolder(jobid), file.filename);
try {
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await Promise.race([
fileTypeFromFile(relativeFilePath),
new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), 5000))
]);
return {
type,
size: file.size,
src: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
file.filename
]),
thumbnail: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
relativeThumbPath
]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
name: file.filename,
path: relativeFilePath,
thumbnailPath: relativeThumbPath
};
} catch (error) {
// Return basic info if thumbnail/type fails
return {
type: undefined,
size: file.size,
src: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
file.filename
]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, "assets", "file.svg"]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
name: file.filename,
path: relativeFilePath,
thumbnailPath: ""
};
}
};
return (await Promise.all(files.map(processFile))).filter((r): r is MediaFile => r !== null);
}
async function processExistingBillFiles(jobid: string, invoice_number: string): Promise<MediaFile[]> {
let filesList: fs.Dirent[] = (
await fs.readdir(PathToRoBillsFolder(jobid), {
withFileTypes: true
})
).filter(
(f) =>
f.isFile() &&
!/(^|\/)\.[^\/\.]/g.test(f.name) &&
(invoice_number !== "" ? f.name.toLowerCase().includes(invoice_number.toLowerCase()) : true) &&
ListableChecker(f)
);
const processFile = async (file: fs.Dirent): Promise<MediaFile | null> => {
const relativeFilePath: string = path.join(PathToRoBillsFolder(jobid), file.name);
try {
if (!(await fs.pathExists(relativeFilePath))) {
return null;
}
const fileStats = await Promise.race([
fs.stat(relativeFilePath),
new Promise<never>((_, reject) => setTimeout(() => reject(new Error("File stat timeout")), 5000))
]);
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await Promise.race([
fileTypeFromFile(relativeFilePath),
new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), 5000))
]);
return {
type,
size: fileStats.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, FolderPaths.BillsSubDir, file.name]),
thumbnail: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
relativeThumbPath
]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.name,
name: file.name,
path: relativeFilePath,
thumbnailPath: relativeThumbPath
};
} catch (error) {
try {
const fileStats = await fs.stat(relativeFilePath);
return {
type,
size: file.size,
src: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
file.filename
]),
thumbnail: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
relativeThumbPath
]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
relativeFilePath
};
})
);
} else {
let filesList: fs.Dirent[] = (
await fs.readdir(PathToRoBillsFolder(jobid), {
withFileTypes: true
})
).filter(
(f) =>
f.isFile() &&
!/(^|\/)\.[^\/\.]/g.test(f.name) &&
(invoice_number !== "" ? f.name.toLowerCase().includes(invoice_number.toLowerCase()) : true) &&
ListableChecker(f)
);
ret = await Promise.all(
filesList.map(async (file) => {
const relativeFilePath: string = path.join(PathToRoBillsFolder(jobid), file.name);
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await fileTypeFromFile(relativeFilePath);
const fileSize = await fs.stat(relativeFilePath);
return {
type,
size: fileSize.size,
type: undefined,
size: fileStats.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, FolderPaths.BillsSubDir, file.name]),
thumbnail: GenerateUrl([
FolderPaths.StaticPath,
FolderPaths.JobsFolder,
jobid,
FolderPaths.BillsSubDir,
relativeThumbPath
]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, "assets", "file.svg"]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.name,
relativeFilePath
name: file.name,
path: relativeFilePath,
thumbnailPath: ""
};
})
);
}
} catch {
return null;
}
}
};
res.json(ret);
return (await Promise.all(filesList.map(processFile))).filter((r): r is MediaFile => r !== null);
}

236
bills/billsUploadMedia.ts
View File

@@ -1,3 +1,4 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import dotenv from "dotenv";
import { Request, Response } from "express";
import fs from "fs-extra";
@@ -6,7 +7,7 @@ import path, { resolve } from "path";
import { logger } from "../server.js";
import GenerateThumbnail from "../util/generateThumbnail.js";
import { generateUniqueBillFilename } from "../util/generateUniqueFilename.js";
import { ConvertHeicFiles } from "../util/heicConverter.js";
import { convertHeicFiles } from "../util/heicConverter.js";
import { PathToRoBillsFolder, PathToVendorBillsFile } from "../util/pathGenerators.js";
import { BillsListMedia } from "./billsListMedia.js";
@@ -14,72 +15,209 @@ dotenv.config({
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const BILLS_UPLOAD_QUEUE_NAME = "billsUploadProcessingQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const billsUploadProcessingQueue = new Queue(BILLS_UPLOAD_QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 1000 }
}
});
const billsUploadQueueEvents = new QueueEvents(BILLS_UPLOAD_QUEUE_NAME, {
connection: connectionOpts
});
// Bills upload processing worker
const billsUploadWorker = new Worker(
BILLS_UPLOAD_QUEUE_NAME,
async (
job: Job<{
files: Express.Multer.File[];
vendorid: string;
invoice_number: string;
skipThumbnails: boolean;
duplicateToVendor: boolean;
}>
) => {
const { files, vendorid, invoice_number, skipThumbnails, duplicateToVendor } = job.data;
try {
logger.debug(`Processing bills upload with ${files.length} files`);
await job.updateProgress(10);
// Convert HEIC files if any
await convertHeicFiles(files);
await job.updateProgress(30);
if (!skipThumbnails) {
// Filter out HEIC files since they've been converted to JPEG already
const filesToThumbnail = files.filter((file) => {
const isHeic =
file.mimetype === "image/heic" ||
file.mimetype === "image/heif" ||
/\.heic$/i.test(file.filename) ||
/\.heif$/i.test(file.filename);
return !isHeic;
});
logger.debug(
`Generating thumbnails for ${filesToThumbnail.length} bill files (excluding HEIC)`,
filesToThumbnail.map((f) => f.filename)
);
// Generate thumbnails
const thumbnailPromises = filesToThumbnail.map((file) => GenerateThumbnail(file.path));
await Promise.all(thumbnailPromises);
await job.updateProgress(60);
}
// Duplicate to vendor folder if enabled
if (duplicateToVendor && vendorid && invoice_number) {
await Promise.all(
files.map(async (file) => {
const target = path.join(PathToVendorBillsFile(vendorid), file.filename);
await fs.ensureDir(path.dirname(target));
await fs.copyFile(file.path, target);
})
);
await job.updateProgress(90);
}
logger.debug(`Bills upload processing completed`);
await job.updateProgress(100);
return { success: true, filesProcessed: files.length };
} catch (error) {
logger.error(`Error processing bills upload:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 2 // Allow 2 bills upload processing jobs to run concurrently
}
);
// Event listeners for bills upload queue and worker
billsUploadProcessingQueue.on("waiting", (job) => {
logger.debug(`[BillsUploadQueue] Job waiting: ${job.data.invoice_number}`);
});
billsUploadProcessingQueue.on("error", (error) => {
logger.error(`[BillsUploadQueue] Queue error:`, error);
});
billsUploadWorker.on("ready", () => {
logger.debug("[BillsUploadWorker] Worker ready");
});
billsUploadWorker.on("active", (job, prev) => {
logger.debug(`[BillsUploadWorker] Job ${job.id} active (previous: ${prev})`);
});
billsUploadWorker.on("completed", async (job) => {
logger.debug(`[BillsUploadWorker] Job ${job.id} completed`);
await job.remove();
logger.debug(`[BillsUploadWorker] Job ${job.id} removed from Redis`);
});
billsUploadWorker.on("failed", (jobId, reason) => {
logger.error(`[BillsUploadWorker] Job ${jobId} failed: ${reason}`);
});
billsUploadWorker.on("error", (error) => {
logger.error(`[BillsUploadWorker] Worker error:`, error);
});
billsUploadWorker.on("stalled", (job) => {
logger.error(`[BillsUploadWorker] Worker stalled: ${job}`);
});
billsUploadWorker.on("ioredis:close", () => {
logger.error("[BillsUploadWorker] Redis connection closed");
});
export const BillsMediaUploadMulter = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
destination(req, file, cb) {
const jobid: string = (req.body.jobid || "").trim();
const DestinationFolder: string = PathToRoBillsFolder(jobid);
fs.ensureDirSync(DestinationFolder);
cb(jobid === "" || jobid === null ? new Error("Job ID not specified.") : null, DestinationFolder);
if (!jobid) {
cb(new Error("Job ID not specified."), "");
return;
}
const destinationFolder = PathToRoBillsFolder(jobid);
fs.ensureDir(destinationFolder)
.then(() => cb(null, destinationFolder))
.catch((err) => cb(err as Error, ""));
},
filename: function (req, file, cb) {
logger.info("Uploading file: ", {
file: path.basename(file.originalname)
});
filename(req, file, cb) {
logger.info("Uploading file:", { file: path.basename(file.originalname) });
const invoice_number: string = (req.body.invoice_number || "").trim();
cb(
invoice_number === "" || invoice_number === null ? new Error("Invoice number not specified.") : null,
generateUniqueBillFilename(file, invoice_number)
);
if (!invoice_number) {
cb(new Error("Invoice number not specified."), "");
return;
}
cb(null, generateUniqueBillFilename(file, invoice_number));
}
})
});
export async function BillsUploadMedia(req: Request, res: Response) {
try {
if (!req.files) {
res.send({
const files = req.files as Express.Multer.File[] | undefined;
if (!files || files.length === 0) {
logger.warning("Upload contained no files.");
res.status(400).json({
status: false,
message: "No file uploaded"
});
} else {
await ConvertHeicFiles(req.files as Express.Multer.File[]);
const thumbnailGenerationQueue: Promise<string>[] = [];
//for each file.path, generate the thumbnail.
(req.files as Express.Multer.File[]).forEach((file) => {
thumbnailGenerationQueue.push(GenerateThumbnail(file.path));
});
await Promise.all(thumbnailGenerationQueue);
//Check to see if it should be duplicated to the vendor.
if (process.env.DUPLICATE_BILL_TO_VENDOR) {
const copyQueue: Promise<void>[] = [];
//for each file.path, generate the thumbnail.
(req.files as Express.Multer.File[]).forEach((file) => {
const vendorid: string = (req.body.vendorid || "").trim();
const invoice_number: string = (req.body.invoice_number || "").trim();
copyQueue.push(
(async () => {
const target: string = path.join(PathToVendorBillsFile(vendorid), file.filename);
await fs.ensureDir(path.dirname(target));
await fs.copyFile(file.path, target);
})()
);
//Copy Queue is not awaited - we don't care if it finishes before we serve up the thumbnails from the jobs directory.
});
}
BillsListMedia(req, res);
return;
}
const vendorid: string = (req.body.vendorid || "").trim();
const invoice_number: string = (req.body.invoice_number || "").trim();
const skipThumbnails = req.body.skip_thumbnail === "true";
const duplicateToVendor = process.env.DUPLICATE_BILL_TO_VENDOR === "true";
// If skipping thumbnails, queue job without waiting and return immediately
if (skipThumbnails) {
await billsUploadProcessingQueue.add("processBillsUpload", {
files,
vendorid,
invoice_number,
skipThumbnails: true,
duplicateToVendor
});
res.sendStatus(200);
return;
}
// Add bills upload processing job to queue and wait for completion
logger.debug(`Adding bills upload processing job for ${files.length} files`);
const job = await billsUploadProcessingQueue.add("processBillsUpload", {
files,
vendorid,
invoice_number,
skipThumbnails: false,
duplicateToVendor
});
// Wait for the job to complete
await job.waitUntilFinished(billsUploadQueueEvents);
logger.debug(`Bills upload processing job completed`);
// Delegate to BillsListMedia to respond with updated media list
await BillsListMedia(req, res);
} catch (error) {
logger.error("Error while uploading Bill Media", {
files: req.files,
error
error: (error as Error).message
});
res.status(500).send(error);
res.status(500).json({ error: (error as Error).message });
}
}

15
jobs/jobRequestValidator.ts
View File

@@ -1,11 +1,16 @@
import { NextFunction, Request, Response } from "express";
const validateJobRequest: (req: Request, res: Response, next: NextFunction) => void = (req, res, next) => {
const jobId: string = (req.body.jobid || "").trim();
if (jobId === "") {
return res.status(400).json({ error: "No RO Number has been specified." });
const validateJobRequest = (req: Request, res: Response, next: NextFunction) => {
try {
const jobid: string = (req.body.jobid || "").trim();
if (!jobid) {
res.status(400).json({ error: "No RO Number has been specified." });
return;
}
next();
} catch (error) {
res.status(500).json({ error: "Error validating job request.", details: (error as Error).message });
}
next();
};
export default validateJobRequest;

379
jobs/jobsDeleteMedia.ts
View File

@@ -1,64 +1,351 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import { Request, Response } from "express";
import fs from "fs-extra";
import path from "path";
import { logger } from "../server.js";
import MediaFile from "../util/interfaces/MediaFile.js";
import ListableChecker from "../util/listableChecker.js";
import { PathToRoBillsFolder, PathToRoFolder } from "../util/pathGenerators.js";
import { PathToRoBillsFolder, PathToRoFolder, PathToVendorBillsFile } from "../util/pathGenerators.js";
import { BillsRelativeFilePath, FolderPaths, JobRelativeFilePath } from "../util/serverInit.js";
const DELETE_QUEUE_NAME = "deleteQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const deleteQueue = new Queue(DELETE_QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: 10,
removeOnFail: 5,
attempts: 3,
backoff: { type: "exponential", delay: 2000 }
}
});
const deleteQueueEvents = new QueueEvents(DELETE_QUEUE_NAME, {
connection: connectionOpts
});
const deleteWorker = new Worker(
DELETE_QUEUE_NAME,
async (job: Job<{ jobid: string; files: string[] }>) => {
const { jobid, files } = job.data;
logger.debug(`[DeleteWorker] Starting delete operation for job ${jobid} with ${files.length} files`);
try {
await job.updateProgress(5);
const result = await processDeleteOperation(jobid, files, job);
await job.updateProgress(100);
logger.debug(`[DeleteWorker] Completed delete operation for job ${jobid}`);
return result;
} catch (error) {
logger.error(`[DeleteWorker] Error deleting files for job ${jobid}:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 2 // Limit concurrent delete operations
}
);
// Worker event listeners for logging
deleteWorker.on("ready", () => {
logger.debug("[DeleteWorker] Worker is ready");
});
deleteWorker.on("active", (job, prev) => {
logger.debug(`[DeleteWorker] Job ${job.id} active (previous: ${prev})`);
});
deleteWorker.on("completed", async (job) => {
logger.debug(`[DeleteWorker] Job ${job.id} completed`);
});
deleteWorker.on("failed", (job, err) => {
logger.error(`[DeleteWorker] Job ${job?.id} failed:`, err);
});
deleteWorker.on("stalled", (jobId) => {
logger.error(`[DeleteWorker] Job stalled: ${jobId}`);
});
deleteWorker.on("error", (err) => {
logger.error("[DeleteWorker] Worker error:", err);
});
// Queue event listeners
deleteQueue.on("waiting", (job) => {
logger.debug(`[DeleteQueue] Job waiting in queue: job ${job.data.jobid} - ${job.data.files.length} files`);
});
deleteQueue.on("error", (err) => {
logger.error("[DeleteQueue] Queue error:", err);
});
async function processDeleteOperation(
jobid: string,
files: string[],
job?: Job
): Promise<{ deleted: number; failed: number }> {
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug("Deleting media for job: " + PathToRoFolder(jobid));
try {
// Setup lists for both file locations
async function readFilteredDir(dirPath: string): Promise<fs.Dirent[]> {
const filtered: fs.Dirent[] = [];
try {
const dir = await fs.opendir(dirPath);
for await (const dirent of dir) {
if (dirent.isFile() && ListableChecker(dirent)) {
filtered.push(dirent);
}
}
} catch (err) {
logger.error(`Failed to read directory: ${dirPath}`, err);
}
return filtered;
}
const jobFileList = await readFilteredDir(PathToRoFolder(jobid));
const billFileList = await readFilteredDir(PathToRoBillsFolder(jobid));
if (job) await job.updateProgress(15);
// Helper function for safe file deletion
const safeUnlink = async (filePath: string, logPrefix: string = "[DeleteWorker] ") => {
try {
// lstat first to ensure it's a file and the handle is safe to unlink
if (await fs.pathExists(filePath)) {
const stats = await fs.lstat(filePath);
if (stats.isFile()) {
await fs.unlink(filePath);
logger.debug(`${logPrefix}Deleted: ${filePath}`);
}
}
} catch (err) {
logger.warn(`${logPrefix}Failed to delete ${filePath}: ${err}`);
}
};
// Helper to delete a file and its thumbnails
const deleteFileWithThumbs = async (mediaFile: MediaFile, logPrefix: string) => {
try {
await safeUnlink(mediaFile.path, logPrefix);
// Delete thumbnails
const thumbDir = path.dirname(mediaFile.thumbnailPath);
const baseThumb = path.basename(mediaFile.thumbnailPath, path.extname(mediaFile.thumbnailPath));
logger.debug(`${logPrefix}Deleting thumbnails from: ${thumbDir}, baseThumb: ${baseThumb}`);
for (const ext of [".jpg", ".png"]) {
const thumbPath = path.join(thumbDir, `${baseThumb}${ext}`);
await safeUnlink(thumbPath, logPrefix);
}
// Delete ConvertedOriginal file if it exists
// The ConvertedOriginal folder contains the original files with the same filename but original extension
const convertedOriginalDir = path.join(path.dirname(mediaFile.path), FolderPaths.ConvertedOriginalSubDir);
try {
if (await fs.pathExists(convertedOriginalDir)) {
const convertedOriginalFiles = await fs.readdir(convertedOriginalDir);
const currentFileName = path.basename(mediaFile.path, path.extname(mediaFile.path));
logger.debug(`Looking for ConvertedOriginal files with base name: ${currentFileName}`, {
convertedOriginalDir,
currentFileName,
availableFiles: convertedOriginalFiles
});
for (const file of convertedOriginalFiles) {
const fileBaseName = path.basename(file, path.extname(file));
// Match files that have the same base name (same filename, potentially different extension)
if (fileBaseName === currentFileName) {
const convertedOriginalPath = path.join(convertedOriginalDir, file);
await safeUnlink(convertedOriginalPath, logPrefix);
logger.debug(`Found and deleted ConvertedOriginal file: ${convertedOriginalPath}`);
}
}
}
} catch (error) {
logger.warn(`Error checking/deleting ConvertedOriginal files for ${mediaFile.path}:`, error);
}
} catch (error) {
logger.error(`${logPrefix}Error in deleteFileWithThumbs for ${mediaFile.path}:`, error);
throw error;
}
};
// Convert to MediaFile objects for better type safety
const jobMediaFiles: MediaFile[] = jobFileList.map((file) => {
const thumbName = file.name.replace(/\.[^/.]+$/, ".jpg");
const thumbPath = path.join(FolderPaths.Jobs, jobid, FolderPaths.ThumbsSubDir, thumbName);
const filePath = JobRelativeFilePath(jobid, file.name);
return {
name: file.name,
path: filePath,
thumbnailPath: thumbPath,
src: filePath,
thumbnail: thumbPath,
thumbnailHeight: 0,
thumbnailWidth: 0,
filename: file.name
};
});
// Delete job files with proper error handling
const jobDeletions = jobMediaFiles
.filter((mediaFile) => files.includes(path.basename(mediaFile.filename)))
.map((mediaFile) => deleteFileWithThumbs(mediaFile, "[DeleteWorker] "));
// Prepare bill media files
const billMediaFiles: MediaFile[] = billFileList.map((file) => {
const thumbName = file.name.replace(/\.[^/.]+$/, ".jpg");
const thumbPath = path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, FolderPaths.ThumbsSubDir, thumbName);
const filePath = BillsRelativeFilePath(jobid, file.name);
return {
name: file.name,
path: filePath,
thumbnailPath: thumbPath,
src: filePath,
thumbnail: thumbPath,
thumbnailHeight: 0,
thumbnailWidth: 0,
filename: file.name
};
});
// Delete bill files using the helper function
const billDeletions = billMediaFiles
.filter((mediaFile) => files.includes(path.basename(mediaFile.filename)))
.map((mediaFile) => deleteFileWithThumbs(mediaFile, "[DeleteWorker] Bill: "));
// Delete vendor duplicates if DUPLICATE_BILL_TO_VENDOR is enabled
const vendorDeletions: Promise<any>[] = [];
const duplicateToVendor = process.env.DUPLICATE_BILL_TO_VENDOR === "true";
if (duplicateToVendor) {
const billFilesToDelete = billMediaFiles
.filter((mediaFile) => files.includes(path.basename(mediaFile.filename)))
.map(mediaFile => path.basename(mediaFile.filename));
for (const billFile of billFilesToDelete) {
vendorDeletions.push(
(async () => {
try {
// Search for this file in all vendor directories
const vendorsDir = FolderPaths.Vendors;
if (await fs.pathExists(vendorsDir)) {
const vendors = await fs.readdir(vendorsDir, { withFileTypes: true });
for (const vendor of vendors) {
if (vendor.isDirectory()) {
const vendorFilePath = path.join(vendorsDir, vendor.name, billFile);
if (await fs.pathExists(vendorFilePath)) {
await safeUnlink(vendorFilePath, "[DeleteWorker] Vendor: ");
logger.debug(`[DeleteWorker] Deleted vendor file: ${vendorFilePath}`);
}
}
}
}
} catch (error) {
logger.warn(`[DeleteWorker] Failed to delete vendor copies for ${billFile}:`, error);
}
})()
);
}
}
if (job) await job.updateProgress(80);
// Wait for all deletions to complete
const results = await Promise.allSettled([...jobDeletions, ...billDeletions, ...vendorDeletions]);
const failed = results.filter(r => r.status === 'rejected').length;
const deleted = results.filter(r => r.status === 'fulfilled').length;
logger.debug(`[DeleteWorker] Delete operation completed: ${deleted} successful, ${failed} failed`);
return { deleted, failed };
} catch (error) {
logger.error("[DeleteWorker] Error in processDeleteOperation:", error);
throw error;
}
}
export async function JobsDeleteMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
const files: string[] = req.body.files || [];
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug("Deleteing media for job: " + PathToRoFolder(jobid));
let ret: MediaFile[];
try {
// Setup lists for both file locations
const jobFileList: fs.Dirent[] = (
await fs.readdir(PathToRoFolder(jobid), {
withFileTypes: true
})
).filter((f) => f.isFile() && ListableChecker(f));
const billFileList: fs.Dirent[] = (
await fs.readdir(PathToRoBillsFolder(jobid), {
withFileTypes: true
})
).filter((f) => f.isFile() && ListableChecker(f));
if (!files.length) {
res.status(400).json({ error: "files must be specified." });
return;
}
// Check both list for the files that need to be deleted
await Promise.all(
jobFileList.map(async (file) => {
if (files.includes(path.parse(path.basename(file.name)).base)) {
// File is in the set of requested files.
await fs.remove(JobRelativeFilePath(jobid, file.name));
await fs.remove(
path.join(FolderPaths.Jobs, jobid, FolderPaths.ThumbsSubDir, file.name.replace(/\.[^/.]+$/, ".png"))
);
}
})
);
await Promise.all(
billFileList.map(async (file) => {
if (files.includes(path.parse(path.basename(file.name)).base)) {
// File is in the set of requested files.
await fs.remove(BillsRelativeFilePath(jobid, file.name));
await fs.remove(
path.join(
FolderPaths.Jobs,
jobid,
FolderPaths.BillsSubDir,
FolderPaths.ThumbsSubDir,
file.name.replace(/\.[^/.]+$/, ".png")
)
);
}
})
);
// For small operations (1-5 files), process synchronously for immediate feedback
if (files.length <= 5) {
logger.debug("Processing small delete operation synchronously");
await processDeleteOperation(jobid, files);
res.sendStatus(200);
return;
}
// For larger operations, use BullMQ but still return success immediately
logger.debug(`[JobsDeleteMedia] Queuing delete operation for ${files.length} files`);
const job = await deleteQueue.add("deleteMedia", { jobid, files });
// Return success immediately (optimistic response)
res.sendStatus(200);
// Process in background - if it fails, files will still be there on next refresh
job.waitUntilFinished(deleteQueueEvents)
.then(() => {
logger.debug(`[JobsDeleteMedia] Background delete completed for job ${job.id}`);
})
.catch((error) => {
logger.error(`[JobsDeleteMedia] Background delete failed for job ${job.id}:`, error);
});
if (!res.headersSent) res.sendStatus(200);
} catch (error) {
logger.error("Error deleting job media.", { jobid, error });
if (!res.headersSent) res.status(500).json(error);
if (!res.headersSent) res.status(500).json({ error: "Failed to delete media", details: error });
}
}
/**
* Get the status of a delete operation job
*/
export async function JobsDeleteStatus(req: Request, res: Response) {
const { jobId } = req.params;
try {
const job = await Job.fromId(deleteQueue, jobId);
if (!job) {
res.status(404).json({ error: "Job not found" });
return;
}
const state = await job.getState();
const progress = job.progress;
res.json({
jobId,
state,
progress,
data: job.data,
finishedOn: job.finishedOn,
processedOn: job.processedOn,
failedReason: job.failedReason
});
} catch (error) {
logger.error("Error getting delete job status:", error);
res.status(500).json({ error: "Failed to get job status" });
}
}

75
jobs/jobsDownloadMedia.ts
View File

@@ -12,77 +12,60 @@ export async function jobsDownloadMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
try {
//Do we need all files or just some files?
const files: string[] = req.body.files || [];
const zip: JSZip = new JSZip();
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug(`Generating batch download for Job ID ${jobid}`, files);
//Prepare the zip file.
const jobFileList: fs.Dirent[] = (
await fs.readdir(PathToRoFolder(jobid), {
withFileTypes: true
})
await fs.readdir(PathToRoFolder(jobid), { withFileTypes: true })
).filter((f) => f.isFile() && ListableChecker(f));
const billFileList: fs.Dirent[] = (
await fs.readdir(PathToRoBillsFolder(jobid), {
withFileTypes: true
})
await fs.readdir(PathToRoBillsFolder(jobid), { withFileTypes: true })
).filter((f) => f.isFile() && ListableChecker(f));
if (files.length === 0) {
//Get everything.
// Helper to add files to the zip
const addFilesToZip = async (
fileList: fs.Dirent[],
relativePathFn: (jobid: string, filename: string) => string
) => {
await Promise.all(
jobFileList.map(async (file) => {
//Do something async
const fileOnDisk: Buffer = await fs.readFile(JobRelativeFilePath(jobid, file.name));
zip.file(path.parse(path.basename(file.name)).base, fileOnDisk);
})
);
await Promise.all(
billFileList.map(async (file) => {
//Do something async
const fileOnDisk: Buffer = await fs.readFile(BillsRelativeFilePath(jobid, file.name));
zip.file(path.parse(path.basename(file.name)).base, fileOnDisk);
})
);
} else {
//Get the files that are in the list and see which are requested.
await Promise.all(
jobFileList.map(async (file) => {
if (files.includes(path.parse(path.basename(file.name)).base)) {
// File is in the set of requested files.
const fileOnDisk: Buffer = await fs.readFile(JobRelativeFilePath(jobid, file.name));
zip.file(path.parse(path.basename(file.name)).base, fileOnDisk);
fileList.map(async (file) => {
const baseName = path.basename(file.name);
if (files.length === 0 || files.includes(baseName)) {
try {
const fileOnDisk: Buffer = await fs.readFile(relativePathFn(jobid, file.name));
zip.file(baseName, fileOnDisk);
} catch (err) {
logger.warn(`Could not add file to zip: ${file.name}`, err);
}
}
})
);
await Promise.all(
billFileList.map(async (file) => {
if (files.includes(path.parse(path.basename(file.name)).base)) {
// File is in the set of requested files.
const fileOnDisk: Buffer = await fs.readFile(BillsRelativeFilePath(jobid, file.name));
zip.file(path.parse(path.basename(file.name)).base, fileOnDisk);
}
})
);
}
//Send it as a response to download it automatically.
// res.setHeader("Content-disposition", "attachment; filename=" + filename);
};
await addFilesToZip(jobFileList, JobRelativeFilePath);
await addFilesToZip(billFileList, BillsRelativeFilePath);
// Set headers for download
res.setHeader("Content-Disposition", `attachment; filename="${jobid}.zip"`);
res.setHeader("Content-Type", "application/zip");
zip
.generateNodeStream({
type: "nodebuffer",
streamFiles: true
//encodeFileName: (filename) => `${jobid}.zip`,
})
.pipe(res);
.pipe(res)
.on("finish", () => {
logger.debug(`Zip stream finished for Job ID ${jobid}`);
});
} catch (error) {
logger.error("Error downloading job media.", {
jobid,
error: (error as Error).message
});
res.status(500).json((error as Error).message);
if (!res.headersSent) res.status(500).json((error as Error).message);
}
}

163
jobs/jobsListMedia.ts
View File

@@ -14,56 +14,14 @@ export async function JobsListMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug("Listing media for job: " + PathToRoFolder(jobid));
let ret: MediaFile[];
try {
let ret: MediaFile[];
if (req.files) {
//We just uploaded files, we're going to send only those back.
ret = await Promise.all(
(req.files as Express.Multer.File[]).map(async (file) => {
const relativeFilePath: string = JobRelativeFilePath(jobid, file.filename);
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await fileTypeFromFile(relativeFilePath);
return {
type,
size: file.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file.filename]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, relativeThumbPath]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
relativeFilePath
};
})
);
ret = await processUploadedFiles(req.files as Express.Multer.File[], jobid);
} else {
const filesList: fs.Dirent[] = (
await fs.readdir(PathToRoFolder(jobid), {
withFileTypes: true
})
).filter((f) => f.isFile() && ListableChecker(f));
ret = await Promise.all(
filesList.map(async (file) => {
const relativeFilePath: string = JobRelativeFilePath(jobid, file.name);
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await fileTypeFromFile(relativeFilePath);
const fileSize = await fs.stat(relativeFilePath);
return {
type,
size: fileSize.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file.name]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, relativeThumbPath]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.name,
relativeFilePath
};
})
);
ret = await processExistingFiles(jobid);
}
if (!res.headersSent) res.json(ret);
@@ -72,3 +30,114 @@ export async function JobsListMedia(req: Request, res: Response) {
if (!res.headersSent) res.status(500).json(error);
}
}
async function processUploadedFiles(files: Express.Multer.File[], jobid: string): Promise<MediaFile[]> {
const processFile = async (file: Express.Multer.File): Promise<MediaFile> => {
const relativeFilePath: string = JobRelativeFilePath(jobid, file.filename);
try {
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await Promise.race([
fileTypeFromFile(relativeFilePath),
new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), 5000))
]);
return {
type,
size: file.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file.filename]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, relativeThumbPath]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
name: file.filename,
path: relativeFilePath,
thumbnailPath: relativeThumbPath
};
} catch (error) {
logger.error(`Error processing uploaded file ${file.filename}:`, error);
return {
type: undefined,
size: file.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file.filename]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, "assets", "file.svg"]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file.filename,
name: file.filename,
path: relativeFilePath,
thumbnailPath: ""
};
}
};
return (await Promise.all(files.map(processFile))).filter((r): r is MediaFile => r !== null);
}
async function processExistingFiles(jobid: string): Promise<MediaFile[]> {
const dirPath = PathToRoFolder(jobid);
const mediaFiles: MediaFile[] = [];
const dir = await fs.opendir(dirPath);
for await (const dirent of dir) {
if (!dirent.isFile() || !ListableChecker(dirent)) continue;
const file = dirent.name;
const relativeFilePath: string = JobRelativeFilePath(jobid, file);
try {
if (!(await fs.pathExists(relativeFilePath))) {
logger.warn(`File no longer exists: ${relativeFilePath}`);
continue;
}
const fileStats = await Promise.race([
fs.stat(relativeFilePath),
new Promise<never>((_, reject) => setTimeout(() => reject(new Error("File stat timeout")), 5000))
]);
const relativeThumbPath: string = await GenerateThumbnail(relativeFilePath);
const type: FileTypeResult | undefined = await Promise.race([
fileTypeFromFile(relativeFilePath),
new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), 5000))
]);
mediaFiles.push({
type,
size: fileStats.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, relativeThumbPath]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file,
name: file,
path: relativeFilePath,
thumbnailPath: relativeThumbPath
});
} catch (error) {
logger.error(`Error processing existing file ${file}:`, error);
try {
const fileStats = await fs.stat(relativeFilePath);
mediaFiles.push({
type: undefined,
size: fileStats.size,
src: GenerateUrl([FolderPaths.StaticPath, FolderPaths.JobsFolder, jobid, file]),
thumbnail: GenerateUrl([FolderPaths.StaticPath, "assets", "file.svg"]),
thumbnailHeight: 250,
thumbnailWidth: 250,
filename: file,
name: file,
path: relativeFilePath,
thumbnailPath: ""
});
} catch (statError) {
logger.error(`Could not get stats for ${file}:`, statError);
}
}
}
return mediaFiles;
}

388
jobs/jobsMoveMedia.ts
View File

@@ -1,3 +1,4 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import { Request, Response } from "express";
import fs from "fs-extra";
import path from "path";
@@ -7,90 +8,299 @@ import { PathToRoBillsFolder, PathToRoFolder } from "../util/pathGenerators.js";
import { FolderPaths } from "../util/serverInit.js";
import { JobsListMedia } from "./jobsListMedia.js";
const MOVE_QUEUE_NAME = "moveQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const moveQueue = new Queue(MOVE_QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: 10,
removeOnFail: 5,
attempts: 3,
backoff: { type: "exponential", delay: 2000 }
}
});
const moveQueueEvents = new QueueEvents(MOVE_QUEUE_NAME, {
connection: connectionOpts
});
const moveWorker = new Worker(
MOVE_QUEUE_NAME,
async (job: Job<{ jobid: string; from_jobid: string; files: string[] }>) => {
const { jobid, from_jobid, files } = job.data;
logger.debug(`[MoveWorker] Starting move operation from ${from_jobid} to ${jobid} for ${files.length} files`);
try {
await job.updateProgress(5);
const result = await processMoveOperation(jobid, from_jobid, files, job);
await job.updateProgress(100);
logger.debug(`[MoveWorker] Completed move operation from ${from_jobid} to ${jobid}`);
return result;
} catch (error) {
logger.error(`[MoveWorker] Error moving files from ${from_jobid} to ${jobid}:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 2 // Limit concurrent move operations to avoid I/O overwhelm
}
);
// Worker event listeners for logging
moveWorker.on("ready", () => {
logger.debug("[MoveWorker] Worker is ready");
});
moveWorker.on("active", (job, prev) => {
logger.debug(`[MoveWorker] Job ${job.id} active (previous: ${prev})`);
});
moveWorker.on("completed", async (job) => {
logger.debug(`[MoveWorker] Job ${job.id} completed`);
});
moveWorker.on("failed", (job, err) => {
logger.error(`[MoveWorker] Job ${job?.id} failed:`, err);
});
moveWorker.on("stalled", (jobId) => {
logger.error(`[MoveWorker] Job stalled: ${jobId}`);
});
moveWorker.on("error", (err) => {
logger.error("[MoveWorker] Worker error:", err);
});
// Queue event listeners
moveQueue.on("waiting", (job) => {
logger.debug(`[MoveQueue] Job waiting in queue: ${job.data.from_jobid} -> ${job.data.jobid}`);
});
moveQueue.on("error", (err) => {
logger.error("[MoveQueue] Queue error:", err);
});
async function processMoveOperation(
jobid: string,
from_jobid: string,
files: string[],
job?: Job
): Promise<{ moved: number; failed: number }> {
try {
const jobFileList: string[] = [];
const jobDir = await fs.opendir(PathToRoFolder(from_jobid));
for await (const dirent of jobDir) {
if (dirent.isFile() && ListableChecker(dirent)) jobFileList.push(dirent.name);
}
const billFileList: string[] = [];
const billDir = await fs.opendir(PathToRoBillsFolder(from_jobid));
for await (const dirent of billDir) {
if (dirent.isFile() && ListableChecker(dirent)) billFileList.push(dirent.name);
}
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug("Moving job based media.", { jobid, from_jobid, files });
if (job) await job.updateProgress(15);
const moveOps: Promise<any>[] = [];
let processedFiles = 0;
const totalFiles = files.length;
for (const file of files) {
if (jobFileList.includes(file)) {
// Move main file
moveOps.push(
fs
.move(path.join(FolderPaths.Jobs, from_jobid, file), path.join(FolderPaths.Jobs, jobid, file), {
overwrite: true
})
.then(() => logger.debug(`[MoveWorker] Moved main file: ${file}`))
.catch((err) => logger.warn(`[MoveWorker] Failed to move main file ${file}:`, err))
);
// Move thumbnails
const baseThumb = file.replace(/\.[^/.]+$/, "");
for (const ext of [".jpg", ".png"]) {
moveOps.push(
fs
.move(
path.join(FolderPaths.Jobs, from_jobid, FolderPaths.ThumbsSubDir, `${baseThumb}${ext}`),
path.join(FolderPaths.Jobs, jobid, FolderPaths.ThumbsSubDir, `${baseThumb}${ext}`),
{ overwrite: true }
)
.then(() => logger.debug(`[MoveWorker] Moved thumbnail: ${baseThumb}${ext}`))
.catch(() => {}) // Thumbnails might not exist
);
}
// Move ConvertedOriginal file if it exists
const baseFileName = file.replace(/\.[^/.]+$/, "");
const sourceConvertedDir = path.join(FolderPaths.Jobs, from_jobid, FolderPaths.ConvertedOriginalSubDir);
const targetConvertedDir = path.join(FolderPaths.Jobs, jobid, FolderPaths.ConvertedOriginalSubDir);
moveOps.push(
(async () => {
try {
if (await fs.pathExists(sourceConvertedDir)) {
const convertedOriginalFiles = await fs.readdir(sourceConvertedDir);
for (const convertedFile of convertedOriginalFiles) {
const convertedFileBaseName = path.basename(convertedFile, path.extname(convertedFile));
if (convertedFileBaseName === baseFileName) {
await fs.ensureDir(targetConvertedDir);
await fs.move(
path.join(sourceConvertedDir, convertedFile),
path.join(targetConvertedDir, convertedFile),
{ overwrite: true }
);
logger.debug(`[MoveWorker] Moved ConvertedOriginal: ${convertedFile}`);
}
}
}
} catch (error) {
logger.warn(`[MoveWorker] Failed to move ConvertedOriginal for ${file}:`, error);
}
})()
);
}
if (billFileList.includes(file)) {
// Move bill file
moveOps.push(
fs
.move(
path.join(FolderPaths.Jobs, from_jobid, FolderPaths.BillsSubDir, file),
path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, file),
{ overwrite: true }
)
.then(() => logger.debug(`[MoveWorker] Moved bill file: ${file}`))
.catch((err) => logger.warn(`[MoveWorker] Failed to move bill file ${file}:`, err))
);
// Move bill thumbnails
const baseThumb = file.replace(/\.[^/.]+$/, "");
for (const ext of [".jpg", ".png"]) {
moveOps.push(
fs
.move(
path.join(
FolderPaths.Jobs,
from_jobid,
FolderPaths.BillsSubDir,
FolderPaths.ThumbsSubDir,
`${baseThumb}${ext}`
),
path.join(
FolderPaths.Jobs,
jobid,
FolderPaths.BillsSubDir,
FolderPaths.ThumbsSubDir,
`${baseThumb}${ext}`
),
{ overwrite: true }
)
.then(() => logger.debug(`[MoveWorker] Moved bill thumbnail: ${baseThumb}${ext}`))
.catch(() => {}) // Thumbnails might not exist
);
}
// Move bill ConvertedOriginal file if it exists
const billBaseFileName = file.replace(/\.[^/.]+$/, "");
const sourceBillConvertedDir = path.join(FolderPaths.Jobs, from_jobid, FolderPaths.BillsSubDir, FolderPaths.ConvertedOriginalSubDir);
const targetBillConvertedDir = path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, FolderPaths.ConvertedOriginalSubDir);
moveOps.push(
(async () => {
try {
if (await fs.pathExists(sourceBillConvertedDir)) {
const convertedOriginalFiles = await fs.readdir(sourceBillConvertedDir);
for (const convertedFile of convertedOriginalFiles) {
const convertedFileBaseName = path.basename(convertedFile, path.extname(convertedFile));
if (convertedFileBaseName === billBaseFileName) {
await fs.ensureDir(targetBillConvertedDir);
await fs.move(
path.join(sourceBillConvertedDir, convertedFile),
path.join(targetBillConvertedDir, convertedFile),
{ overwrite: true }
);
logger.debug(`[MoveWorker] Moved bill ConvertedOriginal: ${convertedFile}`);
}
}
}
} catch (error) {
logger.warn(`[MoveWorker] Failed to move bill ConvertedOriginal for ${file}:`, error);
}
})()
);
}
processedFiles++;
if (job) {
const progress = 15 + Math.round((processedFiles / totalFiles) * 70);
await job.updateProgress(progress);
}
}
if (job) await job.updateProgress(90);
const results = await Promise.allSettled(moveOps);
const failed = results.filter(r => r.status === 'rejected').length;
const moved = results.filter(r => r.status === 'fulfilled').length;
logger.debug(`[MoveWorker] Move operation completed: ${moved} successful, ${failed} failed`);
return { moved, failed };
} catch (error) {
logger.error("[MoveWorker] Error in processMoveOperation:", error);
throw error;
}
}
export async function JobsMoveMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
const from_jobid: string = (req.body.from_jobid || "").trim();
const files: string[] = req.body.files; //Just file names.
const files: string[] = req.body.files || [];
try {
//Validate the request is valid and contains everything that it needs.
if (from_jobid === "") {
res.status(400).json({ error: "from_jobid must be specified. " });
if (!from_jobid) {
res.status(400).json({ error: "from_jobid must be specified." });
return;
}
if (files.length === 0) {
res.status(400).json({ error: "files must be specified. " });
if (!files.length) {
res.status(400).json({ error: "files must be specified." });
return;
}
// Setup lists for both file locations
const jobFileList: string[] = (
await fs.readdir(PathToRoFolder(from_jobid), {
withFileTypes: true
})
)
.filter((f) => f.isFile() && ListableChecker(f))
.map((dirent) => dirent.name);
const billFileList: string[] = (
await fs.readdir(PathToRoBillsFolder(from_jobid), {
withFileTypes: true
})
)
.filter((f) => f.isFile() && ListableChecker(f))
.map((dirent) => dirent.name);
//Make sure the destination RO directory exists.
await fs.ensureDir(PathToRoFolder(jobid));
logger.debug("Moving job based media.", { jobid, from_jobid, files });
const movingQueue: Promise<void>[] = [];
files.forEach((file) => {
if (jobFileList.includes(file)) {
movingQueue.push(
fs.move(path.join(FolderPaths.Jobs, from_jobid, file), path.join(FolderPaths.Jobs, jobid, file))
);
movingQueue.push(
fs.move(
path.join(FolderPaths.Jobs, from_jobid, FolderPaths.ThumbsSubDir, file.replace(/\.[^/.]+$/, ".png")),
path.join(FolderPaths.Jobs, jobid, FolderPaths.ThumbsSubDir, file.replace(/\.[^/.]+$/, ".png"))
)
);
}
if (billFileList.includes(file)) {
movingQueue.push(
fs.move(
path.join(FolderPaths.Jobs, from_jobid, FolderPaths.BillsSubDir, file),
path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, file)
)
);
movingQueue.push(
fs.move(
path.join(
FolderPaths.Jobs,
from_jobid,
FolderPaths.BillsSubDir,
FolderPaths.ThumbsSubDir,
file.replace(/\.[^/.]+$/, ".png")
),
path.join(
FolderPaths.Jobs,
jobid,
FolderPaths.BillsSubDir,
FolderPaths.ThumbsSubDir,
file.replace(/\.[^/.]+$/, ".png")
)
)
);
}
});
//Use AllSettled as it allows for individual moves to fail.
//e.g. if the thumbnail does not exist.
await Promise.allSettled(movingQueue);
// For small operations (1-3 files), process synchronously for immediate feedback
if (files.length <= 3) {
logger.debug("Processing small move operation synchronously");
await processMoveOperation(jobid, from_jobid, files);
JobsListMedia(req, res);
return;
}
// For larger operations, use BullMQ but still return updated file list
logger.debug(`[JobsMoveMedia] Queuing move operation for ${files.length} files`);
const job = await moveQueue.add("moveMedia", { jobid, from_jobid, files });
// Return the updated file list immediately (optimistic update)
JobsListMedia(req, res);
// Process in background - if it fails, files will be back on next refresh
job.waitUntilFinished(moveQueueEvents)
.then(() => {
logger.debug(`[JobsMoveMedia] Background move completed for job ${job.id}`);
})
.catch((error) => {
logger.error(`[JobsMoveMedia] Background move failed for job ${job.id}:`, error);
});
} catch (err) {
logger.error("Error moving job media", {
from_jobid,
@@ -98,6 +308,38 @@ export async function JobsMoveMedia(req: Request, res: Response) {
files,
err
});
res.status(500).send(err);
res.status(500).json({ error: "Failed to queue move operation", details: err });
}
}
/**
* Get the status of a move operation job
*/
export async function JobsMoveStatus(req: Request, res: Response) {
const { jobId } = req.params;
try {
const job = await Job.fromId(moveQueue, jobId);
if (!job) {
res.status(404).json({ error: "Job not found" });
return;
}
const state = await job.getState();
const progress = job.progress;
res.json({
jobId,
state,
progress,
data: job.data,
finishedOn: job.finishedOn,
processedOn: job.processedOn,
failedReason: job.failedReason
});
} catch (error) {
logger.error("Error getting move job status:", error);
res.status(500).json({ error: "Failed to get job status" });
}
}

204
jobs/jobsUploadMedia.ts
View File

@@ -1,3 +1,5 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import dotenv from "dotenv";
import { Request, Response } from "express";
import fs from "fs-extra";
import multer from "multer";
@@ -5,10 +7,151 @@ import path from "path";
import { logger } from "../server.js";
import GenerateThumbnail from "../util/generateThumbnail.js";
import generateUniqueFilename from "../util/generateUniqueFilename.js";
import { ConvertHeicFiles } from "../util/heicConverter.js";
import { convertHeicFiles } from "../util/heicConverter.js";
import { PathToRoFolder } from "../util/pathGenerators.js";
import { JobsListMedia } from "./jobsListMedia.js";
dotenv.config({
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const UPLOAD_QUEUE_NAME = "uploadProcessingQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const uploadProcessingQueue = new Queue(UPLOAD_QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 1000 }
}
});
const uploadQueueEvents = new QueueEvents(UPLOAD_QUEUE_NAME, {
connection: connectionOpts
});
// Upload processing worker
const uploadWorker = new Worker(
UPLOAD_QUEUE_NAME,
async (
job: Job<{
files: Express.Multer.File[];
jobid: string;
skipThumbnails: boolean;
}>
) => {
const { files, jobid, skipThumbnails } = job.data;
try {
logger.debug(`Processing upload for job ${jobid} with ${files.length} files, skipThumbnails: ${skipThumbnails}`);
await job.updateProgress(10);
// Convert HEIC files if any
await convertHeicFiles(files);
await job.updateProgress(40);
if (!skipThumbnails) {
// Log file states after conversion for debugging
logger.debug("File states after HEIC conversion:", files.map(f => ({
filename: f.filename,
mimetype: f.mimetype,
path: f.path
})));
// Check if converted files actually exist on disk
for (const file of files) {
const exists = await fs.pathExists(file.path);
logger.debug(`File existence check: ${file.filename} at ${file.path} - exists: ${exists}`);
}
// After HEIC conversion, all files should be ready for thumbnail generation
// Only exclude files that are still HEIC (which shouldn't happen after conversion)
const filesToThumbnail = [];
for (const file of files) {
const isStillHeic =
file.mimetype === "image/heic" ||
file.mimetype === "image/heif" ||
/\.heic$/i.test(file.filename) ||
/\.heif$/i.test(file.filename);
logger.debug(`File ${file.filename} - mimetype: ${file.mimetype}, isStillHeic: ${isStillHeic}, including: ${!isStillHeic}`);
if (!isStillHeic) {
filesToThumbnail.push(file);
}
}
logger.debug(
`Generating thumbnails for ${filesToThumbnail.length} files (excluding HEIC)`,
filesToThumbnail.map((f) => f.filename)
);
// Generate thumbnails
logger.debug(`About to generate thumbnails for ${filesToThumbnail.length} files`);
const thumbnailPromises = filesToThumbnail.map((file, index) => {
logger.debug(`Starting thumbnail generation for file ${index + 1}/${filesToThumbnail.length}: ${file.filename} at ${file.path}`);
return GenerateThumbnail(file.path);
});
const thumbnailResults = await Promise.all(thumbnailPromises);
logger.debug(`Thumbnail generation completed. Results:`, thumbnailResults);
await job.updateProgress(90);
}
logger.debug(`Upload processing completed for job ${jobid}`);
await job.updateProgress(100);
return { success: true, filesProcessed: files.length };
} catch (error) {
logger.error(`Error processing upload for job ${jobid}:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 2 // Allow 2 upload processing jobs to run concurrently
}
);
// Event listeners for upload queue and worker
uploadProcessingQueue.on("waiting", (job) => {
logger.debug(`[UploadQueue] Job waiting: ${job.data.jobid}`);
});
uploadProcessingQueue.on("error", (error) => {
logger.error(`[UploadQueue] Queue error:`, error);
});
uploadWorker.on("ready", () => {
logger.debug("[UploadWorker] Worker ready");
});
uploadWorker.on("active", (job, prev) => {
logger.debug(`[UploadWorker] Job ${job.id} active (previous: ${prev})`);
});
uploadWorker.on("completed", async (job) => {
logger.debug(`[UploadWorker] Job ${job.id} completed`);
await job.remove();
logger.debug(`[UploadWorker] Job ${job.id} removed from Redis`);
});
uploadWorker.on("failed", (jobId, reason) => {
logger.error(`[UploadWorker] Job ${jobId} failed: ${reason}`);
});
uploadWorker.on("error", (error) => {
logger.error(`[UploadWorker] Worker error:`, error);
});
uploadWorker.on("stalled", (job) => {
logger.error(`[UploadWorker] Worker stalled: ${job}`);
});
uploadWorker.on("ioredis:close", () => {
logger.error("[UploadWorker] Redis connection closed");
});
export const JobMediaUploadMulter = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
@@ -30,37 +173,46 @@ export async function jobsUploadMedia(req: Request, res: Response) {
const jobid: string = (req.body.jobid || "").trim();
try {
if (!req.files || (req.files as Express.Multer.File[]).length === 0) {
const files = req.files as Express.Multer.File[] | undefined;
if (!files || files.length === 0) {
logger.warning("Upload contained no files.");
res.status(400).send({
status: false,
message: "No file uploaded"
});
} else {
//If we want to skip waiting for everything, just send it back that we're good.
if (req.body.skip_thumbnail) {
req.headers.skipReponse === "true";
res.sendStatus(200);
}
//Check if there's a heic in the file set. If so, modify the file set.
await ConvertHeicFiles(req.files as Express.Multer.File[]);
logger.debug(
"Creating thumbnails for newly uploaded media",
(req.files as Express.Multer.File[]).map((f) => f.filename)
);
const thumbnailGenerationQueue: Promise<string>[] = [];
//for each file.path, generate the thumbnail.
(req.files as Express.Multer.File[]).forEach((file) => {
thumbnailGenerationQueue.push(GenerateThumbnail(file.path));
});
await Promise.all(thumbnailGenerationQueue);
JobsListMedia(req, res);
return;
}
// Check if client wants to skip waiting for response but still do processing
if (req.body.skip_thumbnail) {
logger.debug("Client requested skip_thumbnail - responding immediately but processing in background");
// Set header to indicate we're skipping the response wait
req.headers.skipResponse = "true";
res.sendStatus(200);
// Continue with background processing (conversion + thumbnails)
const job = await uploadProcessingQueue.add("processUpload", {
files,
jobid,
skipThumbnails: false // Always do thumbnails, just don't wait for completion
});
return;
}
// Add upload processing job to queue and wait for completion
logger.debug(`Adding upload processing job for ${files.length} files with thumbnails enabled`);
const job = await uploadProcessingQueue.add("processUpload", {
files,
jobid,
skipThumbnails: false
});
// Wait for the job to complete
await job.waitUntilFinished(uploadQueueEvents);
logger.debug(`Upload processing job completed for job ${jobid}`);
// Return the updated media list
JobsListMedia(req, res);
} catch (error) {
logger.error("Error uploading job media.", {
jobid,

1574
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

36
package.json
View File

@@ -13,22 +13,22 @@
"makeitpretty": "prettier --write \"**/*.{css,js,json,jsx,scss,ts}\""
},
"dependencies": {
"@types/compression": "^1.7.5",
"axios": "^1.8.1",
"body-parser": "^1.20.3",
"bullmq": "^5.41.7",
"@types/compression": "^1.8.1",
"axios": "^1.10.0",
"body-parser": "^2.2.0",
"bullmq": "^5.56.4",
"compression": "^1.8.0",
"cors": "^2.8.5",
"dotenv": "16.4.7",
"express": "^4.21.2",
"file-type": "^20.4.0",
"dotenv": "17.2.0",
"express": "^5.1.0",
"file-type": "^21.0.0",
"fs-extra": "^11.3.0",
"gm": "^1.25.1",
"helmet": "^8.0.0",
"helmet": "^8.1.0",
"image-thumbnail": "^1.0.17",
"jszip": "^3.10.1",
"morgan": "^1.10.0",
"multer": "^1.4.4",
"multer": "^2.0.1",
"nocache": "^4.0.0",
"response-time": "^2.3.3",
"simple-thumbnail": "^1.6.5",
@@ -36,19 +36,19 @@
"winston-daily-rotate-file": "^5.0.0"
},
"devDependencies": {
"@types/cors": "^2.8.17",
"@types/express": "^5.0.0",
"@types/cors": "^2.8.19",
"@types/express": "^5.0.3",
"@types/fs-extra": "^11.0.4",
"@types/gm": "^1.25.4",
"@types/image-thumbnail": "^1.0.4",
"@types/morgan": "^1.9.9",
"@types/multer": "^1.4.12",
"@types/node": "^22.13.9",
"@types/response-time": "^2.3.8",
"nodemon": "^3.1.9",
"prettier": "^3.5.3",
"@types/morgan": "^1.9.10",
"@types/multer": "^2.0.0",
"@types/node": "^24.0.13",
"@types/response-time": "^2.3.9",
"nodemon": "^3.1.10",
"prettier": "^3.6.2",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.8.2"
"typescript": "^5.8.3"
}
}

8
redis-docker-compose.yml
View File

@@ -1,8 +0,0 @@
version: '3'
services:
redis-server:
image: "redis:alpine"
command: redis-server
ports:
- "6379:6379"

73
server.ts
View File

@@ -26,6 +26,7 @@ dotenv.config({
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
// Logger setup
const commonTransportConfig = {
maxSize: "20m",
maxFiles: 14,
@@ -42,10 +43,6 @@ const baseFormat = winston.format.combine(
winston.format.prettyPrint()
);
const consoleTransport = new winston.transports.Console({
format: winston.format.combine(winston.format.colorize(), winston.format.timestamp(), winston.format.simple())
});
export const logger = winston.createLogger({
format: baseFormat,
level: "http",
@@ -54,15 +51,13 @@ export const logger = winston.createLogger({
new DailyRotateFile({
filename: path.join(FolderPaths.Root, "logs", "exceptions-%DATE%.log"),
...commonTransportConfig
}),
consoleTransport
})
],
rejectionHandlers: [
new DailyRotateFile({
filename: path.join(FolderPaths.Root, "logs", "rejections-%DATE%.log"),
...commonTransportConfig
}),
consoleTransport
})
],
transports: [
new DailyRotateFile({
@@ -78,55 +73,48 @@ export const logger = winston.createLogger({
new DailyRotateFile({
filename: path.join(FolderPaths.Root, "logs", "ALL-%DATE%.log"),
...commonTransportConfig
}),
new winston.transports.Console({
format: winston.format.combine(winston.format.colorize(), winston.format.timestamp(), winston.format.simple())
})
]
});
logger.add(
new winston.transports.Console({
format: winston.format.combine(winston.format.colorize(), winston.format.timestamp(), winston.format.simple())
})
);
// App init
const app: Express = express();
const port = process.env.PORT;
app.disable("x-powered-by");
app.set("trust proxy", 1);
// Middleware order: security, logging, parsing, etc.
app.set("etag", false);
app.use(helmet({ crossOriginResourcePolicy: { policy: "cross-origin" } }));
app.use(nocache());
app.use(cors());
app.use(compression());
app.use(responseTime());
app.use(bodyParser.json({ limit: "1000mb" }));
app.use(bodyParser.urlencoded({ limit: "1000mb", extended: true }));
app.use((req, res, next) => {
res.setHeader("Connection", "keep-alive");
next();
});
app.use(nocache());
app.use(bodyParser.json({ limit: "1000mb" }));
app.use(bodyParser.urlencoded({ limit: "1000mb", extended: true }));
app.use(responseTime());
app.use(cors());
const morganMiddleware = morgan(
"combined", //":method :url :status :res[content-length] - :response-time ms"
{
stream: {
write: (message) => logger.http(message.trim())
}
}
);
const morganMiddleware = morgan("combined", {
stream: { write: (message) => logger.http(message.trim()) }
});
app.use(morganMiddleware);
app.use(helmet({ crossOriginResourcePolicy: { policy: "cross-origin" } }));
// Job endpoints
app.post("/jobs/list", ValidateImsToken, validateJobRequest, JobsListMedia);
app.post("/jobs/upload", ValidateImsToken, JobMediaUploadMulter.array("file"), validateJobRequest, jobsUploadMedia);
app.post("/jobs/download", ValidateImsToken, validateJobRequest, jobsDownloadMedia);
app.post(
"/jobs/move", //JobRequestValidator,
ValidateImsToken,
JobsMoveMedia
);
app.post(
"/jobs/delete", //JobRequestValidator,
ValidateImsToken,
JobsDeleteMedia
);
app.post("/jobs/move", ValidateImsToken, JobsMoveMedia);
app.post("/jobs/delete", ValidateImsToken, JobsDeleteMedia);
// Bill endpoints
app.post("/bills/list", BillRequestValidator, BillsListMedia);
app.post(
"/bills/upload",
@@ -136,16 +124,19 @@ app.post(
BillsUploadMedia
);
app.get("/", ValidateImsToken, (req: express.Request, res: express.Response) => {
// Health and root
app.get("/", ValidateImsToken, (req, res) => {
res.send("IMS running.");
});
app.get("/health", (req: express.Request, res: express.Response) => {
app.get("/health", (req, res) => {
res.status(200).send("OK");
});
// Static files
InitServer();
app.use(FolderPaths.StaticPath, express.static(FolderPaths.Root, { etag: false, maxAge: 30 * 1000 }));
app.use("/assets", express.static("./assets", { etag: false, maxAge: 30 * 1000 }));
app.listen(port, () => {
logger.info(`ImEX Media Server is running at http://localhost:${port}`);
});

207
util/generateThumbnail.ts
View File

@@ -1,7 +1,8 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import { fileTypeFromFile } from "file-type";
import { FileTypeResult } from "file-type/core";
import fs from "fs-extra";
import { access } from "fs/promises";
import { access, FileHandle, open as fsOpen } from "fs/promises";
import gm from "gm";
import imageThumbnail from "image-thumbnail";
import path from "path";
@@ -9,75 +10,187 @@ import { logger } from "../server.js";
import { AssetPaths, FolderPaths } from "./serverInit.js";
//@ts-ignore
import simpleThumb from "simple-thumbnail";
//const ffmpeg = require("ffmpeg-static");
/** @returns {string} Returns the relative path from the file to the thumbnail on the server. This must be converted to a URL. */
export default async function GenerateThumbnail(file: string) {
// const type: core.FileTypeResult | undefined = await ft.fileTypeFromFile(file);
const type: FileTypeResult | undefined = await fileTypeFromFile(file);
let thumbPath: string = path.join(
path.dirname(file),
FolderPaths.ThumbsSubDir,
`${path.parse(path.basename(file)).name}.jpg`
);
const QUEUE_NAME = "thumbnailQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const thumbnailQueue = new Queue(QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 1000 }
}
});
const thumbnailQueueEvents = new QueueEvents(QUEUE_NAME, {
connection: connectionOpts
});
const thumbnailWorker = new Worker(
QUEUE_NAME,
async (job: Job<{ file: string }>) => {
const { file } = job.data;
logger.debug(`[ThumbnailWorker] Starting thumbnail generation for ${file}`);
try {
await job.updateProgress(10);
const result = await processThumbnail(file, job);
await job.updateProgress(100);
logger.debug(`[ThumbnailWorker] Completed thumbnail generation for ${file}`);
return result;
} catch (error) {
logger.error(`[ThumbnailWorker] Error generating thumbnail for ${file}:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 1
}
);
// Worker event listeners for logging
thumbnailWorker.on("ready", () => {
logger.debug("[ThumbnailWorker] Worker is ready");
});
thumbnailWorker.on("active", (job, prev) => {
logger.debug(`[ThumbnailWorker] Job ${job.id} active (previous: ${prev})`);
});
thumbnailWorker.on("completed", async (job) => {
logger.debug(`[ThumbnailWorker] Job ${job.id} completed`);
await job.remove();
logger.debug(`[ThumbnailWorker] Job ${job.id} removed from Redis`);
});
thumbnailWorker.on("failed", (job, err) => {
logger.error(`[ThumbnailWorker] Job ${job?.id} failed:`, err);
});
thumbnailWorker.on("stalled", (jobId) => {
logger.error(`[ThumbnailWorker] Job stalled: ${jobId}`);
});
thumbnailWorker.on("error", (err) => {
logger.error("[ThumbnailWorker] Worker error:", err);
});
thumbnailWorker.on("ioredis:close", () => {
logger.error("[ThumbnailWorker] Redis connection closed");
});
// Queue event listeners
thumbnailQueue.on("waiting", (job) => {
logger.debug(`[ThumbnailQueue] Job waiting in queue: ${job.data.file}`);
});
thumbnailQueue.on("error", (err) => {
logger.error("[ThumbnailQueue] Queue error:", err);
});
async function processThumbnail(file: string, job?: Job): Promise<string> {
let fileHandle: FileHandle | null = null;
let thumbPath = "unknown";
try {
//Ensure the thumbs directory exists.
await access(file, fs.constants.R_OK);
// Wait for stable file size to avoid incomplete thumbnails
const size1 = (await fs.stat(file)).size;
await new Promise((r) => setTimeout(r, 100));
const size2 = (await fs.stat(file)).size;
if (size1 !== size2) {
throw new Error("File is still being written to, skipping thumbnail generation");
}
fileHandle = await fsOpen(file, "r");
const type: FileTypeResult | undefined = await fileTypeFromFile(file);
const baseName = path.parse(path.basename(file)).name;
thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`);
await fs.ensureDir(path.dirname(thumbPath));
//Check for existing thumbnail
if (await fs.pathExists(thumbPath)) {
logger.debug("Using existing thumbnail:", thumbPath);
logger.debug(`[ThumbnailWorker] Using existing thumbnail: ${thumbPath}`);
return path.relative(path.dirname(file), thumbPath);
}
//Check to see if the file is an image, PDF, or video.
if (!type?.mime) {
throw new Error("Unknown file type");
}
if (!type?.mime) throw new Error("Unknown file type");
if (type?.mime === "application/pdf" || type?.mime === "image/heic" || type?.mime === "image/heif") {
await GeneratePdfThumbnail(file, thumbPath);
} else if (type?.mime.startsWith("video")) {
await simpleThumb(file, thumbPath, "250x?", {
// path: ffmpeg,
});
await fileHandle.close();
fileHandle = null;
if (["application/pdf", "image/heic", "image/heif"].includes(type.mime)) {
logger.debug(`[ThumbnailWorker] Generating PDF/HEIC thumbnail for: ${file}`);
await generatePdfThumbnail(file, thumbPath);
} else if (type.mime.startsWith("video")) {
logger.debug(`[ThumbnailWorker] Generating video thumbnail for: ${file}`);
await simpleThumb(file, thumbPath, "250x?");
} else {
logger.debug("Thumbnail being created for : " + thumbPath);
//Ignoring typescript as the interface is lacking a parameter.
// @ts-ignore
const thumbnail = await imageThumbnail(file, {
logger.debug(`[ThumbnailWorker] Generating image thumbnail for: ${file}`);
const thumbnailBuffer = await imageThumbnail(file, {
responseType: "buffer",
height: 250,
width: 250,
failOnError: false
width: 250
});
await fs.writeFile(thumbPath, thumbnail);
await fs.writeFile(thumbPath, thumbnailBuffer);
}
return path.relative(path.dirname(file), thumbPath);
} catch (err) {
logger.error("Error when genenerating thumbnail:", {
} catch (error) {
logger.error("[ThumbnailWorker] Error generating thumbnail:", {
thumbPath,
err,
message: (err as Error).message
error,
message: (error as Error).message
});
return path.relative(path.dirname(file), AssetPaths.File);
} finally {
if (fileHandle) {
try {
await fileHandle.close();
} catch (closeError) {
logger.error("[ThumbnailWorker] Error closing file handle:", closeError);
}
}
}
}
async function GeneratePdfThumbnail(file: string, thumbPath: string) {
const fileOnDisk: Buffer = await fs.readFile(file);
return new Promise<string>((resolve, reject) => {
gm(fileOnDisk)
.selectFrame(0)
async function generatePdfThumbnail(file: string, thumbPath: string): Promise<string> {
return new Promise((resolve, reject) => {
gm(file + "[0]") // first page only
.setFormat("jpg")
.resize(250, 250, "!")
.quality(75)
.write(thumbPath, (error) => {
if (error) {
reject(error);
} else {
resolve(thumbPath);
}
});
.write(thumbPath, (err) => (err ? reject(err) : resolve(thumbPath)));
});
}
/**
* Add a thumbnail generation job and wait for result.
* Returns the relative path to the thumbnail or default file icon on error.
*/
export default async function GenerateThumbnail(file: string): Promise<string> {
const baseName = path.parse(path.basename(file)).name;
const thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`);
if (await fs.pathExists(thumbPath)) {
logger.debug(`[GenerateThumbnail] Returning existing thumbnail immediately for ${file}`);
return path.relative(path.dirname(file), thumbPath);
}
try {
logger.debug(`[GenerateThumbnail] Adding job to queue for ${file}`);
const job = await thumbnailQueue.add("generate", { file });
const result = await job.waitUntilFinished(thumbnailQueueEvents);
logger.debug(`[GenerateThumbnail] Job completed for ${file}`);
return result as string;
} catch (error) {
logger.error(`[GenerateThumbnail] Job failed for ${file}:`, error);
return path.relative(path.dirname(file), AssetPaths.File);
}
}

291
util/heicConverter.ts
View File

@@ -1,4 +1,4 @@
import { Job, Queue, Worker } from "bullmq";
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import dotenv from "dotenv";
import { fileTypeFromFile } from "file-type";
import { FileTypeResult } from "file-type/core";
@@ -10,60 +10,112 @@ import { logger } from "../server.js";
import { generateUniqueHeicFilename } from "./generateUniqueFilename.js";
import { FolderPaths } from "./serverInit.js";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const HeicQueue = new Queue("HEIC Queue", {
connection: {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: function (err) {
const targetError = "READONLY";
return err.message.includes(targetError);
}
},
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: {
type: "exponential",
delay: 1000
}
}
});
const cleanupINTERVAL = 1000 * 60 * 10;
setInterval(cleanupQueue, cleanupINTERVAL);
dotenv.config({
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const imageMagick = gm.subClass({ imageMagick: true });
const QUEUE_NAME = "heicQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const heicQueue = new Queue(QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 1000 }
}
});
// Re-added QueueEvents for waiting on job completion
const heicQueueEvents = new QueueEvents(QUEUE_NAME, {
connection: connectionOpts
});
const CLEANUP_INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
setInterval(cleanupQueue, CLEANUP_INTERVAL_MS);
async function cleanupQueue() {
const ONE_HOUR = 1000 * 60 * 60;
const SIX_HOURS = ONE_HOUR * 6;
try {
// Clean completed jobs older than 1 hour
await HeicQueue.clean(ONE_HOUR, 500, "completed");
const ONE_HOUR = 60 * 60 * 1000;
const SIX_HOURS = 6 * ONE_HOUR;
// Clean failed jobs older than 24 hours
await HeicQueue.clean(SIX_HOURS, 500, "failed");
await heicQueue.clean(ONE_HOUR, 500, "completed");
await heicQueue.clean(SIX_HOURS, 500, "failed");
// Get queue health
const jobCounts = await HeicQueue.getJobCounts();
logger.log("debug", `Queue status: ${JSON.stringify(jobCounts)}`);
const counts = await heicQueue.getJobCounts();
logger.debug(`HEIC Queue status: ${JSON.stringify(counts)}`);
} catch (error) {
logger.log("error", `Queue cleanup error: ${error}`);
logger.error("HEIC Queue cleanup error:", error);
}
}
export async function ConvertHeicFiles(files: Express.Multer.File[]) {
const validFiles = await filterValidHeicFiles(files);
/**
* Filter files to include only valid HEIC images.
*/
async function filterHeicFiles(files: Express.Multer.File[]) {
const valid: Express.Multer.File[] = [];
for (const file of files) {
const type: FileTypeResult | undefined = await fileTypeFromFile(file.path);
if (type?.mime === "image/heic") valid.push(file);
}
return valid;
}
const jobs = validFiles.map((file) => ({
/**
* Handle original file based on environment variable.
*/
async function handleOriginalFile(fileInfo: { path: string; destination: string; originalFilename: string }) {
try {
if (process.env.KEEP_CONVERTED_ORIGINALS) {
const destDir = path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir);
await fs.ensureDir(destDir);
await fs.move(fileInfo.path, path.join(destDir, fileInfo.originalFilename), { overwrite: true });
} else {
await fs.unlink(fileInfo.path);
}
} catch (error) {
logger.error("Error handling original file:", error);
throw error;
}
}
/**
* Convert HEIC to JPEG using GraphicsMagick stream.
*/
async function convertToJpeg(inputPath: string, outputPath: string): Promise<string> {
return new Promise((resolve, reject) => {
const readStream = fs.createReadStream(inputPath);
const writeStream = fs.createWriteStream(outputPath);
gm(readStream)
.setFormat("jpg")
.stream()
.pipe(writeStream)
.on("finish", () => resolve(outputPath))
.on("error", reject);
});
}
/**
* Add HEIC files to the conversion queue and wait for job completion.
*/
export async function convertHeicFiles(files: Express.Multer.File[]) {
const heicFiles = await filterHeicFiles(files);
if (heicFiles.length === 0) {
logger.debug("No HEIC files found to convert.");
return;
}
const jobs = heicFiles.map((file) => ({
name: file.filename,
data: {
convertedFileName: generateUniqueHeicFilename(file),
@@ -75,134 +127,99 @@ export async function ConvertHeicFiles(files: Express.Multer.File[]) {
}
}));
await HeicQueue.addBulk(jobs);
const fileMap = new Map(files.map((file, index) => [file.filename, index]));
jobs.forEach((job) => {
const fileIndex = fileMap.get(job.data.fileInfo.originalFilename);
if (fileIndex !== undefined) {
files[fileIndex].filename = job.data.convertedFileName;
files[fileIndex].mimetype = "image/jpeg";
}
});
}
async function filterValidHeicFiles(files: Express.Multer.File[]) {
const validFiles = [];
for (const file of files) {
const type: FileTypeResult | undefined = await fileTypeFromFile(file.path);
if (type?.mime === "image/heic") {
validFiles.push(file);
// Add jobs and wait for completion of each before proceeding
for (const jobData of jobs) {
try {
const job = await heicQueue.add(jobData.name, jobData.data);
await job.waitUntilFinished(heicQueueEvents);
logger.debug(`Job ${job.id} finished successfully.`);
} catch (error) {
logger.error(`Job for ${jobData.data.fileInfo.originalFilename} failed:`, error);
// Depending on your error handling strategy you might rethrow or continue
}
}
return validFiles;
}
async function handleOriginalFile(fileInfo: { path: string; destination: string; originalFilename: string }) {
try {
if (process.env.KEEP_CONVERTED_ORIGINALS) {
await fs.ensureDir(path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir));
await fs.move(
fileInfo.path,
`${path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir)}/${fileInfo.originalFilename}`
);
} else {
await fs.unlink(fileInfo.path);
// Update original files list with new names, mimetype, and path
const filenameToIndex = new Map(files.map((f, i) => [f.filename, i]));
for (const { data } of jobs) {
const idx = filenameToIndex.get(data.fileInfo.originalFilename);
if (idx !== undefined) {
const oldPath = files[idx].path;
files[idx].filename = data.convertedFileName;
files[idx].mimetype = "image/jpeg";
files[idx].path = path.join(data.fileInfo.destination, data.convertedFileName);
logger.debug(`Updated file entry: ${data.fileInfo.originalFilename} -> ${data.convertedFileName}`, {
oldPath,
newPath: files[idx].path,
newMimetype: files[idx].mimetype
});
}
} catch (error) {
logger.log("error", `Error handling original file: ${error}`);
throw error;
}
}
async function ConvertToJpeg(file: string, newPath: string) {
// const fileOnDisk: Buffer = await fs.readFile(file);
// return new Promise<string>((resolve, reject) => {
// imageMagick(fileOnDisk)
// .setFormat("jpg")
// .write(newPath, (error) => {
// if (error) reject(error.message);
// resolve(newPath);
// });
// });
return new Promise<string>((resolve, reject) => {
const readStream = fs.createReadStream(file);
const writeStream = fs.createWriteStream(newPath);
imageMagick(readStream)
.setFormat("jpg")
.stream()
.pipe(writeStream)
.on("finish", () => resolve(newPath))
.on("error", (error) => reject(error.message));
});
}
const HeicWorker = new Worker(
"HEIC Queue",
async (job: Job) => {
// Worker processing HEIC conversion jobs
const heicWorker = new Worker(
QUEUE_NAME,
async (
job: Job<{
fileInfo: { path: string; destination: string; originalFilename: string };
convertedFileName: string;
}>
) => {
const { fileInfo, convertedFileName } = job.data;
try {
logger.log("debug", `Attempting to Convert ${fileInfo.originalFilename} image to JPEG from HEIC.`);
logger.debug(`Converting ${fileInfo.originalFilename} from HEIC to JPEG.`);
await job.updateProgress(10);
await ConvertToJpeg(fileInfo.path, `${fileInfo.destination}/${convertedFileName}`);
const outputPath = path.join(fileInfo.destination, convertedFileName);
await convertToJpeg(fileInfo.path, outputPath);
await job.updateProgress(50);
await handleOriginalFile(fileInfo);
logger.log("debug", `Converted ${fileInfo.originalFilename} image to JPEG from HEIC.`);
logger.debug(`Successfully converted ${fileInfo.originalFilename} to JPEG.`);
await job.updateProgress(100);
return true;
} catch (error) {
logger.log(
"error",
`QUEUE ERROR: Error converting ${fileInfo.originalFilename} image to JPEG from HEIC. ${JSON.stringify(error)}`
);
logger.error(`Error converting ${fileInfo.originalFilename}:`, error);
throw error;
}
},
{
connection: {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: function (err) {
const targetError = "READONLY";
return err.message.includes(targetError);
}
},
connection: connectionOpts,
concurrency: 1
}
);
HeicQueue.on("waiting", (job) => {
logger.log("debug", `[BULLMQ] Job is waiting in queue! ${job.data.convertedFileName}`);
// Event listeners for queue and worker
heicQueue.on("waiting", (job) => {
logger.debug(`[heicQueue] Job waiting in queue: ${job.data.convertedFileName}`);
});
HeicQueue.on("error", (error) => {
logger.log("error", `[BULLMQ] Queue Error! ${error}`);
heicQueue.on("error", (error) => {
logger.error(`[heicQueue] Queue error:`, error);
});
HeicWorker.on("ready", () => {
logger.log("debug", `[BULLMQ] Worker Ready`);
heicWorker.on("ready", () => {
logger.debug("[heicWorker] Worker ready");
});
HeicWorker.on("active", (job, prev) => {
logger.log("debug", `[BULLMQ] Job ${job.id} is now active; previous status was ${prev}`);
heicWorker.on("active", (job, prev) => {
logger.debug(`[heicWorker] Job ${job.id} active (previous: ${prev})`);
});
HeicWorker.on("completed", async (job, returnvalue) => {
logger.log("debug", `[BULLMQ] ${job.id} has completed and returned ${returnvalue}`);
heicWorker.on("completed", async (job) => {
logger.debug(`[heicWorker] Job ${job.id} completed`);
await job.remove();
logger.log("debug", `Job ${job.id} removed from Redis`);
logger.debug(`Job ${job.id} removed from Redis`);
});
HeicWorker.on("failed", (jobId, failedReason) => {
logger.log("error", `[BULLMQ] ${jobId} has failed with reason ${failedReason}`);
heicWorker.on("failed", (jobId, reason) => {
logger.error(`[heicWorker] Job ${jobId} failed: ${reason}`);
});
HeicWorker.on("error", (error) => {
logger.log("error", `[BULLMQ] There was a queue error! ${error}`);
heicWorker.on("error", (error) => {
logger.error(`[heicWorker] Worker error:`, error);
});
HeicWorker.on("stalled", (error) => {
logger.log("error", `[BULLMQ] There was a worker stall! ${error}`);
heicWorker.on("stalled", (job) => {
logger.error(`[heicWorker] Worker stalled: ${job}`);
});
HeicWorker.on("ioredis:close", () => {
logger.log("error", `[BULLMQ] Redis connection closed!`);
heicWorker.on("ioredis:close", () => {
logger.error("[heicWorker] Redis connection closed");
});

13
util/interfaces/MediaFile.ts
View File

@@ -1,11 +1,14 @@
import internal from "stream";
import { FileTypeResult } from "file-type/core";
interface MediaFile {
export default interface MediaFile {
type?: FileTypeResult | undefined;
size?: number;
src: string;
filename: string;
name: string;
path: string;
thumbnailPath: string;
thumbnail: string;
thumbnailHeight: number;
thumbnailWidth: number;
filename: string;
}
export default MediaFile;

18
util/serverInit.ts
View File

@@ -8,17 +8,21 @@ dotenv.config({
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const RootDirectory = process.env.MEDIA_PATH!.replace("~", os.homedir);
// Resolve root directory, supporting ~ for home
const RootDirectory = (process.env.MEDIA_PATH || "").replace("~", os.homedir);
// Folder names
const JobsFolder = "Jobs";
const VendorsFolder = "Vendors";
// Folder paths object
export const FolderPaths = {
Root: RootDirectory,
Jobs: path.join(RootDirectory, JobsFolder),
Vendors: path.join(RootDirectory, VendorsFolder),
ThumbsSubDir: "/thumbs",
BillsSubDir: "/bills",
ConvertedOriginalSubDir: "/ConvertedOriginal",
ThumbsSubDir: "thumbs",
BillsSubDir: "bills",
ConvertedOriginalSubDir: "ConvertedOriginal",
StaticPath: "/static",
JobsFolder,
VendorsFolder
@@ -28,19 +32,25 @@ export const AssetPaths = {
File: "/assets/file.png"
};
// Utility functions for relative file paths
export function JobRelativeFilePath(jobid: string, filename: string) {
return path.join(FolderPaths.Jobs, jobid, filename);
}
export function BillsRelativeFilePath(jobid: string, filename: string) {
return path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, filename);
}
// Ensure all required directories exist at startup
export default function InitServer() {
logger.info(`Ensuring Root media path exists: ${FolderPaths.Root}`);
ensureDirSync(FolderPaths.Root);
logger.info(`Ensuring Jobs media path exists: ${FolderPaths.Jobs}`);
ensureDirSync(FolderPaths.Jobs);
logger.info(`Ensuring Vendors media path exists: ${FolderPaths.Vendors}`);
ensureDirSync(FolderPaths.Vendors);
logger.info("Folder Paths", FolderPaths);
logger.info("IMS Token set to: " + (process.env.IMS_TOKEN || "").trim());
}

27
util/validateToken.ts
View File

@@ -1,23 +1,32 @@
import dotenv from "dotenv";
import { NextFunction, Request, Response } from "express";
import { resolve } from "path";
import { logger } from "../server.js";
dotenv.config({
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
export default function ValidateImsToken(req: Request, res: Response, next: NextFunction) {
const jobid: string = (req.body.jobid || "").trim();
try {
const IMS_TOKEN: string = (process.env.IMS_TOKEN || "").trim();
const IMS_TOKEN: string = (process.env.IMS_TOKEN || "").trim();
if (IMS_TOKEN === "") {
next();
} else {
if (req.headers.ims_token !== IMS_TOKEN) {
res.sendStatus(401);
} else {
if (!IMS_TOKEN) {
logger.debug("IMS_TOKEN not set, skipping token validation.");
next();
return;
}
const token = req.headers.ims_token || req.headers["ims-token"] || req.headers["x-ims-token"];
if (token !== IMS_TOKEN) {
logger.warn("Invalid IMS token provided.", { provided: token });
res.sendStatus(401);
return;
}
next();
} catch (error) {
logger.error("Error validating IMS token.", { error: (error as Error).message });
if (!res.headersSent) res.status(500).json({ error: "Error validating IMS token." });
}
}