admin/bodyshop-media-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.0.14

Browse Source
This commit is contained in:
Allan Carr
2025-07-13 00:17:08 -07:00
parent 231130267f
commit 7f782d5a64
19 changed files with 2564 additions and 1416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
bills/billsUploadMedia.ts
View File

@@ -1,3 +1,4 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import dotenv from "dotenv";
import { Request, Response } from "express";
import fs from "fs-extra";
@@ -6,7 +7,7 @@ import path, { resolve } from "path";
import { logger } from "../server.js";
import GenerateThumbnail from "../util/generateThumbnail.js";
import { generateUniqueBillFilename } from "../util/generateUniqueFilename.js";
import { ConvertHeicFiles } from "../util/heicConverter.js";
import { convertHeicFiles } from "../util/heicConverter.js";
import { PathToRoBillsFolder, PathToVendorBillsFile } from "../util/pathGenerators.js";
import { BillsListMedia } from "./billsListMedia.js";
@@ -14,72 +15,209 @@ dotenv.config({
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const BILLS_UPLOAD_QUEUE_NAME = "billsUploadProcessingQueue";
const connectionOpts = {
host: "localhost",
port: 6379,
maxRetriesPerRequest: 3,
enableReadyCheck: true,
reconnectOnError: (err: Error) => err.message.includes("READONLY")
};
const billsUploadProcessingQueue = new Queue(BILLS_UPLOAD_QUEUE_NAME, {
connection: connectionOpts,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
backoff: { type: "exponential", delay: 1000 }
}
});
const billsUploadQueueEvents = new QueueEvents(BILLS_UPLOAD_QUEUE_NAME, {
connection: connectionOpts
});
// Bills upload processing worker
const billsUploadWorker = new Worker(
BILLS_UPLOAD_QUEUE_NAME,
async (
job: Job<{
files: Express.Multer.File[];
vendorid: string;
invoice_number: string;
skipThumbnails: boolean;
duplicateToVendor: boolean;
}>
) => {
const { files, vendorid, invoice_number, skipThumbnails, duplicateToVendor } = job.data;
try {
logger.debug(`Processing bills upload with ${files.length} files`);
await job.updateProgress(10);
// Convert HEIC files if any
await convertHeicFiles(files);
await job.updateProgress(30);
if (!skipThumbnails) {
// Filter out HEIC files since they've been converted to JPEG already
const filesToThumbnail = files.filter((file) => {
const isHeic =
file.mimetype === "image/heic" ||
file.mimetype === "image/heif" ||
/\.heic$/i.test(file.filename) ||
/\.heif$/i.test(file.filename);
return !isHeic;
});
logger.debug(
`Generating thumbnails for ${filesToThumbnail.length} bill files (excluding HEIC)`,
filesToThumbnail.map((f) => f.filename)
);
// Generate thumbnails
const thumbnailPromises = filesToThumbnail.map((file) => GenerateThumbnail(file.path));
await Promise.all(thumbnailPromises);
await job.updateProgress(60);
}
// Duplicate to vendor folder if enabled
if (duplicateToVendor && vendorid && invoice_number) {
await Promise.all(
files.map(async (file) => {
const target = path.join(PathToVendorBillsFile(vendorid), file.filename);
await fs.ensureDir(path.dirname(target));
await fs.copyFile(file.path, target);
})
);
await job.updateProgress(90);
}
logger.debug(`Bills upload processing completed`);
await job.updateProgress(100);
return { success: true, filesProcessed: files.length };
} catch (error) {
logger.error(`Error processing bills upload:`, error);
throw error;
}
},
{
connection: connectionOpts,
concurrency: 2 // Allow 2 bills upload processing jobs to run concurrently
}
);
// Event listeners for bills upload queue and worker
billsUploadProcessingQueue.on("waiting", (job) => {
logger.debug(`[BillsUploadQueue] Job waiting: ${job.data.invoice_number}`);
});
billsUploadProcessingQueue.on("error", (error) => {
logger.error(`[BillsUploadQueue] Queue error:`, error);
});
billsUploadWorker.on("ready", () => {
logger.debug("[BillsUploadWorker] Worker ready");
});
billsUploadWorker.on("active", (job, prev) => {
logger.debug(`[BillsUploadWorker] Job ${job.id} active (previous: ${prev})`);
});
billsUploadWorker.on("completed", async (job) => {
logger.debug(`[BillsUploadWorker] Job ${job.id} completed`);
await job.remove();
logger.debug(`[BillsUploadWorker] Job ${job.id} removed from Redis`);
});
billsUploadWorker.on("failed", (jobId, reason) => {
logger.error(`[BillsUploadWorker] Job ${jobId} failed: ${reason}`);
});
billsUploadWorker.on("error", (error) => {
logger.error(`[BillsUploadWorker] Worker error:`, error);
});
billsUploadWorker.on("stalled", (job) => {
logger.error(`[BillsUploadWorker] Worker stalled: ${job}`);
});
billsUploadWorker.on("ioredis:close", () => {
logger.error("[BillsUploadWorker] Redis connection closed");
});
export const BillsMediaUploadMulter = multer({
storage: multer.diskStorage({
destination: function (req, file, cb) {
destination(req, file, cb) {
const jobid: string = (req.body.jobid || "").trim();
const DestinationFolder: string = PathToRoBillsFolder(jobid);
fs.ensureDirSync(DestinationFolder);
cb(jobid === "" || jobid === null ? new Error("Job ID not specified.") : null, DestinationFolder);
if (!jobid) {
cb(new Error("Job ID not specified."), "");
return;
}
const destinationFolder = PathToRoBillsFolder(jobid);
fs.ensureDir(destinationFolder)
.then(() => cb(null, destinationFolder))
.catch((err) => cb(err as Error, ""));
},
filename: function (req, file, cb) {
logger.info("Uploading file: ", {
file: path.basename(file.originalname)
});
filename(req, file, cb) {
logger.info("Uploading file:", { file: path.basename(file.originalname) });
const invoice_number: string = (req.body.invoice_number || "").trim();
cb(
invoice_number === "" || invoice_number === null ? new Error("Invoice number not specified.") : null,
generateUniqueBillFilename(file, invoice_number)
);
if (!invoice_number) {
cb(new Error("Invoice number not specified."), "");
return;
}
cb(null, generateUniqueBillFilename(file, invoice_number));
}
})
});
export async function BillsUploadMedia(req: Request, res: Response) {
try {
if (!req.files) {
res.send({
const files = req.files as Express.Multer.File[] | undefined;
if (!files || files.length === 0) {
logger.warning("Upload contained no files.");
res.status(400).json({
status: false,
message: "No file uploaded"
});
} else {
await ConvertHeicFiles(req.files as Express.Multer.File[]);
const thumbnailGenerationQueue: Promise<string>[] = [];
//for each file.path, generate the thumbnail.
(req.files as Express.Multer.File[]).forEach((file) => {
thumbnailGenerationQueue.push(GenerateThumbnail(file.path));
});
await Promise.all(thumbnailGenerationQueue);
//Check to see if it should be duplicated to the vendor.
if (process.env.DUPLICATE_BILL_TO_VENDOR) {
const copyQueue: Promise<void>[] = [];
//for each file.path, generate the thumbnail.
(req.files as Express.Multer.File[]).forEach((file) => {
const vendorid: string = (req.body.vendorid || "").trim();
const invoice_number: string = (req.body.invoice_number || "").trim();
copyQueue.push(
(async () => {
const target: string = path.join(PathToVendorBillsFile(vendorid), file.filename);
await fs.ensureDir(path.dirname(target));
await fs.copyFile(file.path, target);
})()
);
//Copy Queue is not awaited - we don't care if it finishes before we serve up the thumbnails from the jobs directory.
});
}
BillsListMedia(req, res);
return;
}
const vendorid: string = (req.body.vendorid || "").trim();
const invoice_number: string = (req.body.invoice_number || "").trim();
const skipThumbnails = req.body.skip_thumbnail === "true";
const duplicateToVendor = process.env.DUPLICATE_BILL_TO_VENDOR === "true";
// If skipping thumbnails, queue job without waiting and return immediately
if (skipThumbnails) {
await billsUploadProcessingQueue.add("processBillsUpload", {
files,
vendorid,
invoice_number,
skipThumbnails: true,
duplicateToVendor
});
res.sendStatus(200);
return;
}
// Add bills upload processing job to queue and wait for completion
logger.debug(`Adding bills upload processing job for ${files.length} files`);
const job = await billsUploadProcessingQueue.add("processBillsUpload", {
files,
vendorid,
invoice_number,
skipThumbnails: false,
duplicateToVendor
});
// Wait for the job to complete
await job.waitUntilFinished(billsUploadQueueEvents);
logger.debug(`Bills upload processing job completed`);
// Delegate to BillsListMedia to respond with updated media list
await BillsListMedia(req, res);
} catch (error) {
logger.error("Error while uploading Bill Media", {
files: req.files,
error
error: (error as Error).message
});
res.status(500).send(error);
res.status(500).json({ error: (error as Error).message });
}
}