1.0.14
This commit is contained in:
Allan Carr
@@ -1,7 +1,8 @@
|
||||
import { Job, Queue, QueueEvents, Worker } from "bullmq";
|
||||
import { fileTypeFromFile } from "file-type";
|
||||
import { FileTypeResult } from "file-type/core";
|
||||
import fs from "fs-extra";
|
||||
import { access } from "fs/promises";
|
||||
import { access, FileHandle, open as fsOpen } from "fs/promises";
|
||||
import gm from "gm";
|
||||
import imageThumbnail from "image-thumbnail";
|
||||
import path from "path";
|
||||
@@ -9,75 +10,187 @@ import { logger } from "../server.js";
|
||||
import { AssetPaths, FolderPaths } from "./serverInit.js";
|
||||
//@ts-ignore
|
||||
import simpleThumb from "simple-thumbnail";
|
||||
//const ffmpeg = require("ffmpeg-static");
|
||||
|
||||
/** @returns {string} Returns the relative path from the file to the thumbnail on the server. This must be converted to a URL. */
|
||||
export default async function GenerateThumbnail(file: string) {
|
||||
// const type: core.FileTypeResult | undefined = await ft.fileTypeFromFile(file);
|
||||
const type: FileTypeResult | undefined = await fileTypeFromFile(file);
|
||||
let thumbPath: string = path.join(
|
||||
path.dirname(file),
|
||||
FolderPaths.ThumbsSubDir,
|
||||
`${path.parse(path.basename(file)).name}.jpg`
|
||||
);
|
||||
const QUEUE_NAME = "thumbnailQueue";
|
||||
|
||||
const connectionOpts = {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
maxRetriesPerRequest: 3,
|
||||
enableReadyCheck: true,
|
||||
reconnectOnError: (err: Error) => err.message.includes("READONLY")
|
||||
};
|
||||
|
||||
const thumbnailQueue = new Queue(QUEUE_NAME, {
|
||||
connection: connectionOpts,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 3,
|
||||
backoff: { type: "exponential", delay: 1000 }
|
||||
}
|
||||
});
|
||||
|
||||
const thumbnailQueueEvents = new QueueEvents(QUEUE_NAME, {
|
||||
connection: connectionOpts
|
||||
});
|
||||
|
||||
const thumbnailWorker = new Worker(
|
||||
QUEUE_NAME,
|
||||
async (job: Job<{ file: string }>) => {
|
||||
const { file } = job.data;
|
||||
logger.debug(`[ThumbnailWorker] Starting thumbnail generation for ${file}`);
|
||||
try {
|
||||
await job.updateProgress(10);
|
||||
|
||||
const result = await processThumbnail(file, job);
|
||||
|
||||
await job.updateProgress(100);
|
||||
logger.debug(`[ThumbnailWorker] Completed thumbnail generation for ${file}`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(`[ThumbnailWorker] Error generating thumbnail for ${file}:`, error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
connection: connectionOpts,
|
||||
concurrency: 1
|
||||
}
|
||||
);
|
||||
|
||||
// Worker event listeners for logging
|
||||
thumbnailWorker.on("ready", () => {
|
||||
logger.debug("[ThumbnailWorker] Worker is ready");
|
||||
});
|
||||
thumbnailWorker.on("active", (job, prev) => {
|
||||
logger.debug(`[ThumbnailWorker] Job ${job.id} active (previous: ${prev})`);
|
||||
});
|
||||
thumbnailWorker.on("completed", async (job) => {
|
||||
logger.debug(`[ThumbnailWorker] Job ${job.id} completed`);
|
||||
await job.remove();
|
||||
logger.debug(`[ThumbnailWorker] Job ${job.id} removed from Redis`);
|
||||
});
|
||||
thumbnailWorker.on("failed", (job, err) => {
|
||||
logger.error(`[ThumbnailWorker] Job ${job?.id} failed:`, err);
|
||||
});
|
||||
thumbnailWorker.on("stalled", (jobId) => {
|
||||
logger.error(`[ThumbnailWorker] Job stalled: ${jobId}`);
|
||||
});
|
||||
thumbnailWorker.on("error", (err) => {
|
||||
logger.error("[ThumbnailWorker] Worker error:", err);
|
||||
});
|
||||
thumbnailWorker.on("ioredis:close", () => {
|
||||
logger.error("[ThumbnailWorker] Redis connection closed");
|
||||
});
|
||||
|
||||
// Queue event listeners
|
||||
thumbnailQueue.on("waiting", (job) => {
|
||||
logger.debug(`[ThumbnailQueue] Job waiting in queue: ${job.data.file}`);
|
||||
});
|
||||
thumbnailQueue.on("error", (err) => {
|
||||
logger.error("[ThumbnailQueue] Queue error:", err);
|
||||
});
|
||||
|
||||
async function processThumbnail(file: string, job?: Job): Promise<string> {
|
||||
let fileHandle: FileHandle | null = null;
|
||||
let thumbPath = "unknown";
|
||||
|
||||
try {
|
||||
//Ensure the thumbs directory exists.
|
||||
await access(file, fs.constants.R_OK);
|
||||
|
||||
// Wait for stable file size to avoid incomplete thumbnails
|
||||
const size1 = (await fs.stat(file)).size;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
const size2 = (await fs.stat(file)).size;
|
||||
if (size1 !== size2) {
|
||||
throw new Error("File is still being written to, skipping thumbnail generation");
|
||||
}
|
||||
|
||||
fileHandle = await fsOpen(file, "r");
|
||||
|
||||
const type: FileTypeResult | undefined = await fileTypeFromFile(file);
|
||||
const baseName = path.parse(path.basename(file)).name;
|
||||
|
||||
thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`);
|
||||
await fs.ensureDir(path.dirname(thumbPath));
|
||||
|
||||
//Check for existing thumbnail
|
||||
if (await fs.pathExists(thumbPath)) {
|
||||
logger.debug("Using existing thumbnail:", thumbPath);
|
||||
logger.debug(`[ThumbnailWorker] Using existing thumbnail: ${thumbPath}`);
|
||||
return path.relative(path.dirname(file), thumbPath);
|
||||
}
|
||||
|
||||
//Check to see if the file is an image, PDF, or video.
|
||||
if (!type?.mime) {
|
||||
throw new Error("Unknown file type");
|
||||
}
|
||||
if (!type?.mime) throw new Error("Unknown file type");
|
||||
|
||||
if (type?.mime === "application/pdf" || type?.mime === "image/heic" || type?.mime === "image/heif") {
|
||||
await GeneratePdfThumbnail(file, thumbPath);
|
||||
} else if (type?.mime.startsWith("video")) {
|
||||
await simpleThumb(file, thumbPath, "250x?", {
|
||||
// path: ffmpeg,
|
||||
});
|
||||
await fileHandle.close();
|
||||
fileHandle = null;
|
||||
|
||||
if (["application/pdf", "image/heic", "image/heif"].includes(type.mime)) {
|
||||
logger.debug(`[ThumbnailWorker] Generating PDF/HEIC thumbnail for: ${file}`);
|
||||
await generatePdfThumbnail(file, thumbPath);
|
||||
} else if (type.mime.startsWith("video")) {
|
||||
logger.debug(`[ThumbnailWorker] Generating video thumbnail for: ${file}`);
|
||||
await simpleThumb(file, thumbPath, "250x?");
|
||||
} else {
|
||||
logger.debug("Thumbnail being created for : " + thumbPath);
|
||||
//Ignoring typescript as the interface is lacking a parameter.
|
||||
// @ts-ignore
|
||||
const thumbnail = await imageThumbnail(file, {
|
||||
logger.debug(`[ThumbnailWorker] Generating image thumbnail for: ${file}`);
|
||||
const thumbnailBuffer = await imageThumbnail(file, {
|
||||
responseType: "buffer",
|
||||
height: 250,
|
||||
width: 250,
|
||||
failOnError: false
|
||||
width: 250
|
||||
});
|
||||
await fs.writeFile(thumbPath, thumbnail);
|
||||
await fs.writeFile(thumbPath, thumbnailBuffer);
|
||||
}
|
||||
|
||||
return path.relative(path.dirname(file), thumbPath);
|
||||
} catch (err) {
|
||||
logger.error("Error when genenerating thumbnail:", {
|
||||
} catch (error) {
|
||||
logger.error("[ThumbnailWorker] Error generating thumbnail:", {
|
||||
thumbPath,
|
||||
err,
|
||||
message: (err as Error).message
|
||||
error,
|
||||
message: (error as Error).message
|
||||
});
|
||||
return path.relative(path.dirname(file), AssetPaths.File);
|
||||
} finally {
|
||||
if (fileHandle) {
|
||||
try {
|
||||
await fileHandle.close();
|
||||
} catch (closeError) {
|
||||
logger.error("[ThumbnailWorker] Error closing file handle:", closeError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function GeneratePdfThumbnail(file: string, thumbPath: string) {
|
||||
const fileOnDisk: Buffer = await fs.readFile(file);
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
gm(fileOnDisk)
|
||||
.selectFrame(0)
|
||||
async function generatePdfThumbnail(file: string, thumbPath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
gm(file + "[0]") // first page only
|
||||
.setFormat("jpg")
|
||||
.resize(250, 250, "!")
|
||||
.quality(75)
|
||||
.write(thumbPath, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(thumbPath);
|
||||
}
|
||||
});
|
||||
.write(thumbPath, (err) => (err ? reject(err) : resolve(thumbPath)));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a thumbnail generation job and wait for result.
|
||||
* Returns the relative path to the thumbnail or default file icon on error.
|
||||
*/
|
||||
export default async function GenerateThumbnail(file: string): Promise<string> {
|
||||
const baseName = path.parse(path.basename(file)).name;
|
||||
const thumbPath = path.join(path.dirname(file), FolderPaths.ThumbsSubDir, `${baseName}.jpg`);
|
||||
|
||||
if (await fs.pathExists(thumbPath)) {
|
||||
logger.debug(`[GenerateThumbnail] Returning existing thumbnail immediately for ${file}`);
|
||||
return path.relative(path.dirname(file), thumbPath);
|
||||
}
|
||||
|
||||
try {
|
||||
logger.debug(`[GenerateThumbnail] Adding job to queue for ${file}`);
|
||||
const job = await thumbnailQueue.add("generate", { file });
|
||||
const result = await job.waitUntilFinished(thumbnailQueueEvents);
|
||||
logger.debug(`[GenerateThumbnail] Job completed for ${file}`);
|
||||
return result as string;
|
||||
} catch (error) {
|
||||
logger.error(`[GenerateThumbnail] Job failed for ${file}:`, error);
|
||||
return path.relative(path.dirname(file), AssetPaths.File);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Job, Queue, Worker } from "bullmq";
|
||||
import { Job, Queue, QueueEvents, Worker } from "bullmq";
|
||||
import dotenv from "dotenv";
|
||||
import { fileTypeFromFile } from "file-type";
|
||||
import { FileTypeResult } from "file-type/core";
|
||||
@@ -10,60 +10,112 @@ import { logger } from "../server.js";
|
||||
import { generateUniqueHeicFilename } from "./generateUniqueFilename.js";
|
||||
import { FolderPaths } from "./serverInit.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const HeicQueue = new Queue("HEIC Queue", {
|
||||
connection: {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
maxRetriesPerRequest: 3,
|
||||
enableReadyCheck: true,
|
||||
reconnectOnError: function (err) {
|
||||
const targetError = "READONLY";
|
||||
return err.message.includes(targetError);
|
||||
}
|
||||
},
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 1000
|
||||
}
|
||||
}
|
||||
});
|
||||
const cleanupINTERVAL = 1000 * 60 * 10;
|
||||
setInterval(cleanupQueue, cleanupINTERVAL);
|
||||
|
||||
dotenv.config({
|
||||
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
||||
});
|
||||
|
||||
const imageMagick = gm.subClass({ imageMagick: true });
|
||||
const QUEUE_NAME = "heicQueue";
|
||||
|
||||
const connectionOpts = {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
maxRetriesPerRequest: 3,
|
||||
enableReadyCheck: true,
|
||||
reconnectOnError: (err: Error) => err.message.includes("READONLY")
|
||||
};
|
||||
|
||||
const heicQueue = new Queue(QUEUE_NAME, {
|
||||
connection: connectionOpts,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 3,
|
||||
backoff: { type: "exponential", delay: 1000 }
|
||||
}
|
||||
});
|
||||
|
||||
// Re-added QueueEvents for waiting on job completion
|
||||
const heicQueueEvents = new QueueEvents(QUEUE_NAME, {
|
||||
connection: connectionOpts
|
||||
});
|
||||
|
||||
const CLEANUP_INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
|
||||
setInterval(cleanupQueue, CLEANUP_INTERVAL_MS);
|
||||
|
||||
async function cleanupQueue() {
|
||||
const ONE_HOUR = 1000 * 60 * 60;
|
||||
const SIX_HOURS = ONE_HOUR * 6;
|
||||
try {
|
||||
// Clean completed jobs older than 1 hour
|
||||
await HeicQueue.clean(ONE_HOUR, 500, "completed");
|
||||
const ONE_HOUR = 60 * 60 * 1000;
|
||||
const SIX_HOURS = 6 * ONE_HOUR;
|
||||
|
||||
// Clean failed jobs older than 24 hours
|
||||
await HeicQueue.clean(SIX_HOURS, 500, "failed");
|
||||
await heicQueue.clean(ONE_HOUR, 500, "completed");
|
||||
await heicQueue.clean(SIX_HOURS, 500, "failed");
|
||||
|
||||
// Get queue health
|
||||
const jobCounts = await HeicQueue.getJobCounts();
|
||||
logger.log("debug", `Queue status: ${JSON.stringify(jobCounts)}`);
|
||||
const counts = await heicQueue.getJobCounts();
|
||||
logger.debug(`HEIC Queue status: ${JSON.stringify(counts)}`);
|
||||
} catch (error) {
|
||||
logger.log("error", `Queue cleanup error: ${error}`);
|
||||
logger.error("HEIC Queue cleanup error:", error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function ConvertHeicFiles(files: Express.Multer.File[]) {
|
||||
const validFiles = await filterValidHeicFiles(files);
|
||||
/**
|
||||
* Filter files to include only valid HEIC images.
|
||||
*/
|
||||
async function filterHeicFiles(files: Express.Multer.File[]) {
|
||||
const valid: Express.Multer.File[] = [];
|
||||
for (const file of files) {
|
||||
const type: FileTypeResult | undefined = await fileTypeFromFile(file.path);
|
||||
if (type?.mime === "image/heic") valid.push(file);
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
|
||||
const jobs = validFiles.map((file) => ({
|
||||
/**
|
||||
* Handle original file based on environment variable.
|
||||
*/
|
||||
async function handleOriginalFile(fileInfo: { path: string; destination: string; originalFilename: string }) {
|
||||
try {
|
||||
if (process.env.KEEP_CONVERTED_ORIGINALS) {
|
||||
const destDir = path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir);
|
||||
await fs.ensureDir(destDir);
|
||||
await fs.move(fileInfo.path, path.join(destDir, fileInfo.originalFilename), { overwrite: true });
|
||||
} else {
|
||||
await fs.unlink(fileInfo.path);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Error handling original file:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert HEIC to JPEG using GraphicsMagick stream.
|
||||
*/
|
||||
async function convertToJpeg(inputPath: string, outputPath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const readStream = fs.createReadStream(inputPath);
|
||||
const writeStream = fs.createWriteStream(outputPath);
|
||||
|
||||
gm(readStream)
|
||||
.setFormat("jpg")
|
||||
.stream()
|
||||
.pipe(writeStream)
|
||||
.on("finish", () => resolve(outputPath))
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add HEIC files to the conversion queue and wait for job completion.
|
||||
*/
|
||||
export async function convertHeicFiles(files: Express.Multer.File[]) {
|
||||
const heicFiles = await filterHeicFiles(files);
|
||||
|
||||
if (heicFiles.length === 0) {
|
||||
logger.debug("No HEIC files found to convert.");
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = heicFiles.map((file) => ({
|
||||
name: file.filename,
|
||||
data: {
|
||||
convertedFileName: generateUniqueHeicFilename(file),
|
||||
@@ -75,134 +127,99 @@ export async function ConvertHeicFiles(files: Express.Multer.File[]) {
|
||||
}
|
||||
}));
|
||||
|
||||
await HeicQueue.addBulk(jobs);
|
||||
|
||||
const fileMap = new Map(files.map((file, index) => [file.filename, index]));
|
||||
jobs.forEach((job) => {
|
||||
const fileIndex = fileMap.get(job.data.fileInfo.originalFilename);
|
||||
if (fileIndex !== undefined) {
|
||||
files[fileIndex].filename = job.data.convertedFileName;
|
||||
files[fileIndex].mimetype = "image/jpeg";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function filterValidHeicFiles(files: Express.Multer.File[]) {
|
||||
const validFiles = [];
|
||||
for (const file of files) {
|
||||
const type: FileTypeResult | undefined = await fileTypeFromFile(file.path);
|
||||
if (type?.mime === "image/heic") {
|
||||
validFiles.push(file);
|
||||
// Add jobs and wait for completion of each before proceeding
|
||||
for (const jobData of jobs) {
|
||||
try {
|
||||
const job = await heicQueue.add(jobData.name, jobData.data);
|
||||
await job.waitUntilFinished(heicQueueEvents);
|
||||
logger.debug(`Job ${job.id} finished successfully.`);
|
||||
} catch (error) {
|
||||
logger.error(`Job for ${jobData.data.fileInfo.originalFilename} failed:`, error);
|
||||
// Depending on your error handling strategy you might rethrow or continue
|
||||
}
|
||||
}
|
||||
return validFiles;
|
||||
}
|
||||
|
||||
async function handleOriginalFile(fileInfo: { path: string; destination: string; originalFilename: string }) {
|
||||
try {
|
||||
if (process.env.KEEP_CONVERTED_ORIGINALS) {
|
||||
await fs.ensureDir(path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir));
|
||||
await fs.move(
|
||||
fileInfo.path,
|
||||
`${path.join(fileInfo.destination, FolderPaths.ConvertedOriginalSubDir)}/${fileInfo.originalFilename}`
|
||||
);
|
||||
} else {
|
||||
await fs.unlink(fileInfo.path);
|
||||
// Update original files list with new names, mimetype, and path
|
||||
const filenameToIndex = new Map(files.map((f, i) => [f.filename, i]));
|
||||
for (const { data } of jobs) {
|
||||
const idx = filenameToIndex.get(data.fileInfo.originalFilename);
|
||||
if (idx !== undefined) {
|
||||
const oldPath = files[idx].path;
|
||||
files[idx].filename = data.convertedFileName;
|
||||
files[idx].mimetype = "image/jpeg";
|
||||
files[idx].path = path.join(data.fileInfo.destination, data.convertedFileName);
|
||||
logger.debug(`Updated file entry: ${data.fileInfo.originalFilename} -> ${data.convertedFileName}`, {
|
||||
oldPath,
|
||||
newPath: files[idx].path,
|
||||
newMimetype: files[idx].mimetype
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.log("error", `Error handling original file: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function ConvertToJpeg(file: string, newPath: string) {
|
||||
// const fileOnDisk: Buffer = await fs.readFile(file);
|
||||
|
||||
// return new Promise<string>((resolve, reject) => {
|
||||
// imageMagick(fileOnDisk)
|
||||
// .setFormat("jpg")
|
||||
// .write(newPath, (error) => {
|
||||
// if (error) reject(error.message);
|
||||
// resolve(newPath);
|
||||
// });
|
||||
// });
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const readStream = fs.createReadStream(file);
|
||||
const writeStream = fs.createWriteStream(newPath);
|
||||
|
||||
imageMagick(readStream)
|
||||
.setFormat("jpg")
|
||||
.stream()
|
||||
.pipe(writeStream)
|
||||
.on("finish", () => resolve(newPath))
|
||||
.on("error", (error) => reject(error.message));
|
||||
});
|
||||
}
|
||||
|
||||
const HeicWorker = new Worker(
|
||||
"HEIC Queue",
|
||||
async (job: Job) => {
|
||||
// Worker processing HEIC conversion jobs
|
||||
const heicWorker = new Worker(
|
||||
QUEUE_NAME,
|
||||
async (
|
||||
job: Job<{
|
||||
fileInfo: { path: string; destination: string; originalFilename: string };
|
||||
convertedFileName: string;
|
||||
}>
|
||||
) => {
|
||||
const { fileInfo, convertedFileName } = job.data;
|
||||
try {
|
||||
logger.log("debug", `Attempting to Convert ${fileInfo.originalFilename} image to JPEG from HEIC.`);
|
||||
logger.debug(`Converting ${fileInfo.originalFilename} from HEIC to JPEG.`);
|
||||
await job.updateProgress(10);
|
||||
await ConvertToJpeg(fileInfo.path, `${fileInfo.destination}/${convertedFileName}`);
|
||||
|
||||
const outputPath = path.join(fileInfo.destination, convertedFileName);
|
||||
await convertToJpeg(fileInfo.path, outputPath);
|
||||
|
||||
await job.updateProgress(50);
|
||||
await handleOriginalFile(fileInfo);
|
||||
logger.log("debug", `Converted ${fileInfo.originalFilename} image to JPEG from HEIC.`);
|
||||
|
||||
logger.debug(`Successfully converted ${fileInfo.originalFilename} to JPEG.`);
|
||||
await job.updateProgress(100);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.log(
|
||||
"error",
|
||||
`QUEUE ERROR: Error converting ${fileInfo.originalFilename} image to JPEG from HEIC. ${JSON.stringify(error)}`
|
||||
);
|
||||
logger.error(`Error converting ${fileInfo.originalFilename}:`, error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
connection: {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
maxRetriesPerRequest: 3,
|
||||
enableReadyCheck: true,
|
||||
reconnectOnError: function (err) {
|
||||
const targetError = "READONLY";
|
||||
return err.message.includes(targetError);
|
||||
}
|
||||
},
|
||||
connection: connectionOpts,
|
||||
concurrency: 1
|
||||
}
|
||||
);
|
||||
|
||||
HeicQueue.on("waiting", (job) => {
|
||||
logger.log("debug", `[BULLMQ] Job is waiting in queue! ${job.data.convertedFileName}`);
|
||||
// Event listeners for queue and worker
|
||||
heicQueue.on("waiting", (job) => {
|
||||
logger.debug(`[heicQueue] Job waiting in queue: ${job.data.convertedFileName}`);
|
||||
});
|
||||
HeicQueue.on("error", (error) => {
|
||||
logger.log("error", `[BULLMQ] Queue Error! ${error}`);
|
||||
heicQueue.on("error", (error) => {
|
||||
logger.error(`[heicQueue] Queue error:`, error);
|
||||
});
|
||||
|
||||
HeicWorker.on("ready", () => {
|
||||
logger.log("debug", `[BULLMQ] Worker Ready`);
|
||||
heicWorker.on("ready", () => {
|
||||
logger.debug("[heicWorker] Worker ready");
|
||||
});
|
||||
HeicWorker.on("active", (job, prev) => {
|
||||
logger.log("debug", `[BULLMQ] Job ${job.id} is now active; previous status was ${prev}`);
|
||||
heicWorker.on("active", (job, prev) => {
|
||||
logger.debug(`[heicWorker] Job ${job.id} active (previous: ${prev})`);
|
||||
});
|
||||
HeicWorker.on("completed", async (job, returnvalue) => {
|
||||
logger.log("debug", `[BULLMQ] ${job.id} has completed and returned ${returnvalue}`);
|
||||
heicWorker.on("completed", async (job) => {
|
||||
logger.debug(`[heicWorker] Job ${job.id} completed`);
|
||||
await job.remove();
|
||||
logger.log("debug", `Job ${job.id} removed from Redis`);
|
||||
logger.debug(`Job ${job.id} removed from Redis`);
|
||||
});
|
||||
HeicWorker.on("failed", (jobId, failedReason) => {
|
||||
logger.log("error", `[BULLMQ] ${jobId} has failed with reason ${failedReason}`);
|
||||
heicWorker.on("failed", (jobId, reason) => {
|
||||
logger.error(`[heicWorker] Job ${jobId} failed: ${reason}`);
|
||||
});
|
||||
HeicWorker.on("error", (error) => {
|
||||
logger.log("error", `[BULLMQ] There was a queue error! ${error}`);
|
||||
heicWorker.on("error", (error) => {
|
||||
logger.error(`[heicWorker] Worker error:`, error);
|
||||
});
|
||||
HeicWorker.on("stalled", (error) => {
|
||||
logger.log("error", `[BULLMQ] There was a worker stall! ${error}`);
|
||||
heicWorker.on("stalled", (job) => {
|
||||
logger.error(`[heicWorker] Worker stalled: ${job}`);
|
||||
});
|
||||
HeicWorker.on("ioredis:close", () => {
|
||||
logger.log("error", `[BULLMQ] Redis connection closed!`);
|
||||
heicWorker.on("ioredis:close", () => {
|
||||
logger.error("[heicWorker] Redis connection closed");
|
||||
});
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import internal from "stream";
|
||||
import { FileTypeResult } from "file-type/core";
|
||||
|
||||
interface MediaFile {
|
||||
export default interface MediaFile {
|
||||
type?: FileTypeResult | undefined;
|
||||
size?: number;
|
||||
src: string;
|
||||
filename: string;
|
||||
name: string;
|
||||
path: string;
|
||||
thumbnailPath: string;
|
||||
thumbnail: string;
|
||||
thumbnailHeight: number;
|
||||
thumbnailWidth: number;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
export default MediaFile;
|
||||
|
||||
@@ -8,17 +8,21 @@ dotenv.config({
|
||||
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
||||
});
|
||||
|
||||
const RootDirectory = process.env.MEDIA_PATH!.replace("~", os.homedir);
|
||||
// Resolve root directory, supporting ~ for home
|
||||
const RootDirectory = (process.env.MEDIA_PATH || "").replace("~", os.homedir);
|
||||
|
||||
// Folder names
|
||||
const JobsFolder = "Jobs";
|
||||
const VendorsFolder = "Vendors";
|
||||
|
||||
// Folder paths object
|
||||
export const FolderPaths = {
|
||||
Root: RootDirectory,
|
||||
Jobs: path.join(RootDirectory, JobsFolder),
|
||||
Vendors: path.join(RootDirectory, VendorsFolder),
|
||||
ThumbsSubDir: "/thumbs",
|
||||
BillsSubDir: "/bills",
|
||||
ConvertedOriginalSubDir: "/ConvertedOriginal",
|
||||
ThumbsSubDir: "thumbs",
|
||||
BillsSubDir: "bills",
|
||||
ConvertedOriginalSubDir: "ConvertedOriginal",
|
||||
StaticPath: "/static",
|
||||
JobsFolder,
|
||||
VendorsFolder
|
||||
@@ -28,19 +32,25 @@ export const AssetPaths = {
|
||||
File: "/assets/file.png"
|
||||
};
|
||||
|
||||
// Utility functions for relative file paths
|
||||
export function JobRelativeFilePath(jobid: string, filename: string) {
|
||||
return path.join(FolderPaths.Jobs, jobid, filename);
|
||||
}
|
||||
export function BillsRelativeFilePath(jobid: string, filename: string) {
|
||||
return path.join(FolderPaths.Jobs, jobid, FolderPaths.BillsSubDir, filename);
|
||||
}
|
||||
|
||||
// Ensure all required directories exist at startup
|
||||
export default function InitServer() {
|
||||
logger.info(`Ensuring Root media path exists: ${FolderPaths.Root}`);
|
||||
ensureDirSync(FolderPaths.Root);
|
||||
|
||||
logger.info(`Ensuring Jobs media path exists: ${FolderPaths.Jobs}`);
|
||||
ensureDirSync(FolderPaths.Jobs);
|
||||
|
||||
logger.info(`Ensuring Vendors media path exists: ${FolderPaths.Vendors}`);
|
||||
ensureDirSync(FolderPaths.Vendors);
|
||||
|
||||
logger.info("Folder Paths", FolderPaths);
|
||||
logger.info("IMS Token set to: " + (process.env.IMS_TOKEN || "").trim());
|
||||
}
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
import dotenv from "dotenv";
|
||||
import { NextFunction, Request, Response } from "express";
|
||||
import { resolve } from "path";
|
||||
import { logger } from "../server.js";
|
||||
|
||||
dotenv.config({
|
||||
path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
||||
});
|
||||
|
||||
export default function ValidateImsToken(req: Request, res: Response, next: NextFunction) {
|
||||
const jobid: string = (req.body.jobid || "").trim();
|
||||
try {
|
||||
const IMS_TOKEN: string = (process.env.IMS_TOKEN || "").trim();
|
||||
|
||||
const IMS_TOKEN: string = (process.env.IMS_TOKEN || "").trim();
|
||||
|
||||
if (IMS_TOKEN === "") {
|
||||
next();
|
||||
} else {
|
||||
if (req.headers.ims_token !== IMS_TOKEN) {
|
||||
res.sendStatus(401);
|
||||
} else {
|
||||
if (!IMS_TOKEN) {
|
||||
logger.debug("IMS_TOKEN not set, skipping token validation.");
|
||||
next();
|
||||
return;
|
||||
}
|
||||
|
||||
const token = req.headers.ims_token || req.headers["ims-token"] || req.headers["x-ims-token"];
|
||||
if (token !== IMS_TOKEN) {
|
||||
logger.warn("Invalid IMS token provided.", { provided: token });
|
||||
res.sendStatus(401);
|
||||
return;
|
||||
}
|
||||
|
||||
next();
|
||||
} catch (error) {
|
||||
logger.error("Error validating IMS token.", { error: (error as Error).message });
|
||||
if (!res.headersSent) res.status(500).json({ error: "Error validating IMS token." });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user