From c1e1dff7d2533e23521e5ed80ea91d4509ffdd46 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 08:51:41 -0700 Subject: [PATCH] IO-3281 resolve imgproxy download failures. --- ...s-documents-imgproxy-gallery.component.jsx | 8 +++- package-lock.json | 12 +++++- package.json | 3 +- server/media/imgproxy-media.js | 42 +++++++------------ 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx index f99485dc8..8ada3616f 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx @@ -98,7 +98,13 @@ function JobsDocumentsImgproxyComponent({ jobId={jobId} totalSize={totalSize} billId={billId} - callbackAfterUpload={billsCallback || fetchThumbnails || refetch} + callbackAfterUpload={ + billsCallback || + function () { + isFunction(refetch) && refetch(); + isFunction(fetchThumbnails) && fetchThumbnails(); + } + } ignoreSizeLimit={ignoreSizeLimit} /> diff --git a/package-lock.json b/package-lock.json index 82ec89bb2..79cc67430 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.28.0", @@ -13057,6 +13058,15 @@ "node": ">=8" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index b57c9f6b2..7b9279bbf 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.28.0", diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index e30aee90e..c6ea1a9ce 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -20,6 +20,7 @@ const { GET_DOCUMENTS_BY_IDS, DELETE_MEDIA_DOCUMENTS } = require("../graphql-client/queries"); +const yazl = require("yazl"); const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN. const imgproxySalt = process.env.IMGPROXY_SALT; @@ -174,55 +175,45 @@ const downloadFiles = async (req, res) => { try { logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); - //Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components. const client = req.userGraphQLClient; - - //Query for the keys of the document IDs const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); - //Using the Keys, get all the S3 links, zip them, and send back to the client. const s3client = new S3Client({ region: InstanceRegion() }); - const archiveStream = archiver("zip"); - - archiveStream.on("error", (error) => { - console.error("Archival encountered an error:", error); - throw new Error(error); - }); - + const zipfile = new yazl.ZipFile(); const passThrough = new stream.PassThrough(); - archiveStream.pipe(passThrough); + // Pipe the zipfile output to the passThrough stream + zipfile.outputStream.pipe(passThrough); - for (const key of data.documents.map((d) => d.key)) { + // Add each file to the zip as a stream + for (const doc of data.documents) { + const key = doc.key; const response = await s3client.send( new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key }) ); - - archiveStream.append(response.Body, { name: path.basename(key) }); + // response.Body is a readable stream + zipfile.addReadStream(response.Body, path.basename(key)); } - await archiveStream.finalize(); + // Finalize the zip after all files are added + zipfile.end(); const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`; + // Upload the zip stream to S3 const parallelUploads3 = new Upload({ client: s3client, - queueSize: 4, // optional concurrency configuration - leavePartsOnError: false, // optional manually handle dropped parts + queueSize: 4, + leavePartsOnError: false, params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough } }); - // Disabled progress logging for upload, uncomment if needed - // parallelUploads3.on("httpUploadProgress", (progress) => { - // console.log(progress); - // }); - await parallelUploads3.done(); - //Generate the presigned URL to download it. + // Generate the presigned URL to download it. const presignedUrl = await getSignedUrl( s3client, new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }), @@ -230,9 +221,8 @@ const downloadFiles = async (req, res) => { ); return res.json({ success: true, url: presignedUrl }); - //Iterate over them, build the link based on the media type, and return the array. } catch (error) { - logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobId, { + logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { jobId, billid, message: error.message,