Merged in feature/IO-3281-imgproxy-download-hotfix (pull request #2376)

IO-3281 resolve imgproxy download failures.

client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx

@@ -98,7 +98,13 @@ function JobsDocumentsImgproxyComponent({
               jobId={jobId}
               totalSize={totalSize}
               billId={billId}
-              callbackAfterUpload={billsCallback || fetchThumbnails || refetch}
+              callbackAfterUpload={
+                billsCallback ||
+                function () {
+                  isFunction(refetch) && refetch();
+                  isFunction(fetchThumbnails) && fetchThumbnails();
+                }
+              }
               ignoreSizeLimit={ignoreSizeLimit}
             />
           </Card>

package-lock.json

@@ -63,7 +63,8 @@
         "winston": "^3.17.0",
         "winston-cloudwatch": "^6.3.0",
         "xml2js": "^0.6.2",
-        "xmlbuilder2": "^3.1.1"
+        "xmlbuilder2": "^3.1.1",
+        "yazl": "^3.3.1"
       },
       "devDependencies": {
         "@eslint/js": "^9.28.0",
@@ -13057,6 +13058,15 @@
         "node": ">=8"
       }
     },
+    "node_modules/yazl": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz",
+      "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==",
+      "license": "MIT",
+      "dependencies": {
+        "buffer-crc32": "^1.0.0"
+      }
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

package.json

@@ -70,7 +70,8 @@
     "winston": "^3.17.0",
     "winston-cloudwatch": "^6.3.0",
     "xml2js": "^0.6.2",
-    "xmlbuilder2": "^3.1.1"
+    "xmlbuilder2": "^3.1.1",
+    "yazl": "^3.3.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.28.0",

server/media/imgproxy-media.js

@@ -20,6 +20,7 @@ const {
   GET_DOCUMENTS_BY_IDS,
   DELETE_MEDIA_DOCUMENTS
 } = require("../graphql-client/queries");
+const yazl = require("yazl");
 
 const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN.
 const imgproxySalt = process.env.IMGPROXY_SALT;
@@ -174,55 +175,45 @@ const downloadFiles = async (req, res) => {
   try {
     logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids });
 
-    //Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
     const client = req.userGraphQLClient;
-
-    //Query for the keys of the document IDs
     const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
 
-    //Using the Keys, get all the S3 links, zip them, and send back to the client.
     const s3client = new S3Client({ region: InstanceRegion() });
-    const archiveStream = archiver("zip");
+    const zipfile = new yazl.ZipFile();
-
-    archiveStream.on("error", (error) => {
-      console.error("Archival encountered an error:", error);
-      throw new Error(error);
-    });
-
     const passThrough = new stream.PassThrough();
 
-    archiveStream.pipe(passThrough);
+    // Pipe the zipfile output to the passThrough stream
+    zipfile.outputStream.pipe(passThrough);
 
-    for (const key of data.documents.map((d) => d.key)) {
+    // Add each file to the zip as a stream
+    for (const doc of data.documents) {
+      const key = doc.key;
       const response = await s3client.send(
         new GetObjectCommand({
           Bucket: imgproxyDestinationBucket,
           Key: key
         })
       );
-
+      // response.Body is a readable stream
-      archiveStream.append(response.Body, { name: path.basename(key) });
+      zipfile.addReadStream(response.Body, path.basename(key));
     }
 
-    await archiveStream.finalize();
+    // Finalize the zip after all files are added
+    zipfile.end();
 
     const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`;
 
+    // Upload the zip stream to S3
     const parallelUploads3 = new Upload({
       client: s3client,
-      queueSize: 4, // optional concurrency configuration
+      queueSize: 4,
-      leavePartsOnError: false, // optional manually handle dropped parts
+      leavePartsOnError: false,
       params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough }
     });
 
-    // Disabled progress logging for upload, uncomment if needed
-    // parallelUploads3.on("httpUploadProgress", (progress) => {
-    //   console.log(progress);
-    // });
-
     await parallelUploads3.done();
 
-    //Generate the presigned URL to download it.
+    // Generate the presigned URL to download it.
     const presignedUrl = await getSignedUrl(
       s3client,
       new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }),
@@ -230,9 +221,8 @@ const downloadFiles = async (req, res) => {
     );
 
     return res.json({ success: true, url: presignedUrl });
-    //Iterate over them, build the link based on the media type, and return the array.
   } catch (error) {
-    logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobId, {
+    logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, {
       jobId,
       billid,
       message: error.message,