Merge branch 'master-AIO' into feature/IO-3255-simplified-part-management

server/data/usageReport.js

@@ -35,7 +35,7 @@ exports.default = async (req, res) => {
     //Query the usage data.
     const queryResults = await client.request(queries.STATUS_UPDATE, {
       today: moment().startOf("day").subtract(7, "days"),
-      period: moment().subtract(90, "days").startOf("day")
+      period: moment().subtract(365, "days").startOf("day")
     });
 
     //Massage the data.
@@ -66,7 +66,7 @@ exports.default = async (req, res) => {
 Usage Report for ${moment().format("MM/DD/YYYY")} for Rome Online Customers. 
         
 Notes:
-  - Days Since Creation: The number of days since the shop was created. Only shops created in the last 90 days are included.
+  - Days Since Creation: The number of days since the shop was created. Only shops created in the last 365 days are included.
   - Updated values should be higher than created values.
   - Counts are inclusive of the last 7 days of data.
         `,

server/media/imgproxy-media.js

@@ -20,6 +20,7 @@ const {
   GET_DOCUMENTS_BY_IDS,
   DELETE_MEDIA_DOCUMENTS
 } = require("../graphql-client/queries");
+const yazl = require("yazl");
 
 const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN.
 const imgproxySalt = process.env.IMGPROXY_SALT;
@@ -102,13 +103,7 @@ const getThumbnailUrls = async (req, res) => {
       //<Cloudfront_to_lambda>/<hmac with SHA of entire request URI path (with base64 encoded URL if needed), beginning with un-encoded/un-hashed Salt>/<remainder of url - resize params >/< base 64 URL encoded to image path>
       //When working with documents from Cloudinary, the URL does not include the extension.
 
-      let key;
-
-      if (/\.[^/.]+$/.test(document.key)) {
-        key = document.key;
-      } else {
-        key = `${document.key}.${document.extension.toLowerCase()}`;
-      }
+      let key = keyStandardize(document)
       // Build the S3 path to the object.
       const fullS3Path = `s3://${imgproxyDestinationBucket}/${key}`;
       const base64UrlEncodedKeyString = base64UrlEncode(fullS3Path);
@@ -168,78 +163,73 @@ const getThumbnailUrls = async (req, res) => {
  * @returns {Promise<*>}
  */
 const downloadFiles = async (req, res) => {
-  //Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk
   const { jobId, billid, documentids } = req.body;
 
+  logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids });
+
+  const client = req.userGraphQLClient;
+  let data;
   try {
-    logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids });
-
-    //Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
-    const client = req.userGraphQLClient;
-
-    //Query for the keys of the document IDs
-    const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
-
-    //Using the Keys, get all the S3 links, zip them, and send back to the client.
-    const s3client = new S3Client({ region: InstanceRegion() });
-    const archiveStream = archiver("zip");
-
-    archiveStream.on("error", (error) => {
-      console.error("Archival encountered an error:", error);
-      throw new Error(error);
-    });
-
-    const passThrough = new stream.PassThrough();
-
-    archiveStream.pipe(passThrough);
-
-    for (const key of data.documents.map((d) => d.key)) {
-      const response = await s3client.send(
-        new GetObjectCommand({
-          Bucket: imgproxyDestinationBucket,
-          Key: key
-        })
-      );
-
-      archiveStream.append(response.Body, { name: path.basename(key) });
-    }
-
-    await archiveStream.finalize();
-
-    const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`;
-
-    const parallelUploads3 = new Upload({
-      client: s3client,
-      queueSize: 4, // optional concurrency configuration
-      leavePartsOnError: false, // optional manually handle dropped parts
-      params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough }
-    });
-
-    // Disabled progress logging for upload, uncomment if needed
-    // parallelUploads3.on("httpUploadProgress", (progress) => {
-    //   console.log(progress);
-    // });
-
-    await parallelUploads3.done();
-
-    //Generate the presigned URL to download it.
-    const presignedUrl = await getSignedUrl(
-      s3client,
-      new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }),
-      { expiresIn: 360 }
-    );
-
-    return res.json({ success: true, url: presignedUrl });
-    //Iterate over them, build the link based on the media type, and return the array.
+    data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
   } catch (error) {
-    logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobId, {
+    logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, {
       jobId,
       billid,
       message: error.message,
       stack: error.stack
     });
+    return res.status(400).json({ message: error.message });
+  }
 
-    return res.status(400).json({ message: error.message, stack: error.stack });
+  const s3client = new S3Client({ region: InstanceRegion() });
+  const zipfile = new yazl.ZipFile();
+
+  const filename = `archive-${jobId || "na"}-${new Date().toISOString().replace(/[:.]/g, "-")}.zip`;
+  res.setHeader("Content-Type", "application/zip");
+  res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
+
+  // Handle zipfile stream errors
+  zipfile.outputStream.on("error", (err) => {
+    logger.log("imgproxy-download-zipstream-error", "ERROR", req.user?.email, jobId, { message: err.message, stack: err.stack });
+    // Cannot send another response here, just destroy the connection
+    res.destroy(err);
+  });
+
+  zipfile.outputStream.pipe(res);
+
+  try {
+    for (const doc of data.documents) {
+      let key = keyStandardize(doc)
+      let response;
+      try {
+        response = await s3client.send(
+          new GetObjectCommand({
+            Bucket: imgproxyDestinationBucket,
+            Key: key
+          })
+        );
+      } catch (err) {
+        logger.log("imgproxy-download-s3-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack });
+        // Optionally, skip this file or add a placeholder file in the zip
+        continue;
+      }
+      // Attach error handler to S3 stream
+      response.Body.on("error", (err) => {
+        logger.log("imgproxy-download-s3stream-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack });
+        res.destroy(err);
+      });
+      zipfile.addReadStream(response.Body, path.basename(key));
+    }
+    zipfile.end();
+  } catch (error) {
+    logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, {
+      jobId,
+      billid,
+      message: error.message,
+      stack: error.stack
+    });
+    // Cannot send another response here, just destroy the connection
+    res.destroy(error);
   }
 };
 
@@ -392,6 +382,15 @@ const moveFiles = async (req, res) => {
   }
 };
 
+const keyStandardize = (doc) => {
+  if (/\.[^/.]+$/.test(doc.key)) {
+    return doc.key;
+  } else {
+    return `${doc.key}.${doc.extension.toLowerCase()}`;
+  }
+};
+
+
 module.exports = {
   generateSignedUploadUrls,
   getThumbnailUrls,