From 633d5668f091b78dbff32fd57a58aafb71c9c412 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Mon, 23 Jun 2025 15:08:36 -0700 Subject: [PATCH 1/6] IO-3279 Set usage report to 1 year. --- server/data/usageReport.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/data/usageReport.js b/server/data/usageReport.js index af1d6ec75..ead4b0bba 100644 --- a/server/data/usageReport.js +++ b/server/data/usageReport.js @@ -35,7 +35,7 @@ exports.default = async (req, res) => { //Query the usage data. const queryResults = await client.request(queries.STATUS_UPDATE, { today: moment().startOf("day").subtract(7, "days"), - period: moment().subtract(90, "days").startOf("day") + period: moment().subtract(365, "days").startOf("day") }); //Massage the data. @@ -66,7 +66,7 @@ exports.default = async (req, res) => { Usage Report for ${moment().format("MM/DD/YYYY")} for Rome Online Customers. Notes: - - Days Since Creation: The number of days since the shop was created. Only shops created in the last 90 days are included. + - Days Since Creation: The number of days since the shop was created. Only shops created in the last 365 days are included. - Updated values should be higher than created values. - Counts are inclusive of the last 7 days of data. `, From c1e1dff7d2533e23521e5ed80ea91d4509ffdd46 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 08:51:41 -0700 Subject: [PATCH 2/6] IO-3281 resolve imgproxy download failures. --- ...s-documents-imgproxy-gallery.component.jsx | 8 +++- package-lock.json | 12 +++++- package.json | 3 +- server/media/imgproxy-media.js | 42 +++++++------------ 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx index f99485dc8..8ada3616f 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx @@ -98,7 +98,13 @@ function JobsDocumentsImgproxyComponent({ jobId={jobId} totalSize={totalSize} billId={billId} - callbackAfterUpload={billsCallback || fetchThumbnails || refetch} + callbackAfterUpload={ + billsCallback || + function () { + isFunction(refetch) && refetch(); + isFunction(fetchThumbnails) && fetchThumbnails(); + } + } ignoreSizeLimit={ignoreSizeLimit} /> diff --git a/package-lock.json b/package-lock.json index 82ec89bb2..79cc67430 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.28.0", @@ -13057,6 +13058,15 @@ "node": ">=8" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index b57c9f6b2..7b9279bbf 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.28.0", diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index e30aee90e..c6ea1a9ce 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -20,6 +20,7 @@ const { GET_DOCUMENTS_BY_IDS, DELETE_MEDIA_DOCUMENTS } = require("../graphql-client/queries"); +const yazl = require("yazl"); const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN. const imgproxySalt = process.env.IMGPROXY_SALT; @@ -174,55 +175,45 @@ const downloadFiles = async (req, res) => { try { logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); - //Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components. const client = req.userGraphQLClient; - - //Query for the keys of the document IDs const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); - //Using the Keys, get all the S3 links, zip them, and send back to the client. const s3client = new S3Client({ region: InstanceRegion() }); - const archiveStream = archiver("zip"); - - archiveStream.on("error", (error) => { - console.error("Archival encountered an error:", error); - throw new Error(error); - }); - + const zipfile = new yazl.ZipFile(); const passThrough = new stream.PassThrough(); - archiveStream.pipe(passThrough); + // Pipe the zipfile output to the passThrough stream + zipfile.outputStream.pipe(passThrough); - for (const key of data.documents.map((d) => d.key)) { + // Add each file to the zip as a stream + for (const doc of data.documents) { + const key = doc.key; const response = await s3client.send( new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key }) ); - - archiveStream.append(response.Body, { name: path.basename(key) }); + // response.Body is a readable stream + zipfile.addReadStream(response.Body, path.basename(key)); } - await archiveStream.finalize(); + // Finalize the zip after all files are added + zipfile.end(); const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`; + // Upload the zip stream to S3 const parallelUploads3 = new Upload({ client: s3client, - queueSize: 4, // optional concurrency configuration - leavePartsOnError: false, // optional manually handle dropped parts + queueSize: 4, + leavePartsOnError: false, params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough } }); - // Disabled progress logging for upload, uncomment if needed - // parallelUploads3.on("httpUploadProgress", (progress) => { - // console.log(progress); - // }); - await parallelUploads3.done(); - //Generate the presigned URL to download it. + // Generate the presigned URL to download it. const presignedUrl = await getSignedUrl( s3client, new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }), @@ -230,9 +221,8 @@ const downloadFiles = async (req, res) => { ); return res.json({ success: true, url: presignedUrl }); - //Iterate over them, build the link based on the media type, and return the array. } catch (error) { - logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobId, { + logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { jobId, billid, message: error.message, From 27d28e7ffcc141d2da8d72df8fe3cb943ec884e3 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 09:42:45 -0700 Subject: [PATCH 3/6] IO-3281 Adjust zip to stream. --- ...nt-imgproxy-gallery.download.component.jsx | 31 ++++++++++--------- .../shop-employees-form.component.jsx | 2 +- server/media/imgproxy-media.js | 31 +++++-------------- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx index 8644115fd..9e49d0d97 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx @@ -56,22 +56,25 @@ export function JobsDocumentsImgproxyDownloadButton({ bodyshop, galleryImages, i const handleDownload = async () => { logImEXEvent("jobs_documents_download"); setLoading(true); - const zipUrl = await axios({ - url: "/media/imgproxy/download", - method: "POST", - data: { jobId, documentids: imagesToDownload.map((_) => _.id) } - }); + try { + const response = await axios({ + url: "/media/imgproxy/download", + method: "POST", + responseType: "blob", + data: { jobId, documentids: imagesToDownload.map((_) => _.id) }, + onDownloadProgress: downloadProgress + }); - const theDownloadedZip = await cleanAxios({ - url: zipUrl.data.url, - method: "GET", - responseType: "arraybuffer", - onDownloadProgress: downloadProgress - }); - setLoading(false); - setDownload(null); + setLoading(false); + setDownload(null); - standardMediaDownload(theDownloadedZip.data); + // Use the response data (Blob) to trigger download + standardMediaDownload(response.data); + } catch (error) { + setLoading(false); + setDownload(null); + // handle error (optional) + } }; return ( diff --git a/client/src/components/shop-employees/shop-employees-form.component.jsx b/client/src/components/shop-employees/shop-employees-form.component.jsx index 563ad835b..a44e26d4a 100644 --- a/client/src/components/shop-employees/shop-employees-form.component.jsx +++ b/client/src/components/shop-employees/shop-employees-form.component.jsx @@ -383,7 +383,7 @@ export function ShopEmployeesFormComponent({ bodyshop }) { title={() => } columns={columns} rowKey={"id"} - dataSource={data ? data.employees_by_pk.employee_vacations : []} + dataSource={data?.employees_by_pk?.employee_vacations ?? []} /> ); diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index c6ea1a9ce..b11f4b539 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -180,10 +180,14 @@ const downloadFiles = async (req, res) => { const s3client = new S3Client({ region: InstanceRegion() }); const zipfile = new yazl.ZipFile(); - const passThrough = new stream.PassThrough(); - // Pipe the zipfile output to the passThrough stream - zipfile.outputStream.pipe(passThrough); + // Set response headers for zip download + const filename = `archive-${jobId || "na"}-${new Date().toISOString().replace(/[:.]/g, "-")}.zip`; + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", `attachment; filename="${filename}"`); + + // Pipe the zipfile output directly to the response + zipfile.outputStream.pipe(res); // Add each file to the zip as a stream for (const doc of data.documents) { @@ -200,27 +204,8 @@ const downloadFiles = async (req, res) => { // Finalize the zip after all files are added zipfile.end(); + // No need to send a JSON response, as the zip is streamed directly - const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`; - - // Upload the zip stream to S3 - const parallelUploads3 = new Upload({ - client: s3client, - queueSize: 4, - leavePartsOnError: false, - params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough } - }); - - await parallelUploads3.done(); - - // Generate the presigned URL to download it. - const presignedUrl = await getSignedUrl( - s3client, - new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }), - { expiresIn: 360 } - ); - - return res.json({ success: true, url: presignedUrl }); } catch (error) { logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { jobId, From f2a2653eae004ecac7c52e8688db812dc624ec44 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 15:36:03 -0700 Subject: [PATCH 4/6] IO-3281 Prevent broken stream reseting HTTP headers. --- ...cument-imgproxy-gallery.download.component.jsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx index 9e49d0d97..65140d6b0 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx @@ -46,11 +46,16 @@ export function JobsDocumentsImgproxyDownloadButton({ bodyshop, galleryImages, i } function standardMediaDownload(bufferData) { - const a = document.createElement("a"); - const url = window.URL.createObjectURL(new Blob([bufferData])); - a.href = url; - a.download = `${identifier || "documents"}.zip`; - a.click(); + try { + const a = document.createElement("a"); + const url = window.URL.createObjectURL(new Blob([bufferData])); + a.href = url; + a.download = `${identifier || "documents"}.zip`; + a.click(); + } catch (error) { + setLoading(false); + setDownload(null); + } } const handleDownload = async () => { From 0c80abb3cab3238601dc7a6865f55bfa4185f600 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 15:48:06 -0700 Subject: [PATCH 5/6] IO-3281 missed file in previous commit. --- server/media/imgproxy-media.js | 91 +++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index b11f4b539..55d88f938 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -169,43 +169,14 @@ const getThumbnailUrls = async (req, res) => { * @returns {Promise<*>} */ const downloadFiles = async (req, res) => { - //Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk const { jobId, billid, documentids } = req.body; + logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); + + const client = req.userGraphQLClient; + let data; try { - logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); - - const client = req.userGraphQLClient; - const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); - - const s3client = new S3Client({ region: InstanceRegion() }); - const zipfile = new yazl.ZipFile(); - - // Set response headers for zip download - const filename = `archive-${jobId || "na"}-${new Date().toISOString().replace(/[:.]/g, "-")}.zip`; - res.setHeader("Content-Type", "application/zip"); - res.setHeader("Content-Disposition", `attachment; filename="${filename}"`); - - // Pipe the zipfile output directly to the response - zipfile.outputStream.pipe(res); - - // Add each file to the zip as a stream - for (const doc of data.documents) { - const key = doc.key; - const response = await s3client.send( - new GetObjectCommand({ - Bucket: imgproxyDestinationBucket, - Key: key - }) - ); - // response.Body is a readable stream - zipfile.addReadStream(response.Body, path.basename(key)); - } - - // Finalize the zip after all files are added - zipfile.end(); - // No need to send a JSON response, as the zip is streamed directly - + data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); } catch (error) { logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { jobId, @@ -213,8 +184,58 @@ const downloadFiles = async (req, res) => { message: error.message, stack: error.stack }); + return res.status(400).json({ message: error.message }); + } - return res.status(400).json({ message: error.message, stack: error.stack }); + const s3client = new S3Client({ region: InstanceRegion() }); + const zipfile = new yazl.ZipFile(); + + const filename = `archive-${jobId || "na"}-${new Date().toISOString().replace(/[:.]/g, "-")}.zip`; + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", `attachment; filename="${filename}"`); + + // Handle zipfile stream errors + zipfile.outputStream.on("error", (err) => { + logger.log("imgproxy-download-zipstream-error", "ERROR", req.user?.email, jobId, { message: err.message, stack: err.stack }); + // Cannot send another response here, just destroy the connection + res.destroy(err); + }); + + zipfile.outputStream.pipe(res); + + try { + for (const doc of data.documents) { + const key = doc.key; + let response; + try { + response = await s3client.send( + new GetObjectCommand({ + Bucket: imgproxyDestinationBucket, + Key: key + }) + ); + } catch (err) { + logger.log("imgproxy-download-s3-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack }); + // Optionally, skip this file or add a placeholder file in the zip + continue; + } + // Attach error handler to S3 stream + response.Body.on("error", (err) => { + logger.log("imgproxy-download-s3stream-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack }); + res.destroy(err); + }); + zipfile.addReadStream(response.Body, path.basename(key)); + } + zipfile.end(); + } catch (error) { + logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { + jobId, + billid, + message: error.message, + stack: error.stack + }); + // Cannot send another response here, just destroy the connection + res.destroy(error); } }; From bd0c4ceae20a39a33fea01addc865b8b5bdd63fa Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 25 Jun 2025 16:32:47 -0700 Subject: [PATCH 6/6] IO-3281 Resolve key issue for downloads. --- server/media/imgproxy-media.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index 55d88f938..46a8dd840 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -103,13 +103,7 @@ const getThumbnailUrls = async (req, res) => { /////< base 64 URL encoded to image path> //When working with documents from Cloudinary, the URL does not include the extension. - let key; - - if (/\.[^/.]+$/.test(document.key)) { - key = document.key; - } else { - key = `${document.key}.${document.extension.toLowerCase()}`; - } + let key = keyStandardize(document) // Build the S3 path to the object. const fullS3Path = `s3://${imgproxyDestinationBucket}/${key}`; const base64UrlEncodedKeyString = base64UrlEncode(fullS3Path); @@ -205,7 +199,7 @@ const downloadFiles = async (req, res) => { try { for (const doc of data.documents) { - const key = doc.key; + let key = keyStandardize(doc) let response; try { response = await s3client.send( @@ -388,6 +382,15 @@ const moveFiles = async (req, res) => { } }; +const keyStandardize = (doc) => { + if (/\.[^/.]+$/.test(doc.key)) { + return doc.key; + } else { + return `${doc.key}.${doc.extension.toLowerCase()}`; + } +}; + + module.exports = { generateSignedUploadUrls, getThumbnailUrls,