diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx index 8644115fd..65140d6b0 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-document-imgproxy-gallery.download.component.jsx @@ -46,32 +46,40 @@ export function JobsDocumentsImgproxyDownloadButton({ bodyshop, galleryImages, i } function standardMediaDownload(bufferData) { - const a = document.createElement("a"); - const url = window.URL.createObjectURL(new Blob([bufferData])); - a.href = url; - a.download = `${identifier || "documents"}.zip`; - a.click(); + try { + const a = document.createElement("a"); + const url = window.URL.createObjectURL(new Blob([bufferData])); + a.href = url; + a.download = `${identifier || "documents"}.zip`; + a.click(); + } catch (error) { + setLoading(false); + setDownload(null); + } } const handleDownload = async () => { logImEXEvent("jobs_documents_download"); setLoading(true); - const zipUrl = await axios({ - url: "/media/imgproxy/download", - method: "POST", - data: { jobId, documentids: imagesToDownload.map((_) => _.id) } - }); + try { + const response = await axios({ + url: "/media/imgproxy/download", + method: "POST", + responseType: "blob", + data: { jobId, documentids: imagesToDownload.map((_) => _.id) }, + onDownloadProgress: downloadProgress + }); - const theDownloadedZip = await cleanAxios({ - url: zipUrl.data.url, - method: "GET", - responseType: "arraybuffer", - onDownloadProgress: downloadProgress - }); - setLoading(false); - setDownload(null); + setLoading(false); + setDownload(null); - standardMediaDownload(theDownloadedZip.data); + // Use the response data (Blob) to trigger download + standardMediaDownload(response.data); + } catch (error) { + setLoading(false); + setDownload(null); + // handle error (optional) + } }; return ( diff --git a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx index f99485dc8..8ada3616f 100644 --- a/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx +++ b/client/src/components/jobs-documents-imgproxy-gallery/jobs-documents-imgproxy-gallery.component.jsx @@ -98,7 +98,13 @@ function JobsDocumentsImgproxyComponent({ jobId={jobId} totalSize={totalSize} billId={billId} - callbackAfterUpload={billsCallback || fetchThumbnails || refetch} + callbackAfterUpload={ + billsCallback || + function () { + isFunction(refetch) && refetch(); + isFunction(fetchThumbnails) && fetchThumbnails(); + } + } ignoreSizeLimit={ignoreSizeLimit} /> diff --git a/client/src/components/shop-employees/shop-employees-form.component.jsx b/client/src/components/shop-employees/shop-employees-form.component.jsx index 563ad835b..a44e26d4a 100644 --- a/client/src/components/shop-employees/shop-employees-form.component.jsx +++ b/client/src/components/shop-employees/shop-employees-form.component.jsx @@ -383,7 +383,7 @@ export function ShopEmployeesFormComponent({ bodyshop }) { title={() => } columns={columns} rowKey={"id"} - dataSource={data ? data.employees_by_pk.employee_vacations : []} + dataSource={data?.employees_by_pk?.employee_vacations ?? []} /> ); diff --git a/package-lock.json b/package-lock.json index 58ac9324e..ac831e2ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,7 +63,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.29.0", @@ -13072,6 +13073,15 @@ "node": ">=8" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 61eaf07d8..199ae1f76 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "winston": "^3.17.0", "winston-cloudwatch": "^6.3.0", "xml2js": "^0.6.2", - "xmlbuilder2": "^3.1.1" + "xmlbuilder2": "^3.1.1", + "yazl": "^3.3.1" }, "devDependencies": { "@eslint/js": "^9.29.0", diff --git a/server/data/usageReport.js b/server/data/usageReport.js index af1d6ec75..ead4b0bba 100644 --- a/server/data/usageReport.js +++ b/server/data/usageReport.js @@ -35,7 +35,7 @@ exports.default = async (req, res) => { //Query the usage data. const queryResults = await client.request(queries.STATUS_UPDATE, { today: moment().startOf("day").subtract(7, "days"), - period: moment().subtract(90, "days").startOf("day") + period: moment().subtract(365, "days").startOf("day") }); //Massage the data. @@ -66,7 +66,7 @@ exports.default = async (req, res) => { Usage Report for ${moment().format("MM/DD/YYYY")} for Rome Online Customers. Notes: - - Days Since Creation: The number of days since the shop was created. Only shops created in the last 90 days are included. + - Days Since Creation: The number of days since the shop was created. Only shops created in the last 365 days are included. - Updated values should be higher than created values. - Counts are inclusive of the last 7 days of data. `, diff --git a/server/media/imgproxy-media.js b/server/media/imgproxy-media.js index e30aee90e..46a8dd840 100644 --- a/server/media/imgproxy-media.js +++ b/server/media/imgproxy-media.js @@ -20,6 +20,7 @@ const { GET_DOCUMENTS_BY_IDS, DELETE_MEDIA_DOCUMENTS } = require("../graphql-client/queries"); +const yazl = require("yazl"); const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN. const imgproxySalt = process.env.IMGPROXY_SALT; @@ -102,13 +103,7 @@ const getThumbnailUrls = async (req, res) => { /////< base 64 URL encoded to image path> //When working with documents from Cloudinary, the URL does not include the extension. - let key; - - if (/\.[^/.]+$/.test(document.key)) { - key = document.key; - } else { - key = `${document.key}.${document.extension.toLowerCase()}`; - } + let key = keyStandardize(document) // Build the S3 path to the object. const fullS3Path = `s3://${imgproxyDestinationBucket}/${key}`; const base64UrlEncodedKeyString = base64UrlEncode(fullS3Path); @@ -168,78 +163,73 @@ const getThumbnailUrls = async (req, res) => { * @returns {Promise<*>} */ const downloadFiles = async (req, res) => { - //Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk const { jobId, billid, documentids } = req.body; + logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); + + const client = req.userGraphQLClient; + let data; try { - logger.log("imgproxy-download", "DEBUG", req.user?.email, jobId, { billid, jobId, documentids }); - - //Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components. - const client = req.userGraphQLClient; - - //Query for the keys of the document IDs - const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); - - //Using the Keys, get all the S3 links, zip them, and send back to the client. - const s3client = new S3Client({ region: InstanceRegion() }); - const archiveStream = archiver("zip"); - - archiveStream.on("error", (error) => { - console.error("Archival encountered an error:", error); - throw new Error(error); - }); - - const passThrough = new stream.PassThrough(); - - archiveStream.pipe(passThrough); - - for (const key of data.documents.map((d) => d.key)) { - const response = await s3client.send( - new GetObjectCommand({ - Bucket: imgproxyDestinationBucket, - Key: key - }) - ); - - archiveStream.append(response.Body, { name: path.basename(key) }); - } - - await archiveStream.finalize(); - - const archiveKey = `archives/${jobId || "na"}/archive-${new Date().toISOString()}.zip`; - - const parallelUploads3 = new Upload({ - client: s3client, - queueSize: 4, // optional concurrency configuration - leavePartsOnError: false, // optional manually handle dropped parts - params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passThrough } - }); - - // Disabled progress logging for upload, uncomment if needed - // parallelUploads3.on("httpUploadProgress", (progress) => { - // console.log(progress); - // }); - - await parallelUploads3.done(); - - //Generate the presigned URL to download it. - const presignedUrl = await getSignedUrl( - s3client, - new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }), - { expiresIn: 360 } - ); - - return res.json({ success: true, url: presignedUrl }); - //Iterate over them, build the link based on the media type, and return the array. + data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids }); } catch (error) { - logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobId, { + logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { jobId, billid, message: error.message, stack: error.stack }); + return res.status(400).json({ message: error.message }); + } - return res.status(400).json({ message: error.message, stack: error.stack }); + const s3client = new S3Client({ region: InstanceRegion() }); + const zipfile = new yazl.ZipFile(); + + const filename = `archive-${jobId || "na"}-${new Date().toISOString().replace(/[:.]/g, "-")}.zip`; + res.setHeader("Content-Type", "application/zip"); + res.setHeader("Content-Disposition", `attachment; filename="${filename}"`); + + // Handle zipfile stream errors + zipfile.outputStream.on("error", (err) => { + logger.log("imgproxy-download-zipstream-error", "ERROR", req.user?.email, jobId, { message: err.message, stack: err.stack }); + // Cannot send another response here, just destroy the connection + res.destroy(err); + }); + + zipfile.outputStream.pipe(res); + + try { + for (const doc of data.documents) { + let key = keyStandardize(doc) + let response; + try { + response = await s3client.send( + new GetObjectCommand({ + Bucket: imgproxyDestinationBucket, + Key: key + }) + ); + } catch (err) { + logger.log("imgproxy-download-s3-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack }); + // Optionally, skip this file or add a placeholder file in the zip + continue; + } + // Attach error handler to S3 stream + response.Body.on("error", (err) => { + logger.log("imgproxy-download-s3stream-error", "ERROR", req.user?.email, jobId, { key, message: err.message, stack: err.stack }); + res.destroy(err); + }); + zipfile.addReadStream(response.Body, path.basename(key)); + } + zipfile.end(); + } catch (error) { + logger.log("imgproxy-download-error", "ERROR", req.user?.email, jobId, { + jobId, + billid, + message: error.message, + stack: error.stack + }); + // Cannot send another response here, just destroy the connection + res.destroy(error); } }; @@ -392,6 +382,15 @@ const moveFiles = async (req, res) => { } }; +const keyStandardize = (doc) => { + if (/\.[^/.]+$/.test(doc.key)) { + return doc.key; + } else { + return `${doc.key}.${doc.extension.toLowerCase()}`; + } +}; + + module.exports = { generateSignedUploadUrls, getThumbnailUrls,