From f0461270de839d38dd360975b9b8435a19953f98 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Mon, 2 Dec 2024 08:52:18 -0800 Subject: [PATCH 01/13] IO-3047 Accounting ID on Owner Page Signed-off-by: Allan Carr --- .../owner-detail-form/owner-detail-form.component.jsx | 3 +++ client/src/graphql/owners.queries.js | 1 + client/src/translations/en_us/common.json | 1 + client/src/translations/es/common.json | 1 + client/src/translations/fr/common.json | 1 + 5 files changed, 7 insertions(+) diff --git a/client/src/components/owner-detail-form/owner-detail-form.component.jsx b/client/src/components/owner-detail-form/owner-detail-form.component.jsx index 419de25fa..5de4231f7 100644 --- a/client/src/components/owner-detail-form/owner-detail-form.component.jsx +++ b/client/src/components/owner-detail-form/owner-detail-form.component.jsx @@ -25,6 +25,9 @@ export default function OwnerDetailFormComponent({ form, loading }) { + + + diff --git a/client/src/graphql/owners.queries.js b/client/src/graphql/owners.queries.js index e28fc8aa2..d169263f4 100644 --- a/client/src/graphql/owners.queries.js +++ b/client/src/graphql/owners.queries.js @@ -48,6 +48,7 @@ export const QUERY_OWNER_BY_ID = gql` query QUERY_OWNER_BY_ID($id: uuid!) { owners_by_pk(id: $id) { id + accountingid allow_text_message ownr_addr1 ownr_addr2 diff --git a/client/src/translations/en_us/common.json b/client/src/translations/en_us/common.json index e86e9936e..ab41c29c0 100644 --- a/client/src/translations/en_us/common.json +++ b/client/src/translations/en_us/common.json @@ -2394,6 +2394,7 @@ "selectexistingornew": "Select an existing owner record or create a new one. " }, "fields": { + "accountingid": "Accounting ID", "address": "Address", "allow_text_message": "Permission to Text?", "name": "Name", diff --git a/client/src/translations/es/common.json b/client/src/translations/es/common.json index 8b9026489..4e0650522 100644 --- a/client/src/translations/es/common.json +++ b/client/src/translations/es/common.json @@ -2394,6 +2394,7 @@ "selectexistingornew": "" }, "fields": { + "accountingid": "", "address": "Dirección", "allow_text_message": "Permiso de texto?", "name": "Nombre", diff --git a/client/src/translations/fr/common.json b/client/src/translations/fr/common.json index 8f9330ea0..edab92e9b 100644 --- a/client/src/translations/fr/common.json +++ b/client/src/translations/fr/common.json @@ -2394,6 +2394,7 @@ "selectexistingornew": "" }, "fields": { + "accountingid": "", "address": "Adresse", "allow_text_message": "Autorisation de texte?", "name": "Prénom", From 241322fa307341fb1b0eaef9e1889b69c32a8b39 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Mon, 2 Dec 2024 11:09:30 -0800 Subject: [PATCH 02/13] IO-3046 purchase_return_ratio_excel Signed-off-by: Allan Carr --- client/src/translations/en_us/common.json | 1 + client/src/translations/es/common.json | 1 + client/src/translations/fr/common.json | 1 + client/src/utils/TemplateConstants.js | 13 +++++++++++++ 4 files changed, 16 insertions(+) diff --git a/client/src/translations/en_us/common.json b/client/src/translations/en_us/common.json index e86e9936e..7ce773a14 100644 --- a/client/src/translations/en_us/common.json +++ b/client/src/translations/en_us/common.json @@ -3057,6 +3057,7 @@ "production_not_production_status": "Production not in Production Status", "production_over_time": "Production Level over Time", "psr_by_make": "Percent of Sales by Vehicle Make", + "purchase_return_ratio_excel": "Purchase & Return Ratio - Excel", "purchase_return_ratio_grouped_by_vendor_detail": "Purchase & Return Ratio by Vendor (Detail)", "purchase_return_ratio_grouped_by_vendor_summary": "Purchase & Return Ratio by Vendor (Summary)", "purchases_by_cost_center_detail": "Purchases by Cost Center (Detail)", diff --git a/client/src/translations/es/common.json b/client/src/translations/es/common.json index 8b9026489..783b27f7e 100644 --- a/client/src/translations/es/common.json +++ b/client/src/translations/es/common.json @@ -3057,6 +3057,7 @@ "production_not_production_status": "", "production_over_time": "", "psr_by_make": "", + "purchase_return_ratio_excel": "", "purchase_return_ratio_grouped_by_vendor_detail": "", "purchase_return_ratio_grouped_by_vendor_summary": "", "purchases_by_cost_center_detail": "", diff --git a/client/src/translations/fr/common.json b/client/src/translations/fr/common.json index 8f9330ea0..bd8466003 100644 --- a/client/src/translations/fr/common.json +++ b/client/src/translations/fr/common.json @@ -3057,6 +3057,7 @@ "production_not_production_status": "", "production_over_time": "", "psr_by_make": "", + "purchase_return_ratio_excel": "", "purchase_return_ratio_grouped_by_vendor_detail": "", "purchase_return_ratio_grouped_by_vendor_summary": "", "purchases_by_cost_center_detail": "", diff --git a/client/src/utils/TemplateConstants.js b/client/src/utils/TemplateConstants.js index 9f80986d2..c193dbb49 100644 --- a/client/src/utils/TemplateConstants.js +++ b/client/src/utils/TemplateConstants.js @@ -2184,6 +2184,19 @@ export const TemplateList = (type, context) => { }, group: "payroll", adp_payroll: true + }, + purchase_return_ratio_excel: { + title: i18n.t("reportcenter.templates.purchase_return_ratio_excel"), + subject: i18n.t("reportcenter.templates.purchase_return_ratio_excel"), + key: "purchase_return_ratio_excel", + //idtype: "vendor", + reporttype: "excel", + disabled: false, + rangeFilter: { + object: i18n.t("reportcenter.labels.objects.bills"), + field: i18n.t("bills.fields.date") + }, + group: "purchases" } } : {}), From a885bdec7433b5432cd253bdeec4ad7f099e7ac7 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 4 Dec 2024 14:22:04 -0800 Subject: [PATCH 03/13] IO-3051 canvas-handler optimization Signed-off-by: Allan Carr --- .../validateCanvasRequestMiddleware.js | 20 ++ server/render/canvas-handler.js | 187 ++++++++++-------- server/routes/renderRoutes.js | 3 +- 3 files changed, 131 insertions(+), 79 deletions(-) create mode 100644 server/middleware/validateCanvasRequestMiddleware.js diff --git a/server/middleware/validateCanvasRequestMiddleware.js b/server/middleware/validateCanvasRequestMiddleware.js new file mode 100644 index 000000000..4a3bdc12f --- /dev/null +++ b/server/middleware/validateCanvasRequestMiddleware.js @@ -0,0 +1,20 @@ +const { isObject } = require("lodash"); + +const validateCanvasRequestMiddleware = (req, res, next) => { + const { w, h, values, keys, override } = req.body; + if (!values || !keys) { + return res.status(400).send("Missing required data"); + } + if (override && !isObject(override)) { + return res.status(400).send("Override must be an object"); + } + if (w && (!Number.isFinite(w) || w <= 0)) { + return res.status(400).send("Width must be a positive number"); + } + if (h && (!Number.isFinite(h) || h <= 0)) { + return res.status(400).send("Height must be a positive number"); + } + next(); +}; + +module.exports = validateCanvasRequestMiddleware; diff --git a/server/render/canvas-handler.js b/server/render/canvas-handler.js index 0af2a7f53..a0fd0a602 100644 --- a/server/render/canvas-handler.js +++ b/server/render/canvas-handler.js @@ -5,89 +5,120 @@ const logger = require("../utils/logger"); const { backgroundColors, borderColors } = require("./canvas-colors"); const { isObject, defaultsDeep, isNumber } = require("lodash"); +let isProcessing = false; +const requestQueue = []; + +const processCanvasRequest = async (req, res) => { + try { + const { w, h, values, keys, override } = req.body; + logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); + + // Set dimensions with defaults + const width = isNumber(w) ? w : 500; + const height = isNumber(h) ? h : 275; + + const configuration = { + type: "doughnut", + data: { + labels: keys, + datasets: [ + { + data: values, + backgroundColor: backgroundColors, + borderColor: borderColors, + borderWidth: 1 + } + ] + }, + options: { + animation: false, + devicePixelRatio: 4, + responsive: false, + maintainAspectRatio: true, + circumference: 180, + rotation: -90, + plugins: { + legend: { + labels: { + boxWidth: 20, + font: { + family: "'Montserrat'", + size: 10, + style: "normal", + weight: "normal" + } + }, + position: "left" + } + } + } + }; + + // If we have a valid override object, merge it with the default configuration object. + // This allows for you to override the default configuration with a custom one. + const defaults = () => { + if (!override || !isObject(override)) { + return configuration; + } + return defaultsDeep(override, configuration); + }; + + // Generate chart + let canvas = createCanvas(width, height); + let ctx = canvas.getContext("2d"); + let chart = new Chart(ctx, defaults()); + const result = canvas.toDataURL(); + + chart.destroy(); + canvas.width = 0; + canvas.height = 0; + ctx = null; + canvas = null; + chart = null; + + res.status(200).send(result); + } catch (error) { + if (chart) chart.destroy(); + if (canvas) { + canvas.width = 0; + canvas.height = 0; + } + ctx = null; + canvas = null; + chart = null; + + logger.log("inbound-canvas-creation", "error", "jsr", null, { error: error.message, stack: error.stack }); + res.status(500).send("Error generating canvas"); + } +}; + +const processNextInQueue = async () => { + if (requestQueue.length === 0) { + isProcessing = false; + return; + } + + const { req, res } = requestQueue.shift(); + await processCanvasRequest(req, res); + processNextInQueue(); +}; + exports.canvastest = function (req, res) { - //console.log("Incoming test request.", req); res.status(200).send("OK"); }; -exports.canvas = function (req, res) { - const { w, h, values, keys, override } = req.body; - //console.log("Incoming Canvas Request:", w, h, values, keys, override); - logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); - // Gate required values - if (!values || !keys) { - res.status(400).send("Missing required data"); +exports.canvas = async function (req, res) { + if (isProcessing) { + if (requestQueue.length >= 100) { + // Set a maximum queue size + return res.status(503).send("Server is busy. Please try again later."); + } + requestQueue.push({ req, res }); + logger.log("inbound-canvas-creation-queue", "debug", "jsr", null, { queue: requestQueue.length }); return; } - // Override must be an object if it exists - if (override && !isObject(override)) { - res.status(400).send("Override must be an object"); - return; - } - - // Set the default Width and Height - let [width, height] = [500, 275]; - - // Allow for custom width and height - if (isNumber(w)) { - width = w; - } - if (isNumber(h)) { - height = h; - } - - const configuration = { - type: "doughnut", - data: { - labels: keys, - datasets: [ - { - data: values, - backgroundColor: backgroundColors, - borderColor: borderColors, - borderWidth: 1 - } - ] - }, - options: { - devicePixelRatio: 4, - responsive: false, - maintainAspectRatio: true, - circumference: 180, - rotation: -90, - plugins: { - legend: { - labels: { - boxWidth: 20, - font: { - family: "'Montserrat'", - size: 10, - style: "normal", - weight: "normal" - } - }, - position: "left" - } - } - } - }; - - // If we have a valid override object, merge it with the default configuration object. - // This allows for you to override the default configuration with a custom one. - const defaults = () => { - if (!override || !isObject(override)) { - return configuration; - } - return defaultsDeep(override, configuration); - }; - - res.status(200).send( - (() => { - const canvas = createCanvas(width, height); - const ctx = canvas.getContext("2d"); - new Chart(ctx, defaults()); - return canvas.toDataURL(); - })() - ); + isProcessing = true; + await processCanvasRequest(req, res); + processNextInQueue(); }; diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index 13288d989..2657d6a76 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -2,10 +2,11 @@ const express = require("express"); const router = express.Router(); const { inlinecss } = require("../render/inlinecss"); const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebaseIdTokenMiddleware"); +const validateCanvasRequestMiddleware = require("../middleware/validateCanvasRequestMiddleware"); const { canvas } = require("../render/canvas-handler"); // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", validateFirebaseIdTokenMiddleware, canvas); +router.post("/canvas", [validateFirebaseIdTokenMiddleware, validateCanvasRequestMiddleware], canvas); module.exports = router; From 006a2a5dca9400b11b6ce291e8d1be200d45282a Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 4 Dec 2024 15:59:05 -0800 Subject: [PATCH 04/13] IO-3050 QBO BillEmail required if NeedToSend Signed-off-by: Allan Carr --- server/accounting/qbo/qbo-receivables.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/accounting/qbo/qbo-receivables.js b/server/accounting/qbo/qbo-receivables.js index f57c6711a..c7c74928b 100644 --- a/server/accounting/qbo/qbo-receivables.js +++ b/server/accounting/qbo/qbo-receivables.js @@ -328,6 +328,7 @@ async function InsertOwner(oauthClient, qbo_realmId, req, job, isThreeTier, pare PostalCode: job.ownr_zip, CountrySubDivisionCode: job.ownr_st }, + ...(job.ownr_ea ? { BillEmail: { Address: job.ownr_ea.trim() } } : {}), ...(isThreeTier ? { Job: true, @@ -395,7 +396,7 @@ async function InsertJob(oauthClient, qbo_realmId, req, job, parentTierRef) { PostalCode: job.ownr_zip, CountrySubDivisionCode: job.ownr_st }, - + ...(job.ownr_ea ? { BillEmail: { Address: job.ownr_ea.trim() } } : {}), Job: true, ParentRef: { value: parentTierRef.Id @@ -556,7 +557,8 @@ async function InsertInvoice(oauthClient, qbo_realmId, req, job, bodyshop, paren Line3: `${job.ownr_city || ""}, ${job.ownr_st || ""} ${job.ownr_zip || ""}`.trim(), Line2: job.ownr_addr1 || "", Line1: `${job.ownr_fn || ""} ${job.ownr_ln || ""} ${job.ownr_co_nm || ""}` - } + }, + ...(job.ownr_ea ? { BillEmail: { Address: job.ownr_ea.trim() } } : {}) }; logger.log("qbo-receivable-objectlog", "DEBUG", req.user.email, job.id, { @@ -673,7 +675,8 @@ async function InsertInvoiceMultiPayerInvoice( Line3: `${job.ownr_city || ""}, ${job.ownr_st || ""} ${job.ownr_zip || ""}`.trim(), Line2: job.ownr_addr1 || "", Line1: `${job.ownr_fn || ""} ${job.ownr_ln || ""} ${job.ownr_co_nm || ""}` - } + }, + ...(job.ownr_ea ? { BillEmail: { Address: job.ownr_ea.trim() } } : {}) }; logger.log("qbo-receivable-objectlog", "DEBUG", req.user.email, job.id, { From 2effe5ef50d73e1f87da883c4ae5f142ceacd2ba Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 4 Dec 2024 18:30:22 -0800 Subject: [PATCH 05/13] IO-3042 Jobs Marked as Total Loss Signed-off-by: Allan Carr --- client/src/translations/en_us/common.json | 1 + client/src/translations/es/common.json | 1 + client/src/translations/fr/common.json | 1 + client/src/utils/TemplateConstants.js | 13 ++++++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/client/src/translations/en_us/common.json b/client/src/translations/en_us/common.json index e86e9936e..0edba955d 100644 --- a/client/src/translations/en_us/common.json +++ b/client/src/translations/en_us/common.json @@ -3082,6 +3082,7 @@ "timetickets": "Time Tickets", "timetickets_employee": "Employee Time Tickets", "timetickets_summary": "Time Tickets Summary", + "total_loss_jobs": "Jobs Marked as Total Loss", "unclaimed_hrs": "Unflagged Hours", "void_ros": "Void ROs", "work_in_progress_committed_labour": "Work in Progress - Committed Labor", diff --git a/client/src/translations/es/common.json b/client/src/translations/es/common.json index 8b9026489..b752bf47e 100644 --- a/client/src/translations/es/common.json +++ b/client/src/translations/es/common.json @@ -3082,6 +3082,7 @@ "timetickets": "", "timetickets_employee": "", "timetickets_summary": "", + "total_loss_jobs": "", "unclaimed_hrs": "", "void_ros": "", "work_in_progress_committed_labour": "", diff --git a/client/src/translations/fr/common.json b/client/src/translations/fr/common.json index 8f9330ea0..c46314c63 100644 --- a/client/src/translations/fr/common.json +++ b/client/src/translations/fr/common.json @@ -3082,6 +3082,7 @@ "timetickets": "", "timetickets_employee": "", "timetickets_summary": "", + "total_loss_jobs": "", "unclaimed_hrs": "", "void_ros": "", "work_in_progress_committed_labour": "", diff --git a/client/src/utils/TemplateConstants.js b/client/src/utils/TemplateConstants.js index 9f80986d2..043d5fa3a 100644 --- a/client/src/utils/TemplateConstants.js +++ b/client/src/utils/TemplateConstants.js @@ -2184,7 +2184,18 @@ export const TemplateList = (type, context) => { }, group: "payroll", adp_payroll: true - } + }, + total_loss_jobs: { + title: i18n.t("reportcenter.templates.total_loss_jobs"), + subject: i18n.t("reportcenter.templates.total_loss_jobs"), + key: "total_loss_jobs", + disabled: false, + rangeFilter: { + object: i18n.t("reportcenter.labels.objects.jobs"), + field: i18n.t("jobs.fields.date_open") + }, + group: "jobs" + }, } : {}), ...(!type || type === "courtesycarcontract" From 6336e7568fe0b7b61ce8281130f0e1bfb16c099d Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 11:26:23 -0800 Subject: [PATCH 06/13] feature/IO-3052-Skia-Canvas-Handler: Initial commit Signed-off-by: Dave Richer --- Dockerfile | 14 ++- package-lock.json | 98 ++++++++++++++++++++ package.json | 1 + server/render/canvas-handler.js | 153 +++++++++++++++++++++++--------- server/routes/renderRoutes.js | 5 +- 5 files changed, 227 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index b1d253808..16e9d2159 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ RUN dnf install -y git \ && dnf install -y nodejs \ && dnf clean all - # Install dependencies required by node-canvas RUN dnf install -y \ gcc \ @@ -19,9 +18,22 @@ RUN dnf install -y \ libpng-devel \ make \ python3 \ + fontconfig \ + freetype \ python3-pip \ + wget \ + unzip \ && dnf clean all +# Install Montserrat fonts +RUN cd /tmp \ + && wget https://images.imex.online/fonts/montserrat.zip -O montserrat.zip \ + && unzip montserrat.zip -d montserrat \ + && mv montserrat/montserrat/*.ttf /usr/share/fonts \ + && fc-cache -fv \ + && rm -rf /tmp/montserrat /tmp/montserrat.zip \ + && echo "Montserrat fonts installed and cached successfully." + # Set the working directory WORKDIR /app diff --git a/package-lock.json b/package-lock.json index 6d1d243cc..cceafbf49 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "recursive-diff": "^1.0.9", "redis": "^4.7.0", "rimraf": "^6.0.1", + "skia-canvas": "^2.0.0", "soap": "^1.1.6", "socket.io": "^4.8.1", "socket.io-adapter": "^2.5.5", @@ -3734,6 +3735,15 @@ "node": ">=6" } }, + "node_modules/cargo-cp-artifact": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.9.tgz", + "integrity": "sha512-6F+UYzTaGB+awsTXg0uSJA1/b/B3DDJzpKVRu0UmyI7DmNeaAl2RFHuTGIN6fEgpadRxoXGb7gbC1xo4C3IdyA==", + "license": "MIT", + "bin": { + "cargo-cp-artifact": "bin/cargo-cp-artifact.js" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -6757,6 +6767,12 @@ "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==" }, + "node_modules/parenthesis": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/parenthesis/-/parenthesis-3.1.8.tgz", + "integrity": "sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==", + "license": "MIT" + }, "node_modules/parse5": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", @@ -6789,6 +6805,12 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -7459,6 +7481,73 @@ "is-arrayish": "^0.3.1" } }, + "node_modules/skia-canvas": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/skia-canvas/-/skia-canvas-2.0.0.tgz", + "integrity": "sha512-wpYkmr9mCxBme5HAnlm6YOEiuaN9tIm9CL+HN8e5AFD4K2FAJXCcWiWvc9+LM8jUXt+AyYXgiwUTBxdQ6P+PEg==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.11", + "cargo-cp-artifact": "^0.1", + "glob": "^11.0.0", + "path-browserify": "^1.0.1", + "simple-get": "^4.0.1", + "string-split-by": "^1.0.0" + } + }, + "node_modules/skia-canvas/node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/skia-canvas/node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/skia-canvas/node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/slick": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/slick/-/slick-1.12.2.tgz", @@ -7851,6 +7940,15 @@ } ] }, + "node_modules/string-split-by": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string-split-by/-/string-split-by-1.0.0.tgz", + "integrity": "sha512-KaJKY+hfpzNyet/emP81PJA9hTVSfxNLS9SFTWxdCnnW1/zOOwiV248+EfoX7IQFcBaOp4G5YE6xTJMF+pLg6A==", + "license": "MIT", + "dependencies": { + "parenthesis": "^3.1.5" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", diff --git a/package.json b/package.json index 4fd96c3ff..5889ea532 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "recursive-diff": "^1.0.9", "redis": "^4.7.0", "rimraf": "^6.0.1", + "skia-canvas": "^2.0.0", "soap": "^1.1.6", "socket.io": "^4.8.1", "socket.io-adapter": "^2.5.5", diff --git a/server/render/canvas-handler.js b/server/render/canvas-handler.js index 0af2a7f53..df3cc8f02 100644 --- a/server/render/canvas-handler.js +++ b/server/render/canvas-handler.js @@ -1,43 +1,28 @@ const { createCanvas } = require("canvas"); +const { Canvas, FontLibrary } = require("skia-canvas"); +const { performance } = require("perf_hooks"); const Chart = require("chart.js/auto"); const logger = require("../utils/logger"); const { backgroundColors, borderColors } = require("./canvas-colors"); const { isObject, defaultsDeep, isNumber } = require("lodash"); -exports.canvastest = function (req, res) { - //console.log("Incoming test request.", req); - res.status(200).send("OK"); -}; +try { + FontLibrary.use("Montserrat", [ + "/usr/share/fonts/Montserrat-Regular.ttf", + "/usr/share/fonts/Montserrat-Bold.ttf", + "/usr/share/fonts/Montserrat-Italic.ttf" + ]); +} catch (error) { + console.error( + "Error loading fonts Skia Canvas Fonts, please be sure to install Montserrat font package", + error.message + ); +} -exports.canvas = function (req, res) { - const { w, h, values, keys, override } = req.body; - //console.log("Incoming Canvas Request:", w, h, values, keys, override); - logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); - // Gate required values - if (!values || !keys) { - res.status(400).send("Missing required data"); - return; - } - - // Override must be an object if it exists - if (override && !isObject(override)) { - res.status(400).send("Override must be an object"); - return; - } - - // Set the default Width and Height - let [width, height] = [500, 275]; - - // Allow for custom width and height - if (isNumber(w)) { - width = w; - } - if (isNumber(h)) { - height = h; - } - - const configuration = { +// Utility to create a chart configuration +const getChartConfiguration = (keys, values, override) => { + const defaultConfiguration = { type: "doughnut", data: { labels: keys, @@ -53,6 +38,7 @@ exports.canvas = function (req, res) { options: { devicePixelRatio: 4, responsive: false, + animation: false, maintainAspectRatio: true, circumference: 180, rotation: -90, @@ -73,21 +59,106 @@ exports.canvas = function (req, res) { } }; - // If we have a valid override object, merge it with the default configuration object. - // This allows for you to override the default configuration with a custom one. - const defaults = () => { - if (!override || !isObject(override)) { - return configuration; - } - return defaultsDeep(override, configuration); - }; + return defaultsDeep(override || {}, defaultConfiguration); +}; + +// Utility to validate input +const validateCanvasInput = ({ values, keys, override }, res) => { + if (!Array.isArray(values) || !Array.isArray(keys)) { + res.status(400).send("Invalid input: 'values' and 'keys' must be arrays."); + return false; + } + + if (values.some((value) => typeof value !== "number")) { + res.status(400).send("Invalid input: 'values' must be an array of numbers."); + return false; + } + + if (keys.some((key) => typeof key !== "string")) { + res.status(400).send("Invalid input: 'keys' must be an array of strings."); + return false; + } + + if (override && !isObject(override)) { + res.status(400).send("Override must be an object"); + return false; + } + + return true; +}; + +exports.canvastest = function (req, res) { + //console.log("Incoming test request.", req); + res.status(200).send("OK"); +}; + +exports.canvas = function (req, res) { + const startTime = performance.now(); + + const { w, h, values, keys, override } = req.body; + //console.log("Incoming Canvas Request:", w, h, values, keys, override); + logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); + + if (!validateCanvasInput(req.body, res)) return; + + // Set the default Width and Height + let [width, height] = [500, 275]; + + // Allow for custom width and height + if (isNumber(w)) { + width = w; + } + if (isNumber(h)) { + height = h; + } + + const configuration = getChartConfiguration(keys, values, override); res.status(200).send( (() => { const canvas = createCanvas(width, height); const ctx = canvas.getContext("2d"); - new Chart(ctx, defaults()); + new Chart(ctx, configuration); return canvas.toDataURL(); })() ); + console.log("Canvas generation time:", performance.now() - startTime, "ms"); +}; + +exports.canvasSkia = async function (req, res) { + const startTime = performance.now(); + const { w, h, values, keys, override } = req.body; + + // Log incoming request for debugging + logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); + + if (!validateCanvasInput(req.body, res)) return; + + // Default width and height + const width = typeof w === "number" && w > 0 ? w : 500; + const height = typeof h === "number" && h > 0 ? h : 275; + + const configuration = getChartConfiguration(keys, values, override); + + try { + // Create a canvas and get the 2D rendering context + const canvas = new Canvas(width, height); + const ctx = canvas.getContext("2d"); + + // Render the chart + new Chart(ctx, configuration); + + // Convert the canvas to a Base64-encoded image + const chartImage = (await canvas.toBuffer("image/png")).toString("base64"); + const dataURL = `data:image/png;base64,${chartImage}`; + + // Send the Base64-encoded image as the response + res.status(200).send(dataURL); + console.log("Canvas generation time:", performance.now() - startTime, "ms"); + } catch (error) { + // Log and handle rendering errors + logger.log("canvas-error", "error", "jsr", null, { error: error.message }); + console.error("Error generating chart:", error.message); + res.status(500).send("Failed to generate canvas."); + } }; diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index 13288d989..083d58b39 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -2,10 +2,11 @@ const express = require("express"); const router = express.Router(); const { inlinecss } = require("../render/inlinecss"); const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebaseIdTokenMiddleware"); -const { canvas } = require("../render/canvas-handler"); +const { canvas, canvasSkia } = require("../render/canvas-handler"); // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", validateFirebaseIdTokenMiddleware, canvas); +router.post("/canvas", canvas); +router.post("/canvas-skia", canvasSkia); module.exports = router; From 86f3179bc09dbb16b8d222ee6db4e4fbca42b262 Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 11:26:36 -0800 Subject: [PATCH 07/13] feature/IO-3052-Skia-Canvas-Handler: Initial commit Signed-off-by: Dave Richer --- server/routes/renderRoutes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index 083d58b39..d1b64566e 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -6,7 +6,7 @@ const { canvas, canvasSkia } = require("../render/canvas-handler"); // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", canvas); -router.post("/canvas-skia", canvasSkia); +router.post("/canvas", validateFirebaseIdTokenMiddleware, canvas); +router.post("/canvas-skia", validateFirebaseIdTokenMiddleware, canvasSkia); module.exports = router; From 50c99f7a1e009808d26c385336783d8b9ee09cd7 Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 11:52:14 -0800 Subject: [PATCH 08/13] feature/IO-3052-Skia-Canvas-Handler: Cleanup Signed-off-by: Dave Richer --- .../validateCanvasInputMiddleware.js | 25 +++++++++++ server/render/canvas-handler.js | 42 ++----------------- server/routes/renderRoutes.js | 5 ++- 3 files changed, 31 insertions(+), 41 deletions(-) create mode 100644 server/middleware/validateCanvasInputMiddleware.js diff --git a/server/middleware/validateCanvasInputMiddleware.js b/server/middleware/validateCanvasInputMiddleware.js new file mode 100644 index 000000000..daf3775ba --- /dev/null +++ b/server/middleware/validateCanvasInputMiddleware.js @@ -0,0 +1,25 @@ +const { isObject } = require("lodash"); + +const validateCanvasInputMiddleware = (req, res, next) => { + const { values, keys, override } = req.body; + + if (!Array.isArray(values) || !Array.isArray(keys)) { + return res.status(400).send("Invalid input: 'values' and 'keys' must be arrays."); + } + + if (values.some((value) => typeof value !== "number")) { + return res.status(400).send("Invalid input: 'values' must be an array of numbers."); + } + + if (keys.some((key) => typeof key !== "string")) { + return res.status(400).send("Invalid input: 'keys' must be an array of strings."); + } + + if (override && !isObject(override)) { + return res.status(400).send("Override must be an object"); + } + + next(); // Proceed to the next middleware or route handler +}; + +module.exports = validateCanvasInputMiddleware; diff --git a/server/render/canvas-handler.js b/server/render/canvas-handler.js index df3cc8f02..b35926935 100644 --- a/server/render/canvas-handler.js +++ b/server/render/canvas-handler.js @@ -1,11 +1,9 @@ const { createCanvas } = require("canvas"); const { Canvas, FontLibrary } = require("skia-canvas"); -const { performance } = require("perf_hooks"); const Chart = require("chart.js/auto"); -const logger = require("../utils/logger"); const { backgroundColors, borderColors } = require("./canvas-colors"); -const { isObject, defaultsDeep, isNumber } = require("lodash"); +const { defaultsDeep, isNumber } = require("lodash"); try { FontLibrary.use("Montserrat", [ @@ -62,45 +60,16 @@ const getChartConfiguration = (keys, values, override) => { return defaultsDeep(override || {}, defaultConfiguration); }; -// Utility to validate input -const validateCanvasInput = ({ values, keys, override }, res) => { - if (!Array.isArray(values) || !Array.isArray(keys)) { - res.status(400).send("Invalid input: 'values' and 'keys' must be arrays."); - return false; - } - - if (values.some((value) => typeof value !== "number")) { - res.status(400).send("Invalid input: 'values' must be an array of numbers."); - return false; - } - - if (keys.some((key) => typeof key !== "string")) { - res.status(400).send("Invalid input: 'keys' must be an array of strings."); - return false; - } - - if (override && !isObject(override)) { - res.status(400).send("Override must be an object"); - return false; - } - - return true; -}; - exports.canvastest = function (req, res) { - //console.log("Incoming test request.", req); res.status(200).send("OK"); }; exports.canvas = function (req, res) { - const startTime = performance.now(); + const { logger } = req; const { w, h, values, keys, override } = req.body; - //console.log("Incoming Canvas Request:", w, h, values, keys, override); logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); - if (!validateCanvasInput(req.body, res)) return; - // Set the default Width and Height let [width, height] = [500, 275]; @@ -122,18 +91,15 @@ exports.canvas = function (req, res) { return canvas.toDataURL(); })() ); - console.log("Canvas generation time:", performance.now() - startTime, "ms"); }; exports.canvasSkia = async function (req, res) { - const startTime = performance.now(); + const { logger } = req; const { w, h, values, keys, override } = req.body; // Log incoming request for debugging logger.log("inbound-canvas-creation", "debug", "jsr", null, { w, h, values, keys, override }); - if (!validateCanvasInput(req.body, res)) return; - // Default width and height const width = typeof w === "number" && w > 0 ? w : 500; const height = typeof h === "number" && h > 0 ? h : 275; @@ -154,11 +120,9 @@ exports.canvasSkia = async function (req, res) { // Send the Base64-encoded image as the response res.status(200).send(dataURL); - console.log("Canvas generation time:", performance.now() - startTime, "ms"); } catch (error) { // Log and handle rendering errors logger.log("canvas-error", "error", "jsr", null, { error: error.message }); - console.error("Error generating chart:", error.message); res.status(500).send("Failed to generate canvas."); } }; diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index d1b64566e..c1423f698 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -3,10 +3,11 @@ const router = express.Router(); const { inlinecss } = require("../render/inlinecss"); const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebaseIdTokenMiddleware"); const { canvas, canvasSkia } = require("../render/canvas-handler"); +const validateCanvasInputMiddleware = require("../middleware/validateCanvasInputMiddleware"); // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", validateFirebaseIdTokenMiddleware, canvas); -router.post("/canvas-skia", validateFirebaseIdTokenMiddleware, canvasSkia); +router.post("/canvas", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvas); +router.post("/canvas-skia", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvasSkia); module.exports = router; From 8f752d575a99c0e013c603d2b04a97dd324af5c3 Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 12:13:49 -0800 Subject: [PATCH 09/13] feature/IO-3052-Skia-Canvas-Handler: Optimizations Signed-off-by: Dave Richer --- server/render/canvas-handler.js | 63 ++++++++++++++------------------- server/routes/renderRoutes.js | 4 +-- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/server/render/canvas-handler.js b/server/render/canvas-handler.js index 946b6d082..0da0cc9a2 100644 --- a/server/render/canvas-handler.js +++ b/server/render/canvas-handler.js @@ -5,6 +5,8 @@ const Chart = require("chart.js/auto"); const { backgroundColors, borderColors } = require("./canvas-colors"); const { defaultsDeep, isNumber } = require("lodash"); +const CANVAS_QUEUE_LIMIT = 100; + let isProcessing = false; const requestQueue = []; @@ -83,9 +85,7 @@ const processCanvasRequest = async (req, res, isSkia = false) => { const chart = new Chart(ctx, configuration); // Generate and send the image - const chartImage = isSkia - ? (await canvas.toBuffer("image/png")).toString("base64") - : canvas.toDataURL(); + const chartImage = isSkia ? (await canvas.toBuffer("image/png")).toString("base64") : canvas.toDataURL(); chart.destroy(); res.status(200).send(isSkia ? `data:image/png;base64,${chartImage}` : chartImage); @@ -95,49 +95,40 @@ const processCanvasRequest = async (req, res, isSkia = false) => { } }; -const processNextInQueue = async () => { - if (requestQueue.length === 0) { - isProcessing = false; - return; +const enqueueRequest = (req, res, isSkia) => { + if (requestQueue.length >= CANVAS_QUEUE_LIMIT) { + res.status(503).send("Server is busy. Please try again later."); + return false; } + requestQueue.push({ req, res, isSkia }); + req.logger.log("inbound-canvas-creation-queue", "debug", "jsr", null, { queue: requestQueue.length }); + return true; +}; - const { req, res, isSkia } = requestQueue.shift(); - await processCanvasRequest(req, res, isSkia); - processNextInQueue(); +const processNextInQueue = async () => { + while (requestQueue.length > 0) { + const { req, res, isSkia } = requestQueue.shift(); + try { + await processCanvasRequest(req, res, isSkia); + } catch (err) { + console.error("canvas-queue-error", "error", "jsr", null, { error: err.message }); + } + } + isProcessing = false; }; exports.canvastest = function (req, res) { res.status(200).send("OK"); }; -exports.canvas = async function (req, res) { - if (isProcessing) { - if (requestQueue.length >= 100) { - // Set a maximum queue size - return res.status(503).send("Server is busy. Please try again later."); - } - requestQueue.push({ req, res, isSkia: false }); - req.logger.log("inbound-canvas-creation-queue", "debug", "jsr", null, { queue: requestQueue.length }); - return; - } - +exports.canvas = async (req, res) => { + if (isProcessing || !enqueueRequest(req, res, false)) return; isProcessing = true; - await processCanvasRequest(req, res, false); - processNextInQueue(); + processNextInQueue().catch((err) => console.error("canvas-processing-error", { error: err.message })); }; -exports.canvasSkia = async function (req, res) { - if (isProcessing) { - if (requestQueue.length >= 100) { - // Set a maximum queue size - return res.status(503).send("Server is busy. Please try again later."); - } - requestQueue.push({ req, res, isSkia: true }); - req.logger.log("inbound-canvas-creation-queue", "debug", "jsr", null, { queue: requestQueue.length }); - return; - } - +exports.canvasSkia = async (req, res) => { + if (isProcessing || !enqueueRequest(req, res, true)) return; isProcessing = true; - await processCanvasRequest(req, res, true); - processNextInQueue(); + processNextInQueue().catch((err) => console.error("canvas-processing-error", { error: err.message })); }; diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index c1423f698..fda1b5846 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -7,7 +7,7 @@ const validateCanvasInputMiddleware = require("../middleware/validateCanvasInput // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvas); -router.post("/canvas-skia", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvasSkia); +router.post("/canvas", [validateCanvasInputMiddleware], canvas); +router.post("/canvas-skia", [validateCanvasInputMiddleware], canvasSkia); module.exports = router; From c84fbcaba129e7e79baf179b632df73c5bc94578 Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 12:14:06 -0800 Subject: [PATCH 10/13] feature/IO-3052-Skia-Canvas-Handler: Optimizations Signed-off-by: Dave Richer --- server/routes/renderRoutes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/routes/renderRoutes.js b/server/routes/renderRoutes.js index fda1b5846..c1423f698 100644 --- a/server/routes/renderRoutes.js +++ b/server/routes/renderRoutes.js @@ -7,7 +7,7 @@ const validateCanvasInputMiddleware = require("../middleware/validateCanvasInput // Define the route for inline CSS rendering router.post("/inlinecss", validateFirebaseIdTokenMiddleware, inlinecss); -router.post("/canvas", [validateCanvasInputMiddleware], canvas); -router.post("/canvas-skia", [validateCanvasInputMiddleware], canvasSkia); +router.post("/canvas", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvas); +router.post("/canvas-skia", [validateFirebaseIdTokenMiddleware, validateCanvasInputMiddleware], canvasSkia); module.exports = router; From 20bddb43b6c92be17aea792094951d6731ca2f2d Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 12:16:32 -0800 Subject: [PATCH 11/13] feature/IO-3052-Skia-Canvas-Handler: Fix missing checks Signed-off-by: Dave Richer --- server/middleware/validateCanvasInputMiddleware.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/middleware/validateCanvasInputMiddleware.js b/server/middleware/validateCanvasInputMiddleware.js index daf3775ba..a03bb12ec 100644 --- a/server/middleware/validateCanvasInputMiddleware.js +++ b/server/middleware/validateCanvasInputMiddleware.js @@ -1,7 +1,7 @@ const { isObject } = require("lodash"); const validateCanvasInputMiddleware = (req, res, next) => { - const { values, keys, override } = req.body; + const { values, keys, override, w, h } = req.body; if (!Array.isArray(values) || !Array.isArray(keys)) { return res.status(400).send("Invalid input: 'values' and 'keys' must be arrays."); @@ -19,6 +19,13 @@ const validateCanvasInputMiddleware = (req, res, next) => { return res.status(400).send("Override must be an object"); } + if (w && (!Number.isFinite(w) || w <= 0)) { + return res.status(400).send("Width must be a positive number"); + } + if (h && (!Number.isFinite(h) || h <= 0)) { + return res.status(400).send("Height must be a positive number"); + } + next(); // Proceed to the next middleware or route handler }; From bfde72eed8831fcc164db3e8267d18fa044abc4b Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 5 Dec 2024 12:29:11 -0800 Subject: [PATCH 12/13] feature/IO-3052-Skia-Canvas-Handler: Fix missing checks Signed-off-by: Dave Richer --- server/render/canvas-handler.js | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/server/render/canvas-handler.js b/server/render/canvas-handler.js index 0da0cc9a2..5b810dd73 100644 --- a/server/render/canvas-handler.js +++ b/server/render/canvas-handler.js @@ -77,21 +77,36 @@ const processCanvasRequest = async (req, res, isSkia = false) => { const configuration = getChartConfiguration(keys, values, override); + // Placeholders to allow fine control over GAC + let canvas = null; + let ctx = null; + let chart = null; + let chartImage = null; + try { - const canvas = isSkia ? new Canvas(width, height) : createCanvas(width, height); - const ctx = canvas.getContext("2d"); + // Create the canvas + canvas = isSkia ? new Canvas(width, height) : createCanvas(width, height); + ctx = canvas.getContext("2d"); // Render the chart - const chart = new Chart(ctx, configuration); + chart = new Chart(ctx, configuration); // Generate and send the image - const chartImage = isSkia ? (await canvas.toBuffer("image/png")).toString("base64") : canvas.toDataURL(); + chartImage = isSkia ? (await canvas.toBuffer("image/png")).toString("base64") : canvas.toDataURL(); - chart.destroy(); res.status(200).send(isSkia ? `data:image/png;base64,${chartImage}` : chartImage); } catch (error) { + // Log the error and send the response logger.log("canvas-error", "error", "jsr", null, { error: error.message }); res.status(500).send("Failed to generate canvas."); + } finally { + // Cleanup resources + if (chart) { + chart.destroy(); + } + ctx = null; // Explicitly nullify for garbage collection + canvas = null; // Explicitly nullify for garbage collection + chartImage = null; } }; From 83a19528807f1675a89a42ed79bad97b6f413052 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Thu, 5 Dec 2024 15:30:37 -0800 Subject: [PATCH 13/13] IO-3051 Replace inlince css with juice. --- package-lock.json | 304 +++++++++++++++++++++++++++++++++++++ package.json | 1 + server/render/inlinecss.js | 40 +++-- 3 files changed, 331 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6d1d243cc..40a207036 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,7 @@ "intuit-oauth": "^4.1.3", "ioredis": "^5.4.1", "json-2-csv": "^5.5.6", + "juice": "^11.0.0", "lodash": "^4.17.21", "moment": "^2.30.1", "moment-timezone": "^0.5.46", @@ -3359,6 +3360,15 @@ "node": ">= 6.0.0" } }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -3927,6 +3937,15 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/component-emitter": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", @@ -4604,6 +4623,31 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", + "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -4690,6 +4734,18 @@ "node": ">=6" } }, + "node_modules/escape-goat": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-3.0.0.tgz", + "integrity": "sha512-w3PwNZJwRxlp47QGzhuEBldEqVHHhh8/tIPcl6ecf2Bou99cdAt0knihBV0Ecc7CGxYduXVBDheH1K2oADRlvw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -6046,6 +6102,69 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/juice": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/juice/-/juice-11.0.0.tgz", + "integrity": "sha512-sGF8hPz9/Wg+YXbaNDqc1Iuoaw+J/P9lBHNQKXAGc9pPNjCd4fyPai0Zxj7MRtdjMr0lcgk5PjEIkP2b8R9F3w==", + "license": "MIT", + "dependencies": { + "cheerio": "^1.0.0", + "commander": "^12.1.0", + "mensch": "^0.3.4", + "slick": "^1.12.2", + "web-resource-inliner": "^7.0.0" + }, + "bin": { + "juice": "bin/juice" + }, + "engines": { + "node": ">=18.17" + } + }, + "node_modules/juice/node_modules/cheerio": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=18.17" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/juice/node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, "node_modules/jwa": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", @@ -6298,6 +6417,12 @@ "cssom": "^0.5.0" } }, + "node_modules/mensch": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/mensch/-/mensch-0.3.4.tgz", + "integrity": "sha512-IAeFvcOnV9V0Yk+bFhYR07O3yNina9ANIN5MoXBKYJ/RLYPurd2d0yw14MDhpr9/momp0WofT1bPUh3hkzdi/g==", + "license": "MIT" + }, "node_modules/merge-descriptors": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", @@ -6780,6 +6905,18 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -8294,6 +8431,15 @@ "node": ">= 4.0.0" } }, + "node_modules/undici": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", + "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -8355,6 +8501,15 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/valid-data-url": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-3.0.1.tgz", + "integrity": "sha512-jOWVmzVceKlVVdwjNSenT4PbGghU0SBIizAev8ofZVgivk/TVHXSbNL8LP6M3spZvkR9/QolkyJavGSX5Cs0UA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -8379,6 +8534,131 @@ "node": ">=6.0" } }, + "node_modules/web-resource-inliner": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/web-resource-inliner/-/web-resource-inliner-7.0.0.tgz", + "integrity": "sha512-NlfnGF8MY9ZUwFjyq3vOUBx7KwF8bmE+ywR781SB0nWB6MoMxN4BA8gtgP1KGTZo/O/AyWJz7HZpR704eaj4mg==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1", + "escape-goat": "^3.0.0", + "htmlparser2": "^5.0.0", + "mime": "^2.4.6", + "valid-data-url": "^3.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/web-resource-inliner/node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/dom-serializer/node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/domhandler": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-3.3.0.tgz", + "integrity": "sha512-J1C5rIANUbuYK+FuFL98650rihynUOEzRLxW+90bKZRWB6A1X1Tf82GxR1qAWLyfNPRvjqfip3Q5tdYlmAa9lA==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.0.1" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/domutils/node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/htmlparser2": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-5.0.1.tgz", + "integrity": "sha512-vKZZra6CSe9qsJzh0BjBGXo8dvzNsq/oGvsjfRdOrrryfeD9UOBEEQdeoqCRmKZchF5h2zOBMQ6YuQ0uRUmdbQ==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^3.3.0", + "domutils": "^2.4.2", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/fb55/htmlparser2?sponsor=1" + } + }, + "node_modules/web-resource-inliner/node_modules/mime": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -8407,6 +8687,30 @@ "node": ">=0.8.0" } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/whatwg-mimetype": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", diff --git a/package.json b/package.json index 4fd96c3ff..f45f6a13f 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "intuit-oauth": "^4.1.3", "ioredis": "^5.4.1", "json-2-csv": "^5.5.6", + "juice": "^11.0.0", "lodash": "^4.17.21", "moment": "^2.30.1", "moment-timezone": "^0.5.46", diff --git a/server/render/inlinecss.js b/server/render/inlinecss.js index 07fb68052..74700b210 100644 --- a/server/render/inlinecss.js +++ b/server/render/inlinecss.js @@ -3,24 +3,36 @@ require("dotenv").config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) }); const logger = require("../utils/logger"); -const inlineCssTool = require("inline-css"); +//const inlineCssTool = require("inline-css"); +const juice = require("juice"); -exports.inlinecss = (req, res) => { +exports.inlinecss = async (req, res) => { //Perform request validation - logger.log("email-inline-css", "DEBUG", req.user.email, null, null); const { html, url } = req.body; - - inlineCssTool(html, { url: url }) - .then((inlinedHtml) => { - res.send(inlinedHtml); - }) - .catch((error) => { - logger.log("email-inline-css-error", "ERROR", req.user.email, null, { - error - }); - - res.send(error); + try { + const inlinedHtml = juice(html, { + applyAttributesTableElements: false, + preserveMediaQueries: false, + applyWidthAttributes: false }); + res.send(inlinedHtml); + } catch (error) { + logger.log("email-inline-css-error", "ERROR", req.user.email, null, { + error + }); + res.send(error.message); + } + + // inlineCssTool(html, { url: url }) + // .then((inlinedHtml) => { + // res.send(inlinedHtml); + // }) + // .catch((error) => { + // logger.log("email-inline-css-error", "ERROR", req.user.email, null, { + // error + // }); + + // }); };