diff --git a/.platform/hooks/predeploy/00-install-fonts.sh b/.platform/hooks/predeploy/00-install-fonts.sh new file mode 100644 index 000000000..af9d08b0b --- /dev/null +++ b/.platform/hooks/predeploy/00-install-fonts.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Install required packages +dnf install -y fontconfig freetype + +# Move to the /tmp directory for temporary download and extraction +cd /tmp + +# Download the Montserrat font zip file +wget https://images.imex.online/fonts/montserrat.zip -O montserrat.zip + +# Unzip the downloaded font file +unzip montserrat.zip -d montserrat + +# Move the font files to the system fonts directory +mv montserrat/*.ttf /usr/share/fonts + +# Rebuild the font cache +fc-cache -fv + +# Clean up +rm -rf /tmp/montserrat /tmp/montserrat.zip + +echo "Montserrat fonts installed and cached successfully." diff --git a/client/src/components/dashboard-components/job-lifecycle/job-lifecycle-dashboard.component.jsx b/client/src/components/dashboard-components/job-lifecycle/job-lifecycle-dashboard.component.jsx index 38d939c89..5f52d62c6 100644 --- a/client/src/components/dashboard-components/job-lifecycle/job-lifecycle-dashboard.component.jsx +++ b/client/src/components/dashboard-components/job-lifecycle/job-lifecycle-dashboard.component.jsx @@ -1,10 +1,10 @@ import { Card, Table, Tag } from "antd"; -import LoadingSkeleton from "../../loading-skeleton/loading-skeleton.component"; -import { useTranslation } from "react-i18next"; -import React, { useEffect, useState } from "react"; -import dayjs from "../../../utils/day"; -import DashboardRefreshRequired from "../refresh-required.component"; import axios from "axios"; +import React, { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import dayjs from "../../../utils/day"; +import LoadingSkeleton from "../../loading-skeleton/loading-skeleton.component"; +import DashboardRefreshRequired from "../refresh-required.component"; const fortyFiveDaysAgo = () => dayjs().subtract(45, "day").toLocaleString(); @@ -46,6 +46,11 @@ export default function JobLifecycleDashboardComponent({ data, bodyshop, ...card dataIndex: "humanReadable", key: "humanReadable" }, + { + title: t("job_lifecycle.columns.average_human_readable"), + dataIndex: "averageHumanReadable", + key: "averageHumanReadable" + }, { title: t("job_lifecycle.columns.status_count"), key: "statusCount", diff --git a/client/src/translations/en_us/common.json b/client/src/translations/en_us/common.json index 38f13e032..d91c369fd 100644 --- a/client/src/translations/en_us/common.json +++ b/client/src/translations/en_us/common.json @@ -1338,6 +1338,8 @@ }, "job_lifecycle": { "columns": { + "average_human_readable": "Average Human Readable", + "average_value": "Average Value", "duration": "Duration", "end": "End", "human_readable": "Human Readable", diff --git a/client/src/translations/es/common.json b/client/src/translations/es/common.json index ed7760e9d..7658ca3ce 100644 --- a/client/src/translations/es/common.json +++ b/client/src/translations/es/common.json @@ -1338,6 +1338,8 @@ }, "job_lifecycle": { "columns": { + "average_human_readable": "", + "average_value": "", "duration": "", "end": "", "human_readable": "", diff --git a/client/src/translations/fr/common.json b/client/src/translations/fr/common.json index 4a64bcc55..a109a127d 100644 --- a/client/src/translations/fr/common.json +++ b/client/src/translations/fr/common.json @@ -1338,6 +1338,8 @@ }, "job_lifecycle": { "columns": { + "average_human_readable": "", + "average_value": "", "duration": "", "end": "", "human_readable": "", diff --git a/docker-compose.yml b/docker-compose.yml index 503d874b3..50dc56932 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,6 +167,27 @@ services: # volumes: # - redis-insight-data:/db +# ##Optional Container for SFTP/SSH Server for testing +# ssh-sftp-server: +# image: atmoz/sftp:alpine # Using an image with SFTP support +# container_name: ssh-sftp-server +# hostname: ssh-sftp-server +# networks: +# - redis-cluster-net +# ports: +# - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) +# volumes: +# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key +# - ./upload:/home/user/upload # Mount a local directory for SFTP uploads +# environment: +# - SFTP_USERS=user:password:1001:100:upload +# command: > +# /bin/sh -c " +# echo 'Match User user' >> /etc/ssh/sshd_config && +# sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config && +# /usr/sbin/sshd -D +# " + networks: redis-cluster-net: driver: bridge diff --git a/server/accounting/qbo/qbo-payables.js b/server/accounting/qbo/qbo-payables.js index 196520de0..04b0c7e08 100644 --- a/server/accounting/qbo/qbo-payables.js +++ b/server/accounting/qbo/qbo-payables.js @@ -167,7 +167,7 @@ async function QueryVendorRecord(oauthClient, qbo_realmId, req, bill) { async function InsertVendorRecord(oauthClient, qbo_realmId, req, bill) { const Vendor = { - DisplayName: bill.vendor.name + DisplayName: StandardizeName(bill.vendor.name) }; try { const result = await oauthClient.makeApiCall({ diff --git a/server/accounting/qbo/qbo.js b/server/accounting/qbo/qbo.js index 74cc3c742..5c7e9d874 100644 --- a/server/accounting/qbo/qbo.js +++ b/server/accounting/qbo/qbo.js @@ -10,7 +10,7 @@ function urlBuilder(realmId, object, query = null) { } function StandardizeName(str) { - return str.replace(new RegExp(/'/g), "\\'"); + return str.replace(new RegExp(/'/g), "\\'").trim(); } exports.urlBuilder = urlBuilder; diff --git a/server/data/chatter.js b/server/data/chatter.js index e610f9791..b879bff59 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -22,135 +22,128 @@ const ftpSetup = { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; + +const allcsvsToUpload = []; +const allErrors = []; + exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { res.sendStatus(403); return; } - + // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { res.sendStatus(401); return; } - //Query for the List of Bodyshop Clients. - logger.log("chatter-start", "DEBUG", "api", null, null); - const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); - const specificShopIds = req.body.bodyshopIds; // ['uuid] - const { start, end, skipUpload } = req.body; //YYYY-MM-DD - - const allcsvsToUpload = []; - const allErrors = []; try { - for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) { - logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { - shopname: bodyshop.shopname - }); - try { - const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, { - bodyshopid: bodyshop.id, - start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }); + //Query for the List of Bodyshop Clients. + logger.log("chatter-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); + const specificShopIds = req.body.bodyshopIds; // ['uuid]; - const chatterObject = jobs.map((j) => { - return { - poc_trigger_code: bodyshops_by_pk.chatterid, - firstname: j.ownr_co_nm ? null : j.ownr_fn, - lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln, - transaction_id: j.ro_number, - email: j.ownr_ea, - phone_number: j.ownr_ph1 - }; - }); + const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); + const batchSize = 10; - allcsvsToUpload.push({ - count: chatterObject.length, - csv: ret, - filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` - }); + const shopsToProcess = + specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; + logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); - logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { - shopname: bodyshop.shopname - }); - } catch (error) { - //Error at the shop level. - logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { - ...error - }); - - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - shopname: bodyshop.shopname, - fatal: true, - errors: [error.toString()] - }); - } finally { - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - shopname: bodyshop.shopname - }); - } + if (shopsToProcess.length === 0) { + logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null); + res.sendStatus(200); + return; } - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); + for (let i = 0; i < shopsToProcess.length; i += batchSize) { + const batch = shopsToProcess.slice(i, i + batchSize); + await processBatch(batch, start, end); + + if (skipUpload) { + for (const csvObj of allcsvsToUpload) { + fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); + } + } else { + await uploadViaSFTP(allcsvsToUpload); } sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )} - ` + Uploaded: ${JSON.stringify( + allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + null, + 2 + )}` }); - res.json(allcsvsToUpload); - return; + + logger.log("chatter-end", "DEBUG", "api", null, null); + res.sendStatus(200); } - - const sftp = new Client(); - sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors })); - try { - //Get the private key from AWS Secrets Manager. - ftpSetup.privateKey = await getPrivateKey(); - - //Connect to the FTP and upload all. - await sftp.connect(ftpSetup); - - for (const csvObj of allcsvsToUpload) { - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename }); - - const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`); - logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult }); - } - } catch (error) { - logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error }); - } finally { - sftp.end(); - } - sendServerEmail({ - subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )}` - }); - res.sendStatus(200); } catch (error) { - res.status(200).json(error); + logger.log("chatter-shopsToProcess-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + res.status(500).json({ error: error.message, stack: error.stack }); } }; +async function processBatch(batch, start, end) { + for (const bodyshop of batch) { + try { + logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + + const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, { + bodyshopid: bodyshop.id, + start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }); + + const chatterObject = jobs.map((j) => { + return { + poc_trigger_code: bodyshops_by_pk.chatterid, + firstname: j.ownr_co_nm ? null : j.ownr_fn, + lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln, + transaction_id: j.ro_number, + email: j.ownr_ea, + phone_number: j.ownr_ph1 + }; + }); + + const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); + + allcsvsToUpload.push({ + count: chatterObject.length, + csv: ret, + filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` + }); + + logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + } catch (error) { + //Error at the shop level. + logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname, + fatal: true, + errors: [error.toString()] + }); + } finally { + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname + }); + } + } +} + async function getPrivateKey() { // Connect to AWS Secrets Manager const client = new SecretsManagerClient({ region: "ca-central-1" }); @@ -160,10 +153,49 @@ async function getPrivateKey() { try { const { SecretString, SecretBinary } = await client.send(command); if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null); - const chatterPrivateKey = SecretString ? JSON.parse(SecretString) : JSON.parse(Buffer.from(SecretBinary, "base64").toString("ascii")); - return chatterPrivateKey.private_key; + const chatterPrivateKey = SecretString + ? SecretString + : Buffer.from(SecretBinary, "base64").toString("ascii"); + return chatterPrivateKey; } catch (error) { - logger.log("chatter-get-private-key", "ERROR", "api", null, error); - throw err; + logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack }); + throw error; + } +} + +async function uploadViaSFTP(allcsvsToUpload) { + const sftp = new Client(); + sftp.on("error", (errors) => + logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + ); + try { + //Get the private key from AWS Secrets Manager. + const privateKey = await getPrivateKey(); + + //Connect to the FTP and upload all. + await sftp.connect({ ...ftpSetup, privateKey }); + + for (const csvObj of allcsvsToUpload) { + try { + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename }); + csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); + logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { + filename: csvObj.filename, + result: csvObj.result + }); + } catch (error) { + logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { + filename: csvObj.filename, + error: error.message, + stack: error.stack + }); + throw error; + } + } + } catch (error) { + logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + throw error; + } finally { + sftp.end(); } } diff --git a/server/job/job-lifecycle.js b/server/job/job-lifecycle.js index 744639bcf..7076069f6 100644 --- a/server/job/job-lifecycle.js +++ b/server/job/job-lifecycle.js @@ -78,16 +78,20 @@ const jobLifecycle = async (req, res) => { Object.keys(flatGroupedAllDurations).forEach((status) => { const value = flatGroupedAllDurations[status].reduce((acc, curr) => acc + curr.value, 0); const humanReadable = durationToHumanReadable(moment.duration(value)); - const percentage = (value / finalTotal) * 100; + const percentage = finalTotal > 0 ? (value / finalTotal) * 100 : 0; const color = getLifecycleStatusColor(status); const roundedPercentage = `${Math.round(percentage)}%`; + const averageValue = _.size(jobIDs) > 0 ? value / jobIDs.length : 0; + const averageHumanReadable = durationToHumanReadable(moment.duration(averageValue)); finalSummations.push({ status, value, humanReadable, percentage, color, - roundedPercentage + roundedPercentage, + averageValue, + averageHumanReadable }); }); @@ -100,7 +104,12 @@ const jobLifecycle = async (req, res) => { totalStatuses: finalSummations.length, total: finalTotal, statusCounts: finalStatusCounts, - humanReadable: durationToHumanReadable(moment.duration(finalTotal)) + humanReadable: durationToHumanReadable(moment.duration(finalTotal)), + averageValue: _.size(jobIDs) > 0 ? finalTotal / jobIDs.length : 0, + averageHumanReadable: + _.size(jobIDs) > 0 + ? durationToHumanReadable(moment.duration(finalTotal / jobIDs.length)) + : durationToHumanReadable(moment.duration(0)) } }); }; diff --git a/upload/.gitignore b/upload/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/upload/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore