From 7cbabf8697cc0367645fefa582be4514934c947d Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Mon, 18 Nov 2024 12:02:08 -0800 Subject: [PATCH 1/5] IO-3027 Turn Off SFTP Logging for Production ENV Signed-off-by: Allan Carr --- package-lock.json | 11 ++++++----- package.json | 2 +- server/data/autohouse.js | 4 +++- server/data/chatter.js | 4 +++- server/data/claimscorp.js | 4 +++- server/data/kaizen.js | 4 +++- 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2f0edfad5..92fb059f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", @@ -7760,16 +7760,17 @@ } }, "node_modules/ssh2-sftp-client": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz", - "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz", + "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==", + "license": "Apache-2.0", "dependencies": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", "ssh2": "^1.15.0" }, "engines": { - "node": ">=16.20.2" + "node": ">=18.20.4" }, "funding": { "type": "individual", diff --git a/package.json b/package.json index 2a3c24ded..3090a4efa 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 7d8447f68..1384771e6 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,7 +25,9 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..8fa217149 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -17,7 +17,9 @@ const ftpSetup = { port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index cafa03df1..b8c77a018 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,7 +24,9 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/kaizen.js b/server/data/kaizen.js index b79fe4745..bbb758a93 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,7 +23,9 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } From 45ac56e0bc2011131c12f10ad721853be01fde36 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 20 Nov 2024 10:57:59 -0800 Subject: [PATCH 2/5] IO-2921 Chatter modifications as per Dave Bourbeau @ Chatter Signed-off-by: Allan Carr --- certs/{id_rsa => id_rsa.key} | 0 certs/io-ftp-test.key | 12 ++++++ certs/io-ftp-test.key.pub | 1 + certs/io-ftp-test.ppk | 12 ++++++ docker-build.ps1 | 14 +++++++ docker-build.sh | 16 ++++++++ docker-compose.yml | 9 +++-- server/data/chatter.js | 75 +++++++++++++++--------------------- 8 files changed, 91 insertions(+), 48 deletions(-) rename certs/{id_rsa => id_rsa.key} (100%) create mode 100644 certs/io-ftp-test.key create mode 100644 certs/io-ftp-test.key.pub create mode 100644 certs/io-ftp-test.ppk create mode 100644 docker-build.ps1 create mode 100644 docker-build.sh diff --git a/certs/id_rsa b/certs/id_rsa.key similarity index 100% rename from certs/id_rsa rename to certs/id_rsa.key diff --git a/certs/io-ftp-test.key b/certs/io-ftp-test.key new file mode 100644 index 000000000..ed1e6c8ef --- /dev/null +++ b/certs/io-ftp-test.key @@ -0,0 +1,12 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS +1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5 +U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY +PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt +8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ +AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA +Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk +ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv +5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2 +lvLWZ0cC10ZXN0LWtleQECAwQ= +-----END OPENSSH PRIVATE KEY----- diff --git a/certs/io-ftp-test.key.pub b/certs/io-ftp-test.key.pub new file mode 100644 index 000000000..128a6cb41 --- /dev/null +++ b/certs/io-ftp-test.key.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key diff --git a/certs/io-ftp-test.ppk b/certs/io-ftp-test.ppk new file mode 100644 index 000000000..ced0cec5e --- /dev/null +++ b/certs/io-ftp-test.ppk @@ -0,0 +1,12 @@ +PuTTY-User-Key-File-3: ecdsa-sha2-nistp521 +Encryption: none +Comment: io-ftp-test-key +Public-Lines: 4 +AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt +2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+ +J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX +Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== +Private-Lines: 2 +AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP +wYC0wLMW4bJAf+kjqUnj4wGocoTe +Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54 diff --git a/docker-build.ps1 b/docker-build.ps1 new file mode 100644 index 000000000..663acfc32 --- /dev/null +++ b/docker-build.ps1 @@ -0,0 +1,14 @@ +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 000000000..8fc253f13 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c9cdc14b..c5c456ee6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,8 +114,8 @@ services: " aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1 - aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa - aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 + aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key + aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 " # Node App: The Main IMEX API @@ -169,7 +169,7 @@ services: # - redis-insight-data:/db # ##Optional Container for SFTP/SSH Server for testing -# ssh-sftp-server: +# ssh-sftp-server: # image: atmoz/sftp:alpine # Using an image with SFTP support # container_name: ssh-sftp-server # hostname: ssh-sftp-server @@ -178,9 +178,10 @@ services: # ports: # - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) # volumes: -# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key +# - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # environment: +# # - SFTP_USERS=user::1000::upload # - SFTP_USERS=user:password:1000::upload # command: > # /bin/sh -c " diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..282ff6421 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -23,7 +23,7 @@ const ftpSetup = { } }; -const allcsvsToUpload = []; +const allChatterObjects = []; const allErrors = []; exports.default = async (req, res) => { @@ -62,29 +62,26 @@ exports.default = async (req, res) => { return; } - const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); - } - } else { - await uploadViaSFTP(allcsvsToUpload); - } - })(); - batchPromises.push(batchPromise); + await processBatch(batch, start, end); } - await Promise.all(batchPromises); + + const csvToUpload = { + count: allChatterObjects.length, + csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), + filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` + }; + if (skipUpload) { + await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); + } else { + await uploadViaSFTP(csvToUpload); + } + await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), - null, - 2 - )}` + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n + Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}` }); logger.log("chatter-end", "DEBUG", "api", null, null); @@ -116,15 +113,7 @@ async function processBatch(batch, start, end) { phone_number: j.ownr_ph1 }; }); - - const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); - - allcsvsToUpload.push({ - count: chatterObject.length, - csv: ret, - filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` - }); - + allChatterObjects.push(...chatterObject); logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); @@ -166,7 +155,7 @@ async function getPrivateKey() { } } -async function uploadViaSFTP(allcsvsToUpload) { +async function uploadViaSFTP(csvToUpload) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -178,21 +167,19 @@ async function uploadViaSFTP(allcsvsToUpload) { //Connect to the FTP and upload all. await sftp.connect({ ...ftpSetup, privateKey }); - for (const csvObj of allcsvsToUpload) { - try { - csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { - filename: csvObj.filename, - result: csvObj.result - }); - } catch (error) { - logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { - filename: csvObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`); + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { + filename: csvToUpload.filename, + result: csvToUpload.result + }); + } catch (error) { + logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { + filename: csvToUpload.filename, + error: csvToUpload.message, + stack: csvToUpload.stack + }); + throw error; } } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); From 8ad1dd83c6e21936580475a616ace4d875679292 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 14:24:31 -0800 Subject: [PATCH 3/5] IO-3027 Datapump Refactor Remove Batch and sftp transfer for each shop during processing Signed-off-by: Allan Carr --- server/data/autohouse.js | 99 ++++++++++++++++++++------------------- server/data/claimscorp.js | 99 ++++++++++++++++++++------------------- server/data/kaizen.js | 98 +++++++++++++++++++------------------- 3 files changed, 152 insertions(+), 144 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 1384771e6..19078c274 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,17 +25,15 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -57,12 +55,13 @@ exports.default = async (req, res) => { try { logger.log("autohouse-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -71,27 +70,18 @@ exports.default = async (req, res) => { logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -103,8 +93,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -134,12 +124,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), + filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, + count: autoHouseObject.AutoHouseExport.RepairOrder.length + }; - allxmlsToUpload.push({ - count: autoHouseObject.AutoHouseExport.RepairOrder.length, - xml: ret, - filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + autohouseid: bodyshop.autohouseid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -171,7 +175,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -180,21 +184,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index b8c77a018..3db09cbcf 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,17 +24,15 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -56,12 +54,13 @@ exports.default = async (req, res) => { try { logger.log("claimscorp-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -70,27 +69,18 @@ exports.default = async (req, res) => { logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -102,8 +92,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -137,12 +127,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: claimsCorpObject.DataFeed.ShopInfo.RO.length + }; - allxmlsToUpload.push({ - count: claimsCorpObject.DataFeed.ShopInfo.RO.length, - xml: ret, - filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -174,7 +178,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -183,21 +187,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index bbb758a93..eb6abc056 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,17 +23,15 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -55,12 +53,13 @@ exports.default = async (req, res) => { try { logger.log("kaizen-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -69,27 +68,18 @@ exports.default = async (req, res) => { logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -101,8 +91,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -135,12 +125,25 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: kaizenObject.DataFeed.ShopInfo.Jobs.length + }; - allxmlsToUpload.push({ - count: kaizenObject.DataFeed.ShopInfo.Jobs.length, - xml: ret, - filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -172,7 +175,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -181,21 +184,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); From dd4ba8a467ccc01f88426dcbc6f3a9949d09fd0a Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 14:40:36 -0800 Subject: [PATCH 4/5] IO-2921 Chatter Datapump Adjustment Signed-off-by: Allan Carr --- server/data/chatter.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/server/data/chatter.js b/server/data/chatter.js index 282ff6421..8c3f8e634 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -17,15 +17,15 @@ const ftpSetup = { port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allChatterObjects = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -47,12 +47,13 @@ exports.default = async (req, res) => { try { logger.log("chatter-start", "DEBUG", "api", null, null); + const allChatterObjects = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -62,16 +63,14 @@ exports.default = async (req, res) => { return; } - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - await processBatch(batch, start, end); - } + await processBatch(shopsToProcess, start, end, allChatterObjects, allErrors); const csvToUpload = { count: allChatterObjects.length, csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` }; + if (skipUpload) { await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); } else { @@ -90,8 +89,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processBatch(shopsToProcess, start, end, allChatterObjects, allErrors) { + for (const bodyshop of shopsToProcess) { try { logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname From e202bf9a8966626cdf050d96c94ded2ab2a17e87 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 16:18:23 -0800 Subject: [PATCH 5/5] IO-3027 Add in bodyshop.id to logging in SFTP Signed-off-by: Allan Carr --- server/data/autohouse.js | 12 ++++++++---- server/data/claimscorp.js | 15 +++++++++++---- server/data/kaizen.js | 12 ++++++++---- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 19078c274..81679522b 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -125,6 +125,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, @@ -178,7 +179,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -186,13 +190,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { + logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -200,7 +204,7 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index 3db09cbcf..70737bd03 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -128,6 +128,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, @@ -181,7 +182,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -189,13 +193,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { + logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -203,7 +207,10 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { + error: error.message, + stack: error.stack + }); throw error; } finally { sftp.end(); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index eb6abc056..a7a6aaf91 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -126,6 +126,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, @@ -178,7 +179,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -186,13 +190,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { + logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -200,7 +204,7 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end();