diff --git a/certs/id_rsa b/certs/id_rsa.key similarity index 100% rename from certs/id_rsa rename to certs/id_rsa.key diff --git a/certs/io-ftp-test.key b/certs/io-ftp-test.key new file mode 100644 index 000000000..ed1e6c8ef --- /dev/null +++ b/certs/io-ftp-test.key @@ -0,0 +1,12 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS +1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5 +U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY +PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt +8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ +AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA +Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk +ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv +5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2 +lvLWZ0cC10ZXN0LWtleQECAwQ= +-----END OPENSSH PRIVATE KEY----- diff --git a/certs/io-ftp-test.key.pub b/certs/io-ftp-test.key.pub new file mode 100644 index 000000000..128a6cb41 --- /dev/null +++ b/certs/io-ftp-test.key.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key diff --git a/certs/io-ftp-test.ppk b/certs/io-ftp-test.ppk new file mode 100644 index 000000000..ced0cec5e --- /dev/null +++ b/certs/io-ftp-test.ppk @@ -0,0 +1,12 @@ +PuTTY-User-Key-File-3: ecdsa-sha2-nistp521 +Encryption: none +Comment: io-ftp-test-key +Public-Lines: 4 +AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt +2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+ +J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX +Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== +Private-Lines: 2 +AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP +wYC0wLMW4bJAf+kjqUnj4wGocoTe +Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54 diff --git a/docker-build.ps1 b/docker-build.ps1 new file mode 100644 index 000000000..663acfc32 --- /dev/null +++ b/docker-build.ps1 @@ -0,0 +1,14 @@ +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 000000000..8fc253f13 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c9cdc14b..c5c456ee6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,8 +114,8 @@ services: " aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1 - aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa - aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 + aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key + aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 " # Node App: The Main IMEX API @@ -169,7 +169,7 @@ services: # - redis-insight-data:/db # ##Optional Container for SFTP/SSH Server for testing -# ssh-sftp-server: +# ssh-sftp-server: # image: atmoz/sftp:alpine # Using an image with SFTP support # container_name: ssh-sftp-server # hostname: ssh-sftp-server @@ -178,9 +178,10 @@ services: # ports: # - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) # volumes: -# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key +# - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # environment: +# # - SFTP_USERS=user::1000::upload # - SFTP_USERS=user:password:1000::upload # command: > # /bin/sh -c " diff --git a/package-lock.json b/package-lock.json index d95191db5..920f070a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -54,7 +54,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", @@ -8740,16 +8740,17 @@ } }, "node_modules/ssh2-sftp-client": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz", - "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz", + "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==", + "license": "Apache-2.0", "dependencies": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", "ssh2": "^1.15.0" }, "engines": { - "node": ">=16.20.2" + "node": ">=18.20.4" }, "funding": { "type": "individual", diff --git a/package.json b/package.json index f51af45b5..eec843a5c 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 7d8447f68..81679522b 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,15 +25,15 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -55,12 +55,13 @@ exports.default = async (req, res) => { try { logger.log("autohouse-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -69,27 +70,18 @@ exports.default = async (req, res) => { logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -101,8 +93,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -132,12 +124,27 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), + filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, + count: autoHouseObject.AutoHouseExport.RepairOrder.length + }; - allxmlsToUpload.push({ - count: autoHouseObject.AutoHouseExport.RepairOrder.length, - xml: ret, - filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + autohouseid: bodyshop.autohouseid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -169,33 +176,35 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..8c3f8e634 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -17,15 +17,15 @@ const ftpSetup = { port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allcsvsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -47,12 +47,13 @@ exports.default = async (req, res) => { try { logger.log("chatter-start", "DEBUG", "api", null, null); + const allChatterObjects = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -62,29 +63,24 @@ exports.default = async (req, res) => { return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); - } - } else { - await uploadViaSFTP(allcsvsToUpload); - } - })(); - batchPromises.push(batchPromise); + await processBatch(shopsToProcess, start, end, allChatterObjects, allErrors); + + const csvToUpload = { + count: allChatterObjects.length, + csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), + filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` + }; + + if (skipUpload) { + await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); + } else { + await uploadViaSFTP(csvToUpload); } - await Promise.all(batchPromises); + await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), - null, - 2 - )}` + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n + Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}` }); logger.log("chatter-end", "DEBUG", "api", null, null); @@ -93,8 +89,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processBatch(shopsToProcess, start, end, allChatterObjects, allErrors) { + for (const bodyshop of shopsToProcess) { try { logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname @@ -116,15 +112,7 @@ async function processBatch(batch, start, end) { phone_number: j.ownr_ph1 }; }); - - const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); - - allcsvsToUpload.push({ - count: chatterObject.length, - csv: ret, - filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` - }); - + allChatterObjects.push(...chatterObject); logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); @@ -166,7 +154,7 @@ async function getPrivateKey() { } } -async function uploadViaSFTP(allcsvsToUpload) { +async function uploadViaSFTP(csvToUpload) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -178,21 +166,19 @@ async function uploadViaSFTP(allcsvsToUpload) { //Connect to the FTP and upload all. await sftp.connect({ ...ftpSetup, privateKey }); - for (const csvObj of allcsvsToUpload) { - try { - csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { - filename: csvObj.filename, - result: csvObj.result - }); - } catch (error) { - logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { - filename: csvObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`); + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { + filename: csvToUpload.filename, + result: csvToUpload.result + }); + } catch (error) { + logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { + filename: csvToUpload.filename, + error: csvToUpload.message, + stack: csvToUpload.stack + }); + throw error; } } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index cafa03df1..70737bd03 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,15 +24,15 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -54,12 +54,13 @@ exports.default = async (req, res) => { try { logger.log("claimscorp-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -68,27 +69,18 @@ exports.default = async (req, res) => { logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -100,8 +92,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -135,12 +127,27 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: claimsCorpObject.DataFeed.ShopInfo.RO.length + }; - allxmlsToUpload.push({ - count: claimsCorpObject.DataFeed.ShopInfo.RO.length, - xml: ret, - filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -172,33 +179,38 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { + error: error.message, + stack: error.stack + }); throw error; } finally { sftp.end(); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index b79fe4745..a7a6aaf91 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,15 +23,15 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -53,12 +53,13 @@ exports.default = async (req, res) => { try { logger.log("kaizen-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -67,27 +68,18 @@ exports.default = async (req, res) => { logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -99,8 +91,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -133,12 +125,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: kaizenObject.DataFeed.ShopInfo.Jobs.length + }; - allxmlsToUpload.push({ - count: kaizenObject.DataFeed.ShopInfo.Jobs.length, - xml: ret, - filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -170,33 +176,35 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end();