From 7cbabf8697cc0367645fefa582be4514934c947d Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Mon, 18 Nov 2024 12:02:08 -0800 Subject: [PATCH 1/3] IO-3027 Turn Off SFTP Logging for Production ENV Signed-off-by: Allan Carr --- package-lock.json | 11 ++++++----- package.json | 2 +- server/data/autohouse.js | 4 +++- server/data/chatter.js | 4 +++- server/data/claimscorp.js | 4 +++- server/data/kaizen.js | 4 +++- 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2f0edfad5..92fb059f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", @@ -7760,16 +7760,17 @@ } }, "node_modules/ssh2-sftp-client": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz", - "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz", + "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==", + "license": "Apache-2.0", "dependencies": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", "ssh2": "^1.15.0" }, "engines": { - "node": ">=16.20.2" + "node": ">=18.20.4" }, "funding": { "type": "individual", diff --git a/package.json b/package.json index 2a3c24ded..3090a4efa 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "soap": "^1.1.5", "socket.io": "^4.8.0", "socket.io-adapter": "^2.5.5", - "ssh2-sftp-client": "^10.0.3", + "ssh2-sftp-client": "^11.0.0", "twilio": "^4.23.0", "uuid": "^10.0.0", "winston": "^3.15.0", diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 7d8447f68..1384771e6 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,7 +25,9 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..8fa217149 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -17,7 +17,9 @@ const ftpSetup = { port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index cafa03df1..b8c77a018 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,7 +24,9 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } diff --git a/server/data/kaizen.js b/server/data/kaizen.js index b79fe4745..bbb758a93 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,7 +23,9 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } From 45ac56e0bc2011131c12f10ad721853be01fde36 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 20 Nov 2024 10:57:59 -0800 Subject: [PATCH 2/3] IO-2921 Chatter modifications as per Dave Bourbeau @ Chatter Signed-off-by: Allan Carr --- certs/{id_rsa => id_rsa.key} | 0 certs/io-ftp-test.key | 12 ++++++ certs/io-ftp-test.key.pub | 1 + certs/io-ftp-test.ppk | 12 ++++++ docker-build.ps1 | 14 +++++++ docker-build.sh | 16 ++++++++ docker-compose.yml | 9 +++-- server/data/chatter.js | 75 +++++++++++++++--------------------- 8 files changed, 91 insertions(+), 48 deletions(-) rename certs/{id_rsa => id_rsa.key} (100%) create mode 100644 certs/io-ftp-test.key create mode 100644 certs/io-ftp-test.key.pub create mode 100644 certs/io-ftp-test.ppk create mode 100644 docker-build.ps1 create mode 100644 docker-build.sh diff --git a/certs/id_rsa b/certs/id_rsa.key similarity index 100% rename from certs/id_rsa rename to certs/id_rsa.key diff --git a/certs/io-ftp-test.key b/certs/io-ftp-test.key new file mode 100644 index 000000000..ed1e6c8ef --- /dev/null +++ b/certs/io-ftp-test.key @@ -0,0 +1,12 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS +1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5 +U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY +PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt +8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ +AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA +Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk +ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv +5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2 +lvLWZ0cC10ZXN0LWtleQECAwQ= +-----END OPENSSH PRIVATE KEY----- diff --git a/certs/io-ftp-test.key.pub b/certs/io-ftp-test.key.pub new file mode 100644 index 000000000..128a6cb41 --- /dev/null +++ b/certs/io-ftp-test.key.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key diff --git a/certs/io-ftp-test.ppk b/certs/io-ftp-test.ppk new file mode 100644 index 000000000..ced0cec5e --- /dev/null +++ b/certs/io-ftp-test.ppk @@ -0,0 +1,12 @@ +PuTTY-User-Key-File-3: ecdsa-sha2-nistp521 +Encryption: none +Comment: io-ftp-test-key +Public-Lines: 4 +AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt +2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+ +J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX +Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== +Private-Lines: 2 +AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP +wYC0wLMW4bJAf+kjqUnj4wGocoTe +Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54 diff --git a/docker-build.ps1 b/docker-build.ps1 new file mode 100644 index 000000000..663acfc32 --- /dev/null +++ b/docker-build.ps1 @@ -0,0 +1,14 @@ +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 000000000..8fc253f13 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c9cdc14b..c5c456ee6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,8 +114,8 @@ services: " aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1 - aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa - aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 + aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key + aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 " # Node App: The Main IMEX API @@ -169,7 +169,7 @@ services: # - redis-insight-data:/db # ##Optional Container for SFTP/SSH Server for testing -# ssh-sftp-server: +# ssh-sftp-server: # image: atmoz/sftp:alpine # Using an image with SFTP support # container_name: ssh-sftp-server # hostname: ssh-sftp-server @@ -178,9 +178,10 @@ services: # ports: # - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) # volumes: -# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key +# - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # environment: +# # - SFTP_USERS=user::1000::upload # - SFTP_USERS=user:password:1000::upload # command: > # /bin/sh -c " diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..282ff6421 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -23,7 +23,7 @@ const ftpSetup = { } }; -const allcsvsToUpload = []; +const allChatterObjects = []; const allErrors = []; exports.default = async (req, res) => { @@ -62,29 +62,26 @@ exports.default = async (req, res) => { return; } - const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); - } - } else { - await uploadViaSFTP(allcsvsToUpload); - } - })(); - batchPromises.push(batchPromise); + await processBatch(batch, start, end); } - await Promise.all(batchPromises); + + const csvToUpload = { + count: allChatterObjects.length, + csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), + filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` + }; + if (skipUpload) { + await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); + } else { + await uploadViaSFTP(csvToUpload); + } + await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), - null, - 2 - )}` + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n + Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}` }); logger.log("chatter-end", "DEBUG", "api", null, null); @@ -116,15 +113,7 @@ async function processBatch(batch, start, end) { phone_number: j.ownr_ph1 }; }); - - const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); - - allcsvsToUpload.push({ - count: chatterObject.length, - csv: ret, - filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` - }); - + allChatterObjects.push(...chatterObject); logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); @@ -166,7 +155,7 @@ async function getPrivateKey() { } } -async function uploadViaSFTP(allcsvsToUpload) { +async function uploadViaSFTP(csvToUpload) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -178,21 +167,19 @@ async function uploadViaSFTP(allcsvsToUpload) { //Connect to the FTP and upload all. await sftp.connect({ ...ftpSetup, privateKey }); - for (const csvObj of allcsvsToUpload) { - try { - csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { - filename: csvObj.filename, - result: csvObj.result - }); - } catch (error) { - logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { - filename: csvObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`); + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { + filename: csvToUpload.filename, + result: csvToUpload.result + }); + } catch (error) { + logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { + filename: csvToUpload.filename, + error: csvToUpload.message, + stack: csvToUpload.stack + }); + throw error; } } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); From dd4ba8a467ccc01f88426dcbc6f3a9949d09fd0a Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 14:40:36 -0800 Subject: [PATCH 3/3] IO-2921 Chatter Datapump Adjustment Signed-off-by: Allan Carr --- server/data/chatter.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/server/data/chatter.js b/server/data/chatter.js index 282ff6421..8c3f8e634 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -17,15 +17,15 @@ const ftpSetup = { port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, - debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allChatterObjects = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -47,12 +47,13 @@ exports.default = async (req, res) => { try { logger.log("chatter-start", "DEBUG", "api", null, null); + const allChatterObjects = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -62,16 +63,14 @@ exports.default = async (req, res) => { return; } - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - await processBatch(batch, start, end); - } + await processBatch(shopsToProcess, start, end, allChatterObjects, allErrors); const csvToUpload = { count: allChatterObjects.length, csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` }; + if (skipUpload) { await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); } else { @@ -90,8 +89,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processBatch(shopsToProcess, start, end, allChatterObjects, allErrors) { + for (const bodyshop of shopsToProcess) { try { logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname