diff --git a/certs/id_rsa b/certs/id_rsa.key similarity index 100% rename from certs/id_rsa rename to certs/id_rsa.key diff --git a/certs/io-ftp-test.key b/certs/io-ftp-test.key new file mode 100644 index 000000000..ed1e6c8ef --- /dev/null +++ b/certs/io-ftp-test.key @@ -0,0 +1,12 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS +1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5 +U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY +PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt +8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ +AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA +Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk +ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv +5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2 +lvLWZ0cC10ZXN0LWtleQECAwQ= +-----END OPENSSH PRIVATE KEY----- diff --git a/certs/io-ftp-test.key.pub b/certs/io-ftp-test.key.pub new file mode 100644 index 000000000..128a6cb41 --- /dev/null +++ b/certs/io-ftp-test.key.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key diff --git a/certs/io-ftp-test.ppk b/certs/io-ftp-test.ppk new file mode 100644 index 000000000..ced0cec5e --- /dev/null +++ b/certs/io-ftp-test.ppk @@ -0,0 +1,12 @@ +PuTTY-User-Key-File-3: ecdsa-sha2-nistp521 +Encryption: none +Comment: io-ftp-test-key +Public-Lines: 4 +AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt +2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+ +J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX +Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== +Private-Lines: 2 +AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP +wYC0wLMW4bJAf+kjqUnj4wGocoTe +Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54 diff --git a/docker-build.ps1 b/docker-build.ps1 new file mode 100644 index 000000000..663acfc32 --- /dev/null +++ b/docker-build.ps1 @@ -0,0 +1,14 @@ +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh new file mode 100644 index 000000000..8fc253f13 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Stop and remove all containers, images, and networks from the Compose file +docker compose down --rmi all + +# Prune all unused Docker objects including volumes +docker system prune --all --volumes --force + +# Prune unused build cache +docker builder prune --all --force + +# Prune all unused volumes +docker volume prune --all --force + +# Rebuild and start the containers +docker compose up --build \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c9cdc14b..c5c456ee6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,8 +114,8 @@ services: " aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1 - aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa - aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 + aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key + aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 " # Node App: The Main IMEX API @@ -169,7 +169,7 @@ services: # - redis-insight-data:/db # ##Optional Container for SFTP/SSH Server for testing -# ssh-sftp-server: +# ssh-sftp-server: # image: atmoz/sftp:alpine # Using an image with SFTP support # container_name: ssh-sftp-server # hostname: ssh-sftp-server @@ -178,9 +178,10 @@ services: # ports: # - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) # volumes: -# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key +# - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # environment: +# # - SFTP_USERS=user::1000::upload # - SFTP_USERS=user:password:1000::upload # command: > # /bin/sh -c " diff --git a/server/data/chatter.js b/server/data/chatter.js index 8890fafe9..282ff6421 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -23,7 +23,7 @@ const ftpSetup = { } }; -const allcsvsToUpload = []; +const allChatterObjects = []; const allErrors = []; exports.default = async (req, res) => { @@ -62,29 +62,26 @@ exports.default = async (req, res) => { return; } - const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); - } - } else { - await uploadViaSFTP(allcsvsToUpload); - } - })(); - batchPromises.push(batchPromise); + await processBatch(batch, start, end); } - await Promise.all(batchPromises); + + const csvToUpload = { + count: allChatterObjects.length, + csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }), + filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv` + }; + if (skipUpload) { + await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv); + } else { + await uploadViaSFTP(csvToUpload); + } + await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), - null, - 2 - )}` + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n + Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}` }); logger.log("chatter-end", "DEBUG", "api", null, null); @@ -116,15 +113,7 @@ async function processBatch(batch, start, end) { phone_number: j.ownr_ph1 }; }); - - const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); - - allcsvsToUpload.push({ - count: chatterObject.length, - csv: ret, - filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` - }); - + allChatterObjects.push(...chatterObject); logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); @@ -166,7 +155,7 @@ async function getPrivateKey() { } } -async function uploadViaSFTP(allcsvsToUpload) { +async function uploadViaSFTP(csvToUpload) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -178,21 +167,19 @@ async function uploadViaSFTP(allcsvsToUpload) { //Connect to the FTP and upload all. await sftp.connect({ ...ftpSetup, privateKey }); - for (const csvObj of allcsvsToUpload) { - try { - csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { - filename: csvObj.filename, - result: csvObj.result - }); - } catch (error) { - logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { - filename: csvObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`); + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { + filename: csvToUpload.filename, + result: csvToUpload.result + }); + } catch (error) { + logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { + filename: csvToUpload.filename, + error: csvToUpload.message, + stack: csvToUpload.stack + }); + throw error; } } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });