IO-2921 Chatter modifications as per Dave Bourbeau @ Chatter

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>

certs/io-ftp-test.key

@@ -0,0 +1,12 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS
+1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5
+U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY
+PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt
+8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ
+AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA
+Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk
+ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv
+5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2
+lvLWZ0cC10ZXN0LWtleQECAwQ=
+-----END OPENSSH PRIVATE KEY-----

certs/io-ftp-test.key.pub

@@ -0,0 +1 @@
+ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key

certs/io-ftp-test.ppk

@@ -0,0 +1,12 @@
+PuTTY-User-Key-File-3: ecdsa-sha2-nistp521
+Encryption: none
+Comment: io-ftp-test-key
+Public-Lines: 4
+AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt
+2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+
+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX
+Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w==
+Private-Lines: 2
+AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP
+wYC0wLMW4bJAf+kjqUnj4wGocoTe
+Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54

docker-build.ps1

@@ -0,0 +1,14 @@
+# Stop and remove all containers, images, and networks from the Compose file
+docker compose down --rmi all
+
+# Prune all unused Docker objects including volumes
+docker system prune --all --volumes --force
+
+# Prune unused build cache
+docker builder prune --all --force
+
+# Prune all unused volumes
+docker volume prune --all --force
+
+# Rebuild and start the containers
+docker compose up --build

docker-build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Stop and remove all containers, images, and networks from the Compose file
+docker compose down --rmi all
+
+# Prune all unused Docker objects including volumes
+docker system prune --all --volumes --force
+
+# Prune unused build cache
+docker builder prune --all --force
+
+# Prune all unused volumes
+docker volume prune --all --force
+
+# Rebuild and start the containers
+docker compose up --build

docker-compose.yml

@@ -114,8 +114,8 @@ services:
       "
         aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1
         aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1
-        aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa
+        aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
-        aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1     
+        aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
         aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
       "
   # Node App: The Main IMEX API
@@ -169,7 +169,7 @@ services:
 #      - redis-insight-data:/db
 
 # ##Optional Container for SFTP/SSH Server for testing
-#   ssh-sftp-server:
+#   ssh-sftp-server: 
 #     image: atmoz/sftp:alpine  # Using an image with SFTP support
 #     container_name: ssh-sftp-server
 #     hostname: ssh-sftp-server
@@ -178,9 +178,10 @@ services:
 #     ports:
 #       - "2222:22"  # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
 #     volumes:
-#       - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro  # Mount the SSH public key
+#       - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro  # Mount the SSH public key as authorized_keys
 #       - ./upload:/home/user/upload  # Mount a local directory for SFTP uploads
 #     environment:
+#       # - SFTP_USERS=user::1000::upload
 #       - SFTP_USERS=user:password:1000::upload
 #     command: >
 #       /bin/sh -c "

server/data/chatter.js

@@ -23,7 +23,7 @@ const ftpSetup = {
   }
 };
 
-const allcsvsToUpload = [];
+const allChatterObjects = [];
 const allErrors = [];
 
 exports.default = async (req, res) => {
@@ -62,29 +62,26 @@ exports.default = async (req, res) => {
       return;
     }
 
-    const batchPromises = [];
     for (let i = 0; i < shopsToProcess.length; i += batchSize) {
       const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
+      await processBatch(batch, start, end);
-        await processBatch(batch, start, end);
-        if (skipUpload) {
-          for (const csvObj of allcsvsToUpload) {
-            await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
-          }
-        } else {
-          await uploadViaSFTP(allcsvsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
     }
-    await Promise.all(batchPromises);
+
+    const csvToUpload = {
+      count: allChatterObjects.length,
+      csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }),
+      filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv`
+    };
+    if (skipUpload) {
+      await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv);
+    } else {
+      await uploadViaSFTP(csvToUpload);
+    }
+
     await sendServerEmail({
       subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n
-        allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+      Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}`
-        null,
-        2
-      )}`
     });
 
     logger.log("chatter-end", "DEBUG", "api", null, null);
@@ -116,15 +113,7 @@ async function processBatch(batch, start, end) {
           phone_number: j.ownr_ph1
         };
       });
-
+      allChatterObjects.push(...chatterObject);
-      const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
-
-      allcsvsToUpload.push({
-        count: chatterObject.length,
-        csv: ret,
-        filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
-      });
-
       logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
         shopname: bodyshop.shopname
       });
@@ -166,7 +155,7 @@ async function getPrivateKey() {
   }
 }
 
-async function uploadViaSFTP(allcsvsToUpload) {
+async function uploadViaSFTP(csvToUpload) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
     logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
@@ -178,21 +167,19 @@ async function uploadViaSFTP(allcsvsToUpload) {
     //Connect to the FTP and upload all.
     await sftp.connect({ ...ftpSetup, privateKey });
 
-    for (const csvObj of allcsvsToUpload) {
+    try {
-      try {
+      csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`);
-        csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`);
+      logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
-        logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
+        filename: csvToUpload.filename,
-          filename: csvObj.filename,
+        result: csvToUpload.result
-          result: csvObj.result
+      });
-        });
+    } catch (error) {
-      } catch (error) {
+      logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
-        logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
+        filename: csvToUpload.filename,
-          filename: csvObj.filename,
+        error: csvToUpload.message,
-          error: error.message,
+        stack: csvToUpload.stack
-          stack: error.stack
+      });
-        });
+      throw error;
-        throw error;
-      }
     }
   } catch (error) {
     logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });