Merged in feature/IO-2921-CARSTAR-Canada-Chatter-Integration (pull request #1946)

Feature/IO-2921 CARSTAR Canada Chatter Integration
2024-11-22 04:39:49 +00:00
parent 289a666b6d 1a6e8bc5ba
commit 1b6eab8488
13 changed files with 115 additions and 66 deletions
--- a/certs/id_rsa.key
+++ b/certs/id_rsa.key
--- a/certs/io-ftp-test.key
+++ b/certs/io-ftp-test.key
@@ -0,0 +1,12 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS
+1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5
+U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY
+PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt
+8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ
+AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA
+Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk
+ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv
+5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2
+lvLWZ0cC10ZXN0LWtleQECAwQ=
+-----END OPENSSH PRIVATE KEY-----
--- a/certs/io-ftp-test.key.pub
+++ b/certs/io-ftp-test.key.pub
@@ -0,0 +1 @@
+ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key
--- a/certs/io-ftp-test.ppk
+++ b/certs/io-ftp-test.ppk
@@ -0,0 +1,12 @@
+PuTTY-User-Key-File-3: ecdsa-sha2-nistp521
+Encryption: none
+Comment: io-ftp-test-key
+Public-Lines: 4
+AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt
+2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+
+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX
+Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w==
+Private-Lines: 2
+AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP
+wYC0wLMW4bJAf+kjqUnj4wGocoTe
+Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54
--- a/docker-build.ps1
+++ b/docker-build.ps1
@@ -0,0 +1,14 @@
+# Stop and remove all containers, images, and networks from the Compose file
+docker compose down --rmi all
+
+# Prune all unused Docker objects including volumes
+docker system prune --all --volumes --force
+
+# Prune unused build cache
+docker builder prune --all --force
+
+# Prune all unused volumes
+docker volume prune --all --force
+
+# Rebuild and start the containers
+docker compose up --build
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Stop and remove all containers, images, and networks from the Compose file
+docker compose down --rmi all
+
+# Prune all unused Docker objects including volumes
+docker system prune --all --volumes --force
+
+# Prune unused build cache
+docker builder prune --all --force
+
+# Prune all unused volumes
+docker volume prune --all --force
+
+# Rebuild and start the containers
+docker compose up --build
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -114,8 +114,8 @@ services:
      "
        aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1
        aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1
-        aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa
-        aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1     
+        aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
+        aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
      "
  # Node App: The Main IMEX API
@@ -169,7 +169,7 @@ services:
 #      - redis-insight-data:/db

 # ##Optional Container for SFTP/SSH Server for testing
-#   ssh-sftp-server:
+#   ssh-sftp-server: 
 #     image: atmoz/sftp:alpine  # Using an image with SFTP support
 #     container_name: ssh-sftp-server
 #     hostname: ssh-sftp-server
@@ -178,9 +178,10 @@ services:
 #     ports:
 #       - "2222:22"  # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
 #     volumes:
-#       - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro  # Mount the SSH public key
+#       - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro  # Mount the SSH public key as authorized_keys
 #       - ./upload:/home/user/upload  # Mount a local directory for SFTP uploads
 #     environment:
+#       # - SFTP_USERS=user::1000::upload
 #       - SFTP_USERS=user:password:1000::upload
 #     command: >
 #       /bin/sh -c "
--- a/package-lock.json
+++ b/package-lock.json
@@ -54,7 +54,7 @@
        "soap": "^1.1.5",
        "socket.io": "^4.8.0",
        "socket.io-adapter": "^2.5.5",
-        "ssh2-sftp-client": "^10.0.3",
+        "ssh2-sftp-client": "^11.0.0",
        "twilio": "^4.23.0",
        "uuid": "^10.0.0",
        "winston": "^3.15.0",
@@ -8740,16 +8740,17 @@
      }
    },
    "node_modules/ssh2-sftp-client": {
-      "version": "10.0.3",
-      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz",
-      "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==",
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz",
+      "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==",
+      "license": "Apache-2.0",
      "dependencies": {
        "concat-stream": "^2.0.0",
        "promise-retry": "^2.0.1",
        "ssh2": "^1.15.0"
      },
      "engines": {
-        "node": ">=16.20.2"
+        "node": ">=18.20.4"
      },
      "funding": {
        "type": "individual",
--- a/package.json
+++ b/package.json
@@ -64,7 +64,7 @@
    "soap": "^1.1.5",
    "socket.io": "^4.8.0",
    "socket.io-adapter": "^2.5.5",
-    "ssh2-sftp-client": "^10.0.3",
+    "ssh2-sftp-client": "^11.0.0",
    "twilio": "^4.23.0",
    "uuid": "^10.0.0",
    "winston": "^3.15.0",
--- a/server/data/autohouse.js
+++ b/server/data/autohouse.js
@@ -25,7 +25,9 @@ const ftpSetup = {
  port: process.env.AUTOHOUSE_PORT,
  username: process.env.AUTOHOUSE_USER,
  password: process.env.AUTOHOUSE_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
  algorithms: {
    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
  }
--- a/server/data/chatter.js
+++ b/server/data/chatter.js
@@ -17,15 +17,15 @@ const ftpSetup = {
  port: process.env.CHATTER_PORT,
  username: process.env.CHATTER_USER,
  privateKey: null,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
  algorithms: {
    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
  }
 };

-const allcsvsToUpload = [];
-const allErrors = [];
-
 exports.default = async (req, res) => {
  // Only process if in production environment.
  if (process.env.NODE_ENV !== "production") {
@@ -47,12 +47,13 @@ exports.default = async (req, res) => {

  try {
    logger.log("chatter-start", "DEBUG", "api", null, null);
+    const allChatterObjects = [];
+    const allErrors = [];
+
    const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients.
    const specificShopIds = req.body.bodyshopIds; // ['uuid];
    const { start, end, skipUpload } = req.body; //YYYY-MM-DD

-    const batchSize = 10;
-
    const shopsToProcess =
      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
    logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -62,29 +63,24 @@ exports.default = async (req, res) => {
      return;
    }

-    const batchPromises = [];
-    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
-      const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
-        await processBatch(batch, start, end);
-        if (skipUpload) {
-          for (const csvObj of allcsvsToUpload) {
-            await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
-          }
-        } else {
-          await uploadViaSFTP(allcsvsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
+    await processBatch(shopsToProcess, start, end, allChatterObjects, allErrors);
+
+    const csvToUpload = {
+      count: allChatterObjects.length,
+      csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }),
+      filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv`
+    };
+
+    if (skipUpload) {
+      await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv);
+    } else {
+      await uploadViaSFTP(csvToUpload);
    }
-    await Promise.all(batchPromises);
+
    await sendServerEmail({
      subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
-        null,
-        2
-      )}`
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n
+      Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}`
    });

    logger.log("chatter-end", "DEBUG", "api", null, null);
@@ -93,8 +89,8 @@ exports.default = async (req, res) => {
  }
 };

-async function processBatch(batch, start, end) {
-  for (const bodyshop of batch) {
+async function processBatch(shopsToProcess, start, end, allChatterObjects, allErrors) {
+  for (const bodyshop of shopsToProcess) {
    try {
      logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
        shopname: bodyshop.shopname
@@ -116,15 +112,7 @@ async function processBatch(batch, start, end) {
          phone_number: j.ownr_ph1
        };
      });
-
-      const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
-
-      allcsvsToUpload.push({
-        count: chatterObject.length,
-        csv: ret,
-        filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
-      });
-
+      allChatterObjects.push(...chatterObject);
      logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
        shopname: bodyshop.shopname
      });
@@ -166,7 +154,7 @@ async function getPrivateKey() {
  }
 }

-async function uploadViaSFTP(allcsvsToUpload) {
+async function uploadViaSFTP(csvToUpload) {
  const sftp = new Client();
  sftp.on("error", (errors) =>
    logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
@@ -178,21 +166,19 @@ async function uploadViaSFTP(allcsvsToUpload) {
    //Connect to the FTP and upload all.
    await sftp.connect({ ...ftpSetup, privateKey });

-    for (const csvObj of allcsvsToUpload) {
-      try {
-        csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`);
-        logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
-          filename: csvObj.filename,
-          result: csvObj.result
-        });
-      } catch (error) {
-        logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
-          filename: csvObj.filename,
-          error: error.message,
-          stack: error.stack
-        });
-        throw error;
-      }
+    try {
+      csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`);
+      logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
+        filename: csvToUpload.filename,
+        result: csvToUpload.result
+      });
+    } catch (error) {
+      logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
+        filename: csvToUpload.filename,
+        error: csvToUpload.message,
+        stack: csvToUpload.stack
+      });
+      throw error;
    }
  } catch (error) {
    logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
--- a/server/data/claimscorp.js
+++ b/server/data/claimscorp.js
@@ -24,7 +24,9 @@ const ftpSetup = {
  port: process.env.CLAIMSCORP_PORT,
  username: process.env.CLAIMSCORP_USER,
  password: process.env.CLAIMSCORP_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
  algorithms: {
    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
  }
--- a/server/data/kaizen.js
+++ b/server/data/kaizen.js
@@ -23,7 +23,9 @@ const ftpSetup = {
  port: process.env.KAIZEN_PORT,
  username: process.env.KAIZEN_USER,
  password: process.env.KAIZEN_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
  algorithms: {
    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
  }
				`@@ -0,0 +1 @@`
				`ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key`