IO-3027 Add in bodyshop.id to logging in SFTP

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>

package-lock.json

@@ -53,7 +53,7 @@
         "soap": "^1.1.5",
         "socket.io": "^4.8.0",
         "socket.io-adapter": "^2.5.5",
-        "ssh2-sftp-client": "^10.0.3",
+        "ssh2-sftp-client": "^11.0.0",
         "twilio": "^4.23.0",
         "uuid": "^10.0.0",
         "winston": "^3.15.0",
@@ -7760,16 +7760,17 @@
       }
     },
     "node_modules/ssh2-sftp-client": {
-      "version": "10.0.3",
-      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz",
-      "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==",
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz",
+      "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==",
+      "license": "Apache-2.0",
       "dependencies": {
         "concat-stream": "^2.0.0",
         "promise-retry": "^2.0.1",
         "ssh2": "^1.15.0"
       },
       "engines": {
-        "node": ">=16.20.2"
+        "node": ">=18.20.4"
       },
       "funding": {
         "type": "individual",

package.json

@@ -63,7 +63,7 @@
     "soap": "^1.1.5",
     "socket.io": "^4.8.0",
     "socket.io-adapter": "^2.5.5",
-    "ssh2-sftp-client": "^10.0.3",
+    "ssh2-sftp-client": "^11.0.0",
     "twilio": "^4.23.0",
     "uuid": "^10.0.0",
     "winston": "^3.15.0",

server/data/autohouse.js

@@ -25,15 +25,15 @@ const ftpSetup = {
   port: process.env.AUTOHOUSE_PORT,
   username: process.env.AUTOHOUSE_USER,
   password: process.env.AUTOHOUSE_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
-const allxmlsToUpload = [];
-const allErrors = [];
-
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -55,12 +55,13 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("autohouse-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
+
     const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-    const batchSize = 10;
-
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -69,27 +70,18 @@ exports.default = async (req, res) => {
       logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
-    const batchPromises = [];
-    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
-      const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
-        await processBatch(batch, start, end);
 
-        if (skipUpload) {
-          for (const xmlObj of allxmlsToUpload) {
-            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-          }
-        } else {
-          await uploadViaSFTP(allxmlsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
-    }
-    await Promise.all(batchPromises);
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
     await sendServerEmail({
       subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
         null,
         2
       )}`
@@ -101,8 +93,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processBatch(batch, start, end) {
-  for (const bodyshop of batch) {
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
     const erroredJobs = [];
     try {
       logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -132,12 +124,27 @@ async function processBatch(batch, start, end) {
         });
       }
 
-      const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true });
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }),
+        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`,
+        count: autoHouseObject.AutoHouseExport.RepairOrder.length
+      };
 
-      allxmlsToUpload.push({
-        count: autoHouseObject.AutoHouseExport.RepairOrder.length,
-        xml: ret,
-        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        autohouseid: bodyshop.autohouseid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
       });
 
       logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -169,33 +176,35 @@ async function processBatch(batch, start, end) {
   }
 }
 
-async function uploadViaSFTP(allxmlsToUpload) {
+async function uploadViaSFTP(xmlObj) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+    logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    for (const xmlObj of allxmlsToUpload) {
-      try {
-        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-        logger.log("autohouse-sftp-upload", "DEBUG", "api", null, {
-          filename: xmlObj.filename,
-          result: xmlObj.result
-        });
-      } catch (error) {
-        logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, {
-          filename: xmlObj.filename,
-          error: error.message,
-          stack: error.stack
-        });
-        throw error;
-      }
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
     }
   } catch (error) {
-    logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();

server/data/chatter.js

@@ -17,7 +17,9 @@ const ftpSetup = {
   port: process.env.CHATTER_PORT,
   username: process.env.CHATTER_USER,
   privateKey: null,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }

server/data/claimscorp.js

@@ -24,15 +24,15 @@ const ftpSetup = {
   port: process.env.CLAIMSCORP_PORT,
   username: process.env.CLAIMSCORP_USER,
   password: process.env.CLAIMSCORP_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
-const allxmlsToUpload = [];
-const allErrors = [];
-
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -54,12 +54,13 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("claimscorp-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
+
     const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-    const batchSize = 10;
-
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -68,27 +69,18 @@ exports.default = async (req, res) => {
       logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
-    const batchPromises = [];
-    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
-      const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
-        await processBatch(batch, start, end);
 
-        if (skipUpload) {
-          for (const xmlObj of allxmlsToUpload) {
-            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-          }
-        } else {
-          await uploadViaSFTP(allxmlsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
-    }
-    await Promise.all(batchPromises);
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
     await sendServerEmail({
       subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
         null,
         2
       )}`
@@ -100,8 +92,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processBatch(batch, start, end) {
-  for (const bodyshop of batch) {
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
     const erroredJobs = [];
     try {
       logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -135,12 +127,27 @@ async function processBatch(batch, start, end) {
         });
       }
 
-      const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true });
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }),
+        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
+        count: claimsCorpObject.DataFeed.ShopInfo.RO.length
+      };
 
-      allxmlsToUpload.push({
-        count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
-        xml: ret,
-        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
       });
 
       logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -172,33 +179,38 @@ async function processBatch(batch, start, end) {
   }
 }
 
-async function uploadViaSFTP(allxmlsToUpload) {
+async function uploadViaSFTP(xmlObj) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    for (const xmlObj of allxmlsToUpload) {
-      try {
-        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-        logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
-          filename: xmlObj.filename,
-          result: xmlObj.result
-        });
-      } catch (error) {
-        logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, {
-          filename: xmlObj.filename,
-          error: error.message,
-          stack: error.stack
-        });
-        throw error;
-      }
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
     }
   } catch (error) {
-    logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: error.message,
+      stack: error.stack
+    });
     throw error;
   } finally {
     sftp.end();

server/data/kaizen.js

@@ -23,15 +23,15 @@ const ftpSetup = {
   port: process.env.KAIZEN_PORT,
   username: process.env.KAIZEN_USER,
   password: process.env.KAIZEN_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
-const allxmlsToUpload = [];
-const allErrors = [];
-
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -53,12 +53,13 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("kaizen-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
+
     const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-    const batchSize = 10;
-
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -67,27 +68,18 @@ exports.default = async (req, res) => {
       logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
-    const batchPromises = [];
-    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
-      const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
-        await processBatch(batch, start, end);
 
-        if (skipUpload) {
-          for (const xmlObj of allxmlsToUpload) {
-            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-          }
-        } else {
-          await uploadViaSFTP(allxmlsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
-    }
-    await Promise.all(batchPromises);
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
     await sendServerEmail({
       subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
         null,
         2
       )}`
@@ -99,8 +91,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processBatch(batch, start, end) {
-  for (const bodyshop of batch) {
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
     const erroredJobs = [];
     try {
       logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -133,12 +125,26 @@ async function processBatch(batch, start, end) {
         });
       }
 
-      const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }),
+        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
+        count: kaizenObject.DataFeed.ShopInfo.Jobs.length
+      };
 
-      allxmlsToUpload.push({
-        count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
-        xml: ret,
-        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
       });
 
       logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -170,33 +176,35 @@ async function processBatch(batch, start, end) {
   }
 }
 
-async function uploadViaSFTP(allxmlsToUpload) {
+async function uploadViaSFTP(xmlObj) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+    logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    for (const xmlObj of allxmlsToUpload) {
-      try {
-        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-        logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
-          filename: xmlObj.filename,
-          result: xmlObj.result
-        });
-      } catch (error) {
-        logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
-          filename: xmlObj.filename,
-          error: error.message,
-          stack: error.stack
-        });
-        throw error;
-      }
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
     }
   } catch (error) {
-    logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();