IO-2921 Chatter Final mods and Cron Trigger

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>

hasura/metadata/cron_triggers.yaml

@@ -6,6 +6,15 @@
   headers:
     - name: x-imex-auth
       value_from_env: DATAPUMP_AUTH
+- name: Chatter Data Pump
+  webhook: '{{HASURA_API_URL}}/data/chatter'
+  schedule: 45 5 * * *
+  include_in_metadata: true
+  payload: {}
+  headers:
+    - name: x-imex-auth
+      value_from_env: DATAPUMP_AUTH
+  comment: ""
 - name: Claimscorp Data Pump
   webhook: '{{HASURA_API_URL}}/data/cc'
   schedule: 30 6 * * *

server/data/autohouse.js

@@ -25,15 +25,17 @@ const ftpSetup = {
   port: process.env.AUTOHOUSE_PORT,
   username: process.env.AUTOHOUSE_USER,
   password: process.env.AUTOHOUSE_PASSWORD,
-  debug:
-    process.env.NODE_ENV !== "production"
-      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
-      : () => {},
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
+const allxmlsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -55,13 +57,12 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("autohouse-start", "DEBUG", "api", null, null);
-    const allXMLResults = [];
-    const allErrors = [];
-
     const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
+    const batchSize = 10;
+
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -70,18 +71,27 @@ exports.default = async (req, res) => {
       logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
 
-    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
-
+        if (skipUpload) {
+          for (const xmlObj of allxmlsToUpload) {
+            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+          }
+        } else {
+          await uploadViaSFTP(allxmlsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
+    }
+    await Promise.all(batchPromises);
     await sendServerEmail({
       subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allXMLResults.map((x) => ({
-          imexshopid: x.imexshopid,
-          filename: x.filename,
-          count: x.count,
-          result: x.result
-        })),
+        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
         null,
         2
       )}`
@@ -93,8 +103,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
-  for (const bodyshop of shopsToProcess) {
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
     const erroredJobs = [];
     try {
       logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -124,27 +134,12 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
         });
       }
 
-      const xmlObj = {
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }),
-        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`,
-        count: autoHouseObject.AutoHouseExport.RepairOrder.length
-      };
+      const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true });
 
-      if (skipUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      } else {
-        await uploadViaSFTP(xmlObj);
-      }
-
-      allXMLResults.push({
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        autohouseid: bodyshop.autohouseid,
-        count: xmlObj.count,
-        filename: xmlObj.filename,
-        result: xmlObj.result
+      allxmlsToUpload.push({
+        count: autoHouseObject.AutoHouseExport.RepairOrder.length,
+        xml: ret,
+        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`
       });
 
       logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -176,35 +171,33 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
   }
 }
 
-async function uploadViaSFTP(xmlObj) {
+async function uploadViaSFTP(allxmlsToUpload) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
-      error: errors.message,
-      stack: errors.stack
-    })
+    logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    try {
-      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-      logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
-        imexshopid: xmlObj.imexshopid,
-        filename: xmlObj.filename,
-        result: xmlObj.result
-      });
-    } catch (error) {
-      logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
-        filename: xmlObj.filename,
-        error: error.message,
-        stack: error.stack
-      });
-      throw error;
+    for (const xmlObj of allxmlsToUpload) {
+      try {
+        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+        logger.log("autohouse-sftp-upload", "DEBUG", "api", null, {
+          filename: xmlObj.filename,
+          result: xmlObj.result
+        });
+      } catch (error) {
+        logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, {
+          filename: xmlObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
     }
   } catch (error) {
-    logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
+    logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();

server/data/chatter.js

@@ -109,7 +109,8 @@ async function processBatch(shopsToProcess, start, end, allChatterObjects, allEr
           lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln,
           transaction_id: j.ro_number,
           email: j.ownr_ea,
-          phone_number: j.ownr_ph1
+          phone_number: j.ownr_ph1,
+          transaction_time: (j.actual_delivery && moment(j.actual_delivery).tz(bodyshop.timezone).format("YYYYMMDD-HHmm")) || ""
         };
       });
       allChatterObjects.push(...chatterObject);

server/data/claimscorp.js

@@ -24,15 +24,17 @@ const ftpSetup = {
   port: process.env.CLAIMSCORP_PORT,
   username: process.env.CLAIMSCORP_USER,
   password: process.env.CLAIMSCORP_PASSWORD,
-  debug:
-    process.env.NODE_ENV !== "production"
-      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
-      : () => {},
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
+const allxmlsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -54,13 +56,12 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("claimscorp-start", "DEBUG", "api", null, null);
-    const allXMLResults = [];
-    const allErrors = [];
-
     const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
+    const batchSize = 10;
+
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -69,18 +70,27 @@ exports.default = async (req, res) => {
       logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
 
-    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
-
+        if (skipUpload) {
+          for (const xmlObj of allxmlsToUpload) {
+            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+          }
+        } else {
+          await uploadViaSFTP(allxmlsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
+    }
+    await Promise.all(batchPromises);
     await sendServerEmail({
       subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allXMLResults.map((x) => ({
-          imexshopid: x.imexshopid,
-          filename: x.filename,
-          count: x.count,
-          result: x.result
-        })),
+        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
         null,
         2
       )}`
@@ -92,8 +102,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
-  for (const bodyshop of shopsToProcess) {
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
     const erroredJobs = [];
     try {
       logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -127,27 +137,12 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
         });
       }
 
-      const xmlObj = {
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }),
-        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
-        count: claimsCorpObject.DataFeed.ShopInfo.RO.length
-      };
+      const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true });
 
-      if (skipUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      } else {
-        await uploadViaSFTP(xmlObj);
-      }
-
-      allXMLResults.push({
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        claimscorpid: bodyshop.claimscorpid,
-        count: xmlObj.count,
-        filename: xmlObj.filename,
-        result: xmlObj.result
+      allxmlsToUpload.push({
+        count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
+        xml: ret,
+        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
       });
 
       logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -179,38 +174,33 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
   }
 }
 
-async function uploadViaSFTP(xmlObj) {
+async function uploadViaSFTP(allxmlsToUpload) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
-      error: errors.message,
-      stack: errors.stack
-    })
+    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    try {
-      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-      logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
-        imexshopid: xmlObj.imexshopid,
-        filename: xmlObj.filename,
-        result: xmlObj.result
-      });
-    } catch (error) {
-      logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
-        filename: xmlObj.filename,
-        error: error.message,
-        stack: error.stack
-      });
-      throw error;
+    for (const xmlObj of allxmlsToUpload) {
+      try {
+        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+        logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
+          filename: xmlObj.filename,
+          result: xmlObj.result
+        });
+      } catch (error) {
+        logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, {
+          filename: xmlObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
     }
   } catch (error) {
-    logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, {
-      error: error.message,
-      stack: error.stack
-    });
+    logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();

server/data/kaizen.js

@@ -23,15 +23,17 @@ const ftpSetup = {
   port: process.env.KAIZEN_PORT,
   username: process.env.KAIZEN_USER,
   password: process.env.KAIZEN_PASSWORD,
-  debug:
-    process.env.NODE_ENV !== "production"
-      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
-      : () => {},
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
+const allxmlsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -53,13 +55,12 @@ exports.default = async (req, res) => {
 
   try {
     logger.log("kaizen-start", "DEBUG", "api", null, null);
-    const allXMLResults = [];
-    const allErrors = [];
-
     const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
+    const batchSize = 10;
+
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -68,18 +69,27 @@ exports.default = async (req, res) => {
       logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
 
-    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
-
+        if (skipUpload) {
+          for (const xmlObj of allxmlsToUpload) {
+            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+          }
+        } else {
+          await uploadViaSFTP(allxmlsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
+    }
+    await Promise.all(batchPromises);
     await sendServerEmail({
       subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allXMLResults.map((x) => ({
-          imexshopid: x.imexshopid,
-          filename: x.filename,
-          count: x.count,
-          result: x.result
-        })),
+        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
         null,
         2
       )}`
@@ -91,8 +101,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
-  for (const bodyshop of shopsToProcess) {
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
     const erroredJobs = [];
     try {
       logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -125,26 +135,12 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
         });
       }
 
-      const xmlObj = {
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }),
-        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
-        count: kaizenObject.DataFeed.ShopInfo.Jobs.length
-      };
+      const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
 
-      if (skipUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      } else {
-        await uploadViaSFTP(xmlObj);
-      }
-
-      allXMLResults.push({
-        bodyshopid: bodyshop.id,
-        imexshopid: bodyshop.imexshopid,
-        count: xmlObj.count,
-        filename: xmlObj.filename,
-        result: xmlObj.result
+      allxmlsToUpload.push({
+        count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
+        xml: ret,
+        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
       });
 
       logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -176,35 +172,33 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes
   }
 }
 
-async function uploadViaSFTP(xmlObj) {
+async function uploadViaSFTP(allxmlsToUpload) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
-      error: errors.message,
-      stack: errors.stack
-    })
+    logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    try {
-      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-      logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
-        imexshopid: xmlObj.imexshopid,
-        filename: xmlObj.filename,
-        result: xmlObj.result
-      });
-    } catch (error) {
-      logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
-        filename: xmlObj.filename,
-        error: error.message,
-        stack: error.stack
-      });
-      throw error;
+    for (const xmlObj of allxmlsToUpload) {
+      try {
+        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+        logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
+          filename: xmlObj.filename,
+          result: xmlObj.result
+        });
+      } catch (error) {
+        logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
+          filename: xmlObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
     }
   } catch (error) {
-    logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
+    logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();

server/graphql-client/queries.js

@@ -840,6 +840,7 @@ exports.CHATTER_QUERY = `query CHATTER_EXPORT($start: timestamptz, $bodyshopid:
     timezone
   }
   jobs(where: {_and: [{converted: {_eq: true}}, {actual_delivery: {_gt: $start}}, {actual_delivery: {_lte: $end}}, {shopid: {_eq: $bodyshopid}}, {_or: [{ownr_ph1: {_is_null: false}}, {ownr_ea: {_is_null: false}}]}]}) {
+    actual_delivery
     id
     created_at
     ro_number