IO-3027 Datapumps Refactor

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>

.vscode/settings.json

@@ -8,5 +8,35 @@
       "pattern": "**/IMEX.xml",
       "systemId": "logs/IMEX.xsd"
     }
+  ],
+  "cSpell.words": [
+    "antd",
+    "appointmentconfirmation",
+    "appt",
+    "autohouse",
+    "autohouseid",
+    "billlines",
+    "bodyshop",
+    "bodyshopid",
+    "bodyshops",
+    "CIECA",
+    "claimscorp",
+    "claimscorpid",
+    "Dinero",
+    "driveable",
+    "IMEX",
+    "imexshopid",
+    "jobid",
+    "joblines",
+    "Kaizen",
+    "labhrs",
+    "larhrs",
+    "mixdata",
+    "ownr",
+    "promanager",
+    "shopname",
+    "smartscheduling",
+    "timetickets",
+    "touchtime"
   ]
 }

server/data/autohouse.js

@@ -47,7 +47,11 @@ exports.default = async (req, res) => {
   }
 
   // Send immediate response and continue processing.
-  res.status(200).send();
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
 
   try {
     logger.log("autohouse-start", "DEBUG", "api", null, null);
@@ -146,7 +150,7 @@ async function processBatch(batch, start, end) {
       allErrors.push({
         bodyshopid: bodyshop.id,
         imexshopid: bodyshop.imexshopid,
-        autuhouseid: bodyshop.autuhouseid,
+        autohouseid: bodyshop.autohouseid,
         fatal: true,
         errors: [error.toString()]
       });
@@ -154,7 +158,7 @@ async function processBatch(batch, start, end) {
       allErrors.push({
         bodyshopid: bodyshop.id,
         imexshopid: bodyshop.imexshopid,
-        autuhouseid: bodyshop.autuhouseid,
+        autohouseid: bodyshop.autohouseid,
         errors: erroredJobs.map((ej) => ({
           ro_number: ej.job?.ro_number,
           jobid: ej.job?.id,
@@ -609,10 +613,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("autohouse-job-calculate-error", "ERROR", "api", null, {
+    logger.log("autohouse-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
-      error
-    });
-
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };

server/data/chatter.js

@@ -39,7 +39,11 @@ exports.default = async (req, res) => {
   }
 
   // Send immediate response and continue processing.
-  res.status(200).send();
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
 
   try {
     logger.log("chatter-start", "DEBUG", "api", null, null);

server/data/claimscorp.js

@@ -26,174 +26,185 @@ const ftpSetup = {
   password: process.env.CLAIMSCORP_PASSWORD,
   debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
   algorithms: {
-    serverHostKey: ["ssh-rsa", "ssh-dss"]
+    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
+const allxmlsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
     res.sendStatus(403);
     return;
   }
-
+  // Only process if the appropriate token is provided.
-  //Query for the List of Bodyshop Clients.
-  logger.log("claimscorp-start", "DEBUG", "api", null, null);
-  const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS);
-
-  const specificShopIds = req.body.bodyshopIds; // ['uuid]
-  const { start, end, skipUpload } = req.body; //YYYY-MM-DD
   if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
     res.sendStatus(401);
     return;
   }
-  const allxmlsToUpload = [];
+
-  const allErrors = [];
+  // Send immediate response and continue processing.
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
+
   try {
-    for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
+    logger.log("claimscorp-start", "DEBUG", "api", null, null);
+    const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
+    const specificShopIds = req.body.bodyshopIds; // ['uuid];
+    const { start, end, skipUpload } = req.body; //YYYY-MM-DD
+
+    const batchSize = 10;
+
+    const shopsToProcess =
+      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
+    logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
+
+    if (shopsToProcess.length === 0) {
+      logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
+      return;
+    }
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
+
+        if (skipUpload) {
+          for (const xmlObj of allxmlsToUpload) {
+            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+          }
+        } else {
+          await uploadViaSFTP(allxmlsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
+    }
+    await Promise.all(batchPromises);
+    await sendServerEmail({
+      subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        null,
+        2
+      )}`
+    });
+
+    logger.log("claimscorp-end", "DEBUG", "api", null, null);
+  } catch (error) {
+    logger.log("claimscorp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+  }
+};
+
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
+    const erroredJobs = [];
+    try {
       logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
         shopname: bodyshop.shopname
       });
-      const erroredJobs = [];
-      try {
-        const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
-          bodyshopid: bodyshop.id,
-          start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-          ...(end && { end: moment(end).endOf("day") })
-        });
 
-        const claimsCorpObject = {
+      const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
-          DataFeed: {
+        bodyshopid: bodyshop.id,
-            ShopInfo: {
+        start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-              ShopID: bodyshops_by_pk.claimscorpid,
+        ...(end && { end: moment(end).endOf("day") })
-              ShopName: bodyshops_by_pk.shopname,
-              RO: jobs.map((j) =>
-                CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-                  erroredJobs.push({ job: job, error: error.toString() });
-                })
-              )
-            }
-          }
-        };
-
-        if (erroredJobs.length > 0) {
-          logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
-            count: erroredJobs.length,
-            jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          });
-        }
-
-        var ret = builder
-          .create(
-            {
-              // version: "1.0",
-              // encoding: "UTF-8",
-              //keepNullNodes: true,
-            },
-            claimsCorpObject
-          )
-          .end({ allowEmptyTags: true });
-
-        allxmlsToUpload.push({
-          count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
-          xml: ret,
-          filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
-        });
-
-        logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
-          shopname: bodyshop.shopname
-        });
-      } catch (error) {
-        //Error at the shop level.
-        logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, {
-          ...error
-        });
-
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          claimscorpid: bodyshop.claimscorpid,
-          fatal: true,
-          errors: [error.toString()]
-        });
-      } finally {
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          claimscorpid: bodyshop.claimscorpid,
-          errors: erroredJobs.map((ej) => ({
-            ro_number: ej.job?.ro_number,
-            jobid: ej.job?.id,
-            error: ej.error
-          }))
-        });
-      }
-    }
-
-    if (skipUpload) {
-      for (const xmlObj of allxmlsToUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      }
-
-      res.json(allxmlsToUpload);
-      sendServerEmail({
-        subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
-        text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
       });
-      return;
-    }
 
-    let sftp = new Client();
+      const claimsCorpObject = {
-    sftp.on("error", (errors) =>
+        DataFeed: {
-      logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
+          ShopInfo: {
-        ...errors
+            ShopID: bodyshops_by_pk.claimscorpid,
-      })
+            ShopName: bodyshops_by_pk.shopname,
-    );
+            RO: jobs.map((j) =>
-    try {
+              CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-      //Connect to the FTP and upload all.
+                erroredJobs.push({ job: job, error: error.toString() });
+              })
+            )
+          }
+        }
+      };
 
-      await sftp.connect(ftpSetup);
+      if (erroredJobs.length > 0) {
-
+        logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
-      for (const xmlObj of allxmlsToUpload) {
+          count: erroredJobs.length,
-        logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
+          jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          filename: xmlObj.filename
-        });
-
-        const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
-        logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, {
-          uploadResult
         });
       }
 
-      //***TODO  Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
+      const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true });
+
+      allxmlsToUpload.push({
+        count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
+        xml: ret,
+        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
+      });
+
+      logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
     } catch (error) {
-      logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
+      //Error at the shop level.
-        ...error
+      logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
+
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        fatal: true,
+        errors: [error.toString()]
       });
     } finally {
-      sftp.end();
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        errors: erroredJobs.map((ej) => ({
+          ro_number: ej.job?.ro_number,
+          jobid: ej.job?.id,
+          error: ej.error
+        }))
+      });
     }
-    sendServerEmail({
-      subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-    });
-    res.sendStatus(200);
-  } catch (error) {
-    res.status(200).json(error);
   }
-};
+}
+
+async function uploadViaSFTP(allxmlsToUpload) {
+  const sftp = new Client();
+  sftp.on("error", (errors) =>
+    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+  );
+  try {
+    //Connect to the FTP and upload all.
+    await sftp.connect(ftpSetup);
+
+    for (const xmlObj of allxmlsToUpload) {
+      try {
+        logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
+        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+        logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, {
+          filename: xmlObj.filename,
+          result: xmlObj.result
+        });
+      } catch (error) {
+        logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, {
+          filename: xmlObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
+    }
+  } catch (error) {
+    logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    throw error;
+  } finally {
+    sftp.end();
+  }
+}
 
 const CreateRepairOrderTag = (job, errorCallback) => {
   //Level 2
@@ -445,10 +456,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, {
+    logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
-      error
-    });
-
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };

server/data/kaizen.js

@@ -16,8 +16,7 @@ const { sendServerEmail } = require("../email/sendemail");
 const DineroFormat = "0,0.00";
 const DateFormat = "MM/DD/YYYY";
 
-const repairOpCodes = ["OP4", "OP9", "OP10"];
+const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
-const replaceOpCodes = ["OP2", "OP5", "OP11", "OP12"];
 
 const ftpSetup = {
   host: process.env.KAIZEN_HOST,
@@ -30,173 +29,180 @@ const ftpSetup = {
   }
 };
 
+const allxmlsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
     res.sendStatus(403);
     return;
   }
-
+  // Only process if the appropriate token is provided.
-  //Query for the List of Bodyshop Clients.
-  logger.log("kaizen-start", "DEBUG", "api", null, null);
-  const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
-
-  const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, {
-    imexshopid: kaizenShopsIDs
-  });
-
-  const specificShopIds = req.body.bodyshopIds; // ['uuid]
-  const { start, end, skipUpload } = req.body; //YYYY-MM-DD
   if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
     res.sendStatus(401);
     return;
   }
-  const allxmlsToUpload = [];
+
-  const allErrors = [];
+  // Send immediate response and continue processing.
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
+
   try {
-    for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
+    logger.log("kaizen-start", "DEBUG", "api", null, null);
+    const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
+    const specificShopIds = req.body.bodyshopIds; // ['uuid];
+    const { start, end, skipUpload } = req.body; //YYYY-MM-DD
+
+    const batchSize = 10;
+
+    const shopsToProcess =
+      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
+    logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
+
+    if (shopsToProcess.length === 0) {
+      logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
+      return;
+    }
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
+
+        if (skipUpload) {
+          for (const xmlObj of allxmlsToUpload) {
+            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+          }
+        } else {
+          await uploadViaSFTP(allxmlsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
+    }
+    await Promise.all(batchPromises);
+    await sendServerEmail({
+      subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        null,
+        2
+      )}`
+    });
+
+    logger.log("kaizen-end", "DEBUG", "api", null, null);
+  } catch (error) {
+    logger.log("kaizen-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+  }
+};
+
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
+    const erroredJobs = [];
+    try {
       logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
         shopname: bodyshop.shopname
       });
-      const erroredJobs = [];
-      try {
-        const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
-          bodyshopid: bodyshop.id,
-          start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-          ...(end && { end: moment(end).endOf("day") })
-        });
 
-        const kaizenObject = {
+      const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
-          DataFeed: {
+        bodyshopid: bodyshop.id,
-            ShopInfo: {
+        start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-              ShopName: bodyshops_by_pk.shopname,
+        ...(end && { end: moment(end).endOf("day") })
-              Jobs: jobs.map((j) =>
-                CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-                  erroredJobs.push({ job: job, error: error.toString() });
-                })
-              )
-            }
-          }
-        };
-
-        if (erroredJobs.length > 0) {
-          logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
-            count: erroredJobs.length,
-            jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          });
-        }
-
-        var ret = builder
-          .create(
-            {
-              // version: "1.0",
-              // encoding: "UTF-8",
-              //keepNullNodes: true,
-            },
-            kaizenObject
-          )
-          .end({ allowEmptyTags: true });
-
-        allxmlsToUpload.push({
-          count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
-          xml: ret,
-          filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
-        });
-
-        logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
-          shopname: bodyshop.shopname
-        });
-      } catch (error) {
-        //Error at the shop level.
-        logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, {
-          ...error
-        });
-
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname,
-          fatal: true,
-          errors: [error.toString()]
-        });
-      } finally {
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname,
-          errors: erroredJobs.map((ej) => ({
-            ro_number: ej.job?.ro_number,
-            jobid: ej.job?.id,
-            error: ej.error
-          }))
-        });
-      }
-    }
-
-    if (skipUpload) {
-      for (const xmlObj of allxmlsToUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      }
-
-      res.json(allxmlsToUpload);
-      sendServerEmail({
-        subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
-        text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
       });
-      return;
-    }
 
-    let sftp = new Client();
+      const kaizenObject = {
-    sftp.on("error", (errors) =>
+        DataFeed: {
-      logger.log("kaizen-sftp-error", "ERROR", "api", null, {
+          ShopInfo: {
-        ...errors
+            ShopName: bodyshops_by_pk.shopname,
-      })
+            Jobs: jobs.map((j) =>
-    );
+              CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-    try {
+                erroredJobs.push({ job: job, error: error.toString() });
-      //Connect to the FTP and upload all.
+              })
+            )
+          }
+        }
+      };
 
-      await sftp.connect(ftpSetup);
+      if (erroredJobs.length > 0) {
-
+        logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
-      for (const xmlObj of allxmlsToUpload) {
+          count: erroredJobs.length,
-        logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
+          jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          filename: xmlObj.filename
-        });
-
-        const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
-        logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
-          uploadResult
         });
       }
 
-      //***TODO  Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
+      const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
+
+      allxmlsToUpload.push({
+        count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
+        xml: ret,
+        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
+      });
+
+      logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
     } catch (error) {
-      logger.log("kaizen-sftp-error", "ERROR", "api", null, {
+      //Error at the shop level.
-        ...error
+      logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
+
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname,
+        fatal: true,
+        errors: [error.toString()]
       });
     } finally {
-      sftp.end();
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname,
+        errors: erroredJobs.map((ej) => ({
+          ro_number: ej.job?.ro_number,
+          jobid: ej.job?.id,
+          error: ej.error
+        }))
+      });
     }
-    sendServerEmail({
-      subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-    });
-    res.sendStatus(200);
-  } catch (error) {
-    res.status(200).json(error);
   }
-};
+}
+
+async function uploadViaSFTP(allxmlsToUpload) {
+  const sftp = new Client();
+  sftp.on("error", (errors) =>
+    logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+  );
+  try {
+    //Connect to the FTP and upload all.
+    await sftp.connect(ftpSetup);
+
+    for (const xmlObj of allxmlsToUpload) {
+      try {
+        logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
+        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+        logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
+          filename: xmlObj.filename,
+          result: xmlObj.result
+        });
+      } catch (error) {
+        logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
+          filename: xmlObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
+    }
+  } catch (error) {
+    logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    throw error;
+  } finally {
+    sftp.end();
+  }
+}
 
 const CreateRepairOrderTag = (job, errorCallback) => {
   //Level 2
@@ -420,10 +426,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("kaizen-job-calculate-error", "ERROR", "api", null, {
+    logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
-      error
-    });
-
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };