IO-3027 Add in bodyshop.id to logging in SFTP

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>

.platform/hooks/predeploy/00-install-fonts.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Install required packages
+dnf install -y fontconfig freetype
+
+# Move to the /tmp directory for temporary download and extraction
+cd /tmp
+
+# Download the Montserrat font zip file
+wget https://images.imex.online/fonts/montserrat.zip -O montserrat.zip
+
+# Unzip the downloaded font file
+unzip montserrat.zip -d montserrat
+
+# Move the font files to the system fonts directory
+mv montserrat/montserrat/*.ttf /usr/share/fonts
+
+# Rebuild the font cache
+fc-cache -fv
+
+# Clean up
+rm -rf /tmp/montserrat /tmp/montserrat.zip
+
+echo "Montserrat fonts installed and cached successfully."

.vscode/settings.json

@@ -8,5 +8,35 @@
       "pattern": "**/IMEX.xml",
       "systemId": "logs/IMEX.xsd"
     }
+  ],
+  "cSpell.words": [
+    "antd",
+    "appointmentconfirmation",
+    "appt",
+    "autohouse",
+    "autohouseid",
+    "billlines",
+    "bodyshop",
+    "bodyshopid",
+    "bodyshops",
+    "CIECA",
+    "claimscorp",
+    "claimscorpid",
+    "Dinero",
+    "driveable",
+    "IMEX",
+    "imexshopid",
+    "jobid",
+    "joblines",
+    "Kaizen",
+    "labhrs",
+    "larhrs",
+    "mixdata",
+    "ownr",
+    "promanager",
+    "shopname",
+    "smartscheduling",
+    "timetickets",
+    "touchtime"
   ]
 }

client/src/components/dashboard-components/job-lifecycle/job-lifecycle-dashboard.component.jsx

@@ -1,10 +1,10 @@
 import { Card, Table, Tag } from "antd";
-import LoadingSkeleton from "../../loading-skeleton/loading-skeleton.component";
-import { useTranslation } from "react-i18next";
-import React, { useEffect, useState } from "react";
-import dayjs from "../../../utils/day";
-import DashboardRefreshRequired from "../refresh-required.component";
 import axios from "axios";
+import React, { useEffect, useState } from "react";
+import { useTranslation } from "react-i18next";
+import dayjs from "../../../utils/day";
+import LoadingSkeleton from "../../loading-skeleton/loading-skeleton.component";
+import DashboardRefreshRequired from "../refresh-required.component";
 
 const fortyFiveDaysAgo = () => dayjs().subtract(45, "day").toLocaleString();
 
@@ -46,6 +46,11 @@ export default function JobLifecycleDashboardComponent({ data, bodyshop, ...card
       dataIndex: "humanReadable",
       key: "humanReadable"
     },
+    {
+      title: t("job_lifecycle.columns.average_human_readable"),
+      dataIndex: "averageHumanReadable",
+      key: "averageHumanReadable"
+    },
     {
       title: t("job_lifecycle.columns.status_count"),
       key: "statusCount",

client/src/contexts/SocketIO/useSocket.js

@@ -1,8 +1,9 @@
-import { useEffect, useState, useRef } from "react";
+import { useEffect, useRef, useState } from "react";
 import SocketIO from "socket.io-client";
 import { auth } from "../../firebase/firebase.utils";
 import { store } from "../../redux/store";
-import { setWssStatus } from "../../redux/application/application.actions";
+import { addAlerts, setWssStatus } from "../../redux/application/application.actions";
+
 const useSocket = (bodyshop) => {
   const socketRef = useRef(null);
   const [clientId, setClientId] = useState(null);
@@ -31,6 +32,14 @@ const useSocket = (bodyshop) => {
           socketRef.current = socketInstance;
 
           const handleBodyshopMessage = (message) => {
+            if (!message || !message?.type) return;
+
+            switch (message.type) {
+              case "alert-update":
+                store.dispatch(addAlerts(message.payload));
+                break;
+            }
+
             if (!import.meta.env.DEV) return;
             console.log(`Received message for bodyshop ${bodyshop.id}:`, message);
           };
@@ -39,22 +48,22 @@ const useSocket = (bodyshop) => {
             console.log("Socket connected:", socketInstance.id);
             socketInstance.emit("join-bodyshop-room", bodyshop.id);
             setClientId(socketInstance.id);
-            store.dispatch(setWssStatus("connected"))
+            store.dispatch(setWssStatus("connected"));
           };
 
           const handleReconnect = (attempt) => {
             console.log(`Socket reconnected after ${attempt} attempts`);
-            store.dispatch(setWssStatus("connected"))
+            store.dispatch(setWssStatus("connected"));
           };
 
           const handleConnectionError = (err) => {
             console.error("Socket connection error:", err);
-            store.dispatch(setWssStatus("error"))
+            store.dispatch(setWssStatus("error"));
           };
 
           const handleDisconnect = () => {
             console.log("Socket disconnected");
-            store.dispatch(setWssStatus("disconnected"))
+            store.dispatch(setWssStatus("disconnected"));
           };
 
           socketInstance.on("connect", handleConnect);

client/src/pages/manage/manage.page.component.jsx

@@ -1,4 +1,4 @@
-import { FloatButton, Layout, Spin } from "antd";
+import { FloatButton, Layout, notification, Spin } from "antd";
 // import preval from "preval.macro";
 import React, { lazy, Suspense, useContext, useEffect, useState } from "react";
 import { useTranslation } from "react-i18next";
@@ -21,11 +21,12 @@ import ShopSubStatusComponent from "../../components/shop-sub-status/shop-sub-st
 import { requestForToken } from "../../firebase/firebase.utils";
 import SocketContext from "../../contexts/SocketIO/socketContext.jsx";
 import { selectBodyshop, selectInstanceConflict } from "../../redux/user/user.selectors";
-
 import UpdateAlert from "../../components/update-alert/update-alert.component";
 import InstanceRenderManager from "../../utils/instanceRenderMgr.js";
 import "./manage.page.styles.scss";
 import WssStatusDisplayComponent from "../../components/wss-status-display/wss-status-display.component.jsx";
+import { selectAlerts } from "../../redux/application/application.selectors.js";
+import { addAlerts } from "../../redux/application/application.actions.js";
 
 const JobsPage = lazy(() => import("../jobs/jobs.page"));
 
@@ -104,16 +105,80 @@ const { Content, Footer } = Layout;
 
 const mapStateToProps = createStructuredSelector({
   conflict: selectInstanceConflict,
-  bodyshop: selectBodyshop
+  bodyshop: selectBodyshop,
+  alerts: selectAlerts
 });
 
-const mapDispatchToProps = (dispatch) => ({});
+const ALERT_FILE_URL = InstanceRenderManager({
+  imex: "https://images.imex.online/alerts/alerts-imex.json",
+  rome: "https://images.imex.online/alerts/alerts-rome.json"
+});
 
-export function Manage({ conflict, bodyshop }) {
+const mapDispatchToProps = (dispatch) => ({
+  setAlerts: (alerts) => dispatch(addAlerts(alerts))
+});
+
+export function Manage({ conflict, bodyshop, alerts, setAlerts }) {
   const { t } = useTranslation();
   const [chatVisible] = useState(false);
   const { socket, clientId } = useContext(SocketContext);
 
+  // State to track displayed alerts
+  const [displayedAlertIds, setDisplayedAlertIds] = useState([]);
+
+  // Fetch displayed alerts from localStorage on mount
+  useEffect(() => {
+    const displayedAlerts = JSON.parse(localStorage.getItem("displayedAlerts") || "[]");
+    setDisplayedAlertIds(displayedAlerts);
+  }, []);
+
+  // Fetch alerts from the JSON file and dispatch to Redux store
+  useEffect(() => {
+    const fetchAlerts = async () => {
+      try {
+        const response = await fetch(ALERT_FILE_URL);
+        const fetchedAlerts = await response.json();
+        setAlerts(fetchedAlerts);
+      } catch (error) {
+        console.error("Error fetching alerts:", error);
+      }
+    };
+
+    fetchAlerts();
+  }, []);
+
+  // Use useEffect to watch for new alerts
+  useEffect(() => {
+    if (alerts && Object.keys(alerts).length > 0) {
+      // Convert the alerts object into an array
+      const alertArray = Object.values(alerts);
+
+      // Filter out alerts that have already been dismissed
+      const newAlerts = alertArray.filter((alert) => !displayedAlertIds.includes(alert.id));
+
+      newAlerts.forEach((alert) => {
+        // Display the notification
+        notification.open({
+          key: "notification-alerts-" + alert.id,
+          message: alert.message,
+          description: alert.description,
+          type: alert.type || "info",
+          duration: 0,
+          placement: "bottomRight",
+          closable: true,
+          onClose: () => {
+            // When the notification is closed, update displayed alerts state and localStorage
+            setDisplayedAlertIds((prevIds) => {
+              const updatedIds = [...prevIds, alert.id];
+              localStorage.setItem("displayedAlerts", JSON.stringify(updatedIds));
+              return updatedIds;
+            });
+          }
+        });
+      });
+    }
+  }, [alerts, displayedAlertIds]);
+
   useEffect(() => {
     const widgetId = InstanceRenderManager({
       imex: "IABVNO4scRKY11XBQkNr",

client/src/redux/application/application.actions.js

@@ -67,6 +67,12 @@ export const setUpdateAvailable = (isUpdateAvailable) => ({
   type: ApplicationActionTypes.SET_UPDATE_AVAILABLE,
   payload: isUpdateAvailable
 });
+
+export const addAlerts = (alerts) => ({
+  type: ApplicationActionTypes.ADD_ALERTS,
+  payload: alerts
+});
+
 export const setWssStatus = (status) => ({
   type: ApplicationActionTypes.SET_WSS_STATUS,
   payload: status

client/src/redux/application/application.reducer.js

@@ -15,7 +15,8 @@ const INITIAL_STATE = {
     error: null
   },
   jobReadOnly: false,
-  partnerVersion: null
+  partnerVersion: null,
+  alerts: {}
 };
 
 const applicationReducer = (state = INITIAL_STATE, action) => {
@@ -91,6 +92,18 @@ const applicationReducer = (state = INITIAL_STATE, action) => {
     case ApplicationActionTypes.SET_WSS_STATUS: {
       return { ...state, wssStatus: action.payload };
     }
+
+    case ApplicationActionTypes.ADD_ALERTS: {
+      const newAlertsMap = { ...state.alerts };
+
+      action.payload.alerts.forEach((alert) => {
+        newAlertsMap[alert.id] = alert;
+      });
+      return {
+        ...state,
+        alerts: newAlertsMap
+      };
+    }
     default:
       return state;
   }

client/src/redux/application/application.selectors.js

@@ -23,3 +23,4 @@ export const selectOnline = createSelector([selectApplication], (application) =>
 export const selectProblemJobs = createSelector([selectApplication], (application) => application.problemJobs);
 export const selectUpdateAvailable = createSelector([selectApplication], (application) => application.updateAvailable);
 export const selectWssStatus = createSelector([selectApplication], (application) => application.wssStatus);
+export const selectAlerts = createSelector([selectApplication], (application) => application.alerts);

client/src/redux/application/application.types.js

@@ -13,6 +13,7 @@ const ApplicationActionTypes = {
   INSERT_AUDIT_TRAIL: "INSERT_AUDIT_TRAIL",
   SET_PROBLEM_JOBS: "SET_PROBLEM_JOBS",
   SET_UPDATE_AVAILABLE: "SET_UPDATE_AVAILABLE",
-  SET_WSS_STATUS: "SET_WSS_STATUS"
+  SET_WSS_STATUS: "SET_WSS_STATUS",
+  ADD_ALERTS: "ADD_ALERTS"
 };
 export default ApplicationActionTypes;

client/src/translations/en_us/common.json

@@ -1338,6 +1338,8 @@
     },
     "job_lifecycle": {
       "columns": {
+        "average_human_readable": "Average Human Readable",
+        "average_value": "Average Value",
         "duration": "Duration",
         "end": "End",
         "human_readable": "Human Readable",

client/src/translations/es/common.json

@@ -1338,6 +1338,8 @@
     },
     "job_lifecycle": {
       "columns": {
+        "average_human_readable": "",
+        "average_value": "",
         "duration": "",
         "end": "",
         "human_readable": "",

client/src/translations/fr/common.json

@@ -1338,6 +1338,8 @@
     },
     "job_lifecycle": {
       "columns": {
+        "average_human_readable": "",
+        "average_value": "",
         "duration": "",
         "end": "",
         "human_readable": "",

docker-compose.yml

@@ -167,6 +167,28 @@ services:
 #    volumes:
 #      - redis-insight-data:/db
 
+# ##Optional Container for SFTP/SSH Server for testing
+#   ssh-sftp-server:
+#     image: atmoz/sftp:alpine  # Using an image with SFTP support
+#     container_name: ssh-sftp-server
+#     hostname: ssh-sftp-server
+#     networks:
+#       - redis-cluster-net
+#     ports:
+#       - "2222:22"  # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
+#     volumes:
+#       - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro  # Mount the SSH public key
+#       - ./upload:/home/user/upload  # Mount a local directory for SFTP uploads
+#     environment:
+#       - SFTP_USERS=user:password:1000::upload
+#     command: >
+#       /bin/sh -c "
+#         chmod -R 007 /home/user/upload &&
+#         echo 'Match User user' >> /etc/ssh/sshd_config &&
+#         sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config &&
+#         /usr/sbin/sshd -D
+#       "
+
 networks:
   redis-cluster-net:
     driver: bridge

hasura/migrations/1730516697278_idx_timetickets_date/down.sql

@@ -0,0 +1,3 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- CREATE INDEX idx_timetickets_date ON timetickets (date );

hasura/migrations/1730516697278_idx_timetickets_date/up.sql

@@ -0,0 +1 @@
+CREATE INDEX idx_timetickets_date ON timetickets (date );

hasura/migrations/1730517308367_run_sql_migration/down.sql

@@ -0,0 +1,9 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- CREATE INDEX idx_jobs_ownr_fn ON jobs USING gin (ownr_fn gin_trgm_ops);
+-- CREATE INDEX idx_jobs_ownr_ln ON jobs USING gin (ownr_ln gin_trgm_ops);
+-- CREATE INDEX idx_jobs_ownr_co_nm ON jobs USING gin (ownr_co_nm gin_trgm_ops);
+-- CREATE INDEX idx_jobs_clm_no ON jobs USING gin (clm_no gin_trgm_ops);
+-- CREATE INDEX idx_jobs_v_make_desc ON jobs USING gin (v_make_desc gin_trgm_ops);
+-- CREATE INDEX idx_jobs_v_model_desc ON jobs USING gin (v_model_desc gin_trgm_ops);
+-- CREATE INDEX idx_jobs_plate_no ON jobs USING gin (plate_no gin_trgm_ops);

hasura/migrations/1730517308367_run_sql_migration/up.sql

@@ -0,0 +1,7 @@
+CREATE INDEX idx_jobs_ownr_fn ON jobs USING gin (ownr_fn gin_trgm_ops);
+CREATE INDEX idx_jobs_ownr_ln ON jobs USING gin (ownr_ln gin_trgm_ops);
+CREATE INDEX idx_jobs_ownr_co_nm ON jobs USING gin (ownr_co_nm gin_trgm_ops);
+CREATE INDEX idx_jobs_clm_no ON jobs USING gin (clm_no gin_trgm_ops);
+CREATE INDEX idx_jobs_v_make_desc ON jobs USING gin (v_make_desc gin_trgm_ops);
+CREATE INDEX idx_jobs_v_model_desc ON jobs USING gin (v_model_desc gin_trgm_ops);
+CREATE INDEX idx_jobs_plate_no ON jobs USING gin (plate_no gin_trgm_ops);

hasura/migrations/1730518121867_run_sql_migration/down.sql

@@ -0,0 +1,3 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- CREATE INDEX idx_exportlog_createdat_desc ON exportlog (created_at desc);

hasura/migrations/1730518121867_run_sql_migration/up.sql

@@ -0,0 +1 @@
+CREATE INDEX idx_exportlog_createdat_desc ON exportlog (created_at desc);

hasura/migrations/1730521661838_run_sql_migration/down.sql

@@ -0,0 +1,4 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- CREATE index  idx_messages_unread_agg ON messages (read, isoutbound)
+-- WHERE read = false AND isoutbound = false;

hasura/migrations/1730521661838_run_sql_migration/up.sql

@@ -0,0 +1,2 @@
+CREATE index  idx_messages_unread_agg ON messages (read, isoutbound)
+WHERE read = false AND isoutbound = false;

package-lock.json

@@ -53,7 +53,7 @@
         "soap": "^1.1.5",
         "socket.io": "^4.8.0",
         "socket.io-adapter": "^2.5.5",
-        "ssh2-sftp-client": "^10.0.3",
+        "ssh2-sftp-client": "^11.0.0",
         "twilio": "^4.23.0",
         "uuid": "^10.0.0",
         "winston": "^3.15.0",
@@ -7760,16 +7760,17 @@
       }
     },
     "node_modules/ssh2-sftp-client": {
-      "version": "10.0.3",
-      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz",
-      "integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==",
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz",
+      "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==",
+      "license": "Apache-2.0",
       "dependencies": {
         "concat-stream": "^2.0.0",
         "promise-retry": "^2.0.1",
         "ssh2": "^1.15.0"
       },
       "engines": {
-        "node": ">=16.20.2"
+        "node": ">=18.20.4"
       },
       "funding": {
         "type": "individual",

package.json

@@ -63,7 +63,7 @@
     "soap": "^1.1.5",
     "socket.io": "^4.8.0",
     "socket.io-adapter": "^2.5.5",
-    "ssh2-sftp-client": "^10.0.3",
+    "ssh2-sftp-client": "^11.0.0",
     "twilio": "^4.23.0",
     "uuid": "^10.0.0",
     "winston": "^3.15.0",

server/accounting/qbo/qbo-payables.js

@@ -167,7 +167,7 @@ async function QueryVendorRecord(oauthClient, qbo_realmId, req, bill) {
 
 async function InsertVendorRecord(oauthClient, qbo_realmId, req, bill) {
   const Vendor = {
-    DisplayName: bill.vendor.name
+    DisplayName: StandardizeName(bill.vendor.name)
   };
   try {
     const result = await oauthClient.makeApiCall({

server/accounting/qbo/qbo.js

@@ -10,7 +10,7 @@ function urlBuilder(realmId, object, query = null) {
 }
 
 function StandardizeName(str) {
-  return str.replace(new RegExp(/'/g), "\\'");
+  return str.replace(new RegExp(/'/g), "\\'").trim();
 }
 
 exports.urlBuilder = urlBuilder;

server/alerts/alertcheck.js

@@ -0,0 +1,76 @@
+const axios = require("axios");
+const _ = require("lodash");
+const { default: InstanceMgr } = require("../utils/instanceMgr"); // For deep object comparison
+
+// Constants
+const ALERTS_REDIS_KEY = "alerts_data"; // The key under which we'll store alerts in Redis
+const GLOBAL_SOCKET_ID = "global"; // Use 'global' as a socketId to store global data
+
+const ALERT_FILE_URL = InstanceMgr({
+  imex: "https://images.imex.online/alerts/alerts-imex.json",
+  rome: "https://images.imex.online/alerts/alerts-rome.json"
+});
+
+const alertCheck = async (req, res) => {
+  // Access Redis helper functions
+  const { ioRedis, logger } = req;
+  const { getSessionData, setSessionData } = req.sessionUtils;
+
+  try {
+    // Get the JSON Alert file from the server
+    const response = await axios.get(ALERT_FILE_URL);
+    const currentAlerts = response.data;
+    // Retrieve stored alerts from Redis using a global socketId
+    const storedAlerts = await getSessionData(GLOBAL_SOCKET_ID, ALERTS_REDIS_KEY);
+    if (!storedAlerts) {
+      // Alerts not in Redis, store them
+      await setSessionData(GLOBAL_SOCKET_ID, ALERTS_REDIS_KEY, currentAlerts);
+      logger.logger.debug("Alerts added to Redis for the first time.");
+
+      // Emit to clients
+      if (ioRedis) {
+        ioRedis.emit("bodyshop-message", {
+          type: "alert-update",
+          payload: currentAlerts
+        });
+        logger.logger.debug("Alerts emitted to clients for the first time.");
+      } else {
+        logger.log("Socket.IO instance not found. (1)", "error");
+      }
+
+      return res.status(200).send("Alerts added to Redis and emitted to clients.");
+    } else {
+      // Alerts are in Redis, compare them
+      if (!_.isEqual(currentAlerts, storedAlerts)) {
+        // Alerts are different, update Redis and emit to clients
+        await setSessionData(GLOBAL_SOCKET_ID, ALERTS_REDIS_KEY, currentAlerts);
+        logger.logger.debug("Alerts updated in Redis.");
+
+        // Emit the new alerts to all connected clients
+        if (ioRedis) {
+          ioRedis.emit("bodyshop-message", {
+            type: "alert-update",
+            payload: currentAlerts
+          });
+          logger.logger.debug("Alerts emitted to clients after update.");
+        } else {
+          logger.log("Socket.IO instance not found. (2)", "error");
+        }
+
+        return res.status(200).send("Alerts updated in Redis and emitted to clients.");
+      } else {
+        return res.status(200).send("No changes in alerts.");
+      }
+    }
+  } catch (error) {
+    logger.log("Error in alertCheck:", "error", null, null, {
+      error: {
+        message: error.message,
+        stack: error.stack
+      }
+    });
+    return res.status(500).send("Internal server error.");
+  }
+};
+
+module.exports = { alertCheck };

server/data/autohouse.js

@@ -25,15 +25,15 @@ const ftpSetup = {
   port: process.env.AUTOHOUSE_PORT,
   username: process.env.AUTOHOUSE_USER,
   password: process.env.AUTOHOUSE_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
-const allxmlsToUpload = [];
-const allErrors = [];
-
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
@@ -47,16 +47,21 @@ exports.default = async (req, res) => {
   }
 
   // Send immediate response and continue processing.
-  res.status(200).send();
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
 
   try {
     logger.log("autohouse-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
+
     const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-    const batchSize = 10;
-
     const shopsToProcess =
       specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
     logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -65,27 +70,18 @@ exports.default = async (req, res) => {
       logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
-    const batchPromises = [];
-    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
-      const batch = shopsToProcess.slice(i, i + batchSize);
-      const batchPromise = (async () => {
-        await processBatch(batch, start, end);
 
-        if (skipUpload) {
-          for (const xmlObj of allxmlsToUpload) {
-            fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-          }
-        } else {
-          await uploadViaSFTP(allxmlsToUpload);
-        }
-      })();
-      batchPromises.push(batchPromise);
-    }
-    await Promise.all(batchPromises);
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
     await sendServerEmail({
       subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
         null,
         2
       )}`
@@ -97,8 +93,8 @@ exports.default = async (req, res) => {
   }
 };
 
-async function processBatch(batch, start, end) {
-  for (const bodyshop of batch) {
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
     const erroredJobs = [];
     try {
       logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -128,12 +124,27 @@ async function processBatch(batch, start, end) {
         });
       }
 
-      const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true });
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }),
+        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`,
+        count: autoHouseObject.AutoHouseExport.RepairOrder.length
+      };
 
-      allxmlsToUpload.push({
-        count: autoHouseObject.AutoHouseExport.RepairOrder.length,
-        xml: ret,
-        filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        autohouseid: bodyshop.autohouseid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
       });
 
       logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -146,7 +157,7 @@ async function processBatch(batch, start, end) {
       allErrors.push({
         bodyshopid: bodyshop.id,
         imexshopid: bodyshop.imexshopid,
-        autuhouseid: bodyshop.autuhouseid,
+        autohouseid: bodyshop.autohouseid,
         fatal: true,
         errors: [error.toString()]
       });
@@ -154,7 +165,7 @@ async function processBatch(batch, start, end) {
       allErrors.push({
         bodyshopid: bodyshop.id,
         imexshopid: bodyshop.imexshopid,
-        autuhouseid: bodyshop.autuhouseid,
+        autohouseid: bodyshop.autohouseid,
         errors: erroredJobs.map((ej) => ({
           ro_number: ej.job?.ro_number,
           jobid: ej.job?.id,
@@ -165,34 +176,35 @@ async function processBatch(batch, start, end) {
   }
 }
 
-async function uploadViaSFTP(allxmlsToUpload) {
+async function uploadViaSFTP(xmlObj) {
   const sftp = new Client();
   sftp.on("error", (errors) =>
-    logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+    logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
   );
   try {
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
 
-    for (const xmlObj of allxmlsToUpload) {
-      try {
-        logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
-        xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
-        logger.log("autohouse-sftp-upload-result", "DEBUG", "api", null, {
-          filename: xmlObj.filename,
-          result: xmlObj.result
-        });
-      } catch (error) {
-        logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, {
-          filename: xmlObj.filename,
-          error: error.message,
-          stack: error.stack
-        });
-        throw error;
-      }
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
     }
   } catch (error) {
-    logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
     throw error;
   } finally {
     sftp.end();
@@ -609,10 +621,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("autohouse-job-calculate-error", "ERROR", "api", null, {
-      error
-    });
-
+    logger.log("autohouse-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };

server/data/chatter.js

@@ -17,140 +17,140 @@ const ftpSetup = {
   port: process.env.CHATTER_PORT,
   username: process.env.CHATTER_USER,
   privateKey: null,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug: process.env.NODE_ENV !== "production" 
+    ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+    : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
+
+const allcsvsToUpload = [];
+const allErrors = [];
+
 exports.default = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
     res.sendStatus(403);
     return;
   }
-
+  // Only process if the appropriate token is provided.
   if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
     res.sendStatus(401);
     return;
   }
-  //Query for the List of Bodyshop Clients.
-  logger.log("chatter-start", "DEBUG", "api", null, null);
-  const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS);
-  const specificShopIds = req.body.bodyshopIds; // ['uuid]
-  const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-  const allcsvsToUpload = [];
-  const allErrors = [];
+  // Send immediate response and continue processing.
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
+
   try {
-    for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
-      logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
-        shopname: bodyshop.shopname
-      });
-      try {
-        const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, {
-          bodyshopid: bodyshop.id,
-          start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"),
-          ...(end && { end: moment(end).endOf("day") })
-        });
+    logger.log("chatter-start", "DEBUG", "api", null, null);
+    const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients.
+    const specificShopIds = req.body.bodyshopIds; // ['uuid];
+    const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-        const chatterObject = jobs.map((j) => {
-          return {
-            poc_trigger_code: bodyshops_by_pk.chatterid,
-            firstname: j.ownr_co_nm ? null : j.ownr_fn,
-            lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln,
-            transaction_id: j.ro_number,
-            email: j.ownr_ea,
-            phone_number: j.ownr_ph1
-          };
-        });
+    const batchSize = 10;
 
-        const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
+    const shopsToProcess =
+      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
+    logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null);
 
-        allcsvsToUpload.push({
-          count: chatterObject.length,
-          csv: ret,
-          filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
-        });
-
-        logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
-          shopname: bodyshop.shopname
-        });
-      } catch (error) {
-        //Error at the shop level.
-        logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, {
-          ...error
-        });
-
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname,
-          fatal: true,
-          errors: [error.toString()]
-        });
-      } finally {
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname
-        });
-      }
-    }
-
-    if (skipUpload) {
-      for (const csvObj of allcsvsToUpload) {
-        fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
-      }
-
-      sendServerEmail({
-        subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
-        text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-      });
-      res.json(allcsvsToUpload);
+    if (shopsToProcess.length === 0) {
+      logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
 
-    const sftp = new Client();
-    sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors }));
-    try {
-      //Get the private key from AWS Secrets Manager.
-      ftpSetup.privateKey = await getPrivateKey();
-
-      //Connect to the FTP and upload all.
-      await sftp.connect(ftpSetup);
-
-      for (const csvObj of allcsvsToUpload) {
-        logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename });
-
-        const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`);
-        logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult });
-      }
-    } catch (error) {
-      logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error });
-    } finally {
-      sftp.end();
+    const batchPromises = [];
+    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
+      const batch = shopsToProcess.slice(i, i + batchSize);
+      const batchPromise = (async () => {
+        await processBatch(batch, start, end);
+        if (skipUpload) {
+          for (const csvObj of allcsvsToUpload) {
+            await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
+          }
+        } else {
+          await uploadViaSFTP(allcsvsToUpload);
+        }
+      })();
+      batchPromises.push(batchPromise);
     }
-    sendServerEmail({
+    await Promise.all(batchPromises);
+    await sendServerEmail({
       subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+        allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
         null,
         2
       )}`
     });
-    res.sendStatus(200);
+
+    logger.log("chatter-end", "DEBUG", "api", null, null);
   } catch (error) {
-    res.status(200).json(error);
+    logger.log("chatter-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
   }
 };
 
+async function processBatch(batch, start, end) {
+  for (const bodyshop of batch) {
+    try {
+      logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
+
+      const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, {
+        bodyshopid: bodyshop.id,
+        start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"),
+        ...(end && { end: moment(end).endOf("day") })
+      });
+
+      const chatterObject = jobs.map((j) => {
+        return {
+          poc_trigger_code: bodyshops_by_pk.chatterid,
+          firstname: j.ownr_co_nm ? null : j.ownr_fn,
+          lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln,
+          transaction_id: j.ro_number,
+          email: j.ownr_ea,
+          phone_number: j.ownr_ph1
+        };
+      });
+
+      const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
+
+      allcsvsToUpload.push({
+        count: chatterObject.length,
+        csv: ret,
+        filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
+      });
+
+      logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
+    } catch (error) {
+      //Error at the shop level.
+      logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
+
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname,
+        fatal: true,
+        errors: [error.toString()]
+      });
+    } finally {
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname
+      });
+    }
+  }
+}
+
 async function getPrivateKey() {
   // Connect to AWS Secrets Manager
   const client = new SecretsManagerClient({ region: "ca-central-1" });
@@ -160,10 +160,46 @@ async function getPrivateKey() {
   try {
     const { SecretString, SecretBinary } = await client.send(command);
     if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null);
-    const chatterPrivateKey = SecretString ? JSON.parse(SecretString) : JSON.parse(Buffer.from(SecretBinary, "base64").toString("ascii"));
-    return chatterPrivateKey.private_key;
+    const chatterPrivateKey = SecretString ? SecretString : Buffer.from(SecretBinary, "base64").toString("ascii");
+    return chatterPrivateKey;
   } catch (error) {
-    logger.log("chatter-get-private-key", "ERROR", "api", null, error);
-    throw err;
+    logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    throw error;
+  }
+}
+
+async function uploadViaSFTP(allcsvsToUpload) {
+  const sftp = new Client();
+  sftp.on("error", (errors) =>
+    logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
+  );
+  try {
+    //Get the private key from AWS Secrets Manager.
+    const privateKey = await getPrivateKey();
+
+    //Connect to the FTP and upload all.
+    await sftp.connect({ ...ftpSetup, privateKey });
+
+    for (const csvObj of allcsvsToUpload) {
+      try {
+        csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`);
+        logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
+          filename: csvObj.filename,
+          result: csvObj.result
+        });
+      } catch (error) {
+        logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
+          filename: csvObj.filename,
+          error: error.message,
+          stack: error.stack
+        });
+        throw error;
+      }
+    }
+  } catch (error) {
+    logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+    throw error;
+  } finally {
+    sftp.end();
   }
 }

server/data/claimscorp.js

@@ -24,9 +24,12 @@ const ftpSetup = {
   port: process.env.CLAIMSCORP_PORT,
   username: process.env.CLAIMSCORP_USER,
   password: process.env.CLAIMSCORP_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
-    serverHostKey: ["ssh-rsa", "ssh-dss"]
+    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
 };
 
@@ -36,164 +39,183 @@ exports.default = async (req, res) => {
     res.sendStatus(403);
     return;
   }
-
-  //Query for the List of Bodyshop Clients.
-  logger.log("claimscorp-start", "DEBUG", "api", null, null);
-  const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS);
-
-  const specificShopIds = req.body.bodyshopIds; // ['uuid]
-  const { start, end, skipUpload } = req.body; //YYYY-MM-DD
+  // Only process if the appropriate token is provided.
   if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
     res.sendStatus(401);
     return;
   }
-  const allxmlsToUpload = [];
-  const allErrors = [];
+
+  // Send immediate response and continue processing.
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
+
   try {
-    for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
-      logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
-        shopname: bodyshop.shopname
-      });
-      const erroredJobs = [];
-      try {
-        const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
-          bodyshopid: bodyshop.id,
-          start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-          ...(end && { end: moment(end).endOf("day") })
-        });
+    logger.log("claimscorp-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
 
-        const claimsCorpObject = {
-          DataFeed: {
-            ShopInfo: {
-              ShopID: bodyshops_by_pk.claimscorpid,
-              ShopName: bodyshops_by_pk.shopname,
-              RO: jobs.map((j) =>
-                CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-                  erroredJobs.push({ job: job, error: error.toString() });
-                })
-              )
-            }
-          }
-        };
+    const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
+    const specificShopIds = req.body.bodyshopIds; // ['uuid];
+    const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-        if (erroredJobs.length > 0) {
-          logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
-            count: erroredJobs.length,
-            jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          });
-        }
+    const shopsToProcess =
+      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
+    logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
 
-        var ret = builder
-          .create(
-            {
-              // version: "1.0",
-              // encoding: "UTF-8",
-              //keepNullNodes: true,
-            },
-            claimsCorpObject
-          )
-          .end({ allowEmptyTags: true });
-
-        allxmlsToUpload.push({
-          count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
-          xml: ret,
-          filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
-        });
-
-        logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
-          shopname: bodyshop.shopname
-        });
-      } catch (error) {
-        //Error at the shop level.
-        logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, {
-          ...error
-        });
-
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          claimscorpid: bodyshop.claimscorpid,
-          fatal: true,
-          errors: [error.toString()]
-        });
-      } finally {
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          claimscorpid: bodyshop.claimscorpid,
-          errors: erroredJobs.map((ej) => ({
-            ro_number: ej.job?.ro_number,
-            jobid: ej.job?.id,
-            error: ej.error
-          }))
-        });
-      }
-    }
-
-    if (skipUpload) {
-      for (const xmlObj of allxmlsToUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      }
-
-      res.json(allxmlsToUpload);
-      sendServerEmail({
-        subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
-        text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-      });
+    if (shopsToProcess.length === 0) {
+      logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
 
-    let sftp = new Client();
-    sftp.on("error", (errors) =>
-      logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
-        ...errors
-      })
-    );
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
+    await sendServerEmail({
+      subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
+        null,
+        2
+      )}`
+    });
+
+    logger.log("claimscorp-end", "DEBUG", "api", null, null);
+  } catch (error) {
+    logger.log("claimscorp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+  }
+};
+
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
+    const erroredJobs = [];
     try {
-      //Connect to the FTP and upload all.
+      logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
 
-      await sftp.connect(ftpSetup);
+      const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
+        bodyshopid: bodyshop.id,
+        start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
+        ...(end && { end: moment(end).endOf("day") })
+      });
 
-      for (const xmlObj of allxmlsToUpload) {
-        logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
-          filename: xmlObj.filename
-        });
+      const claimsCorpObject = {
+        DataFeed: {
+          ShopInfo: {
+            ShopID: bodyshops_by_pk.claimscorpid,
+            ShopName: bodyshops_by_pk.shopname,
+            RO: jobs.map((j) =>
+              CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
+                erroredJobs.push({ job: job, error: error.toString() });
+              })
+            )
+          }
+        }
+      };
 
-        const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
-        logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, {
-          uploadResult
+      if (erroredJobs.length > 0) {
+        logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
+          count: erroredJobs.length,
+          jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
         });
       }
 
-      //***TODO  Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }),
+        filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
+        count: claimsCorpObject.DataFeed.ShopInfo.RO.length
+      };
+
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+
+      logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
     } catch (error) {
-      logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
-        ...error
+      //Error at the shop level.
+      logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
+
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        fatal: true,
+        errors: [error.toString()]
       });
     } finally {
-      sftp.end();
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        claimscorpid: bodyshop.claimscorpid,
+        errors: erroredJobs.map((ej) => ({
+          ro_number: ej.job?.ro_number,
+          jobid: ej.job?.id,
+          error: ej.error
+        }))
+      });
     }
-    sendServerEmail({
-      subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-    });
-    res.sendStatus(200);
-  } catch (error) {
-    res.status(200).json(error);
   }
-};
+}
+
+async function uploadViaSFTP(xmlObj) {
+  const sftp = new Client();
+  sftp.on("error", (errors) =>
+    logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
+  );
+  try {
+    //Connect to the FTP and upload all.
+    await sftp.connect(ftpSetup);
+
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
+    }
+  } catch (error) {
+    logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: error.message,
+      stack: error.stack
+    });
+    throw error;
+  } finally {
+    sftp.end();
+  }
+}
 
 const CreateRepairOrderTag = (job, errorCallback) => {
   //Level 2
@@ -445,10 +467,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, {
-      error
-    });
-
+    logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };

server/data/kaizen.js

@@ -16,15 +16,17 @@ const { sendServerEmail } = require("../email/sendemail");
 const DineroFormat = "0,0.00";
 const DateFormat = "MM/DD/YYYY";
 
-const repairOpCodes = ["OP4", "OP9", "OP10"];
-const replaceOpCodes = ["OP2", "OP5", "OP11", "OP12"];
+const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
 
 const ftpSetup = {
   host: process.env.KAIZEN_HOST,
   port: process.env.KAIZEN_PORT,
   username: process.env.KAIZEN_USER,
   password: process.env.KAIZEN_PASSWORD,
-  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
+  debug:
+    process.env.NODE_ENV !== "production"
+      ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
@@ -36,167 +38,178 @@ exports.default = async (req, res) => {
     res.sendStatus(403);
     return;
   }
-
-  //Query for the List of Bodyshop Clients.
-  logger.log("kaizen-start", "DEBUG", "api", null, null);
-  const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
-
-  const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, {
-    imexshopid: kaizenShopsIDs
-  });
-
-  const specificShopIds = req.body.bodyshopIds; // ['uuid]
-  const { start, end, skipUpload } = req.body; //YYYY-MM-DD
+  // Only process if the appropriate token is provided.
   if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
     res.sendStatus(401);
     return;
   }
-  const allxmlsToUpload = [];
-  const allErrors = [];
+
+  // Send immediate response and continue processing.
+  res.status(202).json({
+    success: true,
+    message: "Processing request ...",
+    timestamp: new Date().toISOString()
+  });
+
   try {
-    for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
-      logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
-        shopname: bodyshop.shopname
-      });
-      const erroredJobs = [];
-      try {
-        const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
-          bodyshopid: bodyshop.id,
-          start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
-          ...(end && { end: moment(end).endOf("day") })
-        });
+    logger.log("kaizen-start", "DEBUG", "api", null, null);
+    const allXMLResults = [];
+    const allErrors = [];
 
-        const kaizenObject = {
-          DataFeed: {
-            ShopInfo: {
-              ShopName: bodyshops_by_pk.shopname,
-              Jobs: jobs.map((j) =>
-                CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
-                  erroredJobs.push({ job: job, error: error.toString() });
-                })
-              )
-            }
-          }
-        };
+    const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
+    const specificShopIds = req.body.bodyshopIds; // ['uuid];
+    const { start, end, skipUpload } = req.body; //YYYY-MM-DD
 
-        if (erroredJobs.length > 0) {
-          logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
-            count: erroredJobs.length,
-            jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
-          });
-        }
+    const shopsToProcess =
+      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
+    logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
 
-        var ret = builder
-          .create(
-            {
-              // version: "1.0",
-              // encoding: "UTF-8",
-              //keepNullNodes: true,
-            },
-            kaizenObject
-          )
-          .end({ allowEmptyTags: true });
-
-        allxmlsToUpload.push({
-          count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
-          xml: ret,
-          filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
-        });
-
-        logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
-          shopname: bodyshop.shopname
-        });
-      } catch (error) {
-        //Error at the shop level.
-        logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, {
-          ...error
-        });
-
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname,
-          fatal: true,
-          errors: [error.toString()]
-        });
-      } finally {
-        allErrors.push({
-          bodyshopid: bodyshop.id,
-          imexshopid: bodyshop.imexshopid,
-          shopname: bodyshop.shopname,
-          errors: erroredJobs.map((ej) => ({
-            ro_number: ej.job?.ro_number,
-            jobid: ej.job?.id,
-            error: ej.error
-          }))
-        });
-      }
-    }
-
-    if (skipUpload) {
-      for (const xmlObj of allxmlsToUpload) {
-        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
-      }
-
-      res.json(allxmlsToUpload);
-      sendServerEmail({
-        subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
-        text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-      });
+    if (shopsToProcess.length === 0) {
+      logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
       return;
     }
 
-    let sftp = new Client();
-    sftp.on("error", (errors) =>
-      logger.log("kaizen-sftp-error", "ERROR", "api", null, {
-        ...errors
-      })
-    );
+    await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
+
+    await sendServerEmail({
+      subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
+      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
+        allXMLResults.map((x) => ({
+          imexshopid: x.imexshopid,
+          filename: x.filename,
+          count: x.count,
+          result: x.result
+        })),
+        null,
+        2
+      )}`
+    });
+
+    logger.log("kaizen-end", "DEBUG", "api", null, null);
+  } catch (error) {
+    logger.log("kaizen-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
+  }
+};
+
+async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
+  for (const bodyshop of shopsToProcess) {
+    const erroredJobs = [];
     try {
-      //Connect to the FTP and upload all.
+      logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
 
-      await sftp.connect(ftpSetup);
+      const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
+        bodyshopid: bodyshop.id,
+        start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
+        ...(end && { end: moment(end).endOf("day") })
+      });
 
-      for (const xmlObj of allxmlsToUpload) {
-        logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
-          filename: xmlObj.filename
-        });
+      const kaizenObject = {
+        DataFeed: {
+          ShopInfo: {
+            ShopName: bodyshops_by_pk.shopname,
+            Jobs: jobs.map((j) =>
+              CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
+                erroredJobs.push({ job: job, error: error.toString() });
+              })
+            )
+          }
+        }
+      };
 
-        const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
-        logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
-          uploadResult
+      if (erroredJobs.length > 0) {
+        logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
+          count: erroredJobs.length,
+          jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
         });
       }
 
-      //***TODO  Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
+      const xmlObj = {
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }),
+        filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
+        count: kaizenObject.DataFeed.ShopInfo.Jobs.length
+      };
+
+      if (skipUpload) {
+        fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
+      } else {
+        await uploadViaSFTP(xmlObj);
+      }
+
+      allXMLResults.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        count: xmlObj.count,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+
+      logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
+        shopname: bodyshop.shopname
+      });
     } catch (error) {
-      logger.log("kaizen-sftp-error", "ERROR", "api", null, {
-        ...error
+      //Error at the shop level.
+      logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
+
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname,
+        fatal: true,
+        errors: [error.toString()]
       });
     } finally {
-      sftp.end();
+      allErrors.push({
+        bodyshopid: bodyshop.id,
+        imexshopid: bodyshop.imexshopid,
+        shopname: bodyshop.shopname,
+        errors: erroredJobs.map((ej) => ({
+          ro_number: ej.job?.ro_number,
+          jobid: ej.job?.id,
+          error: ej.error
+        }))
+      });
     }
-    sendServerEmail({
-      subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
-      text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
-      Uploaded: ${JSON.stringify(
-        allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
-        null,
-        2
-      )}
-      `
-    });
-    res.sendStatus(200);
-  } catch (error) {
-    res.status(200).json(error);
   }
-};
+}
+
+async function uploadViaSFTP(xmlObj) {
+  const sftp = new Client();
+  sftp.on("error", (errors) =>
+    logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
+      error: errors.message,
+      stack: errors.stack
+    })
+  );
+  try {
+    //Connect to the FTP and upload all.
+    await sftp.connect(ftpSetup);
+
+    try {
+      xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
+      logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
+        imexshopid: xmlObj.imexshopid,
+        filename: xmlObj.filename,
+        result: xmlObj.result
+      });
+    } catch (error) {
+      logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
+        filename: xmlObj.filename,
+        error: error.message,
+        stack: error.stack
+      });
+      throw error;
+    }
+  } catch (error) {
+    logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
+    throw error;
+  } finally {
+    sftp.end();
+  }
+}
 
 const CreateRepairOrderTag = (job, errorCallback) => {
   //Level 2
@@ -420,10 +433,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     };
     return ret;
   } catch (error) {
-    logger.log("kaizen-job-calculate-error", "ERROR", "api", null, {
-      error
-    });
-
+    logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
     errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
   }
 };

server/job/job-lifecycle.js

@@ -78,16 +78,20 @@ const jobLifecycle = async (req, res) => {
   Object.keys(flatGroupedAllDurations).forEach((status) => {
     const value = flatGroupedAllDurations[status].reduce((acc, curr) => acc + curr.value, 0);
     const humanReadable = durationToHumanReadable(moment.duration(value));
-    const percentage = (value / finalTotal) * 100;
+    const percentage = finalTotal > 0 ? (value / finalTotal) * 100 : 0;
     const color = getLifecycleStatusColor(status);
     const roundedPercentage = `${Math.round(percentage)}%`;
+    const averageValue = _.size(jobIDs) > 0 ? value / jobIDs.length : 0;
+    const averageHumanReadable = durationToHumanReadable(moment.duration(averageValue));
     finalSummations.push({
       status,
       value,
       humanReadable,
       percentage,
       color,
-      roundedPercentage
+      roundedPercentage,
+      averageValue,
+      averageHumanReadable
     });
   });
 
@@ -100,7 +104,12 @@ const jobLifecycle = async (req, res) => {
       totalStatuses: finalSummations.length,
       total: finalTotal,
       statusCounts: finalStatusCounts,
-      humanReadable: durationToHumanReadable(moment.duration(finalTotal))
+      humanReadable: durationToHumanReadable(moment.duration(finalTotal)),
+      averageValue: _.size(jobIDs) > 0 ? finalTotal / jobIDs.length : 0,
+      averageHumanReadable:
+        _.size(jobIDs) > 0
+          ? durationToHumanReadable(moment.duration(finalTotal / jobIDs.length))
+          : durationToHumanReadable(moment.duration(0))
     }
   });
 };

server/routes/miscellaneousRoutes.js

@@ -12,6 +12,7 @@ const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebas
 const withUserGraphQLClientMiddleware = require("../middleware/withUserGraphQLClientMiddleware");
 const { taskAssignedEmail, tasksRemindEmail } = require("../email/tasksEmails");
 const { canvastest } = require("../render/canvas-handler");
+const { alertCheck } = require("../alerts/alertcheck");
 
 //Test route to ensure Express is responding.
 router.get("/test", async function (req, res) {
@@ -53,4 +54,7 @@ router.post("/taskHandler", validateFirebaseIdTokenMiddleware, taskHandler.taskH
 // Canvas Test
 router.post("/canvastest", validateFirebaseIdTokenMiddleware, canvastest);
 
+// Alert Check
+router.post("/alertcheck", eventAuthorizationMiddleware, alertCheck);
+
 module.exports = router;

upload/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore