IO-3386 CARFAX RPS

Signed-off-by: Allan Carr <allan@imexsystems.ca>

docker-compose-cluster.yml

@@ -207,6 +207,9 @@ services:
         aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
         aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
         aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
+        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
+        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
+        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
       "
 
 networks:

docker-compose.yml

@@ -120,6 +120,8 @@ services:
         aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-job-totals --create-bucket-configuration LocationConstraint=ca-central-1
         aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket parts-estimates --create-bucket-configuration LocationConstraint=ca-central-1
         aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
+        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
+        aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
       "
   # Node App: The Main IMEX API
   node-app:

hasura/metadata/cron_triggers.yaml

@@ -8,7 +8,16 @@
       value_from_env: DATAPUMP_AUTH
 - name: CARFAX Data Pump
   webhook: '{{HASURA_API_URL}}/data/carfax'
-  schedule: 0 7 * * 6
+  schedule: 0 7 * * 0
+  include_in_metadata: true
+  payload: {}
+  headers:
+    - name: x-imex-auth
+      value_from_env: DATAPUMP_AUTH
+  comment: Project Mexico
+- name: CARFAX RPS Data Pump
+  webhook: '{{HASURA_API_URL}}/data/carfaxrps'
+  schedule: 15 7 * * 0
   include_in_metadata: true
   payload: {}
   headers:

server/data/carfax-rps.js

@@ -1,24 +1,20 @@
 const queries = require("../graphql-client/queries");
-const Dinero = require("dinero.js");
 const moment = require("moment-timezone");
 const logger = require("../utils/logger");
-const InstanceManager = require("../utils/instanceMgr").default;
-const { isString, isEmpty } = require("lodash");
 const fs = require("fs");
 const client = require("../graphql-client/graphql-client").rpsClient;
 const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail");
-const { uploadFileToS3 } = require("../utils/s3");
 const crypto = require("crypto");
-const { ftpSetup, uploadToS3 } = require("./carfax")
+const { ftpSetup, uploadToS3 } = require("./carfax");
 let Client = require("ssh2-sftp-client");
 
 const AHDateFormat = "YYYY-MM-DD";
 
 const NON_ASCII_REGEX = /[^\x20-\x7E]/g;
 
-const carfaxExportRps = async (req, res) => {
+const S3_BUCKET_NAME = "rps-carfax-uploads";
-  const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients.
 
+const carfaxExportRps = async (req, res) => {
   // Only process if in production environment.
   if (process.env.NODE_ENV !== "production") {
     return res.sendStatus(403);
@@ -40,7 +36,7 @@ const carfaxExportRps = async (req, res) => {
     const allXMLResults = [];
     const allErrors = [];
 
-    const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients.
+    const { bodyshops } = await client.request(queries.GET_CARFAX_RPS_SHOPS); //Query for the List of Bodyshop Clients.
     const specificShopIds = req.body.bodyshopIds; // ['uuid];
     const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD
 
@@ -56,7 +52,7 @@ const carfaxExportRps = async (req, res) => {
     await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors);
 
     await sendServerEmail({
-      subject: `Project Mexico Report ${moment().format("MM-DD-YY")}`,
+      subject: `Project Mexico RPS Report ${moment().format("MM-DD-YY")}`,
       text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
         allXMLResults.map((x) => ({
           imexshopid: x.imexshopid,
@@ -77,21 +73,25 @@ const carfaxExportRps = async (req, res) => {
 
 async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) {
   for (const bodyshop of shopsToProcess) {
-    const shopid = bodyshop.imexshopid?.toLowerCase() || bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
+    const shopid = bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
     const erroredJobs = [];
     try {
       logger.log("CARFAX-RPS-start-shop-extract", "DEBUG", "api", bodyshop.id, {
         shopname: bodyshop.shopname
       });
 
-      const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, {
+      const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_RPS_QUERY, {
         bodyshopid: bodyshop.id,
         ...(ignoreDateFilter
           ? {}
           : {
-            start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
+              starttz: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
-            ...(end && { end: moment(end).endOf("day") })
+              ...(end && { endtz: moment(end).endOf("day") }),
-          })
+              start: start
+                ? moment(start).startOf("day").format(AHDateFormat)
+                : moment().subtract(7, "days").startOf("day").format(AHDateFormat),
+              ...(end && { endtz: moment(end).endOf("day").format(AHDateFormat) })
+            })
       });
 
       const carfaxObject = {
@@ -107,7 +107,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
       if (erroredJobs.length > 0) {
         logger.log("CARFAX-RPS-failed-jobs", "ERROR", "api", bodyshop.id, {
           count: erroredJobs.length,
-          jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
+          jobs: JSON.stringify(erroredJobs.map((j) => j.job.id))
         });
       }
 
@@ -121,18 +121,14 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
 
       if (skipUpload) {
         fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json);
-        uploadToS3(jsonObj);
+        uploadToS3(jsonObj, S3_BUCKET_NAME);
       } else {
         await uploadViaSFTP(jsonObj);
 
         await sendMexicoBillingEmail({
-          subject: `${shopid.replace(/_/g, "").toUpperCase()}_Mexico${InstanceManager({
+          subject: `${shopid.replace(/_/g, "").toUpperCase()}_MexicoRPS_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`,
-            imex: "IO",
-            rome: "RO"
-          })}_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`,
           text: `Errors:\n${JSON.stringify(
             erroredJobs.map((ej) => ({
-              ro_number: ej.job?.ro_number,
               jobid: ej.job?.id,
               error: ej.error
             })),
@@ -180,7 +176,6 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
         imexshopid: shopid,
         CARFAXid: bodyshop.CARFAXid,
         errors: erroredJobs.map((ej) => ({
-          ro_number: ej.job?.ro_number,
           jobid: ej.job?.id,
           error: ej.error
         }))
@@ -199,7 +194,7 @@ async function uploadViaSFTP(jsonObj) {
   );
   try {
     // Upload to S3 first.
-    uploadToS3(jsonObj);
+    uploadToS3(jsonObj, S3_BUCKET_NAME);
 
     //Connect to the FTP and upload all.
     await sftp.connect(ftpSetup);
@@ -220,7 +215,10 @@ async function uploadViaSFTP(jsonObj) {
       throw error;
     }
   } catch (error) {
-    logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack });
+    logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, {
+      error: error.message,
+      stack: error.stack
+    });
     throw error;
   } finally {
     sftp.end();
@@ -228,42 +226,27 @@ async function uploadViaSFTP(jsonObj) {
 }
 
 const CreateRepairOrderTag = (job, errorCallback) => {
-  if (!job.job_totals) {
-    errorCallback({
-      jobid: job.id,
-      job: job,
-      ro_number: job.ro_number,
-      error: { toString: () => "No job totals for RO." }
-    });
-    return {};
-  }
-
   try {
+    const subtotalEntry = job.totals.find((total) => total.TTL_TYPECD === "");
+    const subtotal = subtotalEntry ? subtotalEntry.T_AMT : 0;
+
     const ret = {
-      ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"),
+      ro_number: crypto.createHash("md5").update(job.id, "utf8").digest("hex"),
       v_vin: job.v_vin || "",
       v_year: job.v_model_yr
         ? parseInt(job.v_model_yr.match(/\d/g))
           ? parseInt(job.v_model_yr.match(/\d/g).join(""), 10)
           : ""
         : "",
-      v_make: job.v_make_desc || "",
+      v_make: job.v_makedesc || "",
-      v_model: job.v_model_desc || "",
+      v_model: job.v_model || "",
 
-      date_estimated: [job.date_estimated, job.created_at].find((date) => date)
+      date_estimated: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "",
-        ? moment([job.date_open, job.created_at].find((date) => date))
+      data_opened: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "",
-          .tz(job.bodyshop.timezone)
+      date_invoiced: [job.close_date, job.created_at].find((date) => date)
-          .format(AHDateFormat)
+        ? moment([job.close_date, job.created_at].find((date) => date))
-        : "",
+            .tz("America/Winnipeg")
-      data_opened: [job.date_open, job.created_at].find((date) => date)
+            .format(AHDateFormat)
-        ? moment([job.date_open, job.created_at].find((date) => date))
-          .tz(job.bodyshop.timezone)
-          .format(AHDateFormat)
-        : "",
-      date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)
-        ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date))
-          .tz(job.bodyshop.timezone)
-          .format(AHDateFormat)
         : "",
       loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "",
 
@@ -271,11 +254,12 @@ const CreateRepairOrderTag = (job, errorCallback) => {
       loss_desc: job.loss_desc || "",
       theft_ind: job.theft_ind,
       tloss_ind: job.tlos_ind,
-      subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(),
+
+      subtotal: subtotal,
 
       areaofdamage: {
-        impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""),
+        impact1: generateAreaOfDamage(job.impact_1 || ""),
-        impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "")
+        impact2: generateAreaOfDamage(job.impact_2 || "")
       },
 
       jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()]
@@ -283,7 +267,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
     return ret;
   } catch (error) {
     logger.log("CARFAX-RPS-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
-    errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
+    errorCallback({ jobid: job.id, error });
   }
 };
 
@@ -292,7 +276,7 @@ const GenerateDetailLines = (line) => {
     line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null,
     oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null,
     alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null,
-    op_code_desc: line.op_code_desc ? line.op_code_desc.replace(NON_ASCII_REGEX, "") : null,
+    op_code_desc: generateOpCodeDescription(line.lbr_op),
     lbr_ty: generateLaborType(line.mod_lbr_ty),
     lbr_hrs: line.mod_lb_hrs || 0,
     part_qty: line.part_qty || 0,
@@ -394,6 +378,51 @@ const generatePartType = (type) => {
   return partTypeMap[type?.toLowerCase()] || null;
 };
 
+const generateOpCodeDescription = (type) => {
+  const opCodeMap = {
+    OP0: "REMOVE / REPLACE PARTIAL",
+    OP1: "REFINISH / REPAIR",
+    OP10: "REPAIR , PARTIAL",
+    OP100: "REPLACE PRE-PRICED",
+    OP101: "REMOVE/REPLACE RECYCLED PART",
+    OP103: "REMOVE / REPLACE PARTIAL",
+    OP104: "REMOVE / REPLACE PARTIAL LABOUR",
+    OP105: "!!ADJUST MANUALLY!!",
+    OP106: "REPAIR , PARTIAL",
+    OP107: "CHIPGUARD",
+    OP108: "MULTI TONE",
+    OP109: "REPLACE PRE-PRICED",
+    OP11: "REMOVE / REPLACE",
+    OP110: "REFINISH / REPAIR",
+    OP111: "REMOVE / REPLACE",
+    OP112: "REMOVE / REPLACE",
+    OP113: "REPLACE PRE-PRICED",
+    OP114: "REPLACE PRE-PRICED",
+    OP12: "REMOVE / REPLACE PARTIAL",
+    OP120: "REPAIR , PARTIAL",
+    OP13: "ADDITIONAL COSTS",
+    OP14: "ADDITIONAL OPERATIONS",
+    OP15: "BLEND",
+    OP16: "SUBLET",
+    OP17: "POLICY LIMIT ADJUSTMENT",
+    OP18: "APPEAR ALLOWANCE",
+    OP2: "REMOVE / INSTALL",
+    OP24: "CHIPGUARD",
+    OP25: "TWO TONE",
+    OP26: "PAINTLESS DENT REPAIR",
+    OP260: "SUBLET",
+    OP3: "ADDITIONAL LABOR",
+    OP4: "ALIGNMENT",
+    OP5: "OVERHAUL",
+    OP6: "REFINISH",
+    OP7: "INSPECT",
+    OP8: "CHECK / ADJUST",
+    OP9: "REPAIR"
+  };
+
+  return opCodeMap[type?.toUpperCase()] || null;
+};
+
 const errorCode = ({ count, filename, results }) => {
   if (count === 0) return 1;
   if (!filename) return 3;
@@ -405,8 +434,7 @@ const errorCode = ({ count, filename, results }) => {
   return 0;
 };
 
-
 module.exports = {
   default: carfaxExportRps,
   ftpSetup
-}
+};

server/data/carfax.js

@@ -24,7 +24,7 @@ const ftpSetup = {
   debug:
     process.env.NODE_ENV !== "production"
       ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
-      : () => { },
+      : () => {},
   algorithms: {
     serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
   }
@@ -37,12 +37,12 @@ const S3_BUCKET_NAME = InstanceManager({
 const region = InstanceManager.InstanceRegion;
 const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME);
 
-const uploadToS3 = (jsonObj) => {
+const uploadToS3 = (jsonObj, bucketName = S3_BUCKET_NAME) => {
   const webPath = isLocal
-    ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}`
+    ? `https://${bucketName}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}`
-    : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`;
+    : `https://${bucketName}.s3.${region}.amazonaws.com/${jsonObj.filename}`;
 
-  uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json })
+  uploadFileToS3({ bucketName: bucketName, key: jsonObj.filename, content: jsonObj.json })
     .then(() => {
       logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, {
         imexshopid: jsonObj.imexshopid,
@@ -132,9 +132,9 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
         ...(ignoreDateFilter
           ? {}
           : {
-            start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
+              start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
-            ...(end && { end: moment(end).endOf("day") })
+              ...(end && { end: moment(end).endOf("day") })
-          })
+            })
       });
 
       const carfaxObject = {
@@ -295,18 +295,18 @@ const CreateRepairOrderTag = (job, errorCallback) => {
 
       date_estimated: [job.date_estimated, job.created_at].find((date) => date)
         ? moment([job.date_open, job.created_at].find((date) => date))
-          .tz(job.bodyshop.timezone)
+            .tz(job.bodyshop.timezone)
-          .format(AHDateFormat)
+            .format(AHDateFormat)
         : "",
       data_opened: [job.date_open, job.created_at].find((date) => date)
         ? moment([job.date_open, job.created_at].find((date) => date))
-          .tz(job.bodyshop.timezone)
+            .tz(job.bodyshop.timezone)
-          .format(AHDateFormat)
+            .format(AHDateFormat)
         : "",
       date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)
         ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date))
-          .tz(job.bodyshop.timezone)
+            .tz(job.bodyshop.timezone)
-          .format(AHDateFormat)
+            .format(AHDateFormat)
         : "",
       loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "",
 
@@ -448,8 +448,8 @@ const errorCode = ({ count, filename, results }) => {
   return 0;
 };
 
-
 module.exports = {
   default: carfaxExport,
-  ftpSetup, uploadToS3
+  ftpSetup,
-}
+  uploadToS3
+};

server/email/sendemail.js

@@ -68,7 +68,7 @@ const sendServerEmail = async ({ subject, text }) => {
       },
       // eslint-disable-next-line no-unused-vars
       (err, info) => {
-        logger.log("server-email-failure", err ? "error" : "debug", null, null, {
+        logger.log("server-email-send", err ? "error" : "debug", null, null, {
           message: err?.message,
           stack: err?.stack
         });
@@ -103,7 +103,7 @@ const sendMexicoBillingEmail = async ({ subject, text }) => {
       },
       // eslint-disable-next-line no-unused-vars
       (err, info) => {
-        logger.log("server-email-failure", err ? "error" : "debug", null, null, {
+        logger.log("server-email-send", err ? "error" : "debug", null, null, {
           message: err?.message,
           stack: err?.stack
         });
@@ -258,7 +258,7 @@ const sendTaskEmail = async ({ to, subject, type = "text", html, text, attachmen
       // eslint-disable-next-line no-unused-vars
       (err, info) => {
         // (message, type, user, record, meta
-        logger.log("server-email", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack });
+        logger.log("server-email-send", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack });
       }
     );
   } catch (error) {

server/graphql-client/queries.js

@@ -919,6 +919,41 @@ exports.CARFAX_QUERY = `query CARFAX_EXPORT($start: timestamptz, $bodyshopid: uu
   }
 }`;
 
+exports.CARFAX_RPS_QUERY = `query CARFAX_RPS_EXPORT($starttz: timestamptz, $endtz: timestamptz,$start: date, $end: date, $bodyshopid: uuid!) {
+  bodyshops_by_pk(id: $bodyshopid) {
+    id
+    shopname
+  }
+  jobs(where: {_and: [{_or: [{close_date: {_gt: $start, _lte: $end}}, {created_at: {_gt: $starttz, _lte: $endtz}, close_date: {_is_null: true}}]}, {_not: {_and: [{close_date: {_is_null: true}}, {created_at: {_is_null: true}}]}}, {bodyshopid: {_eq: $bodyshopid}}, {v_vin: {_is_null: false}}]}) {
+    close_date
+    created_at
+    id
+    ins_co_nm
+    impact_1
+    impact_2
+    joblines {
+      act_price
+      alt_partno
+      line_desc
+      mod_lb_hrs
+      mod_lbr_ty
+      oem_partno
+      lbr_op
+      part_type
+      part_qty
+    }
+    loss_date
+    loss_desc
+    theft_ind
+    tlos_ind
+    totals
+    v_makedesc
+    v_model
+    v_model_yr
+    v_vin
+  }
+}`;
+
 exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) {
   bodyshops_by_pk(id: $bodyshopid){
     id
@@ -1865,6 +1900,13 @@ exports.GET_CARFAX_SHOPS = `query GET_CARFAX_SHOPS {
   }
 }`;
 
+exports.GET_CARFAX_RPS_SHOPS = `query GET_CARFAX_RPS_SHOPS {
+  bodyshops(where: {carfax_exclude: {_neq: "true"}}){
+      id
+      shopname
+  }
+}`;
+
 exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS {
   bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){
       id