From 9e977d9a58dc4cb7f19ae8220231e5ef076434be Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 2 Oct 2025 18:34:40 -0700 Subject: [PATCH] IO-3386 CARFAX RPS Signed-off-by: Allan Carr --- docker-compose-cluster.yml | 3 + docker-compose.yml | 2 + hasura/metadata/cron_triggers.yaml | 11 ++- server/data/carfax-rps.js | 144 +++++++++++++++++------------ server/data/carfax.js | 34 +++---- server/email/sendemail.js | 6 +- server/graphql-client/queries.js | 42 +++++++++ 7 files changed, 163 insertions(+), 79 deletions(-) diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml index 86defaf50..bbce31dd4 100644 --- a/docker-compose-cluster.yml +++ b/docker-compose-cluster.yml @@ -207,6 +207,9 @@ services: aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 + aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 + aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 + aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 " networks: diff --git a/docker-compose.yml b/docker-compose.yml index ad805e272..261076522 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -120,6 +120,8 @@ services: aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-job-totals --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket parts-estimates --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 + aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 + aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1 " # Node App: The Main IMEX API node-app: diff --git a/hasura/metadata/cron_triggers.yaml b/hasura/metadata/cron_triggers.yaml index a1b4b7f4a..2c8c4c91d 100644 --- a/hasura/metadata/cron_triggers.yaml +++ b/hasura/metadata/cron_triggers.yaml @@ -8,7 +8,16 @@ value_from_env: DATAPUMP_AUTH - name: CARFAX Data Pump webhook: '{{HASURA_API_URL}}/data/carfax' - schedule: 0 7 * * 6 + schedule: 0 7 * * 0 + include_in_metadata: true + payload: {} + headers: + - name: x-imex-auth + value_from_env: DATAPUMP_AUTH + comment: Project Mexico +- name: CARFAX RPS Data Pump + webhook: '{{HASURA_API_URL}}/data/carfaxrps' + schedule: 15 7 * * 0 include_in_metadata: true payload: {} headers: diff --git a/server/data/carfax-rps.js b/server/data/carfax-rps.js index e8c43f7ae..9455ee7fe 100644 --- a/server/data/carfax-rps.js +++ b/server/data/carfax-rps.js @@ -1,24 +1,20 @@ const queries = require("../graphql-client/queries"); -const Dinero = require("dinero.js"); const moment = require("moment-timezone"); const logger = require("../utils/logger"); -const InstanceManager = require("../utils/instanceMgr").default; -const { isString, isEmpty } = require("lodash"); const fs = require("fs"); const client = require("../graphql-client/graphql-client").rpsClient; const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail"); -const { uploadFileToS3 } = require("../utils/s3"); const crypto = require("crypto"); -const { ftpSetup, uploadToS3 } = require("./carfax") +const { ftpSetup, uploadToS3 } = require("./carfax"); let Client = require("ssh2-sftp-client"); const AHDateFormat = "YYYY-MM-DD"; const NON_ASCII_REGEX = /[^\x20-\x7E]/g; -const carfaxExportRps = async (req, res) => { - const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. +const S3_BUCKET_NAME = "rps-carfax-uploads"; +const carfaxExportRps = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { return res.sendStatus(403); @@ -40,7 +36,7 @@ const carfaxExportRps = async (req, res) => { const allXMLResults = []; const allErrors = []; - const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. + const { bodyshops } = await client.request(queries.GET_CARFAX_RPS_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD @@ -56,7 +52,7 @@ const carfaxExportRps = async (req, res) => { await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors); await sendServerEmail({ - subject: `Project Mexico Report ${moment().format("MM-DD-YY")}`, + subject: `Project Mexico RPS Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allXMLResults.map((x) => ({ imexshopid: x.imexshopid, @@ -77,21 +73,25 @@ const carfaxExportRps = async (req, res) => { async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) { for (const bodyshop of shopsToProcess) { - const shopid = bodyshop.imexshopid?.toLowerCase() || bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + const shopid = bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); const erroredJobs = []; try { logger.log("CARFAX-RPS-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); - const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, { + const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_RPS_QUERY, { bodyshopid: bodyshop.id, ...(ignoreDateFilter ? {} : { - start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }) + starttz: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), + ...(end && { endtz: moment(end).endOf("day") }), + start: start + ? moment(start).startOf("day").format(AHDateFormat) + : moment().subtract(7, "days").startOf("day").format(AHDateFormat), + ...(end && { endtz: moment(end).endOf("day").format(AHDateFormat) }) + }) }); const carfaxObject = { @@ -107,7 +107,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat if (erroredJobs.length > 0) { logger.log("CARFAX-RPS-failed-jobs", "ERROR", "api", bodyshop.id, { count: erroredJobs.length, - jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) + jobs: JSON.stringify(erroredJobs.map((j) => j.job.id)) }); } @@ -121,18 +121,14 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat if (skipUpload) { fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json); - uploadToS3(jsonObj); + uploadToS3(jsonObj, S3_BUCKET_NAME); } else { await uploadViaSFTP(jsonObj); await sendMexicoBillingEmail({ - subject: `${shopid.replace(/_/g, "").toUpperCase()}_Mexico${InstanceManager({ - imex: "IO", - rome: "RO" - })}_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`, + subject: `${shopid.replace(/_/g, "").toUpperCase()}_MexicoRPS_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`, text: `Errors:\n${JSON.stringify( erroredJobs.map((ej) => ({ - ro_number: ej.job?.ro_number, jobid: ej.job?.id, error: ej.error })), @@ -180,7 +176,6 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat imexshopid: shopid, CARFAXid: bodyshop.CARFAXid, errors: erroredJobs.map((ej) => ({ - ro_number: ej.job?.ro_number, jobid: ej.job?.id, error: ej.error })) @@ -199,7 +194,7 @@ async function uploadViaSFTP(jsonObj) { ); try { // Upload to S3 first. - uploadToS3(jsonObj); + uploadToS3(jsonObj, S3_BUCKET_NAME); //Connect to the FTP and upload all. await sftp.connect(ftpSetup); @@ -220,7 +215,10 @@ async function uploadViaSFTP(jsonObj) { throw error; } } catch (error) { - logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack }); + logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { + error: error.message, + stack: error.stack + }); throw error; } finally { sftp.end(); @@ -228,42 +226,27 @@ async function uploadViaSFTP(jsonObj) { } const CreateRepairOrderTag = (job, errorCallback) => { - if (!job.job_totals) { - errorCallback({ - jobid: job.id, - job: job, - ro_number: job.ro_number, - error: { toString: () => "No job totals for RO." } - }); - return {}; - } - try { + const subtotalEntry = job.totals.find((total) => total.TTL_TYPECD === ""); + const subtotal = subtotalEntry ? subtotalEntry.T_AMT : 0; + const ret = { - ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"), + ro_number: crypto.createHash("md5").update(job.id, "utf8").digest("hex"), v_vin: job.v_vin || "", v_year: job.v_model_yr ? parseInt(job.v_model_yr.match(/\d/g)) ? parseInt(job.v_model_yr.match(/\d/g).join(""), 10) : "" : "", - v_make: job.v_make_desc || "", - v_model: job.v_model_desc || "", + v_make: job.v_makedesc || "", + v_model: job.v_model || "", - date_estimated: [job.date_estimated, job.created_at].find((date) => date) - ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) - : "", - data_opened: [job.date_open, job.created_at].find((date) => date) - ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) - : "", - date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date) - ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + date_estimated: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "", + data_opened: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "", + date_invoiced: [job.close_date, job.created_at].find((date) => date) + ? moment([job.close_date, job.created_at].find((date) => date)) + .tz("America/Winnipeg") + .format(AHDateFormat) : "", loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "", @@ -271,11 +254,12 @@ const CreateRepairOrderTag = (job, errorCallback) => { loss_desc: job.loss_desc || "", theft_ind: job.theft_ind, tloss_ind: job.tlos_ind, - subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(), + + subtotal: subtotal, areaofdamage: { - impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""), - impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "") + impact1: generateAreaOfDamage(job.impact_1 || ""), + impact2: generateAreaOfDamage(job.impact_2 || "") }, jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()] @@ -283,7 +267,7 @@ const CreateRepairOrderTag = (job, errorCallback) => { return ret; } catch (error) { logger.log("CARFAX-RPS-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); - errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); + errorCallback({ jobid: job.id, error }); } }; @@ -292,7 +276,7 @@ const GenerateDetailLines = (line) => { line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null, oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null, alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null, - op_code_desc: line.op_code_desc ? line.op_code_desc.replace(NON_ASCII_REGEX, "") : null, + op_code_desc: generateOpCodeDescription(line.lbr_op), lbr_ty: generateLaborType(line.mod_lbr_ty), lbr_hrs: line.mod_lb_hrs || 0, part_qty: line.part_qty || 0, @@ -394,6 +378,51 @@ const generatePartType = (type) => { return partTypeMap[type?.toLowerCase()] || null; }; +const generateOpCodeDescription = (type) => { + const opCodeMap = { + OP0: "REMOVE / REPLACE PARTIAL", + OP1: "REFINISH / REPAIR", + OP10: "REPAIR , PARTIAL", + OP100: "REPLACE PRE-PRICED", + OP101: "REMOVE/REPLACE RECYCLED PART", + OP103: "REMOVE / REPLACE PARTIAL", + OP104: "REMOVE / REPLACE PARTIAL LABOUR", + OP105: "!!ADJUST MANUALLY!!", + OP106: "REPAIR , PARTIAL", + OP107: "CHIPGUARD", + OP108: "MULTI TONE", + OP109: "REPLACE PRE-PRICED", + OP11: "REMOVE / REPLACE", + OP110: "REFINISH / REPAIR", + OP111: "REMOVE / REPLACE", + OP112: "REMOVE / REPLACE", + OP113: "REPLACE PRE-PRICED", + OP114: "REPLACE PRE-PRICED", + OP12: "REMOVE / REPLACE PARTIAL", + OP120: "REPAIR , PARTIAL", + OP13: "ADDITIONAL COSTS", + OP14: "ADDITIONAL OPERATIONS", + OP15: "BLEND", + OP16: "SUBLET", + OP17: "POLICY LIMIT ADJUSTMENT", + OP18: "APPEAR ALLOWANCE", + OP2: "REMOVE / INSTALL", + OP24: "CHIPGUARD", + OP25: "TWO TONE", + OP26: "PAINTLESS DENT REPAIR", + OP260: "SUBLET", + OP3: "ADDITIONAL LABOR", + OP4: "ALIGNMENT", + OP5: "OVERHAUL", + OP6: "REFINISH", + OP7: "INSPECT", + OP8: "CHECK / ADJUST", + OP9: "REPAIR" + }; + + return opCodeMap[type?.toUpperCase()] || null; +}; + const errorCode = ({ count, filename, results }) => { if (count === 0) return 1; if (!filename) return 3; @@ -405,8 +434,7 @@ const errorCode = ({ count, filename, results }) => { return 0; }; - module.exports = { default: carfaxExportRps, ftpSetup -} \ No newline at end of file +}; diff --git a/server/data/carfax.js b/server/data/carfax.js index 941ee615b..dbbcf3464 100644 --- a/server/data/carfax.js +++ b/server/data/carfax.js @@ -24,7 +24,7 @@ const ftpSetup = { debug: process.env.NODE_ENV !== "production" ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => { }, + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } @@ -37,12 +37,12 @@ const S3_BUCKET_NAME = InstanceManager({ const region = InstanceManager.InstanceRegion; const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME); -const uploadToS3 = (jsonObj) => { +const uploadToS3 = (jsonObj, bucketName = S3_BUCKET_NAME) => { const webPath = isLocal - ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}` - : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`; + ? `https://${bucketName}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}` + : `https://${bucketName}.s3.${region}.amazonaws.com/${jsonObj.filename}`; - uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json }) + uploadFileToS3({ bucketName: bucketName, key: jsonObj.filename, content: jsonObj.json }) .then(() => { logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, { imexshopid: jsonObj.imexshopid, @@ -132,9 +132,9 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat ...(ignoreDateFilter ? {} : { - start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }) + start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }) }); const carfaxObject = { @@ -295,18 +295,18 @@ const CreateRepairOrderTag = (job, errorCallback) => { date_estimated: [job.date_estimated, job.created_at].find((date) => date) ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", data_opened: [job.date_open, job.created_at].find((date) => date) ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date) ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "", @@ -448,8 +448,8 @@ const errorCode = ({ count, filename, results }) => { return 0; }; - module.exports = { default: carfaxExport, - ftpSetup, uploadToS3 -} \ No newline at end of file + ftpSetup, + uploadToS3 +}; diff --git a/server/email/sendemail.js b/server/email/sendemail.js index c9a72c17f..f953f1bf0 100644 --- a/server/email/sendemail.js +++ b/server/email/sendemail.js @@ -68,7 +68,7 @@ const sendServerEmail = async ({ subject, text }) => { }, // eslint-disable-next-line no-unused-vars (err, info) => { - logger.log("server-email-failure", err ? "error" : "debug", null, null, { + logger.log("server-email-send", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack }); @@ -103,7 +103,7 @@ const sendMexicoBillingEmail = async ({ subject, text }) => { }, // eslint-disable-next-line no-unused-vars (err, info) => { - logger.log("server-email-failure", err ? "error" : "debug", null, null, { + logger.log("server-email-send", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack }); @@ -258,7 +258,7 @@ const sendTaskEmail = async ({ to, subject, type = "text", html, text, attachmen // eslint-disable-next-line no-unused-vars (err, info) => { // (message, type, user, record, meta - logger.log("server-email", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack }); + logger.log("server-email-send", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack }); } ); } catch (error) { diff --git a/server/graphql-client/queries.js b/server/graphql-client/queries.js index 9a3bb1d6e..d6ebfce64 100644 --- a/server/graphql-client/queries.js +++ b/server/graphql-client/queries.js @@ -919,6 +919,41 @@ exports.CARFAX_QUERY = `query CARFAX_EXPORT($start: timestamptz, $bodyshopid: uu } }`; +exports.CARFAX_RPS_QUERY = `query CARFAX_RPS_EXPORT($starttz: timestamptz, $endtz: timestamptz,$start: date, $end: date, $bodyshopid: uuid!) { + bodyshops_by_pk(id: $bodyshopid) { + id + shopname + } + jobs(where: {_and: [{_or: [{close_date: {_gt: $start, _lte: $end}}, {created_at: {_gt: $starttz, _lte: $endtz}, close_date: {_is_null: true}}]}, {_not: {_and: [{close_date: {_is_null: true}}, {created_at: {_is_null: true}}]}}, {bodyshopid: {_eq: $bodyshopid}}, {v_vin: {_is_null: false}}]}) { + close_date + created_at + id + ins_co_nm + impact_1 + impact_2 + joblines { + act_price + alt_partno + line_desc + mod_lb_hrs + mod_lbr_ty + oem_partno + lbr_op + part_type + part_qty + } + loss_date + loss_desc + theft_ind + tlos_ind + totals + v_makedesc + v_model + v_model_yr + v_vin + } +}`; + exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { bodyshops_by_pk(id: $bodyshopid){ id @@ -1865,6 +1900,13 @@ exports.GET_CARFAX_SHOPS = `query GET_CARFAX_SHOPS { } }`; +exports.GET_CARFAX_RPS_SHOPS = `query GET_CARFAX_RPS_SHOPS { + bodyshops(where: {carfax_exclude: {_neq: "true"}}){ + id + shopname + } +}`; + exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS { bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){ id