admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IO-3386 CARFAX RPS

Browse Source 
Signed-off-by: Allan Carr <allan@imexsystems.ca>
This commit is contained in:
Allan Carr
2025-10-02 18:34:40 -07:00
parent 43f822e6ad
commit 9e977d9a58
7 changed files with 163 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docker-compose-cluster.yml
View File

@@ -207,6 +207,9 @@ services:
aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
"
networks:

2
docker-compose.yml
View File

@@ -120,6 +120,8 @@ services:
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-job-totals --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket parts-estimates --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rome-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket rps-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
"
# Node App: The Main IMEX API
node-app:

11
hasura/metadata/cron_triggers.yaml
View File

@@ -8,7 +8,16 @@
value_from_env: DATAPUMP_AUTH
- name: CARFAX Data Pump
webhook: '{{HASURA_API_URL}}/data/carfax'
schedule: 0 7 * * 6
schedule: 0 7 * * 0
include_in_metadata: true
payload: {}
headers:
- name: x-imex-auth
value_from_env: DATAPUMP_AUTH
comment: Project Mexico
- name: CARFAX RPS Data Pump
webhook: '{{HASURA_API_URL}}/data/carfaxrps'
schedule: 15 7 * * 0
include_in_metadata: true
payload: {}
headers:

140
server/data/carfax-rps.js
View File

@@ -1,24 +1,20 @@
const queries = require("../graphql-client/queries");
const Dinero = require("dinero.js");
const moment = require("moment-timezone");
const logger = require("../utils/logger");
const InstanceManager = require("../utils/instanceMgr").default;
const { isString, isEmpty } = require("lodash");
const fs = require("fs");
const client = require("../graphql-client/graphql-client").rpsClient;
const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail");
const { uploadFileToS3 } = require("../utils/s3");
const crypto = require("crypto");
const { ftpSetup, uploadToS3 } = require("./carfax")
const { ftpSetup, uploadToS3 } = require("./carfax");
let Client = require("ssh2-sftp-client");
const AHDateFormat = "YYYY-MM-DD";
const NON_ASCII_REGEX = /[^\x20-\x7E]/g;
const carfaxExportRps = async (req, res) => {
const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients.
const S3_BUCKET_NAME = "rps-carfax-uploads";
const carfaxExportRps = async (req, res) => {
// Only process if in production environment.
if (process.env.NODE_ENV !== "production") {
return res.sendStatus(403);
@@ -40,7 +36,7 @@ const carfaxExportRps = async (req, res) => {
const allXMLResults = [];
const allErrors = [];
const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients.
const { bodyshops } = await client.request(queries.GET_CARFAX_RPS_SHOPS); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD
@@ -56,7 +52,7 @@ const carfaxExportRps = async (req, res) => {
await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors);
await sendServerEmail({
subject: `Project Mexico Report ${moment().format("MM-DD-YY")}`,
subject: `Project Mexico RPS Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allXMLResults.map((x) => ({
imexshopid: x.imexshopid,
@@ -77,20 +73,24 @@ const carfaxExportRps = async (req, res) => {
async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) {
for (const bodyshop of shopsToProcess) {
const shopid = bodyshop.imexshopid?.toLowerCase() || bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
const shopid = bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
const erroredJobs = [];
try {
logger.log("CARFAX-RPS-start-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, {
const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_RPS_QUERY, {
bodyshopid: bodyshop.id,
...(ignoreDateFilter
? {}
: {
start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
starttz: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
...(end && { endtz: moment(end).endOf("day") }),
start: start
? moment(start).startOf("day").format(AHDateFormat)
: moment().subtract(7, "days").startOf("day").format(AHDateFormat),
...(end && { endtz: moment(end).endOf("day").format(AHDateFormat) })
})
});
@@ -107,7 +107,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
if (erroredJobs.length > 0) {
logger.log("CARFAX-RPS-failed-jobs", "ERROR", "api", bodyshop.id, {
count: erroredJobs.length,
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
jobs: JSON.stringify(erroredJobs.map((j) => j.job.id))
});
}
@@ -121,18 +121,14 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
if (skipUpload) {
fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json);
uploadToS3(jsonObj);
uploadToS3(jsonObj, S3_BUCKET_NAME);
} else {
await uploadViaSFTP(jsonObj);
await sendMexicoBillingEmail({
subject: `${shopid.replace(/_/g, "").toUpperCase()}_Mexico${InstanceManager({
imex: "IO",
rome: "RO"
})}_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`,
subject: `${shopid.replace(/_/g, "").toUpperCase()}_MexicoRPS_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`,
text: `Errors:\n${JSON.stringify(
erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
})),
@@ -180,7 +176,6 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
imexshopid: shopid,
CARFAXid: bodyshop.CARFAXid,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
}))
@@ -199,7 +194,7 @@ async function uploadViaSFTP(jsonObj) {
);
try {
// Upload to S3 first.
uploadToS3(jsonObj);
uploadToS3(jsonObj, S3_BUCKET_NAME);
//Connect to the FTP and upload all.
await sftp.connect(ftpSetup);
@@ -220,7 +215,10 @@ async function uploadViaSFTP(jsonObj) {
throw error;
}
} catch (error) {
logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack });
logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, {
error: error.message,
stack: error.stack
});
throw error;
} finally {
sftp.end();
@@ -228,41 +226,26 @@ async function uploadViaSFTP(jsonObj) {
}
const CreateRepairOrderTag = (job, errorCallback) => {
if (!job.job_totals) {
errorCallback({
jobid: job.id,
job: job,
ro_number: job.ro_number,
error: { toString: () => "No job totals for RO." }
});
return {};
}
try {
const subtotalEntry = job.totals.find((total) => total.TTL_TYPECD === "");
const subtotal = subtotalEntry ? subtotalEntry.T_AMT : 0;
const ret = {
ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"),
ro_number: crypto.createHash("md5").update(job.id, "utf8").digest("hex"),
v_vin: job.v_vin || "",
v_year: job.v_model_yr
? parseInt(job.v_model_yr.match(/\d/g))
? parseInt(job.v_model_yr.match(/\d/g).join(""), 10)
: ""
: "",
v_make: job.v_make_desc || "",
v_model: job.v_model_desc || "",
v_make: job.v_makedesc || "",
v_model: job.v_model || "",
date_estimated: [job.date_estimated, job.created_at].find((date) => date)
? moment([job.date_open, job.created_at].find((date) => date))
.tz(job.bodyshop.timezone)
.format(AHDateFormat)
: "",
data_opened: [job.date_open, job.created_at].find((date) => date)
? moment([job.date_open, job.created_at].find((date) => date))
.tz(job.bodyshop.timezone)
.format(AHDateFormat)
: "",
date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)
? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date))
.tz(job.bodyshop.timezone)
date_estimated: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "",
data_opened: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "",
date_invoiced: [job.close_date, job.created_at].find((date) => date)
? moment([job.close_date, job.created_at].find((date) => date))
.tz("America/Winnipeg")
.format(AHDateFormat)
: "",
loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "",
@@ -271,11 +254,12 @@ const CreateRepairOrderTag = (job, errorCallback) => {
loss_desc: job.loss_desc || "",
theft_ind: job.theft_ind,
tloss_ind: job.tlos_ind,
subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(),
subtotal: subtotal,
areaofdamage: {
impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""),
impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "")
impact1: generateAreaOfDamage(job.impact_1 || ""),
impact2: generateAreaOfDamage(job.impact_2 || "")
},
jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()]
@@ -283,7 +267,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
return ret;
} catch (error) {
logger.log("CARFAX-RPS-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
errorCallback({ jobid: job.id, error });
}
};
@@ -292,7 +276,7 @@ const GenerateDetailLines = (line) => {
line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null,
oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null,
alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null,
op_code_desc: line.op_code_desc ? line.op_code_desc.replace(NON_ASCII_REGEX, "") : null,
op_code_desc: generateOpCodeDescription(line.lbr_op),
lbr_ty: generateLaborType(line.mod_lbr_ty),
lbr_hrs: line.mod_lb_hrs || 0,
part_qty: line.part_qty || 0,
@@ -394,6 +378,51 @@ const generatePartType = (type) => {
return partTypeMap[type?.toLowerCase()] || null;
};
const generateOpCodeDescription = (type) => {
const opCodeMap = {
OP0: "REMOVE / REPLACE PARTIAL",
OP1: "REFINISH / REPAIR",
OP10: "REPAIR , PARTIAL",
OP100: "REPLACE PRE-PRICED",
OP101: "REMOVE/REPLACE RECYCLED PART",
OP103: "REMOVE / REPLACE PARTIAL",
OP104: "REMOVE / REPLACE PARTIAL LABOUR",
OP105: "!!ADJUST MANUALLY!!",
OP106: "REPAIR , PARTIAL",
OP107: "CHIPGUARD",
OP108: "MULTI TONE",
OP109: "REPLACE PRE-PRICED",
OP11: "REMOVE / REPLACE",
OP110: "REFINISH / REPAIR",
OP111: "REMOVE / REPLACE",
OP112: "REMOVE / REPLACE",
OP113: "REPLACE PRE-PRICED",
OP114: "REPLACE PRE-PRICED",
OP12: "REMOVE / REPLACE PARTIAL",
OP120: "REPAIR , PARTIAL",
OP13: "ADDITIONAL COSTS",
OP14: "ADDITIONAL OPERATIONS",
OP15: "BLEND",
OP16: "SUBLET",
OP17: "POLICY LIMIT ADJUSTMENT",
OP18: "APPEAR ALLOWANCE",
OP2: "REMOVE / INSTALL",
OP24: "CHIPGUARD",
OP25: "TWO TONE",
OP26: "PAINTLESS DENT REPAIR",
OP260: "SUBLET",
OP3: "ADDITIONAL LABOR",
OP4: "ALIGNMENT",
OP5: "OVERHAUL",
OP6: "REFINISH",
OP7: "INSPECT",
OP8: "CHECK / ADJUST",
OP9: "REPAIR"
};
return opCodeMap[type?.toUpperCase()] || null;
};
const errorCode = ({ count, filename, results }) => {
if (count === 0) return 1;
if (!filename) return 3;
@@ -405,8 +434,7 @@ const errorCode = ({ count, filename, results }) => {
return 0;
};
module.exports = {
default: carfaxExportRps,
ftpSetup
}
};

14
server/data/carfax.js
View File

@@ -37,12 +37,12 @@ const S3_BUCKET_NAME = InstanceManager({
const region = InstanceManager.InstanceRegion;
const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME);
const uploadToS3 = (jsonObj) => {
const uploadToS3 = (jsonObj, bucketName = S3_BUCKET_NAME) => {
const webPath = isLocal
? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}`
: `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`;
? `https://${bucketName}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}`
: `https://${bucketName}.s3.${region}.amazonaws.com/${jsonObj.filename}`;
uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json })
uploadFileToS3({ bucketName: bucketName, key: jsonObj.filename, content: jsonObj.json })
.then(() => {
logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, {
imexshopid: jsonObj.imexshopid,
@@ -448,8 +448,8 @@ const errorCode = ({ count, filename, results }) => {
return 0;
};
module.exports = {
default: carfaxExport,
ftpSetup, uploadToS3
}
ftpSetup,
uploadToS3
};

6
server/email/sendemail.js
View File

@@ -68,7 +68,7 @@ const sendServerEmail = async ({ subject, text }) => {
},
// eslint-disable-next-line no-unused-vars
(err, info) => {
logger.log("server-email-failure", err ? "error" : "debug", null, null, {
logger.log("server-email-send", err ? "error" : "debug", null, null, {
message: err?.message,
stack: err?.stack
});
@@ -103,7 +103,7 @@ const sendMexicoBillingEmail = async ({ subject, text }) => {
},
// eslint-disable-next-line no-unused-vars
(err, info) => {
logger.log("server-email-failure", err ? "error" : "debug", null, null, {
logger.log("server-email-send", err ? "error" : "debug", null, null, {
message: err?.message,
stack: err?.stack
});
@@ -258,7 +258,7 @@ const sendTaskEmail = async ({ to, subject, type = "text", html, text, attachmen
// eslint-disable-next-line no-unused-vars
(err, info) => {
// (message, type, user, record, meta
logger.log("server-email", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack });
logger.log("server-email-send", err ? "error" : "debug", null, null, { message: err?.message, stack: err?.stack });
}
);
} catch (error) {

42
server/graphql-client/queries.js
View File

@@ -919,6 +919,41 @@ exports.CARFAX_QUERY = `query CARFAX_EXPORT($start: timestamptz, $bodyshopid: uu
}
}`;
exports.CARFAX_RPS_QUERY = `query CARFAX_RPS_EXPORT($starttz: timestamptz, $endtz: timestamptz,$start: date, $end: date, $bodyshopid: uuid!) {
bodyshops_by_pk(id: $bodyshopid) {
id
shopname
}
jobs(where: {_and: [{_or: [{close_date: {_gt: $start, _lte: $end}}, {created_at: {_gt: $starttz, _lte: $endtz}, close_date: {_is_null: true}}]}, {_not: {_and: [{close_date: {_is_null: true}}, {created_at: {_is_null: true}}]}}, {bodyshopid: {_eq: $bodyshopid}}, {v_vin: {_is_null: false}}]}) {
close_date
created_at
id
ins_co_nm
impact_1
impact_2
joblines {
act_price
alt_partno
line_desc
mod_lb_hrs
mod_lbr_ty
oem_partno
lbr_op
part_type
part_qty
}
loss_date
loss_desc
theft_ind
tlos_ind
totals
v_makedesc
v_model
v_model_yr
v_vin
}
}`;
exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) {
bodyshops_by_pk(id: $bodyshopid){
id
@@ -1865,6 +1900,13 @@ exports.GET_CARFAX_SHOPS = `query GET_CARFAX_SHOPS {
}
}`;
exports.GET_CARFAX_RPS_SHOPS = `query GET_CARFAX_RPS_SHOPS {
bodyshops(where: {carfax_exclude: {_neq: "true"}}){
id
shopname
}
}`;
exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS {
bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){
id