Merged in feature/IO-3330-CARFAX-Datapump (pull request #2471)
IO-3330 CARFAX Datapump Approved-by: Dave Richer
This commit is contained in:
Allan Carr
409
server/data/carfax.js
Normal file
409
server/data/carfax.js
Normal file
@@ -0,0 +1,409 @@
|
||||
const path = require("path");
|
||||
const queries = require("../graphql-client/queries");
|
||||
const Dinero = require("dinero.js");
|
||||
const moment = require("moment-timezone");
|
||||
const logger = require("../utils/logger");
|
||||
const InstanceManager = require("../utils/instanceMgr").default;
|
||||
const { isString, isEmpty } = require("lodash");
|
||||
const fs = require("fs");
|
||||
const client = require("../graphql-client/graphql-client").client;
|
||||
const { sendServerEmail } = require("../email/sendemail");
|
||||
const { uploadFileToS3 } = require("../utils/s3");
|
||||
const crypto = require("crypto");
|
||||
|
||||
require("dotenv").config({
|
||||
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
||||
});
|
||||
let Client = require("ssh2-sftp-client");
|
||||
|
||||
const AHDateFormat = "YYYY-MM-DD";
|
||||
|
||||
const NON_ASCII_REGEX = /[^\x20-\x7E]/g;
|
||||
|
||||
const ftpSetup = {
|
||||
host: process.env.CARFAX_HOST,
|
||||
port: process.env.CARFAX_PORT,
|
||||
username: process.env.CARFAX_USER,
|
||||
password: process.env.CARFAX_PASSWORD,
|
||||
debug:
|
||||
process.env.NODE_ENV !== "production"
|
||||
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
||||
: () => {},
|
||||
algorithms: {
|
||||
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
||||
}
|
||||
};
|
||||
|
||||
const S3_BUCKET_NAME = InstanceManager({
|
||||
imex: "imex-carfax-uploads",
|
||||
rome: "rome-carfax-uploads"
|
||||
});
|
||||
const region = InstanceManager.InstanceRegion;
|
||||
const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME);
|
||||
|
||||
const uploadToS3 = (jsonObj) => {
|
||||
const webPath = isLocal
|
||||
? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}`
|
||||
: `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`;
|
||||
|
||||
uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json })
|
||||
.then(() => {
|
||||
logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, {
|
||||
imexshopid: jsonObj.imexshopid,
|
||||
filename: jsonObj.filename,
|
||||
webPath
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.log("CARFAX-s3-upload-error", "ERROR", "api", jsonObj.bodyshopid, {
|
||||
imexshopid: jsonObj.imexshopid,
|
||||
filename: jsonObj.filename,
|
||||
webPath,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
exports.default = async (req, res) => {
|
||||
// Only process if in production environment.
|
||||
// if (process.env.NODE_ENV !== "production") {
|
||||
// res.sendStatus(403);
|
||||
// return;
|
||||
// }
|
||||
// Only process if the appropriate token is provided.
|
||||
// if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
|
||||
// res.sendStatus(401);
|
||||
// return;
|
||||
// }
|
||||
|
||||
// Send immediate response and continue processing.
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
message: "Processing request ...",
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
try {
|
||||
logger.log("CARFAX-start", "DEBUG", "api", null, null);
|
||||
const allXMLResults = [];
|
||||
const allErrors = [];
|
||||
|
||||
const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients.
|
||||
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
||||
const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD
|
||||
|
||||
const shopsToProcess =
|
||||
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
||||
logger.log("CARFAX-shopsToProcess-generated", "DEBUG", "api", null, null);
|
||||
|
||||
if (shopsToProcess.length === 0) {
|
||||
logger.log("CARFAX-shopsToProcess-empty", "DEBUG", "api", null, null);
|
||||
return;
|
||||
}
|
||||
|
||||
await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors);
|
||||
|
||||
await sendServerEmail({
|
||||
subject: `CARFAX Report ${moment().format("MM-DD-YY")}`,
|
||||
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
||||
allXMLResults.map((x) => ({
|
||||
imexshopid: x.imexshopid,
|
||||
filename: x.filename,
|
||||
count: x.count,
|
||||
result: x.result
|
||||
})),
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
});
|
||||
|
||||
logger.log("CARFAX-end", "DEBUG", "api", null, null);
|
||||
} catch (error) {
|
||||
logger.log("CARFAX-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) {
|
||||
for (const bodyshop of shopsToProcess) {
|
||||
const erroredJobs = [];
|
||||
try {
|
||||
logger.log("CARFAX-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||
shopname: bodyshop.shopname
|
||||
});
|
||||
|
||||
const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, {
|
||||
bodyshopid: bodyshop.id,
|
||||
...(ignoreDateFilter
|
||||
? {}
|
||||
: {
|
||||
start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"),
|
||||
...(end && { end: moment(end).endOf("day") })
|
||||
})
|
||||
});
|
||||
|
||||
const carfaxObject = {
|
||||
shopid: bodyshop.imexshopid,
|
||||
shop_name: bodyshop.shopname,
|
||||
job: jobs.map((j) =>
|
||||
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
|
||||
erroredJobs.push({ job: job, error: error.toString() });
|
||||
})
|
||||
)
|
||||
};
|
||||
|
||||
if (erroredJobs.length > 0) {
|
||||
logger.log("CARFAX-failed-jobs", "ERROR", "api", bodyshop.id, {
|
||||
count: erroredJobs.length,
|
||||
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
|
||||
});
|
||||
}
|
||||
|
||||
const jsonObj = {
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
json: JSON.stringify(carfaxObject, null, 2),
|
||||
filename: `${bodyshop.imexshopid}_${moment().format("DDMMYYYY_HHMMss")}.json`,
|
||||
count: carfaxObject.job.length
|
||||
};
|
||||
|
||||
if (skipUpload) {
|
||||
fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json);
|
||||
} else {
|
||||
await uploadViaSFTP(jsonObj);
|
||||
}
|
||||
|
||||
allXMLResults.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
count: jsonObj.count,
|
||||
filename: jsonObj.filename,
|
||||
result: jsonObj.result
|
||||
});
|
||||
|
||||
logger.log("CARFAX-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||
shopname: bodyshop.shopname
|
||||
});
|
||||
} catch (error) {
|
||||
//Error at the shop level.
|
||||
logger.log("CARFAX-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
|
||||
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
CARFAXid: bodyshop.CARFAXid,
|
||||
fatal: true,
|
||||
errors: [error.toString()]
|
||||
});
|
||||
} finally {
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
CARFAXid: bodyshop.CARFAXid,
|
||||
errors: erroredJobs.map((ej) => ({
|
||||
ro_number: ej.job?.ro_number,
|
||||
jobid: ej.job?.id,
|
||||
error: ej.error
|
||||
}))
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadViaSFTP(jsonObj) {
|
||||
const sftp = new Client();
|
||||
sftp.on("error", (errors) =>
|
||||
logger.log("CARFAX-sftp-connection-error", "ERROR", "api", jsonObj.bodyshopid, {
|
||||
error: errors.message,
|
||||
stack: errors.stack
|
||||
})
|
||||
);
|
||||
try {
|
||||
// Upload to S3 first.
|
||||
uploadToS3(jsonObj);
|
||||
|
||||
//Connect to the FTP and upload all.
|
||||
await sftp.connect(ftpSetup);
|
||||
|
||||
try {
|
||||
jsonObj.result = await sftp.put(Buffer.from(jsonObj.json), `${jsonObj.filename}`);
|
||||
logger.log("CARFAX-sftp-upload", "DEBUG", "api", jsonObj.bodyshopid, {
|
||||
imexshopid: jsonObj.imexshopid,
|
||||
filename: jsonObj.filename,
|
||||
result: jsonObj.result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.log("CARFAX-sftp-upload-error", "ERROR", "api", jsonObj.bodyshopid, {
|
||||
filename: jsonObj.filename,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.log("CARFAX-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack });
|
||||
throw error;
|
||||
} finally {
|
||||
sftp.end();
|
||||
}
|
||||
}
|
||||
|
||||
const CreateRepairOrderTag = (job, errorCallback) => {
|
||||
if (!job.job_totals) {
|
||||
errorCallback({
|
||||
jobid: job.id,
|
||||
job: job,
|
||||
ro_number: job.ro_number,
|
||||
error: { toString: () => "No job totals for RO." }
|
||||
});
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
const ret = {
|
||||
ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"),
|
||||
v_vin: job.v_vin || "",
|
||||
v_year: job.v_model_yr
|
||||
? parseInt(job.v_model_yr.match(/\d/g))
|
||||
? parseInt(job.v_model_yr.match(/\d/g).join(""), 10)
|
||||
: ""
|
||||
: "",
|
||||
v_make: job.v_make_desc || "",
|
||||
v_model: job.v_model_desc || "",
|
||||
|
||||
date_estimated:
|
||||
(job.date_estimated && moment(job.date_estimated).tz(job.bodyshop.timezone).format(AHDateFormat)) ||
|
||||
(job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) ||
|
||||
"",
|
||||
data_opened:
|
||||
(job.date_open && moment(job.date_open).tz(job.bodyshop.timezone).format(AHDateFormat)) ||
|
||||
(job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) ||
|
||||
"",
|
||||
date_invoiced:
|
||||
(job.date_invoiced && moment(job.date_invoiced).tz(job.bodyshop.timezone).format(AHDateFormat)) || "",
|
||||
loss_date: (job.loss_date && moment(job.loss_date).format(AHDateFormat)) || "",
|
||||
|
||||
ins_co_nm: job.ins_co_nm || "",
|
||||
loss_desc: job.loss_desc || "",
|
||||
theft_ind: job.theft_ind,
|
||||
tloss_ind: job.tlos_ind,
|
||||
subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(),
|
||||
|
||||
areaofdamage: {
|
||||
impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""),
|
||||
impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "")
|
||||
},
|
||||
|
||||
jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()]
|
||||
};
|
||||
return ret;
|
||||
} catch (error) {
|
||||
logger.log("CARFAX-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
|
||||
}
|
||||
};
|
||||
|
||||
const GenerateDetailLines = (line) => {
|
||||
const ret = {
|
||||
line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null,
|
||||
oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null,
|
||||
alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null,
|
||||
lbr_ty: generateLaborType(line.mod_lbr_ty),
|
||||
part_qty: line.part_qty || 0,
|
||||
part_type: generatePartType(line.part_type),
|
||||
act_price: line.act_price || 0
|
||||
};
|
||||
return ret;
|
||||
};
|
||||
|
||||
const generateNullDetailLine = () => {
|
||||
return {
|
||||
line_desc: null,
|
||||
oem_partno: null,
|
||||
alt_partno: null,
|
||||
lbr_ty: null,
|
||||
part_qty: 0,
|
||||
part_type: null,
|
||||
act_price: 0
|
||||
};
|
||||
};
|
||||
|
||||
const generateAreaOfDamage = (loc) => {
|
||||
const areaMap = {
|
||||
"01": "Right Front Corner",
|
||||
"02": "Right Front Side",
|
||||
"03": "Right Side",
|
||||
"04": "Right Rear Side",
|
||||
"05": "Right Rear Corner",
|
||||
"06": "Rear",
|
||||
"07": "Left Rear Corner",
|
||||
"08": "Left Rear Side",
|
||||
"09": "Left Side",
|
||||
10: "Left Front Side",
|
||||
11: "Left Front Corner",
|
||||
12: "Front",
|
||||
13: "Rollover",
|
||||
14: "Uknown",
|
||||
15: "Total Loss",
|
||||
16: "Non-Collision",
|
||||
19: "All Over",
|
||||
25: "Hood",
|
||||
26: "Deck Lid",
|
||||
27: "Roof",
|
||||
28: "Undercarriage",
|
||||
34: "All Over"
|
||||
};
|
||||
return areaMap[loc] || null;
|
||||
};
|
||||
|
||||
const generateLaborType = (type) => {
|
||||
const laborTypeMap = {
|
||||
laa: "Aluminum",
|
||||
lab: "Body",
|
||||
lad: "Diagnostic",
|
||||
lae: "Electrical",
|
||||
laf: "Frame",
|
||||
lag: "Glass",
|
||||
lam: "Mechanical",
|
||||
lar: "Refinish",
|
||||
las: "Structural",
|
||||
lau: "Other - LAU",
|
||||
la1: "Other - LA1",
|
||||
la2: "Other - LA2",
|
||||
la3: "Other - LA3",
|
||||
la4: "Other - LA4",
|
||||
null: "Other",
|
||||
mapa: "Paint Materials",
|
||||
mash: "Shop Materials",
|
||||
rates_subtotal: "Labor Total",
|
||||
"timetickets.labels.shift": "Shift",
|
||||
"timetickets.labels.amshift": "Morning Shift",
|
||||
"timetickets.labels.ambreak": "Morning Break",
|
||||
"timetickets.labels.pmshift": "Afternoon Shift",
|
||||
"timetickets.labels.pmbreak": "Afternoon Break",
|
||||
"timetickets.labels.lunch": "Lunch"
|
||||
};
|
||||
|
||||
return laborTypeMap[type?.toLowerCase()] || null;
|
||||
};
|
||||
|
||||
const generatePartType = (type) => {
|
||||
const partTypeMap = {
|
||||
paa: "Aftermarket",
|
||||
pae: "Existing",
|
||||
pag: "Glass",
|
||||
pal: "LKQ",
|
||||
pan: "OEM",
|
||||
pao: "Other",
|
||||
pas: "Sublet",
|
||||
pasl: "Sublet",
|
||||
ccc: "CC Cleaning",
|
||||
ccd: "CC Damage Waiver",
|
||||
ccdr: "CC Daily Rate",
|
||||
ccf: "CC Refuel",
|
||||
ccm: "CC Mileage",
|
||||
prt_dsmk_total: "Line Item Adjustment"
|
||||
};
|
||||
|
||||
return partTypeMap[type?.toLowerCase()] || null;
|
||||
};
|
||||
@@ -5,4 +5,5 @@ exports.claimscorp = require("./claimscorp").default;
|
||||
exports.kaizen = require("./kaizen").default;
|
||||
exports.usageReport = require("./usageReport").default;
|
||||
exports.podium = require("./podium").default;
|
||||
exports.emsUpload = require("./emsUpload").default;
|
||||
exports.emsUpload = require("./emsUpload").default;
|
||||
exports.carfax = require("./carfax").default;
|
||||
@@ -878,6 +878,43 @@ exports.CHATTER_QUERY = `query CHATTER_EXPORT($start: timestamptz, $bodyshopid:
|
||||
}
|
||||
}`;
|
||||
|
||||
exports.CARFAX_QUERY = `query CARFAX_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) {
|
||||
bodyshops_by_pk(id: $bodyshopid){
|
||||
id
|
||||
shopname
|
||||
imexshopid
|
||||
timezone
|
||||
}
|
||||
jobs(where: {_and: [{converted: {_eq: true}}, {v_vin: {_is_null: false}}, {date_invoiced: {_gt: $start}}, {date_invoiced: {_lte: $end}}, {shopid: {_eq: $bodyshopid}}]}) {
|
||||
id
|
||||
created_at
|
||||
ro_number
|
||||
v_model_yr
|
||||
v_model_desc
|
||||
v_make_desc
|
||||
v_vin
|
||||
date_estimated
|
||||
date_open
|
||||
date_invoiced
|
||||
loss_date
|
||||
ins_co_nm
|
||||
loss_desc
|
||||
theft_ind
|
||||
tlos_ind
|
||||
job_totals
|
||||
area_of_damage
|
||||
joblines(where: {removed: {_eq: false}}) {
|
||||
line_desc
|
||||
oem_partno
|
||||
alt_partno
|
||||
mod_lbr_ty
|
||||
part_qty
|
||||
part_type
|
||||
act_price
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) {
|
||||
bodyshops_by_pk(id: $bodyshopid){
|
||||
id
|
||||
@@ -1816,6 +1853,14 @@ exports.GET_CHATTER_SHOPS = `query GET_CHATTER_SHOPS {
|
||||
}
|
||||
}`;
|
||||
|
||||
exports.GET_CARFAX_SHOPS = `query GET_CARFAX_SHOPS {
|
||||
bodyshops{
|
||||
id
|
||||
shopname
|
||||
imexshopid
|
||||
}
|
||||
}`;
|
||||
|
||||
exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS {
|
||||
bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){
|
||||
id
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const express = require("express");
|
||||
const router = express.Router();
|
||||
const { autohouse, claimscorp, chatter, kaizen, usageReport, podium } = require("../data/data");
|
||||
const { autohouse, claimscorp, chatter, kaizen, usageReport, podium, carfax } = require("../data/data");
|
||||
|
||||
router.post("/ah", autohouse);
|
||||
router.post("/cc", claimscorp);
|
||||
@@ -8,5 +8,6 @@ router.post("/chatter", chatter);
|
||||
router.post("/kaizen", kaizen);
|
||||
router.post("/usagereport", usageReport);
|
||||
router.post("/podium", podium);
|
||||
router.post("/carfax", carfax);
|
||||
|
||||
module.exports = router;
|
||||
|
||||
Reference in New Issue
Block a user