diff --git a/server/data/carfax-rps.js b/server/data/carfax-rps.js new file mode 100644 index 000000000..e8c43f7ae --- /dev/null +++ b/server/data/carfax-rps.js @@ -0,0 +1,412 @@ +const queries = require("../graphql-client/queries"); +const Dinero = require("dinero.js"); +const moment = require("moment-timezone"); +const logger = require("../utils/logger"); +const InstanceManager = require("../utils/instanceMgr").default; +const { isString, isEmpty } = require("lodash"); +const fs = require("fs"); +const client = require("../graphql-client/graphql-client").rpsClient; +const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail"); +const { uploadFileToS3 } = require("../utils/s3"); +const crypto = require("crypto"); +const { ftpSetup, uploadToS3 } = require("./carfax") +let Client = require("ssh2-sftp-client"); + +const AHDateFormat = "YYYY-MM-DD"; + +const NON_ASCII_REGEX = /[^\x20-\x7E]/g; + +const carfaxExportRps = async (req, res) => { + const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. + + // Only process if in production environment. + if (process.env.NODE_ENV !== "production") { + return res.sendStatus(403); + } + // Only process if the appropriate token is provided. + if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { + return res.sendStatus(401); + } + + // Send immediate response and continue processing. + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); + + try { + logger.log("CARFAX-RPS-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + + const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; + const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD + + const shopsToProcess = + specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; + logger.log("CARFAX-RPS-shopsToProcess-generated", "DEBUG", "api", null, null); + + if (shopsToProcess.length === 0) { + logger.log("CARFAX-RPS-shopsToProcess-empty", "DEBUG", "api", null, null); + return; + } + + await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors); + + await sendServerEmail({ + subject: `Project Mexico Report ${moment().format("MM-DD-YY")}`, + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), + null, + 2 + )}` + }); + + logger.log("CARFAX-RPS-end", "DEBUG", "api", null, null); + } catch (error) { + logger.log("CARFAX-RPS-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + } +}; + +async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { + const shopid = bodyshop.imexshopid?.toLowerCase() || bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + const erroredJobs = []; + try { + logger.log("CARFAX-RPS-start-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + + const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, { + bodyshopid: bodyshop.id, + ...(ignoreDateFilter + ? {} + : { + start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }) + }); + + const carfaxObject = { + shopid: shopid, + shop_name: bodyshop.shopname, + job: jobs.map((j) => + CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { + erroredJobs.push({ job: job, error: error.toString() }); + }) + ) + }; + + if (erroredJobs.length > 0) { + logger.log("CARFAX-RPS-failed-jobs", "ERROR", "api", bodyshop.id, { + count: erroredJobs.length, + jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) + }); + } + + const jsonObj = { + bodyshopid: bodyshop.id, + imexshopid: shopid, + json: JSON.stringify(carfaxObject, null, 2), + filename: `${shopid}_${moment().format("DDMMYYYY_HHMMss")}.json`, + count: carfaxObject.job.length + }; + + if (skipUpload) { + fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json); + uploadToS3(jsonObj); + } else { + await uploadViaSFTP(jsonObj); + + await sendMexicoBillingEmail({ + subject: `${shopid.replace(/_/g, "").toUpperCase()}_Mexico${InstanceManager({ + imex: "IO", + rome: "RO" + })}_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`, + text: `Errors:\n${JSON.stringify( + erroredJobs.map((ej) => ({ + ro_number: ej.job?.ro_number, + jobid: ej.job?.id, + error: ej.error + })), + null, + 2 + )}\n\nUploaded:\n${JSON.stringify( + { + bodyshopid: bodyshop.id, + imexshopid: shopid, + count: jsonObj.count, + filename: jsonObj.filename, + result: jsonObj.result + }, + null, + 2 + )}` + }); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: shopid, + count: jsonObj.count, + filename: jsonObj.filename, + result: jsonObj.result + }); + + logger.log("CARFAX-RPS-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + } catch (error) { + //Error at the shop level. + logger.log("CARFAX-RPS-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: shopid, + CARFAXid: bodyshop.CARFAXid, + fatal: true, + errors: [error.toString()] + }); + } finally { + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: shopid, + CARFAXid: bodyshop.CARFAXid, + errors: erroredJobs.map((ej) => ({ + ro_number: ej.job?.ro_number, + jobid: ej.job?.id, + error: ej.error + })) + }); + } + } +} + +async function uploadViaSFTP(jsonObj) { + const sftp = new Client(); + sftp.on("error", (errors) => + logger.log("CARFAX-RPS-sftp-connection-error", "ERROR", "api", jsonObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) + ); + try { + // Upload to S3 first. + uploadToS3(jsonObj); + + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + try { + jsonObj.result = await sftp.put(Buffer.from(jsonObj.json), `${jsonObj.filename}`); + logger.log("CARFAX-RPS-sftp-upload", "DEBUG", "api", jsonObj.bodyshopid, { + imexshopid: jsonObj.imexshopid, + filename: jsonObj.filename, + result: jsonObj.result + }); + } catch (error) { + logger.log("CARFAX-RPS-sftp-upload-error", "ERROR", "api", jsonObj.bodyshopid, { + filename: jsonObj.filename, + error: error.message, + stack: error.stack + }); + throw error; + } + } catch (error) { + logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack }); + throw error; + } finally { + sftp.end(); + } +} + +const CreateRepairOrderTag = (job, errorCallback) => { + if (!job.job_totals) { + errorCallback({ + jobid: job.id, + job: job, + ro_number: job.ro_number, + error: { toString: () => "No job totals for RO." } + }); + return {}; + } + + try { + const ret = { + ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"), + v_vin: job.v_vin || "", + v_year: job.v_model_yr + ? parseInt(job.v_model_yr.match(/\d/g)) + ? parseInt(job.v_model_yr.match(/\d/g).join(""), 10) + : "" + : "", + v_make: job.v_make_desc || "", + v_model: job.v_model_desc || "", + + date_estimated: [job.date_estimated, job.created_at].find((date) => date) + ? moment([job.date_open, job.created_at].find((date) => date)) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) + : "", + data_opened: [job.date_open, job.created_at].find((date) => date) + ? moment([job.date_open, job.created_at].find((date) => date)) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) + : "", + date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date) + ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) + : "", + loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "", + + ins_co_nm: job.ins_co_nm || "", + loss_desc: job.loss_desc || "", + theft_ind: job.theft_ind, + tloss_ind: job.tlos_ind, + subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(), + + areaofdamage: { + impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""), + impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "") + }, + + jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()] + }; + return ret; + } catch (error) { + logger.log("CARFAX-RPS-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); + } +}; + +const GenerateDetailLines = (line) => { + const ret = { + line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null, + oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null, + alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null, + op_code_desc: line.op_code_desc ? line.op_code_desc.replace(NON_ASCII_REGEX, "") : null, + lbr_ty: generateLaborType(line.mod_lbr_ty), + lbr_hrs: line.mod_lb_hrs || 0, + part_qty: line.part_qty || 0, + part_type: generatePartType(line.part_type), + act_price: line.act_price || 0 + }; + return ret; +}; + +const generateNullDetailLine = () => { + return { + line_desc: null, + oem_partno: null, + alt_partno: null, + lbr_ty: null, + part_qty: 0, + part_type: null, + act_price: 0 + }; +}; + +const generateAreaOfDamage = (loc) => { + const areaMap = { + "01": "Right Front Corner", + "02": "Right Front Side", + "03": "Right Side", + "04": "Right Rear Side", + "05": "Right Rear Corner", + "06": "Rear", + "07": "Left Rear Corner", + "08": "Left Rear Side", + "09": "Left Side", + 10: "Left Front Side", + 11: "Left Front Corner", + 12: "Front", + 13: "Rollover", + 14: "Uknown", + 15: "Total Loss", + 16: "Non-Collision", + 19: "All Over", + 25: "Hood", + 26: "Deck Lid", + 27: "Roof", + 28: "Undercarriage", + 34: "All Over" + }; + return areaMap[loc] || null; +}; + +const generateLaborType = (type) => { + const laborTypeMap = { + laa: "Aluminum", + lab: "Body", + lad: "Diagnostic", + lae: "Electrical", + laf: "Frame", + lag: "Glass", + lam: "Mechanical", + lar: "Refinish", + las: "Structural", + lau: "Other - LAU", + la1: "Other - LA1", + la2: "Other - LA2", + la3: "Other - LA3", + la4: "Other - LA4", + null: "Other", + mapa: "Paint Materials", + mash: "Shop Materials", + rates_subtotal: "Labor Total", + "timetickets.labels.shift": "Shift", + "timetickets.labels.amshift": "Morning Shift", + "timetickets.labels.ambreak": "Morning Break", + "timetickets.labels.pmshift": "Afternoon Shift", + "timetickets.labels.pmbreak": "Afternoon Break", + "timetickets.labels.lunch": "Lunch" + }; + + return laborTypeMap[type?.toLowerCase()] || null; +}; + +const generatePartType = (type) => { + const partTypeMap = { + paa: "Aftermarket", + pae: "Existing", + pag: "Glass", + pal: "LKQ", + pan: "OEM", + pao: "Other", + pas: "Sublet", + pasl: "Sublet", + ccc: "CC Cleaning", + ccd: "CC Damage Waiver", + ccdr: "CC Daily Rate", + ccf: "CC Refuel", + ccm: "CC Mileage", + prt_dsmk_total: "Line Item Adjustment" + }; + + return partTypeMap[type?.toLowerCase()] || null; +}; + +const errorCode = ({ count, filename, results }) => { + if (count === 0) return 1; + if (!filename) return 3; + const sftpErrorCode = results?.sftpError?.code; + if (sftpErrorCode && ["ECONNREFUSED", "ENOTFOUND", "ETIMEDOUT", "ECONNRESET"].includes(sftpErrorCode)) { + return 4; + } + if (sftpErrorCode) return 7; + return 0; +}; + + +module.exports = { + default: carfaxExportRps, + ftpSetup +} \ No newline at end of file diff --git a/server/data/carfax.js b/server/data/carfax.js index 34a145dda..941ee615b 100644 --- a/server/data/carfax.js +++ b/server/data/carfax.js @@ -24,7 +24,7 @@ const ftpSetup = { debug: process.env.NODE_ENV !== "production" ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + : () => { }, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } @@ -61,7 +61,7 @@ const uploadToS3 = (jsonObj) => { }); }; -exports.default = async (req, res) => { +const carfaxExport = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { return res.sendStatus(403); @@ -132,9 +132,9 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat ...(ignoreDateFilter ? {} : { - start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }) + start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }) }); const carfaxObject = { @@ -295,18 +295,18 @@ const CreateRepairOrderTag = (job, errorCallback) => { date_estimated: [job.date_estimated, job.created_at].find((date) => date) ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", data_opened: [job.date_open, job.created_at].find((date) => date) ? moment([job.date_open, job.created_at].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", date_invoiced: [job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date) ? moment([job.date_invoiced, job.actual_delivery, job.actual_completion].find((date) => date)) - .tz(job.bodyshop.timezone) - .format(AHDateFormat) + .tz(job.bodyshop.timezone) + .format(AHDateFormat) : "", loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "", @@ -447,3 +447,9 @@ const errorCode = ({ count, filename, results }) => { if (sftpErrorCode) return 7; return 0; }; + + +module.exports = { + default: carfaxExport, + ftpSetup, uploadToS3 +} \ No newline at end of file diff --git a/server/data/data.js b/server/data/data.js index e9b80de5a..0aa7f6e34 100644 --- a/server/data/data.js +++ b/server/data/data.js @@ -7,4 +7,5 @@ exports.usageReport = require("./usageReport").default; exports.podium = require("./podium").default; exports.emsUpload = require("./emsUpload").default; exports.carfax = require("./carfax").default; +exports.carfaxRps = require("./carfax-rps").default; exports.vehicletype = require("./vehicletype/vehicletype").default; \ No newline at end of file diff --git a/server/graphql-client/graphql-client.js b/server/graphql-client/graphql-client.js index 79d86315b..2c42bf361 100644 --- a/server/graphql-client/graphql-client.js +++ b/server/graphql-client/graphql-client.js @@ -1,3 +1,5 @@ +const logger = require("../utils/logger"); + const GraphQLClient = require("graphql-request").GraphQLClient; //New bug introduced with Graphql Request. @@ -11,9 +13,24 @@ const client = new GraphQLClient(process.env.GRAPHQL_ENDPOINT, { } }); +const rpsClient = + process.env.RPS_GRAPHQL_ENDPOINT && process.env.RPS_HASURA_ADMIN_SECRET ? + new GraphQLClient(process.env.RPS_GRAPHQL_ENDPOINT, { + headers: { + "x-hasura-admin-secret": process.env.RPS_HASURA_ADMIN_SECRET + } + }) : null; + +if (!rpsClient) { + //System log to disable RPS functions + logger.log(`RPS secrets are not set. Client is not configured.`, "WARN", "redis", "api", { + }); +} + const unauthorizedClient = new GraphQLClient(process.env.GRAPHQL_ENDPOINT); module.exports = { client, + rpsClient, unauthorizedClient }; diff --git a/server/routes/dataRoutes.js b/server/routes/dataRoutes.js index 8e7bc04fd..c72a2a502 100644 --- a/server/routes/dataRoutes.js +++ b/server/routes/dataRoutes.js @@ -1,6 +1,6 @@ const express = require("express"); const router = express.Router(); -const { autohouse, claimscorp, chatter, kaizen, usageReport, podium, carfax } = require("../data/data"); +const { autohouse, claimscorp, chatter, kaizen, usageReport, podium, carfax, carfaxRps } = require("../data/data"); router.post("/ah", autohouse); router.post("/cc", claimscorp); @@ -9,5 +9,6 @@ router.post("/kaizen", kaizen); router.post("/usagereport", usageReport); router.post("/podium", podium); router.post("/carfax", carfax); +router.post("/carfaxrps", carfaxRps); module.exports = router;