const queries = require("../graphql-client/queries"); const moment = require("moment-timezone"); const logger = require("../utils/logger"); const fs = require("fs"); const client = require("../graphql-client/graphql-client").rpsClient; const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail"); const crypto = require("crypto"); const { ftpSetup, uploadToS3 } = require("./carfax"); let Client = require("ssh2-sftp-client"); const AHDateFormat = "YYYY-MM-DD"; const NON_ASCII_REGEX = /[^\x20-\x7E]/g; const S3_BUCKET_NAME = "rps-carfax-uploads"; const carfaxExportRps = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { return res.sendStatus(403); } // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { return res.sendStatus(401); } // Send immediate response and continue processing. res.status(202).json({ success: true, message: "Processing request ...", timestamp: new Date().toISOString() }); try { logger.log("CARFAX-RPS-start", "DEBUG", "api", null, null); const allJSONResults = []; const allErrors = []; const { bodyshops } = await client.request(queries.GET_CARFAX_RPS_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("CARFAX-RPS-shopsToProcess-generated", "DEBUG", "api", null, null); if (shopsToProcess.length === 0) { logger.log("CARFAX-RPS-shopsToProcess-empty", "DEBUG", "api", null, null); return; } await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allJSONResults, allErrors); await sendServerEmail({ subject: `Project Mexico RPS Report ${moment().format("MM-DD-YY")}`, text: `Total Count: ${allJSONResults.reduce((a, v) => a + v.count, 0)}\nErrors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allJSONResults.map((x) => ({ imexshopid: x.imexshopid, filename: x.filename, count: x.count, result: x.result })), null, 2 )}`, to: ["bradley.rhoades@convenient-brands.com"] }); logger.log("CARFAX-RPS-end", "DEBUG", "api", null, null); } catch (error) { logger.log("CARFAX-RPS-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); } }; async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allJSONResults, allErrors) { for (const bodyshop of shopsToProcess) { const shopid = bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); const erroredJobs = []; try { logger.log("CARFAX-RPS-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_RPS_QUERY, { bodyshopid: bodyshop.id, ...(ignoreDateFilter ? {} : { starttz: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), ...(end && { endtz: moment(end).endOf("day") }), start: start ? moment(start).startOf("day").format(AHDateFormat) : moment().subtract(7, "days").startOf("day").format(AHDateFormat), ...(end && { endtz: moment(end).endOf("day").format(AHDateFormat) }) }) }); const carfaxObject = { shopid: shopid, shop_name: bodyshop.shopname, job: jobs.map((j) => CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { erroredJobs.push({ job: job, error: error.toString() }); }) ) }; if (erroredJobs.length > 0) { logger.log("CARFAX-RPS-failed-jobs", "ERROR", "api", bodyshop.id, { count: erroredJobs.length, jobs: JSON.stringify(erroredJobs.map((j) => j.job.id)) }); } const jsonObj = { bodyshopid: bodyshop.id, imexshopid: shopid, json: JSON.stringify(carfaxObject, null, 2), filename: `${shopid}_${moment().format("DDMMYYYY_HHMMss")}.json`, count: carfaxObject?.job?.length || 0 }; if (skipUpload) { fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json); uploadToS3(jsonObj, S3_BUCKET_NAME); } else { if (jsonObj.count > 0) { await uploadViaSFTP(jsonObj); await sendMexicoBillingEmail({ subject: `${shopid.replace(/_/g, "").toUpperCase()}_MexicoRPS_${moment().format("MMDDYYYY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`, text: `Errors:\n${JSON.stringify( erroredJobs.map((ej) => ({ jobid: ej.job?.id, error: ej.error })), null, 2 )}\n\nUploaded:\n${JSON.stringify( { bodyshopid: bodyshop.id, imexshopid: shopid, count: jsonObj.count, filename: jsonObj.filename, result: jsonObj.result }, null, 2 )}` }); } } jsonObj.count > 0 && allJSONResults.push({ bodyshopid: bodyshop.id, imexshopid: shopid, count: jsonObj.count, filename: jsonObj.filename, result: jsonObj.result || "No Upload Result Available" }); logger.log("CARFAX-RPS-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); } catch (error) { //Error at the shop level. logger.log("CARFAX-RPS-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); allErrors.push({ bodyshopid: bodyshop.id, imexshopid: shopid, CARFAXid: bodyshop.CARFAXid, fatal: true, errors: [error.toString()] }); } finally { allErrors.push({ bodyshopid: bodyshop.id, imexshopid: shopid, CARFAXid: bodyshop.CARFAXid, errors: erroredJobs.map((ej) => ({ jobid: ej.job?.id, error: ej.error })) }); } } } async function uploadViaSFTP(jsonObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("CARFAX-RPS-sftp-connection-error", "ERROR", "api", jsonObj.bodyshopid, { error: errors.message, stack: errors.stack }) ); try { // Upload to S3 first. uploadToS3(jsonObj, S3_BUCKET_NAME); //Connect to the FTP and upload all. await sftp.connect(ftpSetup); try { jsonObj.result = await sftp.put(Buffer.from(jsonObj.json), `${jsonObj.filename}`); logger.log("CARFAX-RPS-sftp-upload", "DEBUG", "api", jsonObj.bodyshopid, { imexshopid: jsonObj.imexshopid, filename: jsonObj.filename, result: jsonObj.result }); } catch (error) { logger.log("CARFAX-RPS-sftp-upload-error", "ERROR", "api", jsonObj.bodyshopid, { filename: jsonObj.filename, error: error.message, stack: error.stack }); throw error; } } catch (error) { logger.log("CARFAX-RPS-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); } } const CreateRepairOrderTag = (job, errorCallback) => { try { const subtotalEntry = job.totals.find((total) => total.TTL_TYPECD === ""); const subtotal = subtotalEntry ? subtotalEntry.T_AMT : 0; const ret = { ro_number: crypto.createHash("md5").update(job.id, "utf8").digest("hex"), v_vin: job.v_vin || "", v_year: (() => { const y = parseInt(job.v_model_yr); return isNaN(y) ? null : y < 100 ? y + (y >= (new Date().getFullYear() + 1) % 100 ? 1900 : 2000) : y; })(), v_make: job.v_makedesc || "", v_model: job.v_model || "", date_estimated: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "", data_opened: moment(job.created_at).tz("America/Winnipeg").format(AHDateFormat) || "", date_invoiced: [job.close_date, job.created_at].find((date) => date) ? moment([job.close_date, job.created_at].find((date) => date)) .tz("America/Winnipeg") .format(AHDateFormat) : "", loss_date: job.loss_date ? moment(job.loss_date).format(AHDateFormat) : "", ins_co_nm: job.ins_co_nm || "", loss_desc: job.loss_desc || "", theft_ind: job.theft_ind, tloss_ind: job.tlos_ind, subtotal: subtotal, areaofdamage: { impact1: generateAreaOfDamage(job.impact_1 || ""), impact2: generateAreaOfDamage(job.impact_2 || "") }, jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()] }; return ret; } catch (error) { logger.log("CARFAX-RPS-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); errorCallback({ jobid: job.id, error }); } }; const GenerateDetailLines = (line) => { const ret = { line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null, oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null, alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null, op_code: line.lbr_op || null, op_code_desc: generateOpCodeDescription(line.lbr_op), lbr_ty: generateLaborType(line.mod_lbr_ty), lbr_hrs: line.mod_lb_hrs || 0, part_qty: line.part_qty || 0, part_type: generatePartType(line.part_type), act_price: line.act_price || 0 }; return ret; }; const generateNullDetailLine = () => { return { line_desc: null, oem_partno: null, alt_partno: null, lbr_ty: null, part_qty: 0, part_type: null, act_price: 0 }; }; const generateAreaOfDamage = (loc) => { const areaMap = { "01": "Right Front Corner", "02": "Right Front Side", "03": "Right Side", "04": "Right Rear Side", "05": "Right Rear Corner", "06": "Rear", "07": "Left Rear Corner", "08": "Left Rear Side", "09": "Left Side", 10: "Left Front Side", 11: "Left Front Corner", 12: "Front", 13: "Rollover", 14: "Uknown", 15: "Total Loss", 16: "Non-Collision", 19: "All Over", 25: "Hood", 26: "Deck Lid", 27: "Roof", 28: "Undercarriage", 34: "All Over" }; return areaMap[loc] || null; }; const generateLaborType = (type) => { const laborTypeMap = { laa: "Aluminum", lab: "Body", lad: "Diagnostic", lae: "Electrical", laf: "Frame", lag: "Glass", lam: "Mechanical", lar: "Refinish", las: "Structural", lau: "Other - LAU", la1: "Other - LA1", la2: "Other - LA2", la3: "Other - LA3", la4: "Other - LA4", null: "Other", mapa: "Paint Materials", mash: "Shop Materials", rates_subtotal: "Labor Total", "timetickets.labels.shift": "Shift", "timetickets.labels.amshift": "Morning Shift", "timetickets.labels.ambreak": "Morning Break", "timetickets.labels.pmshift": "Afternoon Shift", "timetickets.labels.pmbreak": "Afternoon Break", "timetickets.labels.lunch": "Lunch" }; return laborTypeMap[type?.toLowerCase()] || null; }; const generatePartType = (type) => { const partTypeMap = { paa: "Aftermarket", pae: "Existing", pag: "Glass", pal: "LKQ", pan: "OEM", pao: "Other", pas: "Sublet", pasl: "Sublet", ccc: "CC Cleaning", ccd: "CC Damage Waiver", ccdr: "CC Daily Rate", ccf: "CC Refuel", ccm: "CC Mileage", prt_dsmk_total: "Line Item Adjustment" }; return partTypeMap[type?.toLowerCase()] || null; }; const generateOpCodeDescription = (type) => { const opCodeMap = { OP0: "REMOVE / REPLACE PARTIAL", OP1: "REFINISH / REPAIR", OP10: "REPAIR , PARTIAL", OP100: "REPLACE PRE-PRICED", OP101: "REMOVE/REPLACE RECYCLED PART", OP103: "REMOVE / REPLACE PARTIAL", OP104: "REMOVE / REPLACE PARTIAL LABOUR", OP105: "!!ADJUST MANUALLY!!", OP106: "REPAIR , PARTIAL", OP107: "CHIPGUARD", OP108: "MULTI TONE", OP109: "REPLACE PRE-PRICED", OP11: "REMOVE / REPLACE", OP110: "REFINISH / REPAIR", OP111: "REMOVE / REPLACE", OP112: "REMOVE / REPLACE", OP113: "REPLACE PRE-PRICED", OP114: "REPLACE PRE-PRICED", OP12: "REMOVE / REPLACE PARTIAL", OP120: "REPAIR , PARTIAL", OP13: "ADDITIONAL COSTS", OP14: "ADDITIONAL OPERATIONS", OP15: "BLEND", OP16: "SUBLET", OP17: "POLICY LIMIT ADJUSTMENT", OP18: "APPEAR ALLOWANCE", OP2: "REMOVE / INSTALL", OP24: "CHIPGUARD", OP25: "TWO TONE", OP26: "PAINTLESS DENT REPAIR", OP260: "SUBLET", OP3: "ADDITIONAL LABOR", OP4: "ALIGNMENT", OP5: "OVERHAUL", OP6: "REFINISH", OP7: "INSPECT", OP8: "CHECK / ADJUST", OP9: "REPAIR" }; return opCodeMap[type?.toUpperCase()] || null; }; const errorCode = ({ count, filename, results }) => { if (count === 0) return 1; if (!filename) return 3; const sftpErrorCode = results?.sftpError?.code; if (sftpErrorCode && ["ECONNREFUSED", "ENOTFOUND", "ETIMEDOUT", "ECONNRESET"].includes(sftpErrorCode)) { return 4; } if (sftpErrorCode) return 7; return 0; }; module.exports = { default: carfaxExportRps, ftpSetup };