const queries = require("../graphql-client/queries"); const Dinero = require("dinero.js"); const moment = require("moment-timezone"); const logger = require("../utils/logger"); const InstanceManager = require("../utils/instanceMgr").default; const { isString, isEmpty } = require("lodash"); const fs = require("fs"); const client = require("../graphql-client/graphql-client").client; const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail"); const { uploadFileToS3 } = require("../utils/s3"); const crypto = require("crypto"); let Client = require("ssh2-sftp-client"); const AHDateFormat = "YYYY-MM-DD"; const NON_ASCII_REGEX = /[^\x20-\x7E]/g; const ftpSetup = { host: process.env.CARFAX_HOST, port: process.env.CARFAX_PORT, username: process.env.CARFAX_USER, password: process.env.CARFAX_PASSWORD, debug: process.env.NODE_ENV !== "production" ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; const S3_BUCKET_NAME = InstanceManager({ imex: "imex-carfax-uploads", rome: "rome-carfax-uploads" }); const region = InstanceManager.InstanceRegion; const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME); const uploadToS3 = (jsonObj) => { const webPath = isLocal ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}` : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`; uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json }) .then(() => { logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, { imexshopid: jsonObj.imexshopid, filename: jsonObj.filename, webPath }); }) .catch((error) => { logger.log("CARFAX-s3-upload-error", "ERROR", "api", jsonObj.bodyshopid, { imexshopid: jsonObj.imexshopid, filename: jsonObj.filename, webPath, error: error.message, stack: error.stack }); }); }; exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { return res.sendStatus(403); } // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { return res.sendStatus(401); } // Send immediate response and continue processing. res.status(202).json({ success: true, message: "Processing request ...", timestamp: new Date().toISOString() }); try { logger.log("CARFAX-start", "DEBUG", "api", null, null); const allXMLResults = []; const allErrors = []; const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("CARFAX-shopsToProcess-generated", "DEBUG", "api", null, null); if (shopsToProcess.length === 0) { logger.log("CARFAX-shopsToProcess-empty", "DEBUG", "api", null, null); return; } await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors); await sendServerEmail({ subject: `CARFAX Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allXMLResults.map((x) => ({ imexshopid: x.imexshopid, filename: x.filename, count: x.count, result: x.result })), null, 2 )}` }); logger.log("CARFAX-end", "DEBUG", "api", null, null); } catch (error) { logger.log("CARFAX-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); } }; async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) { for (const bodyshop of shopsToProcess) { const shopid = bodyshop.imexshopid?.toLowerCase() || bodyshop.shopname.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); const erroredJobs = []; try { logger.log("CARFAX-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, { bodyshopid: bodyshop.id, ...(ignoreDateFilter ? {} : { start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), ...(end && { end: moment(end).endOf("day") }) }) }); const carfaxObject = { shopid: shopid, shop_name: bodyshop.shopname, job: jobs.map((j) => CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { erroredJobs.push({ job: job, error: error.toString() }); }) ) }; if (erroredJobs.length > 0) { logger.log("CARFAX-failed-jobs", "ERROR", "api", bodyshop.id, { count: erroredJobs.length, jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) }); } const jsonObj = { bodyshopid: bodyshop.id, imexshopid: shopid, json: JSON.stringify(carfaxObject, null, 2), filename: `${shopid}_${moment().format("DDMMYYYY_HHMMss")}.json`, count: carfaxObject.job.length }; if (skipUpload) { fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json); } else { await uploadViaSFTP(jsonObj); } await sendMexicoBillingEmail({ subject: `${shopid.toUpperCase()}_Mexico_${moment().format("MM-DD-YY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`, text: `Errors:\n${JSON.stringify( erroredJobs.map((ej) => ({ ro_number: ej.job?.ro_number, jobid: ej.job?.id, error: ej.error })), null, 2 )}\n\nUploaded:\n${JSON.stringify( { bodyshopid: bodyshop.id, imexshopid: shopid, count: jsonObj.count, filename: jsonObj.filename, result: jsonObj.result }, null, 2 )}` }); allXMLResults.push({ bodyshopid: bodyshop.id, imexshopid: shopid, count: jsonObj.count, filename: jsonObj.filename, result: jsonObj.result }); logger.log("CARFAX-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); } catch (error) { //Error at the shop level. logger.log("CARFAX-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); allErrors.push({ bodyshopid: bodyshop.id, imexshopid: shopid, CARFAXid: bodyshop.CARFAXid, fatal: true, errors: [error.toString()] }); } finally { allErrors.push({ bodyshopid: bodyshop.id, imexshopid: shopid, CARFAXid: bodyshop.CARFAXid, errors: erroredJobs.map((ej) => ({ ro_number: ej.job?.ro_number, jobid: ej.job?.id, error: ej.error })) }); } } } async function uploadViaSFTP(jsonObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("CARFAX-sftp-connection-error", "ERROR", "api", jsonObj.bodyshopid, { error: errors.message, stack: errors.stack }) ); try { // Upload to S3 first. uploadToS3(jsonObj); //Connect to the FTP and upload all. await sftp.connect(ftpSetup); try { jsonObj.result = await sftp.put(Buffer.from(jsonObj.json), `${jsonObj.filename}`); logger.log("CARFAX-sftp-upload", "DEBUG", "api", jsonObj.bodyshopid, { imexshopid: jsonObj.imexshopid, filename: jsonObj.filename, result: jsonObj.result }); } catch (error) { logger.log("CARFAX-sftp-upload-error", "ERROR", "api", jsonObj.bodyshopid, { filename: jsonObj.filename, error: error.message, stack: error.stack }); throw error; } } catch (error) { logger.log("CARFAX-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); } } const CreateRepairOrderTag = (job, errorCallback) => { if (!job.job_totals) { errorCallback({ jobid: job.id, job: job, ro_number: job.ro_number, error: { toString: () => "No job totals for RO." } }); return {}; } try { const ret = { ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"), v_vin: job.v_vin || "", v_year: job.v_model_yr ? parseInt(job.v_model_yr.match(/\d/g)) ? parseInt(job.v_model_yr.match(/\d/g).join(""), 10) : "" : "", v_make: job.v_make_desc || "", v_model: job.v_model_desc || "", date_estimated: (job.date_estimated && moment(job.date_estimated).tz(job.bodyshop.timezone).format(AHDateFormat)) || (job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) || "", data_opened: (job.date_open && moment(job.date_open).tz(job.bodyshop.timezone).format(AHDateFormat)) || (job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) || "", date_invoiced: (job.date_invoiced && moment(job.date_invoiced).tz(job.bodyshop.timezone).format(AHDateFormat)) || "", loss_date: (job.loss_date && moment(job.loss_date).format(AHDateFormat)) || "", ins_co_nm: job.ins_co_nm || "", loss_desc: job.loss_desc || "", theft_ind: job.theft_ind, tloss_ind: job.tlos_ind, subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(), areaofdamage: { impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""), impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "") }, jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()] }; return ret; } catch (error) { logger.log("CARFAX-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); } }; const GenerateDetailLines = (line) => { const ret = { line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null, oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null, alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null, lbr_ty: generateLaborType(line.mod_lbr_ty), part_qty: line.part_qty || 0, part_type: generatePartType(line.part_type), act_price: line.act_price || 0 }; return ret; }; const generateNullDetailLine = () => { return { line_desc: null, oem_partno: null, alt_partno: null, lbr_ty: null, part_qty: 0, part_type: null, act_price: 0 }; }; const generateAreaOfDamage = (loc) => { const areaMap = { "01": "Right Front Corner", "02": "Right Front Side", "03": "Right Side", "04": "Right Rear Side", "05": "Right Rear Corner", "06": "Rear", "07": "Left Rear Corner", "08": "Left Rear Side", "09": "Left Side", 10: "Left Front Side", 11: "Left Front Corner", 12: "Front", 13: "Rollover", 14: "Uknown", 15: "Total Loss", 16: "Non-Collision", 19: "All Over", 25: "Hood", 26: "Deck Lid", 27: "Roof", 28: "Undercarriage", 34: "All Over" }; return areaMap[loc] || null; }; const generateLaborType = (type) => { const laborTypeMap = { laa: "Aluminum", lab: "Body", lad: "Diagnostic", lae: "Electrical", laf: "Frame", lag: "Glass", lam: "Mechanical", lar: "Refinish", las: "Structural", lau: "Other - LAU", la1: "Other - LA1", la2: "Other - LA2", la3: "Other - LA3", la4: "Other - LA4", null: "Other", mapa: "Paint Materials", mash: "Shop Materials", rates_subtotal: "Labor Total", "timetickets.labels.shift": "Shift", "timetickets.labels.amshift": "Morning Shift", "timetickets.labels.ambreak": "Morning Break", "timetickets.labels.pmshift": "Afternoon Shift", "timetickets.labels.pmbreak": "Afternoon Break", "timetickets.labels.lunch": "Lunch" }; return laborTypeMap[type?.toLowerCase()] || null; }; const generatePartType = (type) => { const partTypeMap = { paa: "Aftermarket", pae: "Existing", pag: "Glass", pal: "LKQ", pan: "OEM", pao: "Other", pas: "Sublet", pasl: "Sublet", ccc: "CC Cleaning", ccd: "CC Damage Waiver", ccdr: "CC Daily Rate", ccf: "CC Refuel", ccm: "CC Mileage", prt_dsmk_total: "Line Item Adjustment" }; return partTypeMap[type?.toLowerCase()] || null; }; const errorCode = ({ count, filename, results }) => { if (count === 0) return 1; if (!filename) return 3; const sftpErrorCode = results?.sftpError?.code; if (sftpErrorCode && ["ECONNREFUSED", "ENOTFOUND", "ETIMEDOUT", "ECONNRESET"].includes(sftpErrorCode)) { return 4; } if (sftpErrorCode) return 7; return 0; };