diff --git a/server/accounting/qbo/qbo-payments.js b/server/accounting/qbo/qbo-payments.js index f0d4ccdde..db288efe5 100644 --- a/server/accounting/qbo/qbo-payments.js +++ b/server/accounting/qbo/qbo-payments.js @@ -10,7 +10,6 @@ const queries = require("../../graphql-client/queries"); const { refresh: refreshOauthToken, setNewRefreshToken } = require("./qbo-callback"); const OAuthClient = require("intuit-oauth"); const moment = require("moment-timezone"); -const GraphQLClient = require("graphql-request").GraphQLClient; const { QueryInsuranceCo, InsertInsuranceCo, @@ -28,7 +27,7 @@ exports.default = async (req, res) => { clientId: process.env.QBO_CLIENT_ID, clientSecret: process.env.QBO_SECRET, environment: process.env.NODE_ENV === "production" ? "production" : "sandbox", - redirectUri: process.env.QBO_REDIRECT_URI, + redirectUri: process.env.QBO_REDIRECT_URI }); try { //Fetch the API Access Tokens & Set them for the session. @@ -131,22 +130,20 @@ exports.default = async (req, res) => { // //No error. Mark the payment exported & insert export log. if (elgen) { - const result = await client - .setHeaders({ Authorization: BearerToken }) - .request(queries.QBO_MARK_PAYMENT_EXPORTED, { - paymentId: payment.id, - payment: { - exportedat: moment().tz(bodyshop.timezone) - }, - logs: [ - { - bodyshopid: bodyshop.id, - paymentid: payment.id, - successful: true, - useremail: req.user.email - } - ] - }); + await client.setHeaders({ Authorization: BearerToken }).request(queries.QBO_MARK_PAYMENT_EXPORTED, { + paymentId: payment.id, + payment: { + exportedat: moment().tz(bodyshop.timezone) + }, + logs: [ + { + bodyshopid: bodyshop.id, + paymentid: payment.id, + successful: true, + useremail: req.user.email + } + ] + }); } ret.push({ paymentid: payment.id, success: true }); @@ -156,7 +153,7 @@ exports.default = async (req, res) => { }); //Add the export log error. if (elgen) { - const result = await client.setHeaders({ Authorization: BearerToken }).request(queries.INSERT_EXPORT_LOG, { + await client.setHeaders({ Authorization: BearerToken }).request(queries.INSERT_EXPORT_LOG, { logs: [ { bodyshopid: bodyshop.id, @@ -190,7 +187,7 @@ exports.default = async (req, res) => { } }; -async function InsertPayment(oauthClient, qbo_realmId, req, payment, parentRef, bodyshop) { +async function InsertPayment(oauthClient, qbo_realmId, req, payment, parentRef) { const { paymentMethods, invoices } = await QueryMetaData( oauthClient, qbo_realmId, @@ -227,20 +224,20 @@ async function InsertPayment(oauthClient, qbo_realmId, req, payment, parentRef, PaymentRefNum: payment.transactionid, ...(invoices && invoices.length === 1 && invoices[0] ? { - Line: [ - { - Amount: Dinero({ - amount: Math.round(payment.amount * 100) - }).toFormat(DineroQbFormat), - LinkedTxn: [ - { - TxnId: invoices[0].Id, - TxnType: "Invoice" - } - ] - } - ] - } + Line: [ + { + Amount: Dinero({ + amount: Math.round(payment.amount * 100) + }).toFormat(DineroQbFormat), + LinkedTxn: [ + { + TxnId: invoices[0].Id, + TxnType: "Invoice" + } + ] + } + ] + } : {}) }; logger.log("qbo-payments-objectlog", "DEBUG", req.user.email, payment.id, { @@ -263,7 +260,7 @@ async function InsertPayment(oauthClient, qbo_realmId, req, payment, parentRef, status: result.response?.status, bodyshopid: payment.job.shopid, email: req.user.email - }) + }); setNewRefreshToken(req.user.email, result); return result && result.Bill; } catch (error) { @@ -291,7 +288,7 @@ async function QueryMetaData(oauthClient, qbo_realmId, req, ro_number, isCreditM status: invoice.response?.status, bodyshopid, email: req.user.email - }) + }); const paymentMethods = await oauthClient.makeApiCall({ url: urlBuilder(qbo_realmId, "query", `select * From PaymentMethod`), method: "POST", @@ -306,7 +303,7 @@ async function QueryMetaData(oauthClient, qbo_realmId, req, ro_number, isCreditM status: paymentMethods.response?.status, bodyshopid, email: req.user.email - }) + }); setNewRefreshToken(req.user.email, paymentMethods); // const classes = await oauthClient.makeApiCall({ @@ -358,7 +355,7 @@ async function QueryMetaData(oauthClient, qbo_realmId, req, ro_number, isCreditM status: taxCodes.response?.status, bodyshopid, email: req.user.email - }) + }); const items = await oauthClient.makeApiCall({ url: urlBuilder(qbo_realmId, "query", `select * From Item`), method: "POST", @@ -373,7 +370,7 @@ async function QueryMetaData(oauthClient, qbo_realmId, req, ro_number, isCreditM status: items.response?.status, bodyshopid, email: req.user.email - }) + }); setNewRefreshToken(req.user.email, items); const itemMapping = {}; @@ -412,8 +409,8 @@ async function QueryMetaData(oauthClient, qbo_realmId, req, ro_number, isCreditM }; } -async function InsertCreditMemo(oauthClient, qbo_realmId, req, payment, parentRef, bodyshop) { - const { paymentMethods, invoices, items, taxCodes } = await QueryMetaData( +async function InsertCreditMemo(oauthClient, qbo_realmId, req, payment, parentRef) { + const { invoices, items, taxCodes } = await QueryMetaData( oauthClient, qbo_realmId, req, @@ -449,14 +446,14 @@ async function InsertCreditMemo(oauthClient, qbo_realmId, req, payment, parentRe TaxCodeRef: { value: taxCodes[ - findTaxCode( - { - local: false, - federal: false, - state: false - }, - payment.job.bodyshop.md_responsibility_centers.sales_tax_codes - ) + findTaxCode( + { + local: false, + federal: false, + state: false + }, + payment.job.bodyshop.md_responsibility_centers.sales_tax_codes + ) ] } } @@ -483,12 +480,14 @@ async function InsertCreditMemo(oauthClient, qbo_realmId, req, payment, parentRe status: result.response?.status, bodyshopid: req.user.bodyshopid, email: req.user.email - }) + }); setNewRefreshToken(req.user.email, result); return result && result.Bill; } catch (error) { logger.log("qbo-payables-error", "DEBUG", req.user.email, payment.id, { - error: error && error.message, + error: error, + validationError: JSON.stringify(error?.response?.data), + accountmeta: JSON.stringify({ items, taxCodes }), method: "InsertCreditMemo" }); throw error; diff --git a/server/data/carfax.js b/server/data/carfax.js new file mode 100644 index 000000000..855755f37 --- /dev/null +++ b/server/data/carfax.js @@ -0,0 +1,409 @@ +const path = require("path"); +const queries = require("../graphql-client/queries"); +const Dinero = require("dinero.js"); +const moment = require("moment-timezone"); +const logger = require("../utils/logger"); +const InstanceManager = require("../utils/instanceMgr").default; +const { isString, isEmpty } = require("lodash"); +const fs = require("fs"); +const client = require("../graphql-client/graphql-client").client; +const { sendServerEmail } = require("../email/sendemail"); +const { uploadFileToS3 } = require("../utils/s3"); +const crypto = require("crypto"); + +require("dotenv").config({ + path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) +}); +let Client = require("ssh2-sftp-client"); + +const AHDateFormat = "YYYY-MM-DD"; + +const NON_ASCII_REGEX = /[^\x20-\x7E]/g; + +const ftpSetup = { + host: process.env.CARFAX_HOST, + port: process.env.CARFAX_PORT, + username: process.env.CARFAX_USER, + password: process.env.CARFAX_PASSWORD, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, + algorithms: { + serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] + } +}; + +const S3_BUCKET_NAME = InstanceManager({ + imex: "imex-carfax-uploads", + rome: "rome-carfax-uploads" +}); +const region = InstanceManager.InstanceRegion; +const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME); + +const uploadToS3 = (jsonObj) => { + const webPath = isLocal + ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${jsonObj.filename}` + : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${jsonObj.filename}`; + + uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: jsonObj.filename, content: jsonObj.json }) + .then(() => { + logger.log("CARFAX-s3-upload", "DEBUG", "api", jsonObj.bodyshopid, { + imexshopid: jsonObj.imexshopid, + filename: jsonObj.filename, + webPath + }); + }) + .catch((error) => { + logger.log("CARFAX-s3-upload-error", "ERROR", "api", jsonObj.bodyshopid, { + imexshopid: jsonObj.imexshopid, + filename: jsonObj.filename, + webPath, + error: error.message, + stack: error.stack + }); + }); +}; + +exports.default = async (req, res) => { + // Only process if in production environment. + // if (process.env.NODE_ENV !== "production") { + // res.sendStatus(403); + // return; + // } + // Only process if the appropriate token is provided. + // if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { + // res.sendStatus(401); + // return; + // } + + // Send immediate response and continue processing. + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); + + try { + logger.log("CARFAX-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + + const { bodyshops } = await client.request(queries.GET_CARFAX_SHOPS); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; + const { start, end, skipUpload, ignoreDateFilter } = req.body; //YYYY-MM-DD + + const shopsToProcess = + specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; + logger.log("CARFAX-shopsToProcess-generated", "DEBUG", "api", null, null); + + if (shopsToProcess.length === 0) { + logger.log("CARFAX-shopsToProcess-empty", "DEBUG", "api", null, null); + return; + } + + await processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors); + + await sendServerEmail({ + subject: `CARFAX Report ${moment().format("MM-DD-YY")}`, + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), + null, + 2 + )}` + }); + + logger.log("CARFAX-end", "DEBUG", "api", null, null); + } catch (error) { + logger.log("CARFAX-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + } +}; + +async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDateFilter, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { + const erroredJobs = []; + try { + logger.log("CARFAX-start-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + + const { jobs, bodyshops_by_pk } = await client.request(queries.CARFAX_QUERY, { + bodyshopid: bodyshop.id, + ...(ignoreDateFilter + ? {} + : { + start: start ? moment(start).startOf("day") : moment().subtract(7, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }) + }); + + const carfaxObject = { + shopid: bodyshop.imexshopid, + shop_name: bodyshop.shopname, + job: jobs.map((j) => + CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { + erroredJobs.push({ job: job, error: error.toString() }); + }) + ) + }; + + if (erroredJobs.length > 0) { + logger.log("CARFAX-failed-jobs", "ERROR", "api", bodyshop.id, { + count: erroredJobs.length, + jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) + }); + } + + const jsonObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + json: JSON.stringify(carfaxObject, null, 2), + filename: `${bodyshop.imexshopid}_${moment().format("DDMMYYYY_HHMMss")}.json`, + count: carfaxObject.job.length + }; + + if (skipUpload) { + fs.writeFileSync(`./logs/${jsonObj.filename}`, jsonObj.json); + } else { + await uploadViaSFTP(jsonObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + count: jsonObj.count, + filename: jsonObj.filename, + result: jsonObj.result + }); + + logger.log("CARFAX-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + } catch (error) { + //Error at the shop level. + logger.log("CARFAX-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + CARFAXid: bodyshop.CARFAXid, + fatal: true, + errors: [error.toString()] + }); + } finally { + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + CARFAXid: bodyshop.CARFAXid, + errors: erroredJobs.map((ej) => ({ + ro_number: ej.job?.ro_number, + jobid: ej.job?.id, + error: ej.error + })) + }); + } + } +} + +async function uploadViaSFTP(jsonObj) { + const sftp = new Client(); + sftp.on("error", (errors) => + logger.log("CARFAX-sftp-connection-error", "ERROR", "api", jsonObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) + ); + try { + // Upload to S3 first. + uploadToS3(jsonObj); + + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + try { + jsonObj.result = await sftp.put(Buffer.from(jsonObj.json), `${jsonObj.filename}`); + logger.log("CARFAX-sftp-upload", "DEBUG", "api", jsonObj.bodyshopid, { + imexshopid: jsonObj.imexshopid, + filename: jsonObj.filename, + result: jsonObj.result + }); + } catch (error) { + logger.log("CARFAX-sftp-upload-error", "ERROR", "api", jsonObj.bodyshopid, { + filename: jsonObj.filename, + error: error.message, + stack: error.stack + }); + throw error; + } + } catch (error) { + logger.log("CARFAX-sftp-error", "ERROR", "api", jsonObj.bodyshopid, { error: error.message, stack: error.stack }); + throw error; + } finally { + sftp.end(); + } +} + +const CreateRepairOrderTag = (job, errorCallback) => { + if (!job.job_totals) { + errorCallback({ + jobid: job.id, + job: job, + ro_number: job.ro_number, + error: { toString: () => "No job totals for RO." } + }); + return {}; + } + + try { + const ret = { + ro_number: crypto.createHash("md5").update(job.ro_number, "utf8").digest("hex"), + v_vin: job.v_vin || "", + v_year: job.v_model_yr + ? parseInt(job.v_model_yr.match(/\d/g)) + ? parseInt(job.v_model_yr.match(/\d/g).join(""), 10) + : "" + : "", + v_make: job.v_make_desc || "", + v_model: job.v_model_desc || "", + + date_estimated: + (job.date_estimated && moment(job.date_estimated).tz(job.bodyshop.timezone).format(AHDateFormat)) || + (job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) || + "", + data_opened: + (job.date_open && moment(job.date_open).tz(job.bodyshop.timezone).format(AHDateFormat)) || + (job.created_at && moment(job.created_at).tz(job.bodyshop.timezone).format(AHDateFormat)) || + "", + date_invoiced: + (job.date_invoiced && moment(job.date_invoiced).tz(job.bodyshop.timezone).format(AHDateFormat)) || "", + loss_date: (job.loss_date && moment(job.loss_date).format(AHDateFormat)) || "", + + ins_co_nm: job.ins_co_nm || "", + loss_desc: job.loss_desc || "", + theft_ind: job.theft_ind, + tloss_ind: job.tlos_ind, + subtotal: Dinero(job.job_totals.totals.subtotal).toUnit(), + + areaofdamage: { + impact1: generateAreaOfDamage(job.area_of_damage?.impact1 || ""), + impact2: generateAreaOfDamage(job.area_of_damage?.impact2 || "") + }, + + jobLines: job.joblines.length > 0 ? job.joblines.map((jl) => GenerateDetailLines(jl)) : [generateNullDetailLine()] + }; + return ret; + } catch (error) { + logger.log("CARFAX-job-data-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); + } +}; + +const GenerateDetailLines = (line) => { + const ret = { + line_desc: line.line_desc ? line.line_desc.replace(NON_ASCII_REGEX, "") : null, + oem_partno: line.oem_partno ? line.oem_partno.replace(NON_ASCII_REGEX, "") : null, + alt_partno: line.alt_partno ? line.alt_partno.replace(NON_ASCII_REGEX, "") : null, + lbr_ty: generateLaborType(line.mod_lbr_ty), + part_qty: line.part_qty || 0, + part_type: generatePartType(line.part_type), + act_price: line.act_price || 0 + }; + return ret; +}; + +const generateNullDetailLine = () => { + return { + line_desc: null, + oem_partno: null, + alt_partno: null, + lbr_ty: null, + part_qty: 0, + part_type: null, + act_price: 0 + }; +}; + +const generateAreaOfDamage = (loc) => { + const areaMap = { + "01": "Right Front Corner", + "02": "Right Front Side", + "03": "Right Side", + "04": "Right Rear Side", + "05": "Right Rear Corner", + "06": "Rear", + "07": "Left Rear Corner", + "08": "Left Rear Side", + "09": "Left Side", + 10: "Left Front Side", + 11: "Left Front Corner", + 12: "Front", + 13: "Rollover", + 14: "Uknown", + 15: "Total Loss", + 16: "Non-Collision", + 19: "All Over", + 25: "Hood", + 26: "Deck Lid", + 27: "Roof", + 28: "Undercarriage", + 34: "All Over" + }; + return areaMap[loc] || null; +}; + +const generateLaborType = (type) => { + const laborTypeMap = { + laa: "Aluminum", + lab: "Body", + lad: "Diagnostic", + lae: "Electrical", + laf: "Frame", + lag: "Glass", + lam: "Mechanical", + lar: "Refinish", + las: "Structural", + lau: "Other - LAU", + la1: "Other - LA1", + la2: "Other - LA2", + la3: "Other - LA3", + la4: "Other - LA4", + null: "Other", + mapa: "Paint Materials", + mash: "Shop Materials", + rates_subtotal: "Labor Total", + "timetickets.labels.shift": "Shift", + "timetickets.labels.amshift": "Morning Shift", + "timetickets.labels.ambreak": "Morning Break", + "timetickets.labels.pmshift": "Afternoon Shift", + "timetickets.labels.pmbreak": "Afternoon Break", + "timetickets.labels.lunch": "Lunch" + }; + + return laborTypeMap[type?.toLowerCase()] || null; +}; + +const generatePartType = (type) => { + const partTypeMap = { + paa: "Aftermarket", + pae: "Existing", + pag: "Glass", + pal: "LKQ", + pan: "OEM", + pao: "Other", + pas: "Sublet", + pasl: "Sublet", + ccc: "CC Cleaning", + ccd: "CC Damage Waiver", + ccdr: "CC Daily Rate", + ccf: "CC Refuel", + ccm: "CC Mileage", + prt_dsmk_total: "Line Item Adjustment" + }; + + return partTypeMap[type?.toLowerCase()] || null; +}; diff --git a/server/data/data.js b/server/data/data.js index e89d041b9..9d88f14d8 100644 --- a/server/data/data.js +++ b/server/data/data.js @@ -5,4 +5,5 @@ exports.claimscorp = require("./claimscorp").default; exports.kaizen = require("./kaizen").default; exports.usageReport = require("./usageReport").default; exports.podium = require("./podium").default; -exports.emsUpload = require("./emsUpload").default; \ No newline at end of file +exports.emsUpload = require("./emsUpload").default; +exports.carfax = require("./carfax").default; \ No newline at end of file diff --git a/server/graphql-client/queries.js b/server/graphql-client/queries.js index e05e16508..a05dc9538 100644 --- a/server/graphql-client/queries.js +++ b/server/graphql-client/queries.js @@ -878,6 +878,43 @@ exports.CHATTER_QUERY = `query CHATTER_EXPORT($start: timestamptz, $bodyshopid: } }`; +exports.CARFAX_QUERY = `query CARFAX_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { + bodyshops_by_pk(id: $bodyshopid){ + id + shopname + imexshopid + timezone + } + jobs(where: {_and: [{converted: {_eq: true}}, {v_vin: {_is_null: false}}, {date_invoiced: {_gt: $start}}, {date_invoiced: {_lte: $end}}, {shopid: {_eq: $bodyshopid}}]}) { + id + created_at + ro_number + v_model_yr + v_model_desc + v_make_desc + v_vin + date_estimated + date_open + date_invoiced + loss_date + ins_co_nm + loss_desc + theft_ind + tlos_ind + job_totals + area_of_damage + joblines(where: {removed: {_eq: false}}) { + line_desc + oem_partno + alt_partno + mod_lbr_ty + part_qty + part_type + act_price + } + } +}`; + exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { bodyshops_by_pk(id: $bodyshopid){ id @@ -1816,6 +1853,14 @@ exports.GET_CHATTER_SHOPS = `query GET_CHATTER_SHOPS { } }`; +exports.GET_CARFAX_SHOPS = `query GET_CARFAX_SHOPS { + bodyshops{ + id + shopname + imexshopid + } +}`; + exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS { bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){ id diff --git a/server/routes/dataRoutes.js b/server/routes/dataRoutes.js index f8212c36d..8e7bc04fd 100644 --- a/server/routes/dataRoutes.js +++ b/server/routes/dataRoutes.js @@ -1,6 +1,6 @@ const express = require("express"); const router = express.Router(); -const { autohouse, claimscorp, chatter, kaizen, usageReport, podium } = require("../data/data"); +const { autohouse, claimscorp, chatter, kaizen, usageReport, podium, carfax } = require("../data/data"); router.post("/ah", autohouse); router.post("/cc", claimscorp); @@ -8,5 +8,6 @@ router.post("/chatter", chatter); router.post("/kaizen", kaizen); router.post("/usagereport", usageReport); router.post("/podium", podium); +router.post("/carfax", carfax); module.exports = router;