admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in feature/IO-3330-CARFAX-Datapump-Adjustments (pull request #2572)

Browse Source 
Feature/IO-3330 CARFAX Datapump Adjustments

Approved-by: Dave Richer
This commit is contained in:
Allan Carr
2025-09-23 15:01:58 +00:00
committed by Dave Richer
parent 31cfdf9ea3 038aa82087
commit 3e6d6fdbd1
7 changed files with 183 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
_reference/localEmailViewer/index.js
View File

@@ -1,116 +1,96 @@
// index.js // index.js
import express from 'express'; import express from "express";
import fetch from 'node-fetch'; import fetch from "node-fetch";
import {simpleParser} from 'mailparser'; import { simpleParser } from "mailparser";
const app = express(); const app = express();
const PORT = 3334; const PORT = 3334;
app.get('/', async (req, res) => { app.get("/", async (req, res) => {
try { try {
const response = await fetch('http://localhost:4566/_aws/ses'); const response = await fetch("http://localhost:4566/_aws/ses");
if (!response.ok) { if (!response.ok) {
throw new Error('Network response was not ok'); throw new Error("Network response was not ok");
}
const data = await response.json();
const messagesHtml = await parseMessages(data.messages);
res.send(renderHtml(messagesHtml));
} catch (error) {
console.error('Error fetching messages:', error);
res.status(500).send('Error fetching messages');
} }
const data = await response.json();
const messagesHtml = await parseMessages(data.messages);
res.send(renderHtml(messagesHtml));
} catch (error) {
console.error("Error fetching messages:", error);
res.status(500).send("Error fetching messages");
}
}); });
async function parseMessages(messages) { async function parseMessages(messages) {
const parsedMessages = await Promise.all( const parsedMessages = await Promise.all(
messages.map(async (message, index) => { messages.map(async (message, index) => {
try { try {
const parsed = await simpleParser(message.RawData); const parsed = await simpleParser(message.RawData);
return ` return `
<div class="shadow-md rounded-lg p-4 mb-6" style="background-color: lightgray"> <div class="shadow-md rounded-lg p-4 mb-6" style="background-color: lightgray">
<div class="shadow-md rounded-lg p-4 mb-6" style="background-color: white"> <div class="shadow-md rounded-lg p-4 mb-6" style="background-color: white">
<div class="mb-2"> <div class="mb-2"><span class="font-bold text-lg">Message ${index + 1}</span></div>
<span class="font-bold text-lg">Message ${index + 1}</span> <div class="mb-2"><span class="font-semibold">From:</span> ${message.Source}</div>
</div> <div class="mb-2"><span class="font-semibold">To:</span> ${parsed.to.text || "No To Address"}</div>
<div class="mb-2"> <div class="mb-2"><span class="font-semibold">Subject:</span> ${parsed.subject || "No Subject"}</div>
<span class="font-semibold">From:</span> ${message.Source} <div class="mb-2"><span class="font-semibold">Region:</span> ${message.Region}</div>
</div> <div class="mb-2"><span class="font-semibold">Timestamp:</span> ${message.Timestamp}</div>
<div class="mb-2"> </div>
<span class="font-semibold">Region:</span> ${message.Region} <div class="prose">${parsed.html || parsed.textAsHtml || "No HTML content available"}</div>
</div> </div>
<div class="mb-2"> `;
<span class="font-semibold">Timestamp:</span> ${message.Timestamp} } catch (error) {
</div> console.error("Error parsing email:", error);
</div> return `
<div class="prose"> <div class="bg-white shadow-md rounded-lg p-4 mb-6">
${parsed.html || parsed.textAsHtml || 'No HTML content available'} <div class="mb-2"><span class="font-bold text-lg">Message ${index + 1}</span></div>
</div> <div class="mb-2"><span class="font-semibold">From:</span> ${message.Source}</div>
</div> <div class="mb-2"><span class="font-semibold">Region:</span> ${message.Region}</div>
`; <div class="mb-2"><span class="font-semibold">Timestamp:</span> ${message.Timestamp}</div>
} catch (error) { <div class="text-red-500">Error parsing email content</div>
console.error('Error parsing email:', error); </div>
return ` `;
<div class="bg-white shadow-md rounded-lg p-4 mb-6"> }
<div class="mb-2"> })
<span class="font-bold text-lg">Message ${index + 1}</span> );
</div> return parsedMessages.join("");
<div class="mb-2">
<span class="font-semibold">From:</span> ${message.Source}
</div>
<div class="mb-2">
<span class="font-semibold">Region:</span> ${message.Region}
</div>
<div class="mb-2">
<span class="font-semibold">Timestamp:</span> ${message.Timestamp}
</div>
<div class="text-red-500">
Error parsing email content
</div>
</div>
`;
}
})
);
return parsedMessages.join('');
} }
function renderHtml(messagesHtml) { function renderHtml(messagesHtml) {
return ` return `
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Email Messages Viewer</title> <title>Email Messages Viewer</title>
<script src="https://cdn.tailwindcss.com"></script> <script src="https://cdn.tailwindcss.com"></script>
<style> <style>
body { body {
background-color: #f3f4f6; background-color: #f3f4f6;
font-family: Arial, sans-serif; font-family: Arial, sans-serif;
} }
.container { .container {
max-width: 800px; max-width: 800px;
margin: 50px auto; margin: 50px auto;
padding: 20px; padding: 20px;
} }
.prose { .prose {
line-height: 1.6; line-height: 1.6;
} }
</style> </style>
</head> </head>
<body> <body>
<div class="container bg-white shadow-lg rounded-lg p-6"> <div class="container bg-white shadow-lg rounded-lg p-6">
<h1 class="text-2xl font-bold text-center mb-6">Email Messages Viewer</h1> <h1 class="text-2xl font-bold text-center mb-6">Email Messages Viewer</h1>
<div id="messages-container"> <div id="messages-container">${messagesHtml}</div>
${messagesHtml} </div>
</div> </body>
</div> </html>
</body> `;
</html>
`;
} }
app.listen(PORT, () => { app.listen(PORT, () => {
console.log(`Server is running on http://localhost:${PORT}`); console.log(`Server is running on http://localhost:${PORT}`);
}); });

42
_reference/localEmailViewer/package-lock.json generated
View File

@@ -10,7 +10,7 @@
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"express": "^5.1.0", "express": "^5.1.0",
"mailparser": "^3.7.2", "mailparser": "^3.7.4",
"node-fetch": "^3.3.2" "node-fetch": "^3.3.2"
} }
}, },
@@ -634,9 +634,9 @@
"license": "MIT" "license": "MIT"
}, },
"node_modules/libmime": { "node_modules/libmime": {
"version": "5.3.6", "version": "5.3.7",
"resolved": "https://registry.npmjs.org/libmime/-/libmime-5.3.6.tgz", "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.3.7.tgz",
"integrity": "sha512-j9mBC7eiqi6fgBPAGvKCXJKJSIASanYF4EeA4iBzSG0HxQxmXnR3KbyWqTn4CwsKSebqCv2f5XZfAO6sKzgvwA==", "integrity": "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"encoding-japanese": "2.2.0", "encoding-japanese": "2.2.0",
@@ -661,31 +661,31 @@
} }
}, },
"node_modules/mailparser": { "node_modules/mailparser": {
"version": "3.7.2", "version": "3.7.4",
"resolved": "https://registry.npmjs.org/mailparser/-/mailparser-3.7.2.tgz", "resolved": "https://registry.npmjs.org/mailparser/-/mailparser-3.7.4.tgz",
"integrity": "sha512-iI0p2TCcIodR1qGiRoDBBwboSSff50vQAWytM5JRggLfABa4hHYCf3YVujtuzV454xrOP352VsAPIzviqMTo4Q==", "integrity": "sha512-Beh4yyR4jLq3CZZ32asajByrXnW8dLyKCAQD3WvtTiBnMtFWhxO+wa93F6sJNjDmfjxXs4NRNjw3XAGLqZR3Vg==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"encoding-japanese": "2.2.0", "encoding-japanese": "2.2.0",
"he": "1.2.0", "he": "1.2.0",
"html-to-text": "9.0.5", "html-to-text": "9.0.5",
"iconv-lite": "0.6.3", "iconv-lite": "0.6.3",
"libmime": "5.3.6", "libmime": "5.3.7",
"linkify-it": "5.0.0", "linkify-it": "5.0.0",
"mailsplit": "5.4.2", "mailsplit": "5.4.5",
"nodemailer": "6.9.16", "nodemailer": "7.0.4",
"punycode.js": "2.3.1", "punycode.js": "2.3.1",
"tlds": "1.255.0" "tlds": "1.259.0"
} }
}, },
"node_modules/mailsplit": { "node_modules/mailsplit": {
"version": "5.4.2", "version": "5.4.5",
"resolved": "https://registry.npmjs.org/mailsplit/-/mailsplit-5.4.2.tgz", "resolved": "https://registry.npmjs.org/mailsplit/-/mailsplit-5.4.5.tgz",
"integrity": "sha512-4cczG/3Iu3pyl8JgQ76dKkisurZTmxMrA4dj/e8d2jKYcFTZ7MxOzg1gTioTDMPuFXwTrVuN/gxhkrO7wLg7qA==", "integrity": "sha512-oMfhmvclR689IIaQmIcR5nODnZRRVwAKtqFT407TIvmhX2OLUBnshUTcxzQBt3+96sZVDud9NfSe1NxAkUNXEQ==",
"license": "(MIT OR EUPL-1.1+)", "license": "(MIT OR EUPL-1.1+)",
"dependencies": { "dependencies": {
"libbase64": "1.3.0", "libbase64": "1.3.0",
"libmime": "5.3.6", "libmime": "5.3.7",
"libqp": "2.1.1" "libqp": "2.1.1"
} }
}, },
@@ -793,9 +793,9 @@
} }
}, },
"node_modules/nodemailer": { "node_modules/nodemailer": {
"version": "6.9.16", "version": "7.0.4",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.16.tgz", "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-7.0.4.tgz",
"integrity": "sha512-psAuZdTIRN08HKVd/E8ObdV6NO7NTBY3KsC30F7M4H1OnmLCUNaS56FpYxyb26zWLSyYF9Ozch9KYHhHegsiOQ==", "integrity": "sha512-9O00Vh89/Ld2EcVCqJ/etd7u20UhME0f/NToPfArwPEe1Don1zy4mAIz6ariRr7mJ2RDxtaDzN0WJVdVXPtZaw==",
"license": "MIT-0", "license": "MIT-0",
"engines": { "engines": {
"node": ">=6.0.0" "node": ">=6.0.0"
@@ -1114,9 +1114,9 @@
} }
}, },
"node_modules/tlds": { "node_modules/tlds": {
"version": "1.255.0", "version": "1.259.0",
"resolved": "https://registry.npmjs.org/tlds/-/tlds-1.255.0.tgz", "resolved": "https://registry.npmjs.org/tlds/-/tlds-1.259.0.tgz",
"integrity": "sha512-tcwMRIioTcF/FcxLev8MJWxCp+GUALRhFEqbDoZrnowmKSGqPrl5pqS+Sut2m8BgJ6S4FExCSSpGffZ0Tks6Aw==", "integrity": "sha512-AldGGlDP0PNgwppe2quAvuBl18UcjuNtOnDuUkqhd6ipPqrYYBt3aTxK1QTsBVknk97lS2JcafWMghjGWFtunw==",
"license": "MIT", "license": "MIT",
"bin": { "bin": {
"tlds": "bin.js" "tlds": "bin.js"

2
_reference/localEmailViewer/package.json
View File

@@ -12,7 +12,7 @@
"description": "", "description": "",
"dependencies": { "dependencies": {
"express": "^5.1.0", "express": "^5.1.0",
"mailparser": "^3.7.2", "mailparser": "^3.7.4",
"node-fetch": "^3.3.2" "node-fetch": "^3.3.2"
} }
} }

1
docker-compose.yml
View File

@@ -119,6 +119,7 @@ services:
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-job-totals --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-job-totals --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket parts-estimates --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket parts-estimates --create-bucket-configuration LocationConstraint=ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-carfax-uploads --create-bucket-configuration LocationConstraint=ca-central-1
" "
# Node App: The Main IMEX API # Node App: The Main IMEX API
node-app: node-app:

9
hasura/metadata/cron_triggers.yaml
View File

@@ -6,6 +6,15 @@
headers: headers:
- name: x-imex-auth - name: x-imex-auth
value_from_env: DATAPUMP_AUTH value_from_env: DATAPUMP_AUTH
- name: CARFAX Data Pump
webhook: '{{HASURA_API_URL}}/data/carfax'
schedule: 0 7 * * 6
include_in_metadata: true
payload: {}
headers:
- name: x-imex-auth
value_from_env: DATAPUMP_AUTH
comment: Project Mexico
- name: Chatter Data Pump - name: Chatter Data Pump
webhook: '{{HASURA_API_URL}}/data/chatter' webhook: '{{HASURA_API_URL}}/data/chatter'
schedule: 45 5 * * * schedule: 45 5 * * *

36
server/data/carfax.js
View File

@@ -6,7 +6,7 @@ const InstanceManager = require("../utils/instanceMgr").default;
const { isString, isEmpty } = require("lodash"); const { isString, isEmpty } = require("lodash");
const fs = require("fs"); const fs = require("fs");
const client = require("../graphql-client/graphql-client").client; const client = require("../graphql-client/graphql-client").client;
const { sendServerEmail } = require("../email/sendemail"); const { sendServerEmail, sendMexicoBillingEmail } = require("../email/sendemail");
const { uploadFileToS3 } = require("../utils/s3"); const { uploadFileToS3 } = require("../utils/s3");
const crypto = require("crypto"); const crypto = require("crypto");
@@ -168,6 +168,29 @@ async function processShopData(shopsToProcess, start, end, skipUpload, ignoreDat
await uploadViaSFTP(jsonObj); await uploadViaSFTP(jsonObj);
} }
await sendMexicoBillingEmail({
subject: `${shopid.toUpperCase()}_Mexico_${moment().format("MM-DD-YY")} ROs ${jsonObj.count} Error ${errorCode(jsonObj)}`,
text: `Errors:\n${JSON.stringify(
erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
})),
null,
2
)}\n\nUploaded:\n${JSON.stringify(
{
bodyshopid: bodyshop.id,
imexshopid: shopid,
count: jsonObj.count,
filename: jsonObj.filename,
result: jsonObj.result
},
null,
2
)}`
});
allXMLResults.push({ allXMLResults.push({
bodyshopid: bodyshop.id, bodyshopid: bodyshop.id,
imexshopid: shopid, imexshopid: shopid,
@@ -402,3 +425,14 @@ const generatePartType = (type) => {
return partTypeMap[type?.toLowerCase()] || null; return partTypeMap[type?.toLowerCase()] || null;
}; };
const errorCode = ({ count, filename, results }) => {
if (count === 0) return 1;
if (!filename) return 3;
const sftpErrorCode = results?.sftpError?.code;
if (sftpErrorCode && ["ECONNREFUSED", "ENOTFOUND", "ETIMEDOUT", "ECONNRESET"].includes(sftpErrorCode)) {
return 4;
}
if (sftpErrorCode) return 7;
return 0;
};

36
server/email/sendemail.js
View File

@@ -79,6 +79,41 @@ const sendServerEmail = async ({ subject, text }) => {
} }
}; };
const sendMexicoBillingEmail = async ({ subject, text }) => {
if (process.env.NODE_ENV === undefined) return;
try {
mailer.sendMail(
{
from: InstanceManager({
imex: `ImEX Online API - ${process.env.NODE_ENV} <noreply@imex.online>`,
rome: `Rome Online API - ${process.env.NODE_ENV} <noreply@romeonline.io>`
}),
to: ["mexico@rometech.zohodesk.com"],
subject: subject,
text: text,
ses: {
// optional extra arguments for SendRawEmail
Tags: [
{
Name: "tag_name",
Value: "tag_value"
}
]
}
},
// eslint-disable-next-line no-unused-vars
(err, info) => {
logger.log("server-email-failure", err ? "error" : "debug", null, null, {
message: err?.message,
stack: err?.stack
});
}
);
} catch (error) {
logger.log("server-email-failure", "error", null, null, { message: error?.message, stack: error?.stack });
}
};
const sendWelcomeEmail = async ({ to, resetLink, dateLine, features, bcc }) => { const sendWelcomeEmail = async ({ to, resetLink, dateLine, features, bcc }) => {
try { try {
await mailer.sendMail({ await mailer.sendMail({
@@ -420,6 +455,7 @@ ${body.bounce?.bouncedRecipients.map(
module.exports = { module.exports = {
sendEmail, sendEmail,
sendServerEmail, sendServerEmail,
sendMexicoBillingEmail,
sendTaskEmail, sendTaskEmail,
emailBounce, emailBounce,
sendWelcomeEmail sendWelcomeEmail