admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in hotfix/2024-11-21 (pull request #1947)

Browse Source 
Hotfix/2024 11 21 IO-2921 IO-3027
This commit is contained in:
Allan Carr
2024-11-22 04:41:29 +00:00
parent 289a666b6d fa24d87966
commit d40579694f
13 changed files with 279 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
certs/id_rsa → certs/id_rsa.key
View File

12
certs/io-ftp-test.key Normal file
View File

@@ -0,0 +1,12 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS
1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5
U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY
PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt
8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ
AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA
Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk
ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv
5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2
lvLWZ0cC10ZXN0LWtleQECAwQ=
-----END OPENSSH PRIVATE KEY-----

1
certs/io-ftp-test.key.pub Normal file
View File

@@ -0,0 +1 @@
ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key

12
certs/io-ftp-test.ppk Normal file
View File

@@ -0,0 +1,12 @@
PuTTY-User-Key-File-3: ecdsa-sha2-nistp521
Encryption: none
Comment: io-ftp-test-key
Public-Lines: 4
AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt
2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+
J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX
Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w==
Private-Lines: 2
AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP
wYC0wLMW4bJAf+kjqUnj4wGocoTe
Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54

14
docker-build.ps1 Normal file
View File

@@ -0,0 +1,14 @@
# Stop and remove all containers, images, and networks from the Compose file
docker compose down --rmi all
# Prune all unused Docker objects including volumes
docker system prune --all --volumes --force
# Prune unused build cache
docker builder prune --all --force
# Prune all unused volumes
docker volume prune --all --force
# Rebuild and start the containers
docker compose up --build

16
docker-build.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Stop and remove all containers, images, and networks from the Compose file
docker compose down --rmi all
# Prune all unused Docker objects including volumes
docker system prune --all --volumes --force
# Prune unused build cache
docker builder prune --all --force
# Prune all unused volumes
docker volume prune --all --force
# Rebuild and start the containers
docker compose up --build

9
docker-compose.yml
View File

@@ -114,8 +114,8 @@ services:
" "
aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1
aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1 aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1
aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1 aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1 aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
" "
# Node App: The Main IMEX API # Node App: The Main IMEX API
@@ -169,7 +169,7 @@ services:
# - redis-insight-data:/db # - redis-insight-data:/db
# ##Optional Container for SFTP/SSH Server for testing # ##Optional Container for SFTP/SSH Server for testing
# ssh-sftp-server: # ssh-sftp-server:
# image: atmoz/sftp:alpine # Using an image with SFTP support # image: atmoz/sftp:alpine # Using an image with SFTP support
# container_name: ssh-sftp-server # container_name: ssh-sftp-server
# hostname: ssh-sftp-server # hostname: ssh-sftp-server
@@ -178,9 +178,10 @@ services:
# ports: # ports:
# - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host) # - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
# volumes: # volumes:
# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key # - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys
# - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads
# environment: # environment:
# # - SFTP_USERS=user::1000::upload
# - SFTP_USERS=user:password:1000::upload # - SFTP_USERS=user:password:1000::upload
# command: > # command: >
# /bin/sh -c " # /bin/sh -c "

11
package-lock.json generated
View File

@@ -54,7 +54,7 @@
"soap": "^1.1.5", "soap": "^1.1.5",
"socket.io": "^4.8.0", "socket.io": "^4.8.0",
"socket.io-adapter": "^2.5.5", "socket.io-adapter": "^2.5.5",
"ssh2-sftp-client": "^10.0.3", "ssh2-sftp-client": "^11.0.0",
"twilio": "^4.23.0", "twilio": "^4.23.0",
"uuid": "^10.0.0", "uuid": "^10.0.0",
"winston": "^3.15.0", "winston": "^3.15.0",
@@ -8740,16 +8740,17 @@
} }
}, },
"node_modules/ssh2-sftp-client": { "node_modules/ssh2-sftp-client": {
"version": "10.0.3", "version": "11.0.0",
"resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz", "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-11.0.0.tgz",
"integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==", "integrity": "sha512-lOjgNYtioYquhtgyHwPryFNhllkuENjvCKkUXo18w/Q4UpEffCnEUBfiOTlwFdKIhG1rhrOGnA6DeKPSF2CP6w==",
"license": "Apache-2.0",
"dependencies": { "dependencies": {
"concat-stream": "^2.0.0", "concat-stream": "^2.0.0",
"promise-retry": "^2.0.1", "promise-retry": "^2.0.1",
"ssh2": "^1.15.0" "ssh2": "^1.15.0"
}, },
"engines": { "engines": {
"node": ">=16.20.2" "node": ">=18.20.4"
}, },
"funding": { "funding": {
"type": "individual", "type": "individual",

2
package.json
View File

@@ -64,7 +64,7 @@
"soap": "^1.1.5", "soap": "^1.1.5",
"socket.io": "^4.8.0", "socket.io": "^4.8.0",
"socket.io-adapter": "^2.5.5", "socket.io-adapter": "^2.5.5",
"ssh2-sftp-client": "^10.0.3", "ssh2-sftp-client": "^11.0.0",
"twilio": "^4.23.0", "twilio": "^4.23.0",
"uuid": "^10.0.0", "uuid": "^10.0.0",
"winston": "^3.15.0", "winston": "^3.15.0",

105
server/data/autohouse.js
View File

@@ -25,15 +25,15 @@ const ftpSetup = {
port: process.env.AUTOHOUSE_PORT, port: process.env.AUTOHOUSE_PORT,
username: process.env.AUTOHOUSE_USER, username: process.env.AUTOHOUSE_USER,
password: process.env.AUTOHOUSE_PASSWORD, password: process.env.AUTOHOUSE_PASSWORD,
debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), debug:
process.env.NODE_ENV !== "production"
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
: () => {},
algorithms: { algorithms: {
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
} }
}; };
const allxmlsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => { exports.default = async (req, res) => {
// Only process if in production environment. // Only process if in production environment.
if (process.env.NODE_ENV !== "production") { if (process.env.NODE_ENV !== "production") {
@@ -55,12 +55,13 @@ exports.default = async (req, res) => {
try { try {
logger.log("autohouse-start", "DEBUG", "api", null, null); logger.log("autohouse-start", "DEBUG", "api", null, null);
const allXMLResults = [];
const allErrors = [];
const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid]; const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess = const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null); logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -69,27 +70,18 @@ exports.default = async (req, res) => {
logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
return; return;
} }
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
if (skipUpload) { await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
} else {
await uploadViaSFTP(allxmlsToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({ await sendServerEmail({
subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), allXMLResults.map((x) => ({
imexshopid: x.imexshopid,
filename: x.filename,
count: x.count,
result: x.result
})),
null, null,
2 2
)}` )}`
@@ -101,8 +93,8 @@ exports.default = async (req, res) => {
} }
}; };
async function processBatch(batch, start, end) { async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
for (const bodyshop of batch) { for (const bodyshop of shopsToProcess) {
const erroredJobs = []; const erroredJobs = [];
try { try {
logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -132,12 +124,27 @@ async function processBatch(batch, start, end) {
}); });
} }
const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true }); const xmlObj = {
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }),
filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`,
count: autoHouseObject.AutoHouseExport.RepairOrder.length
};
allxmlsToUpload.push({ if (skipUpload) {
count: autoHouseObject.AutoHouseExport.RepairOrder.length, fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
xml: ret, } else {
filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml` await uploadViaSFTP(xmlObj);
}
allXMLResults.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
autohouseid: bodyshop.autohouseid,
count: xmlObj.count,
filename: xmlObj.filename,
result: xmlObj.result
}); });
logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -169,33 +176,35 @@ async function processBatch(batch, start, end) {
} }
} }
async function uploadViaSFTP(allxmlsToUpload) { async function uploadViaSFTP(xmlObj) {
const sftp = new Client(); const sftp = new Client();
sftp.on("error", (errors) => sftp.on("error", (errors) =>
logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
error: errors.message,
stack: errors.stack
})
); );
try { try {
//Connect to the FTP and upload all. //Connect to the FTP and upload all.
await sftp.connect(ftpSetup); await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) { try {
try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { imexshopid: xmlObj.imexshopid,
filename: xmlObj.filename, filename: xmlObj.filename,
result: xmlObj.result result: xmlObj.result
}); });
} catch (error) { } catch (error) {
logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
filename: xmlObj.filename, filename: xmlObj.filename,
error: error.message, error: error.message,
stack: error.stack stack: error.stack
}); });
throw error; throw error;
}
} }
} catch (error) { } catch (error) {
logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
throw error; throw error;
} finally { } finally {
sftp.end(); sftp.end();

92
server/data/chatter.js
View File

@@ -17,15 +17,15 @@ const ftpSetup = {
port: process.env.CHATTER_PORT, port: process.env.CHATTER_PORT,
username: process.env.CHATTER_USER, username: process.env.CHATTER_USER,
privateKey: null, privateKey: null,
debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), debug:
process.env.NODE_ENV !== "production"
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
: () => {},
algorithms: { algorithms: {
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
} }
}; };
const allcsvsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => { exports.default = async (req, res) => {
// Only process if in production environment. // Only process if in production environment.
if (process.env.NODE_ENV !== "production") { if (process.env.NODE_ENV !== "production") {
@@ -47,12 +47,13 @@ exports.default = async (req, res) => {
try { try {
logger.log("chatter-start", "DEBUG", "api", null, null); logger.log("chatter-start", "DEBUG", "api", null, null);
const allChatterObjects = [];
const allErrors = [];
const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid]; const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess = const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -62,29 +63,24 @@ exports.default = async (req, res) => {
return; return;
} }
const batchPromises = []; await processBatch(shopsToProcess, start, end, allChatterObjects, allErrors);
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize); const csvToUpload = {
const batchPromise = (async () => { count: allChatterObjects.length,
await processBatch(batch, start, end); csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }),
if (skipUpload) { filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv`
for (const csvObj of allcsvsToUpload) { };
await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
} if (skipUpload) {
} else { await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv);
await uploadViaSFTP(allcsvsToUpload); } else {
} await uploadViaSFTP(csvToUpload);
})();
batchPromises.push(batchPromise);
} }
await Promise.all(batchPromises);
await sendServerEmail({ await sendServerEmail({
subject: `Chatter Report ${moment().format("MM-DD-YY")}`, subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n
allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}`
null,
2
)}`
}); });
logger.log("chatter-end", "DEBUG", "api", null, null); logger.log("chatter-end", "DEBUG", "api", null, null);
@@ -93,8 +89,8 @@ exports.default = async (req, res) => {
} }
}; };
async function processBatch(batch, start, end) { async function processBatch(shopsToProcess, start, end, allChatterObjects, allErrors) {
for (const bodyshop of batch) { for (const bodyshop of shopsToProcess) {
try { try {
logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname shopname: bodyshop.shopname
@@ -116,15 +112,7 @@ async function processBatch(batch, start, end) {
phone_number: j.ownr_ph1 phone_number: j.ownr_ph1
}; };
}); });
allChatterObjects.push(...chatterObject);
const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
allcsvsToUpload.push({
count: chatterObject.length,
csv: ret,
filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
});
logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname shopname: bodyshop.shopname
}); });
@@ -166,7 +154,7 @@ async function getPrivateKey() {
} }
} }
async function uploadViaSFTP(allcsvsToUpload) { async function uploadViaSFTP(csvToUpload) {
const sftp = new Client(); const sftp = new Client();
sftp.on("error", (errors) => sftp.on("error", (errors) =>
logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
@@ -178,21 +166,19 @@ async function uploadViaSFTP(allcsvsToUpload) {
//Connect to the FTP and upload all. //Connect to the FTP and upload all.
await sftp.connect({ ...ftpSetup, privateKey }); await sftp.connect({ ...ftpSetup, privateKey });
for (const csvObj of allcsvsToUpload) { try {
try { csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`);
csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvToUpload.filename,
filename: csvObj.filename, result: csvToUpload.result
result: csvObj.result });
}); } catch (error) {
} catch (error) { logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { filename: csvToUpload.filename,
filename: csvObj.filename, error: csvToUpload.message,
error: error.message, stack: csvToUpload.stack
stack: error.stack });
}); throw error;
throw error;
}
} }
} catch (error) { } catch (error) {
logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });

108
server/data/claimscorp.js
View File

@@ -24,15 +24,15 @@ const ftpSetup = {
port: process.env.CLAIMSCORP_PORT, port: process.env.CLAIMSCORP_PORT,
username: process.env.CLAIMSCORP_USER, username: process.env.CLAIMSCORP_USER,
password: process.env.CLAIMSCORP_PASSWORD, password: process.env.CLAIMSCORP_PASSWORD,
debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), debug:
process.env.NODE_ENV !== "production"
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
: () => {},
algorithms: { algorithms: {
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
} }
}; };
const allxmlsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => { exports.default = async (req, res) => {
// Only process if in production environment. // Only process if in production environment.
if (process.env.NODE_ENV !== "production") { if (process.env.NODE_ENV !== "production") {
@@ -54,12 +54,13 @@ exports.default = async (req, res) => {
try { try {
logger.log("claimscorp-start", "DEBUG", "api", null, null); logger.log("claimscorp-start", "DEBUG", "api", null, null);
const allXMLResults = [];
const allErrors = [];
const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid]; const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess = const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -68,27 +69,18 @@ exports.default = async (req, res) => {
logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
return; return;
} }
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
if (skipUpload) { await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
} else {
await uploadViaSFTP(allxmlsToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({ await sendServerEmail({
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), allXMLResults.map((x) => ({
imexshopid: x.imexshopid,
filename: x.filename,
count: x.count,
result: x.result
})),
null, null,
2 2
)}` )}`
@@ -100,8 +92,8 @@ exports.default = async (req, res) => {
} }
}; };
async function processBatch(batch, start, end) { async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
for (const bodyshop of batch) { for (const bodyshop of shopsToProcess) {
const erroredJobs = []; const erroredJobs = [];
try { try {
logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -135,12 +127,27 @@ async function processBatch(batch, start, end) {
}); });
} }
const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); const xmlObj = {
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }),
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
count: claimsCorpObject.DataFeed.ShopInfo.RO.length
};
allxmlsToUpload.push({ if (skipUpload) {
count: claimsCorpObject.DataFeed.ShopInfo.RO.length, fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
xml: ret, } else {
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` await uploadViaSFTP(xmlObj);
}
allXMLResults.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
claimscorpid: bodyshop.claimscorpid,
count: xmlObj.count,
filename: xmlObj.filename,
result: xmlObj.result
}); });
logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -172,33 +179,38 @@ async function processBatch(batch, start, end) {
} }
} }
async function uploadViaSFTP(allxmlsToUpload) { async function uploadViaSFTP(xmlObj) {
const sftp = new Client(); const sftp = new Client();
sftp.on("error", (errors) => sftp.on("error", (errors) =>
logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
error: errors.message,
stack: errors.stack
})
); );
try { try {
//Connect to the FTP and upload all. //Connect to the FTP and upload all.
await sftp.connect(ftpSetup); await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) { try {
try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { imexshopid: xmlObj.imexshopid,
filename: xmlObj.filename, filename: xmlObj.filename,
result: xmlObj.result result: xmlObj.result
}); });
} catch (error) { } catch (error) {
logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
filename: xmlObj.filename, filename: xmlObj.filename,
error: error.message, error: error.message,
stack: error.stack stack: error.stack
}); });
throw error; throw error;
}
} }
} catch (error) { } catch (error) {
logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, {
error: error.message,
stack: error.stack
});
throw error; throw error;
} finally { } finally {
sftp.end(); sftp.end();

104
server/data/kaizen.js
View File

@@ -23,15 +23,15 @@ const ftpSetup = {
port: process.env.KAIZEN_PORT, port: process.env.KAIZEN_PORT,
username: process.env.KAIZEN_USER, username: process.env.KAIZEN_USER,
password: process.env.KAIZEN_PASSWORD, password: process.env.KAIZEN_PASSWORD,
debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), debug:
process.env.NODE_ENV !== "production"
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
: () => {},
algorithms: { algorithms: {
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
} }
}; };
const allxmlsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => { exports.default = async (req, res) => {
// Only process if in production environment. // Only process if in production environment.
if (process.env.NODE_ENV !== "production") { if (process.env.NODE_ENV !== "production") {
@@ -53,12 +53,13 @@ exports.default = async (req, res) => {
try { try {
logger.log("kaizen-start", "DEBUG", "api", null, null); logger.log("kaizen-start", "DEBUG", "api", null, null);
const allXMLResults = [];
const allErrors = [];
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid]; const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess = const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
@@ -67,27 +68,18 @@ exports.default = async (req, res) => {
logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
return; return;
} }
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
if (skipUpload) { await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
} else {
await uploadViaSFTP(allxmlsToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({ await sendServerEmail({
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), allXMLResults.map((x) => ({
imexshopid: x.imexshopid,
filename: x.filename,
count: x.count,
result: x.result
})),
null, null,
2 2
)}` )}`
@@ -99,8 +91,8 @@ exports.default = async (req, res) => {
} }
}; };
async function processBatch(batch, start, end) { async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
for (const bodyshop of batch) { for (const bodyshop of shopsToProcess) {
const erroredJobs = []; const erroredJobs = [];
try { try {
logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -133,12 +125,26 @@ async function processBatch(batch, start, end) {
}); });
} }
const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); const xmlObj = {
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }),
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
count: kaizenObject.DataFeed.ShopInfo.Jobs.length
};
allxmlsToUpload.push({ if (skipUpload) {
count: kaizenObject.DataFeed.ShopInfo.Jobs.length, fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
xml: ret, } else {
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` await uploadViaSFTP(xmlObj);
}
allXMLResults.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
count: xmlObj.count,
filename: xmlObj.filename,
result: xmlObj.result
}); });
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
@@ -170,33 +176,35 @@ async function processBatch(batch, start, end) {
} }
} }
async function uploadViaSFTP(allxmlsToUpload) { async function uploadViaSFTP(xmlObj) {
const sftp = new Client(); const sftp = new Client();
sftp.on("error", (errors) => sftp.on("error", (errors) =>
logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
error: errors.message,
stack: errors.stack
})
); );
try { try {
//Connect to the FTP and upload all. //Connect to the FTP and upload all.
await sftp.connect(ftpSetup); await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) { try {
try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { imexshopid: xmlObj.imexshopid,
filename: xmlObj.filename, filename: xmlObj.filename,
result: xmlObj.result result: xmlObj.result
}); });
} catch (error) { } catch (error) {
logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
filename: xmlObj.filename, filename: xmlObj.filename,
error: error.message, error: error.message,
stack: error.stack stack: error.stack
}); });
throw error; throw error;
}
} }
} catch (error) { } catch (error) {
logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
throw error; throw error;
} finally { } finally {
sftp.end(); sftp.end();