From ce9a77efcfc53273a9817bca5bf6ae9beff68ba2 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 14 Nov 2024 16:15:17 -0800 Subject: [PATCH 1/2] IO-3027 Datapumps Refactor Signed-off-by: Allan Carr --- .vscode/settings.json | 30 ++++ server/data/autohouse.js | 15 +- server/data/chatter.js | 6 +- server/data/claimscorp.js | 294 +++++++++++++++++++------------------ server/data/kaizen.js | 297 +++++++++++++++++++------------------- 5 files changed, 344 insertions(+), 298 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index dd6a1330b..74b58f715 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,35 @@ "pattern": "**/IMEX.xml", "systemId": "logs/IMEX.xsd" } + ], + "cSpell.words": [ + "antd", + "appointmentconfirmation", + "appt", + "autohouse", + "autohouseid", + "billlines", + "bodyshop", + "bodyshopid", + "bodyshops", + "CIECA", + "claimscorp", + "claimscorpid", + "Dinero", + "driveable", + "IMEX", + "imexshopid", + "jobid", + "joblines", + "Kaizen", + "labhrs", + "larhrs", + "mixdata", + "ownr", + "promanager", + "shopname", + "smartscheduling", + "timetickets", + "touchtime" ] } diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 9ea3a415a..161fd3b99 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -47,7 +47,11 @@ exports.default = async (req, res) => { } // Send immediate response and continue processing. - res.status(200).send(); + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); try { logger.log("autohouse-start", "DEBUG", "api", null, null); @@ -146,7 +150,7 @@ async function processBatch(batch, start, end) { allErrors.push({ bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, - autuhouseid: bodyshop.autuhouseid, + autohouseid: bodyshop.autohouseid, fatal: true, errors: [error.toString()] }); @@ -154,7 +158,7 @@ async function processBatch(batch, start, end) { allErrors.push({ bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, - autuhouseid: bodyshop.autuhouseid, + autohouseid: bodyshop.autohouseid, errors: erroredJobs.map((ej) => ({ ro_number: ej.job?.ro_number, jobid: ej.job?.id, @@ -609,10 +613,7 @@ const CreateRepairOrderTag = (job, errorCallback) => { }; return ret; } catch (error) { - logger.log("autohouse-job-calculate-error", "ERROR", "api", null, { - error - }); - + logger.log("autohouse-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); } }; diff --git a/server/data/chatter.js b/server/data/chatter.js index ea8851e94..ea09d3862 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -39,7 +39,11 @@ exports.default = async (req, res) => { } // Send immediate response and continue processing. - res.status(200).send(); + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); try { logger.log("chatter-start", "DEBUG", "api", null, null); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index 6ebc63f81..4304b75b5 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -26,174 +26,185 @@ const ftpSetup = { password: process.env.CLAIMSCORP_PASSWORD, debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), algorithms: { - serverHostKey: ["ssh-rsa", "ssh-dss"] + serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; +const allxmlsToUpload = []; +const allErrors = []; + exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { res.sendStatus(403); return; } - - //Query for the List of Bodyshop Clients. - logger.log("claimscorp-start", "DEBUG", "api", null, null); - const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); - - const specificShopIds = req.body.bodyshopIds; // ['uuid] - const { start, end, skipUpload } = req.body; //YYYY-MM-DD + // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { res.sendStatus(401); return; } - const allxmlsToUpload = []; - const allErrors = []; + + // Send immediate response and continue processing. + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); + try { - for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) { + logger.log("claimscorp-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; + const { start, end, skipUpload } = req.body; //YYYY-MM-DD + + const batchSize = 10; + + const shopsToProcess = + specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; + logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); + + if (shopsToProcess.length === 0) { + logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); + return; + } + const batchPromises = []; + for (let i = 0; i < shopsToProcess.length; i += batchSize) { + const batch = shopsToProcess.slice(i, i + batchSize); + const batchPromise = (async () => { + await processBatch(batch, start, end); + + if (skipUpload) { + for (const xmlObj of allxmlsToUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } + } else { + await uploadViaSFTP(allxmlsToUpload); + } + })(); + batchPromises.push(batchPromise); + } + await Promise.all(batchPromises); + await sendServerEmail({ + subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( + allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + null, + 2 + )}` + }); + + logger.log("claimscorp-end", "DEBUG", "api", null, null); + } catch (error) { + logger.log("claimscorp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + } +}; + +async function processBatch(batch, start, end) { + for (const bodyshop of batch) { + const erroredJobs = []; + try { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); - const erroredJobs = []; - try { - const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, { - bodyshopid: bodyshop.id, - start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }); - const claimsCorpObject = { - DataFeed: { - ShopInfo: { - ShopID: bodyshops_by_pk.claimscorpid, - ShopName: bodyshops_by_pk.shopname, - RO: jobs.map((j) => - CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { - erroredJobs.push({ job: job, error: error.toString() }); - }) - ) - } - } - }; - - if (erroredJobs.length > 0) { - logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, { - count: erroredJobs.length, - jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) - }); - } - - var ret = builder - .create( - { - // version: "1.0", - // encoding: "UTF-8", - //keepNullNodes: true, - }, - claimsCorpObject - ) - .end({ allowEmptyTags: true }); - - allxmlsToUpload.push({ - count: claimsCorpObject.DataFeed.ShopInfo.RO.length, - xml: ret, - filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` - }); - - logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { - shopname: bodyshop.shopname - }); - } catch (error) { - //Error at the shop level. - logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, { - ...error - }); - - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - claimscorpid: bodyshop.claimscorpid, - fatal: true, - errors: [error.toString()] - }); - } finally { - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - claimscorpid: bodyshop.claimscorpid, - errors: erroredJobs.map((ej) => ({ - ro_number: ej.job?.ro_number, - jobid: ej.job?.id, - error: ej.error - })) - }); - } - } - - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - - res.json(allxmlsToUpload); - sendServerEmail({ - subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )} - ` + const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, { + bodyshopid: bodyshop.id, + start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) }); - return; - } - let sftp = new Client(); - sftp.on("error", (errors) => - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { - ...errors - }) - ); - try { - //Connect to the FTP and upload all. + const claimsCorpObject = { + DataFeed: { + ShopInfo: { + ShopID: bodyshops_by_pk.claimscorpid, + ShopName: bodyshops_by_pk.shopname, + RO: jobs.map((j) => + CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { + erroredJobs.push({ job: job, error: error.toString() }); + }) + ) + } + } + }; - await sftp.connect(ftpSetup); - - for (const xmlObj of allxmlsToUpload) { - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename - }); - - const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, { - uploadResult + if (erroredJobs.length > 0) { + logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, { + count: erroredJobs.length, + jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) }); } - //***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml + const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); + + allxmlsToUpload.push({ + count: claimsCorpObject.DataFeed.ShopInfo.RO.length, + xml: ret, + filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` + }); + + logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); } catch (error) { - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { - ...error + //Error at the shop level. + logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + fatal: true, + errors: [error.toString()] }); } finally { - sftp.end(); + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + errors: erroredJobs.map((ej) => ({ + ro_number: ej.job?.ro_number, + jobid: ej.job?.id, + error: ej.error + })) + }); } - sendServerEmail({ - subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )} - ` - }); - res.sendStatus(200); - } catch (error) { - res.status(200).json(error); } -}; +} + +async function uploadViaSFTP(allxmlsToUpload) { + const sftp = new Client(); + sftp.on("error", (errors) => + logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + ); + try { + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + for (const xmlObj of allxmlsToUpload) { + try { + logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename }); + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, { + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; + } + } + } catch (error) { + logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + throw error; + } finally { + sftp.end(); + } +} const CreateRepairOrderTag = (job, errorCallback) => { //Level 2 @@ -445,10 +456,7 @@ const CreateRepairOrderTag = (job, errorCallback) => { }; return ret; } catch (error) { - logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, { - error - }); - + logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); } }; diff --git a/server/data/kaizen.js b/server/data/kaizen.js index c9794acff..f48bb637d 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -16,8 +16,7 @@ const { sendServerEmail } = require("../email/sendemail"); const DineroFormat = "0,0.00"; const DateFormat = "MM/DD/YYYY"; -const repairOpCodes = ["OP4", "OP9", "OP10"]; -const replaceOpCodes = ["OP2", "OP5", "OP11", "OP12"]; +const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"]; const ftpSetup = { host: process.env.KAIZEN_HOST, @@ -30,173 +29,180 @@ const ftpSetup = { } }; +const allxmlsToUpload = []; +const allErrors = []; + exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { res.sendStatus(403); return; } - - //Query for the List of Bodyshop Clients. - logger.log("kaizen-start", "DEBUG", "api", null, null); - const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"]; - - const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { - imexshopid: kaizenShopsIDs - }); - - const specificShopIds = req.body.bodyshopIds; // ['uuid] - const { start, end, skipUpload } = req.body; //YYYY-MM-DD + // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { res.sendStatus(401); return; } - const allxmlsToUpload = []; - const allErrors = []; + + // Send immediate response and continue processing. + res.status(202).json({ + success: true, + message: "Processing request ...", + timestamp: new Date().toISOString() + }); + try { - for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) { + logger.log("kaizen-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; + const { start, end, skipUpload } = req.body; //YYYY-MM-DD + + const batchSize = 10; + + const shopsToProcess = + specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; + logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); + + if (shopsToProcess.length === 0) { + logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); + return; + } + const batchPromises = []; + for (let i = 0; i < shopsToProcess.length; i += batchSize) { + const batch = shopsToProcess.slice(i, i + batchSize); + const batchPromise = (async () => { + await processBatch(batch, start, end); + + if (skipUpload) { + for (const xmlObj of allxmlsToUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } + } else { + await uploadViaSFTP(allxmlsToUpload); + } + })(); + batchPromises.push(batchPromise); + } + await Promise.all(batchPromises); + await sendServerEmail({ + subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( + allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + null, + 2 + )}` + }); + + logger.log("kaizen-end", "DEBUG", "api", null, null); + } catch (error) { + logger.log("kaizen-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + } +}; + +async function processBatch(batch, start, end) { + for (const bodyshop of batch) { + const erroredJobs = []; + try { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); - const erroredJobs = []; - try { - const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, { - bodyshopid: bodyshop.id, - start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"), - ...(end && { end: moment(end).endOf("day") }) - }); - const kaizenObject = { - DataFeed: { - ShopInfo: { - ShopName: bodyshops_by_pk.shopname, - Jobs: jobs.map((j) => - CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { - erroredJobs.push({ job: job, error: error.toString() }); - }) - ) - } - } - }; - - if (erroredJobs.length > 0) { - logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, { - count: erroredJobs.length, - jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) - }); - } - - var ret = builder - .create( - { - // version: "1.0", - // encoding: "UTF-8", - //keepNullNodes: true, - }, - kaizenObject - ) - .end({ allowEmptyTags: true }); - - allxmlsToUpload.push({ - count: kaizenObject.DataFeed.ShopInfo.Jobs.length, - xml: ret, - filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` - }); - - logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { - shopname: bodyshop.shopname - }); - } catch (error) { - //Error at the shop level. - logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { - ...error - }); - - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - shopname: bodyshop.shopname, - fatal: true, - errors: [error.toString()] - }); - } finally { - allErrors.push({ - bodyshopid: bodyshop.id, - imexshopid: bodyshop.imexshopid, - shopname: bodyshop.shopname, - errors: erroredJobs.map((ej) => ({ - ro_number: ej.job?.ro_number, - jobid: ej.job?.id, - error: ej.error - })) - }); - } - } - - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - - res.json(allxmlsToUpload); - sendServerEmail({ - subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )} - ` + const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, { + bodyshopid: bodyshop.id, + start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) }); - return; - } - let sftp = new Client(); - sftp.on("error", (errors) => - logger.log("kaizen-sftp-error", "ERROR", "api", null, { - ...errors - }) - ); - try { - //Connect to the FTP and upload all. + const kaizenObject = { + DataFeed: { + ShopInfo: { + ShopName: bodyshops_by_pk.shopname, + Jobs: jobs.map((j) => + CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) { + erroredJobs.push({ job: job, error: error.toString() }); + }) + ) + } + } + }; - await sftp.connect(ftpSetup); - - for (const xmlObj of allxmlsToUpload) { - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename - }); - - const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`); - logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, { - uploadResult + if (erroredJobs.length > 0) { + logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, { + count: erroredJobs.length, + jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number)) }); } - //***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml + const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); + + allxmlsToUpload.push({ + count: kaizenObject.DataFeed.ShopInfo.Jobs.length, + xml: ret, + filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` + }); + + logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); } catch (error) { - logger.log("kaizen-sftp-error", "ERROR", "api", null, { - ...error + //Error at the shop level. + logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname, + fatal: true, + errors: [error.toString()] }); } finally { - sftp.end(); + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname, + errors: erroredJobs.map((ej) => ({ + ro_number: ej.job?.ro_number, + jobid: ej.job?.id, + error: ej.error + })) + }); } - sendServerEmail({ - subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })), - null, - 2 - )} - ` - }); - res.sendStatus(200); - } catch (error) { - res.status(200).json(error); } -}; +} + +async function uploadViaSFTP(allxmlsToUpload) { + const sftp = new Client(); + sftp.on("error", (errors) => + logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + ); + try { + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + for (const xmlObj of allxmlsToUpload) { + try { + logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename }); + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, { + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; + } + } + } catch (error) { + logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + throw error; + } finally { + sftp.end(); + } +} const CreateRepairOrderTag = (job, errorCallback) => { //Level 2 @@ -420,10 +426,7 @@ const CreateRepairOrderTag = (job, errorCallback) => { }; return ret; } catch (error) { - logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { - error - }); - + logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); errorCallback({ jobid: job.id, ro_number: job.ro_number, error }); } }; From 78678dd3dcf2c7802aa73e5825610ac26554091c Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Fri, 15 Nov 2024 10:04:03 -0800 Subject: [PATCH 2/2] IO-3027 Datapumps Refactor Signed-off-by: Allan Carr --- server/data/autohouse.js | 3 +-- server/data/chatter.js | 3 +-- server/data/claimscorp.js | 3 +-- server/data/kaizen.js | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 161fd3b99..7d8447f68 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -180,9 +180,8 @@ async function uploadViaSFTP(allxmlsToUpload) { for (const xmlObj of allxmlsToUpload) { try { - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename }); xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload-result", "DEBUG", "api", null, { + logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename, result: xmlObj.result }); diff --git a/server/data/chatter.js b/server/data/chatter.js index ea09d3862..8890fafe9 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -180,9 +180,8 @@ async function uploadViaSFTP(allcsvsToUpload) { for (const csvObj of allcsvsToUpload) { try { - logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename }); csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); - logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename, result: csvObj.result }); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index 4304b75b5..cafa03df1 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -183,9 +183,8 @@ async function uploadViaSFTP(allxmlsToUpload) { for (const xmlObj of allxmlsToUpload) { try { - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename }); xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, { + logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename, result: xmlObj.result }); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index f48bb637d..b79fe4745 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -181,9 +181,8 @@ async function uploadViaSFTP(allxmlsToUpload) { for (const xmlObj of allxmlsToUpload) { try { - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename }); xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, { + logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename, result: xmlObj.result });