From 8ad1dd83c6e21936580475a616ace4d875679292 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 14:24:31 -0800 Subject: [PATCH 1/2] IO-3027 Datapump Refactor Remove Batch and sftp transfer for each shop during processing Signed-off-by: Allan Carr --- server/data/autohouse.js | 99 ++++++++++++++++++++------------------- server/data/claimscorp.js | 99 ++++++++++++++++++++------------------- server/data/kaizen.js | 98 +++++++++++++++++++------------------- 3 files changed, 152 insertions(+), 144 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 1384771e6..19078c274 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,17 +25,15 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -57,12 +55,13 @@ exports.default = async (req, res) => { try { logger.log("autohouse-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -71,27 +70,18 @@ exports.default = async (req, res) => { logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -103,8 +93,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -134,12 +124,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), + filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, + count: autoHouseObject.AutoHouseExport.RepairOrder.length + }; - allxmlsToUpload.push({ - count: autoHouseObject.AutoHouseExport.RepairOrder.length, - xml: ret, - filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + autohouseid: bodyshop.autohouseid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -171,7 +175,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -180,21 +184,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index b8c77a018..3db09cbcf 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,17 +24,15 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -56,12 +54,13 @@ exports.default = async (req, res) => { try { logger.log("claimscorp-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -70,27 +69,18 @@ exports.default = async (req, res) => { logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -102,8 +92,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -137,12 +127,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: claimsCorpObject.DataFeed.ShopInfo.RO.length + }; - allxmlsToUpload.push({ - count: claimsCorpObject.DataFeed.ShopInfo.RO.length, - xml: ret, - filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -174,7 +178,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -183,21 +187,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index bbb758a93..eb6abc056 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,17 +23,15 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -55,12 +53,13 @@ exports.default = async (req, res) => { try { logger.log("kaizen-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -69,27 +68,18 @@ exports.default = async (req, res) => { logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -101,8 +91,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -135,12 +125,25 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); + const xmlObj = { + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: kaizenObject.DataFeed.ShopInfo.Jobs.length + }; - allxmlsToUpload.push({ - count: kaizenObject.DataFeed.ShopInfo.Jobs.length, - xml: ret, - filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -172,7 +175,7 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) @@ -181,21 +184,20 @@ async function uploadViaSFTP(allxmlsToUpload) { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); From e202bf9a8966626cdf050d96c94ded2ab2a17e87 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Thu, 21 Nov 2024 16:18:23 -0800 Subject: [PATCH 2/2] IO-3027 Add in bodyshop.id to logging in SFTP Signed-off-by: Allan Carr --- server/data/autohouse.js | 12 ++++++++---- server/data/claimscorp.js | 15 +++++++++++---- server/data/kaizen.js | 12 ++++++++---- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 19078c274..81679522b 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -125,6 +125,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, @@ -178,7 +179,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -186,13 +190,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { + logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -200,7 +204,7 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index 3db09cbcf..70737bd03 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -128,6 +128,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, @@ -181,7 +182,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -189,13 +193,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { + logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -203,7 +207,10 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { + error: error.message, + stack: error.stack + }); throw error; } finally { sftp.end(); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index eb6abc056..a7a6aaf91 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -126,6 +126,7 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes } const xmlObj = { + bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, @@ -178,7 +179,10 @@ async function processShopData(shopsToProcess, start, end, skipUpload, allXMLRes async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. @@ -186,13 +190,13 @@ async function uploadViaSFTP(xmlObj) { try { xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { + logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { imexshopid: xmlObj.imexshopid, filename: xmlObj.filename, result: xmlObj.result }); } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { filename: xmlObj.filename, error: error.message, stack: error.stack @@ -200,7 +204,7 @@ async function uploadViaSFTP(xmlObj) { throw error; } } catch (error) { - logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end();