diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 1384771e6..81679522b 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -25,17 +25,15 @@ const ftpSetup = { port: process.env.AUTOHOUSE_PORT, username: process.env.AUTOHOUSE_USER, password: process.env.AUTOHOUSE_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -57,12 +55,13 @@ exports.default = async (req, res) => { try { logger.log("autohouse-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -71,27 +70,18 @@ exports.default = async (req, res) => { logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -103,8 +93,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -134,12 +124,27 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }), + filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`, + count: autoHouseObject.AutoHouseExport.RepairOrder.length + }; - allxmlsToUpload.push({ - count: autoHouseObject.AutoHouseExport.RepairOrder.length, - xml: ret, - filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + autohouseid: bodyshop.autohouseid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -171,33 +176,35 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("autohouse-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); diff --git a/server/data/claimscorp.js b/server/data/claimscorp.js index b8c77a018..70737bd03 100644 --- a/server/data/claimscorp.js +++ b/server/data/claimscorp.js @@ -24,17 +24,15 @@ const ftpSetup = { port: process.env.CLAIMSCORP_PORT, username: process.env.CLAIMSCORP_USER, password: process.env.CLAIMSCORP_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -56,12 +54,13 @@ exports.default = async (req, res) => { try { logger.log("claimscorp-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -70,27 +69,18 @@ exports.default = async (req, res) => { logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -102,8 +92,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -137,12 +127,27 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: claimsCorpObject.DataFeed.ShopInfo.RO.length + }; - allxmlsToUpload.push({ - count: claimsCorpObject.DataFeed.ShopInfo.RO.length, - xml: ret, - filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + claimscorpid: bodyshop.claimscorpid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -174,33 +179,38 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { + error: error.message, + stack: error.stack + }); throw error; } finally { sftp.end(); diff --git a/server/data/kaizen.js b/server/data/kaizen.js index bbb758a93..a7a6aaf91 100644 --- a/server/data/kaizen.js +++ b/server/data/kaizen.js @@ -23,17 +23,15 @@ const ftpSetup = { port: process.env.KAIZEN_PORT, username: process.env.KAIZEN_USER, password: process.env.KAIZEN_PASSWORD, - debug: process.env.NODE_ENV !== "production" - ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) - : () => {}, + debug: + process.env.NODE_ENV !== "production" + ? (message, ...data) => logger.log(message, "DEBUG", "api", null, data) + : () => {}, algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; -const allxmlsToUpload = []; -const allErrors = []; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -55,12 +53,13 @@ exports.default = async (req, res) => { try { logger.log("kaizen-start", "DEBUG", "api", null, null); + const allXMLResults = []; + const allErrors = []; + const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD - const batchSize = 10; - const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null); @@ -69,27 +68,18 @@ exports.default = async (req, res) => { logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null); return; } - const batchPromises = []; - for (let i = 0; i < shopsToProcess.length; i += batchSize) { - const batch = shopsToProcess.slice(i, i + batchSize); - const batchPromise = (async () => { - await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); - } - } else { - await uploadViaSFTP(allxmlsToUpload); - } - })(); - batchPromises.push(batchPromise); - } - await Promise.all(batchPromises); + await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors); + await sendServerEmail({ subject: `Kaizen Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( - allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), + allXMLResults.map((x) => ({ + imexshopid: x.imexshopid, + filename: x.filename, + count: x.count, + result: x.result + })), null, 2 )}` @@ -101,8 +91,8 @@ exports.default = async (req, res) => { } }; -async function processBatch(batch, start, end) { - for (const bodyshop of batch) { +async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) { + for (const bodyshop of shopsToProcess) { const erroredJobs = []; try { logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -135,12 +125,26 @@ async function processBatch(batch, start, end) { }); } - const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true }); + const xmlObj = { + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }), + filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`, + count: kaizenObject.DataFeed.ShopInfo.Jobs.length + }; - allxmlsToUpload.push({ - count: kaizenObject.DataFeed.ShopInfo.Jobs.length, - xml: ret, - filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml` + if (skipUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } else { + await uploadViaSFTP(xmlObj); + } + + allXMLResults.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + count: xmlObj.count, + filename: xmlObj.filename, + result: xmlObj.result }); logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, { @@ -172,33 +176,35 @@ async function processBatch(batch, start, end) { } } -async function uploadViaSFTP(allxmlsToUpload) { +async function uploadViaSFTP(xmlObj) { const sftp = new Client(); sftp.on("error", (errors) => - logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) + logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, { + error: errors.message, + stack: errors.stack + }) ); try { //Connect to the FTP and upload all. await sftp.connect(ftpSetup); - for (const xmlObj of allxmlsToUpload) { - try { - xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); - logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { - filename: xmlObj.filename, - result: xmlObj.result - }); - } catch (error) { - logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, { - filename: xmlObj.filename, - error: error.message, - stack: error.stack - }); - throw error; - } + try { + xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`); + logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, { + imexshopid: xmlObj.imexshopid, + filename: xmlObj.filename, + result: xmlObj.result + }); + } catch (error) { + logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, { + filename: xmlObj.filename, + error: error.message, + stack: error.stack + }); + throw error; } } catch (error) { - logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); + logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end();