IO-3027 Datapumps Refactor
Signed-off-by: Allan Carr <allan.carr@thinkimex.com>
This commit is contained in:
Allan Carr
@@ -16,8 +16,7 @@ const { sendServerEmail } = require("../email/sendemail");
|
||||
const DineroFormat = "0,0.00";
|
||||
const DateFormat = "MM/DD/YYYY";
|
||||
|
||||
const repairOpCodes = ["OP4", "OP9", "OP10"];
|
||||
const replaceOpCodes = ["OP2", "OP5", "OP11", "OP12"];
|
||||
const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
|
||||
|
||||
const ftpSetup = {
|
||||
host: process.env.KAIZEN_HOST,
|
||||
@@ -30,173 +29,180 @@ const ftpSetup = {
|
||||
}
|
||||
};
|
||||
|
||||
const allxmlsToUpload = [];
|
||||
const allErrors = [];
|
||||
|
||||
exports.default = async (req, res) => {
|
||||
// Only process if in production environment.
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
res.sendStatus(403);
|
||||
return;
|
||||
}
|
||||
|
||||
//Query for the List of Bodyshop Clients.
|
||||
logger.log("kaizen-start", "DEBUG", "api", null, null);
|
||||
const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
|
||||
|
||||
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, {
|
||||
imexshopid: kaizenShopsIDs
|
||||
});
|
||||
|
||||
const specificShopIds = req.body.bodyshopIds; // ['uuid]
|
||||
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
||||
// Only process if the appropriate token is provided.
|
||||
if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
|
||||
res.sendStatus(401);
|
||||
return;
|
||||
}
|
||||
const allxmlsToUpload = [];
|
||||
const allErrors = [];
|
||||
|
||||
// Send immediate response and continue processing.
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
message: "Processing request ...",
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
try {
|
||||
for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
|
||||
logger.log("kaizen-start", "DEBUG", "api", null, null);
|
||||
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
|
||||
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
||||
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
||||
|
||||
const batchSize = 10;
|
||||
|
||||
const shopsToProcess =
|
||||
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
||||
logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
|
||||
|
||||
if (shopsToProcess.length === 0) {
|
||||
logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
|
||||
return;
|
||||
}
|
||||
const batchPromises = [];
|
||||
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
|
||||
const batch = shopsToProcess.slice(i, i + batchSize);
|
||||
const batchPromise = (async () => {
|
||||
await processBatch(batch, start, end);
|
||||
|
||||
if (skipUpload) {
|
||||
for (const xmlObj of allxmlsToUpload) {
|
||||
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
||||
}
|
||||
} else {
|
||||
await uploadViaSFTP(allxmlsToUpload);
|
||||
}
|
||||
})();
|
||||
batchPromises.push(batchPromise);
|
||||
}
|
||||
await Promise.all(batchPromises);
|
||||
await sendServerEmail({
|
||||
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
|
||||
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
});
|
||||
|
||||
logger.log("kaizen-end", "DEBUG", "api", null, null);
|
||||
} catch (error) {
|
||||
logger.log("kaizen-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
async function processBatch(batch, start, end) {
|
||||
for (const bodyshop of batch) {
|
||||
const erroredJobs = [];
|
||||
try {
|
||||
logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||
shopname: bodyshop.shopname
|
||||
});
|
||||
const erroredJobs = [];
|
||||
try {
|
||||
const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
|
||||
bodyshopid: bodyshop.id,
|
||||
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
|
||||
...(end && { end: moment(end).endOf("day") })
|
||||
});
|
||||
|
||||
const kaizenObject = {
|
||||
DataFeed: {
|
||||
ShopInfo: {
|
||||
ShopName: bodyshops_by_pk.shopname,
|
||||
Jobs: jobs.map((j) =>
|
||||
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
|
||||
erroredJobs.push({ job: job, error: error.toString() });
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (erroredJobs.length > 0) {
|
||||
logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
|
||||
count: erroredJobs.length,
|
||||
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
|
||||
});
|
||||
}
|
||||
|
||||
var ret = builder
|
||||
.create(
|
||||
{
|
||||
// version: "1.0",
|
||||
// encoding: "UTF-8",
|
||||
//keepNullNodes: true,
|
||||
},
|
||||
kaizenObject
|
||||
)
|
||||
.end({ allowEmptyTags: true });
|
||||
|
||||
allxmlsToUpload.push({
|
||||
count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
|
||||
xml: ret,
|
||||
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
|
||||
});
|
||||
|
||||
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||
shopname: bodyshop.shopname
|
||||
});
|
||||
} catch (error) {
|
||||
//Error at the shop level.
|
||||
logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, {
|
||||
...error
|
||||
});
|
||||
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
shopname: bodyshop.shopname,
|
||||
fatal: true,
|
||||
errors: [error.toString()]
|
||||
});
|
||||
} finally {
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
shopname: bodyshop.shopname,
|
||||
errors: erroredJobs.map((ej) => ({
|
||||
ro_number: ej.job?.ro_number,
|
||||
jobid: ej.job?.id,
|
||||
error: ej.error
|
||||
}))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (skipUpload) {
|
||||
for (const xmlObj of allxmlsToUpload) {
|
||||
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
||||
}
|
||||
|
||||
res.json(allxmlsToUpload);
|
||||
sendServerEmail({
|
||||
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
|
||||
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
|
||||
Uploaded: ${JSON.stringify(
|
||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
|
||||
null,
|
||||
2
|
||||
)}
|
||||
`
|
||||
const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
|
||||
bodyshopid: bodyshop.id,
|
||||
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
|
||||
...(end && { end: moment(end).endOf("day") })
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let sftp = new Client();
|
||||
sftp.on("error", (errors) =>
|
||||
logger.log("kaizen-sftp-error", "ERROR", "api", null, {
|
||||
...errors
|
||||
})
|
||||
);
|
||||
try {
|
||||
//Connect to the FTP and upload all.
|
||||
const kaizenObject = {
|
||||
DataFeed: {
|
||||
ShopInfo: {
|
||||
ShopName: bodyshops_by_pk.shopname,
|
||||
Jobs: jobs.map((j) =>
|
||||
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
|
||||
erroredJobs.push({ job: job, error: error.toString() });
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await sftp.connect(ftpSetup);
|
||||
|
||||
for (const xmlObj of allxmlsToUpload) {
|
||||
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
|
||||
filename: xmlObj.filename
|
||||
});
|
||||
|
||||
const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
|
||||
logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
|
||||
uploadResult
|
||||
if (erroredJobs.length > 0) {
|
||||
logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
|
||||
count: erroredJobs.length,
|
||||
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
|
||||
});
|
||||
}
|
||||
|
||||
//***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
|
||||
const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
|
||||
|
||||
allxmlsToUpload.push({
|
||||
count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
|
||||
xml: ret,
|
||||
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
|
||||
});
|
||||
|
||||
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||
shopname: bodyshop.shopname
|
||||
});
|
||||
} catch (error) {
|
||||
logger.log("kaizen-sftp-error", "ERROR", "api", null, {
|
||||
...error
|
||||
//Error at the shop level.
|
||||
logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
|
||||
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
shopname: bodyshop.shopname,
|
||||
fatal: true,
|
||||
errors: [error.toString()]
|
||||
});
|
||||
} finally {
|
||||
sftp.end();
|
||||
allErrors.push({
|
||||
bodyshopid: bodyshop.id,
|
||||
imexshopid: bodyshop.imexshopid,
|
||||
shopname: bodyshop.shopname,
|
||||
errors: erroredJobs.map((ej) => ({
|
||||
ro_number: ej.job?.ro_number,
|
||||
jobid: ej.job?.id,
|
||||
error: ej.error
|
||||
}))
|
||||
});
|
||||
}
|
||||
sendServerEmail({
|
||||
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
|
||||
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
|
||||
Uploaded: ${JSON.stringify(
|
||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
|
||||
null,
|
||||
2
|
||||
)}
|
||||
`
|
||||
});
|
||||
res.sendStatus(200);
|
||||
} catch (error) {
|
||||
res.status(200).json(error);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadViaSFTP(allxmlsToUpload) {
|
||||
const sftp = new Client();
|
||||
sftp.on("error", (errors) =>
|
||||
logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
|
||||
);
|
||||
try {
|
||||
//Connect to the FTP and upload all.
|
||||
await sftp.connect(ftpSetup);
|
||||
|
||||
for (const xmlObj of allxmlsToUpload) {
|
||||
try {
|
||||
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
|
||||
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
||||
logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
|
||||
filename: xmlObj.filename,
|
||||
result: xmlObj.result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
|
||||
filename: xmlObj.filename,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||
throw error;
|
||||
} finally {
|
||||
sftp.end();
|
||||
}
|
||||
}
|
||||
|
||||
const CreateRepairOrderTag = (job, errorCallback) => {
|
||||
//Level 2
|
||||
@@ -420,10 +426,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
|
||||
};
|
||||
return ret;
|
||||
} catch (error) {
|
||||
logger.log("kaizen-job-calculate-error", "ERROR", "api", null, {
|
||||
error
|
||||
});
|
||||
|
||||
logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user