IO-3027 Datapumps Refactor

Signed-off-by: Allan Carr <allan.carr@thinkimex.com>
This commit is contained in:
Allan Carr
2024-11-14 16:15:17 -08:00
parent 357d916e0a
commit ce9a77efcf
5 changed files with 344 additions and 298 deletions

View File

@@ -47,7 +47,11 @@ exports.default = async (req, res) => {
}
// Send immediate response and continue processing.
res.status(200).send();
res.status(202).json({
success: true,
message: "Processing request ...",
timestamp: new Date().toISOString()
});
try {
logger.log("autohouse-start", "DEBUG", "api", null, null);
@@ -146,7 +150,7 @@ async function processBatch(batch, start, end) {
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
autuhouseid: bodyshop.autuhouseid,
autohouseid: bodyshop.autohouseid,
fatal: true,
errors: [error.toString()]
});
@@ -154,7 +158,7 @@ async function processBatch(batch, start, end) {
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
autuhouseid: bodyshop.autuhouseid,
autohouseid: bodyshop.autohouseid,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
@@ -609,10 +613,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
};
return ret;
} catch (error) {
logger.log("autohouse-job-calculate-error", "ERROR", "api", null, {
error
});
logger.log("autohouse-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
}
};

View File

@@ -39,7 +39,11 @@ exports.default = async (req, res) => {
}
// Send immediate response and continue processing.
res.status(200).send();
res.status(202).json({
success: true,
message: "Processing request ...",
timestamp: new Date().toISOString()
});
try {
logger.log("chatter-start", "DEBUG", "api", null, null);

View File

@@ -26,174 +26,185 @@ const ftpSetup = {
password: process.env.CLAIMSCORP_PASSWORD,
debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
algorithms: {
serverHostKey: ["ssh-rsa", "ssh-dss"]
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
}
};
const allxmlsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => {
// Only process if in production environment.
if (process.env.NODE_ENV !== "production") {
res.sendStatus(403);
return;
}
//Query for the List of Bodyshop Clients.
logger.log("claimscorp-start", "DEBUG", "api", null, null);
const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS);
const specificShopIds = req.body.bodyshopIds; // ['uuid]
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
// Only process if the appropriate token is provided.
if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
res.sendStatus(401);
return;
}
const allxmlsToUpload = [];
const allErrors = [];
// Send immediate response and continue processing.
res.status(202).json({
success: true,
message: "Processing request ...",
timestamp: new Date().toISOString()
});
try {
for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
logger.log("claimscorp-start", "DEBUG", "api", null, null);
const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
if (shopsToProcess.length === 0) {
logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
return;
}
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
if (skipUpload) {
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
} else {
await uploadViaSFTP(allxmlsToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
null,
2
)}`
});
logger.log("claimscorp-end", "DEBUG", "api", null, null);
} catch (error) {
logger.log("claimscorp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
}
};
async function processBatch(batch, start, end) {
for (const bodyshop of batch) {
const erroredJobs = [];
try {
logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
const erroredJobs = [];
try {
const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
});
const claimsCorpObject = {
DataFeed: {
ShopInfo: {
ShopID: bodyshops_by_pk.claimscorpid,
ShopName: bodyshops_by_pk.shopname,
RO: jobs.map((j) =>
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
erroredJobs.push({ job: job, error: error.toString() });
})
)
}
}
};
if (erroredJobs.length > 0) {
logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
count: erroredJobs.length,
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
});
}
var ret = builder
.create(
{
// version: "1.0",
// encoding: "UTF-8",
//keepNullNodes: true,
},
claimsCorpObject
)
.end({ allowEmptyTags: true });
allxmlsToUpload.push({
count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
xml: ret,
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
});
logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
} catch (error) {
//Error at the shop level.
logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, {
...error
});
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
claimscorpid: bodyshop.claimscorpid,
fatal: true,
errors: [error.toString()]
});
} finally {
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
claimscorpid: bodyshop.claimscorpid,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
}))
});
}
}
if (skipUpload) {
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
res.json(allxmlsToUpload);
sendServerEmail({
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}
`
const { jobs, bodyshops_by_pk } = await client.request(queries.CLAIMSCORP_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
});
return;
}
let sftp = new Client();
sftp.on("error", (errors) =>
logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
...errors
})
);
try {
//Connect to the FTP and upload all.
const claimsCorpObject = {
DataFeed: {
ShopInfo: {
ShopID: bodyshops_by_pk.claimscorpid,
ShopName: bodyshops_by_pk.shopname,
RO: jobs.map((j) =>
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
erroredJobs.push({ job: job, error: error.toString() });
})
)
}
}
};
await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) {
logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
filename: xmlObj.filename
});
const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, {
uploadResult
if (erroredJobs.length > 0) {
logger.log("claimscorp-failed-jobs", "ERROR", "api", bodyshop.id, {
count: erroredJobs.length,
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
});
}
//***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true });
allxmlsToUpload.push({
count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
xml: ret,
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
});
logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
} catch (error) {
logger.log("claimscorp-sftp-error", "ERROR", "api", null, {
...error
//Error at the shop level.
logger.log("claimscorp-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
claimscorpid: bodyshop.claimscorpid,
fatal: true,
errors: [error.toString()]
});
} finally {
sftp.end();
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
claimscorpid: bodyshop.claimscorpid,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
}))
});
}
sendServerEmail({
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}
`
});
res.sendStatus(200);
} catch (error) {
res.status(200).json(error);
}
};
}
async function uploadViaSFTP(allxmlsToUpload) {
const sftp = new Client();
sftp.on("error", (errors) =>
logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
);
try {
//Connect to the FTP and upload all.
await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) {
try {
logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
logger.log("claimscorp-sftp-upload-result", "DEBUG", "api", null, {
filename: xmlObj.filename,
result: xmlObj.result
});
} catch (error) {
logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, {
filename: xmlObj.filename,
error: error.message,
stack: error.stack
});
throw error;
}
}
} catch (error) {
logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
throw error;
} finally {
sftp.end();
}
}
const CreateRepairOrderTag = (job, errorCallback) => {
//Level 2
@@ -445,10 +456,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
};
return ret;
} catch (error) {
logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, {
error
});
logger.log("claimscorp-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
}
};

View File

@@ -16,8 +16,7 @@ const { sendServerEmail } = require("../email/sendemail");
const DineroFormat = "0,0.00";
const DateFormat = "MM/DD/YYYY";
const repairOpCodes = ["OP4", "OP9", "OP10"];
const replaceOpCodes = ["OP2", "OP5", "OP11", "OP12"];
const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
const ftpSetup = {
host: process.env.KAIZEN_HOST,
@@ -30,173 +29,180 @@ const ftpSetup = {
}
};
const allxmlsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => {
// Only process if in production environment.
if (process.env.NODE_ENV !== "production") {
res.sendStatus(403);
return;
}
//Query for the List of Bodyshop Clients.
logger.log("kaizen-start", "DEBUG", "api", null, null);
const kaizenShopsIDs = ["SUMMIT", "STRATHMORE", "SUNRIDGE", "SHAW"];
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, {
imexshopid: kaizenShopsIDs
});
const specificShopIds = req.body.bodyshopIds; // ['uuid]
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
// Only process if the appropriate token is provided.
if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
res.sendStatus(401);
return;
}
const allxmlsToUpload = [];
const allErrors = [];
// Send immediate response and continue processing.
res.status(202).json({
success: true,
message: "Processing request ...",
timestamp: new Date().toISOString()
});
try {
for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
logger.log("kaizen-start", "DEBUG", "api", null, null);
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const batchSize = 10;
const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
if (shopsToProcess.length === 0) {
logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
return;
}
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
if (skipUpload) {
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
} else {
await uploadViaSFTP(allxmlsToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
null,
2
)}`
});
logger.log("kaizen-end", "DEBUG", "api", null, null);
} catch (error) {
logger.log("kaizen-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
}
};
async function processBatch(batch, start, end) {
for (const bodyshop of batch) {
const erroredJobs = [];
try {
logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
const erroredJobs = [];
try {
const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
});
const kaizenObject = {
DataFeed: {
ShopInfo: {
ShopName: bodyshops_by_pk.shopname,
Jobs: jobs.map((j) =>
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
erroredJobs.push({ job: job, error: error.toString() });
})
)
}
}
};
if (erroredJobs.length > 0) {
logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
count: erroredJobs.length,
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
});
}
var ret = builder
.create(
{
// version: "1.0",
// encoding: "UTF-8",
//keepNullNodes: true,
},
kaizenObject
)
.end({ allowEmptyTags: true });
allxmlsToUpload.push({
count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
xml: ret,
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
});
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
} catch (error) {
//Error at the shop level.
logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, {
...error
});
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
fatal: true,
errors: [error.toString()]
});
} finally {
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
}))
});
}
}
if (skipUpload) {
for (const xmlObj of allxmlsToUpload) {
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
}
res.json(allxmlsToUpload);
sendServerEmail({
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}
`
const { jobs, bodyshops_by_pk } = await client.request(queries.KAIZEN_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(5, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
});
return;
}
let sftp = new Client();
sftp.on("error", (errors) =>
logger.log("kaizen-sftp-error", "ERROR", "api", null, {
...errors
})
);
try {
//Connect to the FTP and upload all.
const kaizenObject = {
DataFeed: {
ShopInfo: {
ShopName: bodyshops_by_pk.shopname,
Jobs: jobs.map((j) =>
CreateRepairOrderTag({ ...j, bodyshop: bodyshops_by_pk }, function ({ job, error }) {
erroredJobs.push({ job: job, error: error.toString() });
})
)
}
}
};
await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) {
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
filename: xmlObj.filename
});
const uploadResult = await sftp.put(Buffer.from(xmlObj.xml), `/${xmlObj.filename}`);
logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
uploadResult
if (erroredJobs.length > 0) {
logger.log("kaizen-failed-jobs", "ERROR", "api", bodyshop.id, {
count: erroredJobs.length,
jobs: JSON.stringify(erroredJobs.map((j) => j.job.ro_number))
});
}
//***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml
const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
allxmlsToUpload.push({
count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
xml: ret,
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
});
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
} catch (error) {
logger.log("kaizen-sftp-error", "ERROR", "api", null, {
...error
//Error at the shop level.
logger.log("kaizen-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
fatal: true,
errors: [error.toString()]
});
} finally {
sftp.end();
allErrors.push({
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
errors: erroredJobs.map((ej) => ({
ro_number: ej.job?.ro_number,
jobid: ej.job?.id,
error: ej.error
}))
});
}
sendServerEmail({
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}
`
});
res.sendStatus(200);
} catch (error) {
res.status(200).json(error);
}
};
}
async function uploadViaSFTP(allxmlsToUpload) {
const sftp = new Client();
sftp.on("error", (errors) =>
logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
);
try {
//Connect to the FTP and upload all.
await sftp.connect(ftpSetup);
for (const xmlObj of allxmlsToUpload) {
try {
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, { filename: xmlObj.filename });
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
logger.log("kaizen-sftp-upload-result", "DEBUG", "api", null, {
filename: xmlObj.filename,
result: xmlObj.result
});
} catch (error) {
logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
filename: xmlObj.filename,
error: error.message,
stack: error.stack
});
throw error;
}
}
} catch (error) {
logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
throw error;
} finally {
sftp.end();
}
}
const CreateRepairOrderTag = (job, errorCallback) => {
//Level 2
@@ -420,10 +426,7 @@ const CreateRepairOrderTag = (job, errorCallback) => {
};
return ret;
} catch (error) {
logger.log("kaizen-job-calculate-error", "ERROR", "api", null, {
error
});
logger.log("kaizen-job-calculate-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
errorCallback({ jobid: job.id, ro_number: job.ro_number, error });
}
};