Merged in feature/IO-3027-Datapumps-Refactor (pull request #1943)
IO-3027 Datapump Refactor
This commit is contained in:
@@ -25,17 +25,15 @@ const ftpSetup = {
|
|||||||
port: process.env.AUTOHOUSE_PORT,
|
port: process.env.AUTOHOUSE_PORT,
|
||||||
username: process.env.AUTOHOUSE_USER,
|
username: process.env.AUTOHOUSE_USER,
|
||||||
password: process.env.AUTOHOUSE_PASSWORD,
|
password: process.env.AUTOHOUSE_PASSWORD,
|
||||||
debug: process.env.NODE_ENV !== "production"
|
debug:
|
||||||
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
process.env.NODE_ENV !== "production"
|
||||||
: () => {},
|
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
||||||
|
: () => {},
|
||||||
algorithms: {
|
algorithms: {
|
||||||
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const allxmlsToUpload = [];
|
|
||||||
const allErrors = [];
|
|
||||||
|
|
||||||
exports.default = async (req, res) => {
|
exports.default = async (req, res) => {
|
||||||
// Only process if in production environment.
|
// Only process if in production environment.
|
||||||
if (process.env.NODE_ENV !== "production") {
|
if (process.env.NODE_ENV !== "production") {
|
||||||
@@ -57,12 +55,13 @@ exports.default = async (req, res) => {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
logger.log("autohouse-start", "DEBUG", "api", null, null);
|
logger.log("autohouse-start", "DEBUG", "api", null, null);
|
||||||
|
const allXMLResults = [];
|
||||||
|
const allErrors = [];
|
||||||
|
|
||||||
const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
|
const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients.
|
||||||
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
||||||
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
||||||
|
|
||||||
const batchSize = 10;
|
|
||||||
|
|
||||||
const shopsToProcess =
|
const shopsToProcess =
|
||||||
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
||||||
logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
|
logger.log("autohouse-shopsToProcess-generated", "DEBUG", "api", null, null);
|
||||||
@@ -71,27 +70,18 @@ exports.default = async (req, res) => {
|
|||||||
logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
|
logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const batchPromises = [];
|
|
||||||
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
|
|
||||||
const batch = shopsToProcess.slice(i, i + batchSize);
|
|
||||||
const batchPromise = (async () => {
|
|
||||||
await processBatch(batch, start, end);
|
|
||||||
|
|
||||||
if (skipUpload) {
|
await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
|
||||||
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await uploadViaSFTP(allxmlsToUpload);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
batchPromises.push(batchPromise);
|
|
||||||
}
|
|
||||||
await Promise.all(batchPromises);
|
|
||||||
await sendServerEmail({
|
await sendServerEmail({
|
||||||
subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
|
subject: `Autohouse Report ${moment().format("MM-DD-YY")}`,
|
||||||
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
||||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
|
allXMLResults.map((x) => ({
|
||||||
|
imexshopid: x.imexshopid,
|
||||||
|
filename: x.filename,
|
||||||
|
count: x.count,
|
||||||
|
result: x.result
|
||||||
|
})),
|
||||||
null,
|
null,
|
||||||
2
|
2
|
||||||
)}`
|
)}`
|
||||||
@@ -103,8 +93,8 @@ exports.default = async (req, res) => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
async function processBatch(batch, start, end) {
|
async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
|
||||||
for (const bodyshop of batch) {
|
for (const bodyshop of shopsToProcess) {
|
||||||
const erroredJobs = [];
|
const erroredJobs = [];
|
||||||
try {
|
try {
|
||||||
logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("autohouse-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -134,12 +124,27 @@ async function processBatch(batch, start, end) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const ret = builder.create({}, autoHouseObject).end({ allowEmptyTags: true });
|
const xmlObj = {
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
xml: builder.create({}, autoHouseObject).end({ allowEmptyTags: true }),
|
||||||
|
filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`,
|
||||||
|
count: autoHouseObject.AutoHouseExport.RepairOrder.length
|
||||||
|
};
|
||||||
|
|
||||||
allxmlsToUpload.push({
|
if (skipUpload) {
|
||||||
count: autoHouseObject.AutoHouseExport.RepairOrder.length,
|
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
||||||
xml: ret,
|
} else {
|
||||||
filename: `IM_${bodyshop.autohouseid}_${moment().format("DDMMYYYY_HHMMss")}.xml`
|
await uploadViaSFTP(xmlObj);
|
||||||
|
}
|
||||||
|
|
||||||
|
allXMLResults.push({
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
autohouseid: bodyshop.autohouseid,
|
||||||
|
count: xmlObj.count,
|
||||||
|
filename: xmlObj.filename,
|
||||||
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("autohouse-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -171,33 +176,35 @@ async function processBatch(batch, start, end) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadViaSFTP(allxmlsToUpload) {
|
async function uploadViaSFTP(xmlObj) {
|
||||||
const sftp = new Client();
|
const sftp = new Client();
|
||||||
sftp.on("error", (errors) =>
|
sftp.on("error", (errors) =>
|
||||||
logger.log("autohouse-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
|
logger.log("autohouse-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
|
error: errors.message,
|
||||||
|
stack: errors.stack
|
||||||
|
})
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
//Connect to the FTP and upload all.
|
//Connect to the FTP and upload all.
|
||||||
await sftp.connect(ftpSetup);
|
await sftp.connect(ftpSetup);
|
||||||
|
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
try {
|
||||||
try {
|
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
||||||
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
logger.log("autohouse-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
|
||||||
logger.log("autohouse-sftp-upload", "DEBUG", "api", null, {
|
imexshopid: xmlObj.imexshopid,
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
result: xmlObj.result
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("autohouse-sftp-upload-error", "ERROR", "api", null, {
|
logger.log("autohouse-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
stack: error.stack
|
stack: error.stack
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("autohouse-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
logger.log("autohouse-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
sftp.end();
|
sftp.end();
|
||||||
|
|||||||
@@ -24,17 +24,15 @@ const ftpSetup = {
|
|||||||
port: process.env.CLAIMSCORP_PORT,
|
port: process.env.CLAIMSCORP_PORT,
|
||||||
username: process.env.CLAIMSCORP_USER,
|
username: process.env.CLAIMSCORP_USER,
|
||||||
password: process.env.CLAIMSCORP_PASSWORD,
|
password: process.env.CLAIMSCORP_PASSWORD,
|
||||||
debug: process.env.NODE_ENV !== "production"
|
debug:
|
||||||
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
process.env.NODE_ENV !== "production"
|
||||||
: () => {},
|
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
||||||
|
: () => {},
|
||||||
algorithms: {
|
algorithms: {
|
||||||
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const allxmlsToUpload = [];
|
|
||||||
const allErrors = [];
|
|
||||||
|
|
||||||
exports.default = async (req, res) => {
|
exports.default = async (req, res) => {
|
||||||
// Only process if in production environment.
|
// Only process if in production environment.
|
||||||
if (process.env.NODE_ENV !== "production") {
|
if (process.env.NODE_ENV !== "production") {
|
||||||
@@ -56,12 +54,13 @@ exports.default = async (req, res) => {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
logger.log("claimscorp-start", "DEBUG", "api", null, null);
|
logger.log("claimscorp-start", "DEBUG", "api", null, null);
|
||||||
|
const allXMLResults = [];
|
||||||
|
const allErrors = [];
|
||||||
|
|
||||||
const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
|
const { bodyshops } = await client.request(queries.GET_CLAIMSCORP_SHOPS); //Query for the List of Bodyshop Clients.
|
||||||
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
||||||
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
||||||
|
|
||||||
const batchSize = 10;
|
|
||||||
|
|
||||||
const shopsToProcess =
|
const shopsToProcess =
|
||||||
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
||||||
logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
|
logger.log("claimscorp-shopsToProcess-generated", "DEBUG", "api", null, null);
|
||||||
@@ -70,27 +69,18 @@ exports.default = async (req, res) => {
|
|||||||
logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
|
logger.log("claimscorp-shopsToProcess-empty", "DEBUG", "api", null, null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const batchPromises = [];
|
|
||||||
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
|
|
||||||
const batch = shopsToProcess.slice(i, i + batchSize);
|
|
||||||
const batchPromise = (async () => {
|
|
||||||
await processBatch(batch, start, end);
|
|
||||||
|
|
||||||
if (skipUpload) {
|
await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
|
||||||
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await uploadViaSFTP(allxmlsToUpload);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
batchPromises.push(batchPromise);
|
|
||||||
}
|
|
||||||
await Promise.all(batchPromises);
|
|
||||||
await sendServerEmail({
|
await sendServerEmail({
|
||||||
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
|
subject: `ClaimsCorp Report ${moment().format("MM-DD-YY")}`,
|
||||||
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
||||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
|
allXMLResults.map((x) => ({
|
||||||
|
imexshopid: x.imexshopid,
|
||||||
|
filename: x.filename,
|
||||||
|
count: x.count,
|
||||||
|
result: x.result
|
||||||
|
})),
|
||||||
null,
|
null,
|
||||||
2
|
2
|
||||||
)}`
|
)}`
|
||||||
@@ -102,8 +92,8 @@ exports.default = async (req, res) => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
async function processBatch(batch, start, end) {
|
async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
|
||||||
for (const bodyshop of batch) {
|
for (const bodyshop of shopsToProcess) {
|
||||||
const erroredJobs = [];
|
const erroredJobs = [];
|
||||||
try {
|
try {
|
||||||
logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("claimscorp-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -137,12 +127,27 @@ async function processBatch(batch, start, end) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const ret = builder.create({}, claimsCorpObject).end({ allowEmptyTags: true });
|
const xmlObj = {
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
xml: builder.create({}, claimsCorpObject).end({ allowEmptyTags: true }),
|
||||||
|
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
|
||||||
|
count: claimsCorpObject.DataFeed.ShopInfo.RO.length
|
||||||
|
};
|
||||||
|
|
||||||
allxmlsToUpload.push({
|
if (skipUpload) {
|
||||||
count: claimsCorpObject.DataFeed.ShopInfo.RO.length,
|
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
||||||
xml: ret,
|
} else {
|
||||||
filename: `${bodyshop.claimscorpid}-${moment().format("YYYYMMDDTHHMMss")}.xml`
|
await uploadViaSFTP(xmlObj);
|
||||||
|
}
|
||||||
|
|
||||||
|
allXMLResults.push({
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
claimscorpid: bodyshop.claimscorpid,
|
||||||
|
count: xmlObj.count,
|
||||||
|
filename: xmlObj.filename,
|
||||||
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("claimscorp-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -174,33 +179,38 @@ async function processBatch(batch, start, end) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadViaSFTP(allxmlsToUpload) {
|
async function uploadViaSFTP(xmlObj) {
|
||||||
const sftp = new Client();
|
const sftp = new Client();
|
||||||
sftp.on("error", (errors) =>
|
sftp.on("error", (errors) =>
|
||||||
logger.log("claimscorp-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
|
logger.log("claimscorp-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
|
error: errors.message,
|
||||||
|
stack: errors.stack
|
||||||
|
})
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
//Connect to the FTP and upload all.
|
//Connect to the FTP and upload all.
|
||||||
await sftp.connect(ftpSetup);
|
await sftp.connect(ftpSetup);
|
||||||
|
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
try {
|
||||||
try {
|
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
||||||
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
logger.log("claimscorp-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
|
||||||
logger.log("claimscorp-sftp-upload", "DEBUG", "api", null, {
|
imexshopid: xmlObj.imexshopid,
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
result: xmlObj.result
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("claimscorp-sftp-upload-error", "ERROR", "api", null, {
|
logger.log("claimscorp-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
stack: error.stack
|
stack: error.stack
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("claimscorp-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
logger.log("claimscorp-sftp-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
|
error: error.message,
|
||||||
|
stack: error.stack
|
||||||
|
});
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
sftp.end();
|
sftp.end();
|
||||||
|
|||||||
@@ -23,17 +23,15 @@ const ftpSetup = {
|
|||||||
port: process.env.KAIZEN_PORT,
|
port: process.env.KAIZEN_PORT,
|
||||||
username: process.env.KAIZEN_USER,
|
username: process.env.KAIZEN_USER,
|
||||||
password: process.env.KAIZEN_PASSWORD,
|
password: process.env.KAIZEN_PASSWORD,
|
||||||
debug: process.env.NODE_ENV !== "production"
|
debug:
|
||||||
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
process.env.NODE_ENV !== "production"
|
||||||
: () => {},
|
? (message, ...data) => logger.log(message, "DEBUG", "api", null, data)
|
||||||
|
: () => {},
|
||||||
algorithms: {
|
algorithms: {
|
||||||
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const allxmlsToUpload = [];
|
|
||||||
const allErrors = [];
|
|
||||||
|
|
||||||
exports.default = async (req, res) => {
|
exports.default = async (req, res) => {
|
||||||
// Only process if in production environment.
|
// Only process if in production environment.
|
||||||
if (process.env.NODE_ENV !== "production") {
|
if (process.env.NODE_ENV !== "production") {
|
||||||
@@ -55,12 +53,13 @@ exports.default = async (req, res) => {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
logger.log("kaizen-start", "DEBUG", "api", null, null);
|
logger.log("kaizen-start", "DEBUG", "api", null, null);
|
||||||
|
const allXMLResults = [];
|
||||||
|
const allErrors = [];
|
||||||
|
|
||||||
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
|
const { bodyshops } = await client.request(queries.GET_KAIZEN_SHOPS, { imexshopid: kaizenShopsIDs }); //Query for the List of Bodyshop Clients.
|
||||||
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
const specificShopIds = req.body.bodyshopIds; // ['uuid];
|
||||||
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
|
||||||
|
|
||||||
const batchSize = 10;
|
|
||||||
|
|
||||||
const shopsToProcess =
|
const shopsToProcess =
|
||||||
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
|
||||||
logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
|
logger.log("kaizen-shopsToProcess-generated", "DEBUG", "api", null, null);
|
||||||
@@ -69,27 +68,18 @@ exports.default = async (req, res) => {
|
|||||||
logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
|
logger.log("kaizen-shopsToProcess-empty", "DEBUG", "api", null, null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const batchPromises = [];
|
|
||||||
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
|
|
||||||
const batch = shopsToProcess.slice(i, i + batchSize);
|
|
||||||
const batchPromise = (async () => {
|
|
||||||
await processBatch(batch, start, end);
|
|
||||||
|
|
||||||
if (skipUpload) {
|
await processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors);
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
|
||||||
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await uploadViaSFTP(allxmlsToUpload);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
batchPromises.push(batchPromise);
|
|
||||||
}
|
|
||||||
await Promise.all(batchPromises);
|
|
||||||
await sendServerEmail({
|
await sendServerEmail({
|
||||||
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
|
subject: `Kaizen Report ${moment().format("MM-DD-YY")}`,
|
||||||
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
|
||||||
allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
|
allXMLResults.map((x) => ({
|
||||||
|
imexshopid: x.imexshopid,
|
||||||
|
filename: x.filename,
|
||||||
|
count: x.count,
|
||||||
|
result: x.result
|
||||||
|
})),
|
||||||
null,
|
null,
|
||||||
2
|
2
|
||||||
)}`
|
)}`
|
||||||
@@ -101,8 +91,8 @@ exports.default = async (req, res) => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
async function processBatch(batch, start, end) {
|
async function processShopData(shopsToProcess, start, end, skipUpload, allXMLResults, allErrors) {
|
||||||
for (const bodyshop of batch) {
|
for (const bodyshop of shopsToProcess) {
|
||||||
const erroredJobs = [];
|
const erroredJobs = [];
|
||||||
try {
|
try {
|
||||||
logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("kaizen-start-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -135,12 +125,26 @@ async function processBatch(batch, start, end) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const ret = builder.create({}, kaizenObject).end({ allowEmptyTags: true });
|
const xmlObj = {
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
xml: builder.create({}, kaizenObject).end({ allowEmptyTags: true }),
|
||||||
|
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`,
|
||||||
|
count: kaizenObject.DataFeed.ShopInfo.Jobs.length
|
||||||
|
};
|
||||||
|
|
||||||
allxmlsToUpload.push({
|
if (skipUpload) {
|
||||||
count: kaizenObject.DataFeed.ShopInfo.Jobs.length,
|
fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml);
|
||||||
xml: ret,
|
} else {
|
||||||
filename: `${bodyshop.shopname}-${moment().format("YYYYMMDDTHHMMss")}.xml`
|
await uploadViaSFTP(xmlObj);
|
||||||
|
}
|
||||||
|
|
||||||
|
allXMLResults.push({
|
||||||
|
bodyshopid: bodyshop.id,
|
||||||
|
imexshopid: bodyshop.imexshopid,
|
||||||
|
count: xmlObj.count,
|
||||||
|
filename: xmlObj.filename,
|
||||||
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
logger.log("kaizen-end-shop-extract", "DEBUG", "api", bodyshop.id, {
|
||||||
@@ -172,33 +176,35 @@ async function processBatch(batch, start, end) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadViaSFTP(allxmlsToUpload) {
|
async function uploadViaSFTP(xmlObj) {
|
||||||
const sftp = new Client();
|
const sftp = new Client();
|
||||||
sftp.on("error", (errors) =>
|
sftp.on("error", (errors) =>
|
||||||
logger.log("kaizen-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
|
logger.log("kaizen-sftp-connection-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
|
error: errors.message,
|
||||||
|
stack: errors.stack
|
||||||
|
})
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
//Connect to the FTP and upload all.
|
//Connect to the FTP and upload all.
|
||||||
await sftp.connect(ftpSetup);
|
await sftp.connect(ftpSetup);
|
||||||
|
|
||||||
for (const xmlObj of allxmlsToUpload) {
|
try {
|
||||||
try {
|
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
||||||
xmlObj.result = await sftp.put(Buffer.from(xmlObj.xml), `${xmlObj.filename}`);
|
logger.log("kaizen-sftp-upload", "DEBUG", "api", xmlObj.bodyshopid, {
|
||||||
logger.log("kaizen-sftp-upload", "DEBUG", "api", null, {
|
imexshopid: xmlObj.imexshopid,
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
result: xmlObj.result
|
result: xmlObj.result
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("kaizen-sftp-upload-error", "ERROR", "api", null, {
|
logger.log("kaizen-sftp-upload-error", "ERROR", "api", xmlObj.bodyshopid, {
|
||||||
filename: xmlObj.filename,
|
filename: xmlObj.filename,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
stack: error.stack
|
stack: error.stack
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("kaizen-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
logger.log("kaizen-sftp-error", "ERROR", "api", xmlObj.bodyshopid, { error: error.message, stack: error.stack });
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
sftp.end();
|
sftp.end();
|
||||||
|
|||||||
Reference in New Issue
Block a user