From 655aeb86fcba57d07ad0af5ff73d25859b57e482 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Fri, 8 Nov 2024 23:43:18 -0800 Subject: [PATCH 1/3] IO-2921 Adjust for Promise and change processing Signed-off-by: Allan Carr --- server/data/chatter.js | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/server/data/chatter.js b/server/data/chatter.js index 056e5ba22..ea8851e94 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -37,12 +37,14 @@ exports.default = async (req, res) => { res.sendStatus(401); return; } - try { - //Query for the List of Bodyshop Clients. - logger.log("chatter-start", "DEBUG", "api", null, null); - const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); - const specificShopIds = req.body.bodyshopIds; // ['uuid]; + // Send immediate response and continue processing. + res.status(200).send(); + + try { + logger.log("chatter-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD const batchSize = 10; @@ -53,26 +55,28 @@ exports.default = async (req, res) => { if (shopsToProcess.length === 0) { logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null); - res.sendStatus(200); return; } + const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); - await processBatch(batch, start, end); - - if (skipUpload) { - for (const csvObj of allcsvsToUpload) { - fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); + const batchPromise = (async () => { + await processBatch(batch, start, end); + if (skipUpload) { + for (const csvObj of allcsvsToUpload) { + await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); + } + } else { + await uploadViaSFTP(allcsvsToUpload); } - } else { - await uploadViaSFTP(allcsvsToUpload); - } + })(); + batchPromises.push(batchPromise); } - sendServerEmail({ + await Promise.all(batchPromises); + await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), null, 2 @@ -80,10 +84,8 @@ exports.default = async (req, res) => { }); logger.log("chatter-end", "DEBUG", "api", null, null); - res.sendStatus(200); } catch (error) { - logger.log("chatter-shopsToProcess-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); - res.status(500).json({ error: error.message, stack: error.stack }); + logger.log("chatter-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); } }; @@ -152,9 +154,7 @@ async function getPrivateKey() { try { const { SecretString, SecretBinary } = await client.send(command); if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null); - const chatterPrivateKey = SecretString - ? SecretString - : Buffer.from(SecretBinary, "base64").toString("ascii"); + const chatterPrivateKey = SecretString ? SecretString : Buffer.from(SecretBinary, "base64").toString("ascii"); return chatterPrivateKey; } catch (error) { logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack }); From 5cbf00b0c87bbd622ab77a5099e5aaed17dcfab6 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Sat, 9 Nov 2024 00:32:51 -0800 Subject: [PATCH 2/3] IO-3025 Adjust for promise and change processing Signed-off-by: Allan Carr --- server/data/autohouse.js | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/server/data/autohouse.js b/server/data/autohouse.js index 506ba53d3..9ea3a415a 100644 --- a/server/data/autohouse.js +++ b/server/data/autohouse.js @@ -45,12 +45,14 @@ exports.default = async (req, res) => { res.sendStatus(401); return; } - try { - //Query for the List of Bodyshop Clients. - logger.log("autohouse-start", "DEBUG", "api", null, null); - const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); - const specificShopIds = req.body.bodyshopIds; // ['uuid]; + // Send immediate response and continue processing. + res.status(200).send(); + + try { + logger.log("autohouse-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_AUTOHOUSE_SHOPS); //Query for the List of Bodyshop Clients. + const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD const batchSize = 10; @@ -61,26 +63,28 @@ exports.default = async (req, res) => { if (shopsToProcess.length === 0) { logger.log("autohouse-shopsToProcess-empty", "DEBUG", "api", null, null); - res.sendStatus(200); return; } - + const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); - await processBatch(batch, start, end); + const batchPromise = (async () => { + await processBatch(batch, start, end); - if (skipUpload) { - for (const xmlObj of allxmlsToUpload) { - fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + if (skipUpload) { + for (const xmlObj of allxmlsToUpload) { + fs.writeFileSync(`./logs/${xmlObj.filename}`, xmlObj.xml); + } + } else { + await uploadViaSFTP(allxmlsToUpload); } - } else { - await uploadViaSFTP(allxmlsToUpload); - } + })(); + batchPromises.push(batchPromise); } - sendServerEmail({ + await Promise.all(batchPromises); + await sendServerEmail({ subject: `Autohouse Report ${moment().format("MM-DD-YY")}`, - text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} - Uploaded: ${JSON.stringify( + text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allxmlsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), null, 2 @@ -88,10 +92,8 @@ exports.default = async (req, res) => { }); logger.log("autohouse-end", "DEBUG", "api", null, null); - res.sendStatus(200); } catch (error) { - logger.log("autohouse-shopsToProcess-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); - res.status(500).json({ error: error.message, stack: error.stack }); + logger.log("autohouse-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); } }; From 8d4195b596abce52470fb3c8c83c4d75888bf980 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Sat, 9 Nov 2024 00:35:18 -0800 Subject: [PATCH 3/3] IO-2921 Adjust SFTP setup Signed-off-by: Allan Carr --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 50dc56932..581d74062 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -180,9 +180,10 @@ services: # - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key # - ./upload:/home/user/upload # Mount a local directory for SFTP uploads # environment: -# - SFTP_USERS=user:password:1001:100:upload +# - SFTP_USERS=user:password:1000::upload # command: > # /bin/sh -c " +# chmod -R 007 /home/user/upload && # echo 'Match User user' >> /etc/ssh/sshd_config && # sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config && # /usr/sbin/sshd -D