admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IO-2921 Re-factor as batch and get docker compose dev working for sftp testing

Browse Source 
Signed-off-by: Allan Carr <allan.carr@thinkimex.com>
This commit is contained in:
Allan Carr
2024-11-06 15:42:14 -08:00
parent 542997b1a7
commit 8ec5831ec5
2 changed files with 151 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
docker-compose.yml
View File

@@ -167,6 +167,27 @@ services:
# volumes:
# - redis-insight-data:/db
# ##Optional Container for SFTP/SSH Server for testing
# ssh-sftp-server:
# image: atmoz/sftp:alpine # Using an image with SFTP support
# container_name: ssh-sftp-server
# hostname: ssh-sftp-server
# networks:
# - redis-cluster-net
# ports:
# - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
# volumes:
# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key
# - ./upload:/home/user/upload # Mount a local directory for SFTP uploads
# environment:
# - SFTP_USERS=user:password:1001:100:upload
# command: >
# /bin/sh -c "
# echo 'Match User user' >> /etc/ssh/sshd_config &&
# sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config &&
# /usr/sbin/sshd -D
# "
networks:
redis-cluster-net:
driver: bridge

164
server/data/chatter.js
View File

@@ -22,31 +22,79 @@ const ftpSetup = {
serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
}
};
const allcsvsToUpload = [];
const allErrors = [];
exports.default = async (req, res) => {
// Only process if in production environment.
if (process.env.NODE_ENV !== "production") {
res.sendStatus(403);
return;
}
// Only process if the appropriate token is provided.
if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
res.sendStatus(401);
return;
}
try {
//Query for the List of Bodyshop Clients.
logger.log("chatter-start", "DEBUG", "api", null, null);
const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS);
const specificShopIds = req.body.bodyshopIds; // ['uuid]
const specificShopIds = req.body.bodyshopIds; // ['uuid];
const { start, end, skipUpload } = req.body; //YYYY-MM-DD
const allcsvsToUpload = [];
const allErrors = [];
const batchSize = 10;
const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null);
if (shopsToProcess.length === 0) {
logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null);
res.sendStatus(200);
return;
}
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
await processBatch(batch, start, end);
if (skipUpload) {
for (const csvObj of allcsvsToUpload) {
fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
}
} else {
await uploadViaSFTP(allcsvsToUpload);
}
sendServerEmail({
subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
null,
2
)}`
});
logger.log("chatter-end", "DEBUG", "api", null, null);
res.sendStatus(200);
}
} catch (error) {
logger.log("chatter-shopsToProcess-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
res.status(500).json({ error: error.message, stack: error.stack });
}
};
async function processBatch(batch, start, end) {
for (const bodyshop of batch) {
try {
for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) {
logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
try {
const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"),
@@ -77,9 +125,7 @@ exports.default = async (req, res) => {
});
} catch (error) {
//Error at the shop level.
logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, {
...error
});
logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });
allErrors.push({
bodyshopid: bodyshop.id,
@@ -96,60 +142,7 @@ exports.default = async (req, res) => {
});
}
}
if (skipUpload) {
for (const csvObj of allcsvsToUpload) {
fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
}
sendServerEmail({
subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}
`
});
res.json(allcsvsToUpload);
return;
}
const sftp = new Client();
sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors }));
try {
//Get the private key from AWS Secrets Manager.
ftpSetup.privateKey = await getPrivateKey();
//Connect to the FTP and upload all.
await sftp.connect(ftpSetup);
for (const csvObj of allcsvsToUpload) {
logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename });
const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`);
logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult });
}
} catch (error) {
logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error });
} finally {
sftp.end();
sendServerEmail({
subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))}
Uploaded: ${JSON.stringify(
allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })),
null,
2
)}`
});
res.sendStatus(200);
}
} catch (error) {
res.status(200).json(error);
}
};
}
async function getPrivateKey() {
// Connect to AWS Secrets Manager
@@ -160,10 +153,49 @@ async function getPrivateKey() {
try {
const { SecretString, SecretBinary } = await client.send(command);
if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null);
const chatterPrivateKey = SecretString ? SecretString : Buffer.from(SecretBinary, "base64").toString("ascii");
const chatterPrivateKey = SecretString
? SecretString
: Buffer.from(SecretBinary, "base64").toString("ascii");
return chatterPrivateKey;
} catch (error) {
logger.log("chatter-get-private-key", "ERROR", "api", null, error);
throw err;
logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack });
throw error;
}
}
async function uploadViaSFTP(allcsvsToUpload) {
const sftp = new Client();
sftp.on("error", (errors) =>
logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
);
try {
//Get the private key from AWS Secrets Manager.
const privateKey = await getPrivateKey();
//Connect to the FTP and upload all.
await sftp.connect({ ...ftpSetup, privateKey });
for (const csvObj of allcsvsToUpload) {
try {
logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename });
csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`);
logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, {
filename: csvObj.filename,
result: csvObj.result
});
} catch (error) {
logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
filename: csvObj.filename,
error: error.message,
stack: error.stack
});
throw error;
}
}
} catch (error) {
logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
throw error;
} finally {
sftp.end();
}
}