admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IO-2921 Chatter modifications as per Dave Bourbeau @ Chatter

Browse Source 
Signed-off-by: Allan Carr <allan.carr@thinkimex.com>
This commit is contained in:
Allan Carr
2024-11-20 10:57:59 -08:00
parent 1ff1de8739
commit 45ac56e0bc
8 changed files with 91 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
certs/id_rsa → certs/id_rsa.key
View File

12
certs/io-ftp-test.key Normal file
View File

@@ -0,0 +1,12 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAArAAAABNlY2RzYS
1zaGEyLW5pc3RwNTIxAAAACG5pc3RwNTIxAAAAhQQBYJnAujo17diR0fM2Ze1d1Ft6XHm5
U31pXdFEN+rGC4SoYTdZE8q3relxMS5GwwBOvgvVUuayfid2XS8ls/CMDiMBJAYqEK4CRY
PbbPB7lLnMWsF7muFhvs+SIpPQC+vtDwM2TKlxF0Y8p+iVRpvCADoggsSze7skmJWKmMTt
8jEdEOcAAAEQIyXsOSMl7DkAAAATZWNkc2Etc2hhMi1uaXN0cDUyMQAAAAhuaXN0cDUyMQ
AAAIUEAWCZwLo6Ne3YkdHzNmXtXdRbelx5uVN9aV3RRDfqxguEqGE3WRPKt63pcTEuRsMA
Tr4L1VLmsn4ndl0vJbPwjA4jASQGKhCuAkWD22zwe5S5zFrBe5rhYb7PkiKT0Avr7Q8DNk
ypcRdGPKfolUabwgA6IILEs3u7JJiVipjE7fIxHRDnAAAAQUO5dO9G7i0bxGTP0zV3eIwv
5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZPwYC0wLMW4bJAf+kjqUnj4wGocoTeAAAAD2
lvLWZ0cC10ZXN0LWtleQECAwQ=
-----END OPENSSH PRIVATE KEY-----

1
certs/io-ftp-test.key.pub Normal file
View File

@@ -0,0 +1 @@
ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEXRjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w== io-ftp-test-key

12
certs/io-ftp-test.ppk Normal file
View File

@@ -0,0 +1,12 @@
PuTTY-User-Key-File-3: ecdsa-sha2-nistp521
Encryption: none
Comment: io-ftp-test-key
Public-Lines: 4
AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFgmcC6OjXt
2JHR8zZl7V3UW3pceblTfWld0UQ36sYLhKhhN1kTyret6XExLkbDAE6+C9VS5rJ+
J3ZdLyWz8IwOIwEkBioQrgJFg9ts8HuUucxawXua4WG+z5Iik9AL6+0PAzZMqXEX
Rjyn6JVGm8IAOiCCxLN7uySYlYqYxO3yMR0Q5w==
Private-Lines: 2
AAAAQUO5dO9G7i0bxGTP0zV3eIwv5g0NhrQJfW/bMHS6XWwaxdpr+QZ+DbBJVzZP
wYC0wLMW4bJAf+kjqUnj4wGocoTe
Private-MAC: d67001d47e13c43dc8bdb9c68a25356a96c1c4a6714f3c5a1836fca646b78b54

14
docker-build.ps1 Normal file
View File

@@ -0,0 +1,14 @@
# Stop and remove all containers, images, and networks from the Compose file
docker compose down --rmi all
# Prune all unused Docker objects including volumes
docker system prune --all --volumes --force
# Prune unused build cache
docker builder prune --all --force
# Prune all unused volumes
docker volume prune --all --force
# Rebuild and start the containers
docker compose up --build

16
docker-build.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
# Stop and remove all containers, images, and networks from the Compose file
docker compose down --rmi all
# Prune all unused Docker objects including volumes
docker system prune --all --volumes --force
# Prune unused build cache
docker builder prune --all --force
# Prune all unused volumes
docker volume prune --all --force
# Rebuild and start the containers
docker compose up --build

5
docker-compose.yml
View File

@@ -114,7 +114,7 @@ services:
"
aws --endpoint-url=http://localstack:4566 ses verify-domain-identity --domain imex.online --region ca-central-1
aws --endpoint-url=http://localstack:4566 ses verify-email-identity --email-address noreply@imex.online --region ca-central-1
aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/id_rsa
aws --endpoint-url=http://localstack:4566 secretsmanager create-secret --name CHATTER_PRIVATE_KEY --secret-string file:///tmp/certs/io-ftp-test.key
aws --endpoint-url=http://localstack:4566 logs create-log-group --log-group-name development --region ca-central-1
aws --endpoint-url=http://localstack:4566 s3api create-bucket --bucket imex-large-log --create-bucket-configuration LocationConstraint=ca-central-1
"
@@ -178,9 +178,10 @@ services:
# ports:
# - "2222:22" # Expose port 22 for SSH/SFTP (mapped to 2222 on the host)
# volumes:
# - ./certs/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub:ro # Mount the SSH public key
# - ./certs/io-ftp-test.key.pub:/home/user/.ssh/keys/io-ftp-test.key.pub:ro # Mount the SSH public key as authorized_keys
# - ./upload:/home/user/upload # Mount a local directory for SFTP uploads
# environment:
# # - SFTP_USERS=user::1000::upload
# - SFTP_USERS=user:password:1000::upload
# command: >
# /bin/sh -c "

55
server/data/chatter.js
View File

@@ -23,7 +23,7 @@ const ftpSetup = {
}
};
const allcsvsToUpload = [];
const allChatterObjects = [];
const allErrors = [];
exports.default = async (req, res) => {
@@ -62,29 +62,26 @@ exports.default = async (req, res) => {
return;
}
const batchPromises = [];
for (let i = 0; i < shopsToProcess.length; i += batchSize) {
const batch = shopsToProcess.slice(i, i + batchSize);
const batchPromise = (async () => {
await processBatch(batch, start, end);
}
const csvToUpload = {
count: allChatterObjects.length,
csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }),
filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv`
};
if (skipUpload) {
for (const csvObj of allcsvsToUpload) {
await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv);
}
await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv);
} else {
await uploadViaSFTP(allcsvsToUpload);
await uploadViaSFTP(csvToUpload);
}
})();
batchPromises.push(batchPromise);
}
await Promise.all(batchPromises);
await sendServerEmail({
subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify(
allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })),
null,
2
)}`
text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n
Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}`
});
logger.log("chatter-end", "DEBUG", "api", null, null);
@@ -116,15 +113,7 @@ async function processBatch(batch, start, end) {
phone_number: j.ownr_ph1
};
});
const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" });
allcsvsToUpload.push({
count: chatterObject.length,
csv: ret,
filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv`
});
allChatterObjects.push(...chatterObject);
logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
shopname: bodyshop.shopname
});
@@ -166,7 +155,7 @@ async function getPrivateKey() {
}
}
async function uploadViaSFTP(allcsvsToUpload) {
async function uploadViaSFTP(csvToUpload) {
const sftp = new Client();
sftp.on("error", (errors) =>
logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
@@ -178,22 +167,20 @@ async function uploadViaSFTP(allcsvsToUpload) {
//Connect to the FTP and upload all.
await sftp.connect({ ...ftpSetup, privateKey });
for (const csvObj of allcsvsToUpload) {
try {
csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`);
csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`);
logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
filename: csvObj.filename,
result: csvObj.result
filename: csvToUpload.filename,
result: csvToUpload.result
});
} catch (error) {
logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
filename: csvObj.filename,
error: error.message,
stack: error.stack
filename: csvToUpload.filename,
error: csvToUpload.message,
stack: csvToUpload.stack
});
throw error;
}
}
} catch (error) {
logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
throw error;