From 1bb2212e4aa1f9efdb7308c491fba77219360691 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Tue, 17 Sep 2024 12:55:08 -0700 Subject: [PATCH] IO-2921 CARSTAR Canada Chatter Datapump Signed-off-by: Allan Carr --- server/data/chatter.js | 171 +++++++++++++++++++++++++++++++ server/data/data.js | 1 + server/graphql-client/queries.js | 29 ++++++ server/routes/dataRoutes.js | 3 +- 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 server/data/chatter.js diff --git a/server/data/chatter.js b/server/data/chatter.js new file mode 100644 index 000000000..69c7707a8 --- /dev/null +++ b/server/data/chatter.js @@ -0,0 +1,171 @@ +const path = require("path"); +const queries = require("../graphql-client/queries"); +const moment = require("moment-timezone"); +const converter = require("json-2-csv"); +const _ = require("lodash"); +const logger = require("../utils/logger"); +const fs = require("fs"); +const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager"); +require("dotenv").config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) }); +let Client = require("ssh2-sftp-client"); + +const client = require("../graphql-client/graphql-client").client; +const { sendServerEmail } = require("../email/sendemail"); + +const ftpSetup = { + host: process.env.CHATTER_HOST, + port: process.env.CHATTER_PORT, + username: process.env.CHATTER_USER, + privateKey: null, + debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + algorithms: { + serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] + } +}; + +exports.default = async (req, res) => { + // Only process if in production environment. + if (process.env.NODE_ENV !== "production") { + res.sendStatus(403); + return; + } + + //Query for the List of Bodyshop Clients. + logger.log("chatter-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); + + const specificShopIds = req.body.bodyshopIds; // ['uuid] + const { start, end, skipUpload } = req.body; //YYYY-MM-DD + if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { + res.sendStatus(401); + return; + } + const allcsvsToUpload = []; + const allErrors = []; + try { + for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) { + logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + try { + const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, { + bodyshopid: bodyshop.id, + start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }); + + const chatterObject = jobs.map((j) => { + return { + poc_trigger_code: bodyshops_by_pk.chatterid, + firstname: j.ownr_co_nm ? null : j.ownr_fn, + lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln, + transaction_id: j.ro_number, + email: j.ownr_ea, + phone_number: j.ownr_ph1 + }; + }); + + var ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); + + allcsvsToUpload.push({ + count: chatterObject.length, + csv: ret, + filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` + }); + + logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + } catch (error) { + //Error at the shop level. + logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { + ...error + }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname, + fatal: true, + errors: [error.toString()] + }); + } finally { + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname + }); + } + } + + if (skipUpload) { + for (const csvObj of allcsvsToUpload) { + fs.writeFileSync(`./logs/${csvObj.filename}`, csvObj.csv); + } + + res.json(allcsvsToUpload); + sendServerEmail({ + subject: `Chatter Report ${moment().format("MM-DD-YY")}`, + text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} + Uploaded: ${JSON.stringify( + allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), + null, + 2 + )} + ` + }); + return; + } + + let sftp = new Client(); + sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors })); + try { + //Get the private key from AWS Secrets Manager. + ftpSetup.privateKey = await getPrivateKey(); + + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + for (const csvObj of allcsvsToUpload) { + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename }); + + const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`); + logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult }); + } + + //***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml + } catch (error) { + logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error }); + } finally { + sftp.end(); + } + sendServerEmail({ + subject: `Chatter Report ${moment().format("MM-DD-YY")}`, + text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} + Uploaded: ${JSON.stringify( + allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), + null, + 2 + )}` + }); + res.sendStatus(200); + } catch (error) { + res.status(200).json(error); + } +}; + +async function getPrivateKey() { + // Connect to AWS Secrets Manager + const client = new SecretsManagerClient({ region: "ca-central-1" }); + const command = new GetSecretValueCommand({ SecretId: CHATTER_PRIVATE_KEY }); + + logger.log("chatter-get-private-key", "DEBUG", "api", null, null); + try { + const { SecretString, SecretBinary } = await client.send(command); + if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null); + return SecretString || Buffer.from(SecretBinary, "base64").toString("ascii"); + } catch (error) { + logger.log("chatter-get-private-key", "ERROR", "api", null, error); + throw err; + } +} diff --git a/server/data/data.js b/server/data/data.js index 1656c1878..0f6fcd30c 100644 --- a/server/data/data.js +++ b/server/data/data.js @@ -1,4 +1,5 @@ exports.arms = require("./arms").default; exports.autohouse = require("./autohouse").default; +exports.chatter = require("./chatter").default; exports.claimscorp = require("./claimscorp").default; exports.kaizen = require("./kaizen").default; diff --git a/server/graphql-client/queries.js b/server/graphql-client/queries.js index 65cbef473..423ca673b 100644 --- a/server/graphql-client/queries.js +++ b/server/graphql-client/queries.js @@ -832,6 +832,25 @@ exports.AUTOHOUSE_QUERY = `query AUTOHOUSE_EXPORT($start: timestamptz, $bodyshop } }`; +exports.CHATTER_QUERY = `query CHATTER_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { + bodyshops_by_pk(id: $bodyshopid){ + id + shopname + chatterid + timezone + } + jobs(where: {_and: [{converted: {_eq: true}}, {actual_delivery: {_gt: $start}}, {actual_delivery: {_lte: $end}}, {shopid: {_eq: $bodyshopid}}, {_or: [{ownr_ph1: {_is_null: false}}, {ownr_ea: {_is_null: false}}]}]}) { + id + created_at + ro_number + ownr_fn + ownr_ln + ownr_co_nm + ownr_ph1 + ownr_ea + } +}`; + exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { bodyshops_by_pk(id: $bodyshopid){ id @@ -1732,6 +1751,16 @@ exports.GET_AUTOHOUSE_SHOPS = `query GET_AUTOHOUSE_SHOPS { } }`; +exports.GET_CHATTER_SHOPS = `query GET_CHATTER_SHOPS { + bodyshops(where: {chatterid: {_is_null: false}, _or: {chatterid: {_neq: ""}}}){ + id + shopname + chatterid + imexshopid + timezone + } +}`; + exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS { bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){ id diff --git a/server/routes/dataRoutes.js b/server/routes/dataRoutes.js index d7fe6e640..a12563282 100644 --- a/server/routes/dataRoutes.js +++ b/server/routes/dataRoutes.js @@ -1,9 +1,10 @@ const express = require("express"); const router = express.Router(); -const { autohouse, claimscorp, kaizen } = require("../data/data"); +const { autohouse, claimscorp, chatter, kaizen } = require("../data/data"); router.post("/ah", autohouse); router.post("/cc", claimscorp); +router.post("/chatter", chatter); router.post("/kaizen", kaizen); module.exports = router;