From 0e9ad1258d952472b7422e8a127e090d9a4084ba Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Mon, 16 Sep 2024 16:13:13 -0700 Subject: [PATCH 1/3] IO-2921 ChatterID DB field for Bodyshop Signed-off-by: Allan Carr --- hasura/metadata/tables.yaml | 1 + .../down.sql | 4 ++++ .../up.sql | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/down.sql create mode 100644 hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/up.sql diff --git a/hasura/metadata/tables.yaml b/hasura/metadata/tables.yaml index cfc275500..e5e086300 100644 --- a/hasura/metadata/tables.yaml +++ b/hasura/metadata/tables.yaml @@ -918,6 +918,7 @@ - bill_tax_rates - cdk_configuration - cdk_dealerid + - chatterid - city - claimscorpid - convenient_company diff --git a/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/down.sql b/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/down.sql new file mode 100644 index 000000000..25291f5fd --- /dev/null +++ b/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/down.sql @@ -0,0 +1,4 @@ +-- Could not auto-generate a down migration. +-- Please write an appropriate down migration for the SQL below: +-- alter table "public"."bodyshops" add column "chatterid" text +-- null; diff --git a/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/up.sql b/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/up.sql new file mode 100644 index 000000000..76e217875 --- /dev/null +++ b/hasura/migrations/1726528225033_alter_table_public_bodyshops_add_column_chatterid/up.sql @@ -0,0 +1,2 @@ +alter table "public"."bodyshops" add column "chatterid" text + null; From 1bb2212e4aa1f9efdb7308c491fba77219360691 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Tue, 17 Sep 2024 12:55:08 -0700 Subject: [PATCH 2/3] IO-2921 CARSTAR Canada Chatter Datapump Signed-off-by: Allan Carr --- server/data/chatter.js | 171 +++++++++++++++++++++++++++++++ server/data/data.js | 1 + server/graphql-client/queries.js | 29 ++++++ server/routes/dataRoutes.js | 3 +- 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 server/data/chatter.js diff --git a/server/data/chatter.js b/server/data/chatter.js new file mode 100644 index 000000000..69c7707a8 --- /dev/null +++ b/server/data/chatter.js @@ -0,0 +1,171 @@ +const path = require("path"); +const queries = require("../graphql-client/queries"); +const moment = require("moment-timezone"); +const converter = require("json-2-csv"); +const _ = require("lodash"); +const logger = require("../utils/logger"); +const fs = require("fs"); +const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager"); +require("dotenv").config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) }); +let Client = require("ssh2-sftp-client"); + +const client = require("../graphql-client/graphql-client").client; +const { sendServerEmail } = require("../email/sendemail"); + +const ftpSetup = { + host: process.env.CHATTER_HOST, + port: process.env.CHATTER_PORT, + username: process.env.CHATTER_USER, + privateKey: null, + debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), + algorithms: { + serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] + } +}; + +exports.default = async (req, res) => { + // Only process if in production environment. + if (process.env.NODE_ENV !== "production") { + res.sendStatus(403); + return; + } + + //Query for the List of Bodyshop Clients. + logger.log("chatter-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); + + const specificShopIds = req.body.bodyshopIds; // ['uuid] + const { start, end, skipUpload } = req.body; //YYYY-MM-DD + if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { + res.sendStatus(401); + return; + } + const allcsvsToUpload = []; + const allErrors = []; + try { + for (const bodyshop of specificShopIds ? bodyshops.filter((b) => specificShopIds.includes(b.id)) : bodyshops) { + logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + try { + const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, { + bodyshopid: bodyshop.id, + start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"), + ...(end && { end: moment(end).endOf("day") }) + }); + + const chatterObject = jobs.map((j) => { + return { + poc_trigger_code: bodyshops_by_pk.chatterid, + firstname: j.ownr_co_nm ? null : j.ownr_fn, + lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln, + transaction_id: j.ro_number, + email: j.ownr_ea, + phone_number: j.ownr_ph1 + }; + }); + + var ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); + + allcsvsToUpload.push({ + count: chatterObject.length, + csv: ret, + filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` + }); + + logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { + shopname: bodyshop.shopname + }); + } catch (error) { + //Error at the shop level. + logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { + ...error + }); + + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname, + fatal: true, + errors: [error.toString()] + }); + } finally { + allErrors.push({ + bodyshopid: bodyshop.id, + imexshopid: bodyshop.imexshopid, + shopname: bodyshop.shopname + }); + } + } + + if (skipUpload) { + for (const csvObj of allcsvsToUpload) { + fs.writeFileSync(`./logs/${csvObj.filename}`, csvObj.csv); + } + + res.json(allcsvsToUpload); + sendServerEmail({ + subject: `Chatter Report ${moment().format("MM-DD-YY")}`, + text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} + Uploaded: ${JSON.stringify( + allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), + null, + 2 + )} + ` + }); + return; + } + + let sftp = new Client(); + sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors })); + try { + //Get the private key from AWS Secrets Manager. + ftpSetup.privateKey = await getPrivateKey(); + + //Connect to the FTP and upload all. + await sftp.connect(ftpSetup); + + for (const csvObj of allcsvsToUpload) { + logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename }); + + const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`); + logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult }); + } + + //***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml + } catch (error) { + logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error }); + } finally { + sftp.end(); + } + sendServerEmail({ + subject: `Chatter Report ${moment().format("MM-DD-YY")}`, + text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} + Uploaded: ${JSON.stringify( + allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count })), + null, + 2 + )}` + }); + res.sendStatus(200); + } catch (error) { + res.status(200).json(error); + } +}; + +async function getPrivateKey() { + // Connect to AWS Secrets Manager + const client = new SecretsManagerClient({ region: "ca-central-1" }); + const command = new GetSecretValueCommand({ SecretId: CHATTER_PRIVATE_KEY }); + + logger.log("chatter-get-private-key", "DEBUG", "api", null, null); + try { + const { SecretString, SecretBinary } = await client.send(command); + if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null); + return SecretString || Buffer.from(SecretBinary, "base64").toString("ascii"); + } catch (error) { + logger.log("chatter-get-private-key", "ERROR", "api", null, error); + throw err; + } +} diff --git a/server/data/data.js b/server/data/data.js index 1656c1878..0f6fcd30c 100644 --- a/server/data/data.js +++ b/server/data/data.js @@ -1,4 +1,5 @@ exports.arms = require("./arms").default; exports.autohouse = require("./autohouse").default; +exports.chatter = require("./chatter").default; exports.claimscorp = require("./claimscorp").default; exports.kaizen = require("./kaizen").default; diff --git a/server/graphql-client/queries.js b/server/graphql-client/queries.js index 65cbef473..423ca673b 100644 --- a/server/graphql-client/queries.js +++ b/server/graphql-client/queries.js @@ -832,6 +832,25 @@ exports.AUTOHOUSE_QUERY = `query AUTOHOUSE_EXPORT($start: timestamptz, $bodyshop } }`; +exports.CHATTER_QUERY = `query CHATTER_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { + bodyshops_by_pk(id: $bodyshopid){ + id + shopname + chatterid + timezone + } + jobs(where: {_and: [{converted: {_eq: true}}, {actual_delivery: {_gt: $start}}, {actual_delivery: {_lte: $end}}, {shopid: {_eq: $bodyshopid}}, {_or: [{ownr_ph1: {_is_null: false}}, {ownr_ea: {_is_null: false}}]}]}) { + id + created_at + ro_number + ownr_fn + ownr_ln + ownr_co_nm + ownr_ph1 + ownr_ea + } +}`; + exports.CLAIMSCORP_QUERY = `query CLAIMSCORP_EXPORT($start: timestamptz, $bodyshopid: uuid!, $end: timestamptz) { bodyshops_by_pk(id: $bodyshopid){ id @@ -1732,6 +1751,16 @@ exports.GET_AUTOHOUSE_SHOPS = `query GET_AUTOHOUSE_SHOPS { } }`; +exports.GET_CHATTER_SHOPS = `query GET_CHATTER_SHOPS { + bodyshops(where: {chatterid: {_is_null: false}, _or: {chatterid: {_neq: ""}}}){ + id + shopname + chatterid + imexshopid + timezone + } +}`; + exports.GET_CLAIMSCORP_SHOPS = `query GET_CLAIMSCORP_SHOPS { bodyshops(where: {claimscorpid: {_is_null: false}, _or: {claimscorpid: {_neq: ""}}}){ id diff --git a/server/routes/dataRoutes.js b/server/routes/dataRoutes.js index d7fe6e640..a12563282 100644 --- a/server/routes/dataRoutes.js +++ b/server/routes/dataRoutes.js @@ -1,9 +1,10 @@ const express = require("express"); const router = express.Router(); -const { autohouse, claimscorp, kaizen } = require("../data/data"); +const { autohouse, claimscorp, chatter, kaizen } = require("../data/data"); router.post("/ah", autohouse); router.post("/cc", claimscorp); +router.post("/chatter", chatter); router.post("/kaizen", kaizen); module.exports = router; From 1a5c71048cee78d60c19b0e9b435019426f346d1 Mon Sep 17 00:00:00 2001 From: Allan Carr Date: Wed, 18 Sep 2024 10:04:45 -0700 Subject: [PATCH 3/3] IO-2921 CARSTAR Canada Requested Adjustments Signed-off-by: Allan Carr --- server/data/chatter.js | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/server/data/chatter.js b/server/data/chatter.js index 69c7707a8..85dbe4a9b 100644 --- a/server/data/chatter.js +++ b/server/data/chatter.js @@ -22,7 +22,6 @@ const ftpSetup = { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; - exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { @@ -30,16 +29,16 @@ exports.default = async (req, res) => { return; } - //Query for the List of Bodyshop Clients. - logger.log("chatter-start", "DEBUG", "api", null, null); - const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); - - const specificShopIds = req.body.bodyshopIds; // ['uuid] - const { start, end, skipUpload } = req.body; //YYYY-MM-DD if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { res.sendStatus(401); return; } + //Query for the List of Bodyshop Clients. + logger.log("chatter-start", "DEBUG", "api", null, null); + const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); + const specificShopIds = req.body.bodyshopIds; // ['uuid] + const { start, end, skipUpload } = req.body; //YYYY-MM-DD + const allcsvsToUpload = []; const allErrors = []; try { @@ -65,7 +64,7 @@ exports.default = async (req, res) => { }; }); - var ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); + const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); allcsvsToUpload.push({ count: chatterObject.length, @@ -100,10 +99,9 @@ exports.default = async (req, res) => { if (skipUpload) { for (const csvObj of allcsvsToUpload) { - fs.writeFileSync(`./logs/${csvObj.filename}`, csvObj.csv); + fs.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); } - res.json(allcsvsToUpload); sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, text: `Errors: ${allErrors.map((e) => JSON.stringify(e, null, 2))} @@ -114,10 +112,11 @@ exports.default = async (req, res) => { )} ` }); + res.json(allcsvsToUpload); return; } - let sftp = new Client(); + const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-error", "ERROR", "api", null, { ...errors })); try { //Get the private key from AWS Secrets Manager. @@ -132,8 +131,6 @@ exports.default = async (req, res) => { const uploadResult = await sftp.put(Buffer.from(csvObj.xml), `/${csvObj.filename}`); logger.log("chatter-sftp-upload-result", "DEBUG", "api", null, { uploadResult }); } - - //***TODO Change filing naming when creating the cron job. IM_ShopInternalName_DDMMYYYY_HHMMSS.xml } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { ...error }); } finally {