const path = require("path"); const queries = require("../graphql-client/queries"); const moment = require("moment-timezone"); const converter = require("json-2-csv"); const _ = require("lodash"); const logger = require("../utils/logger"); const fs = require("fs"); const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager"); require("dotenv").config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) }); let Client = require("ssh2-sftp-client"); const client = require("../graphql-client/graphql-client").client; const { sendServerEmail } = require("../email/sendemail"); const ftpSetup = { host: process.env.CHATTER_HOST, port: process.env.CHATTER_PORT, username: process.env.CHATTER_USER, privateKey: null, debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data), algorithms: { serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"] } }; const allcsvsToUpload = []; const allErrors = []; exports.default = async (req, res) => { // Only process if in production environment. if (process.env.NODE_ENV !== "production") { res.sendStatus(403); return; } // Only process if the appropriate token is provided. if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) { res.sendStatus(401); return; } // Send immediate response and continue processing. res.status(202).json({ success: true, message: "Processing request ...", timestamp: new Date().toISOString() }); try { logger.log("chatter-start", "DEBUG", "api", null, null); const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients. const specificShopIds = req.body.bodyshopIds; // ['uuid]; const { start, end, skipUpload } = req.body; //YYYY-MM-DD const batchSize = 10; const shopsToProcess = specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops; logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null); if (shopsToProcess.length === 0) { logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null); return; } const batchPromises = []; for (let i = 0; i < shopsToProcess.length; i += batchSize) { const batch = shopsToProcess.slice(i, i + batchSize); const batchPromise = (async () => { await processBatch(batch, start, end); if (skipUpload) { for (const csvObj of allcsvsToUpload) { await fs.promises.writeFile(`./logs/${csvObj.filename}`, csvObj.csv); } } else { await uploadViaSFTP(allcsvsToUpload); } })(); batchPromises.push(batchPromise); } await Promise.all(batchPromises); await sendServerEmail({ subject: `Chatter Report ${moment().format("MM-DD-YY")}`, text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\nUploaded:\n${JSON.stringify( allcsvsToUpload.map((x) => ({ filename: x.filename, count: x.count, result: x.result })), null, 2 )}` }); logger.log("chatter-end", "DEBUG", "api", null, null); } catch (error) { logger.log("chatter-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); } }; async function processBatch(batch, start, end) { for (const bodyshop of batch) { try { logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, { bodyshopid: bodyshop.id, start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"), ...(end && { end: moment(end).endOf("day") }) }); const chatterObject = jobs.map((j) => { return { poc_trigger_code: bodyshops_by_pk.chatterid, firstname: j.ownr_co_nm ? null : j.ownr_fn, lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln, transaction_id: j.ro_number, email: j.ownr_ea, phone_number: j.ownr_ph1 }; }); const ret = converter.json2csv(chatterObject, { emptyFieldValue: "" }); allcsvsToUpload.push({ count: chatterObject.length, csv: ret, filename: `${bodyshop.shopname}_solicitation_${moment().format("YYYYMMDD")}.csv` }); logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname }); } catch (error) { //Error at the shop level. logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack }); allErrors.push({ bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, shopname: bodyshop.shopname, fatal: true, errors: [error.toString()] }); } finally { allErrors.push({ bodyshopid: bodyshop.id, imexshopid: bodyshop.imexshopid, shopname: bodyshop.shopname }); } } } async function getPrivateKey() { // Connect to AWS Secrets Manager const client = new SecretsManagerClient({ region: "ca-central-1" }); const command = new GetSecretValueCommand({ SecretId: "CHATTER_PRIVATE_KEY" }); logger.log("chatter-get-private-key", "DEBUG", "api", null, null); try { const { SecretString, SecretBinary } = await client.send(command); if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null); const chatterPrivateKey = SecretString ? SecretString : Buffer.from(SecretBinary, "base64").toString("ascii"); return chatterPrivateKey; } catch (error) { logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack }); throw error; } } async function uploadViaSFTP(allcsvsToUpload) { const sftp = new Client(); sftp.on("error", (errors) => logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack }) ); try { //Get the private key from AWS Secrets Manager. const privateKey = await getPrivateKey(); //Connect to the FTP and upload all. await sftp.connect({ ...ftpSetup, privateKey }); for (const csvObj of allcsvsToUpload) { try { csvObj.result = await sftp.put(Buffer.from(csvObj.csv), `${csvObj.filename}`); logger.log("chatter-sftp-upload", "DEBUG", "api", null, { filename: csvObj.filename, result: csvObj.result }); } catch (error) { logger.log("chatter-sftp-upload-error", "ERROR", "api", null, { filename: csvObj.filename, error: error.message, stack: error.stack }); throw error; } } } catch (error) { logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack }); throw error; } finally { sftp.end(); } }