bodyshop/server/data/chatter.js

const path = require("path");
const queries = require("../graphql-client/queries");
const moment = require("moment-timezone");
const converter = require("json-2-csv");
const _ = require("lodash");
const logger = require("../utils/logger");
const fs = require("fs");
const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
require("dotenv").config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) });
let Client = require("ssh2-sftp-client");

const client = require("../graphql-client/graphql-client").client;
const { sendServerEmail } = require("../email/sendemail");

const ftpSetup = {
  host: process.env.CHATTER_HOST,
  port: process.env.CHATTER_PORT,
  username: process.env.CHATTER_USER,
  privateKey: null,
  debug: (message, ...data) => logger.log(message, "DEBUG", "api", null, data),
  algorithms: {
    serverHostKey: ["ssh-rsa", "ssh-dss", "rsa-sha2-256", "rsa-sha2-512", "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384"]
  }
};

const allChatterObjects = [];
const allErrors = [];

exports.default = async (req, res) => {
  // Only process if in production environment.
  if (process.env.NODE_ENV !== "production") {
    res.sendStatus(403);
    return;
  }
  // Only process if the appropriate token is provided.
  if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) {
    res.sendStatus(401);
    return;
  }

  // Send immediate response and continue processing.
  res.status(202).json({
    success: true,
    message: "Processing request ...",
    timestamp: new Date().toISOString()
  });

  try {
    logger.log("chatter-start", "DEBUG", "api", null, null);
    const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS); //Query for the List of Bodyshop Clients.
    const specificShopIds = req.body.bodyshopIds; // ['uuid];
    const { start, end, skipUpload } = req.body; //YYYY-MM-DD

    const batchSize = 10;

    const shopsToProcess =
      specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
    logger.log("chatter-shopsToProcess-generated", "DEBUG", "api", null, null);

    if (shopsToProcess.length === 0) {
      logger.log("chatter-shopsToProcess-empty", "DEBUG", "api", null, null);
      return;
    }

    for (let i = 0; i < shopsToProcess.length; i += batchSize) {
      const batch = shopsToProcess.slice(i, i + batchSize);
      await processBatch(batch, start, end);
    }

    const csvToUpload = {
      count: allChatterObjects.length,
      csv: converter.json2csv(allChatterObjects, { emptyFieldValue: "" }),
      filename: `IMEX_ONLINE_solicitation_${moment().format("YYYYMMDD")}.csv`
    };
    if (skipUpload) {
      await fs.promises.writeFile(`./logs/${csvToUpload.filename}`, csvToUpload.csv);
    } else {
      await uploadViaSFTP(csvToUpload);
    }

    await sendServerEmail({
      subject: `Chatter Report ${moment().format("MM-DD-YY")}`,
      text: `Errors:\n${JSON.stringify(allErrors, null, 2)}\n\n
      Uploaded:\n${JSON.stringify({ filename: csvToUpload.filename, count: csvToUpload.count, result: csvToUpload.result }, null, 2)}`
    });

    logger.log("chatter-end", "DEBUG", "api", null, null);
  } catch (error) {
    logger.log("chatter-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
  }
};

async function processBatch(batch, start, end) {
  for (const bodyshop of batch) {
    try {
      logger.log("chatter-start-shop-extract", "DEBUG", "api", bodyshop.id, {
        shopname: bodyshop.shopname
      });

      const { jobs, bodyshops_by_pk } = await client.request(queries.CHATTER_QUERY, {
        bodyshopid: bodyshop.id,
        start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"),
        ...(end && { end: moment(end).endOf("day") })
      });

      const chatterObject = jobs.map((j) => {
        return {
          poc_trigger_code: bodyshops_by_pk.chatterid,
          firstname: j.ownr_co_nm ? null : j.ownr_fn,
          lastname: j.ownr_co_nm ? j.ownr_co_nm : j.ownr_ln,
          transaction_id: j.ro_number,
          email: j.ownr_ea,
          phone_number: j.ownr_ph1
        };
      });
      allChatterObjects.push(...chatterObject);
      logger.log("chatter-end-shop-extract", "DEBUG", "api", bodyshop.id, {
        shopname: bodyshop.shopname
      });
    } catch (error) {
      //Error at the shop level.
      logger.log("chatter-error-shop", "ERROR", "api", bodyshop.id, { error: error.message, stack: error.stack });

      allErrors.push({
        bodyshopid: bodyshop.id,
        imexshopid: bodyshop.imexshopid,
        shopname: bodyshop.shopname,
        fatal: true,
        errors: [error.toString()]
      });
    } finally {
      allErrors.push({
        bodyshopid: bodyshop.id,
        imexshopid: bodyshop.imexshopid,
        shopname: bodyshop.shopname
      });
    }
  }
}

async function getPrivateKey() {
  // Connect to AWS Secrets Manager
  const client = new SecretsManagerClient({ region: "ca-central-1" });
  const command = new GetSecretValueCommand({ SecretId: "CHATTER_PRIVATE_KEY" });

  logger.log("chatter-get-private-key", "DEBUG", "api", null, null);
  try {
    const { SecretString, SecretBinary } = await client.send(command);
    if (SecretString || SecretBinary) logger.log("chatter-retrieved-private-key", "DEBUG", "api", null, null);
    const chatterPrivateKey = SecretString ? SecretString : Buffer.from(SecretBinary, "base64").toString("ascii");
    return chatterPrivateKey;
  } catch (error) {
    logger.log("chatter-get-private-key", "ERROR", "api", null, { error: error.message, stack: error.stack });
    throw error;
  }
}

async function uploadViaSFTP(csvToUpload) {
  const sftp = new Client();
  sftp.on("error", (errors) =>
    logger.log("chatter-sftp-connection-error", "ERROR", "api", null, { error: errors.message, stack: errors.stack })
  );
  try {
    //Get the private key from AWS Secrets Manager.
    const privateKey = await getPrivateKey();

    //Connect to the FTP and upload all.
    await sftp.connect({ ...ftpSetup, privateKey });

    try {
      csvToUpload.result = await sftp.put(Buffer.from(csvToUpload.csv), `${csvToUpload.filename}`);
      logger.log("chatter-sftp-upload", "DEBUG", "api", null, {
        filename: csvToUpload.filename,
        result: csvToUpload.result
      });
    } catch (error) {
      logger.log("chatter-sftp-upload-error", "ERROR", "api", null, {
        filename: csvToUpload.filename,
        error: csvToUpload.message,
        stack: csvToUpload.stack
      });
      throw error;
    }
  } catch (error) {
    logger.log("chatter-sftp-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
    throw error;
  } finally {
    sftp.end();
  }
}