feature/IO-3029-Enhanced-Logging-File-Based: Final Enhancements

Signed-off-by: Dave Richer <dave@imexsystems.ca>

server.js

@@ -21,7 +21,7 @@ const { applyRedisHelpers } = require("./server/utils/redisHelpers");
 const { applyIOHelpers } = require("./server/utils/ioHelpers");
 const { redisSocketEvents } = require("./server/web-sockets/redisSocketEvents");
 const { ElastiCacheClient, DescribeCacheClustersCommand } = require("@aws-sdk/client-elasticache");
-const { default: InstanceManager } = require("./server/utils/instanceMgr");
+const { InstanceRegion } = require("./server/utils/instanceMgr");
 
 const CLUSTER_RETRY_BASE_DELAY = 100;
 const CLUSTER_RETRY_MAX_DELAY = 5000;
@@ -114,10 +114,7 @@ const applyRoutes = ({ app }) => {
  */
 const getRedisNodesFromAWS = async () => {
   const client = new ElastiCacheClient({
-    region: InstanceManager({
-      imex: "ca-central-1",
-      rome: "us-east-2"
-    })
+    region: InstanceRegion()
   });
 
   const params = {

server/email/mailer.js

@@ -1,6 +1,6 @@
 const { isString, isEmpty } = require("lodash");
 const { defaultProvider } = require("@aws-sdk/credential-provider-node");
-const { default: InstanceManager } = require("../utils/instanceMgr");
+const { InstanceRegion } = require("../utils/instanceMgr");
 const aws = require("@aws-sdk/client-ses");
 const nodemailer = require("nodemailer");
 const logger = require("../utils/logger");
@@ -10,12 +10,7 @@ const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.e
 const sesConfig = {
   apiVersion: "latest",
   credentials: defaultProvider(),
-  region: isLocal
-    ? "ca-central-1"
-    : InstanceManager({
-        imex: "ca-central-1",
-        rome: "us-east-2"
-      })
+  region: InstanceRegion()
 };
 
 if (isLocal) {

server/intellipay/intellipay.js

@@ -17,12 +17,10 @@ require("dotenv").config({
 const domain = process.env.NODE_ENV ? "secure" : "test";
 
 const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
+const { InstanceRegion } = require("../utils/instanceMgr");
 
 const client = new SecretsManagerClient({
-  region: InstanceManager({
-    imex: "ca-central-1",
-    rome: "us-east-2"
-  })
+  region: InstanceRegion()
 });
 
 const gqlClient = require("../graphql-client/graphql-client").client;

server/utils/instanceMgr.js

@@ -44,4 +44,10 @@ function InstanceManager({ args, instance, debug, executeFunction, rome, promana
   return propToReturn === undefined ? null : propToReturn;
 }
 
+exports.InstanceRegion = () =>
+  InstanceManager({
+    imex: "ca-central-1",
+    rome: "us-east-2"
+  });
+
 exports.default = InstanceManager;

server/utils/logger.js

@@ -11,6 +11,7 @@ const { isString, isEmpty } = require("lodash");
 const { networkInterfaces, hostname } = require("node:os");
 const { uploadFileToS3 } = require("./s3");
 const { v4 } = require("uuid");
+const { InstanceRegion } = require("./instanceMgr");
 
 const LOG_LEVELS = {
   error: { level: 0, name: "error" },
@@ -29,14 +30,20 @@ const S3_BUCKET_NAME = InstanceManager({
   rome: "rome-large-log"
 });
 
+const region = InstanceRegion();
+
 const estimateLogSize = (logEntry) => {
   let estimatedSize = 0;
   for (const key in logEntry) {
     if (logEntry.hasOwnProperty(key)) {
       const value = logEntry[key];
+      if (value === undefined || value === null) {
+        estimatedSize += key.length; // Only count the key length if value is undefined or null
+      } else {
         estimatedSize += key.length + (typeof value === "string" ? value.length : JSON.stringify(value).length);
       }
     }
+  }
   return estimatedSize;
 };
 
@@ -50,10 +57,7 @@ const createLogger = () => {
     const winstonCloudwatchTransportDefaults = {
       logGroupName: logGroupName,
       awsOptions: {
-        region: InstanceManager({
-          imex: "ca-central-1",
-          rome: "us-east-2"
-        })
+        region
       },
       jsonMessage: true
     };
@@ -154,17 +158,23 @@ const createLogger = () => {
         meta
       };
 
+      //https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/%5Btest%5D-%5Bip-172-31-42-218%5D-%5B2024-11-14T04-15-52.708Z%5D-%5B7a03efb9-9547-4f6b-acd4-c63a2d58a0c8%5D.json
+
       const uploadLogToS3 = (logEntry, message, type, user) => {
         const uniqueId = v4();
         const dateTimeString = new Date().toISOString().replace(/:/g, "-");
         const envName = process.env?.NODE_ENV ? process.env.NODE_ENV : "";
-        const logStreamName = `[${envName}]-[${internalHostname}]-[${dateTimeString}]-[${uniqueId}].json`;
+        const logStreamName = `${envName}-${internalHostname}-${dateTimeString}-${uniqueId}.json`;
         const logString = JSON.stringify(logEntry);
+        const webPath = isLocal
+          ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${logStreamName}`
+          : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${logStreamName}`;
 
         uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: logStreamName, content: logString })
           .then(() => {
             log("A log file has been uploaded to S3", "info", "S3", null, {
               logStreamName,
+              webPath,
               message: message?.slice(0, 200),
               type,
               user
@@ -173,6 +183,7 @@ const createLogger = () => {
           .catch((err) => {
             log("Error in S3 Upload", "error", "S3", null, {
               logStreamName,
+              webPath,
               message: message?.slice(0, 100),
               type,
               user,
@@ -182,16 +193,14 @@ const createLogger = () => {
       };
 
       const checkAndUploadLog = () => {
-        const logString = JSON.stringify(logEntry);
-        const logSize = Buffer.byteLength(logString, "utf8");
+        const estimatedSize = estimateLogSize(logEntry);
 
-        if (logSize > LOG_LENGTH_LIMIT * 0.9 || logSize > LOG_LENGTH_LIMIT) {
+        if (estimatedSize > LOG_LENGTH_LIMIT * 0.9 || estimatedSize > LOG_LENGTH_LIMIT) {
           uploadLogToS3(logEntry, message, type, user);
           return true;
         }
         return false;
       };
-
       // Upload log immediately if upload is true, otherwise check the log size.
       if (upload) {
         uploadLogToS3(logEntry, message, type, user);

server/utils/s3.js

@@ -7,15 +7,12 @@ const {
   CopyObjectCommand
 } = require("@aws-sdk/client-s3");
 const { defaultProvider } = require("@aws-sdk/credential-provider-node");
-const { default: InstanceManager } = require("./instanceMgr");
+const { InstanceRegion } = require("./instanceMgr");
 const { isString, isEmpty } = require("lodash");
 
 const createS3Client = () => {
   const S3Options = {
-    region: InstanceManager({
-      imex: "ca-central-1",
-      rome: "us-east-2"
-    }),
+    region: InstanceRegion(),
     credentials: defaultProvider()
   };