feature/IO-3029-Enhanced-Logging-File-Based: Add File based S3 Logging.

Signed-off-by: Dave Richer <dave@imexsystems.ca>

server/utils/logger.js

@@ -9,6 +9,8 @@ const winston = require("winston");
 const WinstonCloudWatch = require("winston-cloudwatch");
 const { isString, isEmpty } = require("lodash");
 const { networkInterfaces, hostname } = require("node:os");
+const { uploadFileToS3 } = require("./s3");
+const { v4 } = require("uuid");
 
 const LOG_LEVELS = {
   error: { level: 0, name: "error" },
@@ -20,6 +22,24 @@ const LOG_LEVELS = {
   silly: { level: 6, name: "silly" }
 };
 
+const LOG_LENGTH_LIMIT = 256 * 1024; // 256KB
+
+const S3_BUCKET_NAME = InstanceManager({
+  imex: "imex-large-log",
+  rome: "rome-large-log"
+});
+
+const estimateLogSize = (logEntry) => {
+  let estimatedSize = 0;
+  for (const key in logEntry) {
+    if (logEntry.hasOwnProperty(key)) {
+      const value = logEntry[key];
+      estimatedSize += key.length + (typeof value === "string" ? value.length : JSON.stringify(value).length);
+    }
+  }
+  return estimatedSize;
+};
+
 const normalizeLevel = (level) => (level ? level.toLowerCase() : LOG_LEVELS.debug.name);
 
 const createLogger = () => {
@@ -124,15 +144,56 @@ const createLogger = () => {
       );
     }
 
-    const log = (message, type, user, record, meta) => {
-      winstonLogger.log({
+    const log = (message, type, user, record, meta, upload) => {
+      const logEntry = {
         level: normalizeLevel(type),
         message,
         user,
         record,
         hostname: internalHostname,
         meta
-      });
+      };
+
+      const uploadLogToS3 = (logEntry, message, type, user) => {
+        const uniqueId = v4();
+        const dateTimeString = new Date().toISOString().replace(/:/g, "-");
+        const logStreamName = `${dateTimeString}-${internalHostname}-${uniqueId}`;
+        const logString = JSON.stringify(logEntry);
+
+        uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: logStreamName, content: logString })
+          .then(() => {
+            log("A log file has been uploaded to S3", "info", "S3", null, {
+              logStreamName,
+              message: message?.slice(0, 200),
+              type,
+              user
+            });
+          })
+          .catch((err) => {
+            log("Error in S3 Upload", "error", "S3", null, {
+              logStreamName,
+              message: message?.slice(0, 100),
+              type,
+              user,
+              errorMessage: err?.message?.slice(0, 100)
+            });
+          });
+      };
+
+      const checkAndUploadLog = () => {
+        const logString = JSON.stringify(logEntry);
+        const logSize = Buffer.byteLength(logString, "utf8");
+
+        if (logSize > LOG_LENGTH_LIMIT * 0.9 || logSize > LOG_LENGTH_LIMIT) {
+          uploadLogToS3(logEntry, message, type, user);
+          return true;
+        }
+        return false;
+      };
+
+      if (upload || checkAndUploadLog()) return;
+
+      winstonLogger.log(logEntry);
     };
 
     return {