From 2439755f9e98365cf4d85cc08f426910e353f650 Mon Sep 17 00:00:00 2001 From: Dave Richer Date: Thu, 14 Nov 2024 08:34:21 -0800 Subject: [PATCH] feature/IO-3029-Enhanced-Logging-File-Based: Final Enhancements Signed-off-by: Dave Richer --- server.js | 7 ++----- server/email/mailer.js | 9 ++------- server/intellipay/intellipay.js | 6 ++---- server/utils/instanceMgr.js | 6 ++++++ server/utils/logger.js | 29 +++++++++++++++++++---------- server/utils/s3.js | 7 ++----- 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/server.js b/server.js index a92ab7241..f98ec67da 100644 --- a/server.js +++ b/server.js @@ -21,7 +21,7 @@ const { applyRedisHelpers } = require("./server/utils/redisHelpers"); const { applyIOHelpers } = require("./server/utils/ioHelpers"); const { redisSocketEvents } = require("./server/web-sockets/redisSocketEvents"); const { ElastiCacheClient, DescribeCacheClustersCommand } = require("@aws-sdk/client-elasticache"); -const { default: InstanceManager } = require("./server/utils/instanceMgr"); +const { InstanceRegion } = require("./server/utils/instanceMgr"); const CLUSTER_RETRY_BASE_DELAY = 100; const CLUSTER_RETRY_MAX_DELAY = 5000; @@ -114,10 +114,7 @@ const applyRoutes = ({ app }) => { */ const getRedisNodesFromAWS = async () => { const client = new ElastiCacheClient({ - region: InstanceManager({ - imex: "ca-central-1", - rome: "us-east-2" - }) + region: InstanceRegion() }); const params = { diff --git a/server/email/mailer.js b/server/email/mailer.js index 654f56cb3..6134053b6 100644 --- a/server/email/mailer.js +++ b/server/email/mailer.js @@ -1,6 +1,6 @@ const { isString, isEmpty } = require("lodash"); const { defaultProvider } = require("@aws-sdk/credential-provider-node"); -const { default: InstanceManager } = require("../utils/instanceMgr"); +const { InstanceRegion } = require("../utils/instanceMgr"); const aws = require("@aws-sdk/client-ses"); const nodemailer = require("nodemailer"); const logger = require("../utils/logger"); @@ -10,12 +10,7 @@ const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.e const sesConfig = { apiVersion: "latest", credentials: defaultProvider(), - region: isLocal - ? "ca-central-1" - : InstanceManager({ - imex: "ca-central-1", - rome: "us-east-2" - }) + region: InstanceRegion() }; if (isLocal) { diff --git a/server/intellipay/intellipay.js b/server/intellipay/intellipay.js index c231f8261..080deec90 100644 --- a/server/intellipay/intellipay.js +++ b/server/intellipay/intellipay.js @@ -17,12 +17,10 @@ require("dotenv").config({ const domain = process.env.NODE_ENV ? "secure" : "test"; const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager"); +const { InstanceRegion } = require("../utils/instanceMgr"); const client = new SecretsManagerClient({ - region: InstanceManager({ - imex: "ca-central-1", - rome: "us-east-2" - }) + region: InstanceRegion() }); const gqlClient = require("../graphql-client/graphql-client").client; diff --git a/server/utils/instanceMgr.js b/server/utils/instanceMgr.js index e83e32661..6a01e6904 100644 --- a/server/utils/instanceMgr.js +++ b/server/utils/instanceMgr.js @@ -44,4 +44,10 @@ function InstanceManager({ args, instance, debug, executeFunction, rome, promana return propToReturn === undefined ? null : propToReturn; } +exports.InstanceRegion = () => + InstanceManager({ + imex: "ca-central-1", + rome: "us-east-2" + }); + exports.default = InstanceManager; diff --git a/server/utils/logger.js b/server/utils/logger.js index c2b2a2079..fc8a84c29 100644 --- a/server/utils/logger.js +++ b/server/utils/logger.js @@ -11,6 +11,7 @@ const { isString, isEmpty } = require("lodash"); const { networkInterfaces, hostname } = require("node:os"); const { uploadFileToS3 } = require("./s3"); const { v4 } = require("uuid"); +const { InstanceRegion } = require("./instanceMgr"); const LOG_LEVELS = { error: { level: 0, name: "error" }, @@ -29,12 +30,18 @@ const S3_BUCKET_NAME = InstanceManager({ rome: "rome-large-log" }); +const region = InstanceRegion(); + const estimateLogSize = (logEntry) => { let estimatedSize = 0; for (const key in logEntry) { if (logEntry.hasOwnProperty(key)) { const value = logEntry[key]; - estimatedSize += key.length + (typeof value === "string" ? value.length : JSON.stringify(value).length); + if (value === undefined || value === null) { + estimatedSize += key.length; // Only count the key length if value is undefined or null + } else { + estimatedSize += key.length + (typeof value === "string" ? value.length : JSON.stringify(value).length); + } } } return estimatedSize; @@ -50,10 +57,7 @@ const createLogger = () => { const winstonCloudwatchTransportDefaults = { logGroupName: logGroupName, awsOptions: { - region: InstanceManager({ - imex: "ca-central-1", - rome: "us-east-2" - }) + region }, jsonMessage: true }; @@ -154,17 +158,23 @@ const createLogger = () => { meta }; + //https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/%5Btest%5D-%5Bip-172-31-42-218%5D-%5B2024-11-14T04-15-52.708Z%5D-%5B7a03efb9-9547-4f6b-acd4-c63a2d58a0c8%5D.json + const uploadLogToS3 = (logEntry, message, type, user) => { const uniqueId = v4(); const dateTimeString = new Date().toISOString().replace(/:/g, "-"); const envName = process.env?.NODE_ENV ? process.env.NODE_ENV : ""; - const logStreamName = `[${envName}]-[${internalHostname}]-[${dateTimeString}]-[${uniqueId}].json`; + const logStreamName = `${envName}-${internalHostname}-${dateTimeString}-${uniqueId}.json`; const logString = JSON.stringify(logEntry); + const webPath = isLocal + ? `https://${S3_BUCKET_NAME}.s3.localhost.localstack.cloud:4566/${logStreamName}` + : `https://${S3_BUCKET_NAME}.s3.${region}.amazonaws.com/${logStreamName}`; uploadFileToS3({ bucketName: S3_BUCKET_NAME, key: logStreamName, content: logString }) .then(() => { log("A log file has been uploaded to S3", "info", "S3", null, { logStreamName, + webPath, message: message?.slice(0, 200), type, user @@ -173,6 +183,7 @@ const createLogger = () => { .catch((err) => { log("Error in S3 Upload", "error", "S3", null, { logStreamName, + webPath, message: message?.slice(0, 100), type, user, @@ -182,16 +193,14 @@ const createLogger = () => { }; const checkAndUploadLog = () => { - const logString = JSON.stringify(logEntry); - const logSize = Buffer.byteLength(logString, "utf8"); + const estimatedSize = estimateLogSize(logEntry); - if (logSize > LOG_LENGTH_LIMIT * 0.9 || logSize > LOG_LENGTH_LIMIT) { + if (estimatedSize > LOG_LENGTH_LIMIT * 0.9 || estimatedSize > LOG_LENGTH_LIMIT) { uploadLogToS3(logEntry, message, type, user); return true; } return false; }; - // Upload log immediately if upload is true, otherwise check the log size. if (upload) { uploadLogToS3(logEntry, message, type, user); diff --git a/server/utils/s3.js b/server/utils/s3.js index 132fcc193..f87fc0fd2 100644 --- a/server/utils/s3.js +++ b/server/utils/s3.js @@ -7,15 +7,12 @@ const { CopyObjectCommand } = require("@aws-sdk/client-s3"); const { defaultProvider } = require("@aws-sdk/credential-provider-node"); -const { default: InstanceManager } = require("./instanceMgr"); +const { InstanceRegion } = require("./instanceMgr"); const { isString, isEmpty } = require("lodash"); const createS3Client = () => { const S3Options = { - region: InstanceManager({ - imex: "ca-central-1", - rome: "us-east-2" - }), + region: InstanceRegion(), credentials: defaultProvider() };