Merged in feature/IO-3556-Chattr-Integration (pull request #2976)

Feature/IO-3556 Chattr Integration
This commit is contained in:
Dave Richer
2026-02-10 22:28:54 +00:00
17 changed files with 809 additions and 96 deletions

View File

@@ -0,0 +1,126 @@
const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
const { defaultProvider } = require("@aws-sdk/credential-provider-node");
const { isString, isEmpty } = require("lodash");
const CHATTER_BASE_URL = process.env.CHATTER_API_BASE_URL || "https://api.chatterresearch.com";
const AWS_REGION = process.env.AWS_REGION || "ca-central-1";
// Configure SecretsManager client with localstack support
const secretsClientOptions = {
region: AWS_REGION,
credentials: defaultProvider()
};
const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME);
if (isLocal) {
secretsClientOptions.endpoint = `http://${process.env.LOCALSTACK_HOSTNAME}:4566`;
}
const secretsClient = new SecretsManagerClient(secretsClientOptions);
/**
* Chatter API Client for making requests to the Chatter API
*/
class ChatterApiClient {
constructor({ baseUrl, apiToken }) {
if (!apiToken) throw new Error("ChatterApiClient requires apiToken");
this.baseUrl = String(baseUrl || "").replace(/\/+$/, "");
this.apiToken = apiToken;
}
async createLocation(companyId, payload) {
return this.request(`/api/v1/companies/${companyId}/locations`, {
method: "POST",
body: payload
});
}
async postInteraction(companyId, payload) {
return this.request(`/api/v1/companies/${companyId}/solicitation/interaction`, {
method: "POST",
body: payload
});
}
async request(path, { method = "GET", body } = {}) {
const res = await fetch(this.baseUrl + path, {
method,
headers: {
"Api-Token": this.apiToken,
Accept: "application/json",
...(body ? { "Content-Type": "application/json" } : {})
},
body: body ? JSON.stringify(body) : undefined
});
const text = await res.text();
const data = text ? safeJson(text) : null;
if (!res.ok) {
const err = new Error(`Chatter API error ${res.status} | ${data?.message}`);
err.status = res.status;
err.data = data;
throw err;
}
return data;
}
}
/**
* Safely parse JSON, returning original text if parsing fails
*/
function safeJson(text) {
try {
return JSON.parse(text);
} catch {
return text;
}
}
/**
* Fetches Chatter API token from AWS Secrets Manager
* SecretId format: CHATTER_COMPANY_KEY_<companyId>
*
* @param {string|number} companyId - The company ID
* @returns {Promise<string>} The API token
*/
async function getChatterApiToken(companyId) {
const key = String(companyId ?? "").trim();
if (!key) throw new Error("getChatterApiToken: companyId is required");
// Optional override for development/testing
if (process.env.CHATTER_API_TOKEN) return process.env.CHATTER_API_TOKEN;
const secretId = `CHATTER_COMPANY_KEY_${key}`;
const command = new GetSecretValueCommand({ SecretId: secretId });
const { SecretString, SecretBinary } = await secretsClient.send(command);
const token =
(SecretString && SecretString.trim()) ||
(SecretBinary && Buffer.from(SecretBinary, "base64").toString("ascii").trim()) ||
"";
if (!token) throw new Error(`Chatter API token secret is empty: ${secretId}`);
return token;
}
/**
* Creates a Chatter API client instance
*
* @param {string|number} companyId - The company ID
* @param {string} [baseUrl] - Optional base URL override
* @returns {Promise<ChatterApiClient>} Configured API client
*/
async function createChatterClient(companyId, baseUrl = CHATTER_BASE_URL) {
const apiToken = await getChatterApiToken(companyId);
return new ChatterApiClient({ baseUrl, apiToken });
}
module.exports = {
ChatterApiClient,
getChatterApiToken,
createChatterClient,
safeJson,
CHATTER_BASE_URL
};

View File

@@ -0,0 +1,123 @@
const DEFAULT_COMPANY_ID = process.env.CHATTER_DEFAULT_COMPANY_ID;
const client = require("../graphql-client/graphql-client").client;
const { createChatterClient } = require("./chatter-client");
const InstanceManager = require("../utils/instanceMgr").default;
const GET_BODYSHOP_FOR_CHATTER = `
query GET_BODYSHOP_FOR_CHATTER($id: uuid!) {
bodyshops_by_pk(id: $id) {
id
shopname
address1
city
state
zip_post
imexshopid
chatterid
chatter_company_id
}
}
`;
const UPDATE_BODYSHOP_CHATTER_FIELDS = `
mutation UPDATE_BODYSHOP_CHATTER_FIELDS($id: uuid!, $chatter_company_id: String!, $chatterid: String!) {
update_bodyshops_by_pk(pk_columns: {id: $id}, _set: {chatter_company_id: $chatter_company_id, chatterid: $chatterid}) {
id
chatter_company_id
chatterid
}
}
`;
const createLocation = async (req, res) => {
const { logger } = req;
const { bodyshopID, googlePlaceID } = req.body;
console.dir({ body: req.body });
if (!DEFAULT_COMPANY_ID) {
logger.log("chatter-create-location-no-default-company", "warn", null, null, { bodyshopID });
return res.json({ success: false, message: "No default company set" });
}
if (!googlePlaceID) {
logger.log("chatter-create-location-no-google-place-id", "warn", null, null, { bodyshopID });
return res.json({ success: false, message: "No google place id provided" });
}
if (!bodyshopID) {
logger.log("chatter-create-location-invalid-bodyshop", "warn", null, null, { bodyshopID });
return res.json({ success: false, message: "No bodyshop id" });
}
try {
const { bodyshops_by_pk: bodyshop } = await client.request(GET_BODYSHOP_FOR_CHATTER, { id: bodyshopID });
if (!bodyshop) {
logger.log("chatter-create-location-bodyshop-not-found", "warn", null, null, { bodyshopID });
return res.json({ success: false, message: "Bodyshop not found" });
}
if (bodyshop.chatter_company_id && bodyshop.chatterid) {
logger.log("chatter-create-location-already-exists", "warn", null, null, {
bodyshopID
});
return res.json({ success: false, message: "This Bodyshop already has a location associated with it" });
}
const chatterApi = await createChatterClient(DEFAULT_COMPANY_ID);
const locationIdentifier = `${DEFAULT_COMPANY_ID}-${bodyshop.id}`;
const locationPayload = {
name: bodyshop.shopname,
locationIdentifier: locationIdentifier,
address: bodyshop.address1,
postalCode: bodyshop.zip_post,
state: bodyshop.state,
city: bodyshop.city,
country: InstanceManager({ imex: "Canada", rome: "US" }),
googlePlaceId: googlePlaceID,
status: "active"
};
logger.log("chatter-create-location-calling-api", "info", null, null, { bodyshopID, locationIdentifier });
const response = await chatterApi.createLocation(DEFAULT_COMPANY_ID, locationPayload);
if (!response.location?.id) {
logger.log("chatter-create-location-no-location-id", "error", null, null, { bodyshopID, response });
return res.json({ success: false, message: "No location ID in response", data: response });
}
await client.request(UPDATE_BODYSHOP_CHATTER_FIELDS, {
id: bodyshopID,
chatter_company_id: DEFAULT_COMPANY_ID,
chatterid: String(response.location.id)
});
logger.log("chatter-create-location-success", "info", null, null, {
bodyshopID,
chatter_company_id: DEFAULT_COMPANY_ID,
chatterid: response.location.id,
locationIdentifier
});
return res.json({ success: true, data: response });
} catch (error) {
logger.log("chatter-create-location-error", "error", null, null, {
bodyshopID,
error: error.message,
status: error.status,
data: error.data
});
return res.json({
success: false,
message: error.message || "Failed to create location",
error: error.data
});
}
};
module.exports = createLocation;

347
server/data/chatter-api.js Normal file
View File

@@ -0,0 +1,347 @@
const queries = require("../graphql-client/queries");
const moment = require("moment-timezone");
const logger = require("../utils/logger");
const { ChatterApiClient, getChatterApiToken, CHATTER_BASE_URL } = require("../chatter/chatter-client");
const client = require("../graphql-client/graphql-client").client;
const { sendServerEmail } = require("../email/sendemail");
const CHATTER_EVENT = process.env.CHATTER_SOLICITATION_EVENT || "delivery";
const MAX_CONCURRENCY = Number(process.env.CHATTER_API_CONCURRENCY || 5);
// Client caching (in-memory) - tokens are now cached in Redis
const clientCache = new Map(); // companyId -> ChatterApiClient
const tokenInFlight = new Map(); // companyId -> Promise<string> (for in-flight deduplication)
exports.default = async (req, res) => {
if (process.env.NODE_ENV !== "production") return res.sendStatus(403);
if (req.headers["x-imex-auth"] !== process.env.AUTOHOUSE_AUTH_TOKEN) return res.sendStatus(401);
res.status(202).json({
success: true,
message: "Processing request ...",
timestamp: new Date().toISOString()
});
try {
logger.log("chatter-api-start", "DEBUG", "api", null, null);
const allErrors = [];
const allShopSummaries = [];
// Shops that DO have chatter_company_id
const { bodyshops } = await client.request(queries.GET_CHATTER_SHOPS_WITH_COMPANY);
const specificShopIds = req.body.bodyshopIds;
const { start, end, skipUpload } = req.body; // keep same flag; now acts like "dry run"
const shopsToProcess =
specificShopIds?.length > 0 ? bodyshops.filter((shop) => specificShopIds.includes(shop.id)) : bodyshops;
logger.log("chatter-api-shopsToProcess-generated", "DEBUG", "api", null, { count: shopsToProcess.length });
if (shopsToProcess.length === 0) {
logger.log("chatter-api-shopsToProcess-empty", "DEBUG", "api", null, null);
return;
}
await processBatchApi({
shopsToProcess,
start,
end,
skipUpload,
allShopSummaries,
allErrors,
sessionUtils: req.sessionUtils
});
const totals = allShopSummaries.reduce(
(acc, s) => {
acc.shops += 1;
acc.jobs += s.jobs || 0;
acc.sent += s.sent || 0;
acc.duplicates += s.duplicates || 0;
acc.failed += s.failed || 0;
return acc;
},
{ shops: 0, jobs: 0, sent: 0, duplicates: 0, failed: 0 }
);
await sendServerEmail({
subject: `Chatter API Report ${moment().format("MM-DD-YY")}`,
text:
`Totals:\n${JSON.stringify(totals, null, 2)}\n\n` +
`Shop summaries:\n${JSON.stringify(allShopSummaries, null, 2)}\n\n` +
`Errors:\n${JSON.stringify(allErrors, null, 2)}\n`
});
logger.log("chatter-api-end", "DEBUG", "api", null, totals);
} catch (error) {
logger.log("chatter-api-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
}
};
async function processBatchApi({ shopsToProcess, start, end, skipUpload, allShopSummaries, allErrors, sessionUtils }) {
for (const bodyshop of shopsToProcess) {
const summary = {
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
chatter_company_id: bodyshop.chatter_company_id,
chatterid: bodyshop.chatterid,
jobs: 0,
sent: 0,
duplicates: 0,
failed: 0,
ok: true
};
try {
logger.log("chatter-api-start-shop", "DEBUG", "api", bodyshop.id, { shopname: bodyshop.shopname });
const companyId = parseCompanyId(bodyshop.chatter_company_id);
if (!companyId) {
summary.ok = false;
summary.failed = 0;
allErrors.push({
...pickShop(bodyshop),
fatal: true,
errors: [`Invalid chatter_company_id: "${bodyshop.chatter_company_id}"`]
});
allShopSummaries.push(summary);
continue;
}
const chatterApi = await getChatterApiClient(companyId, sessionUtils);
const { jobs } = await client.request(queries.CHATTER_QUERY, {
bodyshopid: bodyshop.id,
start: start ? moment(start).startOf("day") : moment().subtract(1, "days").startOf("day"),
...(end && { end: moment(end).endOf("day") })
});
summary.jobs = jobs.length;
// concurrency-limited posting
const limit = createConcurrencyLimit(MAX_CONCURRENCY);
const results = await Promise.all(
jobs.map((j) =>
limit(async () => {
const payload = buildInteractionPayload(bodyshop, j);
// keep legacy flag name: skipUpload == dry-run
if (skipUpload) return { ok: true, dryRun: true };
const r = await postInteractionWithPolicy(chatterApi, companyId, payload);
return r;
})
)
);
for (const r of results) {
if (r?.dryRun) continue;
if (r?.ok && r?.duplicate) summary.duplicates += 1;
else if (r?.ok) summary.sent += 1;
else summary.failed += 1;
}
// record failures with some detail (cap to avoid huge emails)
const failures = results
.filter((r) => r && r.ok === false)
.slice(0, 25)
.map((r) => ({ status: r.status, error: r.error }));
if (failures.length) {
summary.ok = false;
allErrors.push({
...pickShop(bodyshop),
fatal: false,
errors: failures
});
}
logger.log("chatter-api-end-shop", "DEBUG", "api", bodyshop.id, summary);
} catch (error) {
summary.ok = false;
logger.log("chatter-api-error-shop", "ERROR", "api", bodyshop.id, {
error: error.message,
stack: error.stack
});
allErrors.push({
...pickShop(bodyshop),
fatal: true,
errors: [error.toString()]
});
} finally {
allShopSummaries.push(summary);
}
}
}
function buildInteractionPayload(bodyshop, j) {
const isCompany = Boolean(j.ownr_co_nm);
const locationIdentifier = `${bodyshop.chatter_company_id}-${bodyshop.id}`;
return {
locationIdentifier: locationIdentifier,
event: CHATTER_EVENT,
transactionId: j.ro_number != null ? String(j.ro_number) : undefined,
timestamp: j.actual_delivery ? moment(j.actual_delivery).tz(bodyshop.timezone).toISOString() : undefined,
firstName: isCompany ? null : j.ownr_fn || null,
lastName: isCompany ? j.ownr_co_nm : j.ownr_ln || null,
emailAddress: j.ownr_ea || undefined,
phoneNumber: j.ownr_ph1 || undefined,
metadata: {
imexShopId: bodyshop.imexshopid,
bodyshopId: bodyshop.id,
jobId: j.id
}
};
}
async function postInteractionWithPolicy(chatterApi, companyId, payload) {
for (let attempt = 0; attempt < 6; attempt++) {
try {
await chatterApi.postInteraction(companyId, payload);
return { ok: true };
} catch (e) {
// duplicate -> treat as successful idempotency outcome
if (e.status === 409) return { ok: true, duplicate: true, error: e.data };
// rate limited -> backoff + retry
if (e.status === 429) {
await sleep(backoffMs(attempt));
continue;
}
return { ok: false, status: e.status, error: e.data ?? e.message };
}
}
return { ok: false, status: 429, error: "rate limit retry exhausted" };
}
function parseCompanyId(val) {
const s = String(val ?? "").trim();
if (!s) return null;
const n = Number(s);
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0) return null;
return n;
}
function pickShop(bodyshop) {
return {
bodyshopid: bodyshop.id,
imexshopid: bodyshop.imexshopid,
shopname: bodyshop.shopname,
chatter_company_id: bodyshop.chatter_company_id,
chatterid: bodyshop.chatterid
};
}
function sleep(ms) {
return new Promise((r) => setTimeout(r, ms));
}
function backoffMs(attempt) {
const base = Math.min(30_000, 500 * 2 ** attempt);
const jitter = Math.floor(Math.random() * 250);
return base + jitter;
}
function createConcurrencyLimit(max) {
let active = 0;
const queue = [];
const next = () => {
if (active >= max) return;
const fn = queue.shift();
if (!fn) return;
active++;
fn()
.catch(() => {})
.finally(() => {
active--;
next();
});
};
return (fn) =>
new Promise((resolve, reject) => {
queue.push(async () => {
try {
resolve(await fn());
} catch (e) {
reject(e);
}
});
next();
});
}
/**
* Returns a per-company Chatter API client, caching both the token and the client.
*/
async function getChatterApiClient(companyId, sessionUtils) {
const key = String(companyId);
const existing = clientCache.get(key);
if (existing) return existing;
const apiToken = await getChatterApiTokenCached(companyId, sessionUtils);
const chatterApi = new ChatterApiClient({ baseUrl: CHATTER_BASE_URL, apiToken });
clientCache.set(key, chatterApi);
return chatterApi;
}
/**
* Fetches the per-company token from AWS Secrets Manager with Redis caching
* SecretId: CHATTER_COMPANY_KEY_<companyId>
*
* Uses Redis caching + in-flight dedupe to avoid hammering Secrets Manager.
*/
async function getChatterApiTokenCached(companyId, sessionUtils) {
const key = String(companyId ?? "").trim();
if (!key) throw new Error("getChatterApiToken: companyId is required");
// Optional override for emergency/dev
if (process.env.CHATTER_API_TOKEN) return process.env.CHATTER_API_TOKEN;
// Check Redis cache if sessionUtils is available
if (sessionUtils?.getChatterToken) {
const cachedToken = await sessionUtils.getChatterToken(key);
if (cachedToken) {
logger.log("chatter-api-get-token-cache-hit", "DEBUG", "api", null, { companyId: key });
return cachedToken;
}
}
// Check for in-flight requests
const inflight = tokenInFlight.get(key);
if (inflight) return inflight;
const p = (async () => {
logger.log("chatter-api-get-token-cache-miss", "DEBUG", "api", null, { companyId: key });
// Fetch token from Secrets Manager using shared function
const token = await getChatterApiToken(companyId);
// Store in Redis cache if sessionUtils is available
if (sessionUtils?.setChatterToken) {
await sessionUtils.setChatterToken(key, token);
}
return token;
})();
tokenInFlight.set(key, p);
try {
return await p;
} finally {
tokenInFlight.delete(key);
}
}

View File

@@ -4,6 +4,8 @@ const converter = require("json-2-csv");
const logger = require("../utils/logger");
const fs = require("fs");
const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
const { defaultProvider } = require("@aws-sdk/credential-provider-node");
const { isString, isEmpty } = require("lodash");
let Client = require("ssh2-sftp-client");
const client = require("../graphql-client/graphql-client").client;
@@ -144,7 +146,18 @@ async function processBatch(shopsToProcess, start, end, allChatterObjects, allEr
async function getPrivateKey() {
// Connect to AWS Secrets Manager
const client = new SecretsManagerClient({ region: "ca-central-1" });
const secretsClientOptions = {
region: "ca-central-1",
credentials: defaultProvider()
};
const isLocal = isString(process.env?.LOCALSTACK_HOSTNAME) && !isEmpty(process.env?.LOCALSTACK_HOSTNAME);
if (isLocal) {
secretsClientOptions.endpoint = `http://${process.env.LOCALSTACK_HOSTNAME}:4566`;
}
const client = new SecretsManagerClient(secretsClientOptions);
const command = new GetSecretValueCommand({ SecretId: "CHATTER_PRIVATE_KEY" });
logger.log("chatter-get-private-key", "DEBUG", "api", null, null);

View File

@@ -9,4 +9,5 @@ exports.emsUpload = require("./emsUpload").default;
exports.carfax = require("./carfax").default;
exports.carfaxRps = require("./carfax-rps").default;
exports.vehicletype = require("./vehicletype/vehicletype").default;
exports.documentAnalytics = require("./analytics/documents").default;
exports.documentAnalytics = require("./analytics/documents").default;
exports.chatterApi = require("chatter-api").default();

View File

@@ -1919,10 +1919,35 @@ exports.GET_AUTOHOUSE_SHOPS = `query GET_AUTOHOUSE_SHOPS {
}`;
exports.GET_CHATTER_SHOPS = `query GET_CHATTER_SHOPS {
bodyshops(where: {chatterid: {_is_null: false}, _or: {chatterid: {_neq: ""}}}){
bodyshops(
where: {
chatterid: { _is_null: false, _neq: "" }
_or: [
{ chatter_company_id: { _is_null: true } }
{ chatter_company_id: { _eq: "" } }
]
}
) {
id
shopname
chatterid
chatter_company_id
imexshopid
timezone
}
}`;
exports.GET_CHATTER_SHOPS_WITH_COMPANY = `query GET_CHATTER_SHOPS_WITH_COMPANY {
bodyshops(
where: {
chatterid: { _is_null: false, _neq: "" }
chatter_company_id: { _is_null: false, _neq: "" }
}
) {
id
shopname
chatterid
chatter_company_id
imexshopid
timezone
}

View File

@@ -0,0 +1,12 @@
const express = require("express");
const createLocation = require("../chatter/createLocation");
const router = express.Router();
const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebaseIdTokenMiddleware");
const validateAdminMiddleware = require("../middleware/validateAdminMiddleware");
router.use(validateFirebaseIdTokenMiddleware);
router.use(validateAdminMiddleware);
router.post("/create-location", createLocation);
module.exports = router;

View File

@@ -1,10 +1,21 @@
const express = require("express");
const router = express.Router();
const { autohouse, claimscorp, chatter, kaizen, usageReport, podium, carfax, carfaxRps } = require("../data/data");
const {
autohouse,
claimscorp,
chatter,
kaizen,
usageReport,
podium,
carfax,
carfaxRps,
chatterApi
} = require("../data/data");
router.post("/ah", autohouse);
router.post("/cc", claimscorp);
router.post("/chatter", chatter);
router.post("/chatter-api", chatterApi);
router.post("/kaizen", kaizen);
router.post("/usagereport", usageReport);
router.post("/podium", podium);

View File

@@ -8,6 +8,12 @@ const client = require("../graphql-client/graphql-client").client;
*/
const BODYSHOP_CACHE_TTL = 3600; // 1 hour
/**
* Chatter API token cache TTL in seconds
* @type {number}
*/
const CHATTER_TOKEN_CACHE_TTL = 3600; // 1 hour
/**
* Generate a cache key for a bodyshop
* @param bodyshopId
@@ -15,6 +21,13 @@ const BODYSHOP_CACHE_TTL = 3600; // 1 hour
*/
const getBodyshopCacheKey = (bodyshopId) => `bodyshop-cache:${bodyshopId}`;
/**
* Generate a cache key for a Chatter API token
* @param companyId
* @returns {`chatter-token:${string}`}
*/
const getChatterTokenCacheKey = (companyId) => `chatter-token:${companyId}`;
/**
* Generate a cache key for a user socket mapping
* @param email
@@ -373,9 +386,53 @@ const applyRedisHelpers = ({ pubClient, app, logger }) => {
*/
const getProviderCache = (ns, field) => getSessionData(`${ns}:provider`, field);
/**
* Get Chatter API token from Redis cache
* @param companyId
* @returns {Promise<string|null>}
*/
const getChatterToken = async (companyId) => {
const key = getChatterTokenCacheKey(companyId);
try {
const token = await pubClient.get(key);
return token;
} catch (error) {
logger.log("get-chatter-token-from-redis", "ERROR", "redis", null, {
companyId,
error: error.message
});
return null;
}
};
/**
* Set Chatter API token in Redis cache
* @param companyId
* @param token
* @returns {Promise<void>}
*/
const setChatterToken = async (companyId, token) => {
const key = getChatterTokenCacheKey(companyId);
try {
await pubClient.set(key, token);
await pubClient.expire(key, CHATTER_TOKEN_CACHE_TTL);
devDebugLogger("chatter-token-cache-set", {
companyId,
action: "Token cached"
});
} catch (error) {
logger.log("set-chatter-token-in-redis", "ERROR", "redis", null, {
companyId,
error: error.message
});
throw error;
}
};
const api = {
getUserSocketMappingKey,
getBodyshopCacheKey,
getChatterTokenCacheKey,
setSessionData,
getSessionData,
clearSessionData,
@@ -390,7 +447,9 @@ const applyRedisHelpers = ({ pubClient, app, logger }) => {
getSessionTransactionData,
clearSessionTransactionData,
setProviderCache,
getProviderCache
getProviderCache,
getChatterToken,
setChatterToken
};
Object.assign(module.exports, api);