IO-3515 PR Comments addressed.
This commit is contained in:
@@ -58,11 +58,11 @@ function BillEnterAiScan({
|
|||||||
setScanLoading(false);
|
setScanLoading(false);
|
||||||
|
|
||||||
// Update form with the extracted data
|
// Update form with the extracted data
|
||||||
if (data.data && data.data.billForm) {
|
if (data?.data?.billForm) {
|
||||||
form.setFieldsValue(data.data.billForm);
|
form.setFieldsValue(data.data.billForm);
|
||||||
await form.validateFields(["billlines"], { recursive: true });
|
await form.validateFields(["billlines"], { recursive: true });
|
||||||
notification.success({
|
notification.success({
|
||||||
title: t(".bills.labels.ai.scancomplete")
|
title: t("bills.labels.ai.scancomplete")
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if (data.status === "FAILED") {
|
} else if (data.status === "FAILED") {
|
||||||
@@ -155,8 +155,6 @@ function BillEnterAiScan({
|
|||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
setScanLoading(false);
|
setScanLoading(false);
|
||||||
console.log("*** ~ BillEnterAiScan ~ error:", error, error.response?.data?.message);
|
|
||||||
|
|
||||||
notification.error({
|
notification.error({
|
||||||
title: t("bills.labels.ai.scanfailed"),
|
title: t("bills.labels.ai.scanfailed"),
|
||||||
description: error.response?.data?.message || error.message || t("bills.labels.ai.generic_failure")
|
description: error.response?.data?.message || error.message || t("bills.labels.ai.generic_failure")
|
||||||
|
|||||||
@@ -280,11 +280,7 @@ export function BillEnterModalLinesComponent({
|
|||||||
{ required: true },
|
{ required: true },
|
||||||
{
|
{
|
||||||
validator: (_, value) => {
|
validator: (_, value) => {
|
||||||
if (Math.abs(parseFloat(value)) < 0.01) {
|
return Math.abs(parseFloat(value)) < 0.01 ? Promise.reject() : Promise.resolve();
|
||||||
return Promise.reject();
|
|
||||||
} else {
|
|
||||||
return Promise.resolve();
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
warningOnly: true
|
warningOnly: true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ const ConfidenceDisplay = ({ rowValue: { confidence, actual_price, actual_cost }
|
|||||||
const parsed_actual_price = parseFloat(actual_price);
|
const parsed_actual_price = parseFloat(actual_price);
|
||||||
const parsed_actual_cost = parseFloat(actual_cost);
|
const parsed_actual_cost = parseFloat(actual_cost);
|
||||||
if (!parsed) {
|
if (!parsed) {
|
||||||
return <span style={{ color: "#999", fontSize: "0.85em" }}>N/A</span>;
|
return <span style={{ color: "#959595", fontSize: "0.85em" }}>N/A</span>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { total, ocr, jobMatch } = parsed;
|
const { total, ocr, jobMatch } = parsed;
|
||||||
|
|||||||
13
server.js
13
server.js
@@ -427,6 +427,12 @@ const loadQueues = async ({ pubClient, logger, redisHelpers, ioRedis }) => {
|
|||||||
chatterApiQueue.on("error", (error) => {
|
chatterApiQueue.on("error", (error) => {
|
||||||
logger.log(`Error in chatterApiQueue: ${error}`, "ERROR", "queue", "api", null, { error: error?.message });
|
logger.log(`Error in chatterApiQueue: ${error}`, "ERROR", "queue", "api", null, { error: error?.message });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Initialize bill-ocr with Redis client
|
||||||
|
const { initializeBillOcr, startSQSPolling } = require("./server/ai/bill-ocr/bill-ocr");
|
||||||
|
initializeBillOcr(pubClient);
|
||||||
|
// Start SQS polling for Textract notifications
|
||||||
|
startSQSPolling();
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -457,13 +463,6 @@ const main = async () => {
|
|||||||
await server.listen(port);
|
await server.listen(port);
|
||||||
logger.log(`Server started on port ${port}`, "INFO", "api");
|
logger.log(`Server started on port ${port}`, "INFO", "api");
|
||||||
|
|
||||||
// Initialize bill-ocr with Redis client
|
|
||||||
const { initializeBillOcr, startSQSPolling } = require("./server/ai/bill-ocr/bill-ocr");
|
|
||||||
initializeBillOcr(pubClient);
|
|
||||||
|
|
||||||
// Start SQS polling for Textract notifications
|
|
||||||
startSQSPolling();
|
|
||||||
logger.log(`Started SQS polling for Textract notifications`, "INFO", "api");
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log(`Server failed to start on port ${port}`, "ERROR", "api", null, { error: error.message });
|
logger.log(`Server failed to start on port ${port}`, "ERROR", "api", null, { error: error.message });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -287,9 +287,8 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body
|
|||||||
|
|
||||||
if (jobs.length === 0) {
|
if (jobs.length === 0) {
|
||||||
throw new Error("No job found for the detected RO/PO number.");
|
throw new Error("No job found for the detected RO/PO number.");
|
||||||
} else {
|
|
||||||
jobid = jobs[0].id;
|
|
||||||
}
|
}
|
||||||
|
jobid = jobs[0].id;
|
||||||
}
|
}
|
||||||
|
|
||||||
const jobData = await client.request(`
|
const jobData = await client.request(`
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
const PDFDocument = require('pdf-lib').PDFDocument;
|
const PDFDocument = require('pdf-lib').PDFDocument;
|
||||||
|
const logger = require("../../utils/logger");
|
||||||
const TEXTRACT_REDIS_PREFIX = `textract:${process.env?.NODE_ENV}`
|
const TEXTRACT_REDIS_PREFIX = `textract:${process.env?.NODE_ENV}`
|
||||||
const TEXTRACT_JOB_TTL = 10 * 60;
|
const TEXTRACT_JOB_TTL = 10 * 60;
|
||||||
|
|
||||||
@@ -141,7 +142,7 @@ async function hasActiveJobs({ redisPubClient }) {
|
|||||||
|
|
||||||
return false;
|
return false;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error checking for active jobs:', error);
|
logger.log("bill-ocr-job-check-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ const { getTextractJobKey, setTextractJob, getTextractJob, getFileType, getPdfPa
|
|||||||
const { extractInvoiceData, processScanData } = require("./bill-ocr-normalize");
|
const { extractInvoiceData, processScanData } = require("./bill-ocr-normalize");
|
||||||
const { generateBillFormData } = require("./bill-ocr-generator");
|
const { generateBillFormData } = require("./bill-ocr-generator");
|
||||||
const logger = require("../../utils/logger");
|
const logger = require("../../utils/logger");
|
||||||
|
|
||||||
// Initialize AWS clients
|
// Initialize AWS clients
|
||||||
const awsConfig = {
|
const awsConfig = {
|
||||||
region: process.env.AWS_AI_REGION || "ca-central-1",
|
region: process.env.AWS_AI_REGION || "ca-central-1",
|
||||||
@@ -39,25 +40,19 @@ async function jobExists(textractJobId) {
|
|||||||
if (!redisPubClient) {
|
if (!redisPubClient) {
|
||||||
throw new Error('Redis client not initialized. Call initializeBillOcr first.');
|
throw new Error('Redis client not initialized. Call initializeBillOcr first.');
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Checking if job exists for Textract job ID:', textractJobId);
|
|
||||||
const key = getTextractJobKey(textractJobId);
|
const key = getTextractJobKey(textractJobId);
|
||||||
const exists = await redisPubClient.exists(key);
|
const exists = await redisPubClient.exists(key);
|
||||||
|
|
||||||
if (exists) {
|
if (exists) {
|
||||||
console.log(`Job found: ${textractJobId}`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('No matching job found in Redis');
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleBillOcr(req, res) {
|
async function handleBillOcr(req, res) {
|
||||||
// Check if file was uploaded
|
// Check if file was uploaded
|
||||||
if (!req.file) {
|
if (!req.file) {
|
||||||
res.status(400).send({ error: 'No file uploaded.' });
|
return res.status(400).send({ error: 'No file uploaded.' });
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The uploaded file is available in request file
|
// The uploaded file is available in request file
|
||||||
@@ -73,7 +68,7 @@ async function handleBillOcr(req, res) {
|
|||||||
const billForm = await generateBillFormData({ processedData: processedData, jobid, bodyshopid, partsorderid, req: req });
|
const billForm = await generateBillFormData({ processedData: processedData, jobid, bodyshopid, partsorderid, req: req });
|
||||||
logger.log("bill-ocr-single-complete", "DEBUG", req.user.email, jobid, { ...processedData, billForm });
|
logger.log("bill-ocr-single-complete", "DEBUG", req.user.email, jobid, { ...processedData, billForm });
|
||||||
|
|
||||||
res.status(200).send({
|
return res.status(200).json({
|
||||||
success: true,
|
success: true,
|
||||||
status: 'COMPLETED',
|
status: 'COMPLETED',
|
||||||
data: { ...processedData, billForm },
|
data: { ...processedData, billForm },
|
||||||
@@ -88,35 +83,35 @@ async function handleBillOcr(req, res) {
|
|||||||
const processedData = await processSinglePageDocument(uploadedFile.buffer);
|
const processedData = await processSinglePageDocument(uploadedFile.buffer);
|
||||||
const billForm = await generateBillFormData({ processedData: processedData, jobid, bodyshopid, partsorderid, req: req });
|
const billForm = await generateBillFormData({ processedData: processedData, jobid, bodyshopid, partsorderid, req: req });
|
||||||
logger.log("bill-ocr-single-complete", "DEBUG", req.user.email, jobid, { ...processedData, billForm });
|
logger.log("bill-ocr-single-complete", "DEBUG", req.user.email, jobid, { ...processedData, billForm });
|
||||||
res.status(200).send({
|
return res.status(200).json({
|
||||||
success: true,
|
success: true,
|
||||||
status: 'COMPLETED',
|
status: 'COMPLETED',
|
||||||
data: { ...processedData, billForm },
|
data: { ...processedData, billForm },
|
||||||
message: 'Invoice processing completed'
|
message: 'Invoice processing completed'
|
||||||
});
|
});
|
||||||
} else {
|
|
||||||
// Start the Textract job (non-blocking) for multi-page documents
|
|
||||||
const jobInfo = await startTextractJob(uploadedFile.buffer, { jobid, bodyshopid, partsorderid });
|
|
||||||
logger.log("bill-ocr-multipage-start", "DEBUG", req.user.email, jobid, jobInfo);
|
|
||||||
|
|
||||||
res.status(202).send({
|
|
||||||
success: true,
|
|
||||||
textractJobId: jobInfo.jobId,
|
|
||||||
message: 'Invoice processing started',
|
|
||||||
statusUrl: `/ai/bill-ocr/status/${jobInfo.jobId}`
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
// Start the Textract job (non-blocking) for multi-page documents
|
||||||
|
const jobInfo = await startTextractJob(uploadedFile.buffer, { jobid, bodyshopid, partsorderid });
|
||||||
|
logger.log("bill-ocr-multipage-start", "DEBUG", req.user.email, jobid, jobInfo);
|
||||||
|
|
||||||
|
return res.status(202).json({
|
||||||
|
success: true,
|
||||||
|
textractJobId: jobInfo.jobId,
|
||||||
|
message: 'Invoice processing started',
|
||||||
|
statusUrl: `/ai/bill-ocr/status/${jobInfo.jobId}`
|
||||||
|
});
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
logger.log("bill-ocr-unsupported-filetype", "WARN", req.user.email, jobid, { fileType });
|
logger.log("bill-ocr-unsupported-filetype", "WARN", req.user.email, jobid, { fileType });
|
||||||
|
|
||||||
res.status(400).send({
|
return res.status(400).json({
|
||||||
error: 'Unsupported file type',
|
error: 'Unsupported file type',
|
||||||
message: 'Please upload a PDF or supported image file (JPEG, PNG, TIFF)'
|
message: 'Please upload a PDF or supported image file (JPEG, PNG, TIFF)'
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("bill-ocr-error", "ERROR", req.user.email, jobid, { error: error.message, stack: error.stack });
|
logger.log("bill-ocr-error", "ERROR", req.user.email, jobid, { error: error.message, stack: error.stack });
|
||||||
res.status(500).send({
|
return res.status(500).json({
|
||||||
error: 'Failed to start invoice processing',
|
error: 'Failed to start invoice processing',
|
||||||
message: error.message
|
message: error.message
|
||||||
});
|
});
|
||||||
@@ -128,15 +123,13 @@ async function handleBillOcrStatus(req, res) {
|
|||||||
|
|
||||||
if (!textractJobId) {
|
if (!textractJobId) {
|
||||||
logger.log("bill-ocr-status-error", "WARN", req.user.email, null, { error: 'No textractJobId found in params' });
|
logger.log("bill-ocr-status-error", "WARN", req.user.email, null, { error: 'No textractJobId found in params' });
|
||||||
res.status(400).send({ error: 'Job ID is required' });
|
return res.status(400).json({ error: 'Job ID is required' });
|
||||||
res.status(400).send({ error: 'Job ID is required' });
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
const jobStatus = await getTextractJob({ redisPubClient, textractJobId });
|
const jobStatus = await getTextractJob({ redisPubClient, textractJobId });
|
||||||
|
|
||||||
if (!jobStatus) {
|
if (!jobStatus) {
|
||||||
res.status(404).send({ error: 'Job not found' });
|
return res.status(404).json({ error: 'Job not found' });
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jobStatus.status === 'COMPLETED') {
|
if (jobStatus.status === 'COMPLETED') {
|
||||||
@@ -169,17 +162,17 @@ async function handleBillOcrStatus(req, res) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("bill-ocr-multipage-error", "ERROR", req.user.email, jobStatus.context.jobid, { ...jobStatus.data, error: error.message, stack: error.stack });
|
logger.log("bill-ocr-multipage-error", "ERROR", req.user.email, jobStatus.context.jobid, { ...jobStatus.data, error: error.message, stack: error.stack });
|
||||||
|
|
||||||
res.status(500).send({
|
return res.status(500).send({
|
||||||
status: 'COMPLETED',
|
status: 'COMPLETED',
|
||||||
error: 'Data processed but failed to generate bill form',
|
error: 'Data processed but failed to generate bill form',
|
||||||
message: error.message,
|
message: error.message,
|
||||||
data: jobStatus.data // Still return the raw processed data
|
data: jobStatus.data // Still return the raw processed data
|
||||||
});
|
});
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res.status(200).send({
|
return res.status(200).send({
|
||||||
status: 'COMPLETED',
|
status: 'COMPLETED',
|
||||||
data: {
|
data: {
|
||||||
...jobStatus.data,
|
...jobStatus.data,
|
||||||
@@ -189,12 +182,12 @@ async function handleBillOcrStatus(req, res) {
|
|||||||
} else if (jobStatus.status === 'FAILED') {
|
} else if (jobStatus.status === 'FAILED') {
|
||||||
logger.log("bill-ocr-multipage-failed", "ERROR", req.user.email, jobStatus.context.jobid, { ...jobStatus.data, error: jobStatus.error, });
|
logger.log("bill-ocr-multipage-failed", "ERROR", req.user.email, jobStatus.context.jobid, { ...jobStatus.data, error: jobStatus.error, });
|
||||||
|
|
||||||
res.status(500).send({
|
return res.status(500).json({
|
||||||
status: 'FAILED',
|
status: 'FAILED',
|
||||||
error: jobStatus.error
|
error: jobStatus.error
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
res.status(200).send({
|
return res.status(200).json({
|
||||||
status: jobStatus.status
|
status: jobStatus.status
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -294,14 +287,13 @@ async function processSQSMessages() {
|
|||||||
const queueUrl = process.env.AWS_TEXTRACT_SQS_QUEUE_URL;
|
const queueUrl = process.env.AWS_TEXTRACT_SQS_QUEUE_URL;
|
||||||
|
|
||||||
if (!queueUrl) {
|
if (!queueUrl) {
|
||||||
console.error('AWS_TEXTRACT_SQS_QUEUE_URL not configured');
|
logger.log("bill-ocr-error", "ERROR", "api", null, { message: "AWS_TEXTRACT_SQS_QUEUE_URL not configured" });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only poll if there are active mutli page jobs in progress
|
// Only poll if there are active mutli page jobs in progress
|
||||||
const hasActive = await hasActiveJobs({ redisPubClient });
|
const hasActive = await hasActiveJobs({ redisPubClient });
|
||||||
if (!hasActive) {
|
if (!hasActive) {
|
||||||
console.log('No active jobs in progress, skipping SQS poll');
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,7 +308,7 @@ async function processSQSMessages() {
|
|||||||
const result = await sqsClient.send(receiveCommand);
|
const result = await sqsClient.send(receiveCommand);
|
||||||
|
|
||||||
if (result.Messages && result.Messages.length > 0) {
|
if (result.Messages && result.Messages.length > 0) {
|
||||||
console.log('Processing', result.Messages.length, 'messages from SQS');
|
logger.log("bill-ocr-sqs-processing", "DEBUG", "api", null, { message: `Processing ${result.Messages.length} messages from SQS` });
|
||||||
for (const message of result.Messages) {
|
for (const message of result.Messages) {
|
||||||
try {
|
try {
|
||||||
// Environment-level filtering: check if this message belongs to this environment
|
// Environment-level filtering: check if this message belongs to this environment
|
||||||
@@ -330,8 +322,6 @@ async function processSQSMessages() {
|
|||||||
ReceiptHandle: message.ReceiptHandle
|
ReceiptHandle: message.ReceiptHandle
|
||||||
});
|
});
|
||||||
await sqsClient.send(deleteCommand);
|
await sqsClient.send(deleteCommand);
|
||||||
} else {
|
|
||||||
console.log('Ignoring message - job not found in this environment');
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
||||||
@@ -342,7 +332,6 @@ async function processSQSMessages() {
|
|||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.log("bill-ocr-sqs-receiving-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
logger.log("bill-ocr-sqs-receiving-error", "ERROR", "api", null, { error: error.message, stack: error.stack });
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -361,7 +350,7 @@ async function shouldProcessMessage(message) {
|
|||||||
const exists = await jobExists(textractJobId);
|
const exists = await jobExists(textractJobId);
|
||||||
return exists;
|
return exists;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error checking if message should be processed:', error);
|
logger.log("bill-ocr-message-check-error", "DEBUG", "api", null, { message: "Error checking if message should be processed", error: error.message, stack: error.stack });
|
||||||
// If we can't parse the message, don't process it
|
// If we can't parse the message, don't process it
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -373,8 +362,7 @@ async function handleTextractNotification(message) {
|
|||||||
try {
|
try {
|
||||||
snsMessage = JSON.parse(body.Message);
|
snsMessage = JSON.parse(body.Message);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error parsing SNS message:', error);
|
logger.log("bill-ocr-handle-textract-error", "DEBUG", "api", null, { message: "Error parsing SNS message - invalid message format.", error: error.message, stack: error.stack, body });
|
||||||
console.log('Invalid message format:', body);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,7 +373,7 @@ async function handleTextractNotification(message) {
|
|||||||
const jobInfo = await getTextractJob({ redisPubClient, textractJobId });
|
const jobInfo = await getTextractJob({ redisPubClient, textractJobId });
|
||||||
|
|
||||||
if (!jobInfo) {
|
if (!jobInfo) {
|
||||||
console.warn(`Job info not found in Redis for Textract job ID: ${textractJobId}`);
|
logger.log("bill-ocr-job-not-found", "DEBUG", "api", null, { message: `Job info not found in Redis for Textract job ID: ${textractJobId}`, textractJobId, snsMessage });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -461,7 +449,7 @@ async function retrieveTextractResults(textractJobId) {
|
|||||||
function startSQSPolling() {
|
function startSQSPolling() {
|
||||||
const pollInterval = setInterval(() => {
|
const pollInterval = setInterval(() => {
|
||||||
processSQSMessages().catch(error => {
|
processSQSMessages().catch(error => {
|
||||||
console.error('SQS polling error:', error);
|
logger.log("bill-ocr-sqs-poll-error", "ERROR", "api", null, { message: error.message, stack: error.stack });
|
||||||
});
|
});
|
||||||
}, 10000); // Poll every 10 seconds
|
}, 10000); // Poll every 10 seconds
|
||||||
return pollInterval;
|
return pollInterval;
|
||||||
|
|||||||
Reference in New Issue
Block a user