From 21f43285bc454556f61019991febe31b19759567 Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Thu, 19 Feb 2026 14:15:57 -0800 Subject: [PATCH] IO-3515 additional cleanup, translations --- bodyshop_translations.babel | 63 +++++++ .../bill-enter-ai-scan.component.jsx | 3 +- .../bill-enter-modal.container.jsx | 2 + .../bill-form/bill-form.component.jsx | 6 +- client/src/translations/en_us/common.json | 3 + client/src/translations/es/common.json | 3 + client/src/translations/fr/common.json | 3 + server/ai/bill-ocr/bill-ocr-generator.js | 158 +++++++++--------- server/ai/bill-ocr/bill-ocr-helpers.js | 2 +- server/ai/bill-ocr/bill-ocr-readme.md | 4 +- 10 files changed, 159 insertions(+), 88 deletions(-) diff --git a/bodyshop_translations.babel b/bodyshop_translations.babel index 4470ae100..cfb312ab1 100644 --- a/bodyshop_translations.babel +++ b/bodyshop_translations.babel @@ -3015,6 +3015,48 @@ errors + + calculating_totals + false + + + + + + en-US + false + + + es-MX + false + + + fr-CA + false + + + + + calculating_totals_generic + false + + + + + + en-US + false + + + es-MX + false + + + fr-CA + false + + + creating false @@ -3642,6 +3684,27 @@ + + generic_failure + false + + + + + + en-US + false + + + es-MX + false + + + fr-CA + false + + + multipage false diff --git a/client/src/components/bill-enter-ai-scan/bill-enter-ai-scan.component.jsx b/client/src/components/bill-enter-ai-scan/bill-enter-ai-scan.component.jsx index 02745ecdc..3d2735333 100644 --- a/client/src/components/bill-enter-ai-scan/bill-enter-ai-scan.component.jsx +++ b/client/src/components/bill-enter-ai-scan/bill-enter-ai-scan.component.jsx @@ -111,7 +111,6 @@ function BillEnterAiScan({ formdata.append("jobid", billEnterModal.context.job?.id); formdata.append("bodyshopid", bodyshop.id); formdata.append("partsorderid", billEnterModal.context.parts_order?.id); - //formdata.append("skipTextract", "true"); // For testing purposes try { const { data, status } = await axios.post("/ai/bill-ocr", formdata); @@ -147,7 +146,7 @@ function BillEnterAiScan({ setScanLoading(false); notification.error({ title: t("bills.labels.ai.scanfailed"), - message: error.response?.data?.message || error.message || "Failed to process invoice" + message: error.response?.data?.message || error.message || t("bills.labels.ai.generic_failure") }); } } diff --git a/client/src/components/bill-enter-modal/bill-enter-modal.container.jsx b/client/src/components/bill-enter-modal/bill-enter-modal.container.jsx index 08ad735b5..5985aed22 100644 --- a/client/src/components/bill-enter-modal/bill-enter-modal.container.jsx +++ b/client/src/components/bill-enter-modal/bill-enter-modal.container.jsx @@ -119,6 +119,8 @@ function BillEnterModalContainer({ billEnterModal, toggleModalVisible, bodyshop, create_ppc, // eslint-disable-next-line no-unused-vars original_actual_price, + // eslint-disable-next-line no-unused-vars + confidence, ...restI } = i; diff --git a/client/src/components/bill-form/bill-form.component.jsx b/client/src/components/bill-form/bill-form.component.jsx index 6a1146073..25fac42d3 100644 --- a/client/src/components/bill-form/bill-form.component.jsx +++ b/client/src/components/bill-form/bill-form.component.jsx @@ -128,7 +128,7 @@ export function BillFormComponent({ ]); useEffect(() => { - console.log("*** Form Watch - jobid changed:", jobIdFormWatch); + // When the jobid is set by AI scan, we need to reload the lines. This prevents having to hoist the apollo query. if (jobIdFormWatch !== null) { if (form.getFieldValue("jobid") !== null && form.getFieldValue("jobid") !== undefined) { loadLines({ variables: { id: form.getFieldValue("jobid") } }); @@ -399,8 +399,8 @@ export function BillFormComponent({ totals = CalculateBillTotal(values); } catch (error) { notification.error({ - title: "Error calculating totals", - message: error.message || "An error occurred while calculating bill totals.", + title: t("bills.errors.calculating_totals"), + message: error.message || t("bills.errors.calculating_totals_generic"), key: "bill_totals_calculation_error" }); } diff --git a/client/src/translations/en_us/common.json b/client/src/translations/en_us/common.json index 89263a568..b70334d93 100644 --- a/client/src/translations/en_us/common.json +++ b/client/src/translations/en_us/common.json @@ -192,6 +192,8 @@ "return": "Return Items" }, "errors": { + "calculating_totals": "Error Calculating Totals", + "calculating_totals_generic": "Please ensure all fields are properly completed. ", "creating": "Error adding bill. {{error}}", "deleting": "Error deleting bill. {{error}}", "existinginventoryline": "This bill cannot be deleted as it is tied to items in inventory.", @@ -228,6 +230,7 @@ "overall": "Overall" }, "disclaimer_title": "AI Scan Beta Disclaimer", + "generic_failure": "Failed to process invoice.", "multipage": "The is a multi-page document. Processing will take a few moments.", "processing": "Analyzing Bill", "scan": "AI Bill Scanner", diff --git a/client/src/translations/es/common.json b/client/src/translations/es/common.json index 27f99dd56..adbb4f5af 100644 --- a/client/src/translations/es/common.json +++ b/client/src/translations/es/common.json @@ -192,6 +192,8 @@ "return": "" }, "errors": { + "calculating_totals": "", + "calculating_totals_generic": "", "creating": "", "deleting": "", "existinginventoryline": "", @@ -228,6 +230,7 @@ "overall": "" }, "disclaimer_title": "", + "generic_failure": "", "multipage": "", "processing": "", "scan": "", diff --git a/client/src/translations/fr/common.json b/client/src/translations/fr/common.json index f781667be..3cee05ac6 100644 --- a/client/src/translations/fr/common.json +++ b/client/src/translations/fr/common.json @@ -192,6 +192,8 @@ "return": "" }, "errors": { + "calculating_totals": "", + "calculating_totals_generic": "", "creating": "", "deleting": "", "existinginventoryline": "", @@ -228,6 +230,7 @@ "overall": "" }, "disclaimer_title": "", + "generic_failure": "", "multipage": "", "processing": "", "scan": "", diff --git a/server/ai/bill-ocr/bill-ocr-generator.js b/server/ai/bill-ocr/bill-ocr-generator.js index 870285c01..31d265111 100644 --- a/server/ai/bill-ocr/bill-ocr-generator.js +++ b/server/ai/bill-ocr/bill-ocr-generator.js @@ -19,93 +19,97 @@ const normalizePrice = (str) => { if (typeof str !== 'string') return str; return str.replace(/[^0-9.-]+/g, ""); }; -const normalizePriceFinal = (str) => { - if (typeof str !== 'string') { - // If it's already a number, format to 2 decimals - const num = parseFloat(str); - return isNaN(num) ? 0 : num; - } - // First, try to extract valid decimal number patterns (e.g., "123.45") - const decimalPattern = /\d+\.\d{1,2}/g; - const decimalMatches = str.match(decimalPattern); +//More complex function. Not necessary at the moment, keeping for reference. +// const normalizePriceFinal = (str) => { +// if (typeof str !== 'string') { +// // If it's already a number, format to 2 decimals +// const num = parseFloat(str); +// return isNaN(num) ? 0 : num; +// } - if (decimalMatches && decimalMatches.length > 0) { - // Found valid decimal number(s) - const numbers = decimalMatches.map(m => parseFloat(m)).filter(n => !isNaN(n) && n > 0); +// // First, try to extract valid decimal number patterns (e.g., "123.45") +// const decimalPattern = /\d+\.\d{1,2}/g; +// const decimalMatches = str.match(decimalPattern); - if (numbers.length === 1) { - return numbers[0]; - } +// if (decimalMatches && decimalMatches.length > 0) { +// // Found valid decimal number(s) +// const numbers = decimalMatches.map(m => parseFloat(m)).filter(n => !isNaN(n) && n > 0); - if (numbers.length > 1) { - // Check if all numbers are the same (e.g., "47.57.47.57" -> [47.57, 47.57]) - const uniqueNumbers = [...new Set(numbers)]; - if (uniqueNumbers.length === 1) { - return uniqueNumbers[0]; - } +// if (numbers.length === 1) { +// return numbers[0]; +// } - // Check if numbers are very close (within 1% tolerance) - const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length; - const allClose = numbers.every(num => Math.abs(num - avg) / avg < 0.01); +// if (numbers.length > 1) { +// // Check if all numbers are the same (e.g., "47.57.47.57" -> [47.57, 47.57]) +// const uniqueNumbers = [...new Set(numbers)]; +// if (uniqueNumbers.length === 1) { +// return uniqueNumbers[0]; +// } - if (allClose) { - return avg; - } +// // Check if numbers are very close (within 1% tolerance) +// const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length; +// const allClose = numbers.every(num => Math.abs(num - avg) / avg < 0.01); - // Return the first number (most likely correct) - return numbers[0]; - } - } +// if (allClose) { +// return avg; +// } - // Fallback: Split on common delimiters and extract all potential numbers - const parts = str.split(/[\/|\\,;]/).map(part => part.trim()).filter(part => part.length > 0); +// // Return the first number (most likely correct) +// return numbers[0]; +// } +// } - if (parts.length > 1) { - // Multiple values detected - extract and parse all valid numbers - const numbers = parts - .map(part => { - const cleaned = part.replace(/[^0-9.-]+/g, ""); - const parsed = parseFloat(cleaned); - return isNaN(parsed) ? null : parsed; - }) - .filter(num => num !== null && num > 0); +// // Fallback: Split on common delimiters and extract all potential numbers +// const parts = str.split(/[\/|\\,;]/).map(part => part.trim()).filter(part => part.length > 0); - if (numbers.length === 0) { - // No valid numbers found, try fallback to basic cleaning - const cleaned = str.replace(/[^0-9.-]+/g, ""); - const parsed = parseFloat(cleaned); - return isNaN(parsed) ? 0 : parsed; - } +// if (parts.length > 1) { +// // Multiple values detected - extract and parse all valid numbers +// const numbers = parts +// .map(part => { +// const cleaned = part.replace(/[^0-9.-]+/g, ""); +// const parsed = parseFloat(cleaned); +// return isNaN(parsed) ? null : parsed; +// }) +// .filter(num => num !== null && num > 0); - if (numbers.length === 1) { - return numbers[0]; - } +// if (numbers.length === 0) { +// // No valid numbers found, try fallback to basic cleaning +// const cleaned = str.replace(/[^0-9.-]+/g, ""); +// const parsed = parseFloat(cleaned); +// return isNaN(parsed) ? 0 : parsed; +// } - // Multiple valid numbers - const uniqueNumbers = [...new Set(numbers)]; +// if (numbers.length === 1) { +// return numbers[0]; +// } - if (uniqueNumbers.length === 1) { - return uniqueNumbers[0]; - } +// // Multiple valid numbers +// const uniqueNumbers = [...new Set(numbers)]; - // Check if numbers are very close (within 1% tolerance) - const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length; - const allClose = numbers.every(num => Math.abs(num - avg) / avg < 0.01); +// if (uniqueNumbers.length === 1) { +// return uniqueNumbers[0]; +// } - if (allClose) { - return avg; - } +// // Check if numbers are very close (within 1% tolerance) +// const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length; +// const allClose = numbers.every(num => Math.abs(num - avg) / avg < 0.01); + +// if (allClose) { +// return avg; +// } + +// // Return the first valid number +// return numbers[0]; +// } + +// // Single value or no delimiters, clean normally +// const cleaned = str.replace(/[^0-9.-]+/g, ""); +// const parsed = parseFloat(cleaned); +// return isNaN(parsed) ? 0 : parsed; +// }; - // Return the first valid number - return numbers[0]; - } - // Single value or no delimiters, clean normally - const cleaned = str.replace(/[^0-9.-]+/g, ""); - const parsed = parseFloat(cleaned); - return isNaN(parsed) ? 0 : parsed; -}; // Helper function to calculate Textract OCR confidence (0-100%) const calculateTextractConfidence = (textractLineItem) => { @@ -149,6 +153,7 @@ const calculateTextractConfidence = (textractLineItem) => { else if (field.normalizedLabel === standardizedFieldsnames.quantity) { weight = 3.5; } + // We generally ignore the key from textract. Keeping for future reference. // else if (key === 'ITEM' || key === 'PRODUCT_CODE') { // weight = 3; // Description and part number are most important // } else if (key === 'PRICE' || key === 'UNIT_PRICE' || key === 'QUANTITY') { @@ -179,7 +184,6 @@ const calculateTextractConfidence = (textractLineItem) => { return Math.round(avgConfidence * 100) / 100; // Round to 2 decimal places }; -// Helper function to calculate match confidence score (0-100%) const calculateMatchConfidence = (matches, bestMatch) => { if (!matches || matches.length === 0 || !bestMatch) { return 0; // No match = 0% confidence @@ -217,7 +221,6 @@ const calculateMatchConfidence = (matches, bestMatch) => { return Math.max(matchConfidence, 1); }; -// Helper function to calculate overall confidence combining OCR and match confidence const calculateOverallConfidence = (ocrConfidence, matchConfidence) => { // If there's no match, OCR confidence doesn't matter much if (matchConfidence === 0) { @@ -318,7 +321,7 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body } `, { - jobid, // TODO: Refactor back in parts orders + jobid, // TODO: Parts order IDs are currently ignore. If receving a parts order, it could be used to more precisely match to joblines. }); //Create fuses of line descriptions for matching. @@ -378,10 +381,8 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body if (!job) { throw new Error('Job not found for bill form data generation.'); } - //Figure out which lines have a match and which don't. //TODO: How do we handle freight lines and core charges? - //Create the form data structure for the bill posting screen. const billFormData = { "jobid": jobid, @@ -392,10 +393,10 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body "total": normalizePrice(processedData.summary?.INVOICE_TOTAL?.value || processedData.summary?.TOTAL?.value), "billlines": joblineMatches.map(jlMatchLine => { const { matches, textractLineItem, } = jlMatchLine - //Matches should be prioritized, take the first one. + //Matches should be pre-sorted, take the first one. const matchToUse = matches.length > 0 ? matches[0] : null; - // Calculate confidence scores (0-100%) + // Calculate confidence scores const ocrConfidence = calculateTextractConfidence(textractLineItem); const matchConfidence = calculateMatchConfidence(matches, matchToUse); const overallConfidence = calculateOverallConfidence(ocrConfidence, matchConfidence); @@ -452,7 +453,7 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body //TODO: Do we need to verify the lines to see if it is a unit price or total price (i.e. quantity * price) const lineObject = { "line_desc": matchToUse?.item?.line_desc || textractLineItem.ITEM?.value || "NO DESCRIPTION", - "quantity": textractLineItem.QUANTITY?.value, // convert to integer? + "quantity": textractLineItem.QUANTITY?.value, "actual_price": normalizePrice(actualPrice), "actual_cost": normalizePrice(actualCost), "cost_center": matchToUse?.item?.part_type @@ -470,7 +471,6 @@ async function generateBillFormData({ processedData, jobid: jobidFromProps, body }, "joblineid": matchToUse?.item?.id || "noline", "confidence": `T${overallConfidence} - O${ocrConfidence} - J${matchConfidence}` - } return lineObject }) diff --git a/server/ai/bill-ocr/bill-ocr-helpers.js b/server/ai/bill-ocr/bill-ocr-helpers.js index fe3e58e2d..6a4ef9086 100644 --- a/server/ai/bill-ocr/bill-ocr-helpers.js +++ b/server/ai/bill-ocr/bill-ocr-helpers.js @@ -1,5 +1,5 @@ const PDFDocument = require('pdf-lib').PDFDocument; -const TEXTRACT_REDIS_PREFIX = `textract:${process.env?.NODE_ENV === "production" ? "PROD" : "TEST"}` +const TEXTRACT_REDIS_PREFIX = `textract:${process.env?.NODE_ENV}` const TEXTRACT_JOB_TTL = 10 * 60; diff --git a/server/ai/bill-ocr/bill-ocr-readme.md b/server/ai/bill-ocr/bill-ocr-readme.md index 1037d9f11..c9709483a 100644 --- a/server/ai/bill-ocr/bill-ocr-readme.md +++ b/server/ai/bill-ocr/bill-ocr-readme.md @@ -5,6 +5,4 @@ Required Infrastructure setup TODO: * Create a rome bucket for uploads, or move to the regular spot. -* How to implement this across environments. -* How to prevent polling for a job that may have errored. -* Handling of HEIC files on upload. \ No newline at end of file +* Add environment variables. \ No newline at end of file