Local history for scrubbing.

This commit is contained in:
Patrick Fic
2026-02-25 10:50:46 -08:00
parent e97f644373
commit 4915e05ac9
15 changed files with 1413 additions and 479 deletions

View File

@@ -0,0 +1,386 @@
import { app } from "electron";
import log from "electron-log/main";
import Database from "better-sqlite3";
import crypto from "crypto";
import path from "path";
import type { RawJobDataObject } from "../decoder/decoder";
export type ScrubHistoryJobRow = {
id: string;
created_at: number;
ownr_name: string;
vehicle: string;
claim_number: string;
pdf_url: string | null;
};
export type ScrubHistoryScrubResultRow = {
id: number;
created_at: number;
job_id: string;
anchor: string | null;
category: string | null;
subcategory: string | null;
left_text: string | null;
right_text: string | null;
linktext: string | null;
};
export type ScrubHistoryItem = {
id: string;
createdAt: number;
ownrName: string;
vehicle: string;
claimNumber: string;
pdfUrl: string | null;
results: Array<{
createdAt: number;
anchor: string | null;
category: string | null;
subcategory: string | null;
left: string | null;
right: string | null;
linktext: string | null;
}>;
};
export type ScrubHistoryPage = {
items: ScrubHistoryItem[];
totalJobs: number;
page: number;
pageSize: number;
totalResults: number;
lastProcessed: number | null;
};
type IdentifiedItem = {
Anchor?: unknown;
Category?: unknown;
SubCategory?: unknown;
L?: unknown;
R?: unknown;
LinkText?: unknown;
};
let db: Database.Database | undefined;
function toNullableString(value: unknown): string | null {
if (value === null || value === undefined) return null;
if (typeof value === "string") return value;
if (typeof value === "number" || typeof value === "boolean") {
return String(value);
}
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}
function getDbPath(): string {
const userDataDir = app.getPath("userData");
return path.join(userDataDir, "scrub-history.sqlite3");
}
function getDb(): Database.Database {
if (db) return db;
const dbPath = getDbPath();
log.info(`[scrub-history-db] opening sqlite db at ${dbPath}`);
db = new Database(dbPath);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
db.exec(`
CREATE TABLE IF NOT EXISTS jobs (
id TEXT PRIMARY KEY,
created_at INTEGER NOT NULL,
ownr_name TEXT NOT NULL,
vehicle TEXT NOT NULL,
claim_number TEXT NOT NULL,
pdf_url TEXT
);
CREATE TABLE IF NOT EXISTS scrub_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at INTEGER NOT NULL,
job_id TEXT NOT NULL,
anchor TEXT,
category TEXT,
subcategory TEXT,
left_text TEXT,
right_text TEXT,
linktext TEXT,
FOREIGN KEY(job_id) REFERENCES jobs(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_scrub_results_job_id ON scrub_results(job_id);
CREATE INDEX IF NOT EXISTS idx_scrub_results_created_at ON scrub_results(created_at DESC);
`);
// Lightweight migrations for existing installs
const ensureJobsColumn = (columnName: string, definition: string): void => {
const cols = db!.prepare("PRAGMA table_info(jobs)").all() as Array<{
name: string;
}>;
const hasCol = cols.some((c) => c.name === columnName);
if (!hasCol) {
log.info(`[scrub-history-db] migrating jobs: adding ${columnName}`);
db!.exec(`ALTER TABLE jobs ADD COLUMN ${definition}`);
}
};
ensureJobsColumn("pdf_url", "pdf_url TEXT");
return db;
}
function buildJobRow(
job: RawJobDataObject,
createdAt: number,
pdfUrl: string | null,
): ScrubHistoryJobRow {
const ownrName = `${job.ownr_fn ?? ""} ${job.ownr_ln ?? ""}`.trim();
const vehicle =
`${job.v_model_yr ?? ""} ${job.v_make_desc ?? ""} ${job.v_model_desc ?? ""}`.trim();
const claimNumber = `${job.clm_no ?? ""}`.trim();
return {
id: crypto.randomUUID(),
created_at: createdAt,
ownr_name: ownrName || "Unknown",
vehicle: vehicle || "Unknown",
claim_number: claimNumber || "Unknown",
pdf_url: pdfUrl,
};
}
export function insertScrubRun(params: {
job: RawJobDataObject;
identifiedItems: unknown;
pdfUrl: string | null;
}): { jobId: string; createdAt: number; insertedResultsCount: number } {
const database = getDb();
const createdAt = Date.now();
const jobRow = buildJobRow(params.job, createdAt, params.pdfUrl);
const items: IdentifiedItem[] = Array.isArray(params.identifiedItems)
? (params.identifiedItems as IdentifiedItem[])
: params.identifiedItems
? [params.identifiedItems as IdentifiedItem]
: [];
const insertTx = database.transaction(() => {
database
.prepare(
"INSERT INTO jobs (id, created_at, ownr_name, vehicle, claim_number, pdf_url) VALUES (?, ?, ?, ?, ?, ?)",
)
.run(
jobRow.id,
jobRow.created_at,
jobRow.ownr_name,
jobRow.vehicle,
jobRow.claim_number,
jobRow.pdf_url,
);
const stmt = database.prepare(
"INSERT INTO scrub_results (created_at, job_id, anchor, category, subcategory, left_text, right_text, linktext) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
);
for (const item of items) {
stmt.run(
createdAt,
jobRow.id,
toNullableString(item.Anchor),
toNullableString(item.Category),
toNullableString(item.SubCategory),
toNullableString(item.L),
toNullableString(item.R),
toNullableString(item.LinkText),
);
}
});
insertTx();
return { jobId: jobRow.id, createdAt, insertedResultsCount: items.length };
}
export function getScrubHistory(): ScrubHistoryItem[] {
const database = getDb();
const jobs = database
.prepare(
"SELECT id, created_at, ownr_name, vehicle, claim_number, pdf_url FROM jobs ORDER BY created_at DESC",
)
.all() as ScrubHistoryJobRow[];
if (jobs.length === 0) return [];
const results = database
.prepare(
"SELECT id, created_at, job_id, anchor, category, subcategory, left_text, right_text, linktext FROM scrub_results ORDER BY created_at DESC, id DESC",
)
.all() as ScrubHistoryScrubResultRow[];
const resultsByJobId = new Map<string, ScrubHistoryItem["results"]>();
for (const r of results) {
const bucket = resultsByJobId.get(r.job_id) ?? [];
bucket.push({
createdAt: r.created_at,
anchor: r.anchor,
category: r.category,
subcategory: r.subcategory,
left: r.left_text,
right: r.right_text,
linktext: r.linktext,
});
resultsByJobId.set(r.job_id, bucket);
}
return jobs.map((j) => ({
id: j.id,
createdAt: j.created_at,
ownrName: j.ownr_name,
vehicle: j.vehicle,
claimNumber: j.claim_number,
pdfUrl: j.pdf_url ?? null,
results: resultsByJobId.get(j.id) ?? [],
}));
}
function normalizePageParams(params: { page: number; pageSize: number }): {
page: number;
pageSize: number;
offset: number;
} {
const page = Number.isFinite(params.page) ? Math.floor(params.page) : 1;
const pageSize = Number.isFinite(params.pageSize)
? Math.floor(params.pageSize)
: 10;
const safePage = Math.max(1, page);
const safePageSize = Math.min(100, Math.max(1, pageSize));
const offset = (safePage - 1) * safePageSize;
return { page: safePage, pageSize: safePageSize, offset };
}
export function getScrubHistoryPage(params: {
page: number;
pageSize: number;
}): ScrubHistoryPage {
const database = getDb();
const { page, pageSize, offset } = normalizePageParams(params);
const totalJobsRow = database
.prepare("SELECT COUNT(1) as c FROM jobs")
.get() as { c: number };
const totalJobs = totalJobsRow?.c ?? 0;
const totalResultsRow = database
.prepare("SELECT COUNT(1) as c FROM scrub_results")
.get() as { c: number };
const totalResults = totalResultsRow?.c ?? 0;
const lastProcessedRow = database
.prepare("SELECT MAX(created_at) as m FROM jobs")
.get() as { m: number | null };
const lastProcessed = lastProcessedRow?.m ?? null;
if (totalJobs === 0) {
return {
items: [],
totalJobs,
page,
pageSize,
totalResults,
lastProcessed,
};
}
const jobs = database
.prepare(
"SELECT id, created_at, ownr_name, vehicle, claim_number, pdf_url FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?",
)
.all(pageSize, offset) as ScrubHistoryJobRow[];
if (jobs.length === 0) {
return {
items: [],
totalJobs,
page,
pageSize,
totalResults,
lastProcessed,
};
}
const jobIds = jobs.map((j) => j.id);
const placeholders = jobIds.map(() => "?").join(",");
const results = database
.prepare(
`SELECT id, created_at, job_id, anchor, category, subcategory, left_text, right_text, linktext FROM scrub_results WHERE job_id IN (${placeholders}) ORDER BY created_at DESC, id DESC`,
)
.all(...jobIds) as ScrubHistoryScrubResultRow[];
const resultsByJobId = new Map<string, ScrubHistoryItem["results"]>();
for (const r of results) {
const bucket = resultsByJobId.get(r.job_id) ?? [];
bucket.push({
createdAt: r.created_at,
anchor: r.anchor,
category: r.category,
subcategory: r.subcategory,
left: r.left_text,
right: r.right_text,
linktext: r.linktext,
});
resultsByJobId.set(r.job_id, bucket);
}
return {
items: jobs.map((j) => ({
id: j.id,
createdAt: j.created_at,
ownrName: j.ownr_name,
vehicle: j.vehicle,
claimNumber: j.claim_number,
pdfUrl: j.pdf_url ?? null,
results: resultsByJobId.get(j.id) ?? [],
})),
totalJobs,
page,
pageSize,
totalResults,
lastProcessed,
};
}
export function deleteScrubHistoryJob(jobId: string): { deletedJobs: number } {
const database = getDb();
const trimmed = jobId.trim();
if (!trimmed) return { deletedJobs: 0 };
const tx = database.transaction(() => {
const res = database.prepare("DELETE FROM jobs WHERE id = ?").run(trimmed);
return res.changes;
});
return { deletedJobs: tx() };
}
export function clearScrubHistory(): { clearedJobs: number } {
const database = getDb();
const tx = database.transaction(() => {
const countBefore = database
.prepare("SELECT COUNT(1) as c FROM jobs")
.get() as { c: number };
database.prepare("DELETE FROM jobs").run();
return countBefore?.c ?? 0;
});
return { clearedJobs: tx() };
}

View File

@@ -50,33 +50,33 @@ async function ImportJob(filepath: string): Promise<void> {
//The below all end up returning parts of the job object.
//Some of them return additional info - e.g. owner or vehicle record data at both the job and corresponding table level.
setAppProgressbar(0.1);
const env: DecodedEnv = await DecodeEnv(extensionlessFilePath);
setAppProgressbar(0.15);
const ad1: DecodedAd1 = await DecodeAD1(extensionlessFilePath);
setAppProgressbar(0.2);
const ad2: DecodedAD2 = await DecodeAD2(extensionlessFilePath);
setAppProgressbar(0.25);
const veh: DecodedVeh = await DecodeVeh(extensionlessFilePath);
setAppProgressbar(0.3);
const lin: DecodedLin = await DecodeLin(extensionlessFilePath);
setAppProgressbar(0.35);
const pfh: DecodedPfh = await DecodePfh(extensionlessFilePath);
setAppProgressbar(0.4);
const pfl: DecodedPfl = await DecodePfl(extensionlessFilePath);
setAppProgressbar(0.45);
const pft: DecodedPft = await DecodePft(extensionlessFilePath);
setAppProgressbar(0.5);
const env: DecodedEnv = await DecodeEnv(extensionlessFilePath);
setAppProgressbar(0.1);
const ad1: DecodedAd1 = await DecodeAD1(extensionlessFilePath);
setAppProgressbar(0.15);
const ad2: DecodedAD2 = await DecodeAD2(extensionlessFilePath);
setAppProgressbar(0.2);
const veh: DecodedVeh = await DecodeVeh(extensionlessFilePath);
setAppProgressbar(0.25);
const lin: DecodedLin = await DecodeLin(extensionlessFilePath);
setAppProgressbar(0.3);
const pfh: DecodedPfh = await DecodePfh(extensionlessFilePath);
setAppProgressbar(0.35);
const pfl: DecodedPfl = await DecodePfl(extensionlessFilePath);
setAppProgressbar(0.4);
const pft: DecodedPft = await DecodePft(extensionlessFilePath);
setAppProgressbar(0.45);
const pfm: DecodedPfm = await DecodePfm(extensionlessFilePath);
setAppProgressbar(0.55);
setAppProgressbar(0.5);
const pfo: DecodedPfo = await DecodePfo(extensionlessFilePath); // TODO: This will be the `cieca_pfo` object
setAppProgressbar(0.6);
setAppProgressbar(0.55);
const stl: DecodedStl = await DecodeStl(extensionlessFilePath); // TODO: This will be the `cieca_stl` object
setAppProgressbar(0.65);
setAppProgressbar(0.6);
const ttl: DecodedTtl = await DecodeTtl(extensionlessFilePath);
setAppProgressbar(0.7);
setAppProgressbar(0.65);
const pfp: DecodedPfp = await DecodePfp(extensionlessFilePath);
setAppProgressbar(0.75);
setAppProgressbar(0.7);
const jobObjectUncleaned: RawJobDataObject = {
...env,
@@ -97,7 +97,6 @@ async function ImportJob(filepath: string): Promise<void> {
// Replace owner information with claimant information if necessary
const jobObject = ReplaceOwnerInfoWithClaimant(jobObjectUncleaned);
setAppProgressbar(0.8);
if (import.meta.env.DEV) {
// Save jobObject to a timestamped JSON file
@@ -132,33 +131,14 @@ async function ImportJob(filepath: string): Promise<void> {
issupplement: false,
jobid: null,
};
setAppProgressbar(0.85);
setAppProgressbar(0.73);
console.log("Available Job record to upload;", newAvailableJob);
setAppProgressbar(0.95);
setAppProgressbar(-1);
const uploadNotification = new Notification({
title: "Job Imported",
//subtitle: `${newAvailableJob.ownr_name} - ${newAvailableJob.vehicle_info}`,
body: `${newAvailableJob.ownr_name} - ${newAvailableJob.vehicle_info}. Click to view.`,
actions: [{ text: "View Job", type: "button" }],
});
uploadNotification.on("click", () => {
shell.openExternal(
`${
store.get("app.isTest")
? import.meta.env.VITE_FE_URL_TEST
: import.meta.env.VITE_FE_URL
}/manage/available`,
);
});
uploadNotification.show();
//Scrub the estimate
const scrubResults = await ScrubEstimate({ job: jobObject });
setAppProgressbar(0.95);
const esApiKey = store.get("settings.esApiKey") as string;
UploadEmsToS3({
extensionlessFilePath,
@@ -167,11 +147,24 @@ async function ImportJob(filepath: string): Promise<void> {
clm_no: jobObject.clm_no ?? "",
ownr_ln: jobObject.ownr_ln ?? "",
});
console.log("Got past the job upload.");
setAppProgressbar(-1);
const uploadNotification = new Notification({
title: "Job Scrubbed",
//subtitle: `${newAvailableJob.ownr_name} - ${newAvailableJob.vehicle_info}`,
body: `${newAvailableJob.ownr_name} - ${newAvailableJob.vehicle_info}. Click to view.`,
actions: [{ text: "View Job", type: "button" }],
});
uploadNotification.on("click", () => {
if (scrubResults?.data?.resultPDFUrl) {
shell.openExternal(scrubResults.data.resultPDFUrl);
}
});
uploadNotification.show();
} catch (error) {
log.error("Error encountered while decoding job. ", errorTypeCheck(error));
const uploadNotificationFailure = new Notification({
title: "Job Upload Failure",
title: "Job Scrub Failure",
body: errorTypeCheck(error).message, //TODO: Remove after debug.
});

View File

@@ -71,7 +71,7 @@ async function UploadEmsToS3({
ownr_ln,
},
{
headers: {},
headers: { "x-api-key": esApiKey },
},
);

View File

@@ -7,6 +7,7 @@ import path from "path";
import { RawJobDataObject } from "../decoder/decoder";
import store from "../store/store";
import ipcTypes from "../../util/ipcTypes.json";
import { insertScrubRun } from "../db/scrub-history-db";
// Function to write job object to logs subfolder
async function writeJobToLogsFolder(job, fileName): Promise<string> {
@@ -84,13 +85,34 @@ async function ScrubEstimate({
}
// Send raw job to Lambda - transformation happens server-side
const result = await axios.post(estimateScrubberUrl, {
esApiKey,
rawJob: job, // Changed from 'estimate' to 'rawJob'
});
const result = await axios.post(
estimateScrubberUrl,
{
esApiKey,
rawJob: job, // Changed from 'estimate' to 'rawJob'
},
{
headers: {
"x-api-key": esApiKey,
},
},
);
const { resultPDFUrl, reportIssueUrl, identified_item } =
result?.data ?? {};
const { resultPDFUrl, identified_item } = result?.data ?? {};
try {
insertScrubRun({
job,
identifiedItems: identified_item.slice(1, identified_item.length), // Remove first item which is the result metadata
pdfUrl: typeof resultPDFUrl === "string" ? resultPDFUrl : null,
});
const mainWindow = BrowserWindow.getAllWindows()[0];
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send(ipcTypes.toRenderer.scrub.historyUpdated);
}
} catch (dbError) {
log.error("Failed to persist scrub history:", dbError);
}
// log.log("Estimate Scrubber Result:", result.data, resultPDFUrl);
// const b = BrowserWindow.getAllWindows()[0];
@@ -112,20 +134,29 @@ async function ScrubEstimate({
return resultPDFUrl;
} catch (error) {
const err = error as AxiosError;
log.error("Error while scrubbing estimate:", error, err.stack);
log.error("Error while scrubbing estimate:", err.message, err.stack);
log.error("Error Response Data:", err.response?.data);
const mainWindow = BrowserWindow.getAllWindows()[0];
if (error.status === 400) {
mainWindow.webContents.send(ipcTypes.toRenderer.scrub.scrubError, {
const status = err.response?.status;
const responseData = err.response?.data;
const responseMessage =
typeof responseData === "string"
? responseData
: responseData
? JSON.stringify(responseData)
: undefined;
if (status === 400) {
mainWindow?.webContents.send(ipcTypes.toRenderer.scrub.scrubError, {
message:
error.response?.data ||
responseMessage ||
"Error encountered sending estimate to Estimate Scrubber.",
});
} else if (error.status === 401) {
mainWindow.webContents.send(ipcTypes.toRenderer.scrub.scrubError, {
} else if (status === 401) {
mainWindow?.webContents.send(ipcTypes.toRenderer.scrub.scrubError, {
message:
err.response?.data || "Authentication with Estimate Scrubber failed."ta,
responseMessage || "Authentication with Estimate Scrubber failed.",
});
}
return "Error: Unable to scrub estimate.";

View File

@@ -23,6 +23,11 @@ import {
ipcMainHandleAuthStateChanged,
ipMainHandleResetPassword,
} from "./ipcMainHandler.user";
import {
ScrubHistoryClearAll,
ScrubHistoryDeleteJob,
ScrubHistoryGetAll,
} from "./ipcMainHandler.scrubHistory";
// Log all IPC messages and their payloads
const logIpcMessages = (): void => {
@@ -56,6 +61,11 @@ ipcMain.on(ipcTypes.toMain.test, () =>
ipcMain.on(ipcTypes.toMain.authStateChanged, ipcMainHandleAuthStateChanged);
ipcMain.on(ipcTypes.toMain.user.resetPassword, ipMainHandleResetPassword);
// Scrub History Handlers
ipcMain.handle(ipcTypes.toMain.scrubHistory.getAll, ScrubHistoryGetAll);
ipcMain.handle(ipcTypes.toMain.scrubHistory.deleteJob, ScrubHistoryDeleteJob);
ipcMain.handle(ipcTypes.toMain.scrubHistory.clearAll, ScrubHistoryClearAll);
// Add debug handlers if in development
if (import.meta.env.DEV) {
log.debug("[IPC Debug Functions] Adding Debug Handlers");

View File

@@ -0,0 +1,67 @@
import { BrowserWindow } from "electron";
import log from "electron-log/main";
import ipcTypes from "../../util/ipcTypes.json";
import {
clearScrubHistory,
deleteScrubHistoryJob,
getScrubHistory,
getScrubHistoryPage,
} from "../db/scrub-history-db";
export async function ScrubHistoryGetAll(
_event: Electron.IpcMainInvokeEvent,
params?: { page?: number; pageSize?: number },
): Promise<
ReturnType<typeof getScrubHistory> | ReturnType<typeof getScrubHistoryPage>
> {
try {
if (
params &&
(params.page !== undefined || params.pageSize !== undefined)
) {
return getScrubHistoryPage({
page: params.page ?? 1,
pageSize: params.pageSize ?? 10,
});
}
return getScrubHistory();
} catch (error) {
log.error("[ScrubHistoryGetAll] failed", error);
return [];
}
}
function notifyHistoryUpdated(): void {
const mainWindow = BrowserWindow.getAllWindows()[0];
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send(ipcTypes.toRenderer.scrub.historyUpdated);
}
}
export async function ScrubHistoryDeleteJob(
_event: Electron.IpcMainInvokeEvent,
jobId: string,
): Promise<{ ok: boolean; deletedJobs: number }> {
try {
const { deletedJobs } = deleteScrubHistoryJob(jobId);
if (deletedJobs > 0) notifyHistoryUpdated();
return { ok: true, deletedJobs };
} catch (error) {
log.error("[ScrubHistoryDeleteJob] failed", error);
return { ok: false, deletedJobs: 0 };
}
}
export async function ScrubHistoryClearAll(): Promise<{
ok: boolean;
clearedJobs: number;
}> {
try {
const { clearedJobs } = clearScrubHistory();
if (clearedJobs > 0) notifyHistoryUpdated();
return { ok: true, clearedJobs };
} catch (error) {
log.error("[ScrubHistoryClearAll] failed", error);
return { ok: false, clearedJobs: 0 };
}
}