admin/bodyshop-media-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor namings.

Browse Source
This commit is contained in:
Patrick Fic
2025-11-06 14:07:49 -08:00
parent 994a35025b
commit ebe3d8821d
3 changed files with 131 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
server.ts
View File

@@ -127,29 +127,23 @@ const loggedBodyshopIds = new Set<string>();
app.use((req, res, next) => { app.use((req, res, next) => {
//Asynchronously check the headers for a bodyshopid. If it exists, write it to a json file in the root directory. Only do this if the bodyshopid has not already been logged once since server start. //Asynchronously check the headers for a bodyshopid. If it exists, write it to a json file in the root directory. Only do this if the bodyshopid has not already been logged once since server start.
const bodyshopId = req.headers.bodyshopid as string; const bodyshopId = req.headers.bodyshopid as string;
console.log("*** ~ loggedBodyshopIds:", loggedBodyshopIds);
if (bodyshopId && !loggedBodyshopIds.has(bodyshopId)) { if (bodyshopId && !loggedBodyshopIds.has(bodyshopId)) {
loggedBodyshopIds.add(bodyshopId); loggedBodyshopIds.add(bodyshopId);
// Asynchronously write to file without blocking the request // Asynchronously write to file without blocking the request
(async () => { (async () => {
try { try {
const fs = await import("fs/promises"); const fs = await import("fs/promises");
const filePath = path.join(FolderPaths.Root, "config.json");
let existingIds: string[] = []; let existingIds: string[] = [];
try { try {
const fileContent = await fs.readFile(filePath, "utf-8"); const fileContent = await fs.readFile(FolderPaths.Config, "utf-8");
const configFile = JSON.parse(fileContent); const configFile = JSON.parse(fileContent);
existingIds = configFile.bodyshopIds || []; existingIds = configFile.bodyshopIds || [];
} catch { } catch {
// File doesn't exist or is invalid, start with empty array // File doesn't exist or is invalid, start with empty array
} }
if (!existingIds.includes(bodyshopId)) { if (!existingIds.includes(bodyshopId)) {
existingIds.push(bodyshopId); existingIds.push(bodyshopId);
await fs.writeFile(filePath, JSON.stringify({ bodyshopIds: existingIds }, null, 2)); await fs.writeFile(FolderPaths.Config, JSON.stringify({ bodyshopIds: existingIds }, null, 2));
logger.info(`Logged new bodyshop ID: ${bodyshopId}`); logger.info(`Logged new bodyshop ID: ${bodyshopId}`);
} }
} catch (error) { } catch (error) {
@@ -157,7 +151,6 @@ app.use((req, res, next) => {
} }
})(); })();
} }
next(); next();
}); });

224
util/s3Sync.ts
View File

@@ -5,6 +5,8 @@ import { readdir, stat as fsStat } from "fs/promises";
import * as path from "path"; import * as path from "path";
import { logger } from "../server.js"; import { logger } from "../server.js";
import { FolderPaths } from "./serverInit.js"; import { FolderPaths } from "./serverInit.js";
import axios from "axios";
import { UUID } from "crypto";
const execAsync = promisify(exec); const execAsync = promisify(exec);
@@ -17,21 +19,25 @@ interface S3SyncConfig {
} }
export interface JobFolderStats { export interface JobFolderStats {
jobId: string; bodyshopid: UUID;
relativePath: string; jobid: UUID | string | null;
documentCount: number; //relativePath: string;
totalSizeBytes: number; document_count: number;
totalSizeMB: number; total_size_bytes: number;
fileTypeStats: { [extension: string]: number }; total_size_mb: number;
file_type_stats: { [extension: string]: number };
} }
export interface JobsDirectoryAnalysis { export interface JobsDirectoryAnalysis {
totalJobs: number; bodyshopid: UUID;
totalDocuments: number; total_jobs: number;
totalSizeBytes: number; total_documents: number;
totalSizeMB: number; total_size_bytes: number;
fileTypeStats: { [extension: string]: number }; total_size_mb: number;
jobs: JobFolderStats[]; file_type_stats: { [extension: string]: number };
media_analytics_details: {
data: JobFolderStats[];
};
} }
export class S3Sync { export class S3Sync {
@@ -162,20 +168,21 @@ export class S3Sync {
if (!(await fs.pathExists(jobsPath))) { if (!(await fs.pathExists(jobsPath))) {
logger.warn(`Jobs directory does not exist: ${jobsPath}`); logger.warn(`Jobs directory does not exist: ${jobsPath}`);
return { return {
totalJobs: 0, bodyshopid,
totalDocuments: 0, total_jobs: 0,
totalSizeBytes: 0, total_documents: 0,
totalSizeMB: 0, total_size_bytes: 0,
fileTypeStats: {}, total_size_mb: 0,
jobs: [] file_type_stats: {},
media_analytics_details: { data: [] }
}; };
} }
const jobFolders = await readdir(jobsPath); const jobFolders = await readdir(jobsPath);
const jobStats: JobFolderStats[] = []; const jobStats: JobFolderStats[] = [];
let totalDocuments = 0; let total_documents = 0;
let totalSizeBytes = 0; let total_size_bytes = 0;
const aggregatedFileTypeStats: { [extension: string]: number } = {}; const aggregated_file_type_stats: { [extension: string]: number } = {};
for (const jobFolder of jobFolders) { for (const jobFolder of jobFolders) {
const jobFolderPath = path.join(jobsPath, jobFolder); const jobFolderPath = path.join(jobsPath, jobFolder);
@@ -185,27 +192,28 @@ export class S3Sync {
if (stat.isDirectory()) { if (stat.isDirectory()) {
const folderStats = await this.analyzeJobFolder(jobsPath, jobFolder); const folderStats = await this.analyzeJobFolder(jobsPath, jobFolder);
jobStats.push(folderStats); jobStats.push(folderStats);
totalDocuments += folderStats.documentCount; total_documents += folderStats.document_count;
totalSizeBytes += folderStats.totalSizeBytes; total_size_bytes += folderStats.total_size_bytes;
// Aggregate file type stats // Aggregate file type stats
for (const [ext, count] of Object.entries(folderStats.fileTypeStats)) { for (const [ext, count] of Object.entries(folderStats.file_type_stats)) {
aggregatedFileTypeStats[ext] = (aggregatedFileTypeStats[ext] || 0) + count; aggregated_file_type_stats[ext] = (aggregated_file_type_stats[ext] || 0) + count;
} }
} }
} }
const analysis: JobsDirectoryAnalysis = { const analysis: JobsDirectoryAnalysis = {
totalJobs: jobStats.length, bodyshopid,
totalDocuments, total_jobs: jobStats.length,
totalSizeBytes, total_documents,
totalSizeMB: Math.round((totalSizeBytes / (1024 * 1024)) * 100) / 100, total_size_bytes,
fileTypeStats: aggregatedFileTypeStats, total_size_mb: Math.round((total_size_bytes / (1024 * 1024)) * 100) / 100,
jobs: jobStats.sort((a, b) => a.jobId.localeCompare(b.jobId)) file_type_stats: aggregated_file_type_stats,
media_analytics_details: { data: jobStats.sort((a, b) => a.jobid?.localeCompare(b.jobid!) || 0) }
}; };
logger.info( logger.info(
`Jobs directory analysis complete: ${analysis.totalJobs} jobs, ${analysis.totalDocuments} documents, ${analysis.totalSizeMB} MB` `Jobs directory analysis complete: ${analysis.total_jobs} jobs, ${analysis.total_documents} documents, ${analysis.total_size_mb} MB`
); );
return analysis; return analysis;
} catch (error) { } catch (error) {
@@ -217,19 +225,20 @@ export class S3Sync {
/** /**
* Analyze a single job folder * Analyze a single job folder
*/ */
private async analyzeJobFolder(jobsPath: string, jobId: string): Promise<JobFolderStats> { private async analyzeJobFolder(jobsPath: string, jobid: string): Promise<JobFolderStats> {
const jobFolderPath = path.join(jobsPath, jobId); const jobFolderPath = path.join(jobsPath, jobid);
const relativePath = path.relative(FolderPaths.Root, jobFolderPath); // const relativePath = path.relative(FolderPaths.Root, jobFolderPath);
const { documentCount, totalSizeBytes, fileTypeStats } = await this.getDirectoryStats(jobFolderPath); const { document_count, total_size_bytes, file_type_stats } = await this.getDirectoryStats(jobFolderPath);
return { return {
jobId, jobid: jobid === "temporary" ? null : (jobid as UUID),
relativePath, bodyshopid,
documentCount, //relativePath,
totalSizeBytes, document_count,
totalSizeMB: Math.round((totalSizeBytes / (1024 * 1024)) * 100) / 100, total_size_bytes,
fileTypeStats total_size_mb: Math.round((total_size_bytes / (1024 * 1024)) * 100) / 100,
file_type_stats
}; };
} }
@@ -238,10 +247,10 @@ export class S3Sync {
*/ */
private async getDirectoryStats( private async getDirectoryStats(
dirPath: string dirPath: string
): Promise<{ documentCount: number; totalSizeBytes: number; fileTypeStats: { [extension: string]: number } }> { ): Promise<{ document_count: number; total_size_bytes: number; file_type_stats: { [extension: string]: number } }> {
let documentCount = 0; let document_count = 0;
let totalSizeBytes = 0; let total_size_bytes = 0;
const fileTypeStats: { [extension: string]: number } = {}; const file_type_stats: { [extension: string]: number } = {};
try { try {
const items = await readdir(dirPath); const items = await readdir(dirPath);
@@ -253,28 +262,28 @@ export class S3Sync {
if (stat.isDirectory()) { if (stat.isDirectory()) {
// Recursively analyze subdirectories // Recursively analyze subdirectories
const subStats = await this.getDirectoryStats(itemPath); const subStats = await this.getDirectoryStats(itemPath);
documentCount += subStats.documentCount; document_count += subStats.document_count;
totalSizeBytes += subStats.totalSizeBytes; total_size_bytes += subStats.total_size_bytes;
// Merge file type stats // Merge file type stats
for (const [ext, count] of Object.entries(subStats.fileTypeStats)) { for (const [ext, count] of Object.entries(subStats.file_type_stats)) {
fileTypeStats[ext] = (fileTypeStats[ext] || 0) + count; file_type_stats[ext] = (file_type_stats[ext] || 0) + count;
} }
} else { } else {
// Count files as documents // Count files as documents
documentCount++; document_count++;
totalSizeBytes += stat.size; total_size_bytes += stat.size;
// Track file extension // Track file extension
const ext = path.extname(item).toLowerCase() || "no-extension"; const ext = path.extname(item).toLowerCase() || "no-extension";
fileTypeStats[ext] = (fileTypeStats[ext] || 0) + 1; file_type_stats[ext] = (file_type_stats[ext] || 0) + 1;
} }
} }
} catch (error) { } catch (error) {
logger.error(`Error analyzing directory ${dirPath}:`, error); logger.error(`Error analyzing directory ${dirPath}:`, error);
} }
return { documentCount, totalSizeBytes, fileTypeStats }; return { document_count, total_size_bytes, file_type_stats };
} }
} }
@@ -310,6 +319,8 @@ export function createS3SyncFromEnv(): S3Sync | null {
export async function analyzeJobsDirectory(): Promise<JobsDirectoryAnalysis> { export async function analyzeJobsDirectory(): Promise<JobsDirectoryAnalysis> {
try { try {
logger.info("Starting Jobs directory analysis..."); logger.info("Starting Jobs directory analysis...");
//Read from the config.json file in the root directory to get the bodyshopid
const bodyshopid: UUID = await getBodyshopIdFromConfig();
const jobsPath = FolderPaths.Jobs; const jobsPath = FolderPaths.Jobs;
@@ -317,20 +328,21 @@ export async function analyzeJobsDirectory(): Promise<JobsDirectoryAnalysis> {
if (!(await fs.pathExists(jobsPath))) { if (!(await fs.pathExists(jobsPath))) {
logger.warn(`Jobs directory does not exist: ${jobsPath}`); logger.warn(`Jobs directory does not exist: ${jobsPath}`);
return { return {
totalJobs: 0, bodyshopid,
totalDocuments: 0, total_jobs: 0,
totalSizeBytes: 0, total_documents: 0,
totalSizeMB: 0, total_size_bytes: 0,
fileTypeStats: {}, total_size_mb: 0,
jobs: [] file_type_stats: {},
media_analytics_details: { data: [] }
}; };
} }
const jobFolders = await readdir(jobsPath); const jobFolders = await readdir(jobsPath);
const jobStats: JobFolderStats[] = []; const jobStats: JobFolderStats[] = [];
let totalDocuments = 0; let total_documents = 0;
let totalSizeBytes = 0; let total_size_bytes = 0;
const aggregatedFileTypeStats: { [extension: string]: number } = {}; const aggregated_file_type_stats: { [extension: string]: number } = {};
for (const jobFolder of jobFolders) { for (const jobFolder of jobFolders) {
const jobFolderPath = path.join(jobsPath, jobFolder); const jobFolderPath = path.join(jobsPath, jobFolder);
@@ -340,34 +352,38 @@ export async function analyzeJobsDirectory(): Promise<JobsDirectoryAnalysis> {
if (stat.isDirectory()) { if (stat.isDirectory()) {
const folderStats = await analyzeJobFolder(jobsPath, jobFolder); const folderStats = await analyzeJobFolder(jobsPath, jobFolder);
jobStats.push(folderStats); jobStats.push(folderStats);
totalDocuments += folderStats.documentCount; total_documents += folderStats.document_count;
totalSizeBytes += folderStats.totalSizeBytes; total_size_bytes += folderStats.total_size_bytes;
// Aggregate file type stats // Aggregate file type stats
for (const [ext, count] of Object.entries(folderStats.fileTypeStats)) { for (const [ext, count] of Object.entries(folderStats.file_type_stats)) {
aggregatedFileTypeStats[ext] = (aggregatedFileTypeStats[ext] || 0) + count; aggregated_file_type_stats[ext] = (aggregated_file_type_stats[ext] || 0) + count;
} }
} }
} }
const analysis: JobsDirectoryAnalysis = { const analysis: JobsDirectoryAnalysis = {
totalJobs: jobStats.length, bodyshopid, //read from the config.json file in the root directory
totalDocuments,
totalSizeBytes, total_jobs: jobStats.length,
totalSizeMB: Math.round((totalSizeBytes / (1024 * 1024)) * 100) / 100, total_documents,
fileTypeStats: aggregatedFileTypeStats, total_size_bytes,
jobs: jobStats.sort((a, b) => a.jobId.localeCompare(b.jobId)) total_size_mb: Math.round((total_size_bytes / (1024 * 1024)) * 100) / 100,
file_type_stats: aggregated_file_type_stats,
media_analytics_details: { data: jobStats.sort((a, b) => a.jobid?.localeCompare(b.jobid!) || 0) }
}; };
logger.info( logger.info(
`Jobs directory analysis complete: ${analysis.totalJobs} jobs, ${analysis.totalDocuments} documents, ${analysis.totalSizeMB} MB` `Jobs directory analysis complete: ${analysis.total_jobs} jobs, ${analysis.total_documents} documents, ${analysis.total_size_mb} MB`
); );
//Add an upload to the IO database to categorize all of this. //Add an upload to the IO database to categorize all of this.
const apiURL = "http://host.docker.internal:4000/analytics/documents";
const result = await axios.post(apiURL, { data: analysis });
return analysis; return analysis;
} catch (error) { } catch (error) {
logger.error("Failed to analyze Jobs directory:", error); logger.error("Failed to analyze Jobs directory:", JSON.stringify(error, null, 4));
throw error; throw error;
} }
} }
@@ -375,19 +391,20 @@ export async function analyzeJobsDirectory(): Promise<JobsDirectoryAnalysis> {
/** /**
* Analyze a single job folder (standalone helper function) * Analyze a single job folder (standalone helper function)
*/ */
async function analyzeJobFolder(jobsPath: string, jobId: string): Promise<JobFolderStats> { async function analyzeJobFolder(jobsPath: string, jobid: string): Promise<JobFolderStats> {
const jobFolderPath = path.join(jobsPath, jobId); const jobFolderPath = path.join(jobsPath, jobid);
const relativePath = path.relative(FolderPaths.Root, jobFolderPath); const relativePath = path.relative(FolderPaths.Root, jobFolderPath);
const { documentCount, totalSizeBytes, fileTypeStats } = await getDirectoryStats(jobFolderPath); const { document_count, total_size_bytes, file_type_stats } = await getDirectoryStats(jobFolderPath);
return { return {
jobId, bodyshopid,
relativePath, jobid,
documentCount, // relativePath,
totalSizeBytes, document_count,
totalSizeMB: Math.round((totalSizeBytes / (1024 * 1024)) * 100) / 100, total_size_bytes,
fileTypeStats total_size_mb: Math.round((total_size_bytes / (1024 * 1024)) * 100) / 100,
file_type_stats
}; };
} }
@@ -396,10 +413,10 @@ async function analyzeJobFolder(jobsPath: string, jobId: string): Promise<JobFol
*/ */
async function getDirectoryStats( async function getDirectoryStats(
dirPath: string dirPath: string
): Promise<{ documentCount: number; totalSizeBytes: number; fileTypeStats: { [extension: string]: number } }> { ): Promise<{ document_count: number; total_size_bytes: number; file_type_stats: { [extension: string]: number } }> {
let documentCount = 0; let document_count = 0;
let totalSizeBytes = 0; let total_size_bytes = 0;
const fileTypeStats: { [extension: string]: number } = {}; const file_type_stats: { [extension: string]: number } = {};
try { try {
const items = await readdir(dirPath); const items = await readdir(dirPath);
@@ -411,26 +428,41 @@ async function getDirectoryStats(
if (stat.isDirectory()) { if (stat.isDirectory()) {
// Recursively analyze subdirectories // Recursively analyze subdirectories
const subStats = await getDirectoryStats(itemPath); const subStats = await getDirectoryStats(itemPath);
documentCount += subStats.documentCount; document_count += subStats.document_count;
totalSizeBytes += subStats.totalSizeBytes; total_size_bytes += subStats.total_size_bytes;
// Merge file type stats // Merge file type stats
for (const [ext, count] of Object.entries(subStats.fileTypeStats)) { for (const [ext, count] of Object.entries(subStats.file_type_stats)) {
fileTypeStats[ext] = (fileTypeStats[ext] || 0) + count; file_type_stats[ext] = (file_type_stats[ext] || 0) + count;
} }
} else { } else {
// Count files as documents // Count files as documents
documentCount++; document_count++;
totalSizeBytes += stat.size; total_size_bytes += stat.size;
// Track file extension // Track file extension
const ext = path.extname(item).toLowerCase() || "no-extension"; const ext = path.extname(item).toLowerCase() || "no-extension";
fileTypeStats[ext] = (fileTypeStats[ext] || 0) + 1; file_type_stats[ext] = (file_type_stats[ext] || 0) + 1;
} }
} }
} catch (error) { } catch (error) {
logger.error(`Error analyzing directory ${dirPath}:`, error); logger.error(`Error analyzing directory ${dirPath}:`, error);
} }
return { documentCount, totalSizeBytes, fileTypeStats }; return { document_count, total_size_bytes, file_type_stats };
} }
let bodyshopid: UUID;
const getBodyshopIdFromConfig = async (): Promise<UUID> => {
if (bodyshopid) return bodyshopid;
try {
const fs = await import("fs/promises"); //Required as fs-extra fails.
const configData = await fs.readFile(FolderPaths.Config, "utf-8");
const config = JSON.parse(configData);
bodyshopid = config.bodyshopIds[0] as UUID;
return bodyshopid;
} catch (error) {
logger.error("Failed to read bodyshopid from config.json:", error, (error as Error).stack);
throw new Error("Could not read bodyshopid from config.json");
}
};

1
util/serverInit.ts
View File

@@ -25,6 +25,7 @@ export const FolderPaths = {
ConvertedOriginalSubDir: "ConvertedOriginal", ConvertedOriginalSubDir: "ConvertedOriginal",
DamagedSubDir: "DamagedOriginal", DamagedSubDir: "DamagedOriginal",
StaticPath: "/static", StaticPath: "/static",
Config: path.join(RootDirectory, "config.json"),
JobsFolder, JobsFolder,
VendorsFolder VendorsFolder
}; };