admin/bodyshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IO-3092 Implement delete, move and download on image proxy. Add imgproxy based components.

Browse Source
This commit is contained in:
Patrick Fic
2025-02-07 13:33:22 -08:00
parent fbb473941c
commit b069b6bc4c
17 changed files with 1491 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
server/graphql-client/queries.js
View File

@@ -2722,3 +2722,30 @@ exports.GET_DOCUMENTS_BY_JOB = `
}
}
}`;
exports.QUERY_TEMPORARY_DOCS = ` query QUERY_TEMPORARY_DOCS {
documents(where: { jobid: { _is_null: true } }, order_by: { takenat: desc }) {
id
name
key
type
extension
size
takenat
}
}`;
exports.GET_DOCUMENTS_BY_IDS = `
query GET_DOCUMENTS_BY_IDS($documentIds: [uuid!]!) {
documents(where: {id: {_in: $documentIds}}, order_by: {takenat: desc}) {
id
name
key
type
extension
size
takenat
}
}
`;

233
server/media/imgprox-media.js
View File

@@ -3,13 +3,30 @@ require("dotenv").config({
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
});
const logger = require("../utils/logger");
const { S3Client, PutObjectCommand, GetObjectCommand } = require("@aws-sdk/client-s3");
const {
S3Client,
PutObjectCommand,
GetObjectCommand,
CopyObjectCommand,
DeleteObjectCommand
} = require("@aws-sdk/client-s3");
const { Upload } = require("@aws-sdk/lib-storage");
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
const crypto = require("crypto");
const { InstanceRegion } = require("../utils/instanceMgr");
const { GET_DOCUMENTS_BY_JOB } = require("../graphql-client/queries");
//TODO: Remove hardcoded values.
const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL;
const {
GET_DOCUMENTS_BY_JOB,
QUERY_TEMPORARY_DOCS,
GET_DOCUMENTS_BY_IDS,
DELETE_MEDIA_DOCUMENTS
} = require("../graphql-client/queries");
const archiver = require("archiver");
const stream = require("node:stream");
const imgproxyBaseUrl =
// `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` || //Direct Lambda function access to bypass CDN.
process.env.IMGPROXY_BASE_URL;
const imgproxyKey = process.env.IMGPROXY_KEY;
const imgproxySalt = process.env.IMGPROXY_SALT;
const imgproxyDestinationBucket = process.env.IMGPROXY_DESTINATION_BUCKET;
@@ -31,10 +48,13 @@ exports.generateSignedUploadUrls = async (req, res) => {
const signedUrls = [];
for (const filename of filenames) {
// TODO: Implement a different, unique file naming convention.
const key = filename; //GenerateKey({ bodyshopid, jobid, filename });
const client = new S3Client({ region: InstanceRegion() });
const command = new PutObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key });
const command = new PutObjectCommand({
Bucket: imgproxyDestinationBucket,
Key: key,
StorageClass: "INTELLIGENT_TIERING"
});
const presignedUrl = await getSignedUrl(client, command, { expiresIn: 360 });
signedUrls.push({ filename, presignedUrl, key });
}
@@ -61,11 +81,14 @@ exports.getThumbnailUrls = async (req, res) => {
const { jobid, billid } = req.body;
try {
//TODO: Query for all documents related to the job.
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
logger.log("imgproxy-thumbnails", "DEBUG", req.user?.email, jobid, { billid, jobid });
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
const client = req.userGraphQLClient;
const data = await client.request(GET_DOCUMENTS_BY_JOB, { jobId: jobid });
//If there's no jobid and no billid, we're in temporary documents.
const data = await (jobid
? client.request(GET_DOCUMENTS_BY_JOB, { jobId: jobid })
: client.request(QUERY_TEMPORARY_DOCS));
const thumbResizeParams = `rs:fill:250:250:1/g:ce`;
const s3client = new S3Client({ region: InstanceRegion() });
@@ -123,7 +146,9 @@ exports.getThumbnailUrls = async (req, res) => {
res.json(proxiedUrls);
//Iterate over them, build the link based on the media type, and return the array.
} catch (error) {
logger.log("imgproxy-get-proxied-urls-error", "ERROR", req.user?.email, jobid, {
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
jobid,
billid,
message: error.message,
stack: error.stack
});
@@ -137,20 +162,194 @@ exports.getBillFiles = async (req, res) => {
exports.downloadFiles = async (req, res) => {
//Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk
const { jobid, billid, documentids } = req.body;
try {
logger.log("imgproxy-download", "DEBUG", req.user?.email, jobid, { billid, jobid, documentids });
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
const client = req.userGraphQLClient;
//Query for the keys of the document IDs
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
//Using the Keys, get all of the S3 links, zip them, and send back to the client.
const s3client = new S3Client({ region: InstanceRegion() });
const archiveStream = archiver("zip");
archiveStream.on("error", (error) => {
console.error("Archival encountered an error:", error);
throw new Error(error);
});
const passthrough = new stream.PassThrough();
archiveStream.pipe(passthrough);
for (const key of data.documents.map((d) => d.key)) {
const response = await s3client.send(new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key }));
// :: `response.Body` is a Buffer
console.log(path.basename(key));
archiveStream.append(response.Body, { name: path.basename(key) });
}
archiveStream.finalize();
const archiveKey = `archives/${jobid}/archive-${new Date().toISOString()}.zip`;
const parallelUploads3 = new Upload({
client: s3client,
queueSize: 4, // optional concurrency configuration
leavePartsOnError: false, // optional manually handle dropped parts
params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passthrough }
});
parallelUploads3.on("httpUploadProgress", (progress) => {
console.log(progress);
});
const uploadResult = await parallelUploads3.done();
//Generate the presigned URL to download it.
const presignedUrl = await getSignedUrl(
s3client,
new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }),
{ expiresIn: 360 }
);
res.json({ success: true, url: presignedUrl });
//Iterate over them, build the link based on the media type, and return the array.
} catch (error) {
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
jobid,
billid,
message: error.message,
stack: error.stack
});
res.status(400).json({ message: error.message, stack: error.stack });
}
};
exports.deleteFiles = async (req, res) => {
//Mark a file for deletion in s3. Lifecycle deletion will actually delete the copy in the future.
//Mark as deleted from the documents section of the database.
const { ids } = req.body;
try {
logger.log("imgproxy-delete-files", "DEBUG", req.user.email, null, { ids });
const client = req.userGraphQLClient;
//Do this to make sure that they are only deleting things that they have access to
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: ids });
const s3client = new S3Client({ region: InstanceRegion() });
const deleteTransactions = [];
data.documents.forEach((document) => {
deleteTransactions.push(
(async () => {
try {
// Delete the original object
const deleteResult = await s3client.send(
new DeleteObjectCommand({
Bucket: imgproxyDestinationBucket,
Key: document.key
})
);
return document;
} catch (error) {
return { document, error: error, bucket: imgproxyDestinationBucket };
}
})()
);
});
const result = await Promise.all(deleteTransactions);
console.log("*** ~ file: imgprox-media.js:260 ~ exports.deleteFiles ~ result:", result);
const errors = result.filter((d) => d.error);
//Delete only the succesful deletes.
const deleteMutationResult = await client.request(DELETE_MEDIA_DOCUMENTS, {
ids: result.filter((t) => !t.error).map((d) => d.id)
});
res.json({ errors, deleteMutationResult });
} catch (error) {
logger.log("imgproxy-delete-files-error", "ERROR", req.user.email, null, {
ids,
message: error.message,
stack: error.stack
});
res.status(400).json({ message: error.message, stack: error.stack });
}
};
//Gerneate a key for the s3 bucket by popping off the extension, add a timestamp, and add back the extension.
//This is to prevent any collisions/duplicates in the bucket.
function GenerateKey({ bodyshopid, jobid, filename }) {
let nameArray = filename.split(".");
let extension = nameArray.pop();
return `${bodyshopid}/${jobid}/${nameArray.join(".")}-${Date.now()}`;
}
exports.moveFiles = async (req, res) => {
const { documents, tojobid } = req.body;
try {
logger.log("imgproxy-move-files", "DEBUG", req.user.email, null, { documents, tojobid });
const s3client = new S3Client({ region: InstanceRegion() });
const moveTransactions = [];
documents.forEach((document) => {
moveTransactions.push(
(async () => {
try {
// Copy the object to the new key
const copyresult = await s3client.send(
new CopyObjectCommand({
Bucket: imgproxyDestinationBucket,
CopySource: `${imgproxyDestinationBucket}/${document.from}`,
Key: document.to,
StorageClass: "INTELLIGENT_TIERING"
})
);
// Delete the original object
const deleteResult = await s3client.send(
new DeleteObjectCommand({
Bucket: imgproxyDestinationBucket,
Key: document.from
})
);
return document;
} catch (error) {
return { id: document.id, from: document.from, error: error, bucket: imgproxyDestinationBucket };
}
})()
);
});
const result = await Promise.all(moveTransactions);
const errors = result.filter((d) => d.error);
let mutations = "";
result
.filter((d) => !d.error)
.forEach((d, idx) => {
//Create mutation text
mutations =
mutations +
`
update_doc${idx}:update_documents_by_pk(pk_columns: { id: "${d.id}" }, _set: {key: "${d.to}", jobid: "${tojobid}"}){
id
}
`;
});
const client = req.userGraphQLClient;
if (mutations !== "") {
const mutationResult = await client.request(`mutation {
${mutations}
}`);
res.json({ errors, mutationResult });
} else {
res.json({ errors: "No images were succesfully moved on remote server. " });
}
} catch (error) {
logger.log("imgproxy-move-files-error", "ERROR", req.user.email, null, {
documents,
tojobid,
message: error.message,
stack: error.stack
});
res.status(400).json({ message: error.message, stack: error.stack });
}
};
function base64UrlEncode(str) {
return Buffer.from(str).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");

15
server/routes/mediaRoutes.js
View File

@@ -1,7 +1,13 @@
const express = require("express");
const router = express.Router();
const { createSignedUploadURL, downloadFiles, renameKeys, deleteFiles } = require("../media/media");
const { generateSignedUploadUrls, getThumbnailUrls } = require("../media/imgprox-media");
const {
generateSignedUploadUrls,
getThumbnailUrls,
downloadFiles: downloadFilesImgproxy,
moveFiles,
deleteFiles: deleteFilesImgproxy
} = require("../media/imgprox-media");
const validateFirebaseIdTokenMiddleware = require("../middleware/validateFirebaseIdTokenMiddleware");
const withUserGraphQLClientMiddleware = require("../middleware/withUserGraphQLClientMiddleware");
@@ -13,7 +19,10 @@ router.post("/download", downloadFiles);
router.post("/rename", renameKeys);
router.post("/delete", deleteFiles);
router.post("/proxy/sign", generateSignedUploadUrls);
router.post("/proxy/thumbnails", getThumbnailUrls);
router.post("/imgproxy/sign", generateSignedUploadUrls);
router.post("/imgproxy/thumbnails", getThumbnailUrls);
router.post("/imgproxy/download", downloadFilesImgproxy);
router.post("/imgproxy/rename", moveFiles);
router.post("/imgproxy/delete", deleteFilesImgproxy);
module.exports = router;