349 lines
12 KiB
JavaScript
349 lines
12 KiB
JavaScript
const path = require("path");
|
|
require("dotenv").config({
|
|
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
|
});
|
|
const logger = require("../utils/logger");
|
|
const {
|
|
S3Client,
|
|
PutObjectCommand,
|
|
GetObjectCommand,
|
|
CopyObjectCommand,
|
|
DeleteObjectCommand
|
|
} = require("@aws-sdk/client-s3");
|
|
const { Upload } = require("@aws-sdk/lib-storage");
|
|
|
|
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
|
|
const crypto = require("crypto");
|
|
const { InstanceRegion } = require("../utils/instanceMgr");
|
|
const {
|
|
GET_DOCUMENTS_BY_JOB,
|
|
QUERY_TEMPORARY_DOCS,
|
|
GET_DOCUMENTS_BY_IDS,
|
|
DELETE_MEDIA_DOCUMENTS
|
|
} = require("../graphql-client/queries");
|
|
const archiver = require("archiver");
|
|
const stream = require("node:stream");
|
|
|
|
const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN.
|
|
const imgproxyKey = process.env.IMGPROXY_KEY;
|
|
const imgproxySalt = process.env.IMGPROXY_SALT;
|
|
const imgproxyDestinationBucket = process.env.IMGPROXY_DESTINATION_BUCKET;
|
|
|
|
//Generate a signed upload link for the S3 bucket.
|
|
//All uploads must be going to the same shop and jobid.
|
|
exports.generateSignedUploadUrls = async (req, res) => {
|
|
const { filenames, bodyshopid, jobid } = req.body;
|
|
try {
|
|
logger.log("imgproxy-upload-start", "DEBUG", req.user?.email, jobid, { filenames, bodyshopid, jobid });
|
|
|
|
const signedUrls = [];
|
|
for (const filename of filenames) {
|
|
const key = filename;
|
|
const client = new S3Client({ region: InstanceRegion() });
|
|
const command = new PutObjectCommand({
|
|
Bucket: imgproxyDestinationBucket,
|
|
Key: key,
|
|
StorageClass: "INTELLIGENT_TIERING"
|
|
});
|
|
const presignedUrl = await getSignedUrl(client, command, { expiresIn: 360 });
|
|
signedUrls.push({ filename, presignedUrl, key });
|
|
}
|
|
|
|
logger.log("imgproxy-upload-success", "DEBUG", req.user?.email, jobid, { signedUrls });
|
|
res.json({
|
|
success: true,
|
|
signedUrls
|
|
});
|
|
} catch (error) {
|
|
res.status(400).json({
|
|
success: false,
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
logger.log("imgproxy-upload-error", "ERROR", req.user?.email, jobid, {
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
}
|
|
};
|
|
|
|
exports.getThumbnailUrls = async (req, res) => {
|
|
const { jobid, billid } = req.body;
|
|
|
|
try {
|
|
logger.log("imgproxy-thumbnails", "DEBUG", req.user?.email, jobid, { billid, jobid });
|
|
|
|
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
|
|
const client = req.userGraphQLClient;
|
|
//If there's no jobid and no billid, we're in temporary documents.
|
|
const data = await (jobid
|
|
? client.request(GET_DOCUMENTS_BY_JOB, { jobId: jobid })
|
|
: client.request(QUERY_TEMPORARY_DOCS));
|
|
|
|
const thumbResizeParams = `rs:fill:250:250:1/g:ce`;
|
|
const s3client = new S3Client({ region: InstanceRegion() });
|
|
const proxiedUrls = [];
|
|
|
|
for (const document of data.documents) {
|
|
//Format to follow:
|
|
//<Cloudfront_to_lambda>/<hmac with SHA of entire request URI path (with base64 encoded URL if needed), beginning with unencoded/unhashed Salt>/<remainder of url - resize params >/< base 64 URL encoded to image path>
|
|
|
|
//When working with documents from Cloudinary, the URL does not include the extension.
|
|
let key;
|
|
if (/\.[^/.]+$/.test(document.key)) {
|
|
key = document.key;
|
|
} else {
|
|
key = `${document.key}.${document.extension.toLowerCase()}`;
|
|
}
|
|
// Build the S3 path to the object.
|
|
const fullS3Path = `s3://${imgproxyDestinationBucket}/${key}`;
|
|
const base64UrlEncodedKeyString = base64UrlEncode(fullS3Path);
|
|
//Thumbnail Generation Block
|
|
const thumbProxyPath = `${thumbResizeParams}/${base64UrlEncodedKeyString}`;
|
|
const thumbHmacSalt = createHmacSha256(`${imgproxySalt}/${thumbProxyPath}`);
|
|
|
|
//Full Size URL block
|
|
|
|
const fullSizeProxyPath = `${base64UrlEncodedKeyString}`;
|
|
const fullSizeHmacSalt = createHmacSha256(`${imgproxySalt}/${fullSizeProxyPath}`);
|
|
|
|
const s3Props = {};
|
|
if (!document.type.startsWith("image")) {
|
|
//If not a picture, we need to get a signed download link to the file using S3 (or cloudfront preferably)
|
|
const command = new GetObjectCommand({
|
|
Bucket: imgproxyDestinationBucket,
|
|
Key: key
|
|
});
|
|
const presignedGetUrl = await getSignedUrl(s3client, command, { expiresIn: 360 });
|
|
s3Props.presignedGetUrl = presignedGetUrl;
|
|
|
|
const originalProxyPath = `raw:1/${base64UrlEncodedKeyString}`;
|
|
const originalHmacSalt = createHmacSha256(`${imgproxySalt}/${originalProxyPath}`);
|
|
s3Props.originalUrlViaProxyPath = `${imgproxyBaseUrl}/${originalHmacSalt}/${originalProxyPath}`;
|
|
}
|
|
|
|
proxiedUrls.push({
|
|
originalUrl: `${imgproxyBaseUrl}/${fullSizeHmacSalt}/${fullSizeProxyPath}`,
|
|
thumbnailUrl: `${imgproxyBaseUrl}/${thumbHmacSalt}/${thumbProxyPath}`,
|
|
fullS3Path,
|
|
base64UrlEncodedKeyString,
|
|
thumbProxyPath,
|
|
...s3Props,
|
|
...document
|
|
});
|
|
}
|
|
|
|
res.json(proxiedUrls);
|
|
//Iterate over them, build the link based on the media type, and return the array.
|
|
} catch (error) {
|
|
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
|
|
jobid,
|
|
billid,
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
res.status(400).json({ message: error.message, stack: error.stack });
|
|
}
|
|
};
|
|
|
|
exports.getBillFiles = async (req, res) => {
|
|
//Givena bill ID, get the documents associated to it.
|
|
};
|
|
|
|
exports.downloadFiles = async (req, res) => {
|
|
//Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk
|
|
const { jobid, billid, documentids } = req.body;
|
|
try {
|
|
logger.log("imgproxy-download", "DEBUG", req.user?.email, jobid, { billid, jobid, documentids });
|
|
|
|
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
|
|
const client = req.userGraphQLClient;
|
|
//Query for the keys of the document IDs
|
|
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
|
|
//Using the Keys, get all of the S3 links, zip them, and send back to the client.
|
|
const s3client = new S3Client({ region: InstanceRegion() });
|
|
const archiveStream = archiver("zip");
|
|
archiveStream.on("error", (error) => {
|
|
console.error("Archival encountered an error:", error);
|
|
throw new Error(error);
|
|
});
|
|
const passthrough = new stream.PassThrough();
|
|
|
|
archiveStream.pipe(passthrough);
|
|
for (const key of data.documents.map((d) => d.key)) {
|
|
const response = await s3client.send(new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key }));
|
|
// :: `response.Body` is a Buffer
|
|
console.log(path.basename(key));
|
|
archiveStream.append(response.Body, { name: path.basename(key) });
|
|
}
|
|
|
|
archiveStream.finalize();
|
|
|
|
const archiveKey = `archives/${jobid}/archive-${new Date().toISOString()}.zip`;
|
|
|
|
const parallelUploads3 = new Upload({
|
|
client: s3client,
|
|
queueSize: 4, // optional concurrency configuration
|
|
leavePartsOnError: false, // optional manually handle dropped parts
|
|
params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passthrough }
|
|
});
|
|
|
|
parallelUploads3.on("httpUploadProgress", (progress) => {
|
|
console.log(progress);
|
|
});
|
|
|
|
const uploadResult = await parallelUploads3.done();
|
|
//Generate the presigned URL to download it.
|
|
const presignedUrl = await getSignedUrl(
|
|
s3client,
|
|
new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }),
|
|
{ expiresIn: 360 }
|
|
);
|
|
|
|
res.json({ success: true, url: presignedUrl });
|
|
//Iterate over them, build the link based on the media type, and return the array.
|
|
} catch (error) {
|
|
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
|
|
jobid,
|
|
billid,
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
res.status(400).json({ message: error.message, stack: error.stack });
|
|
}
|
|
};
|
|
|
|
exports.deleteFiles = async (req, res) => {
|
|
//Mark a file for deletion in s3. Lifecycle deletion will actually delete the copy in the future.
|
|
//Mark as deleted from the documents section of the database.
|
|
const { ids } = req.body;
|
|
try {
|
|
logger.log("imgproxy-delete-files", "DEBUG", req.user.email, null, { ids });
|
|
const client = req.userGraphQLClient;
|
|
|
|
//Do this to make sure that they are only deleting things that they have access to
|
|
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: ids });
|
|
|
|
const s3client = new S3Client({ region: InstanceRegion() });
|
|
|
|
const deleteTransactions = [];
|
|
data.documents.forEach((document) => {
|
|
deleteTransactions.push(
|
|
(async () => {
|
|
try {
|
|
// Delete the original object
|
|
const deleteResult = await s3client.send(
|
|
new DeleteObjectCommand({
|
|
Bucket: imgproxyDestinationBucket,
|
|
Key: document.key
|
|
})
|
|
);
|
|
|
|
return document;
|
|
} catch (error) {
|
|
return { document, error: error, bucket: imgproxyDestinationBucket };
|
|
}
|
|
})()
|
|
);
|
|
});
|
|
|
|
const result = await Promise.all(deleteTransactions);
|
|
const errors = result.filter((d) => d.error);
|
|
|
|
//Delete only the succesful deletes.
|
|
const deleteMutationResult = await client.request(DELETE_MEDIA_DOCUMENTS, {
|
|
ids: result.filter((t) => !t.error).map((d) => d.id)
|
|
});
|
|
|
|
res.json({ errors, deleteMutationResult });
|
|
} catch (error) {
|
|
logger.log("imgproxy-delete-files-error", "ERROR", req.user.email, null, {
|
|
ids,
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
res.status(400).json({ message: error.message, stack: error.stack });
|
|
}
|
|
};
|
|
|
|
exports.moveFiles = async (req, res) => {
|
|
const { documents, tojobid } = req.body;
|
|
try {
|
|
logger.log("imgproxy-move-files", "DEBUG", req.user.email, null, { documents, tojobid });
|
|
const s3client = new S3Client({ region: InstanceRegion() });
|
|
|
|
const moveTransactions = [];
|
|
documents.forEach((document) => {
|
|
moveTransactions.push(
|
|
(async () => {
|
|
try {
|
|
// Copy the object to the new key
|
|
const copyresult = await s3client.send(
|
|
new CopyObjectCommand({
|
|
Bucket: imgproxyDestinationBucket,
|
|
CopySource: `${imgproxyDestinationBucket}/${document.from}`,
|
|
Key: document.to,
|
|
StorageClass: "INTELLIGENT_TIERING"
|
|
})
|
|
);
|
|
|
|
// Delete the original object
|
|
const deleteResult = await s3client.send(
|
|
new DeleteObjectCommand({
|
|
Bucket: imgproxyDestinationBucket,
|
|
Key: document.from
|
|
})
|
|
);
|
|
|
|
return document;
|
|
} catch (error) {
|
|
return { id: document.id, from: document.from, error: error, bucket: imgproxyDestinationBucket };
|
|
}
|
|
})()
|
|
);
|
|
});
|
|
|
|
const result = await Promise.all(moveTransactions);
|
|
const errors = result.filter((d) => d.error);
|
|
|
|
let mutations = "";
|
|
result
|
|
.filter((d) => !d.error)
|
|
.forEach((d, idx) => {
|
|
//Create mutation text
|
|
mutations =
|
|
mutations +
|
|
`
|
|
update_doc${idx}:update_documents_by_pk(pk_columns: { id: "${d.id}" }, _set: {key: "${d.to}", jobid: "${tojobid}"}){
|
|
id
|
|
}
|
|
`;
|
|
});
|
|
|
|
const client = req.userGraphQLClient;
|
|
if (mutations !== "") {
|
|
const mutationResult = await client.request(`mutation {
|
|
${mutations}
|
|
}`);
|
|
res.json({ errors, mutationResult });
|
|
} else {
|
|
res.json({ errors: "No images were succesfully moved on remote server. " });
|
|
}
|
|
} catch (error) {
|
|
logger.log("imgproxy-move-files-error", "ERROR", req.user.email, null, {
|
|
documents,
|
|
tojobid,
|
|
message: error.message,
|
|
stack: error.stack
|
|
});
|
|
res.status(400).json({ message: error.message, stack: error.stack });
|
|
}
|
|
};
|
|
|
|
function base64UrlEncode(str) {
|
|
return Buffer.from(str).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
}
|
|
function createHmacSha256(data) {
|
|
return crypto.createHmac("sha256", imgproxyKey).update(data).digest("base64url");
|
|
}
|