IO-3092 Additional comments, email compatibility.
This commit is contained in:
Patrick Fic
348
server/media/imgproxy-media.js
Normal file
348
server/media/imgproxy-media.js
Normal file
@@ -0,0 +1,348 @@
|
||||
const path = require("path");
|
||||
require("dotenv").config({
|
||||
path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`)
|
||||
});
|
||||
const logger = require("../utils/logger");
|
||||
const {
|
||||
S3Client,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand
|
||||
} = require("@aws-sdk/client-s3");
|
||||
const { Upload } = require("@aws-sdk/lib-storage");
|
||||
|
||||
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
|
||||
const crypto = require("crypto");
|
||||
const { InstanceRegion } = require("../utils/instanceMgr");
|
||||
const {
|
||||
GET_DOCUMENTS_BY_JOB,
|
||||
QUERY_TEMPORARY_DOCS,
|
||||
GET_DOCUMENTS_BY_IDS,
|
||||
DELETE_MEDIA_DOCUMENTS
|
||||
} = require("../graphql-client/queries");
|
||||
const archiver = require("archiver");
|
||||
const stream = require("node:stream");
|
||||
|
||||
const imgproxyBaseUrl = process.env.IMGPROXY_BASE_URL; // `https://u4gzpp5wm437dnm75qa42tvza40fguqr.lambda-url.ca-central-1.on.aws` //Direct Lambda function access to bypass CDN.
|
||||
const imgproxyKey = process.env.IMGPROXY_KEY;
|
||||
const imgproxySalt = process.env.IMGPROXY_SALT;
|
||||
const imgproxyDestinationBucket = process.env.IMGPROXY_DESTINATION_BUCKET;
|
||||
|
||||
//Generate a signed upload link for the S3 bucket.
|
||||
//All uploads must be going to the same shop and jobid.
|
||||
exports.generateSignedUploadUrls = async (req, res) => {
|
||||
const { filenames, bodyshopid, jobid } = req.body;
|
||||
try {
|
||||
logger.log("imgproxy-upload-start", "DEBUG", req.user?.email, jobid, { filenames, bodyshopid, jobid });
|
||||
|
||||
const signedUrls = [];
|
||||
for (const filename of filenames) {
|
||||
const key = filename;
|
||||
const client = new S3Client({ region: InstanceRegion() });
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: imgproxyDestinationBucket,
|
||||
Key: key,
|
||||
StorageClass: "INTELLIGENT_TIERING"
|
||||
});
|
||||
const presignedUrl = await getSignedUrl(client, command, { expiresIn: 360 });
|
||||
signedUrls.push({ filename, presignedUrl, key });
|
||||
}
|
||||
|
||||
logger.log("imgproxy-upload-success", "DEBUG", req.user?.email, jobid, { signedUrls });
|
||||
res.json({
|
||||
success: true,
|
||||
signedUrls
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(400).json({
|
||||
success: false,
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
logger.log("imgproxy-upload-error", "ERROR", req.user?.email, jobid, {
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
exports.getThumbnailUrls = async (req, res) => {
|
||||
const { jobid, billid } = req.body;
|
||||
|
||||
try {
|
||||
logger.log("imgproxy-thumbnails", "DEBUG", req.user?.email, jobid, { billid, jobid });
|
||||
|
||||
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
|
||||
const client = req.userGraphQLClient;
|
||||
//If there's no jobid and no billid, we're in temporary documents.
|
||||
const data = await (jobid
|
||||
? client.request(GET_DOCUMENTS_BY_JOB, { jobId: jobid })
|
||||
: client.request(QUERY_TEMPORARY_DOCS));
|
||||
|
||||
const thumbResizeParams = `rs:fill:250:250:1/g:ce`;
|
||||
const s3client = new S3Client({ region: InstanceRegion() });
|
||||
const proxiedUrls = [];
|
||||
|
||||
for (const document of data.documents) {
|
||||
//Format to follow:
|
||||
//<Cloudfront_to_lambda>/<hmac with SHA of entire request URI path (with base64 encoded URL if needed), beginning with unencoded/unhashed Salt>/<remainder of url - resize params >/< base 64 URL encoded to image path>
|
||||
|
||||
//When working with documents from Cloudinary, the URL does not include the extension.
|
||||
let key;
|
||||
if (/\.[^/.]+$/.test(document.key)) {
|
||||
key = document.key;
|
||||
} else {
|
||||
key = `${document.key}.${document.extension.toLowerCase()}`;
|
||||
}
|
||||
// Build the S3 path to the object.
|
||||
const fullS3Path = `s3://${imgproxyDestinationBucket}/${key}`;
|
||||
const base64UrlEncodedKeyString = base64UrlEncode(fullS3Path);
|
||||
//Thumbnail Generation Block
|
||||
const thumbProxyPath = `${thumbResizeParams}/${base64UrlEncodedKeyString}`;
|
||||
const thumbHmacSalt = createHmacSha256(`${imgproxySalt}/${thumbProxyPath}`);
|
||||
|
||||
//Full Size URL block
|
||||
|
||||
const fullSizeProxyPath = `${base64UrlEncodedKeyString}`;
|
||||
const fullSizeHmacSalt = createHmacSha256(`${imgproxySalt}/${fullSizeProxyPath}`);
|
||||
|
||||
const s3Props = {};
|
||||
if (!document.type.startsWith("image")) {
|
||||
//If not a picture, we need to get a signed download link to the file using S3 (or cloudfront preferably)
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: imgproxyDestinationBucket,
|
||||
Key: key
|
||||
});
|
||||
const presignedGetUrl = await getSignedUrl(s3client, command, { expiresIn: 360 });
|
||||
s3Props.presignedGetUrl = presignedGetUrl;
|
||||
|
||||
const originalProxyPath = `raw:1/${base64UrlEncodedKeyString}`;
|
||||
const originalHmacSalt = createHmacSha256(`${imgproxySalt}/${originalProxyPath}`);
|
||||
s3Props.originalUrlViaProxyPath = `${imgproxyBaseUrl}/${originalHmacSalt}/${originalProxyPath}`;
|
||||
}
|
||||
|
||||
proxiedUrls.push({
|
||||
originalUrl: `${imgproxyBaseUrl}/${fullSizeHmacSalt}/${fullSizeProxyPath}`,
|
||||
thumbnailUrl: `${imgproxyBaseUrl}/${thumbHmacSalt}/${thumbProxyPath}`,
|
||||
fullS3Path,
|
||||
base64UrlEncodedKeyString,
|
||||
thumbProxyPath,
|
||||
...s3Props,
|
||||
...document
|
||||
});
|
||||
}
|
||||
|
||||
res.json(proxiedUrls);
|
||||
//Iterate over them, build the link based on the media type, and return the array.
|
||||
} catch (error) {
|
||||
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
|
||||
jobid,
|
||||
billid,
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
res.status(400).json({ message: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
exports.getBillFiles = async (req, res) => {
|
||||
//Givena bill ID, get the documents associated to it.
|
||||
};
|
||||
|
||||
exports.downloadFiles = async (req, res) => {
|
||||
//Given a series of document IDs or keys, generate a file (or a link) to download all images in bulk
|
||||
const { jobid, billid, documentids } = req.body;
|
||||
try {
|
||||
logger.log("imgproxy-download", "DEBUG", req.user?.email, jobid, { billid, jobid, documentids });
|
||||
|
||||
//Delayed as the key structure may change slightly from what it is currently and will require evaluating mobile components.
|
||||
const client = req.userGraphQLClient;
|
||||
//Query for the keys of the document IDs
|
||||
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: documentids });
|
||||
//Using the Keys, get all of the S3 links, zip them, and send back to the client.
|
||||
const s3client = new S3Client({ region: InstanceRegion() });
|
||||
const archiveStream = archiver("zip");
|
||||
archiveStream.on("error", (error) => {
|
||||
console.error("Archival encountered an error:", error);
|
||||
throw new Error(error);
|
||||
});
|
||||
const passthrough = new stream.PassThrough();
|
||||
|
||||
archiveStream.pipe(passthrough);
|
||||
for (const key of data.documents.map((d) => d.key)) {
|
||||
const response = await s3client.send(new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: key }));
|
||||
// :: `response.Body` is a Buffer
|
||||
console.log(path.basename(key));
|
||||
archiveStream.append(response.Body, { name: path.basename(key) });
|
||||
}
|
||||
|
||||
archiveStream.finalize();
|
||||
|
||||
const archiveKey = `archives/${jobid}/archive-${new Date().toISOString()}.zip`;
|
||||
|
||||
const parallelUploads3 = new Upload({
|
||||
client: s3client,
|
||||
queueSize: 4, // optional concurrency configuration
|
||||
leavePartsOnError: false, // optional manually handle dropped parts
|
||||
params: { Bucket: imgproxyDestinationBucket, Key: archiveKey, Body: passthrough }
|
||||
});
|
||||
|
||||
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||
console.log(progress);
|
||||
});
|
||||
|
||||
const uploadResult = await parallelUploads3.done();
|
||||
//Generate the presigned URL to download it.
|
||||
const presignedUrl = await getSignedUrl(
|
||||
s3client,
|
||||
new GetObjectCommand({ Bucket: imgproxyDestinationBucket, Key: archiveKey }),
|
||||
{ expiresIn: 360 }
|
||||
);
|
||||
|
||||
res.json({ success: true, url: presignedUrl });
|
||||
//Iterate over them, build the link based on the media type, and return the array.
|
||||
} catch (error) {
|
||||
logger.log("imgproxy-thumbnails-error", "ERROR", req.user?.email, jobid, {
|
||||
jobid,
|
||||
billid,
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
res.status(400).json({ message: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
exports.deleteFiles = async (req, res) => {
|
||||
//Mark a file for deletion in s3. Lifecycle deletion will actually delete the copy in the future.
|
||||
//Mark as deleted from the documents section of the database.
|
||||
const { ids } = req.body;
|
||||
try {
|
||||
logger.log("imgproxy-delete-files", "DEBUG", req.user.email, null, { ids });
|
||||
const client = req.userGraphQLClient;
|
||||
|
||||
//Do this to make sure that they are only deleting things that they have access to
|
||||
const data = await client.request(GET_DOCUMENTS_BY_IDS, { documentIds: ids });
|
||||
|
||||
const s3client = new S3Client({ region: InstanceRegion() });
|
||||
|
||||
const deleteTransactions = [];
|
||||
data.documents.forEach((document) => {
|
||||
deleteTransactions.push(
|
||||
(async () => {
|
||||
try {
|
||||
// Delete the original object
|
||||
const deleteResult = await s3client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: imgproxyDestinationBucket,
|
||||
Key: document.key
|
||||
})
|
||||
);
|
||||
|
||||
return document;
|
||||
} catch (error) {
|
||||
return { document, error: error, bucket: imgproxyDestinationBucket };
|
||||
}
|
||||
})()
|
||||
);
|
||||
});
|
||||
|
||||
const result = await Promise.all(deleteTransactions);
|
||||
const errors = result.filter((d) => d.error);
|
||||
|
||||
//Delete only the succesful deletes.
|
||||
const deleteMutationResult = await client.request(DELETE_MEDIA_DOCUMENTS, {
|
||||
ids: result.filter((t) => !t.error).map((d) => d.id)
|
||||
});
|
||||
|
||||
res.json({ errors, deleteMutationResult });
|
||||
} catch (error) {
|
||||
logger.log("imgproxy-delete-files-error", "ERROR", req.user.email, null, {
|
||||
ids,
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
res.status(400).json({ message: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
exports.moveFiles = async (req, res) => {
|
||||
const { documents, tojobid } = req.body;
|
||||
try {
|
||||
logger.log("imgproxy-move-files", "DEBUG", req.user.email, null, { documents, tojobid });
|
||||
const s3client = new S3Client({ region: InstanceRegion() });
|
||||
|
||||
const moveTransactions = [];
|
||||
documents.forEach((document) => {
|
||||
moveTransactions.push(
|
||||
(async () => {
|
||||
try {
|
||||
// Copy the object to the new key
|
||||
const copyresult = await s3client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: imgproxyDestinationBucket,
|
||||
CopySource: `${imgproxyDestinationBucket}/${document.from}`,
|
||||
Key: document.to,
|
||||
StorageClass: "INTELLIGENT_TIERING"
|
||||
})
|
||||
);
|
||||
|
||||
// Delete the original object
|
||||
const deleteResult = await s3client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: imgproxyDestinationBucket,
|
||||
Key: document.from
|
||||
})
|
||||
);
|
||||
|
||||
return document;
|
||||
} catch (error) {
|
||||
return { id: document.id, from: document.from, error: error, bucket: imgproxyDestinationBucket };
|
||||
}
|
||||
})()
|
||||
);
|
||||
});
|
||||
|
||||
const result = await Promise.all(moveTransactions);
|
||||
const errors = result.filter((d) => d.error);
|
||||
|
||||
let mutations = "";
|
||||
result
|
||||
.filter((d) => !d.error)
|
||||
.forEach((d, idx) => {
|
||||
//Create mutation text
|
||||
mutations =
|
||||
mutations +
|
||||
`
|
||||
update_doc${idx}:update_documents_by_pk(pk_columns: { id: "${d.id}" }, _set: {key: "${d.to}", jobid: "${tojobid}"}){
|
||||
id
|
||||
}
|
||||
`;
|
||||
});
|
||||
|
||||
const client = req.userGraphQLClient;
|
||||
if (mutations !== "") {
|
||||
const mutationResult = await client.request(`mutation {
|
||||
${mutations}
|
||||
}`);
|
||||
res.json({ errors, mutationResult });
|
||||
} else {
|
||||
res.json({ errors: "No images were succesfully moved on remote server. " });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.log("imgproxy-move-files-error", "ERROR", req.user.email, null, {
|
||||
documents,
|
||||
tojobid,
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
res.status(400).json({ message: error.message, stack: error.stack });
|
||||
}
|
||||
};
|
||||
|
||||
function base64UrlEncode(str) {
|
||||
return Buffer.from(str).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
}
|
||||
function createHmacSha256(data) {
|
||||
return crypto.createHmac("sha256", imgproxyKey).update(data).digest("base64url");
|
||||
}
|
||||
Reference in New Issue
Block a user