From 3d0791f63349947bfcd76f4a02b7ba41ad3f6714 Mon Sep 17 00:00:00 2001 From: Patrick Fic <> Date: Wed, 20 Jul 2022 14:17:03 -0700 Subject: [PATCH] IO-1998 LMS Zip Download. --- jobs/jobsDownloadMedia.ts | 77 +++++++++++++++++++++++++++++++++++++++ jobs/jobsListMedia.ts | 23 ++++++++---- package.json | 1 + server.ts | 7 ++++ util/listableChecker.ts | 2 +- util/serverInit.ts | 3 ++ yarn.lock | 34 ++++++++++++++++- 7 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 jobs/jobsDownloadMedia.ts diff --git a/jobs/jobsDownloadMedia.ts b/jobs/jobsDownloadMedia.ts new file mode 100644 index 0000000..42973ad --- /dev/null +++ b/jobs/jobsDownloadMedia.ts @@ -0,0 +1,77 @@ +import { Request, Response } from "express"; +import fs from "fs-extra"; +import multer from "multer"; +import path from "path"; +import { logger } from "../server"; +import GenerateThumbnail from "../util/generateThumbnail"; +import generateUniqueFilename from "../util/generateUniqueFilename"; +import { ConvertHeicFiles } from "../util/heicConverter"; +import { PathToRoFolder } from "../util/pathGenerators"; +import { JobsListMedia } from "./jobsListMedia"; +import JSZip from "jszip"; +import ListableChecker from "../util/listableChecker"; +import { JobRelativeFilePath } from "../util/serverInit"; + +//param: files: string[] | array of filenames. +export async function jobsDownloadMedia(req: Request, res: Response) { + const jobid: string = (req.body.jobid || "").trim(); + + try { + //Do we need all files or just some files? + const files: string[] = req.body.files || []; + const zip: JSZip = new JSZip(); + await fs.ensureDir(PathToRoFolder(jobid)); + + logger.debug(`Generating batch download for Job ID ${jobid}`, files); + //Prepare the zip file. + + const filesList: fs.Dirent[] = ( + await fs.readdir(PathToRoFolder(jobid), { + withFileTypes: true, + }) + ).filter((f) => f.isFile() && ListableChecker(f)); + + if (files.length === 0) { + //Get everything. + + await Promise.all( + filesList.map(async (file) => { + //Do something async + const fileOnDisk: Buffer = await fs.readFile( + JobRelativeFilePath(jobid, file.name) + ); + zip.file(path.parse(path.basename(file.name)).base, fileOnDisk); + }) + ); + } else { + //Get the files that are in the list and see which are requested. + await Promise.all( + filesList.map(async (file) => { + if (files.includes(path.parse(path.basename(file.name)).base)) { + // File is in the set of requested files. + const fileOnDisk: Buffer = await fs.readFile( + JobRelativeFilePath(jobid, file.name) + ); + zip.file(path.parse(path.basename(file.name)).base, fileOnDisk); + } + }) + ); + } + //Send it as a response to download it automatically. + // res.setHeader("Content-disposition", "attachment; filename=" + filename); + + zip + .generateNodeStream({ + type: "nodebuffer", + streamFiles: true, + //encodeFileName: (filename) => `${jobid}.zip`, + }) + .pipe(res); + } catch (error) { + logger.error("Error downloading job media.", { + jobid, + error: (error as Error).message, + }); + res.status(500).json((error as Error).message); + } +} diff --git a/jobs/jobsListMedia.ts b/jobs/jobsListMedia.ts index 19ec39f..e3a5544 100644 --- a/jobs/jobsListMedia.ts +++ b/jobs/jobsListMedia.ts @@ -7,7 +7,7 @@ import MediaFile from "../util/interfaces/MediaFile"; import ListableChecker from "../util/listableChecker"; import GenerateUrl from "../util/MediaUrlGen"; import { PathToRoFolder } from "../util/pathGenerators"; -import { FolderPaths } from "../util/serverInit"; +import { FolderPaths, JobRelativeFilePath } from "../util/serverInit"; export async function JobsListMedia(req: Request, res: Response) { const jobid: string = (req.body.jobid || "").trim(); @@ -19,8 +19,13 @@ export async function JobsListMedia(req: Request, res: Response) { //We just uploaded files, we're going to send only those back. ret = await Promise.all( (req.files as Express.Multer.File[]).map(async (file) => { + const relativeFilePath: string = JobRelativeFilePath( + jobid, + file.filename + ); + const relativeThumbPath: string = await GenerateThumbnail( - path.join(FolderPaths.Jobs, jobid, file.filename) + relativeFilePath ); return { src: GenerateUrl([ @@ -38,6 +43,7 @@ export async function JobsListMedia(req: Request, res: Response) { thumbnailHeight: 250, thumbnailWidth: 250, filename: file.filename, + relativeFilePath, }; }) ); @@ -46,15 +52,17 @@ export async function JobsListMedia(req: Request, res: Response) { await fs.readdir(PathToRoFolder(jobid), { withFileTypes: true, }) - ).filter( - (f) => - f.isFile() && !/(^|\/)\.[^\/\.]/g.test(f.name) && ListableChecker(f) - ); + ).filter((f) => f.isFile() && ListableChecker(f)); ret = await Promise.all( filesList.map(async (file) => { + const relativeFilePath: string = JobRelativeFilePath( + jobid, + file.name + ); + const relativeThumbPath: string = await GenerateThumbnail( - path.join(FolderPaths.Jobs, jobid, file.name) + relativeFilePath ); return { src: GenerateUrl([ @@ -72,6 +80,7 @@ export async function JobsListMedia(req: Request, res: Response) { thumbnailHeight: 250, thumbnailWidth: 250, filename: file.name, + relativeFilePath, }; }) ); diff --git a/package.json b/package.json index 1a283d7..849c855 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "heic-convert": "^1.2.4", "helmet": "^5.0.2", "image-thumbnail": "^1.0.14", + "jszip": "^3.10.0", "morgan": "^1.10.0", "multer": "^1.4.4", "response-time": "^2.3.2", diff --git a/server.ts b/server.ts index 0fc94ab..e6d6017 100644 --- a/server.ts +++ b/server.ts @@ -21,6 +21,7 @@ import { BillsUploadMedia, } from "./bills/billsUploadMedia"; import ValidateImsToken from "./util/validateToken"; +import { jobsDownloadMedia } from "./jobs/jobsDownloadMedia"; dotenv.config({ path: resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`), @@ -129,6 +130,12 @@ app.post( JobRequestValidator, jobsUploadMedia ); +app.post( + "/jobs/download", + ValidateImsToken, + JobRequestValidator, + jobsDownloadMedia +); app.post( "/jobs/move", //JobRequestValidator, ValidateImsToken, diff --git a/util/listableChecker.ts b/util/listableChecker.ts index 0bc7462..7c73d63 100644 --- a/util/listableChecker.ts +++ b/util/listableChecker.ts @@ -2,7 +2,7 @@ import fs from "fs-extra"; function ListableChecker(file: fs.Dirent) { if (file.name === "Thumbs.db") return false; - return true; + if (!/(^|\/)\.[^\/\.]/g.test(file.name)) return true; } export default ListableChecker; diff --git a/util/serverInit.ts b/util/serverInit.ts index 609b16d..3895c16 100644 --- a/util/serverInit.ts +++ b/util/serverInit.ts @@ -24,6 +24,9 @@ export const FolderPaths = { VendorsFolder, }; +export function JobRelativeFilePath(jobid: string, filename: string) { + return path.join(FolderPaths.Jobs, jobid, filename); +} export default function InitServer() { logger.info(`Ensuring Root media path exists: ${FolderPaths.Root}`); fs.ensureDirSync(FolderPaths.Root); diff --git a/yarn.lock b/yarn.lock index 3acc5d3..78bae84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1132,6 +1132,11 @@ image-thumbnail@^1.0.14: sharp "^0.28" validator "^13.0.0" +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== + import-lazy@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz" @@ -1282,6 +1287,16 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" +jszip@^3.10.0: + version "3.10.0" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.0.tgz#faf3db2b4b8515425e34effcdbb086750a346061" + integrity sha512-LDfVtOLtOxb9RXkYOwPyNBTQDL4eUbqahtoY6x07GiDJHwSYvn8sHHIw8wINImV3MqbMNve2gSuM1DDqEKk09Q== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + setimmediate "^1.0.5" + keyv@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz" @@ -1306,6 +1321,13 @@ libheif-js@^1.10.0: resolved "https://registry.yarnpkg.com/libheif-js/-/libheif-js-1.12.0.tgz#9ad1ed16a8e6412b4d3d83565d285465a00e7305" integrity sha512-hDs6xQ7028VOwAFwEtM0Q+B2x2NW69Jb2MhQFUbk3rUrHzz4qo5mqS8VrqNgYnSc8TiUGnR691LnO4uIfEE23w== +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + logform@^2.3.2, logform@^2.4.0: version "2.4.0" resolved "https://registry.npmjs.org/logform/-/logform-2.4.0.tgz" @@ -1598,6 +1620,11 @@ package-json@^6.3.0: registry-url "^5.0.0" semver "^6.2.0" +pako@~1.0.2: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parse-cache-control@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz" @@ -1744,7 +1771,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.6, readable-stream@^2.2.2: +readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -1887,6 +1914,11 @@ set-blocking@~2.0.0: resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +setimmediate@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" + integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== + setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz"