diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..2961469 --- /dev/null +++ b/nodemon.json @@ -0,0 +1,12 @@ +{ + "watch": [ + "src" + ], + "ext": "ts", + "execMap": { + "ts": "node --loader ts-node/esm -r tsconfig-paths/register" + }, + "ignore": [ + "src/**/*.spec.ts" + ] + } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index daef5fa..c510a51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "axios": "^1.7.5", "body-parser": "^1.20.2", + "bullmq": "^5.12.12", "cors": "^2.8.5", "dotenv": "16.4.5", "express": "^4.19.2", @@ -36,11 +37,12 @@ "@types/image-thumbnail": "^1.0.4", "@types/morgan": "^1.9.9", "@types/multer": "^1.4.12", - "@types/node": "^22.5.0", + "@types/node": "^22.5.1", "@types/response-time": "^2.3.8", "nodemon": "^3.1.4", "prettier": "^3.3.3", "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", "typescript": "^5.5.4" }, "engines": { @@ -429,6 +431,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -454,6 +462,84 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@sec-ant/readable-stream": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", @@ -609,10 +695,11 @@ } }, "node_modules/@types/node": { - "version": "22.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.0.tgz", - "integrity": "sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg==", + "version": "22.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.1.tgz", + "integrity": "sha512-KkHsxej0j9IW1KKOOAA/XBA0z08UFSrRQHErzEfA3Vgq57eXIMYboIlHJuYIfd+lwCQjtKqUu3UnmKbtUc9yRw==", "dev": true, + "license": "MIT", "dependencies": { "undici-types": "~6.19.2" } @@ -855,6 +942,21 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.12.12", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.12.12.tgz", + "integrity": "sha512-xrWKDj1ZwnGKmrlmFqF6Vmub3WqDFfdBcIRLCooIs5+jeVzbHK7/1usgYSFg2pZiwK6h6eMivTb9WvcKkNW/+w==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "node_modules/busboy": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", @@ -939,6 +1041,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -1124,6 +1235,18 @@ "dev": true, "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", @@ -1173,6 +1296,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1787,6 +1919,53 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1871,6 +2050,19 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "license": "ISC" }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -1927,6 +2119,18 @@ "immediate": "~3.0.5" } }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/logform": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz", @@ -1953,6 +2157,15 @@ "yallist": "^2.1.2" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -2090,6 +2303,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.0.tgz", + "integrity": "sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4.tgz", @@ -2126,11 +2370,33 @@ "node": ">=16.0.0" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/nodemon": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.4.tgz", "integrity": "sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==", "dev": true, + "license": "MIT", "dependencies": { "chokidar": "^3.5.2", "debug": "^4", @@ -2435,6 +2701,27 @@ "node": ">=8.10.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/response-time": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/response-time/-/response-time-2.3.2.tgz", @@ -2668,6 +2955,12 @@ "node": "*" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -2699,6 +2992,16 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/strtok3": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-8.1.0.tgz", @@ -2797,6 +3100,7 @@ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, + "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -2835,11 +3139,25 @@ } } }, + "node_modules/tsconfig-paths": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz", + "integrity": "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "json5": "^2.2.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tslib": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", - "optional": true + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" }, "node_modules/type-is": { "version": "1.6.18", @@ -2865,6 +3183,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -2930,6 +3249,19 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -3224,6 +3556,11 @@ "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", "optional": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -3246,6 +3583,42 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "optional": true + }, "@sec-ant/readable-stream": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", @@ -3394,9 +3767,9 @@ } }, "@types/node": { - "version": "22.5.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.0.tgz", - "integrity": "sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg==", + "version": "22.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.1.tgz", + "integrity": "sha512-KkHsxej0j9IW1KKOOAA/XBA0z08UFSrRQHErzEfA3Vgq57eXIMYboIlHJuYIfd+lwCQjtKqUu3UnmKbtUc9yRw==", "dev": true, "requires": { "undici-types": "~6.19.2" @@ -3595,6 +3968,20 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "bullmq": { + "version": "5.12.12", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.12.12.tgz", + "integrity": "sha512-xrWKDj1ZwnGKmrlmFqF6Vmub3WqDFfdBcIRLCooIs5+jeVzbHK7/1usgYSFg2pZiwK6h6eMivTb9WvcKkNW/+w==", + "requires": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "busboy": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", @@ -3655,6 +4042,11 @@ "readdirp": "~3.6.0" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -3798,6 +4190,14 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "requires": { + "luxon": "^3.2.1" + } + }, "cross-spawn": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", @@ -3837,6 +4237,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4259,6 +4664,37 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4314,6 +4750,12 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, + "json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true + }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -4368,6 +4810,16 @@ "immediate": "~3.0.5" } }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "logform": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz", @@ -4390,6 +4842,11 @@ "yallist": "^2.1.2" } }, + "luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==" + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -4483,6 +4940,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "msgpackr": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.0.tgz", + "integrity": "sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw==", + "requires": { + "msgpackr-extract": "^3.0.2" + } + }, + "msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "optional": true, + "requires": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3", + "node-gyp-build-optional-packages": "5.2.2" + } + }, "multer": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4.tgz", @@ -4508,6 +4988,20 @@ "resolved": "https://registry.npmjs.org/nocache/-/nocache-4.0.0.tgz", "integrity": "sha512-AntnTbmKZvNYIsTVPPwv7dfZdAfo/6H/2ZlZACK66NAOQtIApxkB/6pf/c+s+ACW8vemGJzUCyVTssrzNUK6yQ==" }, + "node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, + "node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "optional": true, + "requires": { + "detect-libc": "^2.0.1" + } + }, "nodemon": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.4.tgz", @@ -4715,6 +5209,19 @@ "picomatch": "^2.2.1" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "response-time": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/response-time/-/response-time-2.3.2.tgz", @@ -4894,6 +5401,11 @@ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -4917,6 +5429,12 @@ "safe-buffer": "~5.1.0" } }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true + }, "strtok3": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-8.1.0.tgz", @@ -4998,11 +5516,21 @@ "yn": "3.1.1" } }, + "tsconfig-paths": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz", + "integrity": "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==", + "dev": true, + "requires": { + "json5": "^2.2.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, "tslib": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", - "optional": true + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" }, "type-is": { "version": "1.6.18", @@ -5061,6 +5589,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index 1c57200..58ab4b2 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ }, "type": "module", "scripts": { - "server": "nodemon --loader=ts-node/esm server", + "server": "nodemon --inspect server.ts", "start": "node dist/server.js", "build": "tsc -p .", "makeitpretty": "prettier --write \"**/*.{css,js,json,jsx,scss,ts}\"" @@ -15,6 +15,7 @@ "dependencies": { "axios": "^1.7.5", "body-parser": "^1.20.2", + "bullmq": "^5.12.12", "cors": "^2.8.5", "dotenv": "16.4.5", "express": "^4.19.2", @@ -40,11 +41,12 @@ "@types/image-thumbnail": "^1.0.4", "@types/morgan": "^1.9.9", "@types/multer": "^1.4.12", - "@types/node": "^22.5.0", + "@types/node": "^22.5.1", "@types/response-time": "^2.3.8", "nodemon": "^3.1.4", "prettier": "^3.3.3", "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", "typescript": "^5.5.4" } } diff --git a/redis-docker-compose.yml b/redis-docker-compose.yml new file mode 100644 index 0000000..ea3ab77 --- /dev/null +++ b/redis-docker-compose.yml @@ -0,0 +1,8 @@ +version: '3' + +services: + redis-server: + image: "redis:alpine" + command: redis-server + ports: + - "6379:6379" \ No newline at end of file diff --git a/util/heicConverter.ts b/util/heicConverter.ts index e5c223d..cf969f4 100644 --- a/util/heicConverter.ts +++ b/util/heicConverter.ts @@ -1,13 +1,19 @@ +import { Job, Queue, QueueEvents, Worker } from "bullmq"; import dotenv from "dotenv"; import { fileTypeFromFile } from "file-type"; import { FileTypeResult } from "file-type/core"; import fs from "fs-extra"; import gm from "gm"; import path from "path"; +import { fileURLToPath } from "url"; import { logger } from "../server.js"; import { generateUniqueHeicFilename } from "./generateUniqueFilename.js"; import { FolderPaths } from "./serverInit.js"; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const HeicQueue = new Queue("HEIC Queue", { connection: { host: "localhost", port: 6379 } }); + dotenv.config({ path: path.resolve(process.cwd(), `.env.${process.env.NODE_ENV || "development"}`) }); @@ -16,19 +22,22 @@ const imageMagick = gm.subClass({ imageMagick: true }); export async function ConvertHeicFiles(files: Express.Multer.File[]) { const validFiles = await filterValidHeicFiles(files); + await Promise.all( validFiles.map(async (file) => { const convertedFileName = generateUniqueHeicFilename(file); - try { - await ConvertToJpeg(file.path, `${file.destination}/${convertedFileName}`); - logger.log("debug", `Converted ${file.filename} image to JPEG from HEIC.`); - await handleOriginalFile(file, convertedFileName); - file.filename = convertedFileName; - file.mimetype = "image/jpeg"; - file.path = `${file.destination}/${convertedFileName}`; - } catch (error) { - logger.log("error", `Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}`); - } + await HeicQueue.add(convertedFileName, { convertedFileName, file }); + + // try { + // await ConvertToJpeg(file.path, `${file.destination}/${convertedFileName}`); + // logger.log("debug", `Converted ${file.filename} image to JPEG from HEIC.`); + // await handleOriginalFile(file, convertedFileName); + // file.filename = convertedFileName; + // file.mimetype = "image/jpeg"; + // file.path = `${file.destination}/${convertedFileName}`; + // } catch (error) { + // logger.log("error", `Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}`); + // } }) ); } @@ -65,3 +74,41 @@ async function ConvertToJpeg(file: string, newPath: string) { }); }); } + +//Previos implementation using sandboxing. Cannot set up because the imports try to launch the server again. +// const processorUrl = pathToFileURL(__dirname + "/heicQueueProcessor.ts"); +// const HeicWorker = new Worker("HEIC Queue", processorUrl, { +// connection: { host: "localhost", port: 6379 } +// }); +const HeicWorker = new Worker( + "HEIC Queue", + async (job: Job) => { + const { file, convertedFileName } = job.data; + try { + await ConvertToJpeg(file.path, `${file.destination}/${convertedFileName}`); + logger.log("debug", `Converted ${file.filename} image to JPEG from HEIC.`); + await handleOriginalFile(file, convertedFileName); + file.filename = convertedFileName; + file.mimetype = "image/jpeg"; + file.path = `${file.destination}/${convertedFileName}`; + } catch (error) { + logger.log("error", `Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}`); + } + }, + { + connection: { host: "localhost", port: 6379 } + } +); + +HeicWorker.on("ready", () => { + console.log(`Worker Ready`); +}); +HeicWorker.on("active", (job, prev) => { + console.log(`Job ${job} is now active; previous status was ${prev}`); +}); +HeicWorker.on("completed", (jobId, returnvalue) => { + console.log(`${jobId} has completed and returned ${returnvalue}`); +}); +HeicWorker.on("failed", (jobId, failedReason) => { + console.log(`${jobId} has failed with reason ${failedReason}`); +}); diff --git a/util/heicQueueProcessor.ts b/util/heicQueueProcessor.ts new file mode 100644 index 0000000..40c69f1 --- /dev/null +++ b/util/heicQueueProcessor.ts @@ -0,0 +1,45 @@ +import { SandboxedJob } from "bullmq"; +import { logger } from "../server.js"; +import dotenv from "dotenv"; +import fs from "fs-extra"; +import path from "path"; +import { FolderPaths } from "./serverInit.js"; +import gm from "gm"; +const imageMagick = gm.subClass({ imageMagick: true }); +export default async function (job: SandboxedJob) { + // Do something with job + console.log("*** We're in the processing Queue") + const { file, convertedFileName } = job.data; + try { + logger.log("debug", `Attempting to convert ${file.filename} image to JPEG from HEIC.`); + await ConvertToJpeg(file.path, `${file.destination}/${convertedFileName}`); + logger.log("debug", `Converted ${file.filename} image to JPEG from HEIC.`); + await handleOriginalFile(file, convertedFileName); + file.filename = convertedFileName; + file.mimetype = "image/jpeg"; + file.path = `${file.destination}/${convertedFileName}`; + } catch (error) { + logger.log("error", `Error converting ${file.filename} image to JPEG from HEIC. ${JSON.stringify(error)}`); + } +} +async function handleOriginalFile(file: Express.Multer.File, convertedFileName: string) { + if (process.env.KEEP_CONVERTED_ORIGINALS) { + await fs.ensureDir(path.join(file.destination, FolderPaths.ConvertedOriginalSubDir)); + await fs.move(file.path, `${path.join(file.destination, FolderPaths.ConvertedOriginalSubDir)}/${file.filename}`); + } else { + await fs.unlink(file.path); + } +} + +async function ConvertToJpeg(file: string, newPath: string) { + const fileOnDisk: Buffer = await fs.readFile(file); + + return new Promise((resolve, reject) => { + imageMagick(fileOnDisk) + .setFormat("jpg") + .write(newPath, (error) => { + if (error) reject(error.message); + resolve(newPath); + }); + }); +} diff --git a/yarn.lock b/yarn.lock index fe5bdc0..1a4f68c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,6 +28,11 @@ resolved "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz" integrity sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@jridgewell/resolve-uri@^3.0.3": version "3.1.2" resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz" @@ -46,6 +51,11 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" +"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3": + version "3.0.3" + resolved "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz" + integrity sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ== + "@sec-ant/readable-stream@^0.4.1": version "0.4.1" resolved "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz" @@ -166,10 +176,10 @@ dependencies: "@types/express" "*" -"@types/node@*", "@types/node@^22.5.0": - version "22.5.0" - resolved "https://registry.npmjs.org/@types/node/-/node-22.5.0.tgz" - integrity sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg== +"@types/node@*", "@types/node@^22.5.1": + version "22.5.1" + resolved "https://registry.npmjs.org/@types/node/-/node-22.5.1.tgz" + integrity sha512-KkHsxej0j9IW1KKOOAA/XBA0z08UFSrRQHErzEfA3Vgq57eXIMYboIlHJuYIfd+lwCQjtKqUu3UnmKbtUc9yRw== dependencies: undici-types "~6.19.2" @@ -342,6 +352,19 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +bullmq@^5.12.12: + version "5.12.12" + resolved "https://registry.npmjs.org/bullmq/-/bullmq-5.12.12.tgz" + integrity sha512-xrWKDj1ZwnGKmrlmFqF6Vmub3WqDFfdBcIRLCooIs5+jeVzbHK7/1usgYSFg2pZiwK6h6eMivTb9WvcKkNW/+w== + dependencies: + cron-parser "^4.6.0" + ioredis "^5.4.1" + msgpackr "^1.10.1" + node-abort-controller "^3.1.1" + semver "^7.5.4" + tslib "^2.0.0" + uuid "^9.0.0" + busboy@^0.2.11: version "0.2.14" resolved "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz" @@ -381,6 +404,11 @@ chokidar@^3.5.2: optionalDependencies: fsevents "~2.3.2" +cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + color-convert@^1.9.3: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" @@ -499,6 +527,13 @@ create-require@^1.1.0: resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron-parser@^4.6.0: + version "4.9.0" + resolved "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + cross-spawn@^4.0.0: version "4.0.2" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz" @@ -521,6 +556,13 @@ debug@^4: dependencies: ms "2.1.2" +debug@^4.3.4: + version "4.3.6" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz" + integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg== + dependencies: + ms "2.1.2" + debug@2.6.9: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" @@ -542,6 +584,11 @@ delayed-stream@~1.0.0: resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@~1.1.0: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" @@ -557,7 +604,7 @@ destroy@1.2.0: resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== -detect-libc@^2.0.3: +detect-libc@^2.0.1, detect-libc@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz" integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw== @@ -893,6 +940,21 @@ inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3, inherits@2.0.4: resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +ioredis@^5.4.1: + version "5.4.1" + resolved "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz" + integrity sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" @@ -952,6 +1014,11 @@ isexe@^2.0.0: resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= +json5@^2.2.2: + version "2.2.3" + resolved "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + jsonfile@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz" @@ -983,6 +1050,16 @@ lie@~3.3.0: dependencies: immediate "~3.0.5" +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + logform@^2.6.0, logform@^2.6.1: version "2.6.1" resolved "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz" @@ -1003,6 +1080,11 @@ lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" +luxon@^3.2.1: + version "3.5.0" + resolved "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz" + integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ== + make-error@^1.1.1: version "1.3.6" resolved "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz" @@ -1090,6 +1172,27 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +msgpackr-extract@^3.0.2: + version "3.0.3" + resolved "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz" + integrity sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA== + dependencies: + node-gyp-build-optional-packages "5.2.2" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.3" + +msgpackr@^1.10.1: + version "1.11.0" + resolved "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.0.tgz" + integrity sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw== + optionalDependencies: + msgpackr-extract "^3.0.2" + multer@^1.4.4: version "1.4.4" resolved "https://registry.npmjs.org/multer/-/multer-1.4.4.tgz" @@ -1114,6 +1217,18 @@ nocache@^4.0.0: resolved "https://registry.npmjs.org/nocache/-/nocache-4.0.0.tgz" integrity sha512-AntnTbmKZvNYIsTVPPwv7dfZdAfo/6H/2ZlZACK66NAOQtIApxkB/6pf/c+s+ACW8vemGJzUCyVTssrzNUK6yQ== +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + +node-gyp-build-optional-packages@5.2.2: + version "5.2.2" + resolved "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz" + integrity sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw== + dependencies: + detect-libc "^2.0.1" + nodemon@^3.1.4: version "3.1.4" resolved "https://registry.npmjs.org/nodemon/-/nodemon-3.1.4.tgz" @@ -1329,6 +1444,18 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== + dependencies: + redis-errors "^1.0.0" + response-time@^2.3.2: version "2.3.2" resolved "https://registry.npmjs.org/response-time/-/response-time-2.3.2.tgz" @@ -1357,7 +1484,7 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -semver@^7.5.3, semver@^7.6.3: +semver@^7.5.3, semver@^7.5.4, semver@^7.6.3: version "7.6.3" resolved "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== @@ -1478,6 +1605,11 @@ stack-trace@0.0.x: resolved "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz" integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + statuses@2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" @@ -1505,6 +1637,11 @@ string_decoder@~0.10.x: resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz" + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== + strtok3@^8.1.0: version "8.1.0" resolved "https://registry.npmjs.org/strtok3/-/strtok3-8.1.0.tgz" @@ -1576,6 +1713,20 @@ ts-node@^10.9.2: v8-compile-cache-lib "^3.0.1" yn "3.1.1" +tsconfig-paths@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz" + integrity sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg== + dependencies: + json5 "^2.2.2" + minimist "^1.2.6" + strip-bom "^3.0.0" + +tslib@^2.0.0: + version "2.7.0" + resolved "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz" + integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA== + type-is@^1.6.4, type-is@~1.6.18: version "1.6.18" resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz" @@ -1629,6 +1780,11 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@^9.0.0: + version "9.0.1" + resolved "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz"