From d30d33a724876822bc2458cfb7fdb9cc14a7add4 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:21:34 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=96=A8=EF=B8=8F=20fix:=20Preserve=20Concu?= =?UTF-8?q?rrent=20PDF=20Extraction=20(#18008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🖨️ fix: Preserve Concurrent PDF Extraction * fix: Synchronize PDF Extraction Bun Lockfile --------- Co-authored-by: Lia --- api/package.json | 2 +- bun.lock | 32 ++-- package-lock.json | 125 +++++++------- packages/api/package.json | 4 +- .../api/src/files/documents/pdf.helper.ts | 154 ++++++++++++++++++ packages/api/src/files/documents/pdf.spec.ts | 71 ++++++++ 6 files changed, 314 insertions(+), 74 deletions(-) create mode 100644 packages/api/src/files/documents/pdf.helper.ts create mode 100644 packages/api/src/files/documents/pdf.spec.ts diff --git a/api/package.json b/api/package.json index 94e4c116f2..95aa456d2b 100644 --- a/api/package.json +++ b/api/package.json @@ -116,7 +116,7 @@ "passport-jwt": "^4.0.1", "passport-ldapauth": "^3.0.1", "passport-local": "^1.0.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.9.1", diff --git a/bun.lock b/bun.lock index c5310e2da7..129814d1f9 100644 --- a/bun.lock +++ b/bun.lock @@ -131,7 +131,7 @@ "passport-jwt": "^4.0.1", "passport-ldapauth": "^3.0.1", "passport-local": "^1.0.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.8.6", @@ -357,7 +357,7 @@ "mammoth": "^1.11.0", "mongodb": "^6.14.2", "nanoid": "^3.3.18", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "rimraf": "^6.1.3", "ts-node": "^10.9.2", "tsdown": "^0.22.2", @@ -425,7 +425,7 @@ "mongoose": "^8.24.1", "nanoid": "^3.3.7", "node-fetch": "2.7.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.8.6", @@ -1637,29 +1637,29 @@ "@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.6.1", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg=="], - "@napi-rs/canvas": ["@napi-rs/canvas@0.1.88", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.88", "@napi-rs/canvas-darwin-arm64": "0.1.88", "@napi-rs/canvas-darwin-x64": "0.1.88", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88", "@napi-rs/canvas-linux-arm64-gnu": "0.1.88", "@napi-rs/canvas-linux-arm64-musl": "0.1.88", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-musl": "0.1.88", "@napi-rs/canvas-win32-arm64-msvc": "0.1.88", "@napi-rs/canvas-win32-x64-msvc": "0.1.88" } }, "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg=="], + "@napi-rs/canvas": ["@napi-rs/canvas@0.1.100", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.100", "@napi-rs/canvas-darwin-arm64": "0.1.100", "@napi-rs/canvas-darwin-x64": "0.1.100", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", "@napi-rs/canvas-linux-arm64-musl": "0.1.100", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-musl": "0.1.100", "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", "@napi-rs/canvas-win32-x64-msvc": "0.1.100" } }, "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA=="], - "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.88", "", { "os": "android", "cpu": "arm64" }, "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ=="], + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.100", "", { "os": "android", "cpu": "arm64" }, "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ=="], - "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@0.1.88", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg=="], + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@0.1.100", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A=="], - "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@0.1.88", "", { "os": "darwin", "cpu": "x64" }, "sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg=="], + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@0.1.100", "", { "os": "darwin", "cpu": "x64" }, "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw=="], - "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@0.1.88", "", { "os": "linux", "cpu": "arm" }, "sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg=="], + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@0.1.100", "", { "os": "linux", "cpu": "arm" }, "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA=="], - "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@0.1.88", "", { "os": "linux", "cpu": "arm64" }, "sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg=="], + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw=="], - "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@0.1.88", "", { "os": "linux", "cpu": "arm64" }, "sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA=="], + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ=="], - "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@0.1.88", "", { "os": "linux", "cpu": "none" }, "sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw=="], + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@0.1.100", "", { "os": "linux", "cpu": "none" }, "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw=="], - "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@0.1.88", "", { "os": "linux", "cpu": "x64" }, "sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw=="], + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg=="], - "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@0.1.88", "", { "os": "linux", "cpu": "x64" }, "sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g=="], + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA=="], - "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@0.1.88", "", { "os": "win32", "cpu": "arm64" }, "sha512-qcIFfEgHrchyYqRrxsCeTQgpJZ/GqHiqPcU/Fvw/ARVlQeDX1VyFH+X+0gCR2tca6UJrq96vnW+5o7buCq+erA=="], + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@0.1.100", "", { "os": "win32", "cpu": "arm64" }, "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw=="], - "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@0.1.88", "", { "os": "win32", "cpu": "x64" }, "sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ=="], + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@0.1.100", "", { "os": "win32", "cpu": "x64" }, "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA=="], "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], @@ -4417,7 +4417,7 @@ "pbkdf2": ["pbkdf2@3.1.7", "", { "dependencies": { "create-hash": "^1.2.0", "create-hmac": "^1.1.7", "ripemd160": "^2.0.3", "safe-buffer": "^5.2.1", "sha.js": "^2.4.12", "to-buffer": "^1.2.2" } }, "sha512-nS4mvFgVwUrecPTRdpdseM2fwpdfnX0/HibA8DgBRQtihUAzHQYJFU2l+OlEtiXtpqkrMvCPDwyXMUnASDt6Qw=="], - "pdfjs-dist": ["pdfjs-dist@5.4.624", "", { "optionalDependencies": { "@napi-rs/canvas": "^0.1.88", "node-readable-to-web-readable-stream": "^0.4.2" } }, "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg=="], + "pdfjs-dist": ["pdfjs-dist@5.5.207", "", { "optionalDependencies": { "@napi-rs/canvas": "^0.1.95", "node-readable-to-web-readable-stream": "^0.4.2" } }, "sha512-WMqqw06w1vUt9ZfT0gOFhMf3wHsWhaCrxGrckGs5Cci6ybDW87IvPaOd2pnBwT6BJuP/CzXDZxjFgmSULLdsdw=="], "pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="], diff --git a/package-lock.json b/package-lock.json index e5ce491ac9..c120cf01dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -146,7 +146,7 @@ "passport-jwt": "^4.0.1", "passport-ldapauth": "^3.0.1", "passport-local": "^1.0.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.9.1", @@ -10256,9 +10256,9 @@ } }, "node_modules/@napi-rs/canvas": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.88.tgz", - "integrity": "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", "license": "MIT", "optional": true, "workspaces": [ @@ -10272,23 +10272,23 @@ "url": "https://github.com/sponsors/Brooooooklyn" }, "optionalDependencies": { - "@napi-rs/canvas-android-arm64": "0.1.88", - "@napi-rs/canvas-darwin-arm64": "0.1.88", - "@napi-rs/canvas-darwin-x64": "0.1.88", - "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88", - "@napi-rs/canvas-linux-arm64-gnu": "0.1.88", - "@napi-rs/canvas-linux-arm64-musl": "0.1.88", - "@napi-rs/canvas-linux-riscv64-gnu": "0.1.88", - "@napi-rs/canvas-linux-x64-gnu": "0.1.88", - "@napi-rs/canvas-linux-x64-musl": "0.1.88", - "@napi-rs/canvas-win32-arm64-msvc": "0.1.88", - "@napi-rs/canvas-win32-x64-msvc": "0.1.88" + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" } }, "node_modules/@napi-rs/canvas-android-arm64": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.88.tgz", - "integrity": "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", "cpu": [ "arm64" ], @@ -10306,9 +10306,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-arm64": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.88.tgz", - "integrity": "sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", "cpu": [ "arm64" ], @@ -10326,9 +10326,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-x64": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.88.tgz", - "integrity": "sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", "cpu": [ "x64" ], @@ -10346,9 +10346,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.88.tgz", - "integrity": "sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", "cpu": [ "arm" ], @@ -10366,12 +10366,15 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-gnu": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.88.tgz", - "integrity": "sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -10386,12 +10389,15 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-musl": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.88.tgz", - "integrity": "sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -10406,12 +10412,15 @@ } }, "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.88.tgz", - "integrity": "sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -10426,12 +10435,15 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-gnu": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.88.tgz", - "integrity": "sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -10446,12 +10458,15 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-musl": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.88.tgz", - "integrity": "sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -10466,9 +10481,9 @@ } }, "node_modules/@napi-rs/canvas-win32-arm64-msvc": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.88.tgz", - "integrity": "sha512-qcIFfEgHrchyYqRrxsCeTQgpJZ/GqHiqPcU/Fvw/ARVlQeDX1VyFH+X+0gCR2tca6UJrq96vnW+5o7buCq+erA==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", "cpu": [ "arm64" ], @@ -10486,9 +10501,9 @@ } }, "node_modules/@napi-rs/canvas-win32-x64-msvc": { - "version": "0.1.88", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.88.tgz", - "integrity": "sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ==", + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", "cpu": [ "x64" ], @@ -35286,15 +35301,15 @@ } }, "node_modules/pdfjs-dist": { - "version": "5.4.624", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.624.tgz", - "integrity": "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==", + "version": "5.5.207", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.5.207.tgz", + "integrity": "sha512-WMqqw06w1vUt9ZfT0gOFhMf3wHsWhaCrxGrckGs5Cci6ybDW87IvPaOd2pnBwT6BJuP/CzXDZxjFgmSULLdsdw==", "license": "Apache-2.0", "engines": { - "node": ">=20.16.0 || >=22.3.0" + "node": ">=20.19.0 || >=22.13.0 || >=24" }, "optionalDependencies": { - "@napi-rs/canvas": "^0.1.88", + "@napi-rs/canvas": "^0.1.95", "node-readable-to-web-readable-stream": "^0.4.2" } }, @@ -42093,7 +42108,7 @@ "mammoth": "^1.11.0", "mongodb": "^6.14.2", "nanoid": "^3.3.18", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "rimraf": "^6.1.3", "ts-node": "^10.9.2", "tsdown": "^0.22.2", @@ -42161,7 +42176,7 @@ "mongoose": "^8.24.1", "nanoid": "^3.3.7", "node-fetch": "2.7.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.9.1", diff --git a/packages/api/package.json b/packages/api/package.json index 9073c8eabb..26dd792e0f 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -89,7 +89,7 @@ "mammoth": "^1.11.0", "mongodb": "^6.14.2", "nanoid": "^3.3.18", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "rimraf": "^6.1.3", "ts-node": "^10.9.2", "tsdown": "^0.22.2", @@ -160,7 +160,7 @@ "mongoose": "^8.24.1", "nanoid": "^3.3.7", "node-fetch": "2.7.0", - "pdfjs-dist": "^5.4.624", + "pdfjs-dist": "^5.5.207", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "re2js": "^2.9.1", diff --git a/packages/api/src/files/documents/pdf.helper.ts b/packages/api/src/files/documents/pdf.helper.ts new file mode 100644 index 0000000000..5bb1786f0a --- /dev/null +++ b/packages/api/src/files/documents/pdf.helper.ts @@ -0,0 +1,154 @@ +import os from 'node:os'; +import path from 'node:path'; +import { promises as fs } from 'node:fs'; +import { parseDocument } from './crud'; + +export interface PdfExtractionResult { + name: string; + pages: number; + text?: string; + bytes?: number; + error?: string; +} + +/** Small, valid PDFs with distinct text on every page and an optional nested page tree. */ +function createPdf(name: string, pages: number, depth = 0): Buffer { + const objects = [ + '', + '<< /Type /Catalog /Pages 2 0 R >>', + '', + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + ]; + const kids: string[] = []; + for (let page = 1; page <= pages; page++) { + const index = objects.length; + const text = name === 'blank' ? '' : `Document ${name} page ${page}`; + const content = `BT /F1 12 Tf 72 720 Td (${text}) Tj ET`; + kids.push(`${index} 0 R`); + objects.push( + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${index + 1} 0 R >>`, + `<< /Length ${content.length} >>\nstream\n${content}\nendstream`, + ); + } + objects[2] = `<< /Type /Pages /Kids [${kids.join(' ')}] /Count ${pages} >>`; + if (depth > 0) { + const start = objects.length; + objects[2] = `<< /Type /Pages /Kids [${start} 0 R] /Count ${pages} >>`; + for (let i = 0; i < depth; i++) { + const parent = i === 0 ? 2 : start + i - 1; + const children = i === depth - 1 ? kids.join(' ') : `${start + i + 1} 0 R`; + objects.push(`<< /Type /Pages /Parent ${parent} 0 R /Kids [${children}] /Count ${pages} >>`); + } + for (let i = 4; i < start; i += 2) { + objects[i] = objects[i].replace('/Parent 2 0 R', `/Parent ${start + depth - 1} 0 R`); + } + } + let pdf = '%PDF-1.7\n'; + const offsets = [0]; + for (let i = 1; i < objects.length; i++) { + offsets.push(pdf.length); + pdf += `${i} 0 obj\n${objects[i]}\nendobj\n`; + } + const startXref = pdf.length; + pdf += `xref\n0 ${objects.length}\n0000000000 65535 f \n`; + pdf += offsets + .slice(1) + .map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`) + .join(''); + pdf += `trailer\n<< /Size ${objects.length} /Root 1 0 R >>\nstartxref\n${startXref}\n%%EOF\n`; + return Buffer.from(pdf, 'ascii'); +} + +async function main(): Promise { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'pdf-extraction-')); + try { + const files = await Promise.all( + [27, 18, 5, 2].map(async (pages, index) => { + const name = String(index); + const filePath = path.join(directory, `${name}.pdf`); + // Deeper page lookup lets the short PDF finish loading during the other extractions. + await fs.writeFile(filePath, createPdf(name, pages, index === 3 ? 20 : 0)); + return { name, pages, path: filePath }; + }), + ); + const extract = async (file: (typeof files)[number]): Promise => { + try { + const result = await parseDocument({ + file: { + path: file.path, + originalname: `${file.name}.pdf`, + mimetype: 'application/pdf', + } as Express.Multer.File, + }); + return { name: file.name, pages: file.pages, text: result.text, bytes: result.bytes }; + } catch (error) { + return { + name: file.name, + pages: file.pages, + error: error instanceof Error ? error.message : String(error), + }; + } + }; + const results: PdfExtractionResult[] = []; + if (process.argv[2] === 'errors') { + for (const name of ['blank', 'invalid', 'oversized']) { + const filePath = path.join(directory, `${name}.pdf`); + await fs.writeFile(filePath, name === 'invalid' ? 'not a PDF' : createPdf('blank', 1)); + try { + await parseDocument({ + file: { + path: filePath, + originalname: `${name}.pdf`, + mimetype: 'application/pdf', + ...(name === 'oversized' ? { size: 15 * 1024 * 1024 + 1 } : {}), + } as Express.Multer.File, + }); + results.push({ name, pages: 1 }); + } catch (error) { + results.push({ + name, + pages: 1, + error: error instanceof Error ? error.message : String(error), + }); + } + } + } else if (process.argv[2] === 'serial') { + for (const file of files) results.push(await extract(file)); + } else { + // Warm the native ESM module and worker before exercising overlapping document loads. + await extract(files[0]); + const readFile = fs.readFile; + const paths = new Set(files.map((file) => file.path)); + const waiting: Array<() => void> = []; + // Release real file reads together so filesystem scheduling cannot serialize PDF loading. + fs.readFile = ((...args: Parameters) => { + const read = readFile(...args); + if (!paths.has(String(args[0]))) return read; + return read.then(async (data) => { + await new Promise((resolve) => { + waiting.push(resolve); + if (waiting.length === files.length) { + for (const release of waiting.splice(0)) release(); + } + }); + return data; + }); + }) as typeof fs.readFile; + try { + for (let round = 0; round < 5; round++) { + results.push(...(await Promise.all(files.map(extract)))); + } + } finally { + fs.readFile = readFile; + } + } + process.send?.(results); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +} + +main().catch((error: Error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/packages/api/src/files/documents/pdf.spec.ts b/packages/api/src/files/documents/pdf.spec.ts new file mode 100644 index 0000000000..04632b3b1d --- /dev/null +++ b/packages/api/src/files/documents/pdf.spec.ts @@ -0,0 +1,71 @@ +import path from 'node:path'; +import { fork } from 'node:child_process'; +import type { PdfExtractionResult } from './pdf.helper'; + +/** Native Node preserves pdf.js's ESM import without changing the workspace's CJS Jest setup. */ +function extractPdfs(mode: 'serial' | 'concurrent' | 'errors'): Promise { + return new Promise((resolve, reject) => { + const child = fork(path.join(__dirname, 'pdf.helper.ts'), [mode], { + execArgv: ['-r', 'ts-node/register/transpile-only'], + env: { + ...process.env, + TS_NODE_PROJECT: path.join(__dirname, '../../../tsconfig.json'), + TS_NODE_COMPILER_OPTIONS: JSON.stringify({ + module: 'NodeNext', + moduleResolution: 'NodeNext', + }), + }, + silent: true, + }); + let results: PdfExtractionResult[] | undefined; + let diagnostics = ''; + child.stdout?.on('data', (chunk: Buffer) => (diagnostics += chunk.toString())); + child.stderr?.on('data', (chunk: Buffer) => (diagnostics += chunk.toString())); + child.on('message', (message: PdfExtractionResult[]) => (results = message)); + child.once('error', reject); + const timer = setTimeout(() => child.kill('SIGKILL'), 12000); + child.once('close', (code, signal) => { + clearTimeout(timer); + if (code !== 0 || results == null) { + reject(new Error(`PDF subprocess failed (${signal ?? code}): ${diagnostics}`)); + return; + } + resolve(results); + }); + }); +} + +function expectComplete(results: PdfExtractionResult[]): void { + for (const result of results) { + expect(result.error).toBeUndefined(); + const expected = Array.from( + { length: result.pages }, + (_, index) => `Document ${result.name} page ${index + 1}\n`, + ).join(''); + expect(result.text).toBe(expected); + expect(result.bytes).toBe(Buffer.byteLength(expected)); + } +} + +describe('real PDF document extraction', () => { + it('extracts every page from serial uploads', async () => { + const results = await extractPdfs('serial'); + expect(results).toHaveLength(4); + expectComplete(results); + }); + + it('preserves complete text for every concurrently extracted PDF', async () => { + const results = await extractPdfs('concurrent'); + expect(results).toHaveLength(20); + expectComplete(results); + }); + + it('rejects invalid, textless and oversized PDFs instead of reporting success', async () => { + const results = await extractPdfs('errors'); + expect(results).toEqual([ + { name: 'blank', pages: 1, error: 'No text found in document' }, + { name: 'invalid', pages: 1, error: expect.stringMatching(/Invalid PDF/) }, + { name: 'oversized', pages: 1, error: expect.stringMatching(/exceeds the 15MB/) }, + ]); + }); +});