mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 06:51:59 +00:00
🖨️ fix: Preserve Concurrent PDF Extraction (#18008)
* 🖨️ fix: Preserve Concurrent PDF Extraction
* fix: Synchronize PDF Extraction Bun Lockfile
---------
Co-authored-by: Lia <lia@librechat.ai>
This commit is contained in:
parent
88b07c9f96
commit
d30d33a724
6 changed files with 314 additions and 74 deletions
|
|
@ -116,7 +116,7 @@
|
|||
"passport-jwt": "^4.0.1",
|
||||
"passport-ldapauth": "^3.0.1",
|
||||
"passport-local": "^1.0.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.9.1",
|
||||
|
|
|
|||
32
bun.lock
32
bun.lock
|
|
@ -131,7 +131,7 @@
|
|||
"passport-jwt": "^4.0.1",
|
||||
"passport-ldapauth": "^3.0.1",
|
||||
"passport-local": "^1.0.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
|
|
@ -357,7 +357,7 @@
|
|||
"mammoth": "^1.11.0",
|
||||
"mongodb": "^6.14.2",
|
||||
"nanoid": "^3.3.18",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"rimraf": "^6.1.3",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsdown": "^0.22.2",
|
||||
|
|
@ -425,7 +425,7 @@
|
|||
"mongoose": "^8.24.1",
|
||||
"nanoid": "^3.3.7",
|
||||
"node-fetch": "2.7.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.8.6",
|
||||
|
|
@ -1637,29 +1637,29 @@
|
|||
|
||||
"@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.6.1", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg=="],
|
||||
|
||||
"@napi-rs/canvas": ["@napi-rs/canvas@0.1.88", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.88", "@napi-rs/canvas-darwin-arm64": "0.1.88", "@napi-rs/canvas-darwin-x64": "0.1.88", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88", "@napi-rs/canvas-linux-arm64-gnu": "0.1.88", "@napi-rs/canvas-linux-arm64-musl": "0.1.88", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-musl": "0.1.88", "@napi-rs/canvas-win32-arm64-msvc": "0.1.88", "@napi-rs/canvas-win32-x64-msvc": "0.1.88" } }, "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg=="],
|
||||
"@napi-rs/canvas": ["@napi-rs/canvas@0.1.100", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.100", "@napi-rs/canvas-darwin-arm64": "0.1.100", "@napi-rs/canvas-darwin-x64": "0.1.100", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", "@napi-rs/canvas-linux-arm64-musl": "0.1.100", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-gnu": "0.1.100", "@napi-rs/canvas-linux-x64-musl": "0.1.100", "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", "@napi-rs/canvas-win32-x64-msvc": "0.1.100" } }, "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA=="],
|
||||
|
||||
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.88", "", { "os": "android", "cpu": "arm64" }, "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ=="],
|
||||
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.100", "", { "os": "android", "cpu": "arm64" }, "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@0.1.88", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg=="],
|
||||
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@0.1.100", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@0.1.88", "", { "os": "darwin", "cpu": "x64" }, "sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg=="],
|
||||
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@0.1.100", "", { "os": "darwin", "cpu": "x64" }, "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@0.1.88", "", { "os": "linux", "cpu": "arm" }, "sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg=="],
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@0.1.100", "", { "os": "linux", "cpu": "arm" }, "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@0.1.88", "", { "os": "linux", "cpu": "arm64" }, "sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg=="],
|
||||
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@0.1.88", "", { "os": "linux", "cpu": "arm64" }, "sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA=="],
|
||||
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@0.1.100", "", { "os": "linux", "cpu": "arm64" }, "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ=="],
|
||||
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@0.1.88", "", { "os": "linux", "cpu": "none" }, "sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw=="],
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@0.1.100", "", { "os": "linux", "cpu": "none" }, "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@0.1.88", "", { "os": "linux", "cpu": "x64" }, "sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw=="],
|
||||
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@0.1.88", "", { "os": "linux", "cpu": "x64" }, "sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g=="],
|
||||
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@0.1.100", "", { "os": "linux", "cpu": "x64" }, "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA=="],
|
||||
|
||||
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@0.1.88", "", { "os": "win32", "cpu": "arm64" }, "sha512-qcIFfEgHrchyYqRrxsCeTQgpJZ/GqHiqPcU/Fvw/ARVlQeDX1VyFH+X+0gCR2tca6UJrq96vnW+5o7buCq+erA=="],
|
||||
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@0.1.100", "", { "os": "win32", "cpu": "arm64" }, "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw=="],
|
||||
|
||||
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@0.1.88", "", { "os": "win32", "cpu": "x64" }, "sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ=="],
|
||||
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@0.1.100", "", { "os": "win32", "cpu": "x64" }, "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA=="],
|
||||
|
||||
"@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="],
|
||||
|
||||
|
|
@ -4417,7 +4417,7 @@
|
|||
|
||||
"pbkdf2": ["pbkdf2@3.1.7", "", { "dependencies": { "create-hash": "^1.2.0", "create-hmac": "^1.1.7", "ripemd160": "^2.0.3", "safe-buffer": "^5.2.1", "sha.js": "^2.4.12", "to-buffer": "^1.2.2" } }, "sha512-nS4mvFgVwUrecPTRdpdseM2fwpdfnX0/HibA8DgBRQtihUAzHQYJFU2l+OlEtiXtpqkrMvCPDwyXMUnASDt6Qw=="],
|
||||
|
||||
"pdfjs-dist": ["pdfjs-dist@5.4.624", "", { "optionalDependencies": { "@napi-rs/canvas": "^0.1.88", "node-readable-to-web-readable-stream": "^0.4.2" } }, "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg=="],
|
||||
"pdfjs-dist": ["pdfjs-dist@5.5.207", "", { "optionalDependencies": { "@napi-rs/canvas": "^0.1.95", "node-readable-to-web-readable-stream": "^0.4.2" } }, "sha512-WMqqw06w1vUt9ZfT0gOFhMf3wHsWhaCrxGrckGs5Cci6ybDW87IvPaOd2pnBwT6BJuP/CzXDZxjFgmSULLdsdw=="],
|
||||
|
||||
"pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="],
|
||||
|
||||
|
|
|
|||
125
package-lock.json
generated
125
package-lock.json
generated
|
|
@ -146,7 +146,7 @@
|
|||
"passport-jwt": "^4.0.1",
|
||||
"passport-ldapauth": "^3.0.1",
|
||||
"passport-local": "^1.0.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.9.1",
|
||||
|
|
@ -10256,9 +10256,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.88.tgz",
|
||||
"integrity": "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz",
|
||||
"integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
|
|
@ -10272,23 +10272,23 @@
|
|||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.88",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.88",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.88",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.88",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.88",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.88",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.88",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.88",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.88",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.88"
|
||||
"@napi-rs/canvas-android-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.100",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.100"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.88.tgz",
|
||||
"integrity": "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
|
|
@ -10306,9 +10306,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.88.tgz",
|
||||
"integrity": "sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
|
|
@ -10326,9 +10326,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.88.tgz",
|
||||
"integrity": "sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz",
|
||||
"integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
|
|
@ -10346,9 +10346,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.88.tgz",
|
||||
"integrity": "sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz",
|
||||
"integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
|
|
@ -10366,12 +10366,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.88.tgz",
|
||||
"integrity": "sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
|
|
@ -10386,12 +10389,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.88.tgz",
|
||||
"integrity": "sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
|
|
@ -10406,12 +10412,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.88.tgz",
|
||||
"integrity": "sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
|
|
@ -10426,12 +10435,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.88.tgz",
|
||||
"integrity": "sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
|
|
@ -10446,12 +10458,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.88.tgz",
|
||||
"integrity": "sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
|
|
@ -10466,9 +10481,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.88.tgz",
|
||||
"integrity": "sha512-qcIFfEgHrchyYqRrxsCeTQgpJZ/GqHiqPcU/Fvw/ARVlQeDX1VyFH+X+0gCR2tca6UJrq96vnW+5o7buCq+erA==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
|
|
@ -10486,9 +10501,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.88",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.88.tgz",
|
||||
"integrity": "sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ==",
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
|
|
@ -35286,15 +35301,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.4.624",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.624.tgz",
|
||||
"integrity": "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==",
|
||||
"version": "5.5.207",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.5.207.tgz",
|
||||
"integrity": "sha512-WMqqw06w1vUt9ZfT0gOFhMf3wHsWhaCrxGrckGs5Cci6ybDW87IvPaOd2pnBwT6BJuP/CzXDZxjFgmSULLdsdw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20.16.0 || >=22.3.0"
|
||||
"node": ">=20.19.0 || >=22.13.0 || >=24"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.88",
|
||||
"@napi-rs/canvas": "^0.1.95",
|
||||
"node-readable-to-web-readable-stream": "^0.4.2"
|
||||
}
|
||||
},
|
||||
|
|
@ -42093,7 +42108,7 @@
|
|||
"mammoth": "^1.11.0",
|
||||
"mongodb": "^6.14.2",
|
||||
"nanoid": "^3.3.18",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"rimraf": "^6.1.3",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsdown": "^0.22.2",
|
||||
|
|
@ -42161,7 +42176,7 @@
|
|||
"mongoose": "^8.24.1",
|
||||
"nanoid": "^3.3.7",
|
||||
"node-fetch": "2.7.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.9.1",
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@
|
|||
"mammoth": "^1.11.0",
|
||||
"mongodb": "^6.14.2",
|
||||
"nanoid": "^3.3.18",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"rimraf": "^6.1.3",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsdown": "^0.22.2",
|
||||
|
|
@ -160,7 +160,7 @@
|
|||
"mongoose": "^8.24.1",
|
||||
"nanoid": "^3.3.7",
|
||||
"node-fetch": "2.7.0",
|
||||
"pdfjs-dist": "^5.4.624",
|
||||
"pdfjs-dist": "^5.5.207",
|
||||
"prom-client": "^15.1.3",
|
||||
"rate-limit-redis": "^4.2.0",
|
||||
"re2js": "^2.9.1",
|
||||
|
|
|
|||
154
packages/api/src/files/documents/pdf.helper.ts
Normal file
154
packages/api/src/files/documents/pdf.helper.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { promises as fs } from 'node:fs';
|
||||
import { parseDocument } from './crud';
|
||||
|
||||
export interface PdfExtractionResult {
|
||||
name: string;
|
||||
pages: number;
|
||||
text?: string;
|
||||
bytes?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Small, valid PDFs with distinct text on every page and an optional nested page tree. */
|
||||
function createPdf(name: string, pages: number, depth = 0): Buffer {
|
||||
const objects = [
|
||||
'',
|
||||
'<< /Type /Catalog /Pages 2 0 R >>',
|
||||
'',
|
||||
'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>',
|
||||
];
|
||||
const kids: string[] = [];
|
||||
for (let page = 1; page <= pages; page++) {
|
||||
const index = objects.length;
|
||||
const text = name === 'blank' ? '' : `Document ${name} page ${page}`;
|
||||
const content = `BT /F1 12 Tf 72 720 Td (${text}) Tj ET`;
|
||||
kids.push(`${index} 0 R`);
|
||||
objects.push(
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${index + 1} 0 R >>`,
|
||||
`<< /Length ${content.length} >>\nstream\n${content}\nendstream`,
|
||||
);
|
||||
}
|
||||
objects[2] = `<< /Type /Pages /Kids [${kids.join(' ')}] /Count ${pages} >>`;
|
||||
if (depth > 0) {
|
||||
const start = objects.length;
|
||||
objects[2] = `<< /Type /Pages /Kids [${start} 0 R] /Count ${pages} >>`;
|
||||
for (let i = 0; i < depth; i++) {
|
||||
const parent = i === 0 ? 2 : start + i - 1;
|
||||
const children = i === depth - 1 ? kids.join(' ') : `${start + i + 1} 0 R`;
|
||||
objects.push(`<< /Type /Pages /Parent ${parent} 0 R /Kids [${children}] /Count ${pages} >>`);
|
||||
}
|
||||
for (let i = 4; i < start; i += 2) {
|
||||
objects[i] = objects[i].replace('/Parent 2 0 R', `/Parent ${start + depth - 1} 0 R`);
|
||||
}
|
||||
}
|
||||
let pdf = '%PDF-1.7\n';
|
||||
const offsets = [0];
|
||||
for (let i = 1; i < objects.length; i++) {
|
||||
offsets.push(pdf.length);
|
||||
pdf += `${i} 0 obj\n${objects[i]}\nendobj\n`;
|
||||
}
|
||||
const startXref = pdf.length;
|
||||
pdf += `xref\n0 ${objects.length}\n0000000000 65535 f \n`;
|
||||
pdf += offsets
|
||||
.slice(1)
|
||||
.map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`)
|
||||
.join('');
|
||||
pdf += `trailer\n<< /Size ${objects.length} /Root 1 0 R >>\nstartxref\n${startXref}\n%%EOF\n`;
|
||||
return Buffer.from(pdf, 'ascii');
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'pdf-extraction-'));
|
||||
try {
|
||||
const files = await Promise.all(
|
||||
[27, 18, 5, 2].map(async (pages, index) => {
|
||||
const name = String(index);
|
||||
const filePath = path.join(directory, `${name}.pdf`);
|
||||
// Deeper page lookup lets the short PDF finish loading during the other extractions.
|
||||
await fs.writeFile(filePath, createPdf(name, pages, index === 3 ? 20 : 0));
|
||||
return { name, pages, path: filePath };
|
||||
}),
|
||||
);
|
||||
const extract = async (file: (typeof files)[number]): Promise<PdfExtractionResult> => {
|
||||
try {
|
||||
const result = await parseDocument({
|
||||
file: {
|
||||
path: file.path,
|
||||
originalname: `${file.name}.pdf`,
|
||||
mimetype: 'application/pdf',
|
||||
} as Express.Multer.File,
|
||||
});
|
||||
return { name: file.name, pages: file.pages, text: result.text, bytes: result.bytes };
|
||||
} catch (error) {
|
||||
return {
|
||||
name: file.name,
|
||||
pages: file.pages,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
};
|
||||
const results: PdfExtractionResult[] = [];
|
||||
if (process.argv[2] === 'errors') {
|
||||
for (const name of ['blank', 'invalid', 'oversized']) {
|
||||
const filePath = path.join(directory, `${name}.pdf`);
|
||||
await fs.writeFile(filePath, name === 'invalid' ? 'not a PDF' : createPdf('blank', 1));
|
||||
try {
|
||||
await parseDocument({
|
||||
file: {
|
||||
path: filePath,
|
||||
originalname: `${name}.pdf`,
|
||||
mimetype: 'application/pdf',
|
||||
...(name === 'oversized' ? { size: 15 * 1024 * 1024 + 1 } : {}),
|
||||
} as Express.Multer.File,
|
||||
});
|
||||
results.push({ name, pages: 1 });
|
||||
} catch (error) {
|
||||
results.push({
|
||||
name,
|
||||
pages: 1,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (process.argv[2] === 'serial') {
|
||||
for (const file of files) results.push(await extract(file));
|
||||
} else {
|
||||
// Warm the native ESM module and worker before exercising overlapping document loads.
|
||||
await extract(files[0]);
|
||||
const readFile = fs.readFile;
|
||||
const paths = new Set(files.map((file) => file.path));
|
||||
const waiting: Array<() => void> = [];
|
||||
// Release real file reads together so filesystem scheduling cannot serialize PDF loading.
|
||||
fs.readFile = ((...args: Parameters<typeof fs.readFile>) => {
|
||||
const read = readFile(...args);
|
||||
if (!paths.has(String(args[0]))) return read;
|
||||
return read.then(async (data) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
waiting.push(resolve);
|
||||
if (waiting.length === files.length) {
|
||||
for (const release of waiting.splice(0)) release();
|
||||
}
|
||||
});
|
||||
return data;
|
||||
});
|
||||
}) as typeof fs.readFile;
|
||||
try {
|
||||
for (let round = 0; round < 5; round++) {
|
||||
results.push(...(await Promise.all(files.map(extract))));
|
||||
}
|
||||
} finally {
|
||||
fs.readFile = readFile;
|
||||
}
|
||||
}
|
||||
process.send?.(results);
|
||||
} finally {
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error: Error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
71
packages/api/src/files/documents/pdf.spec.ts
Normal file
71
packages/api/src/files/documents/pdf.spec.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import path from 'node:path';
|
||||
import { fork } from 'node:child_process';
|
||||
import type { PdfExtractionResult } from './pdf.helper';
|
||||
|
||||
/** Native Node preserves pdf.js's ESM import without changing the workspace's CJS Jest setup. */
|
||||
function extractPdfs(mode: 'serial' | 'concurrent' | 'errors'): Promise<PdfExtractionResult[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = fork(path.join(__dirname, 'pdf.helper.ts'), [mode], {
|
||||
execArgv: ['-r', 'ts-node/register/transpile-only'],
|
||||
env: {
|
||||
...process.env,
|
||||
TS_NODE_PROJECT: path.join(__dirname, '../../../tsconfig.json'),
|
||||
TS_NODE_COMPILER_OPTIONS: JSON.stringify({
|
||||
module: 'NodeNext',
|
||||
moduleResolution: 'NodeNext',
|
||||
}),
|
||||
},
|
||||
silent: true,
|
||||
});
|
||||
let results: PdfExtractionResult[] | undefined;
|
||||
let diagnostics = '';
|
||||
child.stdout?.on('data', (chunk: Buffer) => (diagnostics += chunk.toString()));
|
||||
child.stderr?.on('data', (chunk: Buffer) => (diagnostics += chunk.toString()));
|
||||
child.on('message', (message: PdfExtractionResult[]) => (results = message));
|
||||
child.once('error', reject);
|
||||
const timer = setTimeout(() => child.kill('SIGKILL'), 12000);
|
||||
child.once('close', (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
if (code !== 0 || results == null) {
|
||||
reject(new Error(`PDF subprocess failed (${signal ?? code}): ${diagnostics}`));
|
||||
return;
|
||||
}
|
||||
resolve(results);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function expectComplete(results: PdfExtractionResult[]): void {
|
||||
for (const result of results) {
|
||||
expect(result.error).toBeUndefined();
|
||||
const expected = Array.from(
|
||||
{ length: result.pages },
|
||||
(_, index) => `Document ${result.name} page ${index + 1}\n`,
|
||||
).join('');
|
||||
expect(result.text).toBe(expected);
|
||||
expect(result.bytes).toBe(Buffer.byteLength(expected));
|
||||
}
|
||||
}
|
||||
|
||||
describe('real PDF document extraction', () => {
|
||||
it('extracts every page from serial uploads', async () => {
|
||||
const results = await extractPdfs('serial');
|
||||
expect(results).toHaveLength(4);
|
||||
expectComplete(results);
|
||||
});
|
||||
|
||||
it('preserves complete text for every concurrently extracted PDF', async () => {
|
||||
const results = await extractPdfs('concurrent');
|
||||
expect(results).toHaveLength(20);
|
||||
expectComplete(results);
|
||||
});
|
||||
|
||||
it('rejects invalid, textless and oversized PDFs instead of reporting success', async () => {
|
||||
const results = await extractPdfs('errors');
|
||||
expect(results).toEqual([
|
||||
{ name: 'blank', pages: 1, error: 'No text found in document' },
|
||||
{ name: 'invalid', pages: 1, error: expect.stringMatching(/Invalid PDF/) },
|
||||
{ name: 'oversized', pages: 1, error: expect.stringMatching(/exceeds the 15MB/) },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue