diff --git a/bin/init-mongo.sh b/bin/init-mongo.sh new file mode 100644 index 0000000000..f77a3de967 --- /dev/null +++ b/bin/init-mongo.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +echo "Starting MongoDB initialization..." +sleep 3 + +# Create user using local connection (no port specification needed) +echo "Creating user..." +mongosh --eval " +const adminDb = db.getSiblingDB('admin'); +try { +adminDb.createUser({ + user: 'mongotUser', + pwd: 'mongotPassword', + roles: [{ role: 'searchCoordinator', db: 'admin' }] +}); +print('User mongotUser created successfully'); +} catch (error) { +if (error.code === 11000) { + print('User mongotUser already exists'); +} else { + print('Error creating user: ' + error); +} +} +" +echo "MongoDB initialization completed." diff --git a/config/default.js b/config/default.js index eeec5d6a14..75f37cb60d 100644 --- a/config/default.js +++ b/config/default.js @@ -45,7 +45,7 @@ module.exports = { ezs: { url: String(process.env.WORKERS_URL || 'http://localhost:31976'), verbose: 'ezs:*,-ezs:debug', - timeout: 30000, + timeout: 60000, cacheEnable: false, cacheDelay: 60 * 60 * 12, mainStatement: 'delegate', // use detatch to have a thread dedicated to processing the response, otherwise you can simply use “delegate” @@ -94,6 +94,8 @@ module.exports = { 'distribute-by-decadal', 'distribute-by-interval', 'filter', + 'join-by', + 'similar-by', 'get-fields', 'graph-by', 'segments-precomputed', @@ -101,6 +103,7 @@ module.exports = { 'values-precomputed', 'values-precomputed-nofilter', 'raw-precomputed-nofilter', + 'vsearch-precomputed', 'group-and-sum-with', 'hello-world', 'istex-facet', diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 85b759fbb9..a3a50fd482 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,7 +2,7 @@ version: '3' services: api: - image: node:24.13-alpine + image: node:24.13-slim extra_hosts: # for linux hosts since version 20.10 - "host.docker.internal:host-gateway" volumes: @@ -18,14 +18,14 @@ services: EZMASTER_PUBLIC_URL: ${EZMASTER_PUBLIC_URL} REDIS_URL: 'redis://redis.lodex-net:6379' WORKERS_URL: 'http://workers.lodex-net:31976' - MONGO_HOST: 'mongo.lodex-net:27017' + MONGO_HOST: 'mongod.lodex-net:27017' MAILER_HOST: 'maildev.lodex-net:1025' PRECOMPUTED_URL: 'http://api.lodex-net:3000' CI: ${CI} networks: - lodex-net depends_on: - - mongo + - mongod - redis - workers - maildev @@ -40,7 +40,7 @@ services: memory: 4G dev-server: ## Enable hot-reload in development - image: node:24.13-alpine + image: node:24.13-slim volumes: - .:/app working_dir: /app @@ -58,13 +58,20 @@ services: cpus: '2' memory: 4G - mongo: - image: mongodb/mongodb-community-server:7.0.9-ubi9 - logging: - driver: none - command: --quiet + mongod: + image: mongodb/mongodb-community-server:8.2.3-ubi9 + command: >- + mongod + --config /etc/mongod.conf + --replSetMember=mongod.lodex-net:27017 ports: - "27017:27017" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - mongod_data:/data/db + - ./etc/mongod.conf:/etc/mongod.conf:ro + - ./bin/init-mongo.sh:/docker-entrypoint-initdb.d/init-mongo.sh:ro networks: - lodex-net deploy: @@ -72,7 +79,15 @@ services: limits: cpus: '2' memory: 1G - + qdrant: + image: qdrant/qdrant:v1.17 + networks: + - lodex-net + depends_on: + - api + - workers + ports: + - 6333:6333 redis: image: redis:6 ports: @@ -84,9 +99,8 @@ services: limits: cpus: '1' memory: 1G - workers: - image: node:24.13-alpine + image: node:24.13-slim extra_hosts: # for linux hosts since version 20.10 - "host.docker.internal:host-gateway" volumes: @@ -112,7 +126,7 @@ services: - lodex-net command: npm run production:workers depends_on: - - mongo + - mongod deploy: resources: limits: @@ -125,6 +139,10 @@ services: - "1025:1025" networks: - lodex-net +volumes: + mongod_data: + mongot_data: + networks: lodex-net: name: lodex-net diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 3ba603b027..e8995e19db 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,6 +19,6 @@ services: ports: - "3000:3000" mongo: - image: mongodb/mongodb-community-server:7.0.9-ubi9 + image: mongodb/mongodb-community-server:8.2.3-ubi9 ports: - "27017:27017" diff --git a/docker-compose.spec.yml b/docker-compose.spec.yml index d8fdbe4a5b..6857441718 100644 --- a/docker-compose.spec.yml +++ b/docker-compose.spec.yml @@ -83,7 +83,7 @@ services: cpus: '2' memory: 2G mongo: - image: mongodb/mongodb-community-server:7.0.9-ubi9 + image: mongodb/mongodb-community-server:8.2.3-ubi9 ports: - "27017:27017" deploy: diff --git a/docker-compose.yml b/docker-compose.yml index d15f636017..eac4298601 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,6 @@ services: ports: - "3000:3000" mongo: - image: mongodb/mongodb-community-server:7.0.9-ubi9 + image: mongodb/mongodb-community-server:8.2.3-ubi9 ports: - "27017:27017" diff --git a/etc/mongod.conf b/etc/mongod.conf new file mode 100644 index 0000000000..a9d6b5efa6 --- /dev/null +++ b/etc/mongod.conf @@ -0,0 +1,10 @@ +# mongod.conf +storage: + dbPath: /data/db + +net: + port: 27017 + bindIp: 0.0.0.0 + +replication: + replSetName: rs0 diff --git a/package-lock.json b/package-lock.json index 665d531f95..a46dcfbe6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,9 +31,9 @@ "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "@ezs/analytics": "2.3.6", - "@ezs/basics": "2.9.2", - "@ezs/conditor": "2.13.4", - "@ezs/core": "4.1.3", + "@ezs/basics": "2.11.0", + "@ezs/conditor": "2.13.6", + "@ezs/core": "4.2.0", "@ezs/istex": "1.5.11", "@ezs/lodex": "file:packages/ezsLodex", "@ezs/sparql": "1.2.5", @@ -4021,9 +4021,9 @@ } }, "node_modules/@ezs/basics": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/@ezs/basics/-/basics-2.9.2.tgz", - "integrity": "sha512-efW6/JoD+fdk7wf26DpV0rS67BFgLfQ80aiXY4OjJB/QDUvn5OWpEmOhjJG+j91NjepO1i38zBDqkc8oJ1qtDA==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@ezs/basics/-/basics-2.11.0.tgz", + "integrity": "sha512-/3NnSUddwttVf9A6hgbDNklIzgjK3g8vTv1L3MrfpL63m+bM6V5HisRRzXP5vmHRtERKpswh5NZql0sOwIYeWA==", "license": "MIT", "dependencies": { "async-retry": "1.3.3", @@ -4047,6 +4047,7 @@ "tar-stream": "3.1.6", "tmp-filepath": "2.0.0", "unzipper": "0.10.11", + "uuid-random": "1.3.2", "xml-mapping": "1.7.2", "xml-splitter": "1.2.1" }, @@ -4160,9 +4161,9 @@ "license": "ISC" }, "node_modules/@ezs/conditor": { - "version": "2.13.4", - "resolved": "https://registry.npmjs.org/@ezs/conditor/-/conditor-2.13.4.tgz", - "integrity": "sha512-asQJFrYNMPtrvEqthXU4H3WkN5DtkZ3wDeMyU1ILCrrnJlJK7ugrP3WLRFObJqHSG4OD/Ld3hYV63s97A0nYLw==", + "version": "2.13.6", + "resolved": "https://registry.npmjs.org/@ezs/conditor/-/conditor-2.13.6.tgz", + "integrity": "sha512-s9W/dT8oEOywyseB0YSjWZL31MwukzMzbFgCZPPxrZwKyxDGLWyCXsSgrdagt0TxAJIR4c2JPeousp5stH3WRA==", "license": "CECILL-2.1", "dependencies": { "async-each-series": "^1.1.0", @@ -4170,9 +4171,9 @@ "csv-string": "3.2.0", "debug": "4.3.3", "dotenv": "8.2.0", - "fast-xml-parser": "4.4.1", + "fast-xml-parser": "4.5.4", "fetch-with-proxy": "3.0.1", - "lodash": "4.17.21", + "lodash": "4.17.23", "node-abort-controller": "1.1.0", "progress": "2.0.3", "qs": "6.10.3", @@ -4215,15 +4216,13 @@ } }, "node_modules/@ezs/conditor/node_modules/fast-xml-parser": { - "version": "4.4.1", + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.4.tgz", + "integrity": "sha512-jE8ugADnYOBsu1uaoayVl1tVKAMNOXyjwvv2U6udEA2ORBhDooJDWoGxTkhd4Qn4yh59JVVt/pKXtjPwx9OguQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" } ], "license": "MIT", @@ -4244,6 +4243,12 @@ "tunnel-agent": "^0.6.0" } }, + "node_modules/@ezs/conditor/node_modules/lodash": { + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "license": "MIT" + }, "node_modules/@ezs/conditor/node_modules/qs": { "version": "6.10.3", "license": "BSD-3-Clause", @@ -4262,9 +4267,9 @@ "license": "MIT" }, "node_modules/@ezs/core": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/@ezs/core/-/core-4.1.3.tgz", - "integrity": "sha512-66Eox2ufoT2KClRr7NsdErieHEhp4Xd6UDL2IYV0ok6YEdJ7K6Sam1L0gJ0jktSabBWA9UGFj+FqUtkgXp6z3A==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@ezs/core/-/core-4.2.0.tgz", + "integrity": "sha512-GQQIanOVh7hqZ1gzvMX9mqCbaVzgwKiv6uEb8J4mn+28S9rREvljpoCdBEozO02LDFq0GtD2BthPJv82A/AJ2Q==", "license": "MIT", "dependencies": { "app-module-path": "2.2.0", @@ -4720,6 +4725,27 @@ "version": "1.1.3", "license": "MIT" }, + "node_modules/@huggingface/jinja": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.5.tgz", + "integrity": "sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/transformers": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", + "integrity": "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.3", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "dev": true, @@ -4778,6 +4804,481 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-wasm32/node_modules/@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@ioredis/commands": { "version": "1.2.0", "license": "MIT" @@ -4864,6 +5365,27 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@isaacs/fs-minipass/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "dev": true, @@ -7265,6 +7787,70 @@ "url": "https://opencollective.com/popperjs" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@react-leaflet/core": { "version": "2.1.0", "license": "Hippocratic-2.1", @@ -12127,6 +12713,13 @@ "dev": true, "license": "ISC" }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bowser": { "version": "1.9.4", "license": "MIT" @@ -15136,7 +15729,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.3", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", "license": "Apache-2.0", "engines": { "node": ">=8" @@ -15150,6 +15745,12 @@ "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/dfa": { "version": "1.2.0", "license": "MIT" @@ -15815,6 +16416,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/es6-promise": { "version": "4.2.8", "license": "MIT" @@ -17448,6 +18055,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.3.2", "dev": true, @@ -18064,6 +18677,23 @@ "process": "^0.11.10" } }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, "node_modules/global-dirs": { "version": "3.0.1", "dev": true, @@ -18229,6 +18859,12 @@ "dev": true, "license": "MIT" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/handlebars": { "version": "4.7.8", "dev": true, @@ -23964,6 +24600,12 @@ "node": ">=0.1.90" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest": { "version": "1.0.1", "license": "MIT", @@ -24159,6 +24801,30 @@ "version": "2.0.1", "license": "Python-2.0" }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/matcher/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "license": "MIT", @@ -25931,6 +26597,104 @@ "node_modules/only": { "version": "0.0.2" }, + "node_modules/onnxruntime-common": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "global-agent": "^3.0.0", + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-node/node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/onnxruntime-node/node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/onnxruntime-node/node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/onnxruntime-node/node_modules/tar": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.7.tgz", + "integrity": "sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/onnxruntime-node/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", + "license": "MIT" + }, "node_modules/open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -26467,6 +27231,12 @@ "node": ">=6" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/pm2": { "version": "6.0.13", "resolved": "https://registry.npmjs.org/pm2/-/pm2-6.0.13.tgz", @@ -27080,6 +27850,30 @@ "version": "1.2.4", "license": "ISC" }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-agent": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", @@ -28581,6 +29375,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/robust-predicates": { "version": "3.0.2", "license": "Unlicense" @@ -28994,6 +29805,12 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, "node_modules/semver/node_modules/lru-cache": { "version": "6.0.0", "license": "ISC", @@ -29008,6 +29825,33 @@ "version": "4.0.0", "license": "ISC" }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-blocking": { "version": "2.0.0", "license": "ISC" @@ -29091,6 +29935,62 @@ "version": "1.1.0", "license": "MIT" }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "license": "MIT", @@ -29962,7 +30862,15 @@ } }, "node_modules/strnum": { - "version": "1.0.5", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", + "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], "license": "MIT" }, "node_modules/style-loader": { @@ -33427,7 +34335,7 @@ }, "packages/api": { "name": "@lodex/api", - "version": "16.10.7", + "version": "16.10.8", "dependencies": { "@ezs/transformers": "file:../transformers", "lodash": "4.17.21" @@ -33453,7 +34361,7 @@ }, "packages/common": { "name": "@lodex/common", - "version": "16.10.7", + "version": "16.10.8", "dependencies": { "@ezs/transformers": "file:../transformers" }, @@ -33479,6 +34387,7 @@ "name": "@ezs/lodex", "version": "1.17.0", "dependencies": { + "@huggingface/transformers": "^3.8.1", "lodash": "4.17.21", "mongodb": "6.12.0", "node-object-hash": "2.3.10", diff --git a/package.json b/package.json index b9cabcada9..ae7af45314 100644 --- a/package.json +++ b/package.json @@ -86,9 +86,9 @@ "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "@ezs/analytics": "2.3.6", - "@ezs/basics": "2.9.2", - "@ezs/conditor": "2.13.4", - "@ezs/core": "4.1.3", + "@ezs/basics": "2.11.0", + "@ezs/conditor": "2.13.6", + "@ezs/core": "4.2.0", "@ezs/istex": "1.5.11", "@ezs/lodex": "file:packages/ezsLodex", "@ezs/sparql": "1.2.5", diff --git a/packages/admin-app/src/fields/sourceValue/SourceValueToggle.tsx b/packages/admin-app/src/fields/sourceValue/SourceValueToggle.tsx index 1b0359c570..ec116f940a 100644 --- a/packages/admin-app/src/fields/sourceValue/SourceValueToggle.tsx +++ b/packages/admin-app/src/fields/sourceValue/SourceValueToggle.tsx @@ -411,7 +411,7 @@ export const SourceValueToggle = ({ - + {translate('precomputed_processing')} diff --git a/packages/api/src/services/publishDocuments.ts b/packages/api/src/services/publishDocuments.ts index 6c57b0a39e..6d9e925cae 100644 --- a/packages/api/src/services/publishDocuments.ts +++ b/packages/api/src/services/publishDocuments.ts @@ -1,5 +1,11 @@ import omit from 'lodash/omit'; +import find from 'lodash/find'; import get from 'lodash/get'; +import config from 'config'; +// @ts-expect-error TS(2792): Cannot find module '@ezs/core'. Did you mean to se... Remove this comment to see the full error message +import ezs from '@ezs/core'; +import Lodex from '@ezs/lodex'; +import { PassThrough } from 'stream'; import getDocumentTransformer from './getDocumentTransformer'; import transformAllDocuments from './transformAllDocuments'; @@ -14,6 +20,8 @@ import { import { jobLogger } from '../workers/tools'; import getLogger from './logger'; +ezs.use(Lodex); + export const versionTransformerDecorator = (transformDocument: any, subresourceId = null, hiddenResources = null) => async (document: any, _: any, __: any, publicationDate = new Date()) => { @@ -223,10 +231,32 @@ export const publishDocumentsFactory = }), ); + const input = new PassThrough({ objectMode: true }); + const environment = { + titleFieldName: find(fields, {'overview':1})?.name, + summaryFieldName: find(fields, {'overview':2})?.name, + detail1FieldName: find(fields, {'overview':3})?.name, + detail2FieldName: find(fields, {'overview':4})?.name, + subtitleFieldName: find(fields, {'overview':6})?.name, + }; + const queryString = new URLSearchParams(environment).toString(); + const parameters = { + url: `${config.get('ezs.url')}/publish.ini?${queryString}`, + timeout: config.get('timeout'), + streaming: true, + json: false, + encoder: 'pack', + }; + console.error({parameters}); + input.pipe(ezs('URLConnect', parameters)); + await transformAllDocuments( count, ctx.dataset.findLimitFromSkip, - ctx.publishedDataset.insertBatch, + (documents: any) => { + input.write(documents); + return ctx.publishedDataset.insertBatch(documents) + }, versionTransformerDecorator( transformMainResourceDocument, null, @@ -235,6 +265,7 @@ export const publishDocumentsFactory = undefined, ctx.job, ); + input.end(); ctx.job.isActive() ? jobLogger.info(ctx.job, 'Documents published') diff --git a/packages/ezsLodex/package.json b/packages/ezsLodex/package.json index 80b5389329..b35e7a89a4 100644 --- a/packages/ezsLodex/package.json +++ b/packages/ezsLodex/package.json @@ -9,6 +9,7 @@ "dev": "tsc --watch --preserveWatchOutput" }, "dependencies": { + "@huggingface/transformers": "^3.8.1", "lodash": "4.17.21", "mongodb": "6.12.0", "node-object-hash": "2.3.10", diff --git a/packages/ezsLodex/src/createVectorEmbeddings.ts b/packages/ezsLodex/src/createVectorEmbeddings.ts new file mode 100644 index 0000000000..6519527b33 --- /dev/null +++ b/packages/ezsLodex/src/createVectorEmbeddings.ts @@ -0,0 +1,30 @@ +import get from 'lodash/get.js'; +import set from 'lodash/set.js'; +import { pipeline } from "@huggingface/transformers"; + +const createVectorEmbeddings = async (data: any, feed: any, ctx: any) => { + if (ctx.isLast()) { + return feed.close(); + } + const path = Array().concat(ctx.getParam('path', 'value')).shift(); + const value = get(data, path); + if (!value) { + set(data, path, []); + return feed.send(data); + } + const extractor = await pipeline( + 'feature-extraction', + 'Xenova/all-MiniLM-L6-v2' + ); + const result = await extractor( + [value], + { + pooling: "mean", + normalize: true, + } + ); + set(data, path, Array.from(result.data)); + feed.send(data); +} + +export default createVectorEmbeddings; diff --git a/packages/ezsLodex/src/index.ts b/packages/ezsLodex/src/index.ts index 14ab650020..fa06500eb2 100644 --- a/packages/ezsLodex/src/index.ts +++ b/packages/ezsLodex/src/index.ts @@ -25,10 +25,12 @@ import precomputedSelect from './precomputedSelect.js'; import reduceQuery from './reduceQuery.js'; import runQuery from './runQuery.js'; import runQueryPrecomputed from './runQueryPrecomputed.js'; +import runVSearchPrecomputed from './runVSearchPrecomputed.js'; import saveDocuments from './saveDocuments.js'; import updateDocument from './updateDocument.js'; import updateDocuments from './updateDocuments.js'; import useFieldNames from './useFieldNames.js'; +import createVectorEmbeddings from './createVectorEmbeddings.js'; const funcs = { flattenPatch, @@ -51,6 +53,7 @@ const funcs = { labelizeFieldID, runQuery, runQueryPrecomputed, + runVSearchPrecomputed, reduceQuery, formatOutput, buildContext, @@ -61,6 +64,7 @@ const funcs = { homogenizedObject, updateDocument, updateDocuments, + createVectorEmbeddings, // aliases fixFlatten: flattenPatch.flattenPatch, LodexContext: disabled.disabled, diff --git a/packages/ezsLodex/src/runVSearchPrecomputed.ts b/packages/ezsLodex/src/runVSearchPrecomputed.ts new file mode 100644 index 0000000000..35cb68651c --- /dev/null +++ b/packages/ezsLodex/src/runVSearchPrecomputed.ts @@ -0,0 +1,161 @@ +import zipObject from 'lodash/zipObject.js'; +import unset from 'lodash/unset.js'; +import mongoDatabase from './mongoDatabase.js'; + +let isIndexExist = false; +/** + * Take `Object` containing a MongoDB query and throw the result + * + * The input object must contain a `connectionStringURI` property, containing + * the connection string to MongoDB. + * + * @name LodexRunVSearchPrecomputed + * @param {Object} [valueFieldName=value] field to use as value + * @param {Object} [labelFieldName=id] field to use as label + * @returns {Object} + */ +export default async function LodexRunVSearchPrecomputed(this: any, data: any, feed: any) { + if (this.isLast()) { + return feed.close(); + } + const { ezs } = this; + const { + filter: filterDocuments, + referer, + field, + precomputedId, + connectionStringURI, + skip, + minValue, + maxValue, + maxSize, + orderBy, + queryVector = [], + } = data; + + const collectionName = `pc_${precomputedId}`; + const fds = Array.isArray(field) ? field : [field]; + const fields = fds.filter(Boolean); + const projection = zipObject(fields, Array(fields.length).fill(true)); + + const valueFieldName = this.getParam('valueFieldName'); + const labelFieldName = this.getParam('labelFieldName'); + + const filter = {}; + + if (minValue) { + const minFilter = { + $gte: Number(minValue), + }; + // @ts-expect-error TS(7053): Element implicitly has an 'any' type because expre... Remove this comment to see the full error message + filter[valueFieldName] = { + ...minFilter, + }; + } + if (maxValue) { + const maxFilter = { + $lte: Number(maxValue), + }; + // @ts-expect-error TS(7053): Element implicitly has an 'any' type because expre... Remove this comment to see the full error message + filter[valueFieldName] = { + ...maxFilter, + }; + } + + const db = await mongoDatabase(connectionStringURI); + const collection = db.collection(collectionName); + unset(filterDocuments, '$text'); + + // define your MongoDB Vector Search index + // https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/?deployment-type=self&interface=driver&language=nodejs + const index = { + name: "vector_index", + type: "vectorSearch", + definition: { + "fields": [ + { + "type": "vector", + "numDimensions": 384, + "path": "value", + "similarity": "cosine", + "quantization": "scalar" + } + ] + } + }; + + if (!isIndexExist) { + await collection.createSearchIndex(index); + } + + const postFilter = + Object.keys(filterDocuments).length === 0 + ? {} + : { + documents: { $elemMatch: filterDocuments }, //{ "versions.0.abxD": "2033" } + }; + const aggregatePipeline = [ + { + $vectorSearch: { + index: 'vector_index', + path: 'value', + filter, + queryVector, + numCandidates: 150, + limit: 10, + }, + }, + { + $project: { + _id: 0, + id: 1, + value: { + $meta: "vectorSearchScore" + } + } + }, + { + + $lookup: { + from: 'publishedDataset', + localField: 'id', + foreignField: 'uri', + as: 'documents', + }, + }, + { + $match: postFilter, + }, + ]; + console.log('aggregatePipeline', aggregatePipeline ); + const cursor = collection.aggregate( + aggregatePipeline, + fields.length > 0 + ? { ...projection, allowDiskUse: true } + : { + allowDiskUse: true, + }, + ); + + const count = await collection + // @ts-expect-error TS(2339): Property 'concat' does not exist on type '{}'. + .aggregate(aggregatePipeline.concat({ $count: 'value' })) + .toArray(); + if (count.length === 0) { + return feed.send({ total: 0 }); + } + const path = ['total']; + const value = [count[0] ? count[0].value : 0]; + if (referer) { + path.push('referer'); + value.push(referer); + } + + const stream = cursor + .skip(Number(skip || 0)) + .limit(Number(maxSize || 1000000)) + .stream() + .on('error', (e: any) => feed.stop(e)) + .pipe(ezs('assign', { path, value })); + await feed.flow(stream); +} diff --git a/packages/frontend-common/formats/other/resources-grid/ResourcesGridView.tsx b/packages/frontend-common/formats/other/resources-grid/ResourcesGridView.tsx index 358c4521f4..060da0936a 100644 --- a/packages/frontend-common/formats/other/resources-grid/ResourcesGridView.tsx +++ b/packages/frontend-common/formats/other/resources-grid/ResourcesGridView.tsx @@ -87,7 +87,7 @@ const ResourcesGridView = ({ return (
- {/* + {/* // @ts-expect-error TS2322 */}
    {filteredData.map((entry, index) => ( @@ -96,7 +96,7 @@ const ResourcesGridView = ({ // @ts-expect-error TS2322 className={styles.item} > - {/* + {/* // @ts-expect-error TS2322 */}
    { export default compose( // @ts-expect-error TS2345 - injectData(null, (field) => !!field), + injectData(null, (field) => !!field, true), connect(mapStateToProps), // @ts-expect-error TS2345 )(ResourcesGridView); diff --git a/packages/workers/src/precomputed/vectors.ini b/packages/workers/src/precomputed/vectors.ini new file mode 100644 index 0000000000..c50f32b6bb --- /dev/null +++ b/packages/workers/src/precomputed/vectors.ini @@ -0,0 +1,37 @@ +[use] +plugin = lodex + +[env] +path = generator +value = vector + +# Step 1 (générique): Charger le fichier corpus +[delegate] +file = ./shared/charger.cfg + +# Step 2 (générique): Traiter de manière asynchrone les items reçus +[fork] +standalone = true +logger = ./shared/logger.cfg + +# Step 2.0 (optionnel): Accélère le détachement du fork si l'enrichissement est lent +[fork/delegate] +file = ./shared/buffer.cfg + +# Step 2.1 (spécifique): Lancer un calcul sur tous les items reçus +[fork/delegate] + +# Step 2.1.1 (spécifique): S'assurer d'avoir des tableaux de tableaux +[fork/delegate/createVectorEmbeddings] +path = value + +[fork/delegate/debug] +text = after +# Step 2.2 (générique): Enregistrer le résultat et signaler que le traitement est fini +[fork/delegate] +file = ./shared/recorder.cfg + +# Step 3 : Renvoyer immédiatement un seul élément indiquant comment récupérer le résulat quand il sera prêt +[delegate] +file = ./shared/recipient.cfg + diff --git a/packages/workers/src/publish.ini b/packages/workers/src/publish.ini new file mode 100644 index 0000000000..92359c48d1 --- /dev/null +++ b/packages/workers/src/publish.ini @@ -0,0 +1,96 @@ +[use] +plugin = basics + +[env] +path = database_url +value = http://qdrant.lodex-net:6333 + + +[unpack] +[ungroup] + +[singleton] +merge = false +[singleton/replace] +path = vectors.size +value = 384 +path = vectors.distance +value = Cosine + +[singleton/debug] +text = env('database_url').append('/collections/ma_collection').prepend('PUT ') +[singleton/URLConnect] +url = env('database_url').append('/collections/ma_collection') +method = PUT +json = true +streaming = true +encoder = pack +noerror = true +timeout = 30000 +[singleton/debug] +text = resultat de creation de la collection + +[debug] +text = env('fields') + + + +[replace] +path = id +value = get('uri') + +path = vector +value = get(`versions.0.${env('titleFieldName')}`).toString() + +path = title +value = get(`versions.0.env('titleFieldName')}`).toString() + +path = summary +value = get(`versions.0.env('summaryFieldName')}`).toString() + +path = detail1 +value = get(`versions.0.env('detail1FieldName')}`).toString() + +path = detail2 +value = get(`versions.0.env('detail2FieldName')}`).toString() + +path = subtitle +value = get(`versions.0.env('subtitleFieldName')}`).toString() + +path = date_published +value = get('publicationDate') + +[debug] +text = document + +[createVectorEmbeddings] +path = vector + +[encode] +path = id +mode = uuid + +[group] +size = 5 + +[replace] +path = points +value = self() + +[spawn] +[spawn/debug] +text = env('database_url').append('/collections/ma_collection/points').prepend('PUT ') +[spawn/URLConnect] +url = env('database_url').append('/collections/ma_collection/points') +method = PUT +json = true +streaming = true +encoder = pack +noerror = true +timeout = 30000 + +[debug] +text = resulat indexation + +[OBJCount] +value = true diff --git a/packages/workers/src/routines/join-by.ini b/packages/workers/src/routines/join-by.ini new file mode 100644 index 0000000000..eb0bd2d9d8 --- /dev/null +++ b/packages/workers/src/routines/join-by.ini @@ -0,0 +1,162 @@ +# For one document selected by a URI (otherwise the first of the stream), +# and with one field on parameter, we build a vector from the initial document +# and all documents that shared one or more values of the field +# +# Be carful : works only with @ezs/lodex > 1.0.1 +prepend = delegate?file=../worker.ini +mimeType = application/json +label = join-by + +[use] +plugin = basics +plugin = lodex +plugin = analytics + +[env] +path = connectionStringURI +value = get('connectionStringURI') + +[buildContext] +connectionStringURI = env('connectionStringURI') + +; On garde dans l'environnement quelques paramètres +[env] +path = minValue +value = env('minValue').split('§').map(parseFloat).shift().defaultTo(0) + +path = maxValue +value = env('maxValue').split('§').map(parseFloat).shift().defaultTo(1) + +path = maxSize +value = env('maxSize').parseInt().defaultTo(10) + +path = reverse +value = env('orderBy').split('/').get(1).replace('asc', 0).replace('desc', 1).parseInt() + +path = uri +value = get('filter.uri') + +path = fields +value = get('fields') + +path = field +value = get('field').castArray() + + +; On récupére les documents filtrés +[LodexRunQuery] + +; On supprime les infos inutiles +[filterVersions] +[filterContributions] + +[shift] + +[multiply] +path = key +value = env('field').compact().filter(x => typeof x === 'string') + +[assign] +path = id +value = get('uri') + +path = value +value = get(self.key) + +# Préparation de la requete de rapprochement +[replace] +path = fix('filter.', '["versions.0.', self.key, '"].', '$in').join('') +value = get('value').castArray() + +path = connectionStringURI +value = env('connectionStringURI') + +path = referer.id +value = get('uri') + +path = referer.value +value = get('value') + +path = referer.key +value = env('field.0') + +[assign] + +path = filter.uri.$ne +value = get('referer.id') + +[debug] +text= requete +[LodexRunQuery] + +; On supprime les infos inutiles +[filterVersions] +[filterContributions] + +; find the fields which are selected for title and summary +[assign] +path = titleFieldName +value = env('fields').find({'overview':1}).get('name') + +path = summaryFieldName +value = env('fields').find({'overview':2}).get('name') + +path = detail1FieldName +value = env('fields').find({'overview':3}).get('name') + +path = detail2FieldName +value = env('fields').find({'overview':4}).get('name') + +path = subtitleFieldName +value = env('fields').find({'overview':6}).get('name') + +path = host +value = env('host') + +[assign] +path = id +value = get('uri') + +path = title +value = get(self.titleFieldName).toString() + +path = summary +value = get(self.summaryFieldName).toString() + +path = detail1 +value = get(self.detail1FieldName).toString() + +path = detail2 +value = get(self.detail2FieldName).toString() + +path = subtitle +value = get(self.subtitleFieldName).toString() + +path = date_published +value = get('publicationDate') + +path = url +value = pick(['host', 'uri']).values().join('/') + +; Send only a part of the data +[slice] +start = 1 +size = env('maxSize') + +; Keep only the necessary data +[keep] +path = id +path = url +path = title +path = summary +path = detail1 +path = detail2 +path = subtitle +path = date_published +path = total + +; Create and structured output +[LodexOutput] +keyName = items +indent = true +extract = total diff --git a/packages/workers/src/routines/search.ini b/packages/workers/src/routines/search.ini new file mode 100644 index 0000000000..3ecd8310b4 --- /dev/null +++ b/packages/workers/src/routines/search.ini @@ -0,0 +1,59 @@ +# Export with only the ID & a TITLE + +prepend = delegate?file=../worker.ini +mimeType = application/json +label = all-documents + +[use] +plugin = basics +plugin = lodex +plugin = analytics + +[buildContext] +connectionStringURI = get('connectionStringURI') + +; On garde dans l'environnement quelques paramètres +[env] +path = minValue +value = env('minValue').parseInt().defaultTo(0) + +path = maxValue +value = env('maxValue').parseInt().defaultTo(1000000) + +path = maxSize +value = env('maxSize').parseInt().defaultTo(10) + +path = reverse +value = env('orderBy').split('/').get(1).replace('asc', 0).replace('desc', 1).parseInt() + +path = tuneBy +value = env('orderBy').split('/').get(0).replace('_id', 'id') + +path = sortOrder +value = env('sort').split('/').get(1) + +path = sortOn +value = env('sort').split('/').get(0) + +path = field +value = get('field').castArray() + +; On récupére les documents filtrés +[LodexRunQuery] +sortOn = env('sortOn') +sortOrder = env('sortOrder') + +; On supprime les infos inutiles +[filterVersions] +[filterContributions] + +; On envoit au client uniquement les premiers documents +[slice] +start = 1 +size = env('maxSize') + +# Generate JSON +[LodexOutput] +indent = true +extract = total +extract = fullTotal diff --git a/packages/workers/src/routines/similar-by.ini b/packages/workers/src/routines/similar-by.ini new file mode 100644 index 0000000000..f7b35572a0 --- /dev/null +++ b/packages/workers/src/routines/similar-by.ini @@ -0,0 +1,207 @@ +# For one document selected by a URI (otherwise the first of the stream), +# and with one field on parameter, we build a vector from the initial document +# and all documents that shared one or more values of the field +# +# Be carful : works only with @ezs/lodex > 1.0.1 +prepend = delegate?file=../worker.ini +mimeType = application/json +label = similar-by + +[use] +plugin = basics +plugin = lodex +plugin = analytics + +[env] +path = database_url +value = http://qdrant.lodex-net:6333 + + +path = connectionStringURI +value = get('connectionStringURI') + +[buildContext] +connectionStringURI = env('connectionStringURI') + +; On garde dans l'environnement quelques paramètres +[env] +path = minValue +value = env('minValue').split('§').map(parseFloat).shift().defaultTo(0) + +path = maxValue +value = env('maxValue').split('§').map(parseFloat).shift().defaultTo(1) + +path = maxSize +value = env('maxSize').parseInt().defaultTo(10) + +path = reverse +value = env('orderBy').split('/').get(1).replace('asc', 0).replace('desc', 1).parseInt() + +path = uri +value = get('filter.uri') + +path = fields +value = get('fields') + +path = field +value = get('field').castArray() + + +; On récupére les documents filtrés +[LodexRunQuery] + +; On supprime les infos inutiles +[filterVersions] +[filterContributions] + +[shift] + +[multiply] +path = key +value = env('field').compact().filter(x => typeof x === 'string') + +[assign] +path = id +value = get('uri') + +path = value +value = get(self.key) + +# Préparation de la requete de rapprochement +[replace] +path = query +value = get('value') + +path = limit +value = env('maxSize') + +path = with_payload +value = false + +[createVectorEmbeddings] +path = query + + +[spawn] +[spawn/debug] +text = env('database_url').append('/collections/ma_collection/points/query').prepend('POST ') +[spawn/URLConnect] +url = env('database_url').append('/collections/ma_collection/points/query') +method = POST +json = true +streaming = true +encoder = pack +noerror = true +timeout = 30000 +[spawn/debug] +text = resulat similarité + + +[exchange] +value = get('points') + +[ungroup] + +[greater] +path = score +than = 0.5 +[remove] +test = get('score').gt(1) ; ne marche pas tout le temps + +[decode] +path = id +mode = uuid + +[debug] +text = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +[exchange] +value = get('id') +[group] +size = env('maxSize') + +; Create mongodb filter (@TODO Change versions to a dynamic call when needed) +[replace] +path = filter.uri.$in +value = self() + +[debug] +text = value + +; Get filtered data +[LodexRunQuery] +sortOn = get(`ref_field.${self.sortOn}`) +sortOrder = env('sortOrder') + +; Remove useless data +[filterVersions] +[filterContributions] + +; find the fields which are selected for title and summary +[assign] +path = titleFieldName +value = env('fields').find({'overview':1}).get('name') + +path = summaryFieldName +value = env('fields').find({'overview':2}).get('name') + +path = detail1FieldName +value = env('fields').find({'overview':3}).get('name') + +path = detail2FieldName +value = env('fields').find({'overview':4}).get('name') + +path = subtitleFieldName +value = env('fields').find({'overview':6}).get('name') + +path = host +value = env('host') + +[assign] +path = id +value = get('uri') + +path = title +value = get(self.titleFieldName).toString() + +path = summary +value = get(self.summaryFieldName).toString() + +path = detail1 +value = get(self.detail1FieldName).toString() + +path = detail2 +value = get(self.detail2FieldName).toString() + +path = subtitle +value = get(self.subtitleFieldName).toString() + +path = date_published +value = get('publicationDate') + +path = url +value = pick(['host', 'uri']).values().join('/') + +; Send only a part of the data +[slice] +start = 1 +size = env('maxSize') + +; Keep only the necessary data +[keep] +path = id +path = url +path = title +path = summary +path = detail1 +path = detail2 +path = subtitle +path = date_published +path = total + + +; Create and structured output +[LodexOutput] +keyName = items +indent = true +extract = total diff --git a/packages/workers/src/routines/vsearch-precomputed.ini b/packages/workers/src/routines/vsearch-precomputed.ini new file mode 100644 index 0000000000..94eef608db --- /dev/null +++ b/packages/workers/src/routines/vsearch-precomputed.ini @@ -0,0 +1,31 @@ +prepend = delegate?file=../worker.ini +mimeType = application/json +label = Pre-calculation for vectors search + +[use] +plugin = basics +plugin = lodex + +[buildContext] +connectionStringURI = get('connectionStringURI') +tenant = get('tenant') +precomputedName = get('precomputedName') + +[assign] +path = queryVector +value = get('filter.$text.$search') + +[createVectorEmbeddings] +path = queryVector + +[runVSearchPrecomputed] +valueFieldName = env('precomputedValueColumn').defaultTo('weight') +labelFieldName = env('precomputedLabelColumn').defaultTo('source') + +[exchange] +value = self().omit(['origin', 'documents']) + +[LodexOutput] +indent = true +extract = total + diff --git a/src/app/custom/routines/routines-catalog.json b/src/app/custom/routines/routines-catalog.json index 36357a5c53..60534e3245 100644 --- a/src/app/custom/routines/routines-catalog.json +++ b/src/app/custom/routines/routines-catalog.json @@ -137,6 +137,18 @@ "doc": "https://github.com/Inist-CNRS/lodex-extended/tree/sub-resources-routine/public/routines", "recommendedWith": ["RessourcesGrid","PaginatedTable","UnpaginatedTable"] }, + { + "id": "r-join-by", + "url": "/api/run/join-by/", + "doc": "https://github.com/Inist-CNRS/lodex-extended/tree/sub-resources-routine/public/routines", + "recommendedWith": ["RessourcesGrid","PaginatedTable","UnpaginatedTable"] + }, + { + "id": "r-similar-by", + "url": "/api/run/similar-by/", + "doc": "https://github.com/Inist-CNRS/lodex-extended/tree/sub-resources-routine/public/routines", + "recommendedWith": ["RessourcesGrid","PaginatedTable","UnpaginatedTable"] + }, { "id": "r-get-fields", "url": "/api/run/get-fields/", diff --git a/src/app/custom/routines/routines-precomputed-catalog.json b/src/app/custom/routines/routines-precomputed-catalog.json index 40db8bd910..24775d691a 100644 --- a/src/app/custom/routines/routines-precomputed-catalog.json +++ b/src/app/custom/routines/routines-precomputed-catalog.json @@ -28,5 +28,11 @@ "url": "/api/run/raw-precomputed-nofilter/", "doc": "https://www.lodex.fr/docs/documentation/principales-fonctionnalites-disponibles/les-routines-et-graphes/les-routines/", "recommendedWith": ["BarChart", "BubbleChart", "PieChart"] + }, + { + "id": "r-vsearch-precomputed", + "url": "/api/run/vsearch-precomputed/", + "doc": "https://www.lodex.fr/docs/documentation/principales-fonctionnalites-disponibles/les-routines-et-graphes/les-routines/", + "recommendedWith": [] } ]