From b4448c9d76804bba58036ed76d27830d25ffe7f6 Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Wed, 15 Apr 2026 16:24:35 +0200 Subject: [PATCH 1/3] improve docker compose for dev --- docker-compose.dev.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 85b759fbb..1588ad3c8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -20,7 +20,7 @@ services: WORKERS_URL: 'http://workers.lodex-net:31976' MONGO_HOST: 'mongo.lodex-net:27017' MAILER_HOST: 'maildev.lodex-net:1025' - PRECOMPUTED_URL: 'http://api.lodex-net:3000' + PRECOMPUTED_URL: ${PRECOMPUTED_URL:-http://api.lodex-net:3000} CI: ${CI} networks: - lodex-net @@ -60,9 +60,12 @@ services: mongo: image: mongodb/mongodb-community-server:7.0.9-ubi9 - logging: - driver: none - command: --quiet + memswap_limit: -1 # désactive le swap pour ce container + command: > + mongod + --wiredTigerCacheSizeGB 0.5 + --setParameter maxIndexBuildMemoryUsageMegabytes=500 + --bind_ip_all ports: - "27017:27017" networks: @@ -71,7 +74,7 @@ services: resources: limits: cpus: '2' - memory: 1G + memory: 4G redis: image: redis:6 From 624205af5b75243c19944a80506d672cb3f353ea Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Wed, 15 Apr 2026 16:12:10 +0200 Subject: [PATCH 2/3] no filter no lookup --- .../src/services/precomputed/precomputed.ts | 2 +- packages/ezsLodex/src/runQueryPrecomputed.ts | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/api/src/services/precomputed/precomputed.ts b/packages/api/src/services/precomputed/precomputed.ts index 7ee531241..e51ef149e 100644 --- a/packages/api/src/services/precomputed/precomputed.ts +++ b/packages/api/src/services/precomputed/precomputed.ts @@ -487,7 +487,7 @@ export const processPrecomputed = async (precomputed: any, ctx: any) => { logData = JSON.stringify({ level: 'ok', - message: `[Instance: ${ctx.tenant}] 6/10 - Waiting for response data`, + message: `[Instance: ${ctx.tenant}] 6/10 - Waiting for response data on ${webhookBaseUrl}`, timestamp: new Date(), status: TaskStatus.ON_HOLD, }); diff --git a/packages/ezsLodex/src/runQueryPrecomputed.ts b/packages/ezsLodex/src/runQueryPrecomputed.ts index 942344dee..ad1fd6d59 100644 --- a/packages/ezsLodex/src/runQueryPrecomputed.ts +++ b/packages/ezsLodex/src/runQueryPrecomputed.ts @@ -65,28 +65,28 @@ async function LodexRunQueryPrecomputed(this: any, data: any, feed: any) { const db = await mongoDatabase(connectionStringURI); const collection = db.collection(collectionName); - const postFilter = - Object.keys(filterDocuments).length === 0 - ? {} - : { - documents: { $elemMatch: filterDocuments }, //{ "versions.0.abxD": "2033" } - }; - const aggregatePipeline = [ - { + const aggregatePipeline = []; + + if (Object.keys(filter).length > 0) { + aggregatePipeline.push({ $match: filter, - }, - { + }); + } + if (Object.keys(filterDocuments).length > 0) { + aggregatePipeline.push({ $lookup: { from: 'publishedDataset', localField: 'origin', foreignField: 'uri', as: 'documents', }, - }, - { - $match: postFilter, - }, - ]; + }); + aggregatePipeline.push( { + $match: { + documents: { $elemMatch: filterDocuments }, //{ "versions.0.abxD": "2033" } + }, + }); + } const cursor = collection.aggregate( aggregatePipeline, fields.length > 0 From 9cbdf230fe67417d9b9296415d93aa7c402ad3d7 Mon Sep 17 00:00:00 2001 From: Nicolas Thouvenin Date: Wed, 15 Apr 2026 17:36:12 +0200 Subject: [PATCH 3/3] attempts to imporve query --- packages/ezsLodex/src/runQueryPrecomputed.ts | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/ezsLodex/src/runQueryPrecomputed.ts b/packages/ezsLodex/src/runQueryPrecomputed.ts index ad1fd6d59..0bcafc5ee 100644 --- a/packages/ezsLodex/src/runQueryPrecomputed.ts +++ b/packages/ezsLodex/src/runQueryPrecomputed.ts @@ -75,18 +75,24 @@ async function LodexRunQueryPrecomputed(this: any, data: any, feed: any) { if (Object.keys(filterDocuments).length > 0) { aggregatePipeline.push({ $lookup: { - from: 'publishedDataset', - localField: 'origin', - foreignField: 'uri', + "from": 'publishedDataset', + "let": { origin: '$origin' }, + pipeline: [ + { + $match: { + $expr: { $eq: ['$uri', '$$origin'] }, + ...filterDocuments, // filtre appliqué côté publishedDataset + }, + }, + { + $project: { _id: 1 }, // ne ramener que le strict minimum + }, + ], as: 'documents', }, }); - aggregatePipeline.push( { - $match: { - documents: { $elemMatch: filterDocuments }, //{ "versions.0.abxD": "2033" } - }, - }); } + console.dir(aggregatePipeline, {depth: null}) const cursor = collection.aggregate( aggregatePipeline, fields.length > 0 @@ -116,6 +122,7 @@ async function LodexRunQueryPrecomputed(this: any, data: any, feed: any) { .limit(Number(maxSize || 1000000)) .stream() .on('error', (e: any) => feed.stop(e)) + .pipe(ezs('debug', { text: 'LodexRunQueryPrecomputed', path: '_id' })) .pipe(ezs('assign', { path, value })); await feed.flow(stream); }