Improve precomputed lookup

docker-compose.dev.yml

@@ -20,7 +20,7 @@ services:
             WORKERS_URL: 'http://workers.lodex-net:31976'
             MONGO_HOST: 'mongo.lodex-net:27017'
             MAILER_HOST: 'maildev.lodex-net:1025'
-            PRECOMPUTED_URL: 'http://api.lodex-net:3000'
+            PRECOMPUTED_URL: ${PRECOMPUTED_URL:-http://api.lodex-net:3000}
             CI: ${CI}
         networks:
             - lodex-net
@@ -60,9 +60,12 @@ services:
 
     mongo:
         image: mongodb/mongodb-community-server:7.0.9-ubi9
-        logging:
-            driver: none
-        command: --quiet
+        memswap_limit: -1  # désactive le swap pour ce container
+        command: >
+            mongod
+            --wiredTigerCacheSizeGB 0.5
+            --setParameter maxIndexBuildMemoryUsageMegabytes=500
+            --bind_ip_all
         ports:
             - "27017:27017"
         networks:
@@ -71,7 +74,7 @@ services:
             resources:
                 limits:
                     cpus: '2'
-                    memory: 1G
+                    memory: 4G
 
     redis:
         image: redis:6

packages/api/src/services/precomputed/precomputed.ts

@@ -487,7 +487,7 @@ export const processPrecomputed = async (precomputed: any, ctx: any) => {
 
     logData = JSON.stringify({
         level: 'ok',
-        message: `[Instance: ${ctx.tenant}] 6/10 - Waiting for response data`,
+        message: `[Instance: ${ctx.tenant}] 6/10 - Waiting for response data on ${webhookBaseUrl}`,
         timestamp: new Date(),
         status: TaskStatus.ON_HOLD,
     });

packages/ezsLodex/src/runQueryPrecomputed.ts

@@ -65,28 +65,34 @@ async function LodexRunQueryPrecomputed(this: any, data: any, feed: any) {
     const db = await mongoDatabase(connectionStringURI);
     const collection = db.collection(collectionName);
 
-    const postFilter =
-        Object.keys(filterDocuments).length === 0
-            ? {}
-            : {
-                  documents: { $elemMatch: filterDocuments }, //{ "versions.0.abxD": "2033" }
-              };
-    const aggregatePipeline = [
-        {
+    const aggregatePipeline = [];
+
+    if (Object.keys(filter).length > 0) {
+        aggregatePipeline.push({
             $match: filter,
-        },
-        {
+        });
+    }
+    if (Object.keys(filterDocuments).length > 0) {
+        aggregatePipeline.push({
             $lookup: {
-                from: 'publishedDataset',
-                localField: 'origin',
-                foreignField: 'uri',
+                "from": 'publishedDataset',
+                "let": { origin: '$origin' },
+                pipeline: [
+                    {
+                        $match: {
+                            $expr: { $eq: ['$uri', '$$origin'] },
+                            ...filterDocuments, // filtre appliqué côté publishedDataset
+                        },
+                    },
+                    {
+                        $project: { _id: 1 }, // ne ramener que le strict minimum
+                    },
+                ],
                 as: 'documents',
             },
-        },
-        {
-            $match: postFilter,
-        },
-    ];
+        });
+    }
+    console.dir(aggregatePipeline, {depth: null})
     const cursor = collection.aggregate(
         aggregatePipeline,
         fields.length > 0
@@ -116,6 +122,7 @@ async function LodexRunQueryPrecomputed(this: any, data: any, feed: any) {
         .limit(Number(maxSize || 1000000))
         .stream()
         .on('error', (e: any) => feed.stop(e))
+        .pipe(ezs('debug', { text: 'LodexRunQueryPrecomputed', path: '_id' }))
         .pipe(ezs('assign', { path, value }));
     await feed.flow(stream);
 }