Extend relation filters: JSON paths and single-row matching ($elemMatch)

packages/mongo-knex/lib/convertor.js

@@ -176,6 +176,66 @@ class MongoToKnex {
         Object.assign(this.config, {relations: {}}, config);
     }
 
+    /**
+     * The LHS expression that extracts a JSON scalar for comparison, as a raw
+     * fragment plus its bindings. `->>` unquotes the value on both databases Ghost
+     * runs on (MySQL 8+, SQLite 3.38+): `json_extract` alone returns a MySQL string
+     * scalar *with* its surrounding quotes (`"GB"`), so `LIKE 'G%'` and other string
+     * comparisons would match the quote, not the value — `->>` returns the bare `GB`.
+     */
+    extractJsonColumn(column, path) {
+        return {expr: '?? ->> ?', bindings: [column, path]};
+    }
+
+    /**
+     * Apply one comparison to a query builder, transparently routing a statement
+     * that carries a `jsonPath` through JSON extraction. Without a path it's the
+     * plain `qb[whereType](column, op, value)` call every caller used before.
+     */
+    applyComparison(builder, whereType, column, jsonPath, op, value) {
+        if (jsonPath && jsonPath.length > 0) {
+            const path = `$.${jsonPath.join('.')}`;
+            const {expr, bindings} = this.extractJsonColumn(column, path);
+
+            // A null comparison is IS [NOT] NULL. Handled first so the branches below
+            // only ever see a non-null value, keeping their `${whereType}Raw` method
+            // valid (whereType is a `…Null`/`…NotNull` variant only when value is null,
+            // and there is no `whereNullRaw`).
+            if (value === null) {
+                const rawWhere = whereType.startsWith('or') ? 'orWhereRaw' : 'whereRaw';
+                return builder[rawWhere](`${expr} ${op === '!=' ? 'is not null' : 'is null'}`, bindings);
+            }
+
+            // A regex (contains/startsWith/endsWith) compiles to a LIKE pattern; an
+            // ignore-case match lowers both sides — the extracted value here, the
+            // pattern in processRegExp.
+            if (value instanceof RegExp) {
+                const {source, ignoreCase} = processRegExp(value);
+                const lhs = ignoreCase ? `lower(${expr})` : expr;
+                return builder[`${whereType}Raw`](`${lhs} ${op} ? ESCAPE ?`, [...bindings, source, likeEscapeCharacter]);
+            }
+            // `in`/`not in` (a set, or a negated equality rewritten to `$nin`) isn't
+            // covered by knex's whereJsonPath, so it's spelled out here too.
+            if (op === 'in' || op === 'not in') {
+                const values = _.isArray(value) ? value : [value];
+                const placeholders = values.map(() => '?').join(', ');
+                return builder[`${whereType}Raw`](`${expr} ${op} (${placeholders})`, [...bindings, ...values]);
+            }
+            return builder[`${whereType}Raw`](`${expr} ${op} ?`, [...bindings, value]);
+        }
+
+        // A regex reaches here only via a relation subquery — the top-level path
+        // handles it in buildComparison. Convert it to the same LIKE-with-ESCAPE form
+        // so contains/startsWith/endsWith work on a related column too.
+        if (value instanceof RegExp) {
+            const {source, ignoreCase} = processRegExp(value);
+            const lhs = ignoreCase ? 'lower(??)' : '??';
+            return builder[`${whereType}Raw`](`${lhs} ${op} ? ESCAPE ?`, [column, source, likeEscapeCharacter]);
+        }
+
+        return builder[whereType](column, op, value);
+    }
+
     processWhereType(mode, op, value) {
         if (value === null) {
             return (mode === '$or' ? 'orWhere' : 'where') + (op === '$ne' ? 'NotNull' : 'Null');
@@ -192,7 +252,10 @@ class MongoToKnex {
      * Determine if statement lives on parent table or if statement refers to a relation.
      */
     processStatement(column, op, value) {
-        const [tableName, columnName] = column.split('.');
+        // Segments past `relation.column` (or `column` on a relation's join/target
+        // table) are a JSON path into that column, e.g.
+        // `custom_fields.value_json.country` → column `value_json`, path `['country']`.
+        const [tableName, columnName, ...jsonPath] = column.split('.');
 
         // CASE: relation?
         if (columnName) {
@@ -228,6 +291,7 @@ class MongoToKnex {
                     joinTable: relation.joinTable,
                     table: relation.tableName,
                     column: columnName,
+                    jsonPath: jsonPath,
                     operator: op,
                     value: value,
                     config: relation,
@@ -238,6 +302,7 @@ class MongoToKnex {
             return {
                 table: tableName,
                 column: columnName,
+                jsonPath: jsonPath,
                 operator: op,
                 value: value,
                 config: relation,
@@ -306,6 +371,22 @@ class MongoToKnex {
              */
             const isAggregate = statement.config && statement.config.type === 'aggregate';
 
+            // CASE: all conditions of one `$elemMatch` must match a single related
+            //       row, so they share a subquery keyed by the match's id regardless
+            //       of operator - the "a negation matches a different row" reasoning
+            //       below never applies to them. This is the explicit same-row escape
+            //       hatch: everything outside an $elemMatch keeps the default grouping.
+            if (statement.elemMatchGroup !== undefined) {
+                const elemKey = `${statement.table}_elem_${statement.elemMatchGroup}`;
+
+                if (!group[elemKey]) {
+                    group[elemKey] = {innerWhereStatements: []};
+                }
+
+                group[elemKey].innerWhereStatements.push(statement);
+                return;
+            }
+
             let shouldCreateSubGroup = !isAggregate && isNegationOp(statement.operator);
             if (!shouldCreateSubGroup && !isAggregate && group[statement.table]) {
                 shouldCreateSubGroup = _.some(group[statement.table].innerWhereStatements, (innerStatement) => {
@@ -362,6 +443,9 @@ class MongoToKnex {
      */
     buildRelationQuery(qb, relations, mode) {
         debug(`(buildRelationQuery)`);
+        // The subquery bodies below are knex callbacks where `this` is the query
+        // builder, so hold onto the converter to reach its helpers from inside them.
+        const self = this;
 
         if (debugExtended.enabled) {
             debugExtended(`(buildRelationQuery) ${stringify(relations)}`);
@@ -450,7 +534,7 @@ class MongoToKnex {
                                 statementValue = !_.isArray(statement.value) ? [statement.value] : statement.value;
                             }
 
-                            innerQB[statement.whereType](statementColumn, statementOp, statementValue);
+                            self.applyComparison(innerQB, statement.whereType, statementColumn, statement.jsonPath, statementOp, statementValue);
                         });
 
                         if (debugExtended.enabled) {
@@ -530,7 +614,7 @@ class MongoToKnex {
                                 statementValue = !_.isArray(statement.value) ? [statement.value] : statement.value;
                             }
 
-                            innerQB[statement.whereType](statementColumn, statementOp, statementValue);
+                            self.applyComparison(innerQB, statement.whereType, statementColumn, statement.jsonPath, statementOp, statementValue);
                         });
 
                         if (debugExtended.enabled) {
@@ -736,7 +820,7 @@ class MongoToKnex {
         }
 
         debug(`(buildComparison) whereType: ${whereType}, statement: ${statement}, op: ${op}, comp: ${comp}, value: ${value}`);
-        qb[whereType](column, comp, value);
+        this.applyComparison(qb, whereType, column, processedStatement.jsonPath, comp, value);
     }
 
     /**
@@ -758,14 +842,67 @@ class MongoToKnex {
         //       (unknown operators on aggregate relations were already rejected by
         //       the validateAggregateStatements pre-pass in processJSON)
         _.forIn(sub, (value, op) => {
-            if (isCompOp(op)) {
+            if (op === '$elemMatch') {
+                this.buildElemMatch(qb, mode, statement, value, group);
+            } else if (isCompOp(op)) {
                 this.buildComparison(qb, mode, statement, op, value, group);
             } else {
                 debug('unknown operator');
             }
         });
     }
 
+    /**
+     * `{relation: {$elemMatch: {col: value, otherCol: {$ne: x}}}}`
+     *
+     * Match a single related row against all of the given conditions at once,
+     * emitted as one correlated subquery (`parent.id IN (SELECT … WHERE cond AND
+     * cond …)`). Without it, each condition on a multi-row relation is an
+     * independent existence check - a negation in particular becomes its own
+     * `NOT IN`, so a discriminator+value pair like `key = 'company' AND value != 'x'`
+     * would match different rows. `$elemMatch` is the explicit way to say the whole
+     * group describes one row; everything outside it keeps the default per-condition
+     * grouping untouched.
+     */
+    buildElemMatch(qb, mode, relationName, conditions, group) {
+        const collector = {};
+        const elemMatchGroup = (this.elemMatchSeq = (this.elemMatchSeq || 0) + 1);
+
+        // Inner conditions are always ANDed - a single row satisfies all of them -
+        // so process them as an $and regardless of the mode the match itself sits in.
+        _.forIn(conditions, (conditionValue, conditionColumn) => {
+            this.buildWhereClause(collector, '$and', `${relationName}.${conditionColumn}`, conditionValue, true);
+        });
+
+        const statements = collector.relations;
+        if (!statements || !statements.length) {
+            return;
+        }
+
+        statements.forEach((statement) => {
+            statement.elemMatchGroup = elemMatchGroup;
+        });
+
+        // The subquery itself attaches to the outer query with the outer mode's
+        // conjunction; grouping reads that from the group's first statement.
+        if (mode === '$or') {
+            statements[0].whereType = 'orWhere';
+        }
+
+        // CASE: not part of an outer group - attach the subquery immediately.
+        if (!group) {
+            this.buildRelationQuery(qb, statements, mode);
+            return;
+        }
+
+        // CASE: part of a group - hand the statements to the group's relation flush
+        //       so the subquery composes with sibling relation filters.
+        if (!Object.prototype.hasOwnProperty.call(qb, 'relations')) {
+            qb.relations = [];
+        }
+        qb.relations.push(...statements);
+    }
+
     /**
      * {$and: [{author: 'carl'}, {status: 'draft'}]}}
      * {$and: {author: 'carl'}}

packages/mongo-knex/test/integration/relations.test.js

@@ -1095,6 +1095,37 @@ describe('Relations', function () {
         });
     });
 
+    // These execute against the real database (sqlite3 AND mysql8 in CI), which is
+    // the point: MySQL's `json_extract` returns a scalar string *with* its quotes
+    // (`"London"`), so an anchored LIKE would match the quote, not the value. The
+    // converter uses `->>` to unquote, and only running the query on MySQL proves it.
+    // A .toQuery() string assertion can't - the SQL is identical on both dialects.
+    describe('JSON path extraction', function () {
+        it('matches an equal JSON subfield value', function () {
+            return makeQuery({'posts_meta.meta_json.city': 'London'})
+                .select()
+                .then(result => result.should.matchIds([1]));
+        });
+
+        it('matches a startsWith on a JSON subfield', function () {
+            return makeQuery({'posts_meta.meta_json.city': {$regex: /^Lon/}})
+                .select()
+                .then(result => result.should.matchIds([1]));
+        });
+
+        it('matches an endsWith on a JSON subfield', function () {
+            return makeQuery({'posts_meta.meta_json.city': {$regex: /ton$/}})
+                .select()
+                .then(result => result.should.matchIds([4]));
+        });
+
+        it('matches a case-insensitive contains on a JSON subfield', function () {
+            return makeQuery({'posts_meta.meta_json.city': {$regex: /berlin/i}})
+                .select()
+                .then(result => result.should.matchIds([5]));
+        });
+    });
+
     describe('One-to-One', function () {
         describe('EQUALS $eq', function () {
             it('posts_meta.meta_title equals "Meta of A Whole New World"', function () {

packages/mongo-knex/test/integration/suite1/fixtures/base.json

@@ -138,21 +138,24 @@
             "post_id": 1,
             "meta_title": "Meta of A Whole New World",
             "meta_description": "A whole new world with new horizons to pursue I'll chase them anywhere",
-            "like_count": 10
+            "like_count": 10,
+            "meta_json": "{\"city\":\"London\"}"
         },
         {
             "id": 2,
             "post_id": 4,
             "meta_title": "Meta of Circle of Life",
             "meta_description": "Till we find our place nn the path unwinding in the circle the circle of life.",
-            "like_count": 42
+            "like_count": 42,
+            "meta_json": "{\"city\":\"Boston\"}"
         },
         {
             "id": 3,
             "post_id": 5,
             "meta_title": "Meta of Be Our Guest",
             "meta_description": null,
-            "like_count": 42
+            "like_count": 42,
+            "meta_json": "{\"city\":\"Berlin\"}"
         }
     ]
 }

packages/mongo-knex/test/integration/suite1/schema.js

@@ -34,6 +34,7 @@ module.exports.up = function (knex) {
             table.string('meta_description', 2000).nullable();
             table.string('email_subject', 300).nullable();
             table.integer('like_count').unsigned();
+            table.json('meta_json').nullable();
         }))
         .then(() => knex.schema.createTable('tags', (table) => {
             table.increments('id').primary();