unified: Switch over to using swift-syntax for parsing

unified/extractor/ast_types.yml

@@ -31,13 +31,19 @@ supertypes:
     - throw_expr
     - try_expr
     - switch_expr
+    - unresolved_operator_sequence
     - unsupported_node
   expr_or_pattern:
     - expr
     - pattern
   expr_or_type:
     - expr
     - type_expr
+  # An element of an `unresolved_operator_sequence`: either an operand (`expr`)
+  # or one of the infix operators separating the operands.
+  expr_or_operator:
+    - expr
+    - infix_operator
   pattern:
     - name_pattern
     - tuple_pattern
@@ -137,6 +143,19 @@ named:
     operand: expr
     operator: operator
 
+  # A flat, unresolved operator sequence such as `a <+> b <+> c`.
+  #
+  # Swift's grammar doesn't encode operator precedence, so an operator chain is
+  # first parsed as a flat list of operands and operators. The parser front-end
+  # resolves this into structured `binary_expr` trees when it knows the
+  # operators' precedence (standard-library operators, and operators declared in
+  # the same file). When it encounters an operator whose precedence it can't
+  # determine (e.g. one imported from another module), it leaves that chain
+  # unresolved and emits it here rather than guessing a (possibly wrong)
+  # structure. The `element`s alternate operands (`expr`) and infix operators.
+  unresolved_operator_sequence:
+    element*: expr_or_operator
+
   # Plain assignment
   assign_expr:
     target: expr_or_pattern

unified/extractor/src/languages/mod.rs

@@ -12,6 +12,17 @@ mod swift;
 #[allow(dead_code)]
 pub mod swift_adapter;
 
+/// Swift front-end parser: shells out to `swift-syntax-parse` and adapts its
+/// JSON output via [`swift_adapter`].
+///
+/// Dormant for now: the runtime Swift front-end is still tree-sitter, so
+/// nothing in the binary calls this yet. `allow(dead_code)` for the same
+/// binary-crate reason as [`swift_adapter`]; both allows are removed once the
+/// runtime switches the Swift front-end to swift-syntax.
+#[path = "swift/parse.rs"]
+#[allow(dead_code)]
+pub mod swift_parse;
+
 /// Shared YEAST output AST schema for all languages.
 pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");
 

unified/extractor/src/languages/swift/adapter.rs

@@ -2,9 +2,8 @@
 //! in-memory format the CodeQL desugaring rules operate on.
 //!
 //! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
-//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
-//! so the extractor consumes swift-syntax output without pulling in the Swift
-//! toolchain (the JSON is produced out-of-process).
+//! (`parse_to_json`). This module needs no Swift toolchain, so the extractor
+//! consumes swift-syntax output out-of-process.
 //!
 //! The mapping mirrors tree-sitter's node model, which is what yeast (and the
 //! extractor's rewrite rules) expect:
@@ -24,31 +23,19 @@
 
 use std::collections::BTreeMap;
 
+use codeql_extractor::extractor::ExtraToken;
 use serde_json::Value;
-use yeast::schema::Schema;
 use yeast::{Ast, Id, NodeContent, Point, Range};
 
-/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
+/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
+/// comment/`unexpectedText` [`ExtraToken`]s harvested from it (in source order).
 ///
-/// These are collected into a side channel rather than embedded in the
-/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
+/// The extra tokens are collected into a side channel rather than embedded in
+/// the [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
 /// nodes: they carry a location and text but are not attached to a parent.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct TriviaToken {
-    /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
-    /// `docBlockComment`, `unexpectedText`).
-    pub kind: String,
-    /// The verbatim source text of the piece (e.g. `// comment`).
-    pub text: String,
-    /// The source range the piece occupies.
-    pub range: Range,
-}
-
-/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
-/// comment/`unexpectedText` trivia harvested from it (in source order).
 pub struct AdaptedTree {
     pub ast: Ast,
-    pub trivia: Vec<TriviaToken>,
+    pub extras: Vec<ExtraToken>,
 }
 
 /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
@@ -170,17 +157,18 @@ fn children_of(value: &Value) -> Vec<&Value> {
 /// in the schema on the fly, immediately before the node is created. Children
 /// are built first so a parent's field lists reference existing ids. Any
 /// comment/`unexpectedText` trivia carried by a token is harvested into
-/// `trivia` during the same pass rather than embedded in the tree.
-fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<Id, String> {
+/// `extras` (as [`ExtraToken`]s) during the same pass rather than embedded in
+/// the tree.
+fn build(node: &Value, ast: &mut Ast, extras: &mut Vec<ExtraToken>) -> Result<Id, String> {
     let info = classify(node)?;
-    collect_trivia(node, trivia);
+    collect_extras(node, extras);
 
     let mut fields: BTreeMap<u16, Vec<Id>> = BTreeMap::new();
     for (field, value) in field_entries(node) {
         let field_id = ast.register_field(field);
         let mut ids = Vec::new();
         for child in children_of(value) {
-            ids.push(build(child, ast, trivia)?);
+            ids.push(build(child, ast, extras)?);
         }
         fields.insert(field_id, ids);
     }
@@ -201,9 +189,10 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<I
 }
 
 /// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
-/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
-/// have no trivia keys, so this is a no-op for them.
-fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
+/// filtered to comments/`unexpectedText` upstream) into `out` as
+/// [`ExtraToken`]s. Non-token nodes have no trivia keys, so this is a no-op for
+/// them.
+fn collect_extras(node: &Value, out: &mut Vec<ExtraToken>) {
     for key in ["leadingTrivia", "trailingTrivia"] {
         let Some(Value::Array(pieces)) = node.get(key) else {
             continue;
@@ -220,15 +209,30 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
                 .and_then(Value::as_str)
                 .unwrap_or("")
                 .to_string();
-            out.push(TriviaToken {
-                kind: kind.to_string(),
+            out.push(ExtraToken {
+                kind: trivia_kind_id(kind),
                 text,
                 range,
             });
         }
     }
 }
 
+/// Map a swift-syntax trivia kind name to the stable integer id stored in an
+/// [`ExtraToken`]'s `kind` (and written to the `unified_trivia_tokeninfo`
+/// table). The value is opaque to the QL library (which reads only the text),
+/// but is kept stable and meaningful.
+fn trivia_kind_id(kind: &str) -> usize {
+    match kind {
+        "lineComment" => 1,
+        "blockComment" => 2,
+        "docLineComment" => 3,
+        "docBlockComment" => 4,
+        "unexpectedText" => 5,
+        _ => 0,
+    }
+}
+
 /// Parse a node's `range` into a [`yeast::Range`].
 ///
 /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
@@ -258,21 +262,29 @@ fn parse_range(node: &Value) -> Option<Range> {
     })
 }
 
+/// The authoritative swift-syntax input node-types schema, generated from
+/// swift-syntax (see the schemagen tool). [`json_to_ast`] seeds every parse
+/// with the schema built from this, pre-registering every input kind and field
+/// so rule matching never references a name absent from a given file's tree.
+const SWIFT_NODE_TYPES: &str = include_str!("../../../swift_node_types.yml");
+
 /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
 /// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
-/// from it. Both are produced in a single traversal.
+/// from it. Both are produced in a single traversal. The AST is seeded with the
+/// authoritative swift-syntax schema ([`SWIFT_NODE_TYPES`]); the adapter only
+/// ever consumes swift-syntax input, so the schema is not a parameter.
 pub fn json_to_ast(json: &str) -> Result<AdaptedTree, String> {
     let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?;
 
-    let mut ast = Ast::with_schema(Schema::new());
-    let mut trivia = Vec::new();
-    let root_id = build(&root, &mut ast, &mut trivia)?;
+    let mut ast = Ast::with_schema(yeast::node_types_yaml::schema_from_yaml(SWIFT_NODE_TYPES)?);
+    let mut extras = Vec::new();
+    let root_id = build(&root, &mut ast, &mut extras)?;
     ast.set_root(root_id);
 
-    // Emit trivia in source order (the traversal visits nodes bottom-up).
-    trivia.sort_by_key(|t| t.range.start_byte);
+    // Emit extras in source order (the traversal visits nodes bottom-up).
+    extras.sort_by_key(|t| t.range.start_byte);
 
-    Ok(AdaptedTree { ast, trivia })
+    Ok(AdaptedTree { ast, extras })
 }
 
 #[cfg(test)]
@@ -373,7 +385,7 @@ mod tests {
     }
 
     #[test]
-    fn collects_trivia_into_side_channel() {
+    fn collects_extras_into_side_channel() {
         // A token carrying a trailing line comment in its trivia.
         let json = r#"{
             "kind": "sourceFile",
@@ -395,9 +407,10 @@ mod tests {
         let adapted = json_to_ast(json).expect("adapter should succeed");
 
         // The comment is in the side channel, with its text and location.
-        assert_eq!(adapted.trivia.len(), 1);
-        let comment = &adapted.trivia[0];
-        assert_eq!(comment.kind, "lineComment");
+        assert_eq!(adapted.extras.len(), 1);
+        let comment = &adapted.extras[0];
+        // `lineComment` maps to extra kind id 1.
+        assert_eq!(comment.kind, 1);
         assert_eq!(comment.text, "// c");
         assert_eq!(comment.range.start_byte, 2);
         assert_eq!(comment.range.end_byte, 6);

unified/extractor/src/languages/swift/parse.rs

@@ -0,0 +1,73 @@
+//! Swift front-end parser: shells out to the separate `swift-syntax-parse`
+//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts
+//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
+//!
+//! Running the parser in a separate process keeps the Swift toolchain out of
+//! the extractor's own build: the extractor never links Swift, so working on
+//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call
+//! spawns the parser afresh; a longer-lived parser process could be swapped in
+//! behind this same seam later without touching the extraction pipeline.
+
+use std::io::Write;
+use std::process::{Command, Stdio};
+
+use codeql_extractor::extractor::ParsedTree;
+
+use super::swift_adapter;
+
+/// Environment variable naming the `swift-syntax-parse` executable. When unset,
+/// `swift-syntax-parse` is looked up on `PATH`.
+const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE";
+
+/// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus
+/// side-channel `extra` tokens), ready to be desugared via `run_from_ast`.
+pub fn parse(source: &[u8]) -> Result<ParsedTree, String> {
+    let source =
+        std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?;
+    let json = run_parser(source)?;
+    let mut adapted = swift_adapter::json_to_ast(&json)?;
+    adapted.ast.set_source(source.as_bytes().to_vec());
+    Ok(ParsedTree {
+        ast: adapted.ast,
+        extras: adapted.extras,
+    })
+}
+
+/// The `swift-syntax-parse` executable to invoke.
+fn parse_bin() -> String {
+    std::env::var(PARSE_BIN_ENV).unwrap_or_else(|_| "swift-syntax-parse".to_string())
+}
+
+/// Run the external parser, feeding `source` on stdin and returning its JSON
+/// stdout.
+fn run_parser(source: &str) -> Result<String, String> {
+    let bin = parse_bin();
+    let mut child = Command::new(&bin)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?;
+
+    // The parser reads all of stdin before writing any stdout, so writing the
+    // whole source and then closing stdin (by dropping it) cannot deadlock.
+    child
+        .stdin
+        .take()
+        .expect("child stdin was piped")
+        .write_all(source.as_bytes())
+        .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?;
+
+    let output = child
+        .wait_with_output()
+        .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?;
+    if !output.status.success() {
+        return Err(format!(
+            "Swift parser `{bin}` failed ({}): {}",
+            output.status,
+            String::from_utf8_lossy(&output.stderr).trim()
+        ));
+    }
+    String::from_utf8(output.stdout)
+        .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}"))
+}