From 35c1eaefc72a9e4fec56a3d9c4689a1f488c13f7 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Fri, 30 May 2025 00:23:25 +0000 Subject: [PATCH 01/87] implement the language definition DSL Strongly inspired by the `define-language` macro from Scheme's nanopass framework. --- nanopass/README.md | 2 + nanopass/nanopass.nim | 445 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 nanopass/README.md create mode 100644 nanopass/nanopass.nim diff --git a/nanopass/README.md b/nanopass/README.md new file mode 100644 index 00000000..95bfee28 --- /dev/null +++ b/nanopass/README.md @@ -0,0 +1,2 @@ +This directory contains the implementation of the nanopass framework used by +various parts of Phy. diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim new file mode 100644 index 00000000..b417b449 --- /dev/null +++ b/nanopass/nanopass.nim @@ -0,0 +1,445 @@ +## Implements the nanopass framework, which is a collection of macro DSLs for +## defining intermediate languages (their syntax and grammar) and passes. + +# TODO: +# * implement the DSL for pass definitions +# * add a "compiler definition" macro + +import + std/[macros, sets, strformat, tables] + +type + Elem = object + mvar: string + typ: string + ## the actual type + repeat: bool + + Form = object + tag: string + elems: seq[Elem] + + NamedForm = object + names: seq[string] + ## gives a name to each element of the form + form: int + ## reference to the form + + NonTerminal = object + mvars: seq[string] + ## the meta-variables for ranging over the productions + vars: seq[string] + ## meta-variables used as productions + forms: seq[NamedForm] + ## forms used as productions + + LangDef = object + terminals: Table[string, string] + ## name -> type. The terminals of the language + nterminals: Table[string, NonTerminal] + ## the non-terminals of the language + forms: seq[Form] + ## all forms used in productions + tags: Table[string, uint8] + ## associates an integer ID with each tag + + NonTerminalDef = object + ## Pre-processed non-terminal definition. + name: NimNode + sub: seq[NimNode] + add: seq[NimNode] + +template findIt[T](s: seq[T], predicate: untyped): untyped = + ## Version of ``find`` that allows providing an inline predicate, + ## evaluated for every checked item. + var r = -1 + block: + for i, it {.inject.} in s.pairs: + if predicate: + r = i + r + +proc `==`(x, y: Elem): bool = + x.mvar == y.mvar and x.repeat == y.repeat + +proc `==`(x, y: Form): bool = + ## Compares `x` and `y`, which must belong to the same language, for equality. + x.tag == y.tag and x.elems == y.elems + +proc checkName(target: LangDef, vars: Table[string, string], name: string, + info: NimNode) = + if name in target.terminals: + error(fmt"terminal with name {name} already exists", info) + elif name in target.nterminals: + error(fmt"non-terminal with name {name} already exists", info) + elif name in vars: + error(fmt"'{name}' is already used by a meta-variable for '{vars[name]}'", info) + +proc parseForm(vars: Table[string, string], n: NimNode): (seq[string], Form) = + n[0].expectKind nnkIdent + var form = Form(tag: n[0].strVal) + + for i in 1..= 0 and name[e] in {'0'..'9'}: + dec e + + if e >= 0 and name[e] == '_': + dec e + + elem.mvar = name[0..e] + if elem.mvar notin vars: + error(fmt"no meta-var with name '{elem.mvar}' exists", it) + + elem.typ = vars[elem.mvar] + form.elems.add elem + result[0].add name + result[1] = form + +proc parseRawForm(n: NimNode): Form = + ## Parses a raw form (a form without name information) from the given AST. + n[0].expectKind nnkIdent + result.tag = n[0].strVal + + for i in 1.. 0: + if name notin base.nterminals: + error(fmt"base language has no non-terminal with name '{name}'", + it.name) + + for prod in it.sub.items: + removeProd(base, prod, name) + + if it.add.len == 0 and + base.nterminals[name].vars.len == 0 and + base.nterminals[name].forms.len == 0: + # the non-terminal is and will stay empty, remove it + base.nterminals.del(name) + + if name in base.nterminals: + # remove the old names: + base.nterminals[name].mvars.shrink(0) + + # update the var list with the to-be-inherited non-terminal meta-vars: + for name, it in base.nterminals.pairs: + for n in it.mvars.items: + vars[n] = name + + # only now set the new meta-variable names for inherited non-terminals: + for it in def.items: + let name = it.name[0].strVal + if name in base.nterminals: + for i in 1.. 0 and name notin base.nterminals: + # it's a new non-terminal + checkName(result, vars, name, it.name) + var nt = NonTerminal() + for i in 1.. 0: + for a in it.add.items: + addProd(result, a, name) + + # TODO: implement tag ID computation for terminals + # TODO: re-use tag IDs from the base language + computeTags(result) + +proc makeLanguage(body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code. + body.expectMinLen 1 + var add: seq[NimNode] + var def: seq[NonTerminalDef] + + # second pass: process the productions + proc extract(n: NimNode, list: var seq[NimNode]) = + case n.kind + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], list) + extract(n[2], list) + else: + error("expected '|'", n[0]) + of nnkCall, nnkIdent: + list.add n + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add) + def.add nt + continue + of nnkCall: + add.add it + continue + else: + discard "report an error below" + + error("items must be of the form `a in b`, or `a ::= ...`", it) + + # to keep the implementation simple, a non-extension language is treated + # internally as an empty language definition being extended + buildLanguage(add, @[], def, default(LangDef), body) + +proc makeLanguage(base: LangDef, body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code + ## and `base`. + var add, sub: seq[NimNode] + var def: seq[NonTerminalDef] + + proc extract(n: NimNode, add, sub: var seq[NimNode]) = + case n.kind + of nnkPrefix: + if n[0].eqIdent("+"): + add.add n[1] + elif n[0].eqIdent("-"): + sub.add n[1] + else: + error("expected `+` or `-`", n[0]) + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], add, sub) + extract(n[2], add, sub) + else: + error("expected '|'", n[0]) + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + var handled = false + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + it.expectLen 3 + var nt = NonTerminalDef(name: it[1]) + nt.name.expectKind nnkCall + extract(it[2], nt.add, nt.sub) + def.add nt + handled = true + of nnkPrefix: + if it[0].eqIdent("-"): + sub.add it[1] + handled = true + elif it[0].eqIdent("+"): + add.add it[1] + handled = true + of nnkCall: + # non-terminal with no change in productions + def.add NonTerminalDef(name: it) + handled = true + else: + discard + + if not handled: + error("expected `-a`, `+a`, or `a(...) ::= ...`", it[0]) + + buildLanguage(add, sub, def, base, body) + +macro defineLanguage*(name, body: untyped) = + ## Creates a language definitions and binds it to a const symbol with the + ## given name. + body.expectKind nnkStmtList + body.expectMinLen 1 + let p = bindSym"makeLanguage" + let q = bindSym"quote" + if body[0].kind == nnkCommentStmt: + let filtered = body[1..^1] + result = nnkConstSection.newTree( + nnkConstDef.newTree(name, + newEmptyNode(), + quote do: `p`(`q` do: `filtered`)), + body[0]) + else: + result = nnkConstSection.newTree( + nnkConstDef.newTree(name, + newEmptyNode(), + quote do: `p`(`q` do: `body`))) + +macro defineLanguage*(name, base, body: untyped) = + ## Creates a language definition, extending `base`, and binds it to a const + ## symbol with the given name. Extension doesn't imply a direction in this + ## context. + body.expectKind nnkStmtList + body.expectMinLen 1 + let p = bindSym"makeLanguage" + let q = bindSym"quote" + if body[0].kind == nnkCommentStmt: + let filtered = nnkStmtList.newTree(body[1..^1]) + result = nnkConstSection.newTree( + nnkConstDef.newTree(name, + newEmptyNode(), + quote do: `p`(`base`, `q` do: `filtered`)), + body[0]) + else: + result = nnkConstSection.newTree( + nnkConstDef.newTree(name, + newEmptyNode(), + quote do: `p`(`base`, `q` do: `body`))) From 8602a880c6db599b1bce741055b5ffac4f860306 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Fri, 30 May 2025 00:23:26 +0000 Subject: [PATCH 02/87] start with writing some language definitions Also sketch out how the pass definition macros could look like. --- passes/passes.nim | 184 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 passes/passes.nim diff --git a/passes/passes.nim b/passes/passes.nim new file mode 100644 index 00000000..f6b8e70e --- /dev/null +++ b/passes/passes.nim @@ -0,0 +1,184 @@ +## The home of all intermediate languages and passes, representing the core of +## the compiler. + +import + nanopass/nanopass, + experimental/sexp_parse + +defineLanguage Lsrc: + n(int) + fl(float) + str(string) + # TODO: only allow a type being used as a terminal once + x(string) # identifier + + rec_field(rf) ::= Field(x, e) + field_decl(f) ::= Field(x, t) + + pattern(p) ::= As(x, t) + mrule(mr) ::= Rule(p, e) + expr(e) ::= n | fl | x | + ArrayCons(...e) | + TupleCons(...e) | + RecordCons(rf0, ...rf1) | + Seq(t, ...e) | + Seq(str) | + Call(e, ...e) | + FieldAccess(e, n) | + FieldAccess(e, x) | + At(e0, e1) | + As(e, t) | + And(e0, e1) | + Or(e0, e1) | + If(e0, e1) | + If(e0, e1, e2) | + While(e0, e1) | + Return(e) | + Unreachable() | + Exprs(...e0, e1) | + Asgn(e0, e1) | + Decl(x, e) | + Match(e, mr0, ...mr1) + typ(t) ::= x | VoidTy() | UnitTy() | BoolTy() | IntTy() | FloatTy() | + ArrayTy(n, t) | + SeqTy(t) | + TupleTy(t0, ...t1) | + RecordTy(f0, ...f1) | + UnionTy(t0, ...t1) | + ProcTy(t0, ...t1) + + param_decl(pd) ::= ParamDecl(x, t) + params(pa) ::= Params(...pd) + decl(d) ::= ProcDecl(x, t, pa, e) | TypeDecl(x, t) + module(m) ::= Module(...d) + +defineLanguage L1, Lsrc: + ## Language without And and Or. + expr(e) ::= -And(e, e) | -Or(e, e) + +defineLanguage L2, L1: + ## Language without single-branch `If`. + expr(e) ::= -If(e, e) + +defineLanguage L3, L2: + ## Language that replaces Decl with Let. + expr(e) ::= -Decl(x, e) | +Let(x, e, e) + +defineLanguage L4, L3: + ## Removes the `Match` form. + pattern(p) ::= -As(x, t) + mrule(mr) ::= -Rule(p, e) + expr(e) ::= -Match(e, mr, ...mr) + +defineLanguage L5, L4: + ## Removes the While form and adds the Loop and Break form. + expr(e) ::= -While(e, e) | +Loop(e) | +Break(n) + +defineLanguage L6, L5: + ## Requires explicit copies. + root(ro) ::= +lv | +e + lvalue(lv) ::= + +At(ro, e) | + +FieldAccess(ro, n) + expr(e) ::= + -At(e, e) | + -FieldAccess(e, n) | + +Copy(lv) + +proc parse(s: var SexpParser): Lsrc {.inpass.} = + ## Parses the source language from an S-expression stream. + proc ident(s: var SexpParser): dst.x {.transform.} = + s.space() + s.expect(tkIdent) + build x(s.eatString()) + + proc call(s: var SexpParser): dst.e {.transform.} = + let callee = expr(s) + var x = @[expr(s)] + s.space() + while s.currToken != tkParensRi: + expr(s) + s.space() + + s.eat(tkParensRi) + build Call(callee, x) + + proc expr(s: var SexpParser): dst.e {.transform.} = + s.space() + case s.currToken + of tkParsensLe: + let k = s.getTok() + if k == tkIdent: + case s.currString + of "if": + build If(expr(s), expr(s), expr(s)) + of "while": + build While(expr(s), expr(s)) + of "decl": + build Decl(ident(s), expr(s)) + of "and": + build And(expr(s), expr(s)) + of "or": + build Or(expr(s), expr(s)) + else: + # TODO: implement the remaining keywords + call(s) + else: + call(s) + + of tkString: + build str(s.eatString()) + of tkIdent: + build x(s.eatString()) + of tkInt: + build n(parseInt(s.eatString())) + of tkFloat: + build fl(parseFloat(s.eatString())) + else: + syntaxError(s) + + proc typ(s: var SexpParser): dst.t {.transform.} = + s.space() + # TODO: implement + + proc params(s: var SexprParser): dst.pa {.transform.} = + s.space() + # TODO: implement + + proc top(s: var SexpParser): dst.d {.transform.} = + s.eat(tkParensLe) + s.expect(tkIdent) + case s.currToken + of "proc": + build ProcDecl(ident(s), typ(s), params(s), expr(s)) + of "type": + build TypeDecl(ident(s), typ(s)) + else: + syntaxError() + +proc removeAndOr(_: Lsrc): L1 {.pass.} = + proc expr(_: src.e): dst.e {.transform.} = + And([a], [b]) -> build If(a, b, x("false")) + Or([a], [b]) -> build If(a, x("true"), b) + +proc removeSingleIf(_: L1): L2 {.pass.} = + proc expr(_: src.e): dst.e {.transform.} = + If([a], [b]) -> build If(a, b, TupleCons()) + +proc declToLet(_: L2): L3 {.pass.} = + proc expr(_: src.e): dst.e {.transform.} = + Decl(`x`, `e`) -> build Let(x, e, TupleCons()) + Exprs(`e0`, [last]): + var r = last + var got: seq[dst.e] + for i in countdown(e0.high, 0): + match e0[i]: + Decl(`x`, `e`): + r = + if got.len == 0: build Let(x, e, r) + else: build Let(x, e, Exprs(got, r)) + else: + got.insert expr(e0[i]) + + if got.len == 0: build Exprs(got, r) + else: r From b621ac1af229e92be2211a361f3a6537eda90da6 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 3 Jun 2025 21:27:41 +0000 Subject: [PATCH 03/87] nanopass: disallow duplicate element names --- nanopass/nanopass.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index b417b449..43149a34 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -96,6 +96,9 @@ proc parseForm(vars: Table[string, string], n: NimNode): (seq[string], Form) = if e >= 0 and name[e] == '_': dec e + if name in result[0]: + error(fmt"duplicate use of '{name}'; elements need a unique name", it) + elem.mvar = name[0..e] if elem.mvar notin vars: error(fmt"no meta-var with name '{elem.mvar}' exists", it) From b68098242443689e1f75c7724771f0d1ed2d1445 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 3 Jun 2025 21:27:42 +0000 Subject: [PATCH 04/87] passes: fix duplicate elements names --- passes/passes.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/passes/passes.nim b/passes/passes.nim index f6b8e70e..5e004296 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -23,7 +23,7 @@ defineLanguage Lsrc: RecordCons(rf0, ...rf1) | Seq(t, ...e) | Seq(str) | - Call(e, ...e) | + Call(e0, ...e1) | FieldAccess(e, n) | FieldAccess(e, x) | At(e0, e1) | @@ -62,7 +62,7 @@ defineLanguage L2, L1: defineLanguage L3, L2: ## Language that replaces Decl with Let. - expr(e) ::= -Decl(x, e) | +Let(x, e, e) + expr(e) ::= -Decl(x, e) | +Let(x, e1, e2) defineLanguage L4, L3: ## Removes the `Match` form. From 3019584923bcbea834e9818b0c8f11743ea3220d Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 3 Jun 2025 23:52:54 +0200 Subject: [PATCH 05/87] nanopass: split type sections Co-authored-by: Saem Ghani --- nanopass/nanopass.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 43149a34..69f4e530 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -9,6 +9,7 @@ import std/[macros, sets, strformat, tables] type + # Core types capturing a defined language Elem = object mvar: string typ: string @@ -43,6 +44,8 @@ type tags: Table[string, uint8] ## associates an integer ID with each tag +type + # Intermediate types meant to bridge macro language to core types NonTerminalDef = object ## Pre-processed non-terminal definition. name: NimNode From 7a3fc69259b382a690b5f38e65fcad3d9b8e0ed6 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 5 Jul 2025 00:00:56 +0000 Subject: [PATCH 06/87] nanopass: don't consider metavar names for form equality `Form(e, e)` and `Form(e, b)` are now considered equal when `e` and `b` both range over the same non-terminal. The purpose of meta-variables is to range over types (providing shorthands, more or less) -- they must not introduce new distinct-esque types (non-terminals are effectively types). Some naming and documentation of types and fields is improved/changed, too. --- nanopass/nanopass.nim | 141 +++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 69f4e530..04ddb9f3 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -11,41 +11,50 @@ import type # Core types capturing a defined language Elem = object - mvar: string + ## Element of a form. typ: string ## the actual type repeat: bool Form = object - tag: string + ## Semantic representation of a syntax form. + tag: string # TODO: rename to name elems: seq[Elem] - NamedForm = object - names: seq[string] - ## gives a name to each element of the form - form: int - ## reference to the form + OrigForm = object + ## Source-level-ish representation of a syntax form description. + vars: seq[string] + ## the metavars-as-written for the form's elements + semantic: int + ## index of the semantic representation NonTerminal = object mvars: seq[string] ## the meta-variables for ranging over the productions vars: seq[string] ## meta-variables used as productions - forms: seq[NamedForm] + forms: seq[OrigForm] ## forms used as productions LangDef = object + ## A checked and pre-processed language definition, carrying enough + ## source-level information necessary for implementing, e.g., inheritance. terminals: Table[string, string] - ## name -> type. The terminals of the language + ## the terminals of the language nterminals: Table[string, NonTerminal] ## the non-terminals of the language forms: seq[Form] - ## all forms used in productions + ## all syntax forms present in the language tags: Table[string, uint8] ## associates an integer ID with each tag type # Intermediate types meant to bridge macro language to core types + ParsedForm = object + ## A syntax form description as parsed from NimNode AST. + name: string + elems: seq[tuple[mvar: string, repeat: bool, info: NimNode]] + NonTerminalDef = object ## Pre-processed non-terminal definition. name: NimNode @@ -63,7 +72,7 @@ template findIt[T](s: seq[T], predicate: untyped): untyped = r proc `==`(x, y: Elem): bool = - x.mvar == y.mvar and x.repeat == y.repeat + x.typ == y.typ and x.repeat == y.repeat proc `==`(x, y: Form): bool = ## Compares `x` and `y`, which must belong to the same language, for equality. @@ -78,54 +87,20 @@ proc checkName(target: LangDef, vars: Table[string, string], name: string, elif name in vars: error(fmt"'{name}' is already used by a meta-variable for '{vars[name]}'", info) -proc parseForm(vars: Table[string, string], n: NimNode): (seq[string], Form) = - n[0].expectKind nnkIdent - var form = Form(tag: n[0].strVal) - - for i in 1..= 0 and name[e] in {'0'..'9'}: - dec e - - if e >= 0 and name[e] == '_': - dec e - - if name in result[0]: - error(fmt"duplicate use of '{name}'; elements need a unique name", it) - - elem.mvar = name[0..e] - if elem.mvar notin vars: - error(fmt"no meta-var with name '{elem.mvar}' exists", it) - - elem.typ = vars[elem.mvar] - form.elems.add elem - result[0].add name - result[1] = form - -proc parseRawForm(n: NimNode): Form = - ## Parses a raw form (a form without name information) from the given AST. +proc parseForm(n: NimNode): ParsedForm = + ## Parses a form in the context of a language definition. No semantic checks + ## take place. n[0].expectKind nnkIdent - result.tag = n[0].strVal - + result.name = n[0].strVal for i in 1.. Date: Sat, 5 Jul 2025 00:00:56 +0000 Subject: [PATCH 07/87] nanopass: properly implement node-tags Terminals now have node tags too and node tags are inherited properly. In addition, two forms with the same name but different shapes / element types use different node tags, making them trivial to distinguish in the internal AST representation. --- nanopass/nanopass.nim | 52 ++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 04ddb9f3..2fa85433 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -19,6 +19,10 @@ type Form = object ## Semantic representation of a syntax form. tag: string # TODO: rename to name + id: int + ## the integer ID through which a tree node is identified as being an + ## instance of the form + # TODO: rename to ntag ("node tag") elems: seq[Elem] OrigForm = object @@ -39,14 +43,12 @@ type LangDef = object ## A checked and pre-processed language definition, carrying enough ## source-level information necessary for implementing, e.g., inheritance. - terminals: Table[string, string] + terminals: Table[string, tuple[typ: string, tag: int]] ## the terminals of the language nterminals: Table[string, NonTerminal] ## the non-terminals of the language forms: seq[Form] ## all syntax forms present in the language - tags: Table[string, uint8] - ## associates an integer ID with each tag type # Intermediate types meant to bridge macro language to core types @@ -61,6 +63,12 @@ type sub: seq[NimNode] add: seq[NimNode] +const + RefTag = 128'u8 + ## the node used internally for indirections + FirstTerminalTag = RefTag + 1 + ## the start of the terminals' tag space + template findIt[T](s: seq[T], predicate: untyped): untyped = ## Version of ``find`` that allows providing an inline predicate, ## evaluated for every checked item. @@ -106,17 +114,26 @@ proc addForm(def: var LangDef, form: Form): int = def.forms.add form result = def.forms.high -proc computeTags(def: var LangDef) = - ## Populates the tag table. - var next: uint8 - for tag in def.tags.values: - next = max(tag + 1, next) +proc computeNodeTags(def: var LangDef) = + ## Assigns node tags to forms and terminals. + var next = 0 + for it in def.forms.items: + next = max(it.id + 1, next) # ^^ while simple, this does waste ID space - for it in def.forms.items: - if it.tag notin def.tags: - # TODO: handle overflow - def.tags[it.tag] = next + for it in def.forms.mitems: + # TODO: report an error when the ID overflows the allowed range + if it.id == -1: + it.id = next + inc next + + next = int FirstTerminalTag + for it in def.terminals.values: + next = max(it.tag + 1, next) + + for it in def.terminals.mvalues: + if it.tag == -1: + it.tag = next inc next proc buildLanguage(add, sub: seq[NimNode], @@ -143,7 +160,7 @@ proc buildLanguage(add, sub: seq[NimNode], # apply the terminal removals and carry over the remaining ones: for it in sub.items: let (name, typ) = processTerminal(it) - if name notin base.terminals or base.terminals[name] != typ: + if name notin base.terminals or base.terminals[name].typ != typ: error("terminal does not exist in the base language", it) base.terminals.del(name) @@ -254,12 +271,13 @@ proc buildLanguage(add, sub: seq[NimNode], for it in add.items: let (name, typ) = processTerminal(it) checkName(result, vars, name, it) - result.terminals[name] = typ + # the node tag is filled in later + result.terminals[name] = (typ, -1) vars[name] = name proc addProd(def: var LangDef, n: NimNode, to: string) = proc addForm(def: var LangDef, p: ParsedForm): OrigForm = - var form = Form(tag: p.name) # the ID is computed later + var form = Form(tag: p.name, id: -1) # the ID is computed later for i, (name, repeat, info) in p.elems.pairs: if name notin vars: @@ -308,9 +326,7 @@ proc buildLanguage(add, sub: seq[NimNode], for a in it.add.items: addProd(result, a, name) - # TODO: implement tag ID computation for terminals - # TODO: re-use tag IDs from the base language - computeTags(result) + computeNodeTags(result) proc makeLanguage(body: NimNode): LangDef = ## Creates a language definition from the ``defineLanguage`` DSL code. From 692afc01b5be20b3f61fdd672cf95d09d0a0d10c Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 5 Jul 2025 00:00:56 +0000 Subject: [PATCH 08/87] passes: implement a full set of ILs/passes This is just meant for testing/demonstration purposes. The languages and their order are not well thought out, nor are the various passes properly implemented. Still, it highlights some problems and missing things of the current framework. --- passes/passes.nim | 973 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 889 insertions(+), 84 deletions(-) diff --git a/passes/passes.nim b/passes/passes.nim index 5e004296..a3d577c6 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -2,50 +2,61 @@ ## the compiler. import + std/[ + streams, + strformat, + strutils, + tables + ], nanopass/nanopass, - experimental/sexp_parse + experimental/sexp_parse, + passes/trees, + phy/[reporting, default_reporting] + +type + Symbol = object + Ident = distinct string defineLanguage Lsrc: n(int) fl(float) str(string) - # TODO: only allow a type being used as a terminal once - x(string) # identifier + x(Ident) rec_field(rf) ::= Field(x, e) field_decl(f) ::= Field(x, t) pattern(p) ::= As(x, t) mrule(mr) ::= Rule(p, e) - expr(e) ::= n | fl | x | + expr(e) ::= n | fl | x | str | ArrayCons(...e) | TupleCons(...e) | - RecordCons(rf0, ...rf1) | + RecordCons(rf, ...rf) | Seq(t, ...e) | Seq(str) | - Call(e0, ...e1) | + Call(e, ...e) | FieldAccess(e, n) | FieldAccess(e, x) | - At(e0, e1) | + At(e, e) | As(e, t) | - And(e0, e1) | - Or(e0, e1) | - If(e0, e1) | - If(e0, e1, e2) | - While(e0, e1) | + And(e, e) | + Or(e, e) | + If(e, e) | + If(e, e, e) | + While(e, e) | Return(e) | Unreachable() | - Exprs(...e0, e1) | - Asgn(e0, e1) | + Exprs(...e, e) | + Asgn(e, e) | Decl(x, e) | - Match(e, mr0, ...mr1) - typ(t) ::= x | VoidTy() | UnitTy() | BoolTy() | IntTy() | FloatTy() | + Match(e, mr, ...mr) + typ(t) ::= x | VoidTy() | UnitTy() | BoolTy() | CharTy() | IntTy() | FloatTy() | ArrayTy(n, t) | SeqTy(t) | - TupleTy(t0, ...t1) | - RecordTy(f0, ...f1) | - UnionTy(t0, ...t1) | - ProcTy(t0, ...t1) + TupleTy(t, ...t) | + RecordTy(f, ...f) | + UnionTy(t, ...t) | + ProcTy(t, ...t) param_decl(pd) ::= ParamDecl(x, t) params(pa) ::= Params(...pd) @@ -62,123 +73,917 @@ defineLanguage L2, L1: defineLanguage L3, L2: ## Language that replaces Decl with Let. - expr(e) ::= -Decl(x, e) | +Let(x, e1, e2) + expr(e) ::= -Decl(x, e) | +Let(x, e, e) defineLanguage L4, L3: - ## Removes the `Match` form. - pattern(p) ::= -As(x, t) - mrule(mr) ::= -Rule(p, e) - expr(e) ::= -Match(e, mr, ...mr) + ## Language with symbols instead of raw identifiers. + +s(Symbol) + expr(e) ::= -x | +s | + -Let(x, e, e) | +Let(s, e, e) + typ(t) ::= -x | +s + pattern(p) ::= -As(x, t) | +As(t, s) # the element order is swapped + + param_decl(pd) ::= -ParamDecl(x, t) | +ParamDecl(s, t) + decl(d) ::= -ProcDecl(x, t, pa, e) | +ProcDecl(s, t, pa, e) | + -TypeDecl(x, t) | +TypeDecl(s, t) defineLanguage L5, L4: - ## Removes the While form and adds the Loop and Break form. - expr(e) ::= -While(e, e) | +Loop(e) | +Break(n) + ## Language with type information. + # expressions that always have the same type don't use a type tag + expr(e) ::= -ArrayCons(...e) | +ArrayCons(t, ...e) | + -TupleCons(...e) | +TupleCons(t, e, ...e) | + -RecordCons(rf, ...rf) | +RecordCons(t, rf, ...rf) | + -Seq(str) | +Seq(t, str) | + -Call(e, ...e) | +Call(t, e, ...e) | + -FieldAccess(e, n) | +FieldAccess(t, e, n) | + -FieldAccess(e, x) | +FieldAccess(t, e, x) | + -At(e, e) | +At(t, e, e) | + -As(e, t) | +As(t, e, t) | + -If(e, e, e) | +If(t, e, e, e) | + -Exprs(...e, e) | +Exprs(t, ...e, e) | + -Match(e, mr, ...mr) | +Match(t, e, mr, ...mr) | + +Unit() | + +Prim(t, str, ...e) # primitive calls (i.e., magic operations) + +defineLanguage Lnoas, L5: + ## Language with more specific forms instead of the generic `As`. + expr(e) ::= -As(t, e, t) | +Inj(t, e, t) + +defineLanguage L6, Lnoas: + ## Language with the `Match` from removed. + pattern(p) ::= -As(t, s) + mrule(mr) ::= -Rule(p, e) + expr(e) ::= -Match(t, e, mr, ...mr) | + +Is(t, e, t) | + +Unpack(t, e) + +defineLanguage L7, L6: + ## Language with no tagged unions, only untagged unions. + expr(e) ::= -Is(t, e, t) | -Unpack(t, e) + +defineLanguage L8, L7: + ## Language without records. + rec_field(rf) ::= -Field(x, e) + field_decl(f) ::= -Field(x, t) + expr(e) ::= -RecordCons(t, rf, ...rf) | -FieldAccess(t, e, x) + typ(t) ::= -RecordTy(f, ...f) -defineLanguage L6, L5: - ## Requires explicit copies. - root(ro) ::= +lv | +e +defineLanguage L9, L8: + ## Language where copies are explicit and all aggregate access has a named + ## local as the root. lvalue(lv) ::= - +At(ro, e) | - +FieldAccess(ro, n) + +s | + +At(t, lv, e) | + +FieldAccess(t, lv, n) expr(e) ::= - -At(e, e) | - -FieldAccess(e, n) | - +Copy(lv) + -s | + -At(t, e, e) | + -FieldAccess(t, e, n) | + +lv | + +Copy(t, lv) + +defineLanguage Lseq, L9: + ## Language with no built-in sequence values nor types. + # a "box" is a single-owner single value container + expr(e) ::= -Seq(t, ...e) | -Seq(t, str) | +Box(t, e) | +Unbox(t, e) + typ(t) ::= -SeqTy(t) | +BoxTy(t) + +defineLanguage Lnocopy, Lseq: + ## Language without `Copy` form. + expr(e) ::= -Copy(t, lv) + +defineLanguage Lnobox, Lnocopy: + expr(e) ::= -Box(t, e) | -Unbox(t, e) + typ(t) ::= -BoxTy(t) | +PtrTy() + +defineLanguage Lnocons, Lnobox: + ## Language without aggregate constructors. + init(ie) ::= +e | +Undef() + expr(e) ::= -Let(s, e, e) | +Let(s, ie, e) | + -ArrayCons(t, ...e) | + -TupleCons(t, e, ...e,) + +defineLanguage Lnoletwithval, Lnocons: + ## Language without Let initializers. + init(ie) ::= -e | -Undef() + expr(e) ::= -Let(s, ie, e) | +Let(s, e) + +defineLanguage Lstmt, Lnoletwithval: + ## Language with a statement/expression separation. + expr(e) ::= -Asgn(e, e) | + -Unreachable() | + -Return(e) | + -While(e, e) | + -Exprs(t, ...e, e) | + +Exprs(t, ...st, e) + stmt(st) ::= +Asgn(e, e) | + +Unreachable() | + +Return(e) | + +While(e, st) | + +Pass() | # statement without effect + +Call(t, e, ...e) | + +If(e, st, st) | +If(e, st) | + +Let(s, st) | + +Stmts(...st, st) + decl(d) ::= -ProcDecl(s, t, pa, e) | +ProcDecl(s, t, pa, st) + +defineLanguage L10, Lstmt: + ## Language with no If expressions. + expr(e) ::= -If(t, e, e, e) + +defineLanguage L11, L10: + ## Language with only blob and scalar types. + lvalue(lv) ::= -s | + -FieldAccess(t, lv, n) | + -At(t, lv, e) + expr(e) ::= -lv | + +s | + +Offset(t, e, e, n) | + +Load(t, e) | + +Addr(s) + stmt(st) ::= -Asgn(e, e) | + +Asgn(s, e) | + +Store(e, e) + +defineLanguage L12, L11: + ## Language with only simple callee expressions. + expr(e) ::= -Call(t, e, ...e) | +Call(t, s, ...e) + stmt(st) ::= -Call(t, e, ...e) | +Call(t, s, ...e) + +proc eatString(s: var SexpParser): string = + result = s.captureCurrString() + discard s.getTok() + +proc expect(s: SexpParser, kind: TTokKind) = + if s.currToken != kind: + # TODO: report a proper syntax error + raise ValueError.newException("expected " & $kind & ", got " & $s.currToken) + +proc eat(s: var SexpParser, kind: TTokKind) = + s.expect(kind) + discard s.getTok() proc parse(s: var SexpParser): Lsrc {.inpass.} = ## Parses the source language from an S-expression stream. + # note: the implementation is incomplete and incorrect (there's also no + # formal grammar backing it) + proc expr(s: var SexpParser): dst.e {.transform.} + proc typ(s: var SexpParser): dst.t {.transform.} + proc ident(s: var SexpParser): dst.x {.transform.} = s.space() - s.expect(tkIdent) - build x(s.eatString()) + s.expect(tkSymbol) + terminal Ident(s.eatString()) proc call(s: var SexpParser): dst.e {.transform.} = + # note: does not handle the parenthesis let callee = expr(s) - var x = @[expr(s)] + var args = newSeq[dst.e]() s.space() while s.currToken != tkParensRi: - expr(s) + args.add expr(s) s.space() + build Call(callee, args) + + proc pattern(s: var SexpParser): dst.p {.transform.} = + s.eat(tkParensLe) + s.expect(tkSymbol) + let name = s.eatString() + s.space() + let t = typ(s) s.eat(tkParensRi) - build Call(callee, x) + build As(x(^Ident(name)), t) + + proc rule(s: var SexpParser): dst.mr {.transform.} = + s.eat(tkParensLe) + s.eat(tkSymbol) + # TODO: make sure the symbol is "rule" + s.space() + let p = pattern(s) + s.space() + let body = expr(s) + s.eat(tkParensRi) + build Rule(p, body) proc expr(s: var SexpParser): dst.e {.transform.} = s.space() case s.currToken - of tkParsensLe: + of tkParensLe: let k = s.getTok() - if k == tkIdent: + if k == tkSymbol: case s.currString of "if": - build If(expr(s), expr(s), expr(s)) + discard s.getTok() + result = build If(^expr(s), ^expr(s), ^expr(s)) of "while": - build While(expr(s), expr(s)) + discard s.getTok() + result = build While(^expr(s), ^expr(s)) of "decl": - build Decl(ident(s), expr(s)) + discard s.getTok() + result = build Decl(^ident(s), ^expr(s)) of "and": - build And(expr(s), expr(s)) + discard s.getTok() + result = build And(^expr(s), ^expr(s)) of "or": - build Or(expr(s), expr(s)) + discard s.getTok() + result = build Or(^expr(s), ^expr(s)) + of "as": + discard s.getTok() + result = build As(^expr(s), ^typ(s)) + of "exprs": + discard s.getTok() + s.space() + var elems: seq[dst.e] + while s.currToken != tkParensRi: + elems.add expr(s) + s.space() + + let last = elems.pop() + result = build Exprs(elems, last) + of "match": + discard s.getTok() + let e = expr(s) + s.space() + let first = rule(s) + s.space() + var next: seq[dst.mr] + while s.currToken != tkParensRi: + next.add rule(s) + s.space() + result = build Match(e, first, next) else: # TODO: implement the remaining keywords - call(s) + result = call(s) else: - call(s) + result = call(s) + s.eat(tkParensRi) of tkString: - build str(s.eatString()) - of tkIdent: - build x(s.eatString()) + result = build(dst.e, str(^s.eatString())) + of tkSymbol: + result = build x(^Ident(s.eatString())) of tkInt: - build n(parseInt(s.eatString())) + result = build n(^parseInt(s.eatString())) of tkFloat: - build fl(parseFloat(s.eatString())) + result = build fl(^parseFloat(s.eatString())) else: - syntaxError(s) + raise ValueError.newException($s.currToken) + # syntaxError(s, "expected expression, but got " & ) proc typ(s: var SexpParser): dst.t {.transform.} = s.space() - # TODO: implement + s.eat(tkParensLe) + s.expect(tkSymbol) + let str = s.eatString() + case str + of "bool": result = build BoolTy() + of "int": result = build IntTy() + of "float": result = build FloatTy() + of "union": + let first = typ(s) + s.space() + var e: seq[dst.t] + while s.currToken != tkParensRi: + e.add typ(s) + s.space() + result = build UnionTy(first, e) + else: + result = build x(str) + s.eat(tkParensRi) - proc params(s: var SexprParser): dst.pa {.transform.} = + proc param(s: var SexpParser): dst.pd {.transform.} = + s.eat(tkParensLe) + result = build ParamDecl(^ident(s), ^typ(s)) s.space() - # TODO: implement + s.eat(tkParensRi) + + proc params(s: var SexpParser): dst.pa {.transform.} = + var p: seq[dst.pd] + s.space() + s.eat(tkParensLe) + while s.currToken != tkParensRi: + p.add param(s) + s.space() + s.eat(tkParensRi) + build Params(p) proc top(s: var SexpParser): dst.d {.transform.} = s.eat(tkParensLe) - s.expect(tkIdent) - case s.currToken + s.expect(tkSymbol) + case s.eatString() of "proc": - build ProcDecl(ident(s), typ(s), params(s), expr(s)) + result = build ProcDecl(^ident(s), ^typ(s), ^params(s), ^expr(s)) of "type": - build TypeDecl(ident(s), typ(s)) + result = build TypeDecl(^ident(s), ^typ(s)) else: - syntaxError() - -proc removeAndOr(_: Lsrc): L1 {.pass.} = - proc expr(_: src.e): dst.e {.transform.} = - And([a], [b]) -> build If(a, b, x("false")) - Or([a], [b]) -> build If(a, x("true"), b) - -proc removeSingleIf(_: L1): L2 {.pass.} = - proc expr(_: src.e): dst.e {.transform.} = - If([a], [b]) -> build If(a, b, TupleCons()) - -proc declToLet(_: L2): L3 {.pass.} = - proc expr(_: src.e): dst.e {.transform.} = - Decl(`x`, `e`) -> build Let(x, e, TupleCons()) - Exprs(`e0`, [last]): - var r = last + # TODO: proper syntax error + raise ValueError.newException("") + s.eat(tkParensRi) + + proc module(s: var SexpParser): dst.m {.transform.} = + s.space() + var decls: seq[dst.d] + while s.currToken != tkEof: + decls.add top(s) + s.space() + build Module(decls) + + module(s) + +proc removeAndOr(x: Lsrc): L1 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of And([e0], [e1]): build If(e0, e1, x(^Ident("false"))) + of Or([e0], [e1]): build If(e0, x(^Ident("true")), e1) + +proc removeSingleIf(x: L1): L2 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of If([e0], [e1]): build If(e0, e1, TupleCons([])) + +proc declToLet(x: L2): L3 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of Decl(x, [e]): + build Let(x, e, TupleCons([])) + of Exprs(e0, [e1]): + var r = e1 var got: seq[dst.e] for i in countdown(e0.high, 0): match e0[i]: - Decl(`x`, `e`): - r = - if got.len == 0: build Let(x, e, r) - else: build Let(x, e, Exprs(got, r)) - else: - got.insert expr(e0[i]) + of Decl(x, e): + r = + if got.len == 0: build Let(x, ^expr(e), r) + else: build Let(x, ^expr(e), Exprs(got, r)) + else: + got.insert expr(e0[i]) if got.len == 0: build Exprs(got, r) else: r + +proc symbolize(x: L3, r: ref ReportContext[string]): L4 {.pass.} = + ## Binds all identifiers to symbols. + var ctx: Table[string, Value[Symbol]] + proc error(msg: string) = + r.error(msg) + + proc add(name: sink string): Value[Symbol] = + if name in ctx: + error(fmt"'{name}' has a symbol bound already") + result = terminal Symbol() + ctx[name] = result + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Let(x, e0, e1): + let a = expr(e0) + let s = add(x.val.string) + let b = expr(e1) + # the symbol goes out of scope at the end of the Let + ctx.del(x.val.string) + build Let(s, a, b) + of x: + if x.val.string in ctx: + build ^ctx[x.val.string] + else: + error(fmt"undeclared identifier: '{x.val.string}'") + build s(^Symbol()) # error correction + + proc typ(n: src.t): dst.t {.transform.} = + case n + of x: + if x.val.string in ctx: + build ^ctx[x.val.string] + else: + error(fmt"undeclared identifier: '{x.val.string}'") + build s(^Symbol()) # error correction + + proc pattern(n: src.p): dst.p {.transform.} = + # TODO: don't require this processor (will be fixed by moving to using + # generic procedures internally) + case n + of As(x, t): unreachable() + + proc rule(n: src.mr): dst.mr {.transform.} = + case n + of Rule(p, e): + match p: + of As(x, t): + let s = add(x.val.string) + let e2 = expr(e) + ctx.del(x.val.string) + build Rule(As(^typ(t), s), e2) + + proc param(n: src.pd): dst.pd {.transform.} = + case n + of ParamDecl(x, [t]): + build ParamDecl(^add(x.val.string), t) + + proc params(n: src.pa): dst.pa {.infer.} + + proc decl(n: src.d): dst.d {.transform.} = + case n + of ProcDecl(x, t, pa, e): + let s = add(x.val.string) + build ProcDecl(s, ^typ(t), ^params(pa), ^expr(e)) + of TypeDecl(x, [t]): + build TypeDecl(^add(x.val.string), t) + +proc typeCheck(x: L4): L5 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + # XXX: temporary implementation, just so that something exists + case n + of ArrayCons([e]): + build ArrayCons(UnitTy(), e) + of TupleCons([e]): + if e.len == 0: build Unit() + else: build TupleCons(UnitTy(), ^e[0], e) # FIXME + of RecordCons([rf0], [rf1]): + build RecordCons(UnitTy(), rf0, rf1) + of Seq(str): + build Seq(SeqTy(CharTy()), str) + of Seq([t], [e]): + build Seq(SeqTy(t), [e]) + of Call([e0], [e1]): + build Call(UnitTy(), e0, e1) + of FieldAccess([e], n): + build FieldAccess(UnitTy(), e, n) + of FieldAccess([e], x): + build FieldAccess(UnitTy(), e, x) + of At([e0], [e1]): + build At(UnitTy(), e0, e1) + of If([e0], [e1], [e2]): + build If(UnitTy(), e0, e1, e2) + of As([e], [t]): + build As(t, e, t) + of Exprs([e0], [e1]): + build Exprs(UnitTy(), e0, e1) + of Match([e], [mr0], [mr1]): + build Match(UnitTy(), e, mr0, mr1) + +proc specializeAs(x: L5): Lnoas {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of As([t0], [e], [t1]): + # TODO: handle the non-union-constructor meaning of `As` (i.e., turn + # into an `Exprs`) + build Inj(t0, e, t1) + +proc lowerMatch(x: Lnoas): L6 {.pass.} = + ## Replaces `Match` forms with chains of if expressions. + proc expr(n: src.e): dst.e {.transform.} = + case n + of Match([t], [e], mr0, mr1): + proc wrap(tmp: dst.s, r: src.mr, last: dst.e): dst.e = + match r: + of Rule(p, e): + match p: + of As([t2], s): + build If(t, Is(BoolTy(), tmp, t2), + Let(s, Unpack(t2, tmp), ^expr(e)), last) + + let tmp = terminal Symbol() + var last = build Unit() + # TODO: ^^ use `Unreachable`. Presently doesn't work due to a pass + # ordering issue + for i in countdown(mr1.high, 0): + last = wrap(tmp, mr1[i], last) + build Let(tmp, e, ^wrap(tmp, mr0, last)) + + +proc nounion(x: L6): L7 {.pass.} = + # TODO: use a proper name + proc same(a, b: src.t): bool = + # TODO: implement this somehow. Structural tree equality should likely + # just be provided by the framework + false + + proc computeTag(union, elem: src.t): int = + match union: + of UnionTy(t0, t1): + if same(t0, elem): + 0 + else: + var r = 0 + # for i, it in t1.pairs: + # if same(it.index, elem.index): + # r = i + # break + r + else: + unreachable() + + proc getType(n: src.e): src.t = + # TODO: figure out some way to use '_' for elements (to discard them) + match n: + of Call(t, e0, e1): t + of If(t, e0, e1, e2): t + # TODO: complete + else: unreachable() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Is([t], e, t1): + # replace with a tag comparison + let u = getType(e) + build Prim(t, str(^"eq"), + [FieldAccess(IntTy(), ^expr(e), n(^0)), n(^computeTag(u, t1))]) + of Unpack([t], [e]): + # XXX: how to best get access to the lowered type? + build FieldAccess(t, FieldAccess(t, e, n(^1)), n(^0)) + of Inj(t0, [e], t1): + build TupleCons(^typ(t0), n(^computeTag(t0, t1)), [e]) + + proc typ(n: src.t): dst.t {.transform.} = + case n + of UnionTy([t0], [t1]): + # becomes a tag + union + build TupleTy(IntTy(), [UnionTy(t0, t1)]) + +proc norecord(x: L7): L8 {.pass.} = + # TODO: use a proper name + proc fdef(n: src.rf): dst.e {.transform.} = + case n + of Field(x, [e]): e + + proc fdef(n: src.f): dst.t {.transform.} = + case n + of Field(x, [t]): t + + proc expr(n: src.e): dst.e {.transform.} = + case n + of RecordCons([t], [e0], [e1]): + # TODO: sort the fields while ensuring correct evaluation order + build TupleCons(t, e0, e1) + of FieldAccess([t], [e], x): + build FieldAccess(t, e, n(^0)) # TODO: use the correct index + + proc typ(n: src.t): dst.t {.transform.} = + case n + of RecordTy([t0], [t1]): + # TODO: sort the fields, so that `RecordTy(IntTy(), FloatTy())` and + # `RecordTy(FloatTy(), IntTy())` both produce the same + build TupleTy(t0, t1) + +proc bridge(x: L8): Lnocons {.pass.} = + ## Pass to bridge between some ILs for which the intermediate lowering + ## passes are still missing. + # TODO: implement the intermediate passes + proc typ(n: src.t): dst.t {.transform.} = + case n + of SeqTy(t): build BoolTy() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Seq(t, e): build Unit() + of Seq(t, str): build Unit() + of Let(s, [e1], [e2]): build Let(s, e1, e2) + of ArrayCons(t, e): build Unit() + of TupleCons(t, e0, e1): build Unit() + of FieldAccess(t, e, n): build Unit() + of At(t, e0, e1): build Unit() + +proc simplifyLet(x: Lnocons): Lnoletwithval {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of Let(s, ie, [e]): + match ie: + of Undef(): build Let(s, e) + of e1: build Let(s, Exprs(UnitTy(), [Asgn(s, ^expr(e1))], e)) + +proc exprToStmt(x: Lnoletwithval): Lstmt {.pass.} = + proc etos(n: src.e): dst.st {.transform.} = + case n + # TODO: auto-generate transforms such as, e.g., `Let(s, [st]) -> Let(s, st)` + of Let(s, [st]): build Let(s, st) + of Unit(): build Pass() # a trailing Unit expression becomes Pass + of If(t, [e], [st], e2): + match e2: + of Unit(): build If(e, st) + else: build If(e, st, ^etos(e2)) + of Exprs(_, [st0], [st1]): + build Stmts(st0, st1) + of While([e], [st]): + build While(e, st) + # TODO: also auto-generate the below + of Asgn([e0], [e1]): build Asgn(e0, e1) + of Return([e]): build Return(e) + of Unreachable(): build Unreachable() + else: + # TODO: maybe raise a proper run-time error? It would allow for better + # error messages and thus easier development + unreachable() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Exprs([t], [st], [e]): build Exprs(t, st, e) + of Asgn([e0], [e1]): + build Exprs(UnitTy(), [Asgn(e0, e1)], Unit()) + of While([e], [st]): + build Exprs(UnitTy(), [While(e, st)], Unit()) + # TODO: auto-generate the exception raising? Right now, a static error is + # reported, but - for convenience - the nanopass framework could + # also just leave reporting an error to the runtime (Scheme's + # nanopass framework does it that way, for what it's worth) + of Return(e): + raise ValueError.newException("not an expr") + of Unreachable(): + raise ValueError.newException("not an expr") + + proc decl(n: src.d): dst.d {.transform.} = + # TODO: move this to a separate pass and handle void bodies correctly + case n + of ProcDecl(s, [t], [pa], [e]): build ProcDecl(s, t, pa, Return(e)) + +proc ifExprToStmt(x: Lstmt): L10 {.pass.} = + # TODO: needs to happen earlier + proc expr(n: src.e): dst.e {.transform.} = + case n + of If([t], [e0], [e1], [e2]): + let s = terminal Symbol() + build Let(s, Exprs(t, [If(e0, Asgn(s, e1), Asgn(s, e2))], s)) + +proc blobify(x: L10): L11 {.pass.} = + ## Turns all aggregate types into blob types. + proc lval(n: src.lv): (dst.t, dst.e) = + match n: + of s: + # TODO: use the correct expression type + (build(dst.t, UnitTy()), build(dst.e, Addr(s))) + of At([t], lv, e): + let (elem, x) = lval(lv) + # TODO: use the correct stride (taken from the `elem`) + (t, build(dst.e, Offset(PtrTy(), x, ^expr(e), n(^1)))) + of FieldAccess([t], lv, n): + let (elem, x) = lval(lv) + # TODO: use the correct offset + (t, build(dst.e, Offset(PtrTy(), x, n, n(^1)))) + + proc expr(n: src.e): dst.e {.transform.} = + case n + of lv: + let (t, e) = lval(lv) + build Load(t, e) + + proc stmt(n: src.st): dst.st {.transform.} = + case n + of Asgn(e0, [e1]): + match e0: + of lv: build Store(^(lval(lv)[1]), e1) + else: unreachable() + +proc calleeToSymbol(x: L11): L12 {.pass.} = + ## Removes all complex callee expressions. + proc expr(n: src.e): dst.e {.transform.} = + case n + of Call([t], e0, [e1]): + match e0: + of s: build Call(t, s, e1) + else: + let tmp = terminal Symbol() + build Let(tmp, Exprs(t, [Asgn(tmp, ^expr(e0))], Call(t, tmp, e1))) + + proc stmt(n: src.st): dst.st {.transform.} = + case n + of Call([t], e0, [e1]): + match e0: + of s: build Call(t, s, e1) + else: + let tmp = terminal Symbol() + build Let(tmp, Stmts([Asgn(tmp, ^expr(e0))], Call(t, tmp, e1))) + + +# TODO: fix the symbol binding in the pass DSL such that `vmenv` can be +# imported at the top (at the moment, ambiguity errors ensue) +import vm/[vmmodules, vmspec, vmenv] + +type TypeKind = enum + tkInt, tkFloat, tkPtr, tkBlob + +# XXX: doesn't work yet +#[ +proc genvm(x: L12): VmModule {.outpass.} = + ## VM code generator. Generates a VM module from `x`. + type Assembler = object + code: seq[Instr] + locals: seq[tuple[typ: TypeKind, used: bool]] + map: Table[Value[Symbol], uint32] + + proc instr(c: var Assembler, op: Opcode) = + c.code.add Instr(InstrType(op)) + proc instr(c: var Assembler, op: Opcode, a: int32) = + c.code.add Instr(InstrType(op) or (a.InstrType shl instrAShift)) + proc instr(c: var Assembler, op: Opcode, b: uint16) = + c.code.add Instr(InstrType(op) or (b.InstrType shl instrBShift)) + + proc label(c: Assembler): uint32 = c.code.len.uint32 + proc jump(c: var Assembler, op: Opcode): uint32 = + c.code.add Instr(InstrType(op)) + result = c.code.high.uint32 + proc join(c: var Assembler, pos: uint32) = + c.code[pos] = Instr(InstrType(c.code[pos].opcode)) + proc jump(c: var Assembler, op: Opcode, target: uint32) = + c.code.add Instr(InstrType(op) or (InstrType(c.code.len.uint32 - target) shl instrAShift)) + + proc allocLocal(c: var Assembler, typ: TypeKind): int32 = + for i, it in c.locals.mpairs: + if it.typ == typ and not it.used: + it.used = true + return i.int32 + + c.locals.add (typ, true) + result = c.locals.high.int32 + + proc alloc(c: var Assembler, x: Symbol): int32 = + result = c.allocLocal: + case x.typ.kind + of tkFloat, tkInt: x.typ.kind + else: tkInt + + if x.typ.kind == tkBlob: + # allocate a stack slot + c.instr(opcStackAlloc, x.typ.size) + c.instr(opcPopLocal, result) + + proc free(c: var Assembler, loc: int32, x: Symbol) = + c.locals[loc].used = false + if x.typ.kind == tkBlob: + c.instr(opcStackFree, x.typ.size) + + proc expr(n: src.e, c: var Assembler) = + match n: + of Prim(t, str, e): + case str.val + of "+": + expr(e[0], c) + expr(e[1], c) + c.gen(opcAddInt) + of "<": + expr(e[0], c) + expr(e[1], c) + c.gen(opcLtInt) + of "<=": + expr(e[0], c) + expr(e[1], c) + c.gen(opcLeInt) + of "==": + expr(e[0], c) + expr(e[1], c) + c.gen(opcEqInt) + # of AddChecked(t, e0, e1, x): + # expr(e0, c) + # expr(e1, c) + # c.instr(opcAddChck) + # c.instr(opcPopLocal, x.get) + of Addr(x): + c.instr(opcGetLocal, x) + of Let(s, e): + let x = c.alloc(s) + expr(e, c) + c.free(x) + of Load(t, e): + expr(e) + c.gen(opcLdInt32) + of x: + c.gen(opcGetLocal, c.map[x]) + of Call(t, s, e): + for it in e.items: + expr(it) + c.gen(opcCall, s, x) + of Exprs(st, e): + for it in st.items: + stmt(it, c) + expr(e, c) + + proc stmt(n: src.st, c: var Assembler) = + match n: + of Let(s, st): + let x = c.alloc(s.val) + stmt(st, c) + c.free(x, s.val) + of If(e, st0, st1): + expr(e, c) + let x = c.jump(opcBranch) + stmt(st0, c) + let y = c.jump(opcJmp) + c.join(x) + stmt(st1, c) + c.join(y) + of If(e, st): + expr(e, c) + let x = c.jump(opcBranch) + stmt(st, c) + c.join(x) + of While(e, st): + let lab = c.label() + expr(e, c) + let x = c.jump(opcBranch) + stmt(st, c) + c.jump(opcJmp, lab) + c.join(x) + of Asgn(s, e): + expr(e, c) + c.instr(opcPopLocal, c.map[s]) + of Store(e0, e1): + expr(e0, c) + expr(e1, c) + c.instr(opcWrInt16) + # of MemCopy(e0, e1, e2): + # expr(e0) + # expr(e1) + # expr(e2) + # c.gen(opcMemCopy) + of Return(e): + expr(e, c) + c.instr(opcRet) + of Call(t, s, e1): + for it in e1.items: + expr(it, c) + if s.val.typ.kind == tkPtr: + c.instr(opcGetLocal, c.map[s]) + c.instr(opcIndCall, e1.len) + else: + c.instr(opcCall, s.val, e1.len) + of Unreachable(): + c.instr(opcUnreachable) + + proc prc(name: src.x, ret: src.t, params: src.pa, body: src.st, m: var VmModule) = + var c = Assembler() + let p = unpack(params) + for i, it in p.pairs: + let loc = c.alloc(it.typ) + c.locals[it] = loc + c.instr(opcPopLocal, loc) + stmt(body, c) + # TODO: append to the module + + proc module(n: src.m, m: var VmModule): string = + let procs = unpack(n) + for it in procs.items: + let (x, t, pa, s) = unpack(it) + prc(x, t, pa, s, m) + + module(x, result) +]# + +import std/macros + +macro defineCompiler(name, names: untyped) = + ## Generates a compiler procedures, running the provided passes in + ## sequence. Interim implementation. + var prev = ident"e" + let body = newStmtList() + for it in names.items: + let tmp = genSym(nskLet, "tmp") + if it.kind == nnkIdent: + let name = it.strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, quote do: `it`(`prev`, NodeIndex(0))) + else: + let g = it + g.insert 1, quote do: NodeIndex(0) + g.insert 1, prev + let name = it[0].strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, g) + prev = tmp + result = quote do: + proc `name`(e: Ast[Lsrc]): auto = + `body` + result = `prev` + +# some logic for testing: +var rep = initDefaultReporter[string]() +defineCompiler compile, [ + removeAndOr, + removeSingleIf, + declToLet, + symbolize(rep), + typeCheck, + specializeAs, + lowerMatch, + nounion, + norecord, + bridge, + simplifyLet, + exprToStmt, + ifExprToStmt, + blobify, + calleeToSymbol] + +var s: SexpParser +# s.open(newStringStream(""" +# (proc a (bool) () (exprs (decl x true) (decl y (and x 2)) (and (or 1 2) (and 3 4)))) +# """)) +s.open(newStringStream(""" +(proc a (bool) ((x (int))) (== x 2)) +(proc b (bool) ((x (float))) (== x 2)) +(proc test (bool) ((x (union (int) (float)))) (match x (rule (y (int)) (a x)) (rule (z (float)) (b z)))) +(proc main (bool) () (test (as 1 (union (int) (float))))) +""")) +discard s.getTok() +let m = parse(s) +s.close() + +let got = compile(m) From da880599c83680c20cfc8cc65b7f64ce9a2e11fc Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 18 Aug 2025 22:06:32 +0000 Subject: [PATCH 09/87] nanopass: implement a subset of the major macros Implement a basic version of the `pass`, `transform`, and `build` macros. --- nanopass/asts.nim | 61 ++ nanopass/nanopass.nim | 1291 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 1318 insertions(+), 34 deletions(-) create mode 100644 nanopass/asts.nim diff --git a/nanopass/asts.nim b/nanopass/asts.nim new file mode 100644 index 00000000..71787492 --- /dev/null +++ b/nanopass/asts.nim @@ -0,0 +1,61 @@ +## Implements the nanopass framework specific storage types for ASTs. The types +## are layered on top of `PackedTree `_. + +import passes/trees + +type + # note: the fields are exported so that the nanopass machinery can access + # them. User code should, in most cases, not access the fields directly + Ast*[L: object] = object + tree*: PackedTree[uint8] + ## leaked implementation detail, don't use + + Metavar*[L: object, N: static string] = object + ## Represents a reference to an AST fragment that's a production of non- + ## terminal `N` of language `L`. + # TODO: rename to NonTerminal (currently clashes with the type of the same + # name in `nanopass.nim`) + index*: NodeIndex + ## leaked implementation detail, don't use + + Value*[T] = object + ## Represents a reference to a value with type `T` that's a terminal in + ## an AST. + index*: uint32 + ## leaked implementation detail, don't use + + ChildSlice*[T: Metavar or Value] = object + ## A lightweight reference to a slice of contiguous children of a tree. + start: NodeIndex + len: uint32 + + Storage*[T] = object + ## The container AST fragments use for storing embedded datums. + data: seq[T] # TODO: use a BiTable + +proc slice*[T](start: NodeIndex, len: uint32): ChildSlice[T] = + ChildSlice[T](start: start, len: len) + +iterator items*[T](t: PackedTree[uint8], s: ChildSlice[T]): T = + var c = s.start + for _ in 0.. ``x.abc``. + macro dotAux(fname: static string, e: untyped): untyped = + newDotExpr(e, ident(fname)) + + newCall(bindSym"dotAux", fname, e) + +template matches*[T, U](x: T, _: typedesc[U]): bool = + ## Implements type-based pattern matching, used by the nanopass macros. + when U is PArray: + when T is seq: + matches(default(typeof(x[0])), typeof(U.A)) + else: + false + elif U is PChoice: + # why not just use `or`? Because that would unnecessarily expand the + # second `matches` invocation when the case first invocation was sucessful + when matches(x, typeof(U.A)): + true + else: + matches(x, typeof(U.B)) + elif U is Metavar: + when x is U: + true + else: + matches(x, dot(U.L.meta.nt, U.N)) + else: + x is U + +proc lookup[E; M: tuple](): auto {.compileTime.} = + var x: M + for it in fields(x): + when E is typeof(it[0]): + return it[1] + +template isAtom*(x: uint8): bool = + ## The predicate required for using an uint8 as a ``PackedTree`` tag. + x >= RefTag + +proc `$`(x: Form): string = + result = x.tag + result.add "(" + for i, it in x.elems.pairs: + if i > 0: + result.add ", " + result.add it.typ + result.add ")" + +proc `$`*(x: LangDef): string = + for it in x.forms.items: + result.add $it + result.add "\n" + proc `==`(x, y: Elem): bool = x.typ == y.typ and x.repeat == y.repeat @@ -327,6 +434,8 @@ proc buildLanguage(add, sub: seq[NimNode], addProd(result, a, name) computeNodeTags(result) + # TODO: properly set the entry non-terminal + result.entry = "module" proc makeLanguage(body: NimNode): LangDef = ## Creates a language definition from the ``defineLanguage`` DSL code. @@ -375,6 +484,10 @@ proc makeLanguage(base: LangDef, body: NimNode): LangDef = var add, sub: seq[NimNode] var def: seq[NonTerminalDef] + var body = body + if body.kind != nnkStmtList: + body = newStmtList(body) + proc extract(n: NimNode, add, sub: var seq[NimNode]) = case n.kind of nnkPrefix: @@ -424,43 +537,1153 @@ proc makeLanguage(base: LangDef, body: NimNode): LangDef = buildLanguage(add, sub, def, base, body) -macro defineLanguage*(name, body: untyped) = - ## Creates a language definitions and binds it to a const symbol with the - ## given name. +proc buildLangInfo(def: LangDef): LangInfo = + ## Creates the pass-centric language representation for `def`. + result.map = initTable[string, int](4) + + for name, it in def.terminals.pairs: + result.types.add LangType( + name: it.typ, + mvar: name, + terminal: true, + ntag: it.tag + ) + result.map[name] = high(result.types) + + for name, it in def.nterminals.pairs: + result.types.add LangType( + name: name, + mvar: it.mvars[0], + terminal: false, + forms: mapIt(it.forms, it.semantic) + ) + # add the name-to-type mappings: + result.map[name] = high(result.types) + for x in it.mvars.items: + result.map[x] = high(result.types) + + # now that all name-to-type mappings are present, add the forms and + # the subtype info + for it in def.forms.items: + result.forms.add SForm( + name: it.tag, + ntag: it.id, + elems: mapIt(it.elems, (result.map[it.typ], it.repeat)) + ) + + for name, it in def.nterminals.pairs: + let id = result.map[name] + for v in it.vars.items: + result.types[id].sub.add result.map[v] + +macro makeLanguageType(def: static LangDef, typName: untyped) = + ## Creates the type representing the language defined by `def`. This is the + ## type the nanopass-framework user passes around. + ## + ## The type also stores various information about the language that are + ## needed by the pass-related macros, encoded as types. + let fields = nnkRecList.newTree() + # the metavars are at top level of the type, for easy access by + # the programmer + let mvar = bindSym"Metavar" + for name, it in def.terminals.pairs: + fields.add newIdentDefs(ident(name), + nnkBracketExpr.newTree(bindSym"Value", ident(it.typ))) + for name, it in def.nterminals.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree( + mvar, + ident(typName.strVal), + newStrLitNode(name))) + + let ntType = nnkTupleTy.newTree() + let (csym, fsym, vsym) = (bindSym"PChoice", bindSym"PForm", bindSym"Value") + # add the descriptions for the non-terminals + for name, nt in def.nterminals.pairs: + let ln = ident(typName.strVal) + var p = ident"void" + for f in nt.forms.items: + let id = def.forms[f.semantic].id + p = quote do: + `csym`[`p`, `fsym`[`id`]] + + for v in nt.vars.items: + if v in def.terminals: + let id = ident(def.terminals[v].typ) + p = quote do: + `csym`[`p`, `vsym`[`id`]] + else: + let id = ident(v) + p = quote do: + `csym`[`p`, `ln`.`id`] + + ntType.add newIdentDefs(ident(name), p) + + # everything meant for internal use is stored in an anonymous record in + # the `meta` field, preventing name clashes and the fields showing up + # in auto-complete suggestions + let metaType = nnkTupleTy.newTree( + newIdentDefs(ident"entry", + nnkBracketExpr.newTree(mvar, + ident(typName.strVal), + newStrLitNode(def.entry))), + newIdentDefs(ident"nt", ntType)) + + # create the terminal->tag map: + let tup = nnkTupleConstr.newTree() + for it in def.terminals.values: + let n = it.tag + tup.add nnkTupleConstr.newTree( + ident(it.typ), + nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(n))) + + metaType.add newIdentDefs(ident"term_map", tup) + + fields.add newIdentDefs(ident"meta", metaType) + + result = nnkTypeSection.newTree( + nnkTypeDef.newTree( + typName, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + fields))) + +macro genHelpers(typ, a, b: untyped) = + ## Generates the helper templates `def` and `idef`, which are used for + ## retrieving the `LangDef` and `LangInfo` instance for a language type, + ## respectively. + let (def, info) = (bindSym"LangDef", bindSym"LangInfo") + quote do: + template def(_: typedesc[`typ`]): `def` = `a` + template idef(_: typedesc[`typ`]): `info` = `b` + +proc defineLanguageImpl(name, base, body: NimNode): NimNode = body.expectKind nnkStmtList body.expectMinLen 1 - let p = bindSym"makeLanguage" - let q = bindSym"quote" if body[0].kind == nnkCommentStmt: - let filtered = body[1..^1] - result = nnkConstSection.newTree( - nnkConstDef.newTree(name, - newEmptyNode(), - quote do: `p`(`q` do: `filtered`)), - body[0]) - else: - result = nnkConstSection.newTree( - nnkConstDef.newTree(name, - newEmptyNode(), - quote do: `p`(`q` do: `body`))) + body.del(0) + + let setup1 = + if base.isNil: + genAst(body): makeLanguage(quote do: body) + else: + genAst(body, base): makeLanguage(def(base), quote do: body) + result = genAst(setup1, name): + const + def = setup1 + tmp = buildLangInfo(def) + makeLanguageType(def, name) + genHelpers(name, def, tmp) + +macro defineLanguage*(name, body: untyped) = + ## Creates a language definition and binds it to a const symbol with the + ## given name. + defineLanguageImpl(name, nil, body) macro defineLanguage*(name, base, body: untyped) = ## Creates a language definition, extending `base`, and binds it to a const ## symbol with the given name. Extension doesn't imply a direction in this ## context. - body.expectKind nnkStmtList - body.expectMinLen 1 - let p = bindSym"makeLanguage" - let q = bindSym"quote" - if body[0].kind == nnkCommentStmt: - let filtered = nnkStmtList.newTree(body[1..^1]) - result = nnkConstSection.newTree( - nnkConstDef.newTree(name, - newEmptyNode(), - quote do: `p`(`base`, `q` do: `filtered`)), - body[0]) + defineLanguageImpl(name, base, body) + +# -------- macro helpers ------- + +proc copyLineInfoForTree(n, info: NimNode) = + copyLineInfo(n, info) + for i in 0.. 0: + block outer: + var (i, last) = stack[^1] + let prev = i + while i <= last: + if src[i].kind < RefTag: + last += src[i].val + elif src[i].kind == RefTag: + if i > prev: + # copy everything we got so far + let pos = dst.len + dst.setLen(pos + int(i - prev)) + copyMem(addr dst[pos], addr src[prev], int(i - prev) * size) + + stack[^1] = (i + 1, last) + let next = src[i].val + stack.add (next, next) + break outer + + inc i + + if i > prev: + # copy the rest + let pos = dst.len + dst.setLen(pos + int(i - prev)) + copyMem(addr dst[pos], addr src[prev], int(i - prev) * size) + + stack.shrink(stack.len - 1) + +# ------- build macro ------- + +proc append[L, U](to: var PackedTree[uint8], x: Value[U]) = + to.nodes.add TreeNode[uint8](kind: typeof(lookup[U, L.meta.term_map]()).V) + +proc append[L](to: var PackedTree[uint8], x: Metavar) = + to.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) + +proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = + to.nodes[i] = TreeNode[uint8](kind: RefTag, val: uint32(x.index)) + inc i + +proc append[L](to: var PackedTree[uint8], x: openArray) = + for it in x.items: + append[L](to, it) + +# helpers for `buildImpl` +template len(x: Value): int = 1 +template len(x: Metavar): int = 1 + +macro buildImpl(to: var PackedTree[uint8], lang: static LangInfo, + typ: typedesc[Metavar], e: untyped): untyped = + ## Emits a tree construction for the AST described by `e`, with the syntax + ## from `lang`. + proc cons(lang: LangInfo, n, test, body: NimNode): NimNode {.closure.} + + proc elem(lang: LangInfo, n, test, body: NimNode): NimNode = + ## Processor for element syntax. + case n.kind + of nnkCall: + result = cons(lang, n, test, body) + of nnkBracket: + result = nnkBracket.newTree() + for it in n.items: + result.add elem(lang, it, test, body) + of nnkIdent, nnkSym, nnkAccQuoted: + # some hoisted expression + result = n + body.add genAst(typ, to, n) do: + append[typ.L](to, n) + else: + error("unexpected syntax", n) + + proc form(lang: LangInfo, n, test, body: NimNode): NimNode = + ## Processor for a tree construction. + let tag = n[0].strVal + var elems: seq[NimNode] + var temp = newStmtList() + for i in 1..language pass. + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.body.kind == nnkEmpty: + # a forward declaration, bail + return def + + let to = ident"out.ast" + let ret = def.params[0] + def.body = newStmtList(def.body) + def.body.insert 0, quote do: + # inject a build overload that implicitly uses the output language + template build(body: untyped): untyped {.used.} = + build(`to`, `ret`, body) + + result = def + +macro transform(src, dst: static LangDef, nterm: static string, + form: static int, n: untyped): untyped = + ## Generates the transformation from the given source language form + ## (belonging to non-terminal `nterm`) to a target language form with + ## compatible syntax. + # find a target language form that's a production of the non-terminal and has + # the same shape + # TODO: only require the same name and number of elements, using -> to fit + # the rest. **Edit:** really? That can easily lead to non-obvious + # behaviour + var target = -1 + for it in dst.nterminals[nterm].forms.items: + if dst.forms[it.semantic] == src.forms[form]: + target = it.semantic + break + if target == -1: + return makeError(fmt"cannot generate transformer for '{src.forms[form]}'", n) + + # important: the generated code being efficient is of major importance! Most + # transformations will be auto-generated, and they should thus be as fast as + # possible + # TODO: if none of the child nodes require a processor call, memcopy the + # whole sub-tree + # TODO: (bigger refactor) pass the cursor to the transformers as a var + # parameter, which would eliminate unnecessary tree seeking when + # there's many calls to fully auto-generated non-terminal processors + + let inAst = ident"in.ast" + let to = ident"out.ast" + let id = dst.forms[target].id.uint8 + result = newStmtList() + # add the root node: + let body = quote do: + var tmp {.used.} = `inAst`.child(`n`, 0) + let root = `to`.nodes.len.NodeIndex + var i = `to`.nodes.len + # the node sequence needs to be contiguous, so it's allocated upfront + `to`.nodes.setLen(i + `inAst`.len(`n`) + 1) + `to`.nodes[i] = TreeNode[uint8](kind: `id`, val: `inAst`[`n`].val) + inc i + + # call the transformers and emit the nodes in one go: + for i, it in src.forms[form].elems.pairs: + let fromTerminal = it.typ in src.terminals + let toTerminal = it.typ in dst.terminals + if fromTerminal != toTerminal or + (toTerminal and src.terminals[it.typ].typ != src.terminals[it.typ].typ): + body.add makeError(fmt"cannot generate transformer for {src.forms[form]}", n) + break + + let call = + if fromTerminal: + if src.terminals[it.typ].tag == dst.terminals[it.typ].tag: + # just copy the node + quote do: + `to`.nodes[i] = `inAst`[tmp] + inc i + else: + # repack with the new tag + let tag = dst.terminals[it.typ].tag + quote do: + `to`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `inAst`[tmp].val) + inc i + else: + let append = bindSym"append" + let op = ident"->" + let s = newStrLitNode(it.typ) + let d = newStrLitNode(dst.forms[target].elems[i].typ) + quote do: + `append`(`to`, i, + `op`(Metavar[src, `s`](index: tmp), Metavar[dst, `d`])) + + if it.repeat: + let bias = src.forms[form].elems.len - 1 + body.add quote do: + for _ in 0..<(`inAst`.len(`n`) - `bias`): + `call` + tmp = `inAst`.next(tmp) + else: + body.add quote do: + `call` + tmp = `inAst`.next(tmp) + + result.add body + # the callsite takes care of fitting the index to the right type + result.add ident"root" + +proc ntags(lang: LangInfo, typ: LangType): seq[int] = + ## Returns a list with all possible node tags productions of `typ` can have. + for it in typ.forms.items: + result.add lang.forms[it].ntag + + for it in typ.sub.items: + if lang.types[it].terminal: + result.add lang.types[it].ntag + else: + result.add ntags(lang, lang.types[it]) + +proc matchImpl(lang: LangInfo, src: int, ast, sel, rules: NimNode + ): (seq[NimNode], IntSet) = + ## Implements the core of the `match` macro: + ## 1. makes sure the syntax is correct + ## 2. makes sure the used patterns are unique + ## 3. generates a sequence of transformed 'of' branches, plus a set storing + ## the used forms' tags + var used: IntSet + ## covered form productions (identified by ntag) + + # should nested matching (e.g., ``A(x, B(y))``) be desired, `matchImpl` + # should be factored into two macros macros applied sequentially: + # 1. the first macro does type checking, producing a type form + # 2. the second macro translates the typed form into case/if statements + # combining both steps into one is simple enough when there are no nested + # patterns, but not otherwise + + proc parseVar(n: NimNode): (string, string) = + n.expectKind nnkIdent + let name = n.strVal + var e = name.high + # to get the name of the var, trim trailing numbers and a single underscore + while e >= 0 and name[e] in {'0'..'9'}: + dec e + + if e >= 0 and name[e] == '_': + dec e + + result[0] = name[0..e] + result[1] = name + + proc processIdentPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = + n.expectKind nnkIdent + let (v, nameStr) = parseVar(n) + if v notin lang.map: + error(fmt"no meta-variable with name '{v}'", n) + let id = lang.map[v] + if id notin lang.types[src].sub: + error(fmt"'{v}' is not an immediate production of '{lang.types[src].name}'", n) + + var check, binds: NimNode + let name = ident(nameStr) + if lang.types[id].terminal: + let typ = ident(lang.types[id].name) + let tag = lang.types[id].ntag + used.incl(tag) + # let the compiler report an error for duplicate case labels + check = newIntLitNode(tag) + copyLineInfo(check, n) + binds = newLetStmt(name, quote do: Value[`typ`](index: `ast`[`sel`].val)) + else: + # the pattern binds a non-terminal + let typ = ident(v) + check = nnkCurly.newTree() + let tags = ntags(lang, lang.types[id]) + for tag in tags.items: + check.add nnkConv.newTree(ident"uint8", newIntLitNode(tag)) + # mark the first tag as used, to signal that the non-terminal is handled + used.incl(tags[0]) + + copyLineInfoForTree(check, n) + binds = newLetStmt(name, quote do: src.`typ`(index: `sel`)) + + result = (check, binds) + + proc processPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = + case n.kind + of nnkCall: + n[0].expectKind nnkIdent + # parse the pattern: + var elems = newSeq[tuple[src, dst, name: string]](n.len - 1) + for i in 1.. 0: ident(elems[i].dst) + else: ident(lang.types[it.typ].mvar) + if it.repeat: + let bias = lang.forms[idx].elems.len - 1 + if p.kind == nnkIdent: + # just bind a child slice to the identifier + binds.add newLetStmt(p, quote do: + slice[`origin`](`cursor`, uint32(`ast`.len(`sel`) - `bias`))) + binds.add quote do: + for _ in 0..<`ast`.len(`sel`)-`bias`: + `cursor` = `ast`.next(`cursor`) + else: + # run the selected transformer on all relevant child nodes and + # store the result in a seq + let tmp = genSym() + binds.add newVarStmt(tmp, + quote do: newSeq[`target`](`ast`.len(`sel`)-`bias`)) + let callee = ident"->" + binds.add quote do: + for i in 0..<`ast`.len(`sel`)-`bias`: + `tmp`[i] = `callee`(`origin`(index: `cursor`), `target`) + `cursor` = `ast`.next(`cursor`) + binds.add newLetStmt(p[0], tmp) + else: + # simple case: a single node + if p.kind == nnkIdent: + if lang.types[it.typ].terminal: + binds.add newLetStmt(p, quote do: `origin`(index: `ast`[`cursor`].val)) + else: + binds.add newLetStmt(p, quote do: `origin`(index: `cursor`)) + else: + if lang.types[it.typ].terminal: + binds.add makeError("cannot invoke auto-procesor for terminal", p) + else: + binds.add newLetStmt(p[0], + newCall(ident"->", + nnkObjConstr.newTree(origin, + nnkExprColonExpr.newTree(ident"index", cursor)), + target)) + binds.add quote do: + `cursor` = `ast`.next(`cursor`) + result = (check, binds) + of nnkIdent: + # must be a terminal/non-terminal meta-var + result = processIdentPattern(lang, n) + else: + error("unexpected syntax", n) + + var branches: seq[NimNode] + for it in rules.items: + case it.kind + of nnkOfBranch: + it.expectLen 2 + let (check, binds) = processPattern(lang, it[0]) + branches.add nnkOfBranch.newTree(check, newStmtList(binds, it[1])) + of nnkElse: + # as a guard against malformed run-time inputs, use an 'of' instead + # of an 'else' branch + var ofb = nnkOfBranch.newTree() + # add all remaining forms: + for it in lang.types[src].forms.items: + if not containsOrIncl(used, lang.forms[it].ntag): + ofb.add newIntLitNode(lang.forms[it].ntag) + # also include the tags for sub non-terminals and terminals: + for it in lang.types[src].sub.items: + if lang.types[it].terminal: + if not containsOrIncl(used, lang.types[it].ntag): + ofb.add newIntLitNode(lang.types[it].ntag) + else: + let tags = ntags(lang, lang.types[it]) + if not containsOrIncl(used, tags[0]): + for tag in tags.items: + ofb.add newIntLitNode(tag) + + if ofb.len == 0: + # all forms are handled already. Add an 'else' branch before the + # programmer-provided one so that the compiler can report an + # "unreachable" warning + branches.add nnkElse.newTree(newCall(ident"unreachable")) + branches.add it + else: + copyLineInfo(ofb, it) + ofb.add it[0] + branches.add ofb + else: + error("expected 'of' or 'else'", it) + + result = (branches, used) + +macro processorMatchImpl(lang: static LangInfo, src: static string, + sel: untyped, rules: varargs[untyped]): untyped = + ## Implements the transformation/expansion of a language->language + ## processor's trailing case statement/expression. + let id = lang.map[src] + var (branches, used) = matchImpl(lang, id, ident"in.ast", sel, rules) + + template nt: untyped = lang.types[id] + let sym = bindSym"transform" + # auto-generate the transformers for the forms not manually provided: + for it in nt.forms.items: + let id = lang.forms[it].ntag + if not containsOrIncl(used, id): + branches.add nnkOfBranch.newTree( + newIntLitNode(id), + (quote do: + Metavar[dst, `src`](index: `sym`(def(src), def(dst), `src`, `it`, `sel`)))) + + # auto-generate the branches for missing production terminals and + # non-terminals: + for it in nt.sub.items: + if lang.types[it].terminal: + let id = lang.types[it].ntag + if id notin used: + let to = ident"out.ast" + let input = ident"in.ast" + branches.add nnkOfBranch.newTree( + newIntLitNode(id), + (quote do: + # TODO: use the tag of the destination language + `to`.nodes.add: + TreeNode[uint8](kind: uint8(`id`), val: `input`[`sel`].val) + Metavar[dst, `src`](index: NodeIndex(`to`.nodes.high)))) + else: + # if the first form's tag is used, so is the non-terminal itself + if lang.forms[lang.types[it].forms[0]].ntag notin used: + # TODO: consider inlining the transformer if it's auto-generated. + # More code to emit, but also a little less work at run-time + let branch = nnkOfBranch.newTree() + for tag in ntags(lang, lang.types[it]).items: + branch.add newIntLitNode(tag) + let name = ident(lang.types[it].mvar) + let callee = ident"->" + # dispatch to the processor and convert to the expected type + branch.add quote do: + Metavar[dst, `src`]( + index: `callee`(src.`name`(index: `sel`), dst.`name`).index) + branches.add branch + + let input = ident"in.ast" + result = nnkCaseStmt.newTree(quote do: `input`[`sel`].kind) + result.add branches + if branches[^1].kind != nnkElse: + # the selector is a uint8 and thus the case cannot be exhaustive + result.add nnkElse.newTree(newCall(ident"unreachable")) + +macro matchImpl(lang: static LangInfo, nterm: static string, + ast: PackedTree[uint8], sel: NodeIndex, info: untyped, + rules: varargs[untyped]): untyped = + ## The internal implementation `match` dispatches to. + let (branches, used) = matchImpl(lang, lang.map[nterm], ast, sel, rules) + result = nnkCaseStmt.newTree(quote do: `ast`[`sel`].kind) + copyLineInfoForTree(result, info) + result.add branches + # add a default handler when all possible productions are covered + var allCovered = true + for it in lang.types[lang.map[nterm]].sub.items: + if lang.types[it].terminal: + if lang.types[it].ntag notin used: + allCovered = false + break + elif lang.forms[lang.types[it].forms[0]].ntag notin used: + allCovered = false + break + + if allCovered: + result.add nnkElse.newTree(newCall(ident"unreachable")) + +template match*(ast: PackedTree[uint8], nt: Metavar, + branches: varargs[untyped]): untyped = + ## Provides a convenient way to destructure an AST. Meant to be used as + ## follows: + ## + ## .. code-block:: nim + ## + ## match ast, n: + ## of ...: discard + ## of ...: discard + ## else: discard + bind matchImpl + let idx = nt.index + matchImpl(idef(typeof(nt).L), typeof(nt).N, ast, idx, nt, branches) + +macro genProcessor*(index, nterm: untyped): untyped = + ## Generates the body for a non-terminal processor. + # simply emit an empty processorMatchImpl invocation. All branches will be + # auto-generated + # TODO: if none of the productions require a (direct or indirect) call to a + # custom processor, the source and target productions match, and the + # used tags map to the same ID in the source and target language, use a + # memcopy + let sym = bindSym"processorMatchImpl" + result = quote do: + `sym`(idef(src), `nterm`, `index`) + +proc hasPragma(def: NimNode, name: string): bool = + if def.pragma.kind == nnkPragma: + for it in def.pragma.items: + if it.eqIdent(name): + return true + +macro genAdapter[T1, T2; A, B: static string]( + src: typedesc[Metavar[T1, A]], dst: typedesc[Metavar[T2, B]], + orig: untyped) = + # note: the macro signature is very specific because it acts as the type + # checking for programmer-provided processor signatures + let name = ident("->") + copyLineInfo(name, orig) + result = quote do: + template `name`(n: `src`, _: typedesc[`dst`]): `dst` = + {.line.}: `orig`(n) + +macro transformInOutImpl(lang: static LangDef, name, def: untyped) = + ## Implements the transformation for language->language pass processors. + let name = name + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.params[0].kind == nnkEmpty: + error("a return type is required for a transfomer", def.name) + + if def.body.kind == nnkEmpty: + # a forward declaration. Append the additional adapter procedure + return newStmtList(def, + newCall(bindSym"genAdapter", + copyNimTree(def.params[1][^2]), + copyNimTree(def.params[0]), + copyNimNode(def.name))) + + proc transformCase(n: NimNode): NimNode = + result = genAst(arg=n[0]): + processorMatchImpl(idef(src), typeof(arg).N, arg.index) + for i in 1..language passes. + if def.body.kind != nnkEmpty: + error(".generated must be applied to forward declaration", def) + + def.body = newCall(ident"->", def.params[1][0], copyNimTree(def.params[0])) + result = def + +proc assemblePass(src, dst, def, call: NimNode): NimNode = + ## Assembles the final procedure definition for a pass. `def` is the original + ## proc definition, `call` the call to the pass' implementation. + let input = ident"in.ast" + let output = ident"out.ast" + let hasIn = src != nil + let hasOut = dst != nil + + var body = newStmtList() + let (transformImpl, name) = + if hasIn and hasOut: + (bindSym"transformInOutImpl", dst) + elif hasIn: + (bindSym"transformInImpl", src) + elif hasOut: + (bindSym"transformOutImpl", dst) + else: + unreachable() + + body.add quote do: + template transform(p: untyped) {.used.} = + `transformImpl`(def(`name`), `name`, p) + + if hasIn and hasOut: + let impl = bindSym"generatedImpl" + body.add quote do: + template generated(p: untyped) {.used.} = `impl`(p) + + # alias the source and destination language with known names: + if hasIn: + body.add quote do: + template src: untyped {.used.} = `src` + if hasOut: + body.add quote do: + template dst: untyped {.used.} = `dst` + + if hasIn: + let inj = ident"[]" + body.add quote do: + template match(sel: Metavar, branches: varargs[untyped]): untyped {.used.} = + match(`input`, sel, branches) + + template `inj`(x: ChildSlice, i: int): untyped {.used.} = + `input`[x, i] + + template val[T](v: nanopass.Value[T]): T {.used.} = + # TODO: look up the value of the terminal. Also, return a `lent T` + default(typeof(T)) + + if hasOut: + body.add quote do: + template terminal(x: untyped): untyped {.used.} = + embed(`name`, x) + template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = + build(`output`, n, body) + + if hasIn: + # shadow the input tree with a cursor to prevent a costly copy when + # it's captured by the closure + body.add quote do: + let `input` {.cursor.} = `input`.tree + if hasOut: + body.add quote do: + var `output`: PackedTree[uint8] + let index = `call`.index + # turn the AST with indirections into one without + result = Ast[dst](tree: finish(`output`, index)) + else: + body.add quote do: + result = `call` + + def.body = body + # patch the signature: + if hasIn: + def.params[1][^2] = ident"NodeIndex" + def.params.insert(1, newIdentDefs(ident"in.ast", quote do: Ast[`src`])) + if hasOut: + def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst) + + result = def + +macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = + # create a forward declaration for each transformer: + var preamble = newStmtList() + var i = 0 + while i < def.body.len: + let it = def.body[i] + # not ideal, as it breaks some custom macro/template pragmas, but without + # resorting to typed macros, there's no other way to do this transform + if it.kind == nnkProcDef and it.hasPragma("transform"): + if it.body.kind == nnkEmpty: + # remove the user-defined forward declaration + def.body.del(i) + dec i # undo the following `inc` + else: + let backup = it.body + it.body = newEmptyNode() + preamble.add copyNimTree(it) + it.body = backup + inc i + + # add the generic processor procedure, which all processor invocations + # for processors not supplied by the programmer will end up using + let name = ident"->" + preamble.add quote do: + # note: the signature is overly broad, so that overload resolution + # prefers the more specific adapters created for the programmer-provided + # processors + proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = + genProcessor(n.index, T.N) + + # if the body doesn't end in an expression, add a call to the + # entry processor + if def.body[^1].kind == nnkProcDef: + # ^^ a heuristic, but should work okay enough + def.body.add newCall(ident"->", def.params[1][0], + newDotExpr(newDotExpr(dst, ident"meta"), ident"entry")) + + def.body = newStmtList(preamble, def.body) + + let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) + lambda.params = copyNimTree(def.params) + lambda.params[0] = dstnterm + lambda.params[1][^2] = srcnterm + + let call = newCall(lambda) + # forward the original parameters to the lambda: + for i in 1..language pass, that is a + ## pass, that takes an AST (fragment) of language A and produces an AST of + ## language B. + if p.kind != nnkProcDef: + error(".inpass must be applied to a procedure definition", p) + + var target = p.params[0] + if target.kind == nnkEmpty: + error("a return type is required, but none is provided", p.params[0]) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(input = p.params[1][^2], target, p): + when target is Metavar: + passImpl(input, target.L, input, target, p) + else: + # use the entry non-terminal + passImpl(input, target, input.meta.entry, target.meta.entry, p) + +macro outpass*(p: untyped) = + ## Turns a procedure definition into a language->* pass, that is, a pass + ## that takes an AST (fragment) of language A and produces a value. + if p.kind != nnkProcDef: + error(".outpass must be applied to a procedure definition", p) + + let ret = p.params[0] + if ret.kind == nnkEmpty: + error("a return type is required, but none is provided", ret) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(typ = p.params[1][^2], p): + when typ is Metavar: + outpassImpl(typ, typ, p) + else: + # use the entry non-terminal + outpassImpl(typ, typ.entry, p) From 1f0e97dd632eeb2683be537ac747e534e0c7d222 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 18 Aug 2025 22:06:33 +0000 Subject: [PATCH 10/87] passes: fix a few issues --- passes/passes.nim | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/passes/passes.nim b/passes/passes.nim index a3d577c6..a108875a 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -459,12 +459,6 @@ proc symbolize(x: L3, r: ref ReportContext[string]): L4 {.pass.} = error(fmt"undeclared identifier: '{x.val.string}'") build s(^Symbol()) # error correction - proc pattern(n: src.p): dst.p {.transform.} = - # TODO: don't require this processor (will be fixed by moving to using - # generic procedures internally) - case n - of As(x, t): unreachable() - proc rule(n: src.mr): dst.mr {.transform.} = case n of Rule(p, e): @@ -480,7 +474,7 @@ proc symbolize(x: L3, r: ref ReportContext[string]): L4 {.pass.} = of ParamDecl(x, [t]): build ParamDecl(^add(x.val.string), t) - proc params(n: src.pa): dst.pa {.infer.} + proc params(n: src.pa): dst.pa {.generated.} proc decl(n: src.d): dst.d {.transform.} = case n @@ -504,7 +498,7 @@ proc typeCheck(x: L4): L5 {.pass.} = of Seq(str): build Seq(SeqTy(CharTy()), str) of Seq([t], [e]): - build Seq(SeqTy(t), [e]) + build Seq(SeqTy(t), e) of Call([e0], [e1]): build Call(UnitTy(), e0, e1) of FieldAccess([e], n): @@ -662,7 +656,7 @@ proc exprToStmt(x: Lnoletwithval): Lstmt {.pass.} = match e2: of Unit(): build If(e, st) else: build If(e, st, ^etos(e2)) - of Exprs(_, [st0], [st1]): + of Exprs(t, [st0], [st1]): build Stmts(st0, st1) of While([e], [st]): build While(e, st) From c25b249f13d097fa8045762ed01c928ce95af205 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 18 Aug 2025 22:06:33 +0000 Subject: [PATCH 11/87] passes: remove the deprecated `genSym` usage --- passes/passes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/passes/passes.nim b/passes/passes.nim index a108875a..af778ea6 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -929,7 +929,7 @@ macro defineCompiler(name, names: untyped) = var prev = ident"e" let body = newStmtList() for it in names.items: - let tmp = genSym(nskLet, "tmp") + let tmp = genSym() if it.kind == nnkIdent: let name = it.strVal body.add quote do: echo "-- ", `name` From 05b0a5cadd19e2364312c4fb88ad3909b0e59c1f Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Fri, 10 Oct 2025 19:40:00 +0000 Subject: [PATCH 12/87] nanopass: split the module into multiple modules This allows for better encapsulation and control over which symbols are available to whom. Navigating the code should also become a little easier. --- nanopass/helper.nim | 14 + nanopass/nanopass.nim | 1628 +------------------------------------- nanopass/npbuild.nim | 211 +++++ nanopass/nplang.nim | 83 ++ nanopass/nplangdef.nim | 440 +++++++++++ nanopass/nplanggen.nim | 107 +++ nanopass/npmatch.nim | 270 +++++++ nanopass/nppass.nim | 391 +++++++++ nanopass/nppatterns.nim | 40 + nanopass/nptransform.nim | 96 +++ 10 files changed, 1659 insertions(+), 1621 deletions(-) create mode 100644 nanopass/helper.nim create mode 100644 nanopass/npbuild.nim create mode 100644 nanopass/nplang.nim create mode 100644 nanopass/nplangdef.nim create mode 100644 nanopass/nplanggen.nim create mode 100644 nanopass/npmatch.nim create mode 100644 nanopass/nppass.nim create mode 100644 nanopass/nppatterns.nim create mode 100644 nanopass/nptransform.nim diff --git a/nanopass/helper.nim b/nanopass/helper.nim new file mode 100644 index 00000000..45cd7b76 --- /dev/null +++ b/nanopass/helper.nim @@ -0,0 +1,14 @@ +## Implements some helper and utility routines for working with NimNode AST. + +import std/[macros] + +proc copyLineInfoForTree*(n, info: NimNode) = + copyLineInfo(n, info) + for i in 0.. ``x.abc``. - macro dotAux(fname: static string, e: untyped): untyped = - newDotExpr(e, ident(fname)) - - newCall(bindSym"dotAux", fname, e) - -template matches*[T, U](x: T, _: typedesc[U]): bool = - ## Implements type-based pattern matching, used by the nanopass macros. - when U is PArray: - when T is seq: - matches(default(typeof(x[0])), typeof(U.A)) - else: - false - elif U is PChoice: - # why not just use `or`? Because that would unnecessarily expand the - # second `matches` invocation when the case first invocation was sucessful - when matches(x, typeof(U.A)): - true - else: - matches(x, typeof(U.B)) - elif U is Metavar: - when x is U: - true - else: - matches(x, dot(U.L.meta.nt, U.N)) - else: - x is U - -proc lookup[E; M: tuple](): auto {.compileTime.} = - var x: M - for it in fields(x): - when E is typeof(it[0]): - return it[1] +export nppass.genProcessor, nppass.embed +# TODO: ^^ bind the symbols; don't mix them in template isAtom*(x: uint8): bool = ## The predicate required for using an uint8 as a ``PackedTree`` tag. x >= RefTag -proc `$`(x: Form): string = - result = x.tag - result.add "(" - for i, it in x.elems.pairs: - if i > 0: - result.add ", " - result.add it.typ - result.add ")" - -proc `$`*(x: LangDef): string = - for it in x.forms.items: - result.add $it - result.add "\n" - -proc `==`(x, y: Elem): bool = - x.typ == y.typ and x.repeat == y.repeat - -proc `==`(x, y: Form): bool = - ## Compares `x` and `y`, which must belong to the same language, for equality. - x.tag == y.tag and x.elems == y.elems - -proc checkName(target: LangDef, vars: Table[string, string], name: string, - info: NimNode) = - if name in target.terminals: - error(fmt"terminal with name {name} already exists", info) - elif name in target.nterminals: - error(fmt"non-terminal with name {name} already exists", info) - elif name in vars: - error(fmt"'{name}' is already used by a meta-variable for '{vars[name]}'", info) - -proc parseForm(n: NimNode): ParsedForm = - ## Parses a form in the context of a language definition. No semantic checks - ## take place. - n[0].expectKind nnkIdent - result.name = n[0].strVal - for i in 1.. 0: - if name notin base.nterminals: - error(fmt"base language has no non-terminal with name '{name}'", - it.name) - - for prod in it.sub.items: - removeProd(base, prod, name) - - if it.add.len == 0 and - base.nterminals[name].vars.len == 0 and - base.nterminals[name].forms.len == 0: - # the non-terminal is and will stay empty, remove it - base.nterminals.del(name) - - if name in base.nterminals: - # remove the old names: - base.nterminals[name].mvars.shrink(0) - - # update the var list with the to-be-inherited non-terminal meta-vars: - for name, it in base.nterminals.pairs: - for n in it.mvars.items: - vars[n] = name - - # only now set the new meta-variable names for inherited non-terminals: - for it in def.items: - let name = it.name[0].strVal - if name in base.nterminals: - for i in 1.. 0 and name notin base.nterminals: - # it's a new non-terminal - checkName(result, vars, name, it.name) - var nt = NonTerminal() - for i in 1.. 0: - for a in it.add.items: - addProd(result, a, name) - - computeNodeTags(result) - # TODO: properly set the entry non-terminal - result.entry = "module" - -proc makeLanguage(body: NimNode): LangDef = - ## Creates a language definition from the ``defineLanguage`` DSL code. - body.expectMinLen 1 - var add: seq[NimNode] - var def: seq[NonTerminalDef] - - # second pass: process the productions - proc extract(n: NimNode, list: var seq[NimNode]) = - case n.kind - of nnkInfix: - if n[0].eqIdent("|"): - n.expectLen 3 - extract(n[1], list) - extract(n[2], list) - else: - error("expected '|'", n[0]) - of nnkCall, nnkIdent: - list.add n - else: - error("unexpected syntax: " & $n.kind, n) - - for it in body.items: - case it.kind - of nnkInfix: - if it[0].eqIdent("::="): - var nt = NonTerminalDef(name: it[1]) - extract(it[2], nt.add) - def.add nt - continue - of nnkCall: - add.add it - continue - else: - discard "report an error below" - - error("items must be of the form `a in b`, or `a ::= ...`", it) - - # to keep the implementation simple, a non-extension language is treated - # internally as an empty language definition being extended - buildLanguage(add, @[], def, default(LangDef), body) - -proc makeLanguage(base: LangDef, body: NimNode): LangDef = - ## Creates a language definition from the ``defineLanguage`` DSL code - ## and `base`. - var add, sub: seq[NimNode] - var def: seq[NonTerminalDef] - - var body = body - if body.kind != nnkStmtList: - body = newStmtList(body) - - proc extract(n: NimNode, add, sub: var seq[NimNode]) = - case n.kind - of nnkPrefix: - if n[0].eqIdent("+"): - add.add n[1] - elif n[0].eqIdent("-"): - sub.add n[1] - else: - error("expected `+` or `-`", n[0]) - of nnkInfix: - if n[0].eqIdent("|"): - n.expectLen 3 - extract(n[1], add, sub) - extract(n[2], add, sub) - else: - error("expected '|'", n[0]) - else: - error("unexpected syntax: " & $n.kind, n) - - for it in body.items: - var handled = false - case it.kind - of nnkInfix: - if it[0].eqIdent("::="): - it.expectLen 3 - var nt = NonTerminalDef(name: it[1]) - nt.name.expectKind nnkCall - extract(it[2], nt.add, nt.sub) - def.add nt - handled = true - of nnkPrefix: - if it[0].eqIdent("-"): - sub.add it[1] - handled = true - elif it[0].eqIdent("+"): - add.add it[1] - handled = true - of nnkCall: - # non-terminal with no change in productions - def.add NonTerminalDef(name: it) - handled = true - else: - discard - - if not handled: - error("expected `-a`, `+a`, or `a(...) ::= ...`", it[0]) - - buildLanguage(add, sub, def, base, body) - -proc buildLangInfo(def: LangDef): LangInfo = - ## Creates the pass-centric language representation for `def`. - result.map = initTable[string, int](4) - - for name, it in def.terminals.pairs: - result.types.add LangType( - name: it.typ, - mvar: name, - terminal: true, - ntag: it.tag - ) - result.map[name] = high(result.types) - - for name, it in def.nterminals.pairs: - result.types.add LangType( - name: name, - mvar: it.mvars[0], - terminal: false, - forms: mapIt(it.forms, it.semantic) - ) - # add the name-to-type mappings: - result.map[name] = high(result.types) - for x in it.mvars.items: - result.map[x] = high(result.types) - - # now that all name-to-type mappings are present, add the forms and - # the subtype info - for it in def.forms.items: - result.forms.add SForm( - name: it.tag, - ntag: it.id, - elems: mapIt(it.elems, (result.map[it.typ], it.repeat)) - ) - - for name, it in def.nterminals.pairs: - let id = result.map[name] - for v in it.vars.items: - result.types[id].sub.add result.map[v] - -macro makeLanguageType(def: static LangDef, typName: untyped) = - ## Creates the type representing the language defined by `def`. This is the - ## type the nanopass-framework user passes around. - ## - ## The type also stores various information about the language that are - ## needed by the pass-related macros, encoded as types. - let fields = nnkRecList.newTree() - # the metavars are at top level of the type, for easy access by - # the programmer - let mvar = bindSym"Metavar" - for name, it in def.terminals.pairs: - fields.add newIdentDefs(ident(name), - nnkBracketExpr.newTree(bindSym"Value", ident(it.typ))) - for name, it in def.nterminals.pairs: - for m in it.mvars.items: - fields.add newIdentDefs(ident(m), - nnkBracketExpr.newTree( - mvar, - ident(typName.strVal), - newStrLitNode(name))) - - let ntType = nnkTupleTy.newTree() - let (csym, fsym, vsym) = (bindSym"PChoice", bindSym"PForm", bindSym"Value") - # add the descriptions for the non-terminals - for name, nt in def.nterminals.pairs: - let ln = ident(typName.strVal) - var p = ident"void" - for f in nt.forms.items: - let id = def.forms[f.semantic].id - p = quote do: - `csym`[`p`, `fsym`[`id`]] - - for v in nt.vars.items: - if v in def.terminals: - let id = ident(def.terminals[v].typ) - p = quote do: - `csym`[`p`, `vsym`[`id`]] - else: - let id = ident(v) - p = quote do: - `csym`[`p`, `ln`.`id`] - - ntType.add newIdentDefs(ident(name), p) - - # everything meant for internal use is stored in an anonymous record in - # the `meta` field, preventing name clashes and the fields showing up - # in auto-complete suggestions - let metaType = nnkTupleTy.newTree( - newIdentDefs(ident"entry", - nnkBracketExpr.newTree(mvar, - ident(typName.strVal), - newStrLitNode(def.entry))), - newIdentDefs(ident"nt", ntType)) - - # create the terminal->tag map: - let tup = nnkTupleConstr.newTree() - for it in def.terminals.values: - let n = it.tag - tup.add nnkTupleConstr.newTree( - ident(it.typ), - nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(n))) - - metaType.add newIdentDefs(ident"term_map", tup) - - fields.add newIdentDefs(ident"meta", metaType) - - result = nnkTypeSection.newTree( - nnkTypeDef.newTree( - typName, - newEmptyNode(), - nnkObjectTy.newTree( - newEmptyNode(), - newEmptyNode(), - fields))) - -macro genHelpers(typ, a, b: untyped) = - ## Generates the helper templates `def` and `idef`, which are used for - ## retrieving the `LangDef` and `LangInfo` instance for a language type, - ## respectively. - let (def, info) = (bindSym"LangDef", bindSym"LangInfo") - quote do: - template def(_: typedesc[`typ`]): `def` = `a` - template idef(_: typedesc[`typ`]): `info` = `b` - -proc defineLanguageImpl(name, base, body: NimNode): NimNode = - body.expectKind nnkStmtList - body.expectMinLen 1 - if body[0].kind == nnkCommentStmt: - body.del(0) - - let setup1 = - if base.isNil: - genAst(body): makeLanguage(quote do: body) - else: - genAst(body, base): makeLanguage(def(base), quote do: body) - result = genAst(setup1, name): - const - def = setup1 - tmp = buildLangInfo(def) - makeLanguageType(def, name) - genHelpers(name, def, tmp) - macro defineLanguage*(name, body: untyped) = ## Creates a language definition and binds it to a const symbol with the ## given name. @@ -689,19 +34,6 @@ macro defineLanguage*(name, base, body: untyped) = ## context. defineLanguageImpl(name, base, body) -# -------- macro helpers ------- - -proc copyLineInfoForTree(n, info: NimNode) = - copyLineInfo(n, info) - for i in 0..language pass. - if def.kind notin {nnkProcDef, nnkFuncDef}: - error(".transform must be applied to procedure definition", def) - - if def.body.kind == nnkEmpty: - # a forward declaration, bail - return def - - let to = ident"out.ast" - let ret = def.params[0] - def.body = newStmtList(def.body) - def.body.insert 0, quote do: - # inject a build overload that implicitly uses the output language - template build(body: untyped): untyped {.used.} = - build(`to`, `ret`, body) - - result = def - -macro transform(src, dst: static LangDef, nterm: static string, - form: static int, n: untyped): untyped = - ## Generates the transformation from the given source language form - ## (belonging to non-terminal `nterm`) to a target language form with - ## compatible syntax. - # find a target language form that's a production of the non-terminal and has - # the same shape - # TODO: only require the same name and number of elements, using -> to fit - # the rest. **Edit:** really? That can easily lead to non-obvious - # behaviour - var target = -1 - for it in dst.nterminals[nterm].forms.items: - if dst.forms[it.semantic] == src.forms[form]: - target = it.semantic - break - if target == -1: - return makeError(fmt"cannot generate transformer for '{src.forms[form]}'", n) - - # important: the generated code being efficient is of major importance! Most - # transformations will be auto-generated, and they should thus be as fast as - # possible - # TODO: if none of the child nodes require a processor call, memcopy the - # whole sub-tree - # TODO: (bigger refactor) pass the cursor to the transformers as a var - # parameter, which would eliminate unnecessary tree seeking when - # there's many calls to fully auto-generated non-terminal processors - - let inAst = ident"in.ast" - let to = ident"out.ast" - let id = dst.forms[target].id.uint8 - result = newStmtList() - # add the root node: - let body = quote do: - var tmp {.used.} = `inAst`.child(`n`, 0) - let root = `to`.nodes.len.NodeIndex - var i = `to`.nodes.len - # the node sequence needs to be contiguous, so it's allocated upfront - `to`.nodes.setLen(i + `inAst`.len(`n`) + 1) - `to`.nodes[i] = TreeNode[uint8](kind: `id`, val: `inAst`[`n`].val) - inc i - - # call the transformers and emit the nodes in one go: - for i, it in src.forms[form].elems.pairs: - let fromTerminal = it.typ in src.terminals - let toTerminal = it.typ in dst.terminals - if fromTerminal != toTerminal or - (toTerminal and src.terminals[it.typ].typ != src.terminals[it.typ].typ): - body.add makeError(fmt"cannot generate transformer for {src.forms[form]}", n) - break - - let call = - if fromTerminal: - if src.terminals[it.typ].tag == dst.terminals[it.typ].tag: - # just copy the node - quote do: - `to`.nodes[i] = `inAst`[tmp] - inc i - else: - # repack with the new tag - let tag = dst.terminals[it.typ].tag - quote do: - `to`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `inAst`[tmp].val) - inc i - else: - let append = bindSym"append" - let op = ident"->" - let s = newStrLitNode(it.typ) - let d = newStrLitNode(dst.forms[target].elems[i].typ) - quote do: - `append`(`to`, i, - `op`(Metavar[src, `s`](index: tmp), Metavar[dst, `d`])) - - if it.repeat: - let bias = src.forms[form].elems.len - 1 - body.add quote do: - for _ in 0..<(`inAst`.len(`n`) - `bias`): - `call` - tmp = `inAst`.next(tmp) - else: - body.add quote do: - `call` - tmp = `inAst`.next(tmp) - - result.add body - # the callsite takes care of fitting the index to the right type - result.add ident"root" - -proc ntags(lang: LangInfo, typ: LangType): seq[int] = - ## Returns a list with all possible node tags productions of `typ` can have. - for it in typ.forms.items: - result.add lang.forms[it].ntag - - for it in typ.sub.items: - if lang.types[it].terminal: - result.add lang.types[it].ntag - else: - result.add ntags(lang, lang.types[it]) - -proc matchImpl(lang: LangInfo, src: int, ast, sel, rules: NimNode - ): (seq[NimNode], IntSet) = - ## Implements the core of the `match` macro: - ## 1. makes sure the syntax is correct - ## 2. makes sure the used patterns are unique - ## 3. generates a sequence of transformed 'of' branches, plus a set storing - ## the used forms' tags - var used: IntSet - ## covered form productions (identified by ntag) - - # should nested matching (e.g., ``A(x, B(y))``) be desired, `matchImpl` - # should be factored into two macros macros applied sequentially: - # 1. the first macro does type checking, producing a type form - # 2. the second macro translates the typed form into case/if statements - # combining both steps into one is simple enough when there are no nested - # patterns, but not otherwise - - proc parseVar(n: NimNode): (string, string) = - n.expectKind nnkIdent - let name = n.strVal - var e = name.high - # to get the name of the var, trim trailing numbers and a single underscore - while e >= 0 and name[e] in {'0'..'9'}: - dec e - - if e >= 0 and name[e] == '_': - dec e - - result[0] = name[0..e] - result[1] = name - - proc processIdentPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = - n.expectKind nnkIdent - let (v, nameStr) = parseVar(n) - if v notin lang.map: - error(fmt"no meta-variable with name '{v}'", n) - let id = lang.map[v] - if id notin lang.types[src].sub: - error(fmt"'{v}' is not an immediate production of '{lang.types[src].name}'", n) - - var check, binds: NimNode - let name = ident(nameStr) - if lang.types[id].terminal: - let typ = ident(lang.types[id].name) - let tag = lang.types[id].ntag - used.incl(tag) - # let the compiler report an error for duplicate case labels - check = newIntLitNode(tag) - copyLineInfo(check, n) - binds = newLetStmt(name, quote do: Value[`typ`](index: `ast`[`sel`].val)) - else: - # the pattern binds a non-terminal - let typ = ident(v) - check = nnkCurly.newTree() - let tags = ntags(lang, lang.types[id]) - for tag in tags.items: - check.add nnkConv.newTree(ident"uint8", newIntLitNode(tag)) - # mark the first tag as used, to signal that the non-terminal is handled - used.incl(tags[0]) - - copyLineInfoForTree(check, n) - binds = newLetStmt(name, quote do: src.`typ`(index: `sel`)) - - result = (check, binds) - - proc processPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = - case n.kind - of nnkCall: - n[0].expectKind nnkIdent - # parse the pattern: - var elems = newSeq[tuple[src, dst, name: string]](n.len - 1) - for i in 1.. 0: ident(elems[i].dst) - else: ident(lang.types[it.typ].mvar) - if it.repeat: - let bias = lang.forms[idx].elems.len - 1 - if p.kind == nnkIdent: - # just bind a child slice to the identifier - binds.add newLetStmt(p, quote do: - slice[`origin`](`cursor`, uint32(`ast`.len(`sel`) - `bias`))) - binds.add quote do: - for _ in 0..<`ast`.len(`sel`)-`bias`: - `cursor` = `ast`.next(`cursor`) - else: - # run the selected transformer on all relevant child nodes and - # store the result in a seq - let tmp = genSym() - binds.add newVarStmt(tmp, - quote do: newSeq[`target`](`ast`.len(`sel`)-`bias`)) - let callee = ident"->" - binds.add quote do: - for i in 0..<`ast`.len(`sel`)-`bias`: - `tmp`[i] = `callee`(`origin`(index: `cursor`), `target`) - `cursor` = `ast`.next(`cursor`) - binds.add newLetStmt(p[0], tmp) - else: - # simple case: a single node - if p.kind == nnkIdent: - if lang.types[it.typ].terminal: - binds.add newLetStmt(p, quote do: `origin`(index: `ast`[`cursor`].val)) - else: - binds.add newLetStmt(p, quote do: `origin`(index: `cursor`)) - else: - if lang.types[it.typ].terminal: - binds.add makeError("cannot invoke auto-procesor for terminal", p) - else: - binds.add newLetStmt(p[0], - newCall(ident"->", - nnkObjConstr.newTree(origin, - nnkExprColonExpr.newTree(ident"index", cursor)), - target)) - binds.add quote do: - `cursor` = `ast`.next(`cursor`) - result = (check, binds) - of nnkIdent: - # must be a terminal/non-terminal meta-var - result = processIdentPattern(lang, n) - else: - error("unexpected syntax", n) - - var branches: seq[NimNode] - for it in rules.items: - case it.kind - of nnkOfBranch: - it.expectLen 2 - let (check, binds) = processPattern(lang, it[0]) - branches.add nnkOfBranch.newTree(check, newStmtList(binds, it[1])) - of nnkElse: - # as a guard against malformed run-time inputs, use an 'of' instead - # of an 'else' branch - var ofb = nnkOfBranch.newTree() - # add all remaining forms: - for it in lang.types[src].forms.items: - if not containsOrIncl(used, lang.forms[it].ntag): - ofb.add newIntLitNode(lang.forms[it].ntag) - # also include the tags for sub non-terminals and terminals: - for it in lang.types[src].sub.items: - if lang.types[it].terminal: - if not containsOrIncl(used, lang.types[it].ntag): - ofb.add newIntLitNode(lang.types[it].ntag) - else: - let tags = ntags(lang, lang.types[it]) - if not containsOrIncl(used, tags[0]): - for tag in tags.items: - ofb.add newIntLitNode(tag) - - if ofb.len == 0: - # all forms are handled already. Add an 'else' branch before the - # programmer-provided one so that the compiler can report an - # "unreachable" warning - branches.add nnkElse.newTree(newCall(ident"unreachable")) - branches.add it - else: - copyLineInfo(ofb, it) - ofb.add it[0] - branches.add ofb - else: - error("expected 'of' or 'else'", it) - - result = (branches, used) - -macro processorMatchImpl(lang: static LangInfo, src: static string, - sel: untyped, rules: varargs[untyped]): untyped = - ## Implements the transformation/expansion of a language->language - ## processor's trailing case statement/expression. - let id = lang.map[src] - var (branches, used) = matchImpl(lang, id, ident"in.ast", sel, rules) - - template nt: untyped = lang.types[id] - let sym = bindSym"transform" - # auto-generate the transformers for the forms not manually provided: - for it in nt.forms.items: - let id = lang.forms[it].ntag - if not containsOrIncl(used, id): - branches.add nnkOfBranch.newTree( - newIntLitNode(id), - (quote do: - Metavar[dst, `src`](index: `sym`(def(src), def(dst), `src`, `it`, `sel`)))) - - # auto-generate the branches for missing production terminals and - # non-terminals: - for it in nt.sub.items: - if lang.types[it].terminal: - let id = lang.types[it].ntag - if id notin used: - let to = ident"out.ast" - let input = ident"in.ast" - branches.add nnkOfBranch.newTree( - newIntLitNode(id), - (quote do: - # TODO: use the tag of the destination language - `to`.nodes.add: - TreeNode[uint8](kind: uint8(`id`), val: `input`[`sel`].val) - Metavar[dst, `src`](index: NodeIndex(`to`.nodes.high)))) - else: - # if the first form's tag is used, so is the non-terminal itself - if lang.forms[lang.types[it].forms[0]].ntag notin used: - # TODO: consider inlining the transformer if it's auto-generated. - # More code to emit, but also a little less work at run-time - let branch = nnkOfBranch.newTree() - for tag in ntags(lang, lang.types[it]).items: - branch.add newIntLitNode(tag) - let name = ident(lang.types[it].mvar) - let callee = ident"->" - # dispatch to the processor and convert to the expected type - branch.add quote do: - Metavar[dst, `src`]( - index: `callee`(src.`name`(index: `sel`), dst.`name`).index) - branches.add branch - - let input = ident"in.ast" - result = nnkCaseStmt.newTree(quote do: `input`[`sel`].kind) - result.add branches - if branches[^1].kind != nnkElse: - # the selector is a uint8 and thus the case cannot be exhaustive - result.add nnkElse.newTree(newCall(ident"unreachable")) - -macro matchImpl(lang: static LangInfo, nterm: static string, - ast: PackedTree[uint8], sel: NodeIndex, info: untyped, - rules: varargs[untyped]): untyped = - ## The internal implementation `match` dispatches to. - let (branches, used) = matchImpl(lang, lang.map[nterm], ast, sel, rules) - result = nnkCaseStmt.newTree(quote do: `ast`[`sel`].kind) - copyLineInfoForTree(result, info) - result.add branches - # add a default handler when all possible productions are covered - var allCovered = true - for it in lang.types[lang.map[nterm]].sub.items: - if lang.types[it].terminal: - if lang.types[it].ntag notin used: - allCovered = false - break - elif lang.forms[lang.types[it].forms[0]].ntag notin used: - allCovered = false - break - - if allCovered: - result.add nnkElse.newTree(newCall(ident"unreachable")) - -template match*(ast: PackedTree[uint8], nt: Metavar, - branches: varargs[untyped]): untyped = - ## Provides a convenient way to destructure an AST. Meant to be used as - ## follows: - ## - ## .. code-block:: nim - ## - ## match ast, n: - ## of ...: discard - ## of ...: discard - ## else: discard - bind matchImpl - let idx = nt.index - matchImpl(idef(typeof(nt).L), typeof(nt).N, ast, idx, nt, branches) - -macro genProcessor*(index, nterm: untyped): untyped = - ## Generates the body for a non-terminal processor. - # simply emit an empty processorMatchImpl invocation. All branches will be - # auto-generated - # TODO: if none of the productions require a (direct or indirect) call to a - # custom processor, the source and target productions match, and the - # used tags map to the same ID in the source and target language, use a - # memcopy - let sym = bindSym"processorMatchImpl" - result = quote do: - `sym`(idef(src), `nterm`, `index`) - -proc hasPragma(def: NimNode, name: string): bool = - if def.pragma.kind == nnkPragma: - for it in def.pragma.items: - if it.eqIdent(name): - return true - -macro genAdapter[T1, T2; A, B: static string]( - src: typedesc[Metavar[T1, A]], dst: typedesc[Metavar[T2, B]], - orig: untyped) = - # note: the macro signature is very specific because it acts as the type - # checking for programmer-provided processor signatures - let name = ident("->") - copyLineInfo(name, orig) - result = quote do: - template `name`(n: `src`, _: typedesc[`dst`]): `dst` = - {.line.}: `orig`(n) - -macro transformInOutImpl(lang: static LangDef, name, def: untyped) = - ## Implements the transformation for language->language pass processors. - let name = name - if def.kind notin {nnkProcDef, nnkFuncDef}: - error(".transform must be applied to procedure definition", def) - - if def.params[0].kind == nnkEmpty: - error("a return type is required for a transfomer", def.name) - - if def.body.kind == nnkEmpty: - # a forward declaration. Append the additional adapter procedure - return newStmtList(def, - newCall(bindSym"genAdapter", - copyNimTree(def.params[1][^2]), - copyNimTree(def.params[0]), - copyNimNode(def.name))) - - proc transformCase(n: NimNode): NimNode = - result = genAst(arg=n[0]): - processorMatchImpl(idef(src), typeof(arg).N, arg.index) - for i in 1..language passes. - if def.body.kind != nnkEmpty: - error(".generated must be applied to forward declaration", def) - - def.body = newCall(ident"->", def.params[1][0], copyNimTree(def.params[0])) - result = def - -proc assemblePass(src, dst, def, call: NimNode): NimNode = - ## Assembles the final procedure definition for a pass. `def` is the original - ## proc definition, `call` the call to the pass' implementation. - let input = ident"in.ast" - let output = ident"out.ast" - let hasIn = src != nil - let hasOut = dst != nil - - var body = newStmtList() - let (transformImpl, name) = - if hasIn and hasOut: - (bindSym"transformInOutImpl", dst) - elif hasIn: - (bindSym"transformInImpl", src) - elif hasOut: - (bindSym"transformOutImpl", dst) - else: - unreachable() - - body.add quote do: - template transform(p: untyped) {.used.} = - `transformImpl`(def(`name`), `name`, p) - - if hasIn and hasOut: - let impl = bindSym"generatedImpl" - body.add quote do: - template generated(p: untyped) {.used.} = `impl`(p) - - # alias the source and destination language with known names: - if hasIn: - body.add quote do: - template src: untyped {.used.} = `src` - if hasOut: - body.add quote do: - template dst: untyped {.used.} = `dst` - - if hasIn: - let inj = ident"[]" - body.add quote do: - template match(sel: Metavar, branches: varargs[untyped]): untyped {.used.} = - match(`input`, sel, branches) - - template `inj`(x: ChildSlice, i: int): untyped {.used.} = - `input`[x, i] - - template val[T](v: nanopass.Value[T]): T {.used.} = - # TODO: look up the value of the terminal. Also, return a `lent T` - default(typeof(T)) - - if hasOut: - body.add quote do: - template terminal(x: untyped): untyped {.used.} = - embed(`name`, x) - template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = - build(`output`, n, body) - - if hasIn: - # shadow the input tree with a cursor to prevent a costly copy when - # it's captured by the closure - body.add quote do: - let `input` {.cursor.} = `input`.tree - if hasOut: - body.add quote do: - var `output`: PackedTree[uint8] - let index = `call`.index - # turn the AST with indirections into one without - result = Ast[dst](tree: finish(`output`, index)) - else: - body.add quote do: - result = `call` - - def.body = body - # patch the signature: - if hasIn: - def.params[1][^2] = ident"NodeIndex" - def.params.insert(1, newIdentDefs(ident"in.ast", quote do: Ast[`src`])) - if hasOut: - def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst) - - result = def - -macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = - # create a forward declaration for each transformer: - var preamble = newStmtList() - var i = 0 - while i < def.body.len: - let it = def.body[i] - # not ideal, as it breaks some custom macro/template pragmas, but without - # resorting to typed macros, there's no other way to do this transform - if it.kind == nnkProcDef and it.hasPragma("transform"): - if it.body.kind == nnkEmpty: - # remove the user-defined forward declaration - def.body.del(i) - dec i # undo the following `inc` - else: - let backup = it.body - it.body = newEmptyNode() - preamble.add copyNimTree(it) - it.body = backup - inc i - - # add the generic processor procedure, which all processor invocations - # for processors not supplied by the programmer will end up using - let name = ident"->" - preamble.add quote do: - # note: the signature is overly broad, so that overload resolution - # prefers the more specific adapters created for the programmer-provided - # processors - proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = - genProcessor(n.index, T.N) - - # if the body doesn't end in an expression, add a call to the - # entry processor - if def.body[^1].kind == nnkProcDef: - # ^^ a heuristic, but should work okay enough - def.body.add newCall(ident"->", def.params[1][0], - newDotExpr(newDotExpr(dst, ident"meta"), ident"entry")) - - def.body = newStmtList(preamble, def.body) - - let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) - lambda.params = copyNimTree(def.params) - lambda.params[0] = dstnterm - lambda.params[1][^2] = srcnterm - - let call = newCall(lambda) - # forward the original parameters to the lambda: - for i in 1..language pass, that is a - ## pass, that takes an AST (fragment) of language A and produces an AST of - ## language B. - if p.kind != nnkProcDef: - error(".inpass must be applied to a procedure definition", p) - - var target = p.params[0] - if target.kind == nnkEmpty: - error("a return type is required, but none is provided", p.params[0]) - if p.params.len == 1: - error("the input parameter is missing", p.params) - - result = genAst(input = p.params[1][^2], target, p): - when target is Metavar: - passImpl(input, target.L, input, target, p) - else: - # use the entry non-terminal - passImpl(input, target, input.meta.entry, target.meta.entry, p) - -macro outpass*(p: untyped) = - ## Turns a procedure definition into a language->* pass, that is, a pass - ## that takes an AST (fragment) of language A and produces a value. - if p.kind != nnkProcDef: - error(".outpass must be applied to a procedure definition", p) - - let ret = p.params[0] - if ret.kind == nnkEmpty: - error("a return type is required, but none is provided", ret) - if p.params.len == 1: - error("the input parameter is missing", p.params) - - result = genAst(typ = p.params[1][^2], p): - when typ is Metavar: - outpassImpl(typ, typ, p) - else: - # use the entry non-terminal - outpassImpl(typ, typ.entry, p) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim new file mode 100644 index 00000000..f8949794 --- /dev/null +++ b/nanopass/npbuild.nim @@ -0,0 +1,211 @@ +## Implements the `build` macro, for constructing abstract syntax trees. + +import std/[genasts, macros, strformat, tables] +import passes/trees +import nanopass/[asts, helper, nplang, nplangdef, nppatterns] + +proc lookup[E; M: tuple](): auto {.compileTime.} = + var x: M + for it in fields(x): + when E is typeof(it[0]): + return it[1] + +proc append[L, U](to: var PackedTree[uint8], x: Value[U]) = + to.nodes.add TreeNode[uint8](kind: typeof(lookup[U, L.meta.term_map]()).V) + +proc append[L](to: var PackedTree[uint8], x: Metavar) = + to.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) + +proc append[L](to: var PackedTree[uint8], x: openArray) = + for it in x.items: + append[L](to, it) + +# helpers for `buildImpl` +template len(x: Value): int = 1 +template len(x: Metavar): int = 1 + +macro buildImpl(to: var PackedTree[uint8], lang: static LangInfo, + typ: typedesc[Metavar], e: untyped): untyped = + ## Emits a tree construction for the AST described by `e`, with the syntax + ## from `lang`. + proc cons(lang: LangInfo, n, test, body: NimNode): NimNode {.closure.} + + proc elem(lang: LangInfo, n, test, body: NimNode): NimNode = + ## Processor for element syntax. + case n.kind + of nnkCall: + result = cons(lang, n, test, body) + of nnkBracket: + result = nnkBracket.newTree() + for it in n.items: + result.add elem(lang, it, test, body) + of nnkIdent, nnkSym, nnkAccQuoted: + # some hoisted expression + result = n + body.add genAst(typ, to, n) do: + append[typ.L](to, n) + else: + error("unexpected syntax", n) + + proc form(lang: LangInfo, n, test, body: NimNode): NimNode = + ## Processor for a tree construction. + let tag = n[0].strVal + var elems: seq[NimNode] + var temp = newStmtList() + for i in 1.. 0: + result.add ", " + result.add it.typ + result.add ")" + +proc `$`*(x: LangDef): string = + for it in x.forms.items: + result.add $it + result.add "\n" + +proc `==`(x, y: Elem): bool = + x.typ == y.typ and x.repeat == y.repeat + +proc `==`(x, y: Form): bool = + ## Compares `x` and `y`, which must belong to the same language, for equality. + x.tag == y.tag and x.elems == y.elems + +proc checkName(target: LangDef, vars: Table[string, string], name: string, + info: NimNode) = + if name in target.terminals: + error(fmt"terminal with name {name} already exists", info) + elif name in target.nterminals: + error(fmt"non-terminal with name {name} already exists", info) + elif name in vars: + error(fmt"'{name}' is already used by a meta-variable for '{vars[name]}'", info) + +proc parseForm(n: NimNode): ParsedForm = + ## Parses a form in the context of a language definition. No semantic checks + ## take place. + n[0].expectKind nnkIdent + result.name = n[0].strVal + for i in 1.. 0: + if name notin base.nterminals: + error(fmt"base language has no non-terminal with name '{name}'", + it.name) + + for prod in it.sub.items: + removeProd(base, prod, name) + + if it.add.len == 0 and + base.nterminals[name].vars.len == 0 and + base.nterminals[name].forms.len == 0: + # the non-terminal is and will stay empty, remove it + base.nterminals.del(name) + + if name in base.nterminals: + # remove the old names: + base.nterminals[name].mvars.shrink(0) + + # update the var list with the to-be-inherited non-terminal meta-vars: + for name, it in base.nterminals.pairs: + for n in it.mvars.items: + vars[n] = name + + # only now set the new meta-variable names for inherited non-terminals: + for it in def.items: + let name = it.name[0].strVal + if name in base.nterminals: + for i in 1.. 0 and name notin base.nterminals: + # it's a new non-terminal + checkName(result, vars, name, it.name) + var nt = NonTerminal() + for i in 1.. 0: + for a in it.add.items: + addProd(result, a, name) + + computeNodeTags(result) + # TODO: properly set the entry non-terminal + result.entry = "module" + +proc makeLanguage*(body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code. + body.expectMinLen 1 + var add: seq[NimNode] + var def: seq[NonTerminalDef] + + # second pass: process the productions + proc extract(n: NimNode, list: var seq[NimNode]) = + case n.kind + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], list) + extract(n[2], list) + else: + error("expected '|'", n[0]) + of nnkCall, nnkIdent: + list.add n + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add) + def.add nt + continue + of nnkCall: + add.add it + continue + else: + discard "report an error below" + + error("items must be of the form `a in b`, or `a ::= ...`", it) + + # to keep the implementation simple, a non-extension language is treated + # internally as an empty language definition being extended + buildLanguage(add, @[], def, default(LangDef), body) + +proc makeLanguage*(base: LangDef, body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code + ## and `base`. + var add, sub: seq[NimNode] + var def: seq[NonTerminalDef] + + var body = body + if body.kind != nnkStmtList: + body = newStmtList(body) + + proc extract(n: NimNode, add, sub: var seq[NimNode]) = + case n.kind + of nnkPrefix: + if n[0].eqIdent("+"): + add.add n[1] + elif n[0].eqIdent("-"): + sub.add n[1] + else: + error("expected `+` or `-`", n[0]) + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], add, sub) + extract(n[2], add, sub) + else: + error("expected '|'", n[0]) + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + var handled = false + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + it.expectLen 3 + var nt = NonTerminalDef(name: it[1]) + nt.name.expectKind nnkCall + extract(it[2], nt.add, nt.sub) + def.add nt + handled = true + of nnkPrefix: + if it[0].eqIdent("-"): + sub.add it[1] + handled = true + elif it[0].eqIdent("+"): + add.add it[1] + handled = true + of nnkCall: + # non-terminal with no change in productions + def.add NonTerminalDef(name: it) + handled = true + else: + discard + + if not handled: + error("expected `-a`, `+a`, or `a(...) ::= ...`", it[0]) + + buildLanguage(add, sub, def, base, body) diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim new file mode 100644 index 00000000..67a4d630 --- /dev/null +++ b/nanopass/nplanggen.nim @@ -0,0 +1,107 @@ +## Implements the macros and routines handling the generative part of +## language definition. + +import std/[genasts, macros, tables] +import nanopass/[asts, nplang, nplangdef, nppatterns] + +macro makeLanguageType(def: static LangDef, typName: untyped) = + ## Creates the type representing the language defined by `def`. This is the + ## type the nanopass-framework user passes around. + ## + ## The type also stores various information about the language that are + ## needed by the pass-related macros, encoded as types. + let fields = nnkRecList.newTree() + # the metavars are at top level of the type, for easy access by + # the programmer + let mvar = bindSym"Metavar" + for name, it in def.terminals.pairs: + fields.add newIdentDefs(ident(name), + nnkBracketExpr.newTree(bindSym"Value", ident(it.typ))) + for name, it in def.nterminals.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree( + mvar, + ident(typName.strVal), + newStrLitNode(name))) + + let ntType = nnkTupleTy.newTree() + let (csym, fsym, vsym) = (bindSym"PChoice", bindSym"PForm", bindSym"Value") + # add the descriptions for the non-terminals + for name, nt in def.nterminals.pairs: + let ln = ident(typName.strVal) + var p = ident"void" + for f in nt.forms.items: + let id = def.forms[f.semantic].id + p = quote do: + `csym`[`p`, `fsym`[`id`]] + + for v in nt.vars.items: + if v in def.terminals: + let id = ident(def.terminals[v].typ) + p = quote do: + `csym`[`p`, `vsym`[`id`]] + else: + let id = ident(v) + p = quote do: + `csym`[`p`, `ln`.`id`] + + ntType.add newIdentDefs(ident(name), p) + + # everything meant for internal use is stored in an anonymous record in + # the `meta` field, preventing name clashes and the fields showing up + # in auto-complete suggestions + let metaType = nnkTupleTy.newTree( + newIdentDefs(ident"entry", + nnkBracketExpr.newTree(mvar, + ident(typName.strVal), + newStrLitNode(def.entry))), + newIdentDefs(ident"nt", ntType)) + + # create the terminal->tag map: + let tup = nnkTupleConstr.newTree() + for it in def.terminals.values: + let n = it.tag + tup.add nnkTupleConstr.newTree( + ident(it.typ), + nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(n))) + + metaType.add newIdentDefs(ident"term_map", tup) + + fields.add newIdentDefs(ident"meta", metaType) + + result = nnkTypeSection.newTree( + nnkTypeDef.newTree( + typName, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + fields))) + +macro genHelpers(typ, a, b: untyped) = + ## Generates the helper templates `def` and `idef`, which are used for + ## retrieving the `LangDef` and `LangInfo` instance for a language type, + ## respectively. + let (def, info) = (bindSym"LangDef", bindSym"LangInfo") + quote do: + template def(_: typedesc[`typ`]): `def` = `a` + template idef(_: typedesc[`typ`]): `info` = `b` + +proc defineLanguageImpl*(name, base, body: NimNode): NimNode = + body.expectKind nnkStmtList + body.expectMinLen 1 + if body[0].kind == nnkCommentStmt: + body.del(0) + + let setup1 = + if base.isNil: + genAst(body): makeLanguage(quote do: body) + else: + genAst(body, base): makeLanguage(def(base), quote do: body) + result = genAst(setup1, name): + const + def = setup1 + tmp = buildLangInfo(def) + makeLanguageType(def, name) + genHelpers(name, def, tmp) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim new file mode 100644 index 00000000..1d6be86c --- /dev/null +++ b/nanopass/npmatch.nim @@ -0,0 +1,270 @@ +## Implements the high and low-level `match` macros. + +import std/[macros, intsets, strformat, tables] +import passes/trees +import nanopass/[asts, helper, nplang] + +proc ntags*(lang: LangInfo, typ: LangType): seq[int] = + ## Returns a list with all possible node tags productions of `typ` can have. + for it in typ.forms.items: + result.add lang.forms[it].ntag + + for it in typ.sub.items: + if lang.types[it].terminal: + result.add lang.types[it].ntag + else: + result.add ntags(lang, lang.types[it]) + +proc matchImpl*(lang: LangInfo, src: int, ast, sel, rules: NimNode + ): (seq[NimNode], IntSet) = + ## Implements the core of the `match` macro: + ## 1. makes sure the syntax is correct + ## 2. makes sure the used patterns are unique + ## 3. generates a sequence of transformed 'of' branches, plus a set storing + ## the used forms' tags + var used: IntSet + ## covered form productions (identified by ntag) + + # should nested matching (e.g., ``A(x, B(y))``) be desired, `matchImpl` + # should be factored into two macros macros applied sequentially: + # 1. the first macro does type checking, producing a type form + # 2. the second macro translates the typed form into case/if statements + # combining both steps into one is simple enough when there are no nested + # patterns, but not otherwise + + proc parseVar(n: NimNode): (string, string) = + n.expectKind nnkIdent + let name = n.strVal + var e = name.high + # to get the name of the var, trim trailing numbers and a single underscore + while e >= 0 and name[e] in {'0'..'9'}: + dec e + + if e >= 0 and name[e] == '_': + dec e + + result[0] = name[0..e] + result[1] = name + + proc processIdentPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = + n.expectKind nnkIdent + let (v, nameStr) = parseVar(n) + if v notin lang.map: + error(fmt"no meta-variable with name '{v}'", n) + let id = lang.map[v] + if id notin lang.types[src].sub: + error(fmt"'{v}' is not an immediate production of '{lang.types[src].name}'", n) + + var check, binds: NimNode + let name = ident(nameStr) + if lang.types[id].terminal: + let typ = ident(lang.types[id].name) + let tag = lang.types[id].ntag + used.incl(tag) + # let the compiler report an error for duplicate case labels + check = newIntLitNode(tag) + copyLineInfo(check, n) + binds = newLetStmt(name, quote do: Value[`typ`](index: `ast`[`sel`].val)) + else: + # the pattern binds a non-terminal + let typ = ident(v) + check = nnkCurly.newTree() + let tags = ntags(lang, lang.types[id]) + for tag in tags.items: + check.add nnkConv.newTree(ident"uint8", newIntLitNode(tag)) + # mark the first tag as used, to signal that the non-terminal is handled + used.incl(tags[0]) + + copyLineInfoForTree(check, n) + binds = newLetStmt(name, quote do: src.`typ`(index: `sel`)) + + result = (check, binds) + + proc processPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = + case n.kind + of nnkCall: + n[0].expectKind nnkIdent + # parse the pattern: + var elems = newSeq[tuple[src, dst, name: string]](n.len - 1) + for i in 1.. 0: ident(elems[i].dst) + else: ident(lang.types[it.typ].mvar) + if it.repeat: + let bias = lang.forms[idx].elems.len - 1 + if p.kind == nnkIdent: + # just bind a child slice to the identifier + binds.add newLetStmt(p, quote do: + slice[`origin`](`cursor`, uint32(`ast`.len(`sel`) - `bias`))) + binds.add quote do: + for _ in 0..<`ast`.len(`sel`)-`bias`: + `cursor` = `ast`.next(`cursor`) + else: + # run the selected transformer on all relevant child nodes and + # store the result in a seq + let tmp = genSym() + binds.add newVarStmt(tmp, + quote do: newSeq[`target`](`ast`.len(`sel`)-`bias`)) + let callee = ident"->" + binds.add quote do: + for i in 0..<`ast`.len(`sel`)-`bias`: + `tmp`[i] = `callee`(`origin`(index: `cursor`), `target`) + `cursor` = `ast`.next(`cursor`) + binds.add newLetStmt(p[0], tmp) + else: + # simple case: a single node + if p.kind == nnkIdent: + if lang.types[it.typ].terminal: + binds.add newLetStmt(p, quote do: `origin`(index: `ast`[`cursor`].val)) + else: + binds.add newLetStmt(p, quote do: `origin`(index: `cursor`)) + else: + if lang.types[it.typ].terminal: + binds.add makeError("cannot invoke auto-procesor for terminal", p) + else: + binds.add newLetStmt(p[0], + newCall(ident"->", + nnkObjConstr.newTree(origin, + nnkExprColonExpr.newTree(ident"index", cursor)), + target)) + binds.add quote do: + `cursor` = `ast`.next(`cursor`) + result = (check, binds) + of nnkIdent: + # must be a terminal/non-terminal meta-var + result = processIdentPattern(lang, n) + else: + error("unexpected syntax", n) + + var branches: seq[NimNode] + for it in rules.items: + case it.kind + of nnkOfBranch: + it.expectLen 2 + let (check, binds) = processPattern(lang, it[0]) + branches.add nnkOfBranch.newTree(check, newStmtList(binds, it[1])) + of nnkElse: + # as a guard against malformed run-time inputs, use an 'of' instead + # of an 'else' branch + var ofb = nnkOfBranch.newTree() + # add all remaining forms: + for it in lang.types[src].forms.items: + if not containsOrIncl(used, lang.forms[it].ntag): + ofb.add newIntLitNode(lang.forms[it].ntag) + # also include the tags for sub non-terminals and terminals: + for it in lang.types[src].sub.items: + if lang.types[it].terminal: + if not containsOrIncl(used, lang.types[it].ntag): + ofb.add newIntLitNode(lang.types[it].ntag) + else: + let tags = ntags(lang, lang.types[it]) + if not containsOrIncl(used, tags[0]): + for tag in tags.items: + ofb.add newIntLitNode(tag) + + if ofb.len == 0: + # all forms are handled already. Add an 'else' branch before the + # programmer-provided one so that the compiler can report an + # "unreachable" warning + branches.add nnkElse.newTree(newCall(ident"unreachable")) + branches.add it + else: + copyLineInfo(ofb, it) + ofb.add it[0] + branches.add ofb + else: + error("expected 'of' or 'else'", it) + + result = (branches, used) + + +macro matchImpl(lang: static LangInfo, nterm: static string, + ast: PackedTree[uint8], sel: NodeIndex, info: untyped, + rules: varargs[untyped]): untyped = + ## The internal implementation `match` dispatches to. + let (branches, used) = matchImpl(lang, lang.map[nterm], ast, sel, rules) + result = nnkCaseStmt.newTree(quote do: `ast`[`sel`].kind) + copyLineInfoForTree(result, info) + result.add branches + # add a default handler when all possible productions are covered + var allCovered = true + for it in lang.types[lang.map[nterm]].sub.items: + if lang.types[it].terminal: + if lang.types[it].ntag notin used: + allCovered = false + break + elif lang.forms[lang.types[it].forms[0]].ntag notin used: + allCovered = false + break + + if allCovered: + result.add nnkElse.newTree(newCall(ident"unreachable")) + +template match*(ast: PackedTree[uint8], nt: Metavar, + branches: varargs[untyped]): untyped = + ## Provides a convenient way to destructure an AST. Meant to be used as + ## follows: + ## + ## .. code-block:: nim + ## + ## match ast, n: + ## of ...: discard + ## of ...: discard + ## else: discard + bind matchImpl + let idx = nt.index + matchImpl(idef(typeof(nt).L), typeof(nt).N, ast, idx, nt, branches) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim new file mode 100644 index 00000000..d276409c --- /dev/null +++ b/nanopass/nppass.nim @@ -0,0 +1,391 @@ +## Implements the various pass macros. + +import std/[genasts, macros, packedsets, tables] +import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] + +template embed*(lang: typedesc, arg: untyped): untyped = + # TODO: implement, and don't export + let tmp = arg + Value[typeof(tmp)]() + +macro transformOutImpl(lang: static LangDef, name, def: untyped) = + ## Implements the transformation for processors in an *->language pass. + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.body.kind == nnkEmpty: + # a forward declaration, bail + return def + + let to = ident"out.ast" + let ret = def.params[0] + def.body = newStmtList(def.body) + def.body.insert 0, quote do: + # inject a build overload that implicitly uses the output language + template build(body: untyped): untyped {.used.} = + build(`to`, `ret`, body) + + result = def + +macro processorMatchImpl(lang: static LangInfo, src: static string, + sel: untyped, rules: varargs[untyped]): untyped = + ## Implements the transformation/expansion of a language->language + ## processor's trailing case statement/expression. + let id = lang.map[src] + var (branches, used) = matchImpl(lang, id, ident"in.ast", sel, rules) + + template nt: untyped = lang.types[id] + let sym = bindSym"transform" + # auto-generate the transformers for the forms not manually provided: + for it in nt.forms.items: + let id = lang.forms[it].ntag + if not containsOrIncl(used, id): + branches.add nnkOfBranch.newTree( + newIntLitNode(id), + (quote do: + Metavar[dst, `src`](index: `sym`(def(src), def(dst), `src`, `it`, `sel`)))) + + # auto-generate the branches for missing production terminals and + # non-terminals: + for it in nt.sub.items: + if lang.types[it].terminal: + let id = lang.types[it].ntag + if id notin used: + let to = ident"out.ast" + let input = ident"in.ast" + branches.add nnkOfBranch.newTree( + newIntLitNode(id), + (quote do: + # TODO: use the tag of the destination language + `to`.nodes.add: + TreeNode[uint8](kind: uint8(`id`), val: `input`[`sel`].val) + Metavar[dst, `src`](index: NodeIndex(`to`.nodes.high)))) + else: + # if the first form's tag is used, so is the non-terminal itself + if lang.forms[lang.types[it].forms[0]].ntag notin used: + # TODO: consider inlining the transformer if it's auto-generated. + # More code to emit, but also a little less work at run-time + let branch = nnkOfBranch.newTree() + for tag in ntags(lang, lang.types[it]).items: + branch.add newIntLitNode(tag) + let name = ident(lang.types[it].mvar) + let callee = ident"->" + # dispatch to the processor and convert to the expected type + branch.add quote do: + Metavar[dst, `src`]( + index: `callee`(src.`name`(index: `sel`), dst.`name`).index) + branches.add branch + + let input = ident"in.ast" + result = nnkCaseStmt.newTree(quote do: `input`[`sel`].kind) + result.add branches + if branches[^1].kind != nnkElse: + # the selector is a uint8 and thus the case cannot be exhaustive + result.add nnkElse.newTree(newCall(ident"unreachable")) + +macro genProcessor*(index, nterm: untyped): untyped = + ## Generates the body for a non-terminal processor. + # simply emit an empty processorMatchImpl invocation. All branches will be + # auto-generated + # TODO: if none of the productions require a (direct or indirect) call to a + # custom processor, the source and target productions match, and the + # used tags map to the same ID in the source and target language, use a + # memcopy + let sym = bindSym"processorMatchImpl" + result = quote do: + `sym`(idef(src), `nterm`, `index`) + +proc hasPragma(def: NimNode, name: string): bool = + if def.pragma.kind == nnkPragma: + for it in def.pragma.items: + if it.eqIdent(name): + return true + +macro genAdapter[T1, T2; A, B: static string]( + src: typedesc[Metavar[T1, A]], dst: typedesc[Metavar[T2, B]], + orig: untyped) = + # note: the macro signature is very specific because it acts as the type + # checking for programmer-provided processor signatures + let name = ident("->") + copyLineInfo(name, orig) + result = quote do: + template `name`(n: `src`, _: typedesc[`dst`]): `dst` = + {.line.}: `orig`(n) + +macro transformInOutImpl(lang: static LangDef, name, def: untyped) = + ## Implements the transformation for language->language pass processors. + let name = name + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.params[0].kind == nnkEmpty: + error("a return type is required for a transfomer", def.name) + + if def.body.kind == nnkEmpty: + # a forward declaration. Append the additional adapter procedure + return newStmtList(def, + newCall(bindSym"genAdapter", + copyNimTree(def.params[1][^2]), + copyNimTree(def.params[0]), + copyNimNode(def.name))) + + proc transformCase(n: NimNode): NimNode = + result = genAst(arg=n[0]): + processorMatchImpl(idef(src), typeof(arg).N, arg.index) + for i in 1..language passes. + if def.body.kind != nnkEmpty: + error(".generated must be applied to forward declaration", def) + + def.body = newCall(ident"->", def.params[1][0], copyNimTree(def.params[0])) + result = def + +proc assemblePass(src, dst, def, call: NimNode): NimNode = + ## Assembles the final procedure definition for a pass. `def` is the original + ## proc definition, `call` the call to the pass' implementation. + let input = ident"in.ast" + let output = ident"out.ast" + let hasIn = src != nil + let hasOut = dst != nil + + var body = newStmtList() + let (transformImpl, name) = + if hasIn and hasOut: + (bindSym"transformInOutImpl", dst) + elif hasIn: + (bindSym"transformInImpl", src) + elif hasOut: + (bindSym"transformOutImpl", dst) + else: + unreachable() + + body.add quote do: + template transform(p: untyped) {.used.} = + `transformImpl`(def(`name`), `name`, p) + + if hasIn and hasOut: + let impl = bindSym"generatedImpl" + body.add quote do: + template generated(p: untyped) {.used.} = `impl`(p) + + # alias the source and destination language with known names: + if hasIn: + body.add quote do: + template src: untyped {.used.} = `src` + if hasOut: + body.add quote do: + template dst: untyped {.used.} = `dst` + + if hasIn: + let inj = ident"[]" + body.add quote do: + template match(sel: Metavar, branches: varargs[untyped]): untyped {.used.} = + match(`input`, sel, branches) + + template `inj`(x: ChildSlice, i: int): untyped {.used.} = + `input`[x, i] + + template val[T](v: nanopass.Value[T]): T {.used.} = + # TODO: look up the value of the terminal. Also, return a `lent T` + default(typeof(T)) + + if hasOut: + body.add quote do: + template terminal(x: untyped): untyped {.used.} = + embed(`name`, x) + template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = + build(`output`, n, body) + + if hasIn: + # shadow the input tree with a cursor to prevent a costly copy when + # it's captured by the closure + body.add quote do: + let `input` {.cursor.} = `input`.tree + if hasOut: + body.add quote do: + var `output`: PackedTree[uint8] + let index = `call`.index + # turn the AST with indirections into one without + result = Ast[dst](tree: finish(`output`, index)) + else: + body.add quote do: + result = `call` + + def.body = body + # patch the signature: + if hasIn: + def.params[1][^2] = ident"NodeIndex" + def.params.insert(1, newIdentDefs(ident"in.ast", quote do: Ast[`src`])) + if hasOut: + def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst) + + result = def + +macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = + # create a forward declaration for each transformer: + var preamble = newStmtList() + var i = 0 + while i < def.body.len: + let it = def.body[i] + # not ideal, as it breaks some custom macro/template pragmas, but without + # resorting to typed macros, there's no other way to do this transform + if it.kind == nnkProcDef and it.hasPragma("transform"): + if it.body.kind == nnkEmpty: + # remove the user-defined forward declaration + def.body.del(i) + dec i # undo the following `inc` + else: + let backup = it.body + it.body = newEmptyNode() + preamble.add copyNimTree(it) + it.body = backup + inc i + + # add the generic processor procedure, which all processor invocations + # for processors not supplied by the programmer will end up using + let name = ident"->" + preamble.add quote do: + # note: the signature is overly broad, so that overload resolution + # prefers the more specific adapters created for the programmer-provided + # processors + proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = + genProcessor(n.index, T.N) + + # if the body doesn't end in an expression, add a call to the + # entry processor + if def.body[^1].kind == nnkProcDef: + # ^^ a heuristic, but should work okay enough + def.body.add newCall(ident"->", def.params[1][0], + newDotExpr(newDotExpr(dst, ident"meta"), ident"entry")) + + def.body = newStmtList(preamble, def.body) + + let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) + lambda.params = copyNimTree(def.params) + lambda.params[0] = dstnterm + lambda.params[1][^2] = srcnterm + + let call = newCall(lambda) + # forward the original parameters to the lambda: + for i in 1..language pass, that is a + ## pass, that takes an AST (fragment) of language A and produces an AST of + ## language B. + if p.kind != nnkProcDef: + error(".inpass must be applied to a procedure definition", p) + + var target = p.params[0] + if target.kind == nnkEmpty: + error("a return type is required, but none is provided", p.params[0]) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(input = p.params[1][^2], target, p): + when target is Metavar: + passImpl(input, target.L, input, target, p) + else: + # use the entry non-terminal + passImpl(input, target, input.meta.entry, target.meta.entry, p) + +macro outpass*(p: untyped) = + ## Turns a procedure definition into a language->* pass, that is, a pass + ## that takes an AST (fragment) of language A and produces a value. + if p.kind != nnkProcDef: + error(".outpass must be applied to a procedure definition", p) + + let ret = p.params[0] + if ret.kind == nnkEmpty: + error("a return type is required, but none is provided", ret) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(typ = p.params[1][^2], p): + when typ is Metavar: + outpassImpl(typ, typ, p) + else: + # use the entry non-terminal + outpassImpl(typ, typ.entry, p) diff --git a/nanopass/nppatterns.nim b/nanopass/nppatterns.nim new file mode 100644 index 00000000..5920a8ef --- /dev/null +++ b/nanopass/nppatterns.nim @@ -0,0 +1,40 @@ +## Implements the type-based pattern matching used by the code generated by +## parts of the nanopass framework. + +import std/[macros] + +type + PChoice*[A, B] = object + ## Predicate: T is `A` or `B`. + + PArray*[A] = object + ## Predicate: T is array-like with element `A`. + +macro dot(e, fname: untyped): untyped = + ## ``dot(x, "abc")`` -> ``x.abc``. + macro dotAux(fname: static string, e: untyped): untyped = + newDotExpr(e, ident(fname)) + + newCall(bindSym"dotAux", fname, e) + +template matches*[T, U](x: T, _: typedesc[U]): bool = + ## Implements type-based pattern matching, used by the nanopass macros. + when U is PArray: + when T is seq: + matches(default(typeof(x[0])), typeof(U.A)) + else: + false + elif U is PChoice: + # why not just use `or`? Because that would unnecessarily expand the + # second `matches` invocation when the case first invocation was sucessful + when matches(x, typeof(U.A)): + true + else: + matches(x, typeof(U.B)) + elif U is Metavar: + when x is U: + true + else: + matches(x, dot(U.L.meta.nt, U.N)) + else: + x is U diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim new file mode 100644 index 00000000..b44fd8be --- /dev/null +++ b/nanopass/nptransform.nim @@ -0,0 +1,96 @@ +## Implements the auto-generation of transformers for language forms. + +import std/[macros, strformat, tables] +import passes/[trees] +import nanopass/[asts, helper, nplangdef] + +proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = + to.nodes[i] = TreeNode[uint8](kind: RefTag, val: uint32(x.index)) + inc i + +macro transform*(src, dst: static LangDef, nterm: static string, + form: static int, n: untyped): untyped = + ## Generates the transformation from the given source language form + ## (belonging to non-terminal `nterm`) to a target language form with + ## compatible syntax. + # find a target language form that's a production of the non-terminal and has + # the same shape + # TODO: only require the same name and number of elements, using -> to fit + # the rest. **Edit:** really? That can easily lead to non-obvious + # behaviour + var target = -1 + for it in dst.nterminals[nterm].forms.items: + if dst.forms[it.semantic] == src.forms[form]: + target = it.semantic + break + if target == -1: + return makeError(fmt"cannot generate transformer for '{src.forms[form]}'", n) + + # important: the generated code being efficient is of major importance! Most + # transformations will be auto-generated, and they should thus be as fast as + # possible + # TODO: if none of the child nodes require a processor call, memcopy the + # whole sub-tree + # TODO: (bigger refactor) pass the cursor to the transformers as a var + # parameter, which would eliminate unnecessary tree seeking when + # there's many calls to fully auto-generated non-terminal processors + + let inAst = ident"in.ast" + let to = ident"out.ast" + let id = dst.forms[target].id.uint8 + result = newStmtList() + # add the root node: + let body = quote do: + var tmp {.used.} = `inAst`.child(`n`, 0) + let root = `to`.nodes.len.NodeIndex + var i = `to`.nodes.len + # the node sequence needs to be contiguous, so it's allocated upfront + `to`.nodes.setLen(i + `inAst`.len(`n`) + 1) + `to`.nodes[i] = TreeNode[uint8](kind: `id`, val: `inAst`[`n`].val) + inc i + + # call the transformers and emit the nodes in one go: + for i, it in src.forms[form].elems.pairs: + let fromTerminal = it.typ in src.terminals + let toTerminal = it.typ in dst.terminals + if fromTerminal != toTerminal or + (toTerminal and src.terminals[it.typ].typ != src.terminals[it.typ].typ): + body.add makeError(fmt"cannot generate transformer for {src.forms[form]}", n) + break + + let call = + if fromTerminal: + if src.terminals[it.typ].tag == dst.terminals[it.typ].tag: + # just copy the node + quote do: + `to`.nodes[i] = `inAst`[tmp] + inc i + else: + # repack with the new tag + let tag = dst.terminals[it.typ].tag + quote do: + `to`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `inAst`[tmp].val) + inc i + else: + let append = bindSym"append" + let op = ident"->" + let s = newStrLitNode(it.typ) + let d = newStrLitNode(dst.forms[target].elems[i].typ) + quote do: + `append`(`to`, i, + `op`(Metavar[src, `s`](index: tmp), Metavar[dst, `d`])) + + if it.repeat: + let bias = src.forms[form].elems.len - 1 + body.add quote do: + for _ in 0..<(`inAst`.len(`n`) - `bias`): + `call` + tmp = `inAst`.next(tmp) + else: + body.add quote do: + `call` + tmp = `inAst`.next(tmp) + + result.add body + # the callsite takes care of fitting the index to the right type + result.add ident"root" From 14a2d4c69cff859b251a376d58a8fe17d814067c Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Fri, 10 Oct 2025 19:40:01 +0000 Subject: [PATCH 13/87] nppass: fix generated transformers having the wrong type Automatically generated transformers returned a non-terminal with the name of the *source* type, not the name of the *target* type. --- nanopass/nppass.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index d276409c..3ec73d13 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -43,7 +43,8 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, branches.add nnkOfBranch.newTree( newIntLitNode(id), (quote do: - Metavar[dst, `src`](index: `sym`(def(src), def(dst), `src`, `it`, `sel`)))) + (typeof(result))( + index: `sym`(def(src), def(dst), typeof(result).N, `it`, `sel`)))) # auto-generate the branches for missing production terminals and # non-terminals: @@ -59,7 +60,7 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, # TODO: use the tag of the destination language `to`.nodes.add: TreeNode[uint8](kind: uint8(`id`), val: `input`[`sel`].val) - Metavar[dst, `src`](index: NodeIndex(`to`.nodes.high)))) + (typeof(result))(index: NodeIndex(`to`.nodes.high)))) else: # if the first form's tag is used, so is the non-terminal itself if lang.forms[lang.types[it].forms[0]].ntag notin used: @@ -72,7 +73,7 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, let callee = ident"->" # dispatch to the processor and convert to the expected type branch.add quote do: - Metavar[dst, `src`]( + (typeof(result))( index: `callee`(src.`name`(index: `sel`), dst.`name`).index) branches.add branch @@ -278,7 +279,7 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = # prefers the more specific adapters created for the programmer-provided # processors proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = - genProcessor(n.index, T.N) + genProcessor(n.index, U.N) # if the body doesn't end in an expression, add a call to the # entry processor From b3ca10132c6633d46794f7ef220438c2ce9b671e Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Fri, 10 Oct 2025 19:40:01 +0000 Subject: [PATCH 14/87] nptransform: improve `transform` * allow morphing into forms that don't have the exact same elements * take `LangInfo` instances as input (significantly reduces compile times) --- nanopass/nppass.nim | 2 +- nanopass/nptransform.nim | 94 +++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 3ec73d13..9b20326c 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -44,7 +44,7 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, newIntLitNode(id), (quote do: (typeof(result))( - index: `sym`(def(src), def(dst), typeof(result).N, `it`, `sel`)))) + index: `sym`(idef(src), idef(dst), typeof(result).N, `it`, `sel`)))) # auto-generate the branches for missing production terminals and # non-terminals: diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index b44fd8be..f95d583b 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -2,28 +2,71 @@ import std/[macros, strformat, tables] import passes/[trees] -import nanopass/[asts, helper, nplangdef] +import nanopass/[asts, helper, nplang, nplangdef] + +type + Morphability = enum + None, Ambiguous, Inexact, Exact + +proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = + ## Computes whether form `a` from `src` can be morphed into `b` from `dst`. + if a.elems.len == b.elems.len and a.name == b.name: + result = Exact + for i, it in a.elems.pairs: + if it.repeat != b.elems[i].repeat: + result = None + break + elif src.types[it.typ].name != dst.types[b.elems[i].typ].name: + result = Inexact + else: + result = None + +proc render(lang: LangInfo, form: SForm): string = + result.add form.name + result.add "(" + for i, it in form.elems.pairs: + if i > 0: + result.add ", " + if it.repeat: + result.add "..." + result.add lang.types[it.typ].mvar + result.add ")" proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = to.nodes[i] = TreeNode[uint8](kind: RefTag, val: uint32(x.index)) inc i -macro transform*(src, dst: static LangDef, nterm: static string, +macro transform*(src, dst: static LangInfo, nterm: static string, form: static int, n: untyped): untyped = ## Generates the transformation from the given source language form - ## (belonging to non-terminal `nterm`) to a target language form with - ## compatible syntax. - # find a target language form that's a production of the non-terminal and has - # the same shape - # TODO: only require the same name and number of elements, using -> to fit - # the rest. **Edit:** really? That can easily lead to non-obvious - # behaviour + ## to a compatible target language production of the non-terminal with + ## name `nterm`. + # find a target language form that's a production of the non-terminal and a + # suitable morph target. Exact matches are preferred var target = -1 - for it in dst.nterminals[nterm].forms.items: - if dst.forms[it.semantic] == src.forms[form]: - target = it.semantic - break - if target == -1: + var morphability = None + for it in dst.types[dst.map[nterm]].forms.items: + let m = canMorph(src, dst, src.forms[form], dst.forms[it]) + case m + of None, Ambiguous: + discard "nothing to do" + of Inexact: + case morphability + of Inexact: + morphability = Ambiguous + of Exact, Ambiguous: + discard "keep as is" + of None: + morphability = Inexact + target = it + of Exact: + morphability = m + target = it + + template formatValue(to: var string, x: SForm, prec: string) = + to.add render(src, x) + + if morphability in {None, Ambiguous}: return makeError(fmt"cannot generate transformer for '{src.forms[form]}'", n) # important: the generated code being efficient is of major importance! Most @@ -37,7 +80,7 @@ macro transform*(src, dst: static LangDef, nterm: static string, let inAst = ident"in.ast" let to = ident"out.ast" - let id = dst.forms[target].id.uint8 + let id = dst.forms[target].ntag.uint8 result = newStmtList() # add the root node: let body = quote do: @@ -50,37 +93,38 @@ macro transform*(src, dst: static LangDef, nterm: static string, inc i # call the transformers and emit the nodes in one go: - for i, it in src.forms[form].elems.pairs: - let fromTerminal = it.typ in src.terminals - let toTerminal = it.typ in dst.terminals + for i, a in src.forms[form].elems.pairs: + let b = dst.forms[target].elems[i] + let fromTerminal = src.types[a.typ].terminal + let toTerminal = dst.types[b.typ].terminal if fromTerminal != toTerminal or - (toTerminal and src.terminals[it.typ].typ != src.terminals[it.typ].typ): - body.add makeError(fmt"cannot generate transformer for {src.forms[form]}", n) + (toTerminal and src.types[a.typ].name != dst.types[b.typ].name): + body.add makeError(fmt"cannot generate transformer for '{src.forms[form]}'", n) break let call = if fromTerminal: - if src.terminals[it.typ].tag == dst.terminals[it.typ].tag: + if src.types[a.typ].ntag == dst.types[b.typ].ntag: # just copy the node quote do: `to`.nodes[i] = `inAst`[tmp] inc i else: # repack with the new tag - let tag = dst.terminals[it.typ].tag + let tag = dst.types[b.typ].ntag quote do: `to`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `inAst`[tmp].val) inc i else: let append = bindSym"append" let op = ident"->" - let s = newStrLitNode(it.typ) - let d = newStrLitNode(dst.forms[target].elems[i].typ) + let s = newStrLitNode(src.types[a.typ].name) + let d = newStrLitNode(dst.types[b.typ].name) quote do: `append`(`to`, i, `op`(Metavar[src, `s`](index: tmp), Metavar[dst, `d`])) - if it.repeat: + if a.repeat: let bias = src.forms[form].elems.len - 1 body.add quote do: for _ in 0..<(`inAst`.len(`n`) - `bias`): From af8e83ce24d7533342d83b1bc40b3086e2a11125 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:58:48 +0000 Subject: [PATCH 15/87] nanopass: move `ntags` to `nplang` The function is general enough to warrant it being in the `nplang` module. --- nanopass/nplang.nim | 11 +++++++++++ nanopass/npmatch.nim | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index b9f29bd0..05c5a198 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -81,3 +81,14 @@ proc buildLangInfo*(def: LangDef): LangInfo = let id = result.map[name] for v in it.vars.items: result.types[id].sub.add result.map[v] + +proc ntags*(lang: LangInfo, typ: LangType): seq[int] = + ## Returns a list with all possible node tags productions of `typ` can have. + for it in typ.forms.items: + result.add lang.forms[it].ntag + + for it in typ.sub.items: + if lang.types[it].terminal: + result.add lang.types[it].ntag + else: + result.add ntags(lang, lang.types[it]) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 1d6be86c..58557bef 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -4,17 +4,6 @@ import std/[macros, intsets, strformat, tables] import passes/trees import nanopass/[asts, helper, nplang] -proc ntags*(lang: LangInfo, typ: LangType): seq[int] = - ## Returns a list with all possible node tags productions of `typ` can have. - for it in typ.forms.items: - result.add lang.forms[it].ntag - - for it in typ.sub.items: - if lang.types[it].terminal: - result.add lang.types[it].ntag - else: - result.add ntags(lang, lang.types[it]) - proc matchImpl*(lang: LangInfo, src: int, ast, sel, rules: NimNode ): (seq[NimNode], IntSet) = ## Implements the core of the `match` macro: From 0baeb021802866514f843129283226917c1e6b20 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:58:48 +0000 Subject: [PATCH 16/87] nanopass: implement the first revision of terminals support --- nanopass/nanopass.nim | 2 +- nanopass/npbuild.nim | 9 +++++---- nanopass/nppass.nim | 23 +++++++++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index d2a31f71..8f3179ec 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -16,7 +16,7 @@ export nppatterns.matches export npbuild.build, npmatch.match export nppass.pass, nppass.inpass, nppass.outpass -export nppass.genProcessor, nppass.embed +export nppass.genProcessor # TODO: ^^ bind the symbols; don't mix them in template isAtom*(x: uint8): bool = diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index f8949794..85a86efb 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -11,7 +11,9 @@ proc lookup[E; M: tuple](): auto {.compileTime.} = return it[1] proc append[L, U](to: var PackedTree[uint8], x: Value[U]) = - to.nodes.add TreeNode[uint8](kind: typeof(lookup[U, L.meta.term_map]()).V) + to.nodes.add TreeNode[uint8]( + kind: typeof(lookup[U, L.meta.term_map]()).V, + val: x.index) proc append[L](to: var PackedTree[uint8], x: Metavar) = to.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) @@ -128,10 +130,9 @@ macro buildImpl(to: var PackedTree[uint8], lang: static LangInfo, n.expectLen 2 let valueType = ident(lang.types[lang.map[name]].name) let sym = genSym() - discard n[1] - # TODO: add the value to the environment + let cons = n[1] test.add quote do: - let `sym` = Value[`valueType`]() + let `sym` = Value[`valueType`](index: pack(storage, `cons`)) body.add genAst(typ, to, sym) do: append[typ.L](to, sym) nnkPar.newTree(sym) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 9b20326c..ebfe469c 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -3,10 +3,10 @@ import std/[genasts, macros, packedsets, tables] import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] -template embed*(lang: typedesc, arg: untyped): untyped = - # TODO: implement, and don't export +template embed(lang: typedesc, arg: untyped): untyped = + mixin pack, storage let tmp = arg - Value[typeof(tmp)]() + Value[typeof(tmp)](index: pack(storage, tmp)) macro transformOutImpl(lang: static LangDef, name, def: untyped) = ## Implements the transformation for processors in an *->language pass. @@ -216,16 +216,24 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = `input`[x, i] template val[T](v: nanopass.Value[T]): T {.used.} = - # TODO: look up the value of the terminal. Also, return a `lent T` - default(typeof(T)) + # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) + unpack(storage, v.index, typeof(T)) if hasOut: + let embed = bindSym("embed", brClosed) body.add quote do: template terminal(x: untyped): untyped {.used.} = - embed(`name`, x) + `embed`(`name`, x) template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = build(`output`, n, body) + # temporarily move the storage object into a local, so that it can + # be captured + body.add quote do: + var storage: Literals + swap(storage, st) + defer: swap(storage, st) + if hasIn: # shadow the input tree with a cursor to prevent a costly copy when # it's captured by the closure @@ -249,6 +257,9 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = if hasOut: def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst) + def.params.insert(2 + ord(hasIn), + newIdentDefs(ident"st", nnkVarTy.newTree(ident"Literals"))) + result = def macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = From 5de07be092dd840ef5f60d1f28b4bf1302126e4a Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:58:48 +0000 Subject: [PATCH 17/87] nanopass: use a different approach to terminal storage Instead of requiring the programmer to pass the storage instance to every pass (and carry it around to everywhere the AST is to be used), a ref to the storage is now stored within the AST itself. Storage types not named `Literals` and using pre-existing storage object instance for newly-created AST are not supported yet. --- nanopass/asts.nim | 4 +++- nanopass/npbuild.nim | 13 +++++++++++-- nanopass/nppass.nim | 41 +++++++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 71787492..e5517938 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -6,9 +6,11 @@ import passes/trees type # note: the fields are exported so that the nanopass machinery can access # them. User code should, in most cases, not access the fields directly - Ast*[L: object] = object + Ast*[L: object, Storage: object] = object tree*: PackedTree[uint8] ## leaked implementation detail, don't use + storage*: ref Storage + ## leaked implementation detail, don't use Metavar*[L: object, N: static string] = object ## Represents a reference to an AST fragment that's a production of non- diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 85a86efb..5d51f03e 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -128,11 +128,20 @@ macro buildImpl(to: var PackedTree[uint8], lang: static LangInfo, if name in lang.map: # can only be a terminal n.expectLen 2 - let valueType = ident(lang.types[lang.map[name]].name) + let mvar = ident(name) let sym = genSym() + let tmp = genSym() let cons = n[1] + let storage = ident"io.storage" + # ensure the operand having the right type via a conversion, but only + # when there's no `is`, so as to not interfere with sinking test.add quote do: - let `sym` = Value[`valueType`](index: pack(storage, `cons`)) + let `tmp` = `cons` + let `sym` = dst.`mvar`(index: pack(`storage`[], + (when `tmp` is dst.`mvar`.T: + `tmp` + else: + dst.`mvar`.T(`tmp`)))) body.add genAst(typ, to, sym) do: append[typ.L](to, sym) nnkPar.newTree(sym) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index ebfe469c..30e1c03a 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -3,10 +3,11 @@ import std/[genasts, macros, packedsets, tables] import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] -template embed(lang: typedesc, arg: untyped): untyped = - mixin pack, storage +template embed(storage, arg: untyped): untyped = + ## Implements terminal value construction. + mixin pack let tmp = arg - Value[typeof(tmp)](index: pack(storage, tmp)) + Value[typeof(tmp)](index: pack(storage[], tmp)) macro transformOutImpl(lang: static LangDef, name, def: untyped) = ## Implements the transformation for processors in an *->language pass. @@ -175,6 +176,8 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = ## proc definition, `call` the call to the pass' implementation. let input = ident"in.ast" let output = ident"out.ast" + let storage = ident"io.storage" + let storageTy = ident"Literals" # TODO: don't hardcode let hasIn = src != nil let hasOut = dst != nil @@ -217,22 +220,24 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template val[T](v: nanopass.Value[T]): T {.used.} = # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) - unpack(storage, v.index, typeof(T)) + # XXX: consider renaming this template to `get` + unpack(`storage`[], v.index, typeof(T)) if hasOut: - let embed = bindSym("embed", brClosed) + let embed = bindSym"embed" body.add quote do: template terminal(x: untyped): untyped {.used.} = - `embed`(`name`, x) + `embed`(`storage`, x) template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = build(`output`, n, body) - # temporarily move the storage object into a local, so that it can - # be captured - body.add quote do: - var storage: Literals - swap(storage, st) - defer: swap(storage, st) + if hasIn: + # re-use the data storage object from the input + body.add quote do: + let `storage` = `input`.storage + else: + body.add quote do: + let `storage` = new(`storageTy`) if hasIn: # shadow the input tree with a cursor to prevent a costly copy when @@ -244,7 +249,10 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = var `output`: PackedTree[uint8] let index = `call`.index # turn the AST with indirections into one without - result = Ast[dst](tree: finish(`output`, index)) + result = Ast[dst, `storageTy`]( + tree: finish(`output`, index), + storage: `storage`, + ) else: body.add quote do: result = `call` @@ -253,12 +261,9 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = # patch the signature: if hasIn: def.params[1][^2] = ident"NodeIndex" - def.params.insert(1, newIdentDefs(ident"in.ast", quote do: Ast[`src`])) + def.params.insert(1, newIdentDefs(input, quote do: Ast[`src`, `storageTy`])) if hasOut: - def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst) - - def.params.insert(2 + ord(hasIn), - newIdentDefs(ident"st", nnkVarTy.newTree(ident"Literals"))) + def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst, storageTy) result = def From c08e026aefca7f87d696a3bde3c3ebf1aea4b60d Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:58:49 +0000 Subject: [PATCH 18/87] passes: provide the `Literals` storage type The type is implemented by the new `literals` module and works in much the same way as the literal data storage for `PackedTree`, although currently without the small-integer optimization. --- passes/literals.nim | 57 +++++++++++++++++++++++++++++++++++++++++++++ passes/passes.nim | 11 +++++---- 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 passes/literals.nim diff --git a/passes/literals.nim b/passes/literals.nim new file mode 100644 index 00000000..a40f138c --- /dev/null +++ b/passes/literals.nim @@ -0,0 +1,57 @@ +## Implements the storage for literal data embedded in ASTs. + +# TODO: use bi-tables for the number and string values +# TODO: store small integers inline (like how the packing in `trees` does it) + +type + Literals* = object + ## Storage container for literal data, to be used with nanopass ASTs. + numbers*: seq[uint64] ## a list of bit patterns + strings*: seq[string] + + Ident* = distinct string + +proc pack*(s: var Literals, val: SomeInteger): uint32 {.inline.} = + ## Adds the bit-representation of `val` to `s` and returns an ID to refer + ## to the value with later. + result = uint32(s.numbers.len) + s.numbers.add cast[uint64](val) + +proc pack*(s: var Literals, val: float): uint32 {.inline.} = + ## Adds the bit-representation of `val` to `s` and returns an ID to refer + ## to the value with later. + result = uint32(s.numbers.len) + s.numbers.add cast[uint64](val) + +proc pack*(s: var Literals, val: string): uint32 {.inline.} = + ## Adds the `val` to `s` and returns an ID to refer to the value with later. + result = uint32(s.strings.len) + s.strings.add val + +proc pack*(s: var Literals, val: Ident): uint32 {.inline.} = + ## Adds the `val` to `s` and returns an ID to refer to the value with later. + result = uint32(s.strings.len) + s.strings.add string(val) + +proc unpack*[T: SomeNumber](s: Literals, id: uint32, _: typedesc[T]): T {.inline.} = + ## Returns the bit-representation stored under `id` interpreted as `T`. + when T is uint64: # prevent warnings + s.numbers[id] + else: + cast[T](s.numbers[id]) + +proc unpack*(s: Literals, id: uint32, _: typedesc[string]): lent string {.inline.} = + ## Returns the string stored under `id`. + s.strings[id] + +proc unpack*(s: Literals, id: uint32, _: typedesc[Ident]): lent Ident {.inline.} = + ## Returns the string stored under `id`, treated as an ``Ident``. + s.strings[id].Ident + +# TODO: remove the temporary overloads for objects again + +proc pack*[T: object](s: Literals, val: T): uint32 = + result = 0 + +proc unpack*[T: object](s: Literals, id: uint32, _: typedesc[T]): T = + discard diff --git a/passes/passes.nim b/passes/passes.nim index af778ea6..70927c55 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -8,14 +8,17 @@ import strutils, tables ], + experimental/[ + sexp_parse + ], nanopass/nanopass, - experimental/sexp_parse, - passes/trees, + passes/literals, phy/[reporting, default_reporting] +import passes/trees except Literals + type Symbol = object - Ident = distinct string defineLanguage Lsrc: n(int) @@ -943,7 +946,7 @@ macro defineCompiler(name, names: untyped) = body.add newLetStmt(tmp, g) prev = tmp result = quote do: - proc `name`(e: Ast[Lsrc]): auto = + proc `name`(e: Ast[Lsrc, Literals]): auto = `body` result = `prev` From f7f53e2b93bee85b585ea11a2362a6f0b7900482 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:58:49 +0000 Subject: [PATCH 19/87] nanopass: add an S-expression renderer generator --- nanopass/nanopass.nim | 4 +- nanopass/npsexpr.nim | 119 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 nanopass/npsexpr.nim diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 8f3179ec..94ef1675 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -9,11 +9,11 @@ import passes/[trees], - nanopass/[asts, nplangdef, nplanggen, npmatch, npbuild, nppass, nppatterns] + nanopass/[asts, nplangdef, nplanggen, npmatch, npbuild, nppass, nppatterns, npsexpr] export asts export nppatterns.matches -export npbuild.build, npmatch.match +export npbuild.build, npmatch.match, npsexpr.renderer export nppass.pass, nppass.inpass, nppass.outpass export nppass.genProcessor diff --git a/nanopass/npsexpr.nim b/nanopass/npsexpr.nim new file mode 100644 index 00000000..2897c5c8 --- /dev/null +++ b/nanopass/npsexpr.nim @@ -0,0 +1,119 @@ +## Implements the macros for converting between ASTs and S-expressions. + +# TODO: implement a macro for generating the inverse of `renderer` (i.e., +# S-expr to AST translation) +# TODO: instead of assembling a S-expression directly, the renderer should +# emit a stream of S-expression tokens (ideally as an iterator, but's +# that not really possible today, at least in an efficient manner) + +import std/[macros, tables] +import nanopass/[nplang, helper] +import passes/[trees] + +const + Inp = ident"tree" ## name of the input AST parameter + Pos = ident"pos" ## name of the position parameter + +macro genRenderer(def: static LangInfo, nterm: static string) = + ## Generates the rendering logic for the non-terminal with name `nterm`. + let id = def.map[nterm] + var caseStmt = nnkCaseStmt.newTree(quote do: `Inp`.tree.nodes[`Pos`].kind) + + proc genForType(def: LangInfo, typ: LangType): NimNode = + case typ.terminal + of true: + let name = ident(typ.name) + quote do: + inc `Pos` + toSexp(unpack(`Inp`.storage[], `Inp`.tree.nodes[`Pos` - 1].val, `name`)) + of false: + let name = typ.name + quote do: + render[`name`](`Inp`, `Pos`) + + # form renderers: + for it in def.types[id].forms.items: + var body = nnkStmtList.newTree() + let name = def.forms[it].name + body.add quote do: + let len {.used.} = `Inp`.tree.nodes[`Pos`].val.int + inc `Pos` + result = newSList([newSSymbol(`name`)]) + for i, e in def.forms[it].elems.pairs: + let inner = genForType(def, def.types[e.typ]) + if e.repeat: + let start = def.forms[it].elems.len - 1 + body.add quote do: + for _ in `start`.. Date: Mon, 1 Dec 2025 00:58:50 +0000 Subject: [PATCH 20/87] passes: add a test for the S-expression renderer generator --- passes/passes.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/passes/passes.nim b/passes/passes.nim index 70927c55..6e9447e1 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -9,7 +9,8 @@ import tables ], experimental/[ - sexp_parse + sexp_parse, + sexp ], nanopass/nanopass, passes/literals, @@ -750,6 +751,12 @@ proc calleeToSymbol(x: L11): L12 {.pass.} = let tmp = terminal Symbol() build Let(tmp, Stmts([Asgn(tmp, ^expr(e0))], Call(t, tmp, e1))) +proc toSexp(x: SomeInteger): SexpNode = newSInt(x) +proc toSexp(x: float): SexpNode = newSFloat(x) +proc toSexp(x: string): SexpNode = newSString(x) +proc toSexp(x: Ident): SexpNode = newSSymbol(x.string) + +proc render*(x: Lsrc.m): SexpNode {.renderer.} # TODO: fix the symbol binding in the pass DSL such that `vmenv` can be # imported at the top (at the moment, ambiguity errors ensue) From dc77b6124a992aa85fbd02ff95a7c2881b38c050 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 13 Dec 2025 19:33:18 +0000 Subject: [PATCH 21/87] docs: add the rough skeleton of a manual It's intended to provide the user-facing documentation on how to use the nanopass, while at the same time also representing the (integration) test suite for the framework. --- nanopass/manual.rst | 213 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 nanopass/manual.rst diff --git a/nanopass/manual.rst b/nanopass/manual.rst new file mode 100644 index 00000000..b75ff4ba --- /dev/null +++ b/nanopass/manual.rst @@ -0,0 +1,213 @@ + +Nanopass Framework +================== + +This document describes how to use the nanopass framework, what is allowed, and +what not. It only describes the framework's interface, not how things are +implemented internally. + +Nanopass Compiler Overview +-------------------------- + +A nanopass compiler is a compiler implemented as a series of narrow *passes*, +which *transform*, *analyze*, or otherwise produce abstract syntax trees (=AST) +according to well-defined grammars. + +The idea is that passes focus on small, specific tasks, with tree traversal +boilerplate generated automatically. + +Concepts +-------- + +* a *language* (in the context of the nanopass framework) is a formal grammar. +* a *terminal* is ... +* a *form* is a named schema for a term, made up of zero or more sub-terms +* a *non-terminal* is ... +* a *meta-variable* is a name ranging over a terminal or non-terminal. It may be viewed as an alias. + +.. TODO rewrite this section. It should use the correct terminology while still + being approachable to someone not overly familiar with formal grammars + +Usage +----- + +Language Definition +~~~~~~~~~~~~~~~~~~~ + +Languages are defined via the `defineLanguage` macro, which has two forms. + +From Scratch +~~~~~~~~~~~~ + +The first form `defineLanguage` form takes two arguments, an identifier and +a body, and defines a language from scratch. If the body is valid, a NimSkull +type is bound to the identifier. + +.. note:: + + The type is not meant to be used as a type for values. Rather, it + acts as a namespace through which entities defined in the language are + accessed. + +The body consists of a sequence of terminal and non-terminal definitions. + +.. code-block:: nim + :test: "nim c $1" + + import nanopass/nanopass + + defineLanguage L0: + i(int) # definition of a terminal + expr(e) ::= i # definition of a non-terminal + + # this defines a language `L0` with: + # * a single terminal of type `int`, ranged over by meta-variable `i` + # * a non-terminal with name `expr`, ranged over by meta-variable `e`. Where + # this non-terminal is expected, an `int` value is allowed + +Meta-variables must be unique. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(int) + i(float) # error: 'i' name already in use + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(int) + integer(i) ::= i # error: 'i' name already in use + +All meta-variables defined in the body are accessible in all non-terminals, +regardless of the declarations' order. + +.. code-block:: nim + :test: "nim c $1" + + import nanopass/nanopass + + defineLanguage L0: + expr(e) ::= i + i(int) + +Each non-terminals must have a unique name. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(int) + expr(e) ::= i + expr(b) ::= i # error: 'expr' name already in use + +Non-terminals and meta-variables share a namespace, meaning that it's not +possible to give a name to a non-terminal already used for a meta-variable, +and vice versa. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(int) + i(e) ::= i # error: 'i' already in use + +For terminals, the type expression must be an identifier, more complex +expressions are not allowed. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(ref int) # error: not an identifier + expr(e) ::= i + +The identifier must also refer to a type that exists at the time +`defineLanguage` is expanded. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + i(MyType) + expr(e) ::= i + + type MyType = object # too late, must be defined before the language + +.. TODO continue + +Define Language Via Difference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The second form of `defineLanguage` takes takes three arguments, two +identifiers and a body, and defines a language by applying the diff provided +by the body to the base language. If the result is valid, a NimSkull +type is bound to the identifier. + +The second argument must be the name of an in-scope identifier to which a +language definition was previously bound via `defineLanguage`. + +.. TODO continue + +AST +--- + +.. TODO continue + +Pattern Matching +---------------- + +The nanopass framework provides a facility for comprehending AST fragments +using pattern matching, via the `match` routine. + +.. TODO continue + +Passes +------ + + +.. TODO continue + +The `build` Form +~~~~~~~~~~~~~~~~ + +Passes that produce an AST have a special macro available within their body: +the `build` macro. It's used to create AST fragments in a statically type-safe +manner. + +.. TODO continue + +Value To Language +~~~~~~~~~~~~~~~~~ + +.. TODO continue + +Language To Language +~~~~~~~~~~~~~~~~~~~~ + +.. TODO continue + +Language To Value +~~~~~~~~~~~~~~~~~ + +.. TODO continue From f68c41b01a7e9075635d5e3bd61e3d77b74b018b Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 13 Dec 2025 19:33:19 +0000 Subject: [PATCH 22/87] koch: implement a simple doc building command --- koch.nim | 32 ++++++++++++++++++++++++++++++++ nim.cfg | 3 +++ 2 files changed, 35 insertions(+) create mode 100644 nim.cfg diff --git a/koch.nim b/koch.nim index 73840012..4fca1d3f 100644 --- a/koch.nim +++ b/koch.nim @@ -17,6 +17,7 @@ Commands: generate generates the various language-related modules build-defs verifies the language definitions and generates the textual representation for them + docs builds all documentation """ Programs = { "tester" : ("tools/tester.nim", true), @@ -32,6 +33,10 @@ Commands: DefaultGenerated = "generated" ## the default path for the generated modules + RstList = @[ + "nanopass/manual.rst" + ] + var nimExe = findExe("nim") verbose = true @@ -58,6 +63,14 @@ proc compile(file: sink string, name: string, extra: varargs[string]): bool = args.add file result = run(nimExe, args) +proc rstToHtml(file: sink string, dir: string, extra: varargs[string]): bool = + ## Runs RST-to-HTML conversion on `file`, using `dir` as the + ## output directory. + var args = @["rst2html", "--nimcache:build/docs/", "--outdir:" & dir] + args.add extra + args.add file + result = run(nimExe, args) + proc check(file: sink string, extra: varargs[string]): bool = ## Runs the ``check`` command on the given NimSkull `file`. var args = @["check"] @@ -184,6 +197,23 @@ proc buildDefs(args: string): bool = result = true +proc buildDocs(args: string): bool = + ## Handles building and testing the various documentation. + if args.len > 0: + return false + + let docroot = getCurrentDir() / "build" / "docs" + createDir(docroot) + + for it in RstList.items: + # the output is not relevant at the moment, so it's simply dumped into the + # artifacts directory + if not rstToHtml(it, docroot): + echo "Failure" + quit(1) + + result = true + proc showHelp(): bool = ## Shows the help text. echo HelpText @@ -213,6 +243,8 @@ while true: generate(opts.cmdLineRest) of "build-defs": buildDefs(opts.cmdLineRest) + of "docs": + buildDocs(opts.cmdLineRest) of "help": showHelp() else: diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 00000000..1e07891a --- /dev/null +++ b/nim.cfg @@ -0,0 +1,3 @@ +--path:"." +# ^^ currently needed as a workaround for the NimSkull RST-to-HTML test runner +# providing no direct way to configure module lookup paths From d391e77b688c8fc4736cc3998873b531f5ec10b3 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 13 Dec 2025 19:33:19 +0000 Subject: [PATCH 23/87] ci: integrate doc testing/building --- .github/workflows/build_and_test.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1b06cc60..ce66efff 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -168,3 +168,25 @@ jobs: env: TEST_FILE: tests/expr/t06_call_proc_with_tuple_return_type.test TEST_OUTPUT: "(TupleCons 100 200) : (TupleTy (IntTy) (IntTy))(Done 0)" + + docs: + name: "Test documentation" + runs-on: "ubuntu-latest" + + needs: context + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + filter: tree:0 + + - uses: nim-works/setup-nimskull@0.1.2 + with: + nimskull-version: "${{ needs.context.outputs.nimskull-version }}" + + - name: Build koch + run: nim c --outdir:bin koch.nim + + - name: Build and test documentation + run: bin/koch docs From 2a89f6f01821fb2dfa22e90f4b2aa4d3c779721c Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 20 Dec 2025 21:42:03 +0000 Subject: [PATCH 24/87] nanopass: make terminal definition syntax more regular * use the same shape for terminal definition as for non- definitions * allow having more than one meta-variable for a terminal --- nanopass/nplang.nim | 7 +++++-- nanopass/nplangdef.nim | 42 ++++++++++++++++++++++++++++++------------ nanopass/nplanggen.nim | 22 +++++++++------------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 05c5a198..400b4a27 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -49,12 +49,15 @@ proc buildLangInfo*(def: LangDef): LangInfo = for name, it in def.terminals.pairs: result.types.add LangType( - name: it.typ, - mvar: name, + name: name, + mvar: it.mvars[0], terminal: true, ntag: it.tag ) + # add the name-to-type mappings: result.map[name] = high(result.types) + for x in it.mvars.items: + result.map[x] = high(result.types) for name, it in def.nterminals.pairs: result.types.add LangType( diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 8925d0d2..9364805d 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -25,6 +25,13 @@ type semantic*: int ## index of the semantic representation + Terminal* = object + mvars*: seq[string] + ## the meta-variables for ranging over values of the type + tag*: int + ## the integer ID through which a tree node is identified as being + ## an instance of the terminal + NonTerminal* = object mvars*: seq[string] ## the meta-variables for ranging over the productions @@ -36,7 +43,7 @@ type LangDef* = object ## A checked and pre-processed language definition, carrying enough ## source-level information necessary for implementing, e.g., inheritance. - terminals*: Table[string, tuple[typ: string, tag: int]] + terminals*: Table[string, Terminal] ## the terminals of the language nterminals*: Table[string, NonTerminal] ## the non-terminals of the language @@ -102,7 +109,7 @@ proc checkName(target: LangDef, vars: Table[string, string], name: string, elif name in target.nterminals: error(fmt"non-terminal with name {name} already exists", info) elif name in vars: - error(fmt"'{name}' is already used by a meta-variable for '{vars[name]}'", info) + error(fmt"'{name}' is already the name of a meta-variable for '{vars[name]}'", info) proc parseForm(n: NimNode): ParsedForm = ## Parses a form in the context of a language definition. No semantic checks @@ -159,17 +166,18 @@ proc buildLanguage(add, sub: seq[NimNode], # 1. inherit; carry over everything not explicitly removed # 2. extension; make the additions - proc processTerminal(n: NimNode): (string, string) = + proc processTerminal(n: NimNode): string = n.expectKind nnkCall - n.expectLen 2 + n.expectMinLen 2 n[0].expectKind nnkIdent - n[1].expectKind nnkIdent - result = (n[0].strVal, n[1].strVal) + for i in 1..tag map: let tup = nnkTupleConstr.newTree() - for it in def.terminals.values: + for name, it in def.terminals.pairs: let n = it.tag tup.add nnkTupleConstr.newTree( - ident(it.typ), + ident(name), nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(n))) metaType.add newIdentDefs(ident"term_map", tup) From 47ca6b5c3233f6b9f2676471df9f14032a31d3f3 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 20 Dec 2025 21:42:03 +0000 Subject: [PATCH 25/87] passes: adjust to the new syntax for terminals --- passes/passes.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/passes/passes.nim b/passes/passes.nim index 6e9447e1..0a48f498 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -22,10 +22,10 @@ type Symbol = object defineLanguage Lsrc: - n(int) - fl(float) - str(string) - x(Ident) + int(n) + float(fl) + string(str) + Ident(x) rec_field(rf) ::= Field(x, e) field_decl(f) ::= Field(x, t) @@ -81,7 +81,7 @@ defineLanguage L3, L2: defineLanguage L4, L3: ## Language with symbols instead of raw identifiers. - +s(Symbol) + +Symbol(s) expr(e) ::= -x | +s | -Let(x, e, e) | +Let(s, e, e) typ(t) ::= -x | +s From ead76ff68e417b4c3068b532a603479ed9e27acd Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Sat, 20 Dec 2025 21:42:03 +0000 Subject: [PATCH 26/87] manual: adjust to the new syntax for terminals --- nanopass/manual.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nanopass/manual.rst b/nanopass/manual.rst index b75ff4ba..1a11e88f 100644 --- a/nanopass/manual.rst +++ b/nanopass/manual.rst @@ -57,7 +57,7 @@ The body consists of a sequence of terminal and non-terminal definitions. import nanopass/nanopass defineLanguage L0: - i(int) # definition of a terminal + int(i) # definition of a terminal expr(e) ::= i # definition of a non-terminal # this defines a language `L0` with: @@ -74,8 +74,8 @@ Meta-variables must be unique. import nanopass/nanopass defineLanguage L0: - i(int) - i(float) # error: 'i' name already in use + int(i) + float(i) # error: 'i' name already in use .. code-block:: nim :test: "nim c $1" @@ -84,7 +84,7 @@ Meta-variables must be unique. import nanopass/nanopass defineLanguage L0: - i(int) + int(i) integer(i) ::= i # error: 'i' name already in use All meta-variables defined in the body are accessible in all non-terminals, @@ -97,7 +97,7 @@ regardless of the declarations' order. defineLanguage L0: expr(e) ::= i - i(int) + int(i) Each non-terminals must have a unique name. @@ -108,7 +108,7 @@ Each non-terminals must have a unique name. import nanopass/nanopass defineLanguage L0: - i(int) + int(i) expr(e) ::= i expr(b) ::= i # error: 'expr' name already in use @@ -123,7 +123,7 @@ and vice versa. import nanopass/nanopass defineLanguage L0: - i(int) + int(i) i(e) ::= i # error: 'i' already in use For terminals, the type expression must be an identifier, more complex @@ -136,7 +136,7 @@ expressions are not allowed. import nanopass/nanopass defineLanguage L0: - i(ref int) # error: not an identifier + (ref int)(i) # error: not an identifier expr(e) ::= i The identifier must also refer to a type that exists at the time @@ -149,7 +149,7 @@ The identifier must also refer to a type that exists at the time import nanopass/nanopass defineLanguage L0: - i(MyType) + MyType(i) expr(e) ::= i type MyType = object # too late, must be defined before the language From bcf9e1d02f475c1caeed8a403dfc1cb7d0a04750 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:51 +0000 Subject: [PATCH 27/87] literals: remove `Ident` handling `Ident` is too specific of a type to be hosted by the generic `Literals` implementation. --- passes/literals.nim | 11 ----------- passes/passes.nim | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/passes/literals.nim b/passes/literals.nim index a40f138c..5cf91ec8 100644 --- a/passes/literals.nim +++ b/passes/literals.nim @@ -9,8 +9,6 @@ type numbers*: seq[uint64] ## a list of bit patterns strings*: seq[string] - Ident* = distinct string - proc pack*(s: var Literals, val: SomeInteger): uint32 {.inline.} = ## Adds the bit-representation of `val` to `s` and returns an ID to refer ## to the value with later. @@ -28,11 +26,6 @@ proc pack*(s: var Literals, val: string): uint32 {.inline.} = result = uint32(s.strings.len) s.strings.add val -proc pack*(s: var Literals, val: Ident): uint32 {.inline.} = - ## Adds the `val` to `s` and returns an ID to refer to the value with later. - result = uint32(s.strings.len) - s.strings.add string(val) - proc unpack*[T: SomeNumber](s: Literals, id: uint32, _: typedesc[T]): T {.inline.} = ## Returns the bit-representation stored under `id` interpreted as `T`. when T is uint64: # prevent warnings @@ -44,10 +37,6 @@ proc unpack*(s: Literals, id: uint32, _: typedesc[string]): lent string {.inline ## Returns the string stored under `id`. s.strings[id] -proc unpack*(s: Literals, id: uint32, _: typedesc[Ident]): lent Ident {.inline.} = - ## Returns the string stored under `id`, treated as an ``Ident``. - s.strings[id].Ident - # TODO: remove the temporary overloads for objects again proc pack*[T: object](s: Literals, val: T): uint32 = diff --git a/passes/passes.nim b/passes/passes.nim index 0a48f498..3b5f8264 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -20,6 +20,7 @@ import passes/trees except Literals type Symbol = object + Ident = distinct string defineLanguage Lsrc: int(n) @@ -214,6 +215,12 @@ defineLanguage L12, L11: expr(e) ::= -Call(t, e, ...e) | +Call(t, s, ...e) stmt(st) ::= -Call(t, e, ...e) | +Call(t, s, ...e) +proc pack(s: var Literals, val: Ident): uint32 {.inline.} = + pack(s, string(val)) + +proc unpack(s: Literals, id: uint32, _: typedesc[Ident]): lent Ident {.inline.} = + Ident unpack(s, id, string) + proc eatString(s: var SexpParser): string = result = s.captureCurrString() discard s.getTok() From 390d64310f500c9b464b9ac5ecab81d0de04ddbb Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 28/87] nptransform: move form rendering to `nplang` The routine is general enough to warrant being in the `LangInfo` home module. --- nanopass/nplang.nim | 11 +++++++++++ nanopass/nptransform.nim | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 400b4a27..355b0e1c 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -95,3 +95,14 @@ proc ntags*(lang: LangInfo, typ: LangType): seq[int] = result.add lang.types[it].ntag else: result.add ntags(lang, lang.types[it]) + +proc render*(lang: LangInfo, form: SForm): string = + result.add form.name + result.add "(" + for i, it in form.elems.pairs: + if i > 0: + result.add ", " + if it.repeat: + result.add "..." + result.add lang.types[it.typ].mvar + result.add ")" diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index f95d583b..3cc9bde1 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -21,17 +21,6 @@ proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = else: result = None -proc render(lang: LangInfo, form: SForm): string = - result.add form.name - result.add "(" - for i, it in form.elems.pairs: - if i > 0: - result.add ", " - if it.repeat: - result.add "..." - result.add lang.types[it.typ].mvar - result.add ")" - proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = to.nodes[i] = TreeNode[uint8](kind: RefTag, val: uint32(x.index)) inc i From 717337fe451ee28e853ccebb1f6a05fcc6d4d9c0 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 29/87] asts, nppass: fix some symbol binding issues The nanopass framework should now be usable by just including the `nanopass` module. --- nanopass/asts.nim | 6 ++++++ nanopass/nppass.nim | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index e5517938..f809692b 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -3,6 +3,12 @@ import passes/trees +export trees.NodeIndex +# export some fundamental tree traversal and query operations, so that the +# nanopass implementation doesn't have to use bindSym everywhere +export trees.`[]`, trees.next, trees.child, trees.len +export trees.TreeNode + type # note: the fields are exported so that the nanopass machinery can access # them. User code should, in most cases, not access the fields directly diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 30e1c03a..b64442bd 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -246,7 +246,7 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = let `input` {.cursor.} = `input`.tree if hasOut: body.add quote do: - var `output`: PackedTree[uint8] + var `output`: Ast[dst, `storageTy`].tree let index = `call`.index # turn the AST with indirections into one without result = Ast[dst, `storageTy`]( @@ -260,7 +260,7 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = def.body = body # patch the signature: if hasIn: - def.params[1][^2] = ident"NodeIndex" + def.params[1][^2] = bindSym"NodeIndex" def.params.insert(1, newIdentDefs(input, quote do: Ast[`src`, `storageTy`])) if hasOut: def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst, storageTy) From 20eb1a1dce00c578e73dbf4c6a360c3dae8c7e2e Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 30/87] asts: fix `items` iterator not compiling --- nanopass/asts.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index f809692b..bb9f262c 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -47,7 +47,10 @@ proc slice*[T](start: NodeIndex, len: uint32): ChildSlice[T] = iterator items*[T](t: PackedTree[uint8], s: ChildSlice[T]): T = var c = s.start for _ in 0.. Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 31/87] asts: allow arbitrary integers for `ChildSlice` indexing --- nanopass/asts.nim | 11 +++++++++-- nanopass/nppass.nim | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index bb9f262c..7666e032 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -53,8 +53,15 @@ iterator items*[T](t: PackedTree[uint8], s: ChildSlice[T]): T = yield T(index: t[c].val) c = t.next(c) -proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: int): T = - assert i < int(s.len) +proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: SomeInteger): T = + when compileOption("boundchecks"): + when i is SomeSignedInt: + if i < 0 or uint64(i) >= uint64(s.len): + raise IndexDefect.newException("index out of bounds") + else: + if uint64(i) >= uint64(s.len): + raise IndexDefect.newException("index out of bounds") + var n = s.start for _ in 0.. Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 32/87] asts: add `len` query --- nanopass/asts.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 7666e032..bbb891b6 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -67,6 +67,7 @@ proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: SomeInteger): T = n = t.next(n) result = T(index: n) +proc len*(s: ChildSlice): int = int(s.len) proc high*(s: ChildSlice): int = int(s.len) - 1 # ------- Storage implementation -------- From 690fdc067f3741eb44431efdf92e3572ed5f494e Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:52 +0000 Subject: [PATCH 33/87] asts: remove obsolete `Storage` type --- nanopass/asts.nim | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index bbb891b6..33ab2e87 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -37,10 +37,6 @@ type start: NodeIndex len: uint32 - Storage*[T] = object - ## The container AST fragments use for storing embedded datums. - data: seq[T] # TODO: use a BiTable - proc slice*[T](start: NodeIndex, len: uint32): ChildSlice[T] = ChildSlice[T](start: start, len: len) @@ -69,12 +65,3 @@ proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: SomeInteger): T = proc len*(s: ChildSlice): int = int(s.len) proc high*(s: ChildSlice): int = int(s.len) - 1 - -# ------- Storage implementation -------- - -proc pack*[T](s: Storage[T], val: sink T): uint32 = - s.data.add val - result = s.data.high.uint32 - -proc unpack*[T](s: Storage[T], id: uint32): lent T = - s.data[id] From b05581fe355b5648e2354fad6a4e545f35e7a1f6 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 34/87] literals: implement small-value optimization --- passes/literals.nim | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/passes/literals.nim b/passes/literals.nim index 5cf91ec8..3dafe951 100644 --- a/passes/literals.nim +++ b/passes/literals.nim @@ -1,7 +1,6 @@ ## Implements the storage for literal data embedded in ASTs. # TODO: use bi-tables for the number and string values -# TODO: store small integers inline (like how the packing in `trees` does it) type Literals* = object @@ -9,11 +8,33 @@ type numbers*: seq[uint64] ## a list of bit patterns strings*: seq[string] +const + OverflowShift = 31 + OverflowBit = 1'u32 shl OverflowShift + ## use the most significant bit to flag whether a value is larger than + ## `max(int32)` and overflows into `PackedTree.numbers` + +template castAny[T, U](x: U): T = + ## A `cast` that doesn't warn when casting into the origin type. + when T is U: x + else: cast[T](x) + proc pack*(s: var Literals, val: SomeInteger): uint32 {.inline.} = ## Adds the bit-representation of `val` to `s` and returns an ID to refer ## to the value with later. - result = uint32(s.numbers.len) - s.numbers.add cast[uint64](val) + when sizeof(val) < 4: + # always fits into the packed representation + result = cast[uint32](val) + else: + if (val shr OverflowShift) != 0: + # overflows the packed value range + result = uint32(s.numbers.len) or OverflowBit + when val is SomeSignedInt: + s.numbers.add castAny[uint64](int64(val)) # sign extend + else: + s.numbers.add castAny[uint64](val) + else: + result = castAny[uint32](val) # fits the packed range proc pack*(s: var Literals, val: float): uint32 {.inline.} = ## Adds the bit-representation of `val` to `s` and returns an ID to refer @@ -28,10 +49,10 @@ proc pack*(s: var Literals, val: string): uint32 {.inline.} = proc unpack*[T: SomeNumber](s: Literals, id: uint32, _: typedesc[T]): T {.inline.} = ## Returns the bit-representation stored under `id` interpreted as `T`. - when T is uint64: # prevent warnings - s.numbers[id] + if (id and OverflowBit) != 0: + castAny[T](s.numbers[id and not OverflowBit]) else: - cast[T](s.numbers[id]) + castAny[T](id) proc unpack*(s: Literals, id: uint32, _: typedesc[string]): lent string {.inline.} = ## Returns the string stored under `id`. From b7e420420cc147ddab24931215a6442fe6bf9075 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 35/87] nppass: make `outpass` work again --- nanopass/nppass.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index d51e93d6..d8f9b176 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -337,6 +337,7 @@ macro inpassImpl(name, nterm: typedesc, def: untyped) = macro outpassImpl(name, nterm: typedesc, def: untyped) = let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) lambda.params = copyNimTree(def.params) + lambda.params[1][^2] = nterm let call = newCall(lambda) # forward the original parameters to the lambda: @@ -344,6 +345,8 @@ macro outpassImpl(name, nterm: typedesc, def: untyped) = for j in 0.. Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 36/87] nanopass: close over `Ast` instead of just the tree * change `in.ast` and `out.ast` to be `Ast` instances, rather than packed trees. This gets rid of the `io.storage` local * change `transform` to not refer to the injected locals directly. The input and output AST are now passed via parameters, making the macro less context-dependent --- nanopass/nppass.nim | 56 +++++++++++++++++++--------------------- nanopass/nptransform.nim | 29 ++++++++++----------- 2 files changed, 40 insertions(+), 45 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index d8f9b176..623935a2 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -24,7 +24,7 @@ macro transformOutImpl(lang: static LangDef, name, def: untyped) = def.body.insert 0, quote do: # inject a build overload that implicitly uses the output language template build(body: untyped): untyped {.used.} = - build(`to`, `ret`, body) + build(`to`.tree, `ret`, body) result = def @@ -32,8 +32,10 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, sel: untyped, rules: varargs[untyped]): untyped = ## Implements the transformation/expansion of a language->language ## processor's trailing case statement/expression. + let input = newDotExpr(ident"in.ast", ident"tree") + let output = newDotExpr(ident"out.ast", ident"tree") let id = lang.map[src] - var (branches, used) = matchImpl(lang, id, ident"in.ast", sel, rules) + var (branches, used) = matchImpl(lang, id, input, sel, rules) template nt: untyped = lang.types[id] let sym = bindSym"transform" @@ -45,7 +47,7 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, newIntLitNode(id), (quote do: (typeof(result))( - index: `sym`(idef(src), idef(dst), typeof(result).N, `it`, `sel`)))) + index: `sym`(idef(src), idef(dst), typeof(result).N, `it`, `input`, `output`, `sel`)))) # auto-generate the branches for missing production terminals and # non-terminals: @@ -53,15 +55,13 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, if lang.types[it].terminal: let id = lang.types[it].ntag if id notin used: - let to = ident"out.ast" - let input = ident"in.ast" branches.add nnkOfBranch.newTree( newIntLitNode(id), (quote do: # TODO: use the tag of the destination language - `to`.nodes.add: + `output`.nodes.add: TreeNode[uint8](kind: uint8(`id`), val: `input`[`sel`].val) - (typeof(result))(index: NodeIndex(`to`.nodes.high)))) + (typeof(result))(index: NodeIndex(`output`.nodes.high)))) else: # if the first form's tag is used, so is the non-terminal itself if lang.forms[lang.types[it].forms[0]].ntag notin used: @@ -78,7 +78,6 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, index: `callee`(src.`name`(index: `sel`), dst.`name`).index) branches.add branch - let input = ident"in.ast" result = nnkCaseStmt.newTree(quote do: `input`[`sel`].kind) result.add branches if branches[^1].kind != nnkElse: @@ -155,7 +154,7 @@ macro transformInOutImpl(lang: static LangDef, name, def: untyped) = # inject a build macro overload that implicitly uses the # target non-terminal template build(body: untyped): untyped {.used.} = - build(`to`, `ret`, body) + build(`to`.tree, `ret`, body) result = def @@ -176,7 +175,6 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = ## proc definition, `call` the call to the pass' implementation. let input = ident"in.ast" let output = ident"out.ast" - let storage = ident"io.storage" let storageTy = ident"Literals" # TODO: don't hardcode let hasIn = src != nil let hasOut = dst != nil @@ -213,46 +211,44 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = let inj = ident"[]" body.add quote do: template match(sel: Metavar, branches: varargs[untyped]): untyped {.used.} = - match(`input`, sel, branches) + match(`input`.tree, sel, branches) template `inj`(x: ChildSlice, i: SomeInteger): untyped {.used.} = - `input`[x, i] + `input`.tree[x, i] template val[T](v: nanopass.Value[T]): T {.used.} = # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) # XXX: consider renaming this template to `get` - unpack(`storage`[], v.index, typeof(T)) + unpack(`input`.storage[], v.index, typeof(T)) if hasOut: let embed = bindSym"embed" body.add quote do: template terminal(x: untyped): untyped {.used.} = - `embed`(`storage`, x) + `embed`(`output`.storage, x) template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = - build(`output`, n, body) - - if hasIn: - # re-use the data storage object from the input - body.add quote do: - let `storage` = `input`.storage - else: - body.add quote do: - let `storage` = new(`storageTy`) + build(`output`.tree, n, body) if hasIn: # shadow the input tree with a cursor to prevent a costly copy when # it's captured by the closure body.add quote do: - let `input` {.cursor.} = `input`.tree + let `input` {.cursor.} = `input` if hasOut: body.add quote do: - var `output`: Ast[dst, `storageTy`].tree - let index = `call`.index + var `output` = Ast[dst, `storageTy`]() + if hasIn: + # re-use the storage from the input + body.add quote do: + `output`.storage = `input`.storage + else: + body.add quote do: + `output`.storage = new(`storageTy`) + body.add quote do: + let pos = `call` # turn the AST with indirections into one without - result = Ast[dst, `storageTy`]( - tree: finish(`output`, index), - storage: `storage`, - ) + `output`.tree = finish(`output`.tree, pos.index) + result = move `output` else: body.add quote do: result = `call` diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 3cc9bde1..2342e5a6 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -26,7 +26,8 @@ proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = inc i macro transform*(src, dst: static LangInfo, nterm: static string, - form: static int, n: untyped): untyped = + form: static int, input, output: PackedTree[uint8], + n: untyped): untyped = ## Generates the transformation from the given source language form ## to a compatible target language production of the non-terminal with ## name `nterm`. @@ -67,18 +68,16 @@ macro transform*(src, dst: static LangInfo, nterm: static string, # parameter, which would eliminate unnecessary tree seeking when # there's many calls to fully auto-generated non-terminal processors - let inAst = ident"in.ast" - let to = ident"out.ast" let id = dst.forms[target].ntag.uint8 result = newStmtList() # add the root node: let body = quote do: - var tmp {.used.} = `inAst`.child(`n`, 0) - let root = `to`.nodes.len.NodeIndex - var i = `to`.nodes.len - # the node sequence needs to be contiguous, so it's allocated upfront - `to`.nodes.setLen(i + `inAst`.len(`n`) + 1) - `to`.nodes[i] = TreeNode[uint8](kind: `id`, val: `inAst`[`n`].val) + var tmp {.used.} = `input`.child(`n`, 0) + let root = `output`.nodes.len.NodeIndex + var i = `output`.nodes.len + # the node sequence needs output be contiguous, so it's allocated upfront + `output`.nodes.setLen(i + `input`.len(`n`) + 1) + `output`.nodes[i] = TreeNode[uint8](kind: `id`, val: `input`[`n`].val) inc i # call the transformers and emit the nodes in one go: @@ -96,13 +95,13 @@ macro transform*(src, dst: static LangInfo, nterm: static string, if src.types[a.typ].ntag == dst.types[b.typ].ntag: # just copy the node quote do: - `to`.nodes[i] = `inAst`[tmp] + `output`.nodes[i] = `input`[tmp] inc i else: # repack with the new tag let tag = dst.types[b.typ].ntag quote do: - `to`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `inAst`[tmp].val) + `output`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `input`[tmp].val) inc i else: let append = bindSym"append" @@ -110,19 +109,19 @@ macro transform*(src, dst: static LangInfo, nterm: static string, let s = newStrLitNode(src.types[a.typ].name) let d = newStrLitNode(dst.types[b.typ].name) quote do: - `append`(`to`, i, + `append`(`output`, i, `op`(Metavar[src, `s`](index: tmp), Metavar[dst, `d`])) if a.repeat: let bias = src.forms[form].elems.len - 1 body.add quote do: - for _ in 0..<(`inAst`.len(`n`) - `bias`): + for _ in 0..<(`input`.len(`n`) - `bias`): `call` - tmp = `inAst`.next(tmp) + tmp = `input`.next(tmp) else: body.add quote do: `call` - tmp = `inAst`.next(tmp) + tmp = `input`.next(tmp) result.add body # the callsite takes care of fitting the index to the right type From c4ccb710832278020eb3ecc120f39b51b0e500a4 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 37/87] nanopass: minor cleanup --- nanopass/nanopass.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 94ef1675..70776eb8 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -41,6 +41,12 @@ proc finish*(ast: PackedTree[uint8], n: NodeIndex): PackedTree[uint8] = template dst: untyped = result.nodes const size = sizeof(TreeNode[uint8]) + template append(start, fin: uint32) = + let pos = dst.len + let num = int(fin - start) + dst.setLen(pos + num) + copyMem(addr dst[pos], addr src[start], num * size) + # search for runs of contiguous nodes. When encountering an indirection, # copy the run and move the source cursor to the indirection's # target; repeat. @@ -55,9 +61,7 @@ proc finish*(ast: PackedTree[uint8], n: NodeIndex): PackedTree[uint8] = elif src[i].kind == RefTag: if i > prev: # copy everything we got so far - let pos = dst.len - dst.setLen(pos + int(i - prev)) - copyMem(addr dst[pos], addr src[prev], int(i - prev) * size) + append(prev, i) stack[^1] = (i + 1, last) let next = src[i].val @@ -68,8 +72,6 @@ proc finish*(ast: PackedTree[uint8], n: NodeIndex): PackedTree[uint8] = if i > prev: # copy the rest - let pos = dst.len - dst.setLen(pos + int(i - prev)) - copyMem(addr dst[pos], addr src[prev], int(i - prev) * size) + append(prev, i) stack.shrink(stack.len - 1) From 83ce3e1a421111ed4b2d5bbb1b1e83327804f8dd Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 38/87] nanopass: move `isAtom` and `RefTag` to `asts` --- nanopass/asts.nim | 8 ++++++++ nanopass/nanopass.nim | 6 +----- nanopass/npbuild.nim | 2 +- nanopass/nplangdef.nim | 3 +-- nanopass/nptransform.nim | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 33ab2e87..7679e44e 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -37,6 +37,14 @@ type start: NodeIndex len: uint32 +const + RefTag* = 128'u8 + ## the node used internally for indirections + +template isAtom*(x: uint8): bool = + ## The predicate required for using an uint8 as a ``PackedTree`` tag. + x >= RefTag + proc slice*[T](start: NodeIndex, len: uint32): ChildSlice[T] = ChildSlice[T](start: start, len: len) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 70776eb8..f35343f1 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -9,7 +9,7 @@ import passes/[trees], - nanopass/[asts, nplangdef, nplanggen, npmatch, npbuild, nppass, nppatterns, npsexpr] + nanopass/[asts, nplanggen, npmatch, npbuild, nppass, nppatterns, npsexpr] export asts export nppatterns.matches @@ -19,10 +19,6 @@ export nppass.pass, nppass.inpass, nppass.outpass export nppass.genProcessor # TODO: ^^ bind the symbols; don't mix them in -template isAtom*(x: uint8): bool = - ## The predicate required for using an uint8 as a ``PackedTree`` tag. - x >= RefTag - macro defineLanguage*(name, body: untyped) = ## Creates a language definition and binds it to a const symbol with the ## given name. diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 5d51f03e..27eb9266 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -2,7 +2,7 @@ import std/[genasts, macros, strformat, tables] import passes/trees -import nanopass/[asts, helper, nplang, nplangdef, nppatterns] +import nanopass/[asts, helper, nplang, nppatterns] proc lookup[E; M: tuple](): auto {.compileTime.} = var x: M diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 9364805d..4717b23a 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -1,6 +1,7 @@ ## Implements the language definition parsing and processing. import std/[macros, intsets, sets, strformat, tables] +from nanopass/asts import RefTag type # Core types capturing a defined language @@ -66,8 +67,6 @@ type add: seq[NimNode] const - RefTag* = 128'u8 - ## the node used internally for indirections FirstTerminalTag* = RefTag + 1 ## the start of the terminals' tag space diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 2342e5a6..805901c0 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -2,7 +2,7 @@ import std/[macros, strformat, tables] import passes/[trees] -import nanopass/[asts, helper, nplang, nplangdef] +import nanopass/[asts, helper, nplang] type Morphability = enum From 14a6a0d9b5f9f0bdbdde0755f6381efc911e5b13 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:53 +0000 Subject: [PATCH 39/87] npmatch: rework the pattern matching feature * implement support for nested pattern matching * allow repetition matching in all contexts * support wildcard matching in all form positions (including the name slot!) * support `a -> b` syntax for the pseudo-catamorphisms (i.e., `[...]`) --- nanopass/npmatch.nim | 999 ++++++++++++++++++++++++++++++++++--------- nanopass/nppass.nim | 89 ++-- 2 files changed, 828 insertions(+), 260 deletions(-) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 58557bef..879e5e0f 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -4,247 +4,822 @@ import std/[macros, intsets, strformat, tables] import passes/trees import nanopass/[asts, helper, nplang] -proc matchImpl*(lang: LangInfo, src: int, ast, sel, rules: NimNode - ): (seq[NimNode], IntSet) = - ## Implements the core of the `match` macro: - ## 1. makes sure the syntax is correct - ## 2. makes sure the used patterns are unique - ## 3. generates a sequence of transformed 'of' branches, plus a set storing - ## the used forms' tags - var used: IntSet - ## covered form productions (identified by ntag) - - # should nested matching (e.g., ``A(x, B(y))``) be desired, `matchImpl` - # should be factored into two macros macros applied sequentially: - # 1. the first macro does type checking, producing a type form - # 2. the second macro translates the typed form into case/if statements - # combining both steps into one is simple enough when there are no nested - # patterns, but not otherwise - - proc parseVar(n: NimNode): (string, string) = - n.expectKind nnkIdent - let name = n.strVal - var e = name.high - # to get the name of the var, trim trailing numbers and a single underscore - while e >= 0 and name[e] in {'0'..'9'}: - dec e - - if e >= 0 and name[e] == '_': - dec e - - result[0] = name[0..e] - result[1] = name - - proc processIdentPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = - n.expectKind nnkIdent - let (v, nameStr) = parseVar(n) - if v notin lang.map: - error(fmt"no meta-variable with name '{v}'", n) - let id = lang.map[v] - if id notin lang.types[src].sub: - error(fmt"'{v}' is not an immediate production of '{lang.types[src].name}'", n) - - var check, binds: NimNode - let name = ident(nameStr) - if lang.types[id].terminal: - let typ = ident(lang.types[id].name) - let tag = lang.types[id].ntag - used.incl(tag) - # let the compiler report an error for duplicate case labels - check = newIntLitNode(tag) - copyLineInfo(check, n) - binds = newLetStmt(name, quote do: Value[`typ`](index: `ast`[`sel`].val)) +type + FillProc* = proc(lang: LangInfo, idx: int, n, info: NimNode): NimNode + ## Type for form or type fill callback. + ExpandConfig* = object + fillForm*: FillProc + ## called for filling-in the handling of forms. May be nil + fillType*: FillProc + ## called for filling-in the handling of types. May be nil + +proc defaultFillForm(lang: LangInfo, idx: int, n, info: NimNode): NimNode = + makeError(fmt"missing rule for '{render(lang, lang.forms[idx])}'", info) + +proc defaultFillType(lang: LangInfo, idx: int, n, info: NimNode): NimNode = + makeError(fmt"missing rule for '{lang.types[idx].name}'", info) + +proc parseVar(n: NimNode): string = + n.expectKind nnkIdent + let name = n.strVal + var e = name.high + # to get the name of the var, trim trailing numbers and a single underscore + while e >= 0 and name[e] in {'0'..'9'}: + dec e + + if e >= 0 and name[e] == '_': + dec e + + result = name[0..e] + +proc fits(lang: LangInfo, a, b: int): bool = + ## Computes whether type with id `a` can appear where a type with id `b` + ## is expected. + if a == b: + result = true + elif not lang.types[b].terminal: + for it in lang.types[b].sub.items: + if fits(lang, a, it): + return true + +proc countTags(lang: LangInfo, typ: LangType): int = + result = typ.forms.len + for it in typ.sub.items: + if lang.types[it].terminal: + result += 1 else: - # the pattern binds a non-terminal - let typ = ident(v) - check = nnkCurly.newTree() - let tags = ntags(lang, lang.types[id]) - for tag in tags.items: - check.add nnkConv.newTree(ident"uint8", newIntLitNode(tag)) - # mark the first tag as used, to signal that the non-terminal is handled - used.incl(tags[0]) + result += countTags(lang, lang.types[it]) - copyLineInfoForTree(check, n) - binds = newLetStmt(name, quote do: src.`typ`(index: `sel`)) +proc containsForm(lang: LangInfo, typ: LangType, fid: int): bool = + if fid in typ.forms: + true + else: + for it in typ.sub.items: + if not lang.types[it].terminal and containsForm(lang, lang.types[it], fid): + return true + false - result = (check, binds) +proc makeTyped(e, typ, info: NimNode): NimNode = + typ.copyLineInfo(info) # the type tree carries the source location + nnkExprColonExpr.newTree(e, typ) - proc processPattern(lang: LangInfo, n: NimNode): (NimNode, NimNode) = - case n.kind +proc fitTo(lang: LangInfo, typ: int, pat: NimNode): NimNode = + ## Fits a non-'...' typed pattern to the given type, returning either the + ## fitted pattern or nil. + + proc canMerge(n, into: NimNode): bool = + assert n.kind == into.kind and n.kind == nnkCall + result = true + for i in 1..", newEmptyNode(), quote do: dst.`id`) + copyLineInfo(call, n) + # the matched type needs to be inferred + result = makeTyped( + nnkTupleConstr.newTree(nnkExprEqExpr.newTree(n[0], call)), + newNimNode(nnkNilLit), n) + of nnkInfix: + if n[0].len != 3 or not eqIdent(n[0][0], "->"): + error("expected '->' infix call", n[0]) + let src = parseVar(n[0][1]) + if src notin lang.map: + error(fmt"no meta-variable with name '{src}' exists", n[0][1]) + + let id = ident(parseVar(n[0][2])) + let call = nnkInfix.newTree(ident"->", n[0][1], quote do: dst.`id`) + copyLineInfo(call, n) + # bind the matched value to the first identifier and the result of + # the application to the second identifier + result = makeTyped( + nnkTupleConstr.newTree( + n[0][1], + nnkExprEqExpr.newTree(n[0][2], call)), + newIntLitNode(lang.map[src]), n) + else: + error("expected identifier or '->' infix call'", n[0]) + of nnkPrefix: + if not n[0].eqIdent("..."): + error("only `...` is allowed as a prefix", n[0]) + + let tmp = parsePattern(lang, n[1]) + if tmp[1].kind notin {nnkIntLit, nnkEmpty, nnkNilLit}: + error("only '...' and '...any' are allowed", n[1]) + # the '...' prefix is stripped from the expression + result = makeTyped(tmp[0], nnkBracket.newTree(tmp[1]), n) + of nnkIdent: + if n.eqIdent("_"): + # placeholder, type may be inferred + result = makeTyped(n, newEmptyNode(), n) + elif n.eqIdent("any"): + # an alias for '_' + result = makeTyped(ident"_", newEmptyNode(), n) + else: + # must be a meta-variable + let typ = parseVar(n) + if typ notin lang.map: + error("no meta-variable with the give name exists", n) + + result = makeTyped(n, newIntLitNode(lang.map[typ]), n) + else: + error("syntax error", n) + +proc patternToString(n: NimNode; indent = 0): string = + case n.kind + of nnkCurly: + result = "{" + result.add repr(n[0]) + for i in 1.. 1: + result.add ", " + result.add repr(n[i]) + if n.len > 2: + result.add ", " + result.add patternToString(n[^1], indent) + result.add ")" + of nnkPar: + result = "^" + result.add repr(n[0]) + result.add " " + result.add patternToString(n[1], indent) + of nnkEmpty: + result = "." + of nnkStmtList: + result = "" + of nnkTupleConstr: + result = "(" + result.add repr(n[0]) + result.add ", )" + else: + result = "" + +proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, + config: ExpandConfig): NimNode = + ## Generates the NimSkull code for a match expression `expr`. `els` is either + ## an `nnkElse` tree used for handling the rest of match, or nil, in which + ## case how to handle the rest of a match is dictated by `config`. + let cursor = genSym("cursor") + var stack: seq[NimNode] ## stack of `len` symbols + var top: NimNode + ## the topmost non-union match expression enclosing the currently + ## processed one + var hasUsedElse = false + + proc aux(lang: LangInfo, e, to: NimNode): NimNode = + ## Does the actual work. + proc tm(lang: LangInfo, e, to: NimNode): NimNode = + if e[1].kind != nnkEmpty: + # commit the current cursor to a local with the given name + if e[0].kind == nnkBracket: + let bias = e[2] + let len = stack[^1] # can only be non-empty + to.add newLetStmt(e[1], quote do: (`cursor`, `len` - `bias`)) + else: + to.add newLetStmt(e[1], cursor) + + # only emit a cursor movement when the moved cursor is actually observed + if e[^1].kind in {nnkCall, nnkCurly, nnkPar}: + case e[0].kind + of nnkPar: + let nlen = genSym"len" + to.add quote do: + let `nlen` = `ast`[`cursor`].val + to.add quote do: + `cursor` = `ast`.child(`cursor`, 0) + + stack.add nlen + of nnkEmpty, nnkIntLit: + to.add quote do: + `cursor` = `ast`.next(`cursor`) + of nnkBracket: + let bias = e[2] + let len = stack[^1] # can only be non-empty + to.add quote do: + for _ in 0 ..< (`len` - `bias`): + `cursor` = `ast`.next(`cursor`) + else: + unreachable(e[0].kind) + + result = aux(lang, e[^1], to) + + case e.kind of nnkCall: - n[0].expectKind nnkIdent - # parse the pattern: - var elems = newSeq[tuple[src, dst, name: string]](n.len - 1) - for i in 1.. 0: ident(elems[i].dst) - else: ident(lang.types[it.typ].mvar) - if it.repeat: - let bias = lang.forms[idx].elems.len - 1 - if p.kind == nnkIdent: - # just bind a child slice to the identifier - binds.add newLetStmt(p, quote do: - slice[`origin`](`cursor`, uint32(`ast`.len(`sel`) - `bias`))) - binds.add quote do: - for _ in 0..<`ast`.len(`sel`)-`bias`: - `cursor` = `ast`.next(`cursor`) - else: - # run the selected transformer on all relevant child nodes and - # store the result in a seq - let tmp = genSym() - binds.add newVarStmt(tmp, - quote do: newSeq[`target`](`ast`.len(`sel`)-`bias`)) - let callee = ident"->" - binds.add quote do: - for i in 0..<`ast`.len(`sel`)-`bias`: - `tmp`[i] = `callee`(`origin`(index: `cursor`), `target`) - `cursor` = `ast`.next(`cursor`) - binds.add newLetStmt(p[0], tmp) + unreachable() + + if stack.len == 0: + top = it + + handler.add newStmtList() + discard tm(lang, it, handler[^1]) + caseStmt.add handler + # pop all leftover len entries + stack.setLen(stackLen) + + let info = e[0] + # handle the uncovered values, if any + if els != nil: + let b = genOfBranch(lang, lang.types[typ], used) + if b.len > 0: # are there any uncovered values? + caseStmt.add els + hasUsedElse = true + + if stack.len == 0 and not hasUsedElse: + # the 'else' rule was never used. Add it to the top-level case + # statement such that a warning will be emitted + if caseStmt[^1].kind != nnkElse: + caseStmt.add nnkElse.newTree(newCall(bindSym"unreachable")) + caseStmt.add els + elif stack.len > 0 and config.fillForm != nil: + # uncovered values in nested positions are handled by processing + # the whole production + assert top.kind == nnkCall and top[0].kind == nnkPar + var allCovered = true + for tag in ntags(lang, lang.types[typ]): + if tag notin used: + allCovered = false + break + + if allCovered: + discard "nothing to do" + elif top[0].len == 1: + # simple case. The form ID is known statically + caseStmt.add nnkElse.newTree( + config.fillForm(lang, top[0][0].intVal.int, sel, info)) else: - # simple case: a single node - if p.kind == nnkIdent: - if lang.types[it.typ].terminal: - binds.add newLetStmt(p, quote do: `origin`(index: `ast`[`cursor`].val)) - else: - binds.add newLetStmt(p, quote do: `origin`(index: `cursor`)) + # complex case. The form ID is only known at run-time; dispatch over + # the entry-level node's kind for detecting the form + let inner = nnkCaseStmt.newTree( + quote do: `ast`.nodes[`sel`].kind) + for it in top[0].items: + inner.add nnkOfBranch.newTree(it, + config.fillForm(lang, it.intVal.int, sel, info)) + inner.add nnkElse.newTree(newCall(bindSym"unreachable")) + caseStmt.add nnkElse.newTree(inner) + else: + var + fillForm = config.fillForm + fillType = config.fillType + + # always report an error for nested productions when form-filling + # is unavailable + if fillForm.isNil or stack.len > 0: + fillForm = defaultFillForm + if fillType.isNil or stack.len > 0: + fillType = defaultFillType + + for it in lang.types[typ].forms.items: + if lang.forms[it].ntag notin used: + caseStmt.add nnkOfBranch.newTree( + newIntLitNode(lang.forms[it].ntag), + fillForm(lang, it, cursor, info)) + + # fill in handling for not fully handled subtypes + for it in lang.types[typ].sub.items: + let br = genOfBranch(lang, lang.types[it], used) + if br.len > 0: + br.add fillType(lang, it, cursor, info) + caseStmt.add br + + if caseStmt[^1].kind != nnkElse: + # all allowed node tags are handled, the rest are known to + # be impossible + caseStmt.add nnkElse.newTree(newCall(bindSym"unreachable")) + + to.add caseStmt + # nothing may follow a dispatcher + result = nil + of nnkTupleConstr: + # in-place transform the identdefs into proper ones + + proc transformSource(lang: LangInfo, pos, typ, orig: NimNode): NimNode = + case typ.kind + of nnkIntLit: + # a single item binding + let mvar = ident(lang.types[typ.intVal].mvar) + if pos.kind == nnkIdent: + pos # the source is a bound identifier already else: - if lang.types[it.typ].terminal: - binds.add makeError("cannot invoke auto-procesor for terminal", p) + if lang.types[typ.intVal].terminal: + quote do: `name`.`mvar`(index: `ast`[`pos`].val) else: - binds.add newLetStmt(p[0], - newCall(ident"->", - nnkObjConstr.newTree(origin, - nnkExprColonExpr.newTree(ident"index", cursor)), - target)) - binds.add quote do: - `cursor` = `ast`.next(`cursor`) - result = (check, binds) + quote do: `name`.`mvar`(index: `pos`) + of nnkBracket: + # a list binding + let mvar = ident(lang.types[typ[0].intVal].mvar) + if pos.kind == nnkIdent: + pos # the source is already a slice (bound to an identifier) + else: + quote do: + slice[`name`.`mvar`](`pos`[0], uint32(`pos`[1])) + else: + unreachable() + + for it in e[0].items: + case it[2].kind + of nnkSym: + it[2] = transformSource(lang, it[2], it[1], it[2]) + else: + it[2][1] = transformSource(lang, it[2][1], it[1], it[2]) + it[1] = newEmptyNode() + + to.add e[0] + to.add e[1] + result = nil + of nnkStmtList: + to.add e + result = nil # statement lists are always trailing + else: + unreachable(e.kind) + + result = nnkStmtList.newTree() + result.add newVarStmt(cursor, sel) + discard aux(lang, e, result) + +proc patternToMatch(n: NimNode): tuple[head, tail: NimNode] = + ## Translates a pattern into a match expression. A match expression has the + ## following structure: + ## + ## match ::= (nkCall (nkBracket ) ) + ## | (nkCall (nkPar +) ) + ## | (nkCall ) + ## name ::= | + ## cont ::= + ## | (nkPar ) # leave sub match + ## | (nkCurly +) # union matching + ## | (nkTupleConstr (nkLetSection ...) ...) # binding + ## | (nkStmtList ...) # custom tail logic + var binds: NimNode + proc addBinding(got: NimNode) = + if binds.isNil: + binds = nnkLetSection.newTree() + binds.add got + + let empty = newEmptyNode() # save some allocations + + proc aux(n: NimNode, depth: int): tuple[head, tail: NimNode, depth: int] = + let e = n[0] + let typ = n[1] + case typ.kind + of nnkPar: + result.head = nnkCall.newTree(typ, empty) + result.tail = result.head + assert e.kind == nnkCall + var depth = depth + 1 + for i in 1.. list of identDefs + proc step(n: NimNode, depth: int) = + case n.kind + of nnkCall: + step(n[^1], depth + 1) + var defs: NimNode + if pop(map, depth, defs): + let s = genSym"pos" + n[1] = s + for it in defs.items: + if it[^1].kind in nnkCallKinds: + it[^1][1] = s + else: + it[^1] = s + of nnkCurly: + for i in 1.." - # dispatch to the processor and convert to the expected type - branch.add quote do: - (typeof(result))( - index: `callee`(src.`name`(index: `sel`), dst.`name`).index) - branches.add branch - - result = nnkCaseStmt.newTree(quote do: `input`[`sel`].kind) - result.add branches - if branches[^1].kind != nnkElse: - # the selector is a uint8 and thus the case cannot be exhaustive - result.add nnkElse.newTree(newCall(ident"unreachable")) + # TODO: consider inlining the transformer if it's auto-generated. + # More code to emit, but also a little less work at run-time + let name = ident(lang.types[typ].mvar) + let callee = ident"->" + # dispatch to the processor and convert to the expected type + quote do: + (typeof(result))( + index: `callee`(src.`name`(index: `n`), dst.`name`).index) + + let config = ExpandConfig( + fillForm: fillForm, + fillType: fillType, + ) + + matchImpl(lang, lang.map[src], ident"src", input, sel, rules, config) macro genProcessor*(index, nterm: untyped): untyped = ## Generates the body for a non-terminal processor. @@ -293,6 +278,14 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = genProcessor(n.index, U.N) + proc `name`[T, C, N](s: ChildSlice[T, C], + U: typedesc[Metavar[`dst`, N]]): seq[U] {.closure.} = + # XXX: the explicit .closure annotation works around a closure inference + # compiler bug + result = newSeq[U](s.len) + for i, it in s.pairs: + result[i] = `name`(it, U) + # if the body doesn't end in an expression, add a call to the # entry processor if def.body[^1].kind == nnkProcDef: From 394edbba019e6494dec4753352f798701e68a150 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 40/87] npbuild: rework form construction feature * require explicit `...` prefix for list expansion * allow list expansion in bracket syntax * allow list expansion in forms * remove quoting requirement for literal values (numbers, strings, etc.) * significantly improve error messages for type mismatches --- nanopass/npbuild.nim | 558 ++++++++++++++++++++++++++++++++----------- 1 file changed, 424 insertions(+), 134 deletions(-) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 27eb9266..25db652d 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -4,12 +4,94 @@ import std/[genasts, macros, strformat, tables] import passes/trees import nanopass/[asts, helper, nplang, nppatterns] +type + Candidate = object + ## Represents a potential form candidate matching the input construction. + tag: int ## form tag + types: seq[int] + ## for each operand in the input construction, the required type + min: int ## minimum number of items a dynamic operand must have + max: int ## maximum number of items a dynamic operand can have + +proc toPrefix(cand: Candidate, start: int): NimNode = + ## Turns the type sequence starting at `start` into a prefix (to be used + ## in a prefix tree). + # note: for the sake of efficiency, the prefix tree is stored as an + # AST using a tiny custom grammar + result = nnkBracket.newTree() + for i in start.. c.max): + break + inc at + + tree.insert(at, tup) + of nnkBracket: + var i = i + var j = 0 + while j < tree.len - 1 and tree[j].intVal.int == c.types[i]: + inc i + inc j + + if j == tree.len - 1: + # the sequences are the same so far; merge the tails + insert(tree[j], c, i) + else: + # fork where the sequences start to differ + let old = nnkBracket.newTree(tree[(j + 1)..^1]) + tree.del(j + 1, tree.len - j - 1) + tree[j] = nnkCurly.newTree( + nnkPar.newTree(tree[j], old), + nnkPar.newTree(newIntLitNode(c.types[i]), toPrefix(c, i + 1))) + else: + unreachable() + +# ---------- helper routines ---------- + +{.push stacktrace: off.} + proc lookup[E; M: tuple](): auto {.compileTime.} = var x: M for it in fields(x): when E is typeof(it[0]): return it[1] +proc lengthError(len: int) {.noinline.} = + raise ValueError.newException( + fmt"no form is able to fit the expanded sequence (length = {len}") + proc append[L, U](to: var PackedTree[uint8], x: Value[U]) = to.nodes.add TreeNode[uint8]( kind: typeof(lookup[U, L.meta.term_map]()).V, @@ -22,154 +104,348 @@ proc append[L](to: var PackedTree[uint8], x: openArray) = for it in x.items: append[L](to, it) -# helpers for `buildImpl` -template len(x: Value): int = 1 -template len(x: Metavar): int = 1 +template coerce[T, U](x: U, _: typedesc[Value[T]]): Value[T] = + mixin terminal + when T is U: + terminal(x) # no coercion is necessary + else: + terminal(T(x)) # try a coercion, an error is fine + +{.pop.} + +# ---------- macro implementation ---------- + +proc containsForm(lang: LangInfo, typ: LangType, form: int): bool = + if form in typ.forms: + result = true + else: + result = false + for it in typ.sub.items: + if not lang.types[it].terminal and + containsForm(lang, lang.types[it], form): + result = true + break -macro buildImpl(to: var PackedTree[uint8], lang: static LangInfo, - typ: typedesc[Metavar], e: untyped): untyped = +macro buildImpl(to: var PackedTree[uint8], + lang: static LangInfo, name: static string, + target: typedesc[Metavar], e: untyped): untyped = ## Emits a tree construction for the AST described by `e`, with the syntax ## from `lang`. - proc cons(lang: LangInfo, n, test, body: NimNode): NimNode {.closure.} - proc elem(lang: LangInfo, n, test, body: NimNode): NimNode = - ## Processor for element syntax. - case n.kind - of nnkCall: - result = cons(lang, n, test, body) - of nnkBracket: - result = nnkBracket.newTree() + # the `build` macro is complex, as: + # * there may be multiple forms in a language that have the same name + # * the interpolated operands' types are not known to the macro + # * list expansion is allowed, at least a single one + # In effect, a sort of overload resolution has to be performed for picking + # which form the build syntax ultimately matches. Due to list expansion, + # this cannot always be known at compile-time, in which case disambiguation + # has to happen at *run-time*. In the abstract, the macro works by emitting + # a decision tree (using `when` statements) that selects the form based on + # the operands' types + + proc newMismatchError(src, dst, info: NimNode): NimNode = + result = quote do: + {.error: "expected type fitting " & $`dst` & ", but got " & + $typeof(`src`).} + copyLineInfoForTree(result, info) + + proc makeMatch(src, expect: NimNode): NimNode = + let error = newMismatchError(src, expect, src) + let append = bindSym"append" + result = quote do: + when matches(`src`, `expect`): + `append`[`target`.L](`to`, `src`) + else: + `error` + copyLineInfoForTree(result, src) + + proc addAll(to, n: NimNode) = + if n.kind == nnkStmtList: for it in n.items: - result.add elem(lang, it, test, body) - of nnkIdent, nnkSym, nnkAccQuoted: - # some hoisted expression - result = n - body.add genAst(typ, to, n) do: - append[typ.L](to, n) + to.add it else: - error("unexpected syntax", n) + to.add n + + proc process(lang: LangInfo, typ: LangType, n: NimNode): NimNode {.closure.} = + ## Parses the expression `n` and interprets in the context of `typ`, + ## returning either a `when`-then-else statement, a statement list + ## containing such `when` statement, or, if there's a static type error, + ## a single error statement. + case n.kind + of nnkCall: + if n[0].kind != nnkIdent: + error("constructor must be an identifier", n[0]) + + if typ.terminal: + # expected a terminal, but the constructor can only be that of a form + let mvar = ident(typ.mvar) + result = quote do: + {.error: "expected terminal of type " & $`target`.L.`mvar`.} + copyLineInfoForTree(result, n) + return + + var candidates: seq[Candidate] + + # gather all forms part of `typ` that have a matching name and whose + # shape matches that of the construction + for id, form in lang.forms.pairs: + if form.name != n[0].strVal or not containsForm(lang, typ, id): + continue + + var fpos = 0 # position in form description + var i = 1 + var min, max = 0 + var types = newSeq[int](n.len - 1) + while i < n.len and fpos < form.elems.len: + types[i - 1] = form.elems[fpos].typ + case n[i].kind + of nnkSym, nnkCall, nnkLiterals: + # may only be a single element + if form.elems[fpos].repeat: + break + inc fpos + of nnkBracket: + # must only appear where a list is expected + if not form.elems[fpos].repeat: + break + inc fpos + of nnkPrefix: + # unpack expression. Expands to elements of the exact same type + if min != 0: + error("outside a bracket, only a single expansion is allowed", + n[i]) + + let start = fpos + let fin = form.elems.len - (n.len - i - 1) + while fpos < fin and form.elems[fpos].typ == form.elems[start].typ: + if form.elems[fpos].repeat: + max = high(int) # expanded list doesn't have a max length + else: + min += 1 + inc fpos + + if fpos == start: + break # fits nothing in the receiving form + else: + unreachable(n[i].kind) + inc i + + if fpos < form.elems.len or i < n.len: + continue # shape doesn't match + + if max == 0: + max = min # the expanded list, if any, has a known upper limit + + # shape matches + candidates.add Candidate( + tag: form.ntag, + types: types, + min: min, + max: max) - proc form(lang: LangInfo, n, test, body: NimNode): NimNode = - ## Processor for a tree construction. - let tag = n[0].strVal - var elems: seq[NimNode] - var temp = newStmtList() - for i in 1.. happens when there's a static type error + err = got + else: + body.add emit(lang, c, it[1], n, i+1) + result.add nnkElifBranch.newTree(cond, body) + + if result.len == 0: + # none of the variants are viable + result = err else: - if inp.kind == nnkBracket: - cond = ident"false" # type mismatch - break + result.add nnkElse.newTree( + makeError("not a valid production for this position", n[i])) + of nnkIfExpr: + proc genBody(ctx: Context, tup: NimNode): NimNode = + if ctx.start.isNil: + newStmtList() # the node tag is set already else: - cond.appendCheck(inp, access(lang.types[e.typ])) + let start = ctx.start + let id = tup[2] + quote do: `to`.nodes[`start`].kind = `id` - whenStmt.add nnkElifBranch.newTree(cond, - nnkObjConstr.newTree( - nnkBracketExpr.newTree(bindSym"PForm", newIntLitNode(form.ntag)))) + let expanded = c.expanded + if expanded != nil: + # only when there's a list to expand can more than one shape of a + # form statically match for a construction. Run-time disambiguation + # is required + result = nnkIfStmt.newTree() + for it in t.items: + var cond = ident"true" + if it[0].intVal == it[1].intVal: + # no bound check is needed + let m = it[0] + cond = quote do: len(`expanded`) == `m` + else: + if it[0].intVal > 0: + let val = it[0] + cond = nnkInfix.newTree(ident"and", cond, + quote do: len(`expanded`) >= `val`) + if it[1].intVal < high(int): + let val = it[1] + cond = nnkInfix.newTree(ident"and", cond, + quote do: len(`expanded`) <= `val`) - if whenStmt.len == 0: - test.add makeError(fmt"no form with arity {elems.len} exists", n) - result = ident"true" - else: - whenStmt.add nnkElse.newTree( - makeError("no form with the given shape exists", n)) - - let res = genSym("form") - test.add newLetStmt(res, whenStmt) - - var len = newIntLitNode(0) - for it in elems.items: - let t = - if it.kind == nnkPar: newIntLitNode(1) - elif it.kind == nnkBracket: newIntLitNode(it.len) - else: newCall(bindSym("len", brOpen), it) - - len = nnkInfix.newTree(ident"+", len, t) - body.add quote do: - `to`.nodes.add TreeNode[uint8](kind: typeof(`res`).I.uint8, - val: uint32(`len`)) - body.add temp - # a par distinguishes an inline constructed tree from some embedded value - result = nnkPar.newTree(res) - - proc cons(lang: LangInfo, n, test, body: NimNode): NimNode = - ## Processor for a ``X(...)`` expression, which may either be a terminal - ## construction or inline tree construction. - n[0].expectKind nnkIdent - let name = n[0].strVal - if name in lang.map: - # can only be a terminal - n.expectLen 2 - let mvar = ident(name) - let sym = genSym() - let tmp = genSym() - let cons = n[1] - let storage = ident"io.storage" - # ensure the operand having the right type via a conversion, but only - # when there's no `is`, so as to not interfere with sinking - test.add quote do: - let `tmp` = `cons` - let `sym` = dst.`mvar`(index: pack(`storage`[], - (when `tmp` is dst.`mvar`.T: - `tmp` - else: - dst.`mvar`.T(`tmp`)))) - body.add genAst(typ, to, sym) do: - append[typ.L](to, sym) - nnkPar.newTree(sym) + result.add nnkElifExpr.newTree(cond, genBody(c, it)) + + if result.len == 1 and result[0][0].kind == nnkIdent: + # no check is necessary + result = result[0][1] + else: + result.add nnkElse.newTree( + genAst(expanded) do: lengthError(len(expanded))) + else: + result = genBody(c, t[0]) + of nnkIntLit: + result = process(lang, lang.types[t.intVal], n[i + 1]) + of nnkNilLit: + result = newStmtList() + else: + unreachable() + + proc appendLenExpr(to: var NimNode, n: NimNode, start: int) = + for i in start.. Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 41/87] nanopass: use abstract cursors for tree traversal * introduce the `Cursor` API * change `matchImpl` to use "real" cursors for traversing the tree * change `transform` to use "real" cursors for traversing the tree Using abstract cursors makes it possible to statically swap how traversal works without having to modify `match` and `transform`, making the code more modular. --- nanopass/asts.nim | 39 ++++++++++++++++++++++++++ nanopass/npmatch.nim | 60 +++++++++++++++++++++------------------- nanopass/nppass.nim | 13 +++++---- nanopass/nptransform.nim | 29 ++++++++++--------- 4 files changed, 94 insertions(+), 47 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 7679e44e..fff28c2e 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -37,6 +37,9 @@ type start: NodeIndex len: uint32 + Cursor* = distinct NodeIndex + ## A cursor into a tree where without indirections. + const RefTag* = 128'u8 ## the node used internally for indirections @@ -73,3 +76,39 @@ proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: SomeInteger): T = proc len*(s: ChildSlice): int = int(s.len) proc high*(s: ChildSlice): int = int(s.len) - 1 + +# ----- internal cursor API ----- + +# the cursor interface consists of these routines: +# * ``advance(PackedTree[uint8], var Cursor)``: +# moves the cursor to the sibling of the current node +# * ``get(PackedTree[uint8], Cursor): NodeIndex``: +# returns the resolved index of the node the cursor points to +# * ``pos(Cursor): NodeIndex``: +# returns the unresolved index of the node the cursor points to +# * ``enter(PackedTree[uint8], var Cursor): Savepoint``: +# enters the subtree at the current cursor position +# * ``restore(PackedTree[uint8], var Cursor, Savepoint)``: +# exits the current subtree +# XXX: this should use static interfaces once supported by NimSkull + +{.push stacktrace: off, inline.} + +proc advance*(tree: PackedTree[uint8], cr: var Cursor) {.inline.} = + NodeIndex(cr) = next(tree, NodeIndex(cr)) + +proc get*(tree: PackedTree[uint8], cr: Cursor): NodeIndex {.inline.} = + NodeIndex cr + +template pos*(cr: Cursor): NodeIndex = + NodeIndex cr + +proc enter*(tree: PackedTree[uint8], cr: var Cursor): Cursor {.inline.} = + # nothing to step into and thus no cursor to save + result = cr + cr = Cursor(tree.child(NodeIndex(cr), 0)) + +template restore*(tree: PackedTree[uint8], cr: Cursor, saved: untyped) = + discard # nothing to restore + +{.pop.} diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 879e5e0f..efbe0c12 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -323,7 +323,9 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, ## an `nnkElse` tree used for handling the rest of match, or nil, in which ## case how to handle the rest of a match is dictated by `config`. let cursor = genSym("cursor") - var stack: seq[NimNode] ## stack of `len` symbols + let backup = genSym("backup") + + var stack: seq[tuple[len, saved: NimNode]] ## cursor context stack var top: NimNode ## the topmost non-union match expression enclosing the currently ## processed one @@ -336,31 +338,31 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, # commit the current cursor to a local with the given name if e[0].kind == nnkBracket: let bias = e[2] - let len = stack[^1] # can only be non-empty - to.add newLetStmt(e[1], quote do: (`cursor`, `len` - `bias`)) + let len = stack[^1].len # can only be non-empty + to.add newLetStmt(e[1], quote do: (pos(`cursor`), `len` - `bias`)) else: - to.add newLetStmt(e[1], cursor) + to.add newLetStmt(e[1], quote do: get(`ast`, `cursor`)) # only emit a cursor movement when the moved cursor is actually observed if e[^1].kind in {nnkCall, nnkCurly, nnkPar}: case e[0].kind of nnkPar: let nlen = genSym"len" + let saved = genSym"saved" to.add quote do: - let `nlen` = `ast`[`cursor`].val - to.add quote do: - `cursor` = `ast`.child(`cursor`, 0) + let `nlen` = `ast`[pos(`cursor`)].val + let `saved` = enter(`ast`, `cursor`) - stack.add nlen + stack.add (nlen, saved) of nnkEmpty, nnkIntLit: to.add quote do: - `cursor` = `ast`.next(`cursor`) + advance(`ast`, `cursor`) of nnkBracket: let bias = e[2] - let len = stack[^1] # can only be non-empty + let len = stack[^1].len # can only be non-empty to.add quote do: for _ in 0 ..< (`len` - `bias`): - `cursor` = `ast`.next(`cursor`) + advance(`ast`, `cursor`) else: unreachable(e[0].kind) @@ -372,8 +374,10 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, of nnkPar: # move the current cursor to the end of the subtree and pop it from # the stack - discard stack.pop() - result = aux(lang, e[1], to) + let (_, saved) = stack.pop() + to.add quote do: + restore(`ast`, `cursor`, `saved`) + result = aux(lang, e[0], to) of nnkCurly: # a dispatcher proc genOfBranch(lang: LangInfo, typ: LangType, used: var IntSet, @@ -394,7 +398,7 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, let stackLen = stack.len let typ = e[0].intVal.int - var caseStmt = nnkCaseStmt.newTree(quote do: `ast`[`cursor`].kind) + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`[pos(`cursor`)].kind) var used = initIntSet() for i in 1.. Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 42/87] nanopass: rework type transformers * prefer direct transformers over indirect ones * handle terminals correctly (morphability checks, re-tag when necessary) * improve source location information for transformer errors --- nanopass/nppass.nim | 25 ++++++------------- nanopass/nptransform.nim | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index f81d1129..d85dd0c2 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -38,29 +38,20 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, proc fillForm(lang: LangInfo, form: int, n, info: NimNode): NimNode = ## Generates a form transformer. let sym = bindSym"transform" - quote do: + result = quote do: (typeof(result))( index: `sym`(idef(src), idef(dst), typeof(result).N, `form`, `input`, `output`, `n`)) + copyLineInfo(result[1][1][^1], info) proc fillType(lang: LangInfo, typ: int, n, info: NimNode): NimNode = ## Generates a call to the type transformer for `typ`. - if lang.types[typ].terminal: - let id = lang.types[typ].ntag - quote do: - # TODO: use the tag of the destination language - `output`.nodes.add: - TreeNode[uint8](kind: uint8(`id`), val: `input`[pos(`n`)].val) - (typeof(result))(index: NodeIndex(`output`.nodes.high)) - else: - # TODO: consider inlining the transformer if it's auto-generated. - # More code to emit, but also a little less work at run-time - let name = ident(lang.types[typ].mvar) - let callee = ident"->" - # dispatch to the processor and convert to the expected type - quote do: - (typeof(result))( - index: `callee`(src.`name`(index: pos(`n`)), dst.`name`).index) + let sym = bindSym"transformType" + result = quote do: + (typeof(result))( + index: `sym`(idef(src), idef(dst), typeof(result).N, `typ`, + `input`, `output`, `n`)) + copyLineInfo(result[1][1][^1], info) let config = ExpandConfig( fillForm: fillForm, diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 14f6a08a..07389e9d 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -129,3 +129,56 @@ macro transform*(src, dst: static LangInfo, nterm: static string, result.add body # the callsite takes care of fitting the index to the right type result.add ident"root" + +macro transformType*(src, dst: static LangInfo, nterm: static string, + typ: static int, input, output: PackedTree[uint8], + cursor: untyped): untyped = + ## Transforms the instance of type `typ` (may be either a terminal or non- + ## terminal) at `cursor` to an AST fitting the destination non-terminal + ## `nterm` and appends the result to `output`. + proc contains(lang: LangInfo, typ: LangType, search: int): bool = + for it in typ.sub.items: + result = it == search + if not result and not lang.types[it].terminal: + result = contains(lang, typ, search) + if result: + break + + let dtyp = dst.map.getOrDefault(src.types[typ].name, -1) + if src.types[typ].terminal: + if dtyp == -1: + # target language doesn't have the terminal + result = makeError( + fmt"cannot transform '{src.types[typ].name}' to '{nterm}'", + cursor) + elif contains(dst, dst.types[dst.map[nterm]], dtyp): + if src.types[typ].ntag == dst.types[dtyp].ntag: + # copy the node as it is + result = quote do: + `output`.nodes.add `input`[pos(`cursor`)] + NodeIndex(`output`.nodes.high) + else: + # re-tag the node + let tag = dst.types[dtyp].ntag.uint8 + result = quote do: + `output`.nodes.add TreeNode[uint8]( + kind: `tag`, + val: `input`[get(`input`, `cursor`)].val + ) + NodeIndex(`output`.nodes.high) + else: + # target non-terminal doesn't include the terminal + result = makeError( + fmt"cannot transform terminal '{src.types[typ].name}' to '{nterm}'", + cursor) + else: + let smvar = ident(src.types[typ].mvar) + # prefer a direct processor (i.e. 'a -> a') over 'a -> b' + let dmvar = + if dtyp != -1 and contains(dst, dst.types[dst.map[nterm]], dtyp): + ident(dst.types[dtyp].mvar) + else: + ident(dst.types[dst.map[nterm]].mvar) + + result = quote do: + (src.`smvar`(index: get(`input`, `cursor`)) -> dst.`dmvar`).index From 04f911c49dde1409e194567334a65d5e5f121679 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 43/87] nanopass: rework pass signatures * passes producing an instance of a language now return an AST + non-terminal reference * passes taking an instance of a language as input now accept a non-terminal instead of a raw `NodeIndex` * passes accepting or producing instances of a language not starting at the language's entry point now work correctly --- nanopass/nppass.nim | 55 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index d85dd0c2..d06892d4 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -225,7 +225,7 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = let pos = `call` # turn the AST with indirections into one without `output`.tree = finish(`output`.tree, pos.index) - result = move `output` + result = (move `output`, typeof(pos)(index: NodeIndex(0))) else: body.add quote do: result = `call` @@ -233,10 +233,14 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = def.body = body # patch the signature: if hasIn: - def.params[1][^2] = bindSym"NodeIndex" - def.params.insert(1, newIdentDefs(input, quote do: Ast[`src`, `storageTy`])) + def.params.insert(1, + newIdentDefs(input, + nnkBracketExpr.newTree(ident"Ast", src, storageTy))) if hasOut: - def.params[0] = nnkBracketExpr.newTree(ident"Ast", dst, storageTy) + def.params[0] = + nnkTupleConstr.newTree( + nnkBracketExpr.newTree(ident"Ast", dst, storageTy), + def.params[0]) result = def @@ -282,15 +286,14 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = # entry processor if def.body[^1].kind == nnkProcDef: # ^^ a heuristic, but should work okay enough - def.body.add newCall(ident"->", def.params[1][0], - newDotExpr(newDotExpr(dst, ident"meta"), ident"entry")) + def.body.add newCall(ident"->", def.params[1][0], dstnterm) def.body = newStmtList(preamble, def.body) + def.params[0] = dstnterm + def.params[1][^2] = srcnterm let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) lambda.params = copyNimTree(def.params) - lambda.params[0] = dstnterm - lambda.params[1][^2] = srcnterm let call = newCall(lambda) # forward the original parameters to the lambda: @@ -298,14 +301,13 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = for j in 0..language pass, that is a @@ -366,11 +369,7 @@ macro pass*(p: untyped) = error("the input parameter is missing", p.params) result = genAst(input = p.params[1][^2], target, p): - when target is Metavar: - passImpl(input, target.L, input, target, p) - else: - # use the entry non-terminal - passImpl(input, target, input.meta.entry, target.meta.entry, p) + passImpl(lang(input), lang(target), nterm(input), nterm(target), p) macro outpass*(p: untyped) = ## Turns a procedure definition into a language->* pass, that is, a pass @@ -384,9 +383,5 @@ macro outpass*(p: untyped) = if p.params.len == 1: error("the input parameter is missing", p.params) - result = genAst(typ = p.params[1][^2], p): - when typ is Metavar: - outpassImpl(typ.L, typ, p) - else: - # use the entry non-terminal - outpassImpl(typ, typ.meta.entry, p) + result = genAst(input = p.params[1][^2], p): + outpassImpl(lang(input), nterm(input), p) From 6cfc336847104b59b720727b71d587e040f5f4fb Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 44/87] asts: make cursor logic swappable for `ChildSlice` --- nanopass/asts.nim | 31 ++++++++++++++++++++----------- nanopass/npmatch.nim | 2 +- nanopass/nppass.nim | 7 ++++++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index fff28c2e..aa816651 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -32,9 +32,9 @@ type index*: uint32 ## leaked implementation detail, don't use - ChildSlice*[T: Metavar or Value] = object + ChildSlice*[T: Metavar or Value, Cursor] = object ## A lightweight reference to a slice of contiguous children of a tree. - start: NodeIndex + start: Cursor len: uint32 Cursor* = distinct NodeIndex @@ -48,19 +48,24 @@ template isAtom*(x: uint8): bool = ## The predicate required for using an uint8 as a ``PackedTree`` tag. x >= RefTag -proc slice*[T](start: NodeIndex, len: uint32): ChildSlice[T] = - ChildSlice[T](start: start, len: len) +# ----- slice implementation ----- -iterator items*[T](t: PackedTree[uint8], s: ChildSlice[T]): T = +proc slice*[T, C](start: C, len: uint32): ChildSlice[T, C] = + ChildSlice[T, C](start: start, len: len) + +iterator items*[T, C](t: PackedTree[uint8], s: ChildSlice[T, C]): T = + mixin advance var c = s.start for _ in 0..= uint64(s.len): @@ -71,8 +76,12 @@ proc `[]`*[T](t: PackedTree[uint8], s: ChildSlice[T], i: SomeInteger): T = var n = s.start for _ in 0.. Date: Wed, 7 Jan 2026 00:08:54 +0000 Subject: [PATCH 45/87] asts: implement `IndCursor` --- nanopass/asts.nim | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index aa816651..acbeb952 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -39,6 +39,8 @@ type Cursor* = distinct NodeIndex ## A cursor into a tree where without indirections. + IndCursor* = distinct NodeIndex + ## A cursor into a tree with indirections. const RefTag* = 128'u8 @@ -120,4 +122,34 @@ proc enter*(tree: PackedTree[uint8], cr: var Cursor): Cursor {.inline.} = template restore*(tree: PackedTree[uint8], cr: Cursor, saved: untyped) = discard # nothing to restore +# implementation for a cursor into a tree with indirections follows + +proc advance*(tree: PackedTree[uint8], cr: var IndCursor) = + NodeIndex(cr) = next(tree, NodeIndex(cr)) + +proc get*(tree: PackedTree[uint8], cr: IndCursor): NodeIndex = + if tree[NodeIndex(cr)].kind == 128: + NodeIndex tree[NodeIndex(cr)].val + else: + NodeIndex cr + +template pos*(cr: IndCursor): NodeIndex = + NodeIndex cr + +type Savepoint = tuple[origin: IndCursor, stepped: bool] + +proc enter*(tree: PackedTree[uint8], cr: var IndCursor): Savepoint = + result = (cr, tree[NodeIndex(cr)].kind == 128) + if result.stepped: + cr = IndCursor tree[NodeIndex(cr)].val + else: + cr = IndCursor tree.child(NodeIndex(cr), 0) + +template restore*(tree: PackedTree[uint8], cr: var IndCursor, + saved: Savepoint) = + if saved.stepped: + cr = saved.origin + advance(tree, cr) + # else: the cursor is at the correct position already + {.pop.} From 9bd6aa23715d0178ca34a9b181a7e1120404f1ca Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 46/87] nanopass: implement output introspection In a pass that produces an instance of a language, it's now possible to `match` over output non-terminals. --- nanopass/nppass.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index f88ec036..637e3ce1 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -204,12 +204,31 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = unpack(`input`.storage[], v.index, typeof(T)) if hasOut: + let inj = ident"[]" let embed = bindSym"embed" body.add quote do: template terminal(x: untyped): untyped {.used.} = `embed`(`output`.storage, x) template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = build(`output`.tree, n, body) + template match[N](sel: Metavar[dst, N], branches: varargs[untyped]): untyped {.used.} = + match[dst, N](`output`.tree, IndCursor(sel.index), sel, branches) + template slice[N](T: typedesc[Metavar[dst, N]]): typedesc {.used.} = + ChildSlice[T, IndCursor] + + template `inj`(x: ChildSlice[auto, IndCursor], i: SomeInteger): untyped {.used.} = + `output`.tree[x, i] + + template foreach[T](s: ChildSlice[T, IndCursor], it, body: untyped) {.used.} = + for it in items(`output`.tree, s): + body + + template foreach[T](s: ChildSlice[T, IndCursor], idx, it, body: untyped) {.used.} = + var i = -1 + for it in items(`output`.tree, s): + inc i + let idx = i + body if hasIn: # shadow the input tree with a cursor to prevent a costly copy when From 807b3e37c3e12a13f4d78c45b025ea4c8ef354dd Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 47/87] asts: store pointer to tree in `ChildSlice` While this does make the API less safe (i.e., the reference outliving the tree will result in temporal memory errors), it allows accessing the sequence without access to the underlying tree, removing the need for the `[]` and `foreach` convenience templates. --- nanopass/asts.nim | 42 ++++++++++++++++++++++++++---------------- nanopass/npmatch.nim | 2 +- nanopass/nppass.nim | 19 ------------------- 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index acbeb952..1e89816e 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -33,7 +33,9 @@ type ## leaked implementation detail, don't use ChildSlice*[T: Metavar or Value, Cursor] = object - ## A lightweight reference to a slice of contiguous children of a tree. + ## A lightweight reference to a sequence of sibling nodes. The reference + ## must not outlive the spawned-from tree. + tree: ptr PackedTree[uint8] start: Cursor len: uint32 @@ -52,21 +54,32 @@ template isAtom*(x: uint8): bool = # ----- slice implementation ----- -proc slice*[T, C](start: C, len: uint32): ChildSlice[T, C] = - ChildSlice[T, C](start: start, len: len) +proc slice*[T, C](tree: ptr PackedTree[uint8], start: C, len: uint32 + ): ChildSlice[T, C] = + ## Creates a reference to `len` sibling nodes starting at `start`. + ChildSlice[T, C](tree: tree, start: start, len: len) -iterator items*[T, C](t: PackedTree[uint8], s: ChildSlice[T, C]): T = +template load[T, C](tree: PackedTree[uint8], c: C): T = + mixin get, pos + when T is Metavar: T(index: get(tree, c)) + else: T(index: tree[pos(c)].val) + +iterator items*[T, C](s: ChildSlice[T, C]): T = mixin advance var c = s.start for _ in 0.. Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 48/87] asts, nppass: implement structural equality comparison While possible, implementing structural equality comparison using nanopass pattern matching is both incredible inefficient (both compile- and run-time wise) as well as cumbersome. It being a built-in operation makes much more sense. --- nanopass/asts.nim | 64 +++++++++++++++++++++++++++++++++++++++++++++ nanopass/nppass.nim | 6 +++++ 2 files changed, 70 insertions(+) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 1e89816e..2d6306ae 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -163,3 +163,67 @@ template restore*(tree: PackedTree[uint8], cr: var IndCursor, # else: the cursor is at the correct position already {.pop.} + +# ------ additional tree operations -------- + +proc equal*(tree: PackedTree[uint8], a, b: Cursor): bool = + ## Compares the nodes/sub-trees at `a` and `b` for structural equality. + if pos(a) == pos(b): + return true + + var (a, b) = (a, b) + var i = 1'u32 + while i > 0: + let n = tree[pos(a)] + if n != tree[pos(b)]: + return false + elif not isAtom(n.kind): + i += n.val + advance(tree, a) + advance(tree, b) + dec i + + result = true + +proc equal*(tree: PackedTree[uint8], a, b: IndCursor): bool = + ## Compares the nodes/sub-trees at `a` and `b` for structural equality. + if pos(a) == pos(b): + return true + + # needs a stack for bookkeeping + var (a, b) = (a, b) + var stack: seq[(IndCursor, IndCursor, uint32)] + var i = 1'u32 + while true: + let na = tree[pos(a)] + let nb = tree[pos(b)] + if na != nb: + if na.kind == RefTag: + stack.add (a, b, i) + i = 1 + a = IndCursor(na.val) + continue + elif nb.kind == RefTag: + stack.add (a, b, i) + i = 1 + b = IndCursor(nb.val) + continue + return false + elif na.kind == RefTag: + stack.add (a, b, i) + i = 1 + a = IndCursor(na.val) + b = IndCursor(nb.val) + continue + elif not isAtom(na.kind): + i += na.val + dec i + if i == 0: + if stack.len == 0: + break + (a, b, i) = stack.pop() + else: + advance(tree, a) + advance(tree, b) + + result = true diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index f0dee326..c479398d 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -199,6 +199,9 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = # XXX: consider renaming this template to `get` unpack(`input`.storage[], v.index, typeof(T)) + template equal[N](a, b: Metavar[src, N]): bool {.used.} = + equal(`input`.tree, Cursor(a.index), Cursor(b.index)) + if hasOut: let embed = bindSym"embed" body.add quote do: @@ -211,6 +214,9 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template slice[N](T: typedesc[Metavar[dst, N]]): typedesc {.used.} = ChildSlice[T, IndCursor] + template equal[N](a, b: Metavar[dst, N]): bool {.used.} = + equal(`output`.tree, IndCursor(a.index), IndCursor(b.index)) + if hasIn: # shadow the input tree with a cursor to prevent a costly copy when # it's captured by the closure From 0b55735e2d7c490c2034b88548ea83b43f286841 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 49/87] npparser: implement an AST parser generator --- nanopass/nanopass.nim | 4 +- nanopass/npparser.nim | 277 ++++++++++++++++++++++++++++++++++++++++++ nanopass/npsexpr.nim | 2 - 3 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 nanopass/npparser.nim diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index f35343f1..4f79986c 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -9,11 +9,11 @@ import passes/[trees], - nanopass/[asts, nplanggen, npmatch, npbuild, nppass, nppatterns, npsexpr] + nanopass/[asts, nplanggen, npmatch, npbuild, npparser, nppass, nppatterns, npsexpr] export asts export nppatterns.matches -export npbuild.build, npmatch.match, npsexpr.renderer +export npbuild.build, npmatch.match, npsexpr.renderer, npparser.parser export nppass.pass, nppass.inpass, nppass.outpass export nppass.genProcessor diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim new file mode 100644 index 00000000..3edf0948 --- /dev/null +++ b/nanopass/npparser.nim @@ -0,0 +1,277 @@ +## Implements the `parser <#parser.>`_ macro, for generating an AST parser for +## a language. + +import + std/[genasts, macros, tables], + experimental/[sexp_parse], + nanopass/[asts, nplang, helper] + +import experimental/sexp {.all.} # we need access to the internal parser + +# the core parser logic for a language is implemented in generic routines, +# which themselves call internal macros; the external macro then only expands +# to code calling said generic routines. The benefit: most of the logic for +# parsing an AST is only generated once per language, even when more than one +# parser is generated for a language. This is somewhat problematic for symbol +# binding (for the terminal parsers), however, given how generics work + +proc raiseError(line, col: int, msg: string) {.noreturn.} = + raise ValueError.newException("(" & $line & ", " & $col & ") " & msg) + +macro genTerminalParser(lang: static LangInfo) = + result = newStmtList() + # emit the terminal handlers + for it in lang.types.items: + if it.terminal: + let typ = ident(it.name) + let tag = it.ntag.uint8 + result.add quote do: + block: + let val = tryParse(node, `typ`) + if val.isSome: + return TreeNode[uint8](kind: `tag`, val: pack(lit, val.unsafeGet)) + +proc parseTerminal[L, S](lit: var S, node: SexpNode, line, col: int): TreeNode[uint8] = + ## Implements fallback parsing of terminals. + mixin idef + genTerminalParser(idef(L)) + raiseError(line, col, + "'" & $node & "' is neither a valid language form nor terminal") + +proc parse[L, S](p: var SexpParser, to: var Ast[L, S]) + +proc rawParseForm[L, S](p: var SexpParser, to: var Ast[L, S]) = + ## Parses and appends the elements for a form, without performing any + ## grammar checks. + let start = to.tree.nodes.len + to.tree.nodes.add TreeNode[uint8]() # sub-tree node + var len = 0 + while p.currToken != tkParensRi: + parse(p, to) + space(p) + inc len + + discard getTok(p) # eat the parens + to.tree.nodes[start].val = uint32(len) + # the tag is computed separately + +macro parseFormImpl(lang: static LangInfo) = + ## Expands to the form parser for `lang`. + + # to keep the implementation simple, subtrees are parsed without regard + # to grammar at first. Once a subtree is fully parsed, the grammar check + # takes place and - if the check succeeds - a form tag is assigned to + # the node + + proc mergeInto(m, into: NimNode): NimNode = + ## Merges match expression `m` into `into`. + proc mustTrail(n: NimNode): bool = + n[0].kind == nnkEmpty or n[1].intVal <= 0 + + case into.kind + of nnkCurly: + for i, it in into.pairs: + if it[0] == m[0]: # same head? + into[i] = mergeInto(m, into[i]) + return into + elif mustTrail(it) and not mustTrail(m): + into.insert i, m + return into + + into.add m + into + of nnkCall: + if m[0] == into[0]: + # same head + into[^1] = mergeInto(m[^1], into[^1]) + into + elif mustTrail(into): + # an 'else' always comes last + nnkCurly.newTree(m, into) + else: + nnkCurly.newTree(into, m) + else: + unreachable() + + proc translate(lang: LangInfo, m: NimNode): NimNode = + ## Translates the match expression from the mini-language to NimSkull code. + case m.kind + of nnkIntLit: + result = quote do: + to.tree.nodes[start].kind = `m` + of nnkCurly: + result = nnkIfStmt.newTree() + for it in m.items: + let sub = translate(lang, it) + if sub.kind == nnkIfStmt: + result.add sub[0] + else: + result.add nnkElse.newTree(sub) + of nnkCall: + let head = m[0] + let raiseErr = bindSym"raiseError" + let next = translate(lang, m[^1]) + if head.kind == nnkEmpty: + # matches at the end of the sub-tree + result = quote do: + if cursor.int == to.tree.nodes.len: + `next` + else: + `raiseErr`(line, col, "end of sub-tree expected, but got more nodes") + else: + let tags = nnkCurly.newTree() + let name = lang.types[head.intVal].name + if lang.types[head.intVal].terminal: + tags.add newLit(uint8(lang.types[head.intVal].ntag)) + else: + for it in ntags(lang, lang.types[head.intVal]): + tags.add newLit(uint8(it)) + + if m[1].intVal == 1: # single item? + result = quote do: + if cursor.int < to.tree.nodes.len and + to.tree[cursor].kind in `tags`: + cursor = to.tree.next(cursor) + `next` + else: + `raiseErr`(line, col, "expected production of '" & `name` & "'") + else: + # a list; may be comprised of zero or more items + let bias = -m[1].intVal + result = quote do: + for i in uint32(`bias`).. Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 50/87] npsexpr: rework & use the "unparser" terminology "unparser" makes it clearer that it's the inverse of a parser. In addition, the implementation is reworked such that internally, most generated code is reused across unparsers generated for the same language. --- nanopass/nanopass.nim | 4 +- nanopass/npsexpr.nim | 117 ------------------------------------ nanopass/npunparser.nim | 127 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 119 deletions(-) delete mode 100644 nanopass/npsexpr.nim create mode 100644 nanopass/npunparser.nim diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 4f79986c..742cf93c 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -9,11 +9,11 @@ import passes/[trees], - nanopass/[asts, nplanggen, npmatch, npbuild, npparser, nppass, nppatterns, npsexpr] + nanopass/[asts, nplanggen, npmatch, npbuild, npparser, nppass, nppatterns, npunparser] export asts export nppatterns.matches -export npbuild.build, npmatch.match, npsexpr.renderer, npparser.parser +export npbuild.build, npmatch.match, npunparser.unparser, npparser.parser export nppass.pass, nppass.inpass, nppass.outpass export nppass.genProcessor diff --git a/nanopass/npsexpr.nim b/nanopass/npsexpr.nim deleted file mode 100644 index 277c923b..00000000 --- a/nanopass/npsexpr.nim +++ /dev/null @@ -1,117 +0,0 @@ -## Implements the macros for converting between ASTs and S-expressions. - -# TODO: instead of assembling a S-expression directly, the renderer should -# emit a stream of S-expression tokens (ideally as an iterator, but's -# that not really possible today, at least in an efficient manner) - -import std/[macros, tables] -import nanopass/[nplang, helper] -import passes/[trees] - -const - Inp = ident"tree" ## name of the input AST parameter - Pos = ident"pos" ## name of the position parameter - -macro genRenderer(def: static LangInfo, nterm: static string) = - ## Generates the rendering logic for the non-terminal with name `nterm`. - let id = def.map[nterm] - var caseStmt = nnkCaseStmt.newTree(quote do: `Inp`.tree.nodes[`Pos`].kind) - - proc genForType(def: LangInfo, typ: LangType): NimNode = - case typ.terminal - of true: - let name = ident(typ.name) - quote do: - inc `Pos` - toSexp(unpack(`Inp`.storage[], `Inp`.tree.nodes[`Pos` - 1].val, `name`)) - of false: - let name = typ.name - quote do: - render[`name`](`Inp`, `Pos`) - - # form renderers: - for it in def.types[id].forms.items: - var body = nnkStmtList.newTree() - let name = def.forms[it].name - body.add quote do: - let len {.used.} = `Inp`.tree.nodes[`Pos`].val.int - inc `Pos` - result = newSList([newSSymbol(`name`)]) - for i, e in def.forms[it].elems.pairs: - let inner = genForType(def, def.types[e.typ]) - if e.repeat: - let start = def.forms[it].elems.len - 1 - body.add quote do: - for _ in `start`.. Date: Wed, 7 Jan 2026 00:08:55 +0000 Subject: [PATCH 51/87] passes: move `defineCompiler` to dedicated module --- passes/compilerdef.nim | 30 ++++++++++++++++++++++++++++++ passes/passes.nim | 28 ++-------------------------- 2 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 passes/compilerdef.nim diff --git a/passes/compilerdef.nim b/passes/compilerdef.nim new file mode 100644 index 00000000..c801f56a --- /dev/null +++ b/passes/compilerdef.nim @@ -0,0 +1,30 @@ +## Implements the `defineCompiler <#defineCompiler,untyped,varargs[untyped]>`_ +## macro, for defining nanopass compilers, which are compositions of passes. + +import std/macros + +macro defineCompiler*(name, start, names: untyped) = + ## Generates a compiler procedures, running the provided passes in + ## sequence. Interim implementation. + var prevAst = ident"ast" + var prevPos = ident"it" + let body = newStmtList() + for it in names.items: + let tmp = genSym() + if it.kind == nnkIdent: + let name = it.strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, quote do: `it`(`prevAst`, `prevPos`)) + else: + let g = it + g.insert 1, prevPos + g.insert 1, prevAst + let name = it[0].strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, g) + prevAst = quote do: `tmp`[0] + prevPos = quote do: `tmp`[1] + result = quote do: + proc `name`(ast: Ast[`start`, Literals], it: `start`.meta.entry): auto = + `body` + result = (`prevAst`, `prevPos`) diff --git a/passes/passes.nim b/passes/passes.nim index 3b5f8264..3b3f46c8 100644 --- a/passes/passes.nim +++ b/passes/passes.nim @@ -13,7 +13,7 @@ import sexp ], nanopass/nanopass, - passes/literals, + passes/[literals, compilerdef], phy/[reporting, default_reporting] import passes/trees except Literals @@ -938,31 +938,7 @@ proc genvm(x: L12): VmModule {.outpass.} = module(x, result) ]# -import std/macros - -macro defineCompiler(name, names: untyped) = - ## Generates a compiler procedures, running the provided passes in - ## sequence. Interim implementation. - var prev = ident"e" - let body = newStmtList() - for it in names.items: - let tmp = genSym() - if it.kind == nnkIdent: - let name = it.strVal - body.add quote do: echo "-- ", `name` - body.add newLetStmt(tmp, quote do: `it`(`prev`, NodeIndex(0))) - else: - let g = it - g.insert 1, quote do: NodeIndex(0) - g.insert 1, prev - let name = it[0].strVal - body.add quote do: echo "-- ", `name` - body.add newLetStmt(tmp, g) - prev = tmp - result = quote do: - proc `name`(e: Ast[Lsrc, Literals]): auto = - `body` - result = `prev` +# proc render(x: L4.m): SexpNode {.renderer.} # some logic for testing: var rep = initDefaultReporter[string]() From 8dc5e920b2e99e0f587e38566259cddd3f99aad5 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Wed, 7 Jan 2026 00:08:56 +0000 Subject: [PATCH 52/87] implement the legacy passes using the framework This is intended solely for performance analysis and comparing with the manual implementation, which is why the passes stay as close as possible (in structure as well as spirit) to the original ones. --- passes/passes_legacy.nim | 1365 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1365 insertions(+) create mode 100644 passes/passes_legacy.nim diff --git a/passes/passes_legacy.nim b/passes/passes_legacy.nim new file mode 100644 index 00000000..fd009062 --- /dev/null +++ b/passes/passes_legacy.nim @@ -0,0 +1,1365 @@ + +import nanopass/nanopass +import passes/literals +import std/[tables, intsets, options, algorithm] + +type + Local = distinct uint32 + Global = distinct uint32 + Proc = distinct uint32 + Type = distinct uint32 + +defineLanguage Lskully: + ## Language for the code output by skully. + int64(i) + float64(fl) + string(str) + Local(lo) + Global(g) + Proc(pr) # proc is a reserved word + Type(tid) + + field(f) ::= Field(i, t) + typ(t) ::= tid | Int(i) | UInt(i) | Float(i) | Ptr() | Void() | + Record(i, i, f, ...f) | Union(i, i, t, ...t) | Array(i, i, i, t) | + ProcTy(t, ...t) + + lvalue(lv) ::= lo | g | Deref(t, e) | Field(lv, i) | At(lv, e) + expr(e) ::= i | fl | ProcVal(i) | + Call(t, e, ...e) | Call(pr, ...e) | + Conv(t, t, e) | Reinterp(t, t, e) | Trunc(t, t, e) | + Zext(t, t, e) | Sext(t, t, e) | + Demote(t, t, e) | Promote(t, t, e) | + Nil() | + Load(t, e) | Copy(lv) | Addr(lv) | + Add(t, e, e) | Sub(t, e, e) | Mul(t, e, e) | Div(t, e, e) | Mod(t, e, e) | + BitNot(t, e) | BitAnd(t, e, e) | BitOr(t, e, e) | BitXor(t, e, e) | + AddChck(t, e, e, lo) | SubChck(t, e, e, lo) | MulChck(t, e, e, lo) | + Shl(t, e, e) | Shr(t, e, e) | + Neg(t, e) | + Not(e) | + Eq(t, e, e) | Le(t, e, e) | Lt(t, e, e) + + goto(go) ::= Goto(i) + + target(tgt) ::= Goto(i) | Unwind() + stmt(st) ::= Stmts(st, ...st) | Goto(i) | Join(i) | + CheckedCall(t, e, ...e, tgt) | + CheckedCall(pr, ...e, tgt) | + CheckedCallAsgn(lo, t, e, ...e, tgt) | + CheckedCallAsgn(lo, pr, ...e, tgt) | + Call(pr, ...e) | Call(t, e, ...e) | + Return() | Return(e) | + Raise(e, tgt) | + Branch(e, go, go) | + Blit(e, e, e) | Clear(e, e) | + Store(t, e, e) | Asgn(lv, e) | Except(i, lo) | + Unreachable() | + Loop(i) | + Drop(e) + + params(p) ::= Params(...lo) + locals(locs) ::= Locals(...t) + + init(ini) ::= Data(t, str) | Data(t) + + # the v + decl(d) ::= Import(t, str) | + ProcDef(t, p, locs, st) | + GlobalDef(t, i) | GlobalDef(t, fl) | + GlobalLoc(t, ini) + exprt(exp) ::= Export(str, g) | Export(str, pr) # export is a reserved word + + # + tdecls(td) ::= TypeDefs(...t) + gdecls(gd) ::= GlobalDefs(...d) + pdecls(pd) ::= ProcDefs(...d) + exports(es) ::= List(...exp) + + module(m) ::= Module(td, gd, pd, es) + +defineLanguage L6, Lskully: + ## Language with basic blocks. + bblock(bb) ::= +Block(p, ...st, ex) | +Except(p, ...st, ex) + stmt(st) ::= -Return() | + -Return(e) | + -Goto(i) | + -Raise(e, tgt) | + -Loop(i) | + -Unreachable() | + -Join(i) | + -Except(i, lo) | + -CheckedCall(t, e, ...e, tgt) | + -CheckedCall(pr, ...e, tgt) | + -CheckedCallAsgn(lo, t, e, ...e, tgt) | + -CheckedCallAsgn(lo, pr, ...e, tgt) | + -Branch(e, go, go) | + -Stmts(st, ...st) + exit(ex) ::= +Return() | +Return(e) | +Goto(i) | +Raise(e, tgt) | +Branch(e, go, go) | + +CheckedCall(t, e, ...e, go, tgt) | + +CheckedCall(pr, ...e, go, tgt) | + +CheckedCallAsgn(lo, t, e, ...e, go, tgt) | + +CheckedCallAsgn(lo, pr, ...e, go, tgt) | + +Unreachable() | + +Loop(i) + blocks(bl) ::= +List(bb, ...bb) + decl(d) ::= -ProcDef(t, p, locs, st) | +ProcDef(t, locs, bl) + +defineLanguage L5, L6: + ## Language without mutable global locations. + init(ini) ::= -Data(t) | -Data(t, str) | +Data(i, i) | +Data(i, str) + decl(d) ::= -GlobalLoc(t, ini) | +GlobalDef(t, ini) + lvalue(lv) ::= -g + expr(e) ::= +Copy(g) + +defineLanguage L4, L5: + ## Language with combined, flat paths. + root(ro) ::= +lo | +Deref(t, e) + lvalue(lv) ::= -Field(lv, i) | -At(lv, e) | -Deref(t, e) | + +Path(t, ro, e, ...e) + +defineLanguage L3s2, L4: + ## Language without aggregate parameters. + +defineLanguage L3s1, L3s2: + ## Language without aggregate types. + expr(e) ::= +Offset(e, e, e) + typ(t) ::= +Blob(i, i) | + -Record(i, i, f, ...f) | + -Array(i, i, i, t) | + -Union(i, i, t, ...t) + lvalue(lv) ::= -Path(t, ro, e, ...e) + +defineLanguage L3, L3s1: + ## Language without primitive locals that can have their address taken. + +defineLanguage L2, L3: + ## Language without blob assignments. + +defineLanguage L1, L2: + ## Language with explicit stack management and only primitive types. + typ(t) ::= -Blob(i, i) + expr(e) ::= -Addr(lv) + decl(d) ::= -ProcDef(t, locs, bl) | + +ProcDef(t, i, locs, bl) + +defineLanguage LPtr, L1: + ## Language where only proc types are identified. + +defineLanguage L0, LPtr: + ## Language without pointer types and related operations. + typ(t) ::= -Ptr() + expr(e) ::= -Nil() | -Offset(e, e, e) + +# terminal adapter routines: +template defineTerminal(typ: typedesc) {.dirty.} = + template unpack(lit: Literals, id: uint32, _: typedesc[typ]): typ = + typ(id) + template pack(lit: var Literals, val: typ): uint32 = + uint32(val) + +defineTerminal(Local) +defineTerminal(Global) +defineTerminal(Proc) +defineTerminal(Type) + +template map[T, U](x: ChildSlice[T, auto], p: proc(x: T): U): seq[U] = + let cs = x + var s = newSeq[U](cs.len) + for i, it in x.pairs: + s[i] = p(it) + s + +template mapIt[T](x: ChildSlice[T, auto], body: untyped): untyped = + let cs = x + var s = newSeq[ + typeof( + block: + var it {.inject.}: T + body + it) + ](cs.len) + for i, it {.inject.} in x.pairs: + s[i] = body + s + +proc basicBlocks(ir: Lskully): L6 {.pass.} = + ## Transforms the bodies of procedures into a basic-block-oriented structure. + + proc target(x: src.tgt, map: Table[int64, int]): dst.tgt = + match x: + of Goto(i): build dst.tgt, Goto(i(^map[i.val])) + of Unwind(): build dst.tgt, Unwind() + + proc goto(x: src.go, map: Table[int64, int]): dst.go = + match x: + of Goto(i): build dst.go, Goto(i(^map[i.val])) + + type BBlock = object + isExcept: bool + params: dst.p + stmts: seq[dst.st] + + proc blocks(x: src.st, bbs: var seq[dst.bb], bb: var BBlock, map: Table[int64, int]) = + proc commitBlock(bbs: var seq[dst.bb], bb: var BBlock, ex: dst.ex) = + if bb.isExcept: + bbs.add build(dst.bb, Except(^bb.params, ...bb.stmts, ex)) + else: + bbs.add build(dst.bb, Block(^bb.params, ...bb.stmts, ex)) + bb.stmts.shrink(0) + + proc startBlock(bb: var BBlock) = + bb.isExcept = false + bb.params = build(dst.p, Params([])) + + match x: + of Stmts(...st): + for it in st.items: + blocks(it, bbs, bb, map) + of Goto(i): + commitBlock bbs, bb, build(dst.ex, Goto(i(^map[i.val]))) + of Join(i): + # don't merge an empty entry block with the following block; the latter + # might be a loop start + if map[i.val] > bbs.len: + commitBlock bbs, bb, build(dst.ex, Goto(i(^map[i.val]))) + startBlock(bb) + of Except(_, lo): + assert bb.stmts.len == 0, "control flow falls through into exception handler" + bb.isExcept = true + bb.params = build(dst.p, Params([lo])) + of Asgn([lv], [e]): + bb.stmts.add build(dst.st, Asgn(lv, e)) + of Store([t], [e0], [e1]): + bb.stmts.add build(dst.st, Store(t, e0, e1)) + of Blit([e0], [e1], [e2]): + bb.stmts.add build(dst.st, Blit(e0, e1, e2)) + of Clear([e0], [e1]): + bb.stmts.add build(dst.st, Clear(e0, e1)) + of Call(pr, ...[e]): + bb.stmts.add build(dst.st, Call(pr, ...e)) + of Call([t], [e0], ...[e1]): + bb.stmts.add build(dst.st, Call(t, e0, ...e1)) + of Drop([e]): + bb.stmts.add build(dst.st, Drop(e)) + of CheckedCall([t], [e0], ...[e1], tgt): + commitBlock bbs, bb, build(dst.ex, CheckedCall(t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCall(pr, ...[e], tgt): + commitBlock bbs, bb, build(dst.ex, CheckedCall(pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCallAsgn(lo, [t], [e0], ...[e1], tgt): + commitBlock bbs, bb, build(dst.ex, CheckedCallAsgn(lo, t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCallAsgn(lo, pr, ...[e], tgt): + commitBlock bbs, bb, build(dst.ex, CheckedCallAsgn(lo, pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of Return(): + commitBlock bbs, bb, build(dst.ex, Return()) + of Return([e]): + commitBlock bbs, bb, build(dst.ex, Return(e)) + of Raise([e], tgt): + commitBlock bbs, bb, build(dst.ex, Raise(e, ^target(tgt, map))) + of Branch([e], go0, go1): + commitBlock bbs, bb, build(dst.ex, Branch(e, ^goto(go0, map), ^goto(go1, map))) + of Unreachable(): + commitBlock bbs, bb, build(dst.ex, Unreachable()) + of Loop(i): + commitBlock bbs, bb, build(dst.ex, Loop(i(^map[i.val]))) + + proc scanStmt(ir: src.st, map: var Table[int64, int], wasJoin: var bool, + next: var int) = + match ir: + of Stmts(...st): + for it in st.items: + scanStmt(it, map, wasJoin, next) + of Join(i): + if wasJoin: + # merge joins immediately following each other + map[i.val] = next - 1 + else: + map[i.val] = next + inc next + wasJoin = false + of Except(i, _): + map[i.val] = next + inc next + wasJoin = false + of CheckedCall(...any): + inc next + wasJoin = true + of CheckedCallAsgn(...any): + inc next + wasJoin = true + else: + wasJoin = false + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], [p], [locs], st): + var map: Table[int64, int] + var wasJoin = false + var next = 1 + scanStmt(st, map, wasJoin, next) + var bbs: seq[dst.bb] + var bb = BBlock(isExcept: false, params: p) + blocks(st, bbs, bb, map) + build ProcDef(t, locs, List(...bbs)) + +proc globalsToPointer(ir: L6, ptrsize: int): L5 {.pass.} = + ## Turns global locations into pointer globals. Read access is turned + ## into loads, write access into stores, and taking the address into copies + ## of the pointer global. + let (types, globals) = (; + match ir: + of Module(TypeDefs(...t), GlobalDefs(...d), ...any): + var list = newSeq[src.t](d.len) + for i, it in d.pairs: + match it: + of GlobalLoc(t, _): list[i] = t + of GlobalDef(t, _): list[i] = t + else: unreachable() + (t, list)) + + proc gtype(g: Global): dst.t = + globals[ord g] -> dst.t + + proc sizeAndAlignment(x: L6.t): tuple[size, align: Value[int64]] = + match x: + of tid: + sizeAndAlignment(types[ord tid.val]) + of Union(i1, i2, ...any): (i1, i2) + of Record(i1, i2, ...any): (i1, i2) + of Array(i1, i2, ...any): (i1, i2) + of Int(i): (i, i) + of UInt(i): (i, i) + of Float(i): (i, i) + of Ptr(): (terminal int64(ptrsize), terminal int64(ptrsize)) + else: + unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(g): + build Load(^gtype(g.val), Copy(g)) + of Addr(g): + # the address of the location is now stored in the global + build Copy(g) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of g: + build Deref(^gtype(g.val), Copy(g)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(g, [e]): + build Store(^gtype(g.val), Copy(g), e) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of GlobalLoc(t, Data(_)): + let (size, align) = sizeAndAlignment(t) + build GlobalDef(Ptr(), Data(align, size)) + of GlobalLoc(t, Data(_, str)): + let (_, align) = sizeAndAlignment(t) + build GlobalDef(Ptr(), Data(align, str)) + +proc flattenPaths(ir: L5): L4 {.pass.} = + ## Turns `At` and `Field` expressions into flat `Path` expressions, for easier + ## processing by later passes. + var locals: slice(src.t) + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + + proc arrayElem(x: src.t): src.t = + match x: + of tid: + arrayElem(types[ord tid.val]) + of Array(_, _, _, t): + t + else: + unreachable() + + proc elemAt(x: src.t, i: int64): src.t = + match x: + of tid: + elemAt(types[ord tid.val], i) + of Record(_, _, ...f): + match f[i]: + of Field(_, t): t + of Union(_, _, ...t): + t[i] + else: + unreachable() + + proc filter(x: src.lv, args: var seq[dst.e]): (dst.ro, src.t) = + match x: + of Deref(t, [e]): + (build(dst.ro, Deref(^(t -> dst.t), e)), t) + of Field(lv, i): + let (root, t) = filter(lv, args) + args.add build(dst.e, i) + (root, elemAt(t, i.val)) + of At(lv, [e]): + let (root, t) = filter(lv, args) + args.add e + (root, arrayElem(t)) + of lo: + (build(dst.ro, lo), locals[ord lo.val]) + + proc typ(x: src.t): dst.t {.generated.} + proc bblock(x: src.bb): dst.bb {.generated.} + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of lo: build lo + of _: + var args: seq[dst.e] + let (root, t) = filter(x, args) + assert args.len > 0 + build Path(^typ(t), root, ...args) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc aggregateParams(ir: L4): L3s2 {.pass.} = + ## Turns all aggregate parameters into pointer parameters and replaces returns + ## of aggregate with out parameters. + + # The pass' implementation is fairly involved. Only lvalue expressions can have + # their address taken, meaning that rvalue argument expressions of aggregate + # type have to be assigned to a temporary first, which then has its address + # passed to the procedure. + # + # This leads to problem: an expression can have side effects, and reordering + # it with other (impure) expressions might alter the program's meaning! + # + # Therefore, all (impure) expressions happening before an expression that is + # turned into an assignment also have to be committed to temporaries. To + # implement this, the operands of all operations are iterated from + # *right to left*. + proc resolve(types: slice(src.t), t: src.t): src.t = + match t: + of tid: types[ord tid.val] + else: t + + let (types, signatures) = (; + match ir: + of Module(TypeDefs(...t), _, ProcDefs(...d), ...any): + var sigs = newSeq[src.t](d.len) + for i, it in d.pairs: + # the types for procedures are looked up, resolved, and cached, which + # greatly speeds up later type lookup + match it: + of ProcDef(t1, ...any): sigs[i] = resolve(t, t1) + of Import(t1, _): sigs[i] = resolve(t, t1) + else: unreachable() + (t, sigs)) + + var origLocals: slice(src.t) + var locals: seq[dst.t] + var params: PackedSet[Local] + var stmts: seq[dst.st] ## accumulator for statements + var needsSave: bool + + proc isAggregate(x: src.t): bool = + match x: + of tid: isAggregate(types[ord tid.val]) + of Record(...any): true + of Array(...any): true + of Union(...any): true + else: false + + proc retType(x: src.t): src.t = + match x: + of tid: retType(types[ord tid.val]) + of ProcTy(t, ...any): t + else: unreachable() + + proc paramAt(x: src.t, pos: int): src.t = + match x: + of tid: + paramAt(types[ord tid.val], pos) + of ProcTy(_, ...t): + t[pos] + else: unreachable() + + proc newTemp(t: dst.t): dst.lo = + result = terminal(Local(locals.len)) + locals.add(t) + + proc typ(x: src.t): dst.t {.transform.} + proc expr(x: src.e): dst.e {.transform.} + + proc getType(x: src.e): dst.t = + match x: + of Le(_, _, _): build dst.t, UInt(i(1)) + of Lt(_, _, _): build dst.t, UInt(i(1)) + of Eq(_, _, _): build dst.t, UInt(i(1)) + of Not(_): build dst.t, UInt(i(1)) + of Addr(_): build dst.t, Ptr() + of Nil(): build dst.t, Ptr() + of Copy(g): discard g; build dst.t, Ptr() + of Copy(Path([t], ...any)): t + of Copy(lo): locals[ord lo.val] + of Call(pr, ...any): typ(retType(signatures[ord pr.val])) + of Call(t, ...any): typ(retType(t)) + of ProcVal(_): build dst.t, Ptr() + of i: discard i; unreachable() + of fl: discard fl; unreachable() + of e: + match e: + of any(t, ...any): typ(t) + else: unreachable() + + proc operand(x: src.e): dst.e = + if needsSave: + match x: + of Addr([lv]): build dst.e, Addr(lv) + of Nil(): build dst.e, Nil() + of i: build dst.e, i + of fl: build dst.e, fl + else: + let tmp = newTemp(getType(x)) + needsSave = false + stmts.add build(dst.st, Asgn(tmp, ^expr(x))) + needsSave = true + build dst.e, Copy(tmp) + else: + expr(x) + + proc operands(x: slice(src.e)): seq[dst.e] = + # process in reverse + result.newSeq(x.len) + result[^1] = expr(x[x.high]) + for i in countdown(x.high - 1, 0): + result[i] = operand(x[i]) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of Path([t], ro, ...e): + let e1 = operands(e) + build Path(t, ^root(ro), ...e1) + + proc root(x: src.ro): dst.ro {.transform.} = + case x + of Deref([t], e): + build Deref(t, ^operand(e)) + of lo: + if lo.val in params: + build Deref(^locals[ord lo.val], Copy(lo)) + else: + build lo + + proc args(sig: src.t, e: slice(src.e)): seq[dst.e] = + result = newSeq[dst.e](e.len) + # process in reverse + for i in countdown(e.high, 0): + let it = e[i] + let pt = paramAt(sig, i) + if isAggregate(pt): + let pt = typ(pt) + let tmp = newTemp(pt) + needsSave = false + # ^^ the hoisted expression isn't affected by side effects + match it: + of Call(t, ...e1): + stmts.add build(dst.st, Asgn(tmp, Call(^typ(t), ...args(t, e1)))) + of Call(pr, ...e1): + stmts.add build(dst.st, Asgn(tmp, Call(pr, ...args(signatures[ord pr.val], e1)))) + else: + stmts.add build(dst.st, Asgn(tmp, ^expr(it))) + needsSave = true + result[i] = build(dst.e, Addr(tmp)) + elif needsSave: + let pt = typ(pt) + let tmp = newTemp(pt) + needsSave = false + # ^^ the hoisted expression isn't affected by side effects + stmts.add build(dst.st, Asgn(tmp, ^expr(it))) + needsSave = true + result[i] = build(dst.e, Copy(tmp)) + else: + result[i] = expr(it) + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Addr(lo): + if lo.val in params: + build Copy(lo) + else: + build Addr(lo) + of Call(pr, ...e): + let t = signatures[ord pr.val] + let s = args(t, e) + + let rt = retType(t) + if isAggregate(rt): + let rt = typ(rt) + let tmp = newTemp(rt) + stmts.add build(dst.st, Call(pr, [...s, Addr(tmp)])) + needsSave = true + build Copy(tmp) + else: + build Call(pr, ...s) + of Call(t, e0, ...e1): + # note: e0 needs to be translated last, to keep the reverse processing order + let s = args(t, e1) + let rt = retType(t) + if isAggregate(t): + let rt = typ(rt) + let tmp = newTemp(rt) + stmts.add build(dst.st, Call(^typ(t), ^operand(e0), [...s, Addr(tmp)])) + needsSave = true + build Copy(tmp) + else: + build Call(^typ(t), ^operand(e0), ...s) + of Copy(lo): + if lo.val in params: + build Load(^locals[ord lo.val], Copy(lo)) + else: + build Copy(lo) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Call(t, e0, ...e1): + let args = args(t, e1) + build Call(^typ(t), ^operand(e0), ...args) + of Call(pr, ...e): + build Call(pr, ...args(signatures[ord pr.val], e)) + of Blit(e0, e1, e2): + let e2_1 = expr(e2) + let e1_1 = operand(e1) + let e0_1 = operand(e0) + build Blit(e0_1, e1_1, e2_1) + of Clear(e0, e1): + let e1_1 = expr(e1) + build Clear(^operand(e0), e1_1) + + var outParam: Option[Value[Local]] + var delayed: Table[int64, dst.st] + proc exit(x: src.ex): dst.ex {.transform.} = + case x + of CheckedCallAsgn(lo, t, e0, ...e1, Goto(i), [tgt]): + let args = args(t, e1) + let callee = operand(e0) + if isAggregate(retType(t)): + let tmp = newTemp(typ retType(t)) + # the temporary needs to be copied to the actual target upon landing + delayed[i.val] = build(dst.st, Asgn(lo, Copy(tmp))) + build CheckedCall(^typ(t), callee, [...args, Addr(tmp)], Goto(i), tgt) + else: + build CheckedCallAsgn(lo, ^typ(t), callee, ...args, Goto(i), tgt) + of CheckedCallAsgn(lo, pr, ...e, Goto(i), [tgt]): + let sig = signatures[ord pr.val] + if isAggregate(retType(sig)): + let tmp = newTemp(typ retType(sig)) + # the temporary needs to be copied to the actual target upon landing + delayed[i.val] = build(dst.st, Asgn(lo, Copy(tmp))) + build CheckedCall(pr, [...args(sig, e), Addr(tmp)], Goto(i), tgt) + else: + build CheckedCallAsgn(lo, pr, ...args(sig, e), Goto(i), tgt) + of CheckedCall(pr, ...e, [go], [tgt]): + build CheckedCall(pr, ...args(signatures[ord pr.val], e), go, tgt) + of CheckedCall(t, e0, ...e1, [go], [tgt]): + let args = args(t, e1) + build CheckedCall(^typ(t), ^operand(e0), ...args, go, tgt) + + proc bblock(x: src.bb, pos: int): dst.bb {.transform.} = + proc aux(lo: slice(src.lo), st0: slice(src.st), ex: src.ex, + isExcept: bool, pos: int): dst.bb = + # TODO: allow using child slices directly in expansion positions + var bparams: seq[dst.lo] + for it in lo.items: + if pos == 0 and isAggregate(origLocals[ord it.val]): + params.incl(it.val) + bparams.add it + + if pos == 0 and outParam.isSome: + bparams.add outParam.unsafeGet + + stmts.shrink(0) + var extra: dst.st + if delayed.pop(pos, extra): + stmts.add extra + + for it in st0.items: + needsSave = false + let start = stmts.len + let st = stmt(it) + # the new statements, if any, were added in reverse; correct the order + reverse(stmts.toOpenArray(start, stmts.high)) + stmts.add st + + needsSave = false + + let start = stmts.len + let exit = + if outParam.isSome: + match ex: + of Return([e0 -> e]): + stmts.insert build(dst.st, Store(^getType(e0), Copy(^outParam.unsafeGet), e)), start + build(dst.ex, Return()) + else: exit(ex) + else: exit(ex) + reverse(stmts.toOpenArray(start, stmts.high)) + + if isExcept: + build Except(Params(...bparams), ...stmts, exit) + else: + build Block(Params(...bparams), ...stmts, exit) + + case x + of Block(Params(...lo), ...st, ex): + aux(lo, st, ex, false, pos) + of Except(Params(...lo), ...st, ex): + aux(lo, st, ex, true, pos) + + proc typ(x: src.t): dst.t {.transform.} = + case x + of ProcTy(t0, ...t1): + var got = newSeq[dst.t](t1.len) + for i, it in t1.pairs: + got[i] = + if isAggregate(it): build Ptr() + else: typ(it) + if isAggregate(t0): + build ProcTy(Void(), [...got, Ptr()]) + else: + build ProcTy(^typ(t0), ...got) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t0, Locals(...t1), List(...bb)): + params.clear() + delayed.clear() + origLocals = t1 + locals = map(t1, typ) + if isAggregate(retType(t0)): + outParam = some newTemp(build(dst.t, Ptr())) + else: + outParam = none dst.lo + var blocks = newSeq[dst.bb](bb.len) + for i, it in bb.pairs: + blocks[i] = bblock(it, i) + # turn all aggregate parameter locals into pointers + for it in params.items: + locals[ord it] = build(dst.t, Ptr()) + build ProcDef(^typ(t0), Locals(...locals), List(...blocks)) + +proc aggregatesToBlob(ir: L3s2, ptrsize: uint): L3s1 {.pass.} = + ## Turns all aggregate types into blob types. Path expression are turned into + ## address value arithmetic. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + var locals: slice(src.t) + + proc size(x: src.t): int64 = + match x: + of tid: size(types[ord tid.val]) + of Record(i, ...any): i.val + of Union(i, ...any): i.val + of Array(i, ...any): i.val + of Int(i): i.val + of UInt(i): i.val + of Float(i): i.val + of Ptr(): int64(ptrsize) + else: unreachable() + + proc elementOffset(x: src.t, elem: int64): int64 = + match x: + of tid: + elementOffset(types[ord tid.val], elem) + of Record(_, _, ...f): + match f[elem]: + of Field(i, _): i.val + of Union(...any): + 0 + of Array(_, _, _, t): + size(t) * elem + else: + unreachable() + + proc typeOfElem(x: src.t, elem: int64): src.t = + match x: + of tid: + typeOfElem(types[ord tid.val], elem) + of Record(_, _, ...f): + match f[elem]: + of Field(_, t): t + of Union(_, _, ...t): + t[elem] + of Array(_, _, _, t): + t + else: + unreachable() + + proc typ(x: src.t): dst.t {.transform.} = + case x + of Record(i0, i1, ...any): build Blob(i0, i1) + of Array(i0, i1, ...any): build Blob(i0, i1) + of Union(i0, i1, ...any): build Blob(i0, i1) + + proc path(root: src.ro, elems: slice(src.e)): dst.e = + var typ: src.t + (typ, result) = (; + match root: + of Deref(t, [e]): (t, e) + of lo: (locals[ord lo.val], build(dst.e, Addr(lo)))) + + var offset = 0'i64 + for it in elems.items: + match it: + of i: + # a static field or array access + offset += elementOffset(typ, i.val) + typ = typeOfElem(typ, i.val) + else: + # an array access with a dynamic index + if offset > 0: + # add the static offset computed so far: + result = build(dst.e, Offset(result, i(offset), i(1))) + offset = 0 + + typ = typeOfElem(typ, 0) + # apply the dynamic array element offset: + result = build(dst.e, Offset(result, ^expr(it), i(^size(typ)))) + + if offset > 0: + result = build(dst.e, Offset(result, i(offset), i(1))) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of Path(...any): unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(lv): + match lv: + of Path([t], ro, ...e): + build Load(t, ^path(ro, e)) + else: + build Copy(^lvalue(lv)) + of Addr(lv): + match lv: + of Path(_, ro, ...e): + path(ro, e) # the path becomes an address value + else: + build Addr(^lvalue(lv)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lv, [e]): + match lv: + of Path([t], ro, ...e1): + build Store(t, ^path(ro, e1), e) + else: + build Asgn(^lvalue(lv), e) + + proc bblock(x: src.bb): dst.bb {.generated.} + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc localsToBlob(ir: L3s1, ptrSize: uint): L3 {.pass.} = + ## Turns all non-blob locals that have their address taken into blob locals. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + var marker: PackedSet[Local] + var locals: seq[dst.t] + + proc scan(x: src.bb) = + ## Scans the basic-block for address-of operations and registers all + ## locals that have their address taken in marker. + # this is terrible code, for the largest part consisting of just + # traversal boilerplate. While generation of traversal logic would address + # this issue, the code also exemplifies why having an AST with many + # irregular forms is a bad idea + proc scan(x: src.e) = + match x: + of Addr(lo): + # the expression the pass is actually interested in + marker.incl lo.val + # traversal boilerplate follows + of any(t, ...e): # call and binary operator forms + discard t + for it in e.items: + scan(it) + of Call(pr, ...e): + discard pr + for it in e.items: + scan(it) + of any(t0, t1, e): # the conversion forms + discard t0; discard t1; scan(e) + of any(...e): + for it in e.items: + scan(it) + of any(t, e0, e1, lo): # the check forms + discard t; discard lo; scan(e0); scan(e1) + of Copy(_): discard + of ProcVal(_): discard + of Nil(): discard + of i: discard i + of fl: discard fl + + proc scan(x: src.st) = + # boilerplate... + match x: + of Call(_, ...e): + for it in e.items: + scan(it) + of Asgn(_, e): + scan(e) + of Store(_, e0, e1): + scan(e0); scan(e1) + of any(...e): + for it in e.items: + scan(it) + + proc scan(x: src.ex) = + # boilerplate... + match x: + of CheckedCallAsgn(_, _, ...e, _, _): + for it in e.items: + scan(it) + of CheckedCall(_, ...e, _, _): + for it in e.items: + scan(it) + of Raise(e, _): + scan(e) + of Branch(e, _, _): + scan(e) + of Return(e): + scan(e) + of Goto(_): discard + of Loop(_): discard + of any(): discard + + match x: + of Block(_, ...st, ex): + for it in st.items: scan(it) + scan(ex) + of Except(_, ...st, ex): + for it in st.items: scan(it) + scan(ex) + + proc sizeAndAlignment(x: src.t): (Value[int64], Value[int64]) = + match x: + of tid: sizeAndAlignment(types[ord tid.val]) + of Int(i): (i, i) + of UInt(i): (i, i) + of Float(i): (i, i) + of Ptr(): (terminal(int64(ptrSize)), terminal(int64(ptrSize))) + else: unreachable() + + proc isBlob(x: src.t): bool = + match x: + of tid: isBlob(types[ord tid.val]) + of Blob(_, _): true + else: false + + proc typ(x: src.t): dst.t {.transform.} = + case x + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(lo): + if lo.val in marker: + build Load(^locals[ord lo.val], Addr(lo)) + else: + build Copy(lo) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lo, [e]): + if lo.val in marker: + build Store(^locals[ord lo.val], Addr(lo), e) + else: + build Asgn(lo, e) + + proc newTemp(t: dst.t): dst.lo = + result = terminal Local(locals.len) + locals.add t + + proc bblock(x: src.bb): dst.bb {.transform.} = + proc update(x: src.lo, stmts: var seq[dst.st]): dst.lo = + if x.val in marker: + let typ = locals[ord x.val] + let tmp = newTemp(typ) + stmts.add build(dst.st, Store(typ, Addr(x), Copy(tmp))) + tmp + else: + x + + # if a block parameter (which can only be of primitive type at this + # point) has its address taken, the parameter must be turned back into a + # primitive type. In effect, this means that the real parameter is copied + # to a stack location on block entry that the rest of the procedure + # then uses in place of the real parameter + case x + of Block(Params(...lo), ...[st], [ex]): + var stmts: seq[dst.st] + let params = mapIt(lo, update(it, stmts)) + build Block(Params(...params), [...stmts, ...st], ex) + of Except(Params(...lo), ...[st], [ex]): + var stmts: seq[dst.st] + let params = mapIt(lo, update(it, stmts)) + build Except(Params(...params), [...stmts, ...st], ex) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t0, Locals(...t1), List(...bb)): + marker.clear() + locals = map(t1, typ) + # gather the locals that have their address taken: + for it in bb.items: + scan(it) + + let bbs = map(bb, bblock) + # ^^ potentially modifies the list of locals + + for i, it in t1.pairs: + if Local(i) in marker and not isBlob(it): + let (size, align) = sizeAndAlignment(it) + locals[i] = build(dst.t, Blob(size, align)) + + build ProcDef(^typ(t0), Locals(...locals), List(...bbs)) + +proc legalizeBlobOps(ir: L3): L2 {.pass.} = + ## Turns loads, stores, and assignments of blob types into blit copies. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t + ) + var locals: slice(src.t) + + proc size(x: src.t): int64 = + match x: + of Blob(i, _): i.val.int64 + of tid: size(types[ord tid.val]) + else: unreachable() + + proc isBlob(x: src.t): bool = + match x: + of Blob(...any): true + of tid: isBlob(types[ord tid.val]) + else: false + + proc expr(x: src.e): dst.e {.generated.} + proc typ(x: src.t): dst.t {.generated.} + proc bblock(x: src.bb): dst.bb {.generated.} + + proc operand(x: src.e): dst.e = + match x: + of Copy([lv0 -> lv]): build(dst.e, Addr(lv)) + of Load(_, [e]): e + else: unreachable() + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Store(t, [e0], e1): + if isBlob(t): + build Blit(e0, ^operand(e1), i(^size(t))) + else: + build Store(^typ(t), e0, ^expr(e1)) + of Asgn(lo, e): + let t = locals[ord lo.val] + if isBlob(t): + build Blit(Addr(lo), ^operand(e), i(^size(t))) + else: + build Asgn(lo, ^expr(e)) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc stackAlloc(ir: L2): L1 {.pass.} = + ## Simple stack allocation pass. Adds a frame pointer local to all procedures + ## that use stack memory and turns blob locals into stack locations. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t + ) + var locals: seq[tuple[onStack: bool, offset: uint32]] + var framePointer: dst.e + + func isBlob(x: src.t): bool = + match x: + of tid: isBlob(types[ord tid.val]) + of Blob(_, _): true + else: false + + func sizeAndAlign(x: src.t): (uint32, uint32) = + match x: + of tid: sizeAndAlign(types[ord tid.val]) + of Blob(i0, i1): (i0.val.uint32, i1.val.uint32) + else: unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Addr(lo): + assert locals[ord lo.val].onStack, $ord(lo.val) + if locals[ord lo.val].offset == 0: + # no addition is necessary + framePointer + else: + # XXX: it could make sense to try merging the offset computation into + # an enclosing one, where possible + build Offset(framePointer, i(^locals[ord lo.val].offset), i(1)) + of Copy(lo): + assert not locals[ord lo.val].onStack + build Copy(lo(^locals[ord lo.val].offset)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lo, [e]): + assert not locals[ord lo.val].onStack + build Asgn(lo(^locals[ord lo.val].offset), e) + + proc exit(x: src.ex): dst.ex {.transform.} = + case x + of CheckedCallAsgn(lo, [t0 -> t], ...[e], [go], [tgt]): + # XXX: very inefficient + build CheckedCallAsgn(lo(^locals[ord lo.val].offset), t, ...e, go, tgt) + of CheckedCallAsgn(lo, pr, ...[e], [go], [tgt]): + build CheckedCallAsgn(lo(^locals[ord lo.val].offset), pr, ...e, go, tgt) + + proc bblock(x: src.bb): dst.bb {.generated.} + + proc typ(x: src.t): dst.t {.transform.} = + case x + of Blob(_, _): build Ptr() + + proc mapLocal(x: src.lo): dst.lo = + assert not locals[ord x.val].onStack + terminal Local(locals[ord x.val].offset) + + proc params(x: src.p): dst.p {.transform.} = + case x + of Params(...lo): + var lo1 = newSeq[dst.lo](lo.len) + for i, it in lo.pairs: + lo1[i] = mapLocal(it) + build Params(...lo1) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t, Locals(...t1), List(...bb)): + locals.setLen(t1.len) + var + nextId = 0'u32 + stackOffset = 0'u32 + + # assign a stack location to every blob local. At the moment, blob locals + # are put into consecutive stack location, regardless of whether they have + # disjoint lifetimes or not + for i, it in t1.pairs: + if isBlob(it): + let (size, align) = sizeAndAlign(it) + doAssert align <= 8, "over-aligned locals are currently not supported" + # align the stack offset: + stackOffset = (stackOffset + (align - 1)) and not (align - 1) + locals[i] = (true, stackOffset) + stackOffset += size # reserve the needed space + else: + locals[i] = (false, nextId) + inc nextId + + # make the frame size a multiple of 8, so that the start of stack frame is + # always on an 8 byte boundary + stackOffset = (stackOffset + 7) and not 7'u32 + + if stackOffset == 0: + # the body stays as is, only the header needs to be modified + build ProcDef(^typ(t), i(0), Locals(...map(t1, typ)), List(...map(bb, bblock))) + else: + framePointer = build(dst.e, Copy(lo(^Local(nextId)))) + var filtered = newSeq[dst.t](nextId + 1) + for i, it in locals.pairs: + if not it.onStack: + filtered[it.offset] = typ(t1[i]) + + filtered[^1] = build(dst.t, Ptr()) + + var blocks = newSeq[dst.bb](bb.len) + for i, it in bb.pairs: + if i == 0: + # pass the frame pointer as an extra argument + match it: + of Block(Params(...lo), ...[st], [ex]): + blocks[i] = build(dst.bb, Block(Params([...map(lo, mapLocal), lo(nextid)]), ...st, ex)) + else: + unreachable() + else: + blocks[i] = bblock(it) + + build ProcDef(^typ(t), i(stackOffset), Locals(...filtered), List(...blocks)) + +proc inlineTypes(ir: L1): LPtr {.pass.} = + ## Removes `Blob` types and turns all identified numeric types into inline + ## types. + var types: Table[int, dst.t] + + proc typ(x: src.t): dst.t {.transform.} = + case x + of tid: + var r: dst.t + types.withValue ord(tid.val), val: + r = val[] + do: + r = build tid + r + + proc typedefs(x: src.td): dst.td {.transform.} = + case x + of TypeDefs(...t): + var pos = 0 + # build the lookup table for types to inline: + for i, it in t.pairs: + match it: + of ProcTy(...any): + if pos != i: + types[i] = build(dst.t, tid(pos)) # needs a fixup + inc pos + else: + types[i] = typ(it) + + # inline the types and remove them from the list of typedefs: + var outtypes = newSeq[dst.t](pos) + pos = 0 + for it in t.items: + match it: + of ProcTy(...any): + outtypes[pos] = typ(it) + inc pos + else: + discard "drop the type" + + build TypeDefs(...outtypes) + +proc ptrToInt(ir: LPtr, ptrsize: Positive): L0 {.pass.} = + ## Turns pointer types into unsigned integers. + proc typ(x: src.t): dst.t {.transform.} = + case x + of Ptr(): build UInt(i(ptrsize)) + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Nil(): build i(0) # represented as zero + of Offset([e0], [e1], e2): + match e2: + of i: + if i.val == 1: + return build(Add(UInt(i(ptrsize)), e0, e1)) + else: + discard "nothing to do" + + # the index is scaled + build Add(UInt(i(ptrsize)), e0, Mul(UInt(i(ptrsize)), e1, ^expr(e2))) + of Reinterp(Ptr(), UInt(i), [e0]): + if i.val == 8: e0 # drop the bitcast + else: build Reinterp(UInt(i(ptrsize)), UInt(i), e0) + of Reinterp(UInt(i), Ptr(), [e0]): + if i.val == 8: e0 # drop the bitcast + else: build Reinterp(UInt(i), UInt(i(ptrsize)), e0) + +# ----- compiler implementation ------ + +import experimental/[sexp, sexp_parse] +import passes/compilerdef +import std/[os, streams, strutils] + +proc tryParse[T: Global or Proc or Local or Type](n: SexpNode, _: typedesc[T]): Option[T] = + if n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == $T and n[1].kind == SInt: + some(T(n[1].num)) + else: + none(T) + +proc tryParse(n: SexpNode, _: typedesc[string]): Option[string] = + if n.kind == SString: + some(n.str) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "StringVal" and n[1].kind == SString: + some(n[1].str) + else: + none(string) + +proc tryParse(n: SexpNode, _: typedesc[int64]): Option[int64] = + if n.kind == SInt: + some(int64(n.num)) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "IntVal" and n[1].kind == SInt: + some(int64(n[1].num)) + else: + none(int64) + +proc tryParse(n: SexpNode, _: typedesc[float]): Option[float] = + if n.kind == SFloat: + some(n.fnum) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "FloatVal": + if n[1].kind == SFloat: + some(n[1].fnum) + elif n[1].kind == SSymbol: + some(parseFloat(n[1].symbol)) + else: + none(float) + elif n.kind == SSymbol: + some(parseFloat(n.symbol)) + else: + none(float) + +proc toSexp[T: Local or Global or Proc or Type](x: T): SexpNode = + newSList([newSSymbol($T), newSInt(ord x)]) + +proc toSexp(x: int64): SexpNode = + newSInt(int x) +proc toSexp(x: float): SexpNode = + newSFloat(x) +proc toSexp(x: string): SexpNode = + newSString(x) + +proc parseInput(p: var SexpParser): Lskully.m {.parser.} +proc parseInner(p: var SexpParser): L0.m {.parser.} +proc render(p: L6.m): SexpNode {.unparser.} +proc render(p: L5.m): SexpNode {.unparser.} +proc render(p: L4.m): SexpNode {.unparser.} +proc render(p: L3s2.m): SexpNode {.unparser.} +proc render(p: L3.m): SexpNode {.unparser.} +proc render(p: L2.m): SexpNode {.unparser.} +proc render(p: L1.m): SexpNode {.unparser.} +proc render(p: LPtr.m): SexpNode {.unparser.} +proc render(p: L0.m): SexpNode {.unparser.} + +let f = openFileStream(getExecArgs()[0], fmRead) +var p: SexpParser +p.open(f) +discard p.getTok() + +let (ast, m) = parseInput(p) +f.close() +echo "parsed" + +defineCompiler compile, LSkully, [ + basicBlocks, + globalsToPointer(8), + flattenPaths, + aggregateParams, + aggregatesToBlob(8), + localsToBlob(8), + legalizeBlobOps, + stackAlloc, + inlineTypes, + ptrToInt(8) +] + +discard compile(ast, m) From 314a42c23afd6e5da3e4db216c6e18a7f95f6718 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:56 +0000 Subject: [PATCH 53/87] nanopass: remove the `parser` and `unparser` macros Originally, the macro generated everything itself, but since it nowadays only calls out to generic routines, there's no point in having the macro pragmas at all. They're replaced with simple generic routines, meaning that AST parsing and unparsing is now available for all non-terminals without having to declare any prototype up-front. --- nanopass/nanopass.nim | 2 +- nanopass/npparser.nim | 43 +++++++++++--------------------- nanopass/npunparser.nim | 55 ++++++----------------------------------- 3 files changed, 22 insertions(+), 78 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 742cf93c..9e25d9d0 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -13,7 +13,7 @@ import export asts export nppatterns.matches -export npbuild.build, npmatch.match, npunparser.unparser, npparser.parser +export npbuild.build, npmatch.match, npunparser.unparse, npparser.parseAst export nppass.pass, nppass.inpass, nppass.outpass export nppass.genProcessor diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index 3edf0948..3aca851d 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -1,10 +1,10 @@ -## Implements the `parser <#parser.>`_ macro, for generating an AST parser for -## a language. +## Implements the routines for parsing an S-expression-based AST +## representation into an AST. import std/[genasts, macros, tables], experimental/[sexp_parse], - nanopass/[asts, nplang, helper] + nanopass/[asts, nplang] import experimental/sexp {.all.} # we need access to the internal parser @@ -249,29 +249,14 @@ macro check(lang: static LangInfo, nterm: static string, raiseError(line, col, "expected production of non-terminal '" & nterm & "'") -macro parser*(def: untyped) = - ## Procedure macro that generates a parser for a language's AST. The return - ## type is changed to be a tuple of an AST plus the original return type, - ## which must be a reference to one of the target language's non-terminals. - if def.kind != nnkProcDef: - error("'parser' must be applied to a procdef", def) - if def.body.kind != nnkEmpty: - error("'parser' must be applied to a prototype", def.name) - if def.params.len != 2 or def.params[1].len != 3: - error("prototype must have a single parameter", def.name) - - result = def - let typ = def.params[0] - let param = def.params[1][0] - let err = makeError("parameter must be of type `var SexpParser`", param) - - result.params[0] = quote do: - (Ast[`typ`.L, Literals], `typ`) - result.body = genAst(res=ident"result", param, err, typ): - when param isnot SexpParser: - err - res[0].storage = new(typeof(res[0].storage)) - - let (line, col) = (param.getLine(), param.getColumn()) - parse(param, res[0]) - check(idef(typ.L), typ.N, res[0], 0, line, col) +proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Metavar[L, N]]): (Ast[L, S], T) = + ## Parses the S-expression-based AST representation from `p` into an `Ast`, + ## returning the result, or - in case of an error - raising an exception. + var ast: Ast[L, S] + ast.storage = new S + + let (line, col) = (p.getLine(), p.getColumn()) + parse(p, ast) + check(idef(L), N, ast, 0, line, col) + + result = (ast, T(index: NodeIndex(0))) diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 1ea8a85a..02beeb4b 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -1,4 +1,4 @@ -## Implements the macros for unparsing an ASTs back into an S-expressions. +## Implements the routines for unparsing an ASTs back into S-expressions. # TODO: instead of assembling a S-expression directly, the unparser should # emit a stream of S-expression tokens (ideally as an iterator, but's @@ -8,7 +8,7 @@ import std/[macros, tables] import experimental/[sexp] -import nanopass/[asts, nplang, helper] +import nanopass/[asts, nplang] proc unparse[N: static string, S](ast: Ast[auto, S], pos: var int): SexpNode @@ -79,49 +79,8 @@ proc unparse[N: static string, S](ast: Ast[auto, S], pos: var int): SexpNode = mixin idef unparse(idef(typeof(ast).L), N, ast, pos) -macro unparserImpl(typ: untyped, def: untyped) = - ## The actual implementation of the ``unparser`` macro. - let unparse = bindSym"unparse" - let ast = genSym("ast") - let param = def.params[1][0] - def.params[1][^2] = typ - def.params.insert(1, - newIdentDefs(ast, quote do: Ast[`typ`.L, Literals])) - def.body = quote do: - var pos = `param`.index.int - `unparse`[`typ`.N](`ast`, pos) - - result = def - -macro unparser*(def: untyped): untyped = - ## A procedure macro that generates a body for unparsing a non-terminal at - ## a given position to an S-expression representation. The prototype must - ## have the following form: - ## - ## .. code-block:: nim - ## - ## proc name(x: Metavar[L, ...]): SexpNode - ## - ## and is expanded into: - ## - ## .. code-block:: nim - ## - ## proc name(_: Ast[L, Literals], x: Metavar[L, ...]): SexpNode = ... - ## - ## Terminals are rendered via ``toSexpr`` (with signature - ## ``proc(x: T): SexpNode``) provided by the callsite. - if def.kind != nnkProcDef or def.body.kind != nnkEmpty: - error(".unparser must be applied to a procedure declaration") - elif def.params.len == 2 and def.params[1].len == 1: - error("prototype must have exactly one parameter") - - let - typ = def.params[1][^2] - impl = bindSym"unparserImpl" - error = makeError("parameter type must be a `Metavar`", typ) - - result = quote do: - when `typ` is Metavar: - `impl`(`typ`, `def`) - else: - `error` +proc unparse*[L, S, N](ast: Ast[L, S], at: Metavar[L, N]): SexpNode = + ## Unparses the production at the given position `at`, returning it as a + ## self-contained S-expression. + var pos = at.index.int + unparse[N](ast, pos) From fd5652ff86450ebc7a363bc69653848b08f26e37 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:56 +0000 Subject: [PATCH 54/87] nanopass: move default processor definition into template Beyond preparing for the default processors getting more complex, this also fixes `genProcessor` having to be exported. --- nanopass/nanopass.nim | 3 --- nanopass/nppass.nim | 35 +++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 9e25d9d0..273eb6d3 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -16,9 +16,6 @@ export nppatterns.matches export npbuild.build, npmatch.match, npunparser.unparse, npparser.parseAst export nppass.pass, nppass.inpass, nppass.outpass -export nppass.genProcessor -# TODO: ^^ bind the symbols; don't mix them in - macro defineLanguage*(name, body: untyped) = ## Creates a language definition and binds it to a const symbol with the ## given name. diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index c479398d..505c5337 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -60,7 +60,7 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, matchImpl(lang, lang.map[src], ident"src", input, sel, rules, config) -macro genProcessor*(index, nterm: untyped): untyped = +macro genProcessor(index, nterm: untyped): untyped = ## Generates the body for a non-terminal processor. # simply emit an empty processorMatchImpl invocation. All branches will be # auto-generated @@ -255,6 +255,23 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = result = def +template defineProcessors(dst: untyped) = + ## Helper template for the pass macro implementation. Defines the implicit + ## `->` routines that will be invoked be default unless overridden. + proc `->`[U, X](n: U, T: typedesc[Metavar[dst, X]]): T {.inject.} = + # note: the signature is overly broad so that overload resolution + # prefers the more specific adapters created for the programmer-provided + # processors + genProcessor(n.index, U.N) + + proc `->`[T, C, N](s: ChildSlice[T, C], + U: typedesc[Metavar[dst, N]]): seq[U] {.inject, closure.} = + # XXX: the explicit .closure annotation works around a closure inference + # compiler bug + result = newSeq[U](s.len) + for i, it in s.pairs: + result[i] = it -> U + macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = # create a forward declaration for each transformer: var preamble = newStmtList() @@ -277,21 +294,7 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = # add the generic processor procedure, which all processor invocations # for processors not supplied by the programmer will end up using - let name = ident"->" - preamble.add quote do: - # note: the signature is overly broad, so that overload resolution - # prefers the more specific adapters created for the programmer-provided - # processors - proc `name`[U, X](n: U, T: typedesc[Metavar[`dst`, X]]): T = - genProcessor(n.index, U.N) - - proc `name`[T, C, N](s: ChildSlice[T, C], - U: typedesc[Metavar[`dst`, N]]): seq[U] {.closure.} = - # XXX: the explicit .closure annotation works around a closure inference - # compiler bug - result = newSeq[U](s.len) - for i, it in s.pairs: - result[i] = `name`(it, U) + preamble.add newCall(bindSym"defineProcessors", dst) # if the body doesn't end in an expression, add a call to the # entry processor From 99cb3f0536c39c462a38bec59588b4cd02dac28b Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:56 +0000 Subject: [PATCH 55/87] nptransform: support more general transformers Instead of admitting using non-terminal -> non-terminal transformers, the form transformer generation now also calls `->` for other pairings. This is the first step towards support for terminal -> non-terminal, terminal -> terminal, and non-terminal -> terminal transformers, leaving reporting an error in case automatic transformation is impossible to the `->` routines. --- nanopass/nptransform.nim | 110 +++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 62 deletions(-) diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 07389e9d..c39c5c8e 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -21,8 +21,9 @@ proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = else: result = None -proc append(to: var PackedTree[uint8], i: var int, x: Metavar) = - to.nodes[i] = TreeNode[uint8](kind: RefTag, val: uint32(x.index)) +proc append(to: var PackedTree[uint8], i: var int, + tag: uint8, val: uint32) {.inline.} = + to.nodes[i] = TreeNode[uint8](kind: tag, val: val) inc i macro transform*(src, dst: static LangInfo, nterm: static string, @@ -85,35 +86,25 @@ macro transform*(src, dst: static LangInfo, nterm: static string, # call the transformers and emit the nodes in one go: for i, a in src.forms[form].elems.pairs: let b = dst.forms[target].elems[i] - let fromTerminal = src.types[a.typ].terminal - let toTerminal = dst.types[b.typ].terminal - if fromTerminal != toTerminal or - (toTerminal and src.types[a.typ].name != dst.types[b.typ].name): - body.add makeError( - fmt"cannot generate transformer for '{src.forms[form]}'", cursor) - break + let s = ident(src.types[a.typ].mvar) + let d = ident(dst.types[b.typ].mvar) + let got = + if src.types[a.typ].terminal: + quote do: + src.`s`(index: `input`[pos(`cursor`)].val) + else: + quote do: + src.`s`(index: get(`input`, `cursor`)) + let append = bindSym"append" let call = - if fromTerminal: - if src.types[a.typ].ntag == dst.types[b.typ].ntag: - # just copy the node - quote do: - `output`.nodes[i] = `input`[pos(`cursor`)] - inc i - else: - # repack with the new tag - let tag = dst.types[b.typ].ntag - quote do: - `output`.nodes[i] = TreeNode[uint8](kind: `tag`, val: `input`[pos(`cursor`)].val) - inc i + if dst.types[b.typ].terminal: + let tag = dst.types[b.typ].ntag + quote do: + `append`(`output`, i, uint8(`tag`), (`got` -> dst.`d`).index) else: - let append = bindSym"append" - let op = ident"->" - let s = newStrLitNode(src.types[a.typ].name) - let d = newStrLitNode(dst.types[b.typ].name) quote do: - `append`(`output`, i, - `op`(Metavar[src, `s`](index: get(`input`, `cursor`)), Metavar[dst, `d`])) + `append`(`output`, i, RefTag, (`got` -> dst.`d`).index.uint32) if a.repeat: let bias = src.forms[form].elems.len - 1 @@ -144,41 +135,36 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, if result: break + let smvar = ident(src.types[typ].mvar) + let got = + case src.types[typ].terminal + of true: + quote do: + src.`smvar`(index: `input`[get(`input`, `cursor`)].val) + of false: + quote do: + src.`smvar`(index: get(`input`, `cursor`)) + let dtyp = dst.map.getOrDefault(src.types[typ].name, -1) - if src.types[typ].terminal: - if dtyp == -1: - # target language doesn't have the terminal - result = makeError( - fmt"cannot transform '{src.types[typ].name}' to '{nterm}'", - cursor) - elif contains(dst, dst.types[dst.map[nterm]], dtyp): - if src.types[typ].ntag == dst.types[dtyp].ntag: - # copy the node as it is - result = quote do: - `output`.nodes.add `input`[pos(`cursor`)] - NodeIndex(`output`.nodes.high) - else: - # re-tag the node - let tag = dst.types[dtyp].ntag.uint8 - result = quote do: - `output`.nodes.add TreeNode[uint8]( - kind: `tag`, - val: `input`[get(`input`, `cursor`)].val - ) - NodeIndex(`output`.nodes.high) - else: - # target non-terminal doesn't include the terminal - result = makeError( - fmt"cannot transform terminal '{src.types[typ].name}' to '{nterm}'", - cursor) - else: - let smvar = ident(src.types[typ].mvar) - # prefer a direct processor (i.e. 'a -> a') over 'a -> b' - let dmvar = - if dtyp != -1 and contains(dst, dst.types[dst.map[nterm]], dtyp): - ident(dst.types[dtyp].mvar) - else: - ident(dst.types[dst.map[nterm]].mvar) + # prefer a direct processor (i.e. 'a -> a') over 'a -> b' + if dtyp != -1 and contains(dst, dst.types[dst.map[nterm]], dtyp): + let dmvar = ident(dst.types[dtyp].mvar) + case dst.types[dtyp].terminal + of true: + # a new node needs to be allocated so that a reference to it can + # be returned + let tag = dst.types[dtyp].ntag.uint8 + let tmp = genSym() + result = quote do: + let `tmp` = `got` -> dst.`dmvar` + `output`.nodes.add TreeNode[uint8](kind: `tag`, val: `tmp`.index) + NodeIndex(`output`.nodes.high) + of false: + result = quote do: + (`got` -> dst.`dmvar`).index + else: + # no direct processor is possible; use an indirect processor + let dmvar = ident(dst.types[dst.map[nterm]].mvar) result = quote do: - (src.`smvar`(index: get(`input`, `cursor`)) -> dst.`dmvar`).index + (`got` -> dst.`dmvar`).index From 96fc0cb83b2bc0821d7086982e8ea586b52df375 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:56 +0000 Subject: [PATCH 56/87] nppass: adjust the built-in transformers * add an identity transformer for terminals * make the non-terminal transformer more narrow * support all valid targets with the slice transformer * handle missing transformers better (by reporting a dedicated error) --- nanopass/nppass.nim | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 505c5337..a57eaa31 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -3,6 +3,11 @@ import std/[genasts, macros, packedsets, tables] import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] +macro ctError(str: string, info: untyped) = + ## Like the .error pragma, but with customizable source location information. + copyLineInfo(str, info) + nnkPragma.newTree(nnkExprColonExpr.newTree(ident"error", str)) + template embed(storage, arg: untyped): untyped = ## Implements terminal value construction. mixin pack @@ -258,14 +263,21 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template defineProcessors(dst: untyped) = ## Helper template for the pass macro implementation. Defines the implicit ## `->` routines that will be invoked be default unless overridden. - proc `->`[U, X](n: U, T: typedesc[Metavar[dst, X]]): T {.inject.} = + template `->`[T, U](x: T, _: typedesc[U]): U {.inject.} = + # the fallback processor called when nothing else matches + ctError("cannot generate transformer from '" & $typeof(T) & + " to '" & $typeof(U) & "'", x) + + template `->`[T](v: Value[T], _: typedesc[Value[T]]): Value[T] {.inject.} = + v # nothing to do + + proc `->`[X](n: Metavar, T: typedesc[Metavar[dst, X]]): T {.inject.} = # note: the signature is overly broad so that overload resolution # prefers the more specific adapters created for the programmer-provided # processors - genProcessor(n.index, U.N) + genProcessor(n.index, typeof(n).N) - proc `->`[T, C, N](s: ChildSlice[T, C], - U: typedesc[Metavar[dst, N]]): seq[U] {.inject, closure.} = + proc `->`[T, C, U](s: ChildSlice[T, C], _: typedesc[U]): seq[U] {.inject, closure.} = # XXX: the explicit .closure annotation works around a closure inference # compiler bug result = newSeq[U](s.len) From 5c829d397a2dba56d42733a1e8315fc51a663002 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 57/87] nppass: support declaring all transformer types with `.transform` The internal macro is restructured such that it supports all non-non- terminals as both input and output. In addition, it's now checked that the input and output type are actually valid. --- nanopass/nppass.nim | 126 +++++++++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index a57eaa31..9d7f1964 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -3,11 +3,29 @@ import std/[genasts, macros, packedsets, tables] import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] +type + TypeClass = enum tcNone, tcValue, tcProduction + +macro isPartOf(lang: static LangInfo, lname, typ: untyped): bool = + ## Returns whether typedesc `typ` is a type referring to an entity + ## that's part of `lang`. + result = nil + for i, it in lang.types.pairs: + let call = newCall(ident"is", typ, newDotExpr(lname, ident(it.mvar))) + result = + if result.isNil: call + else: nnkInfix.newTree(ident"or", result, call) + macro ctError(str: string, info: untyped) = ## Like the .error pragma, but with customizable source location information. copyLineInfo(str, info) nnkPragma.newTree(nnkExprColonExpr.newTree(ident"error", str)) +template classify(x: typedesc): TypeClass = + when x is Value: tcValue + elif x is Metavar: tcProduction + else: tcValue + template embed(storage, arg: untyped): untyped = ## Implements terminal value construction. mixin pack @@ -83,16 +101,51 @@ proc hasPragma(def: NimNode, name: string): bool = if it.eqIdent(name): return true -macro genAdapter[T1, T2; A, B: static string]( - src: typedesc[Metavar[T1, A]], dst: typedesc[Metavar[T2, B]], - orig: untyped) = - # note: the macro signature is very specific because it acts as the type - # checking for programmer-provided processor signatures - let name = ident("->") - copyLineInfo(name, orig) - result = quote do: - template `name`(n: `src`, _: typedesc[`dst`]): `dst` = - {.line.}: `orig`(n) +template defineAdapter(src: typedesc, dst: typedesc, name: untyped) = + ## Introduces a definition for the adapter from `->` to `name`. + # the `->` uses a macro instead of a template because: + # * it leaves the call expression to inherit the expansion site's + # line information + # * it only binds `name` to the actual symbol when the macro expands, meaning + # that the symbol is only marked as used when it really is, allowing + # "unused symbol" detection to still work + macro `->`(n: src, _: typedesc[dst]): dst {.used.} = + newCall(ident(astToStr(name)), n) + +macro transformerImpl(sclass, dclass: static TypeClass, + param, body: untyped): untyped = + ## Refines the user-defined body of a processor into a real processor body. + proc transformCase(n: NimNode): NimNode = + result = genAst(arg=n[0]): + processorMatchImpl(idef(src), typeof(arg).N, Cursor(arg.index)) + copyLineInfo(result, n) + for i in 1..language pass processors. @@ -103,41 +156,28 @@ macro transformInOutImpl(lang: static LangDef, name, def: untyped) = if def.params[0].kind == nnkEmpty: error("a return type is required for a transfomer", def.name) - if def.body.kind == nnkEmpty: - # a forward declaration. Append the additional adapter procedure - return newStmtList(def, - newCall(bindSym"genAdapter", - copyNimTree(def.params[1][^2]), - copyNimTree(def.params[0]), - copyNimNode(def.name))) - - proc transformCase(n: NimNode): NimNode = - result = genAst(arg=n[0]): - processorMatchImpl(idef(src), typeof(arg).N, Cursor(arg.index)) - copyLineInfo(result, n) - for i in 1.. Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 58/87] nppass: add the `.manual` pragma It's used in conjunction with the `.transformer` pragma for disabling the procedure being registered with the nanopass framework. --- nanopass/nppass.nim | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 9d7f1964..b5d0c101 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -156,6 +156,15 @@ macro transformInOutImpl(lang: static LangDef, name, def: untyped) = if def.params[0].kind == nnkEmpty: error("a return type is required for a transfomer", def.name) + var isManual = false + block: + let list = def.pragma + for i in 0.. Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 59/87] npbuild: require full AST as input for `build` This makes `build` fully self-contained, meaning that it now works outside of pass context. --- nanopass/npbuild.nim | 67 ++++++++++++++++++++++---------------------- nanopass/nppass.nim | 4 +-- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 25db652d..558e850d 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -92,24 +92,24 @@ proc lengthError(len: int) {.noinline.} = raise ValueError.newException( fmt"no form is able to fit the expanded sequence (length = {len}") -proc append[L, U](to: var PackedTree[uint8], x: Value[U]) = - to.nodes.add TreeNode[uint8]( +proc append[L, U](ast: var Ast[L, auto], x: Value[U]) = + ast.tree.nodes.add TreeNode[uint8]( kind: typeof(lookup[U, L.meta.term_map]()).V, val: x.index) -proc append[L](to: var PackedTree[uint8], x: Metavar) = - to.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) +proc append[L](ast: var Ast[L, auto], x: Metavar) = + ast.tree.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) -proc append[L](to: var PackedTree[uint8], x: openArray) = +proc append[L](ast: var Ast[L, auto], x: openArray) = for it in x.items: - append[L](to, it) + append[L](ast, it) -template coerce[T, U](x: U, _: typedesc[Value[T]]): Value[T] = - mixin terminal +template coerce[S, T, U](s: S, val: U, _: typedesc[Value[T]]): Value[T] = + mixin pack when T is U: - terminal(x) # no coercion is necessary + Value[T](index: pack(s, val)) # no coercion is necessary else: - terminal(T(x)) # try a coercion, an error is fine + Value[T](index: pack(s, T(val))) # try a coercion, an error is fine {.pop.} @@ -126,7 +126,7 @@ proc containsForm(lang: LangInfo, typ: LangType, form: int): bool = result = true break -macro buildImpl(to: var PackedTree[uint8], +macro buildImpl(ast: var Ast, lang: static LangInfo, name: static string, target: typedesc[Metavar], e: untyped): untyped = ## Emits a tree construction for the AST described by `e`, with the syntax @@ -154,7 +154,7 @@ macro buildImpl(to: var PackedTree[uint8], let append = bindSym"append" result = quote do: when matches(`src`, `expect`): - `append`[`target`.L](`to`, `src`) + `append`(`ast`, `src`) else: `error` copyLineInfoForTree(result, src) @@ -180,7 +180,7 @@ macro buildImpl(to: var PackedTree[uint8], # expected a terminal, but the constructor can only be that of a form let mvar = ident(typ.mvar) result = quote do: - {.error: "expected terminal of type " & $`target`.L.`mvar`.} + {.error: "expected terminal of type " & $`ast`.L.`mvar`.} copyLineInfoForTree(result, n) return @@ -302,7 +302,7 @@ macro buildImpl(to: var PackedTree[uint8], else: let start = ctx.start let id = tup[2] - quote do: `to`.nodes[`start`].kind = `id` + quote do: `ast`.tree.nodes[`start`].kind = `id` let expanded = c.expanded if expanded != nil: @@ -378,14 +378,14 @@ macro buildImpl(to: var PackedTree[uint8], # simple case, the tag is known upfront let id = candidates[0].tag.uint8 c.start = nil # no 'start' variable needed - result.add genAst(to, id, lenExpr) do: - to.nodes.add TreeNode[uint8](kind: id, val: uint32(lenExpr)) + result.add genAst(ast, id, lenExpr) do: + ast.tree.nodes.add TreeNode[uint8](kind: id, val: uint32(lenExpr)) else: # the tag is only known at the end c.start = genSym("start") - result.add genAst(to, start=c.start, lenExpr) do: - let start = to.nodes.len - to.nodes.add TreeNode[uint8](kind: 0, val: uint32(lenExpr)) + result.add genAst(ast, start=c.start, lenExpr) do: + let start = ast.tree.nodes.len + ast.tree.nodes.add TreeNode[uint8](kind: 0, val: uint32(lenExpr)) result.addAll emit(lang, c, tree, n, 0) of nnkBracket: @@ -395,23 +395,23 @@ macro buildImpl(to: var PackedTree[uint8], result.addAll process(lang, typ, it) of nnkPrefix: let mvar = ident(typ.mvar) - result = makeMatch(n[1], (genAst(target, mvar) do: PArray[target.L.mvar])) + result = makeMatch(n[1], (genAst(ast, mvar) do: PArray[ast.L.mvar])) else: let mvar = ident(typ.mvar) if typ.terminal: - let expect = quote do: `target`.L.`mvar` + let expect = quote do: `ast`.L.`mvar` let error = newMismatchError(n, expect, n) let append = bindSym"append" result = quote do: when matches(`n`, `expect`) or `n` is `expect`.T: when `n` is `expect`.T: - `append`[`target`.L](`to`, terminal(`n`)) + `append`(`ast`, terminal(`n`)) else: - `append`[`target`.L](`to`, `n`) + `append`(`ast`, `n`) else: `error` else: - result = makeMatch(n, (quote do: `target`.L.`mvar`)) + result = makeMatch(n, (quote do: `ast`.L.`mvar`)) proc hoistCoercions(lang: LangInfo, to, e: NimNode): NimNode = ## Turns all terminal constructions into `let` statements and adds them @@ -425,8 +425,8 @@ macro buildImpl(to: var PackedTree[uint8], let mvar = ident(lang.types[id].mvar) result = genSym() - to.add genAst(result, mvar, src=e[1]) do: - let result = coerce(src, dst.mvar) + to.add genAst(result, ast, mvar, src=e[1]) do: + let result = coerce(ast.storage[], src, typeof(ast).L.mvar) else: result = e for i in 1.. Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 60/87] nanopass: add the "nanopass record" feature In addition to forms, terminals, and non-terminals, languages now also consist of records, which are tuples made up of other records, terminals, or non-terminals. Records are reference-like types, where each construction of a record yields a unique instance. --- nanopass/asts.nim | 14 ++- nanopass/nanopass.nim | 24 +++- nanopass/npbuild.nim | 147 ++++++++++++++++++---- nanopass/nplang.nim | 45 +++++-- nanopass/nplangdef.nim | 171 ++++++++++++++++++++++++-- nanopass/nplanggen.nim | 30 +++++ nanopass/npmatch.nim | 43 ++++--- nanopass/npparser.nim | 258 ++++++++++++++++++++++++++++++++------- nanopass/nppass.nim | 107 +++++++++++++++- nanopass/nptransform.nim | 60 +++++++-- nanopass/npunparser.nim | 106 ++++++++++++---- 11 files changed, 856 insertions(+), 149 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 2d6306ae..48ae4285 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -17,6 +17,8 @@ type ## leaked implementation detail, don't use storage*: ref Storage ## leaked implementation detail, don't use + records*: typeof(L.meta.records) + ## leaked implementation detail, don't use Metavar*[L: object, N: static string] = object ## Represents a reference to an AST fragment that's a production of non- @@ -32,7 +34,12 @@ type index*: uint32 ## leaked implementation detail, don't use - ChildSlice*[T: Metavar or Value, Cursor] = object + RecordRef*[L: object, N: static string] = object + ## A reference to a record of a language. + id*: uint32 + ## leaked implementation detail, don't use + + ChildSlice*[T: Metavar or RecordRef or Value, Cursor] = object ## A lightweight reference to a sequence of sibling nodes. The reference ## must not outlive the spawned-from tree. tree: ptr PackedTree[uint8] @@ -61,8 +68,9 @@ proc slice*[T, C](tree: ptr PackedTree[uint8], start: C, len: uint32 template load[T, C](tree: PackedTree[uint8], c: C): T = mixin get, pos - when T is Metavar: T(index: get(tree, c)) - else: T(index: tree[pos(c)].val) + when T is Metavar: T(index: get(tree, c)) + elif T is RecordRef: T(id: tree[pos(c)].val) + else: T(index: tree[pos(c)].val) iterator items*[T, C](s: ChildSlice[T, C]): T = mixin advance diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 273eb6d3..25824d26 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -2,7 +2,6 @@ ## defining intermediate languages (their syntax and grammar) and passes. # TODO: -# * implement symbol integration # * implement types integration # * implement meta-data support # * add a "compiler definition" macro @@ -15,6 +14,7 @@ export asts export nppatterns.matches export npbuild.build, npmatch.match, npunparser.unparse, npparser.parseAst export nppass.pass, nppass.inpass, nppass.outpass +export nppass.get macro defineLanguage*(name, body: untyped) = ## Creates a language definition and binds it to a const symbol with the @@ -27,9 +27,9 @@ macro defineLanguage*(name, base, body: untyped) = ## context. defineLanguageImpl(name, base, body) -proc finish*(ast: PackedTree[uint8], n: NodeIndex): PackedTree[uint8] = - ## Returns `ast` with all indirections resolved. - # TODO: don't export the procedure +proc resolve(ast: PackedTree[uint8], result: var PackedTree[uint8], n: NodeIndex) = + ## Copies the AST fragment starting at `n` to `result`, resolving all + ## indirections in the process. template src: untyped = ast.nodes template dst: untyped = result.nodes const size = sizeof(TreeNode[uint8]) @@ -68,3 +68,19 @@ proc finish*(ast: PackedTree[uint8], n: NodeIndex): PackedTree[uint8] = append(prev, i) stack.shrink(stack.len - 1) + +proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = + ## Returns `ast` with all indirections in the AST resolved. + var output: PackedTree[uint8] + resolve(ast.tree, output, n) + # also resolve the sub-trees referenced from records: + for s in fields(ast.records): + for tup in s.mitems: + for it in fields(tup): + when it is Metavar: + let got = output.nodes.len + resolve(ast.tree, output, it.index) + it.index = NodeIndex(got) + + ast.tree = output + result = ast diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 558e850d..e85b053e 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -97,6 +97,11 @@ proc append[L, U](ast: var Ast[L, auto], x: Value[U]) = kind: typeof(lookup[U, L.meta.term_map]()).V, val: x.index) +proc append[L](ast: var Ast[L, auto], x: RecordRef) = + ast.tree.nodes.add TreeNode[uint8]( + kind: typeof(lookup[typeof(x), L.meta.record_map]()).V, + val: x.id) + proc append[L](ast: var Ast[L, auto], x: Metavar) = ast.tree.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) @@ -121,14 +126,73 @@ proc containsForm(lang: LangInfo, typ: LangType, form: int): bool = else: result = false for it in typ.sub.items: - if not lang.types[it].terminal and + if lang.types[it].kind == tkNonTerminal and containsForm(lang, lang.types[it], form): result = true break -macro buildImpl(ast: var Ast, - lang: static LangInfo, name: static string, - target: typedesc[Metavar], e: untyped): untyped = +proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode + +proc buildRecord(lang: LangInfo, ast, e: NimNode): NimNode = + ## Translates a record construction form from the `build` language + ## to NimSkull. + assert e.kind == nnkObjConstr + let name = e[0].strVal + let typ = lang.map.getOrDefault(name, -1) + if typ == -1: + return makeError(fmt"no meta-variable or record called '{name}' exists", + e[0]) + elif lang.types[typ].kind != tkRecord: + return makeError(fmt"type '{name}' is not a record", e[0]) + + let tmp = genSym("") + let mvar = ident(lang.types[typ].mvar) + result = newStmtList() + result.add newVarStmt(tmp, quote do: default(typeof(`ast`.records.`mvar`[0]))) + for i in 1.. 0: + if name notin base.records: + error(fmt"base language has no record named '{name}'", it.name) + + var fields = move base.records[name].fields + for field in it.sub.items: + let fname = field[0] + fname.expectKind nnkIdent + block search: + for i in 0.. remove it + base.records.del(name) + else: + base.records[name].fields = fields + + if name in base.records: + # remove the old meta-variables: + base.records[name].mvars.shrink(0) + # update the var list with the to-be-inherited meta-vars: for name, it in base.terminals.pairs: for n in it.mvars.items: @@ -247,6 +301,10 @@ proc buildLanguage(add, sub: seq[NimNode], for n in it.mvars.items: vars[n] = name + for name, it in base.records.pairs: + for n in it.mvars.items: + vars[n] = name + # only now set the new meta-variable names for inherited non-terminals: for it in def.items: let name = it.name[0].strVal @@ -257,6 +315,15 @@ proc buildLanguage(add, sub: seq[NimNode], base.nterminals[name].mvars.add mname vars[mname] = name + for it in records.items: + let name = it.name[0].strVal + if name in base.records: + for i in 1.. 0 and name notin base.nterminals: @@ -345,6 +426,19 @@ proc buildLanguage(add, sub: seq[NimNode], result.nterminals[name] = nt + for it in records.items: + let name = it.name[0].strVal + if it.add.len > 0 and name notin base.records: + # it's a new record + checkName(result, vars, name, it.name) + var rec = Record(tag: -1) # the tag is computed later + for i in 1.. 0: + var record: Record + # temporarily pop the record from the table, so that it can be accessed + # more easily + discard result.records.pop(name, record) + + for field in it.add.items: + let fname = field[0] + let typ = field[1] + fname.expectKind nnkIdent + typ.expectKind nnkIdent + if findIt(record.fields, it.name == fname.strVal) != -1: + error(fmt"record '{name}' already has field named '{fname.strVal}'", + fname) + + # the type name may refer to a meta-variable, but this is resolved + # at a later point + if typ.strVal in vars: + record.fields.add (fname.strVal, typ.strVal, vars[typ.strVal]) + else: + error(fmt"no meta-var with name '{typ.strVal}' exists", typ) + + result.records[name] = record + computeNodeTags(result) # TODO: properly set the entry non-terminal result.entry = "module" @@ -361,6 +482,7 @@ proc makeLanguage*(body: NimNode): LangDef = body.expectMinLen 1 var add: seq[NimNode] var def: seq[NonTerminalDef] + var records: seq[RecordDef] # second pass: process the productions proc extract(n: NimNode, list: var seq[NimNode]) = @@ -381,9 +503,16 @@ proc makeLanguage*(body: NimNode): LangDef = case it.kind of nnkInfix: if it[0].eqIdent("::="): - var nt = NonTerminalDef(name: it[1]) - extract(it[2], nt.add) - def.add nt + if it[2].kind == nnkTupleConstr: + var rec = RecordDef(name: it[1]) + for elem in it[2].items: + elem.expectKind nnkExprColonExpr + rec.add.add elem + records.add rec + else: + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add) + def.add nt continue of nnkCall: add.add it @@ -395,13 +524,14 @@ proc makeLanguage*(body: NimNode): LangDef = # to keep the implementation simple, a non-extension language is treated # internally as an empty language definition being extended - buildLanguage(add, @[], def, default(LangDef), body) + buildLanguage(add, @[], def, records, default(LangDef), body) proc makeLanguage*(base: LangDef, body: NimNode): LangDef = ## Creates a language definition from the ``defineLanguage`` DSL code ## and `base`. var add, sub: seq[NimNode] var def: seq[NonTerminalDef] + var records: seq[RecordDef] var body = body if body.kind != nnkStmtList: @@ -432,10 +562,27 @@ proc makeLanguage*(base: LangDef, body: NimNode): LangDef = of nnkInfix: if it[0].eqIdent("::="): it.expectLen 3 - var nt = NonTerminalDef(name: it[1]) - nt.name.expectKind nnkCall - extract(it[2], nt.add, nt.sub) - def.add nt + if it[2].kind == nnkTupleConstr: + var rec = RecordDef(name: it[1]) + it[2].expectMinLen(1) + + # the + or - prefix is part of the name slot + for elem in it[2].items: + elem.expectKind nnkExprColonExpr + elem[0].expectKind nnkPrefix + if elem[0][0].eqIdent("+"): + rec.add.add nnkExprColonExpr.newTree(elem[0][1], elem[1]) + elif elem[0][0].eqIdent("-"): + rec.sub.add nnkExprColonExpr.newTree(elem[0][1], elem[1]) + else: + error("expected '+' or '-'", elem[0][0]) + + records.add rec + else: + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add, nt.sub) + def.add nt + handled = true of nnkPrefix: if it[0].eqIdent("-"): @@ -454,4 +601,4 @@ proc makeLanguage*(base: LangDef, body: NimNode): LangDef = if not handled: error("expected `-a`, `+a`, or `a(...) ::= ...`", it[0]) - buildLanguage(add, sub, def, base, body) + buildLanguage(add, sub, def, records, base, body) diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim index b292886c..a9f46090 100644 --- a/nanopass/nplanggen.nim +++ b/nanopass/nplanggen.nim @@ -25,6 +25,12 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = mvar, ident(typName.strVal), newStrLitNode(name))) + for name, it in def.records.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree(bindSym"RecordRef", + ident(typName.strVal), + newStrLitNode(name))) let ntType = nnkTupleTy.newTree() let (csym, fsym) = (bindSym"PChoice", bindSym"PForm") @@ -64,6 +70,30 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = metaType.add newIdentDefs(ident"term_map", tup) + # create the record->tag map: + block: + let tup = nnkTupleConstr.newTree() + for it in def.records.values: + tup.add nnkTupleConstr.newTree( + newDotExpr(copyNimTree(typName), ident(it.mvars[0])), + nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(it.tag))) + + if tup.len > 0: + metaType.add newIdentDefs(ident"record_map", tup) + + # create the symbol storage type (a tuple of arrays-of-structs): + let st = nnkTupleTy.newTree() + for name, rec in def.records.pairs: + let tup = nnkTupleTy.newTree() + for (name, mvar, _) in rec.fields.items: + tup.add newIdentDefs(ident(name), newDotExpr(typName, ident(mvar))) + + # expose under the first meta-var there is for the type + st.add newIdentDefs(ident(rec.mvars[0]), + nnkBracketExpr.newTree(ident"seq", tup)) + + metaType.add newIdentDefs(ident"records", st) + fields.add newIdentDefs(ident"meta", metaType) result = nnkTypeSection.newTree( diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 078ec3a4..5e84ab31 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -37,7 +37,7 @@ proc fits(lang: LangInfo, a, b: int): bool = ## is expected. if a == b: result = true - elif not lang.types[b].terminal: + elif lang.types[b].kind == tkNonTerminal: for it in lang.types[b].sub.items: if fits(lang, a, it): return true @@ -45,18 +45,23 @@ proc fits(lang: LangInfo, a, b: int): bool = proc countTags(lang: LangInfo, typ: LangType): int = result = typ.forms.len for it in typ.sub.items: - if lang.types[it].terminal: + case lang.types[it].kind + of tkTerminal, tkRecord: result += 1 - else: + of tkNonTerminal: result += countTags(lang, lang.types[it]) proc containsForm(lang: LangInfo, typ: LangType, fid: int): bool = - if fid in typ.forms: - true - else: - for it in typ.sub.items: - if not lang.types[it].terminal and containsForm(lang, lang.types[it], fid): - return true + case typ.kind + of tkNonTerminal: + if fid in typ.forms: + true + else: + for it in typ.sub.items: + if containsForm(lang, lang.types[it], fid): + return true + false + of tkTerminal, tkRecord: false proc makeTyped(e, typ, info: NimNode): NimNode = @@ -86,7 +91,7 @@ proc fitTo(lang: LangInfo, typ: int, pat: NimNode): NimNode = # the type is inferred from the receiver result = makeTyped(pat[0], newIntLitNode(typ), pat[1]) of nnkCurly: - if lang.types[typ].terminal: + if lang.types[typ].kind in {tkTerminal, tkRecord}: return nil # terminals cannot host any form (they're terminal) # filter out the forms not part of the target type and try to merge the @@ -112,8 +117,7 @@ proc fitTo(lang: LangInfo, typ: int, pat: NimNode): NimNode = else: result = nil of nnkPar: - if not lang.types[typ].terminal and - containsForm(lang, lang.types[typ], pat[1][0].intVal.int): + if containsForm(lang, lang.types[typ], pat[1][0].intVal.int): if countTags(lang, lang.types[typ]) == 1: result = pat # the type is only inhabited by the form else: @@ -383,10 +387,14 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, proc genOfBranch(lang: LangInfo, typ: LangType, used: var IntSet, allowEmpty=true): NimNode = result = nnkOfBranch.newTree() - if typ.terminal: + case typ.kind + of tkTerminal: if not containsOrIncl(used, typ.ntag) or not allowEmpty: result.add newIntLitNode(typ.ntag) - else: + of tkRecord: + if not containsOrIncl(used, typ.rtag) or not allowEmpty: + result.add newIntLitNode(typ.rtag) + of tkNonTerminal: let ntags = ntags(lang, typ) for tag in ntags(lang, typ): if not containsOrIncl(used, tag): @@ -523,9 +531,12 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, if pos.kind == nnkIdent: pos # the source is a bound identifier already else: - if lang.types[typ.intVal].terminal: + case lang.types[typ.intVal].kind + of tkTerminal: quote do: `name`.`mvar`(index: `ast`[`pos`].val) - else: + of tkRecord: + quote do: `name`.`mvar`(id: `ast`[`pos`].val) + of tkNonTerminal: quote do: `name`.`mvar`(index: `pos`) of nnkBracket: # a list binding diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index 3aca851d..299176e5 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -2,12 +2,25 @@ ## representation into an AST. import - std/[genasts, macros, tables], + std/[genasts, macros, strutils, tables, typetraits], experimental/[sexp_parse], nanopass/[asts, nplang] import experimental/sexp {.all.} # we need access to the internal parser +type + Ctx[L, S] = object + # accumulators: + tree: typeof(Ast[L, S].tree) + records: typeof(Ast[L, S].records) + storage: ref S + staging: typeof(Ast[L, S].tree) + ## staging buffer for out-of-band trees embedded in records + # additional parsing state: + maps: array[tupleLen(L.meta.records), Table[int, uint32]] + ## for each record type, keeps track of the declared ID -> real ID + ## mappings + # the core parser logic for a language is implemented in generic routines, # which themselves call internal macros; the external macro then only expands # to code calling said generic routines. The benefit: most of the logic for @@ -15,14 +28,52 @@ import experimental/sexp {.all.} # we need access to the internal parser # parser is generated for a language. This is somewhat problematic for symbol # binding (for the terminal parsers), however, given how generics work +macro mapTypeImpl(lang: static LangInfo, lname, typ: untyped): int = + result = nnkWhenStmt.newTree() + for i, it in lang.types.pairs: + result.add nnkElifBranch.newTree( + newCall(ident"is", typ, newDotExpr(lname, ident(it.mvar))), + newIntLitNode(i)) + + result.add nnkElse.newTree(quote do: {.error: "unreachable".}) + +proc mapType[L, T](): int {.compileTime.} = + ## Maps the type `T` to the integer IDs its known under in `L`. + mapTypeImpl(idef(L), L, T) + +macro tags(lang: static LangInfo, typ: static int): set[uint8] = + ## Returns the node tags for the forms inhabiting `typ`. + case lang.types[typ].kind + of tkRecord: + nnkCurly.newTree(newLit(uint8 lang.types[typ].rtag)) + of tkTerminal: + nnkCurly.newTree(newLit(uint8 lang.types[typ].ntag)) + of tkNonTerminal: + var se = nnkCurly.newTree() + for it in ntags(lang, lang.types[typ]): + se.add newLit(uint8 it) + se + proc raiseError(line, col: int, msg: string) {.noreturn.} = raise ValueError.newException("(" & $line & ", " & $col & ") " & msg) +template check[L](c: Ctx[L, auto], pos: NodeIndex, line, col: int, + t: typedesc) = + ## Makes sure the production at `pos` is one of the non-terminal identified + ## by `nterm`, raising an error if not. + if c.tree[pos].kind notin tags(idef(typeof(L)), mapType[L, t]()): + when t is Metavar: + raiseError(line, col, + "expected production of '" & t.N & "'") + else: + raiseError(line, col, + "expected '" & $t & "'") + macro genTerminalParser(lang: static LangInfo) = result = newStmtList() # emit the terminal handlers for it in lang.types.items: - if it.terminal: + if it.kind == tkTerminal: let typ = ident(it.name) let tag = it.ntag.uint8 result.add quote do: @@ -38,21 +89,129 @@ proc parseTerminal[L, S](lit: var S, node: SexpNode, line, col: int): TreeNode[u raiseError(line, col, "'" & $node & "' is neither a valid language form nor terminal") -proc parse[L, S](p: var SexpParser, to: var Ast[L, S]) +proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) + +proc extract(c: var Ctx, to: var Metavar, pos: int) = + # move the sub-tree over to the out-of-band storage + let start = c.staging.nodes.len + let count = c.tree.nodes.len - pos + c.staging.nodes.setLen(start + count) + copyMem(addr c.staging.nodes[start], addr c.tree.nodes[pos], + sizeof(TreeNode[uint8]) * count) + c.tree.nodes.shrink(pos) + to.index = NodeIndex(start) + +proc extract(c: var Ctx, to: var RecordRef, pos: int) = + to.id = c.tree.nodes.pop().val + +proc extract(c: var Ctx, to: var Value, pos: int) = + to.index = c.tree.nodes.pop().val + +proc parseFieldsImpl[L](c: var Ctx[L, auto], p: var SexpParser, + tup: var tuple) = + ## Parses the body of a record-def and fills `tup` with the values. + for name, it in fieldPairs(tup): + let start = c.tree.nodes.len + space(p) + eat(p, tkParensLe) + if currString(p) != name: + raiseParseErr(p, "expected '" & name & "'") + discard getTok(p) + space(p) + + # parse the field's value... + parse(c, p) + # ...then make sure it's what it should be + check(c, NodeIndex(start), p.getLine(), p.getColumn(), typeof(it)) -proc rawParseForm[L, S](p: var SexpParser, to: var Ast[L, S]) = + extract(c, it, start) + space(p) + eat(p, tkParensRi) + +macro parseFields(lang: static LangInfo, c: var Ctx, name: string) = + ## Selects the record type base on the dynamic value of `name`, parses + ## the record's fields, registers the resulting record with `c`, and + ## appends a record reference to the AST. + result = nnkCaseStmt.newTree(name) + var i = 0 # index of the record type + for typ in lang.types.items: + if typ.kind == tkRecord: + let mvar = ident(typ.mvar) + let tag = typ.rtag + result.add nnkOfBranch.newTree(newStrLitNode(typ.name), + genAst(c, mvar, i, tag) do: + if isDef: + # records may be recursive, so reserve and remember a slot first + let slot = c.records.mvar.len + c.records.mvar.setLen(slot + 1) + if id in c.maps[i]: + raiseError(start[0], start[1], + "a record with the given ID has been defined already") + c.maps[i][id] = uint32(slot) + + var tup: typeof(c.records.mvar[0]) + parseFieldsImpl(c, p, tup) + c.records.mvar[slot] = tup + c.tree.nodes.add TreeNode[uint8]( + kind: uint8(tag), + val: uint32(slot) + ) + else: + c.maps[i].withValue id, val: + c.tree.nodes.add TreeNode[uint8]( + kind: uint8(tag), + val: val[] + ) + do: + raiseError(start[0], start[1], + "record with ID " & $id & " is missing") + ) + inc i + + result.add nnkElse.newTree(genAst(name) do: + raiseError(start[0], start[1], + "there's no symbol type called '" & name & "'") + ) + +proc parseRecord[L, S](c: var Ctx[L, S], p: var SexpParser) = + ## Parses a record definition or reference from `p`. + assert p.currToken == tkKeyword + let isDef {.used.} = + case currString(p) + of ":record-def": true + of ":record": false + else: raiseParseErr(p, "expected ':record' or ':record-def'") + + discard getTok(p) + space(p) + let start {.used.} = (p.getLine(), p.getColumn()) + if p.currToken != tkSymbol: + raiseParseErr(p, "expected symbol") + let name = captureCurrString(p) + discard getTok(p) + space(p) + if p.currToken != tkInt: + raiseParseErr(p, "expected integer") + let id {.used.} = parseInt(p.currString) + discard getTok(p) + + parseFields(idef(L), c, name) + space(p) + eat(p, tkParensRi) + +proc rawParseForm[L, S](c: var Ctx[L, S], p: var SexpParser) = ## Parses and appends the elements for a form, without performing any ## grammar checks. - let start = to.tree.nodes.len - to.tree.nodes.add TreeNode[uint8]() # sub-tree node + let start = c.tree.nodes.len + c.tree.nodes.add TreeNode[uint8]() # sub-tree node var len = 0 while p.currToken != tkParensRi: - parse(p, to) + parse(c, p) space(p) inc len discard getTok(p) # eat the parens - to.tree.nodes[start].val = uint32(len) + c.tree.nodes[start].val = uint32(len) # the tag is computed separately macro parseFormImpl(lang: static LangInfo) = @@ -98,7 +257,7 @@ macro parseFormImpl(lang: static LangInfo) = case m.kind of nnkIntLit: result = quote do: - to.tree.nodes[start].kind = `m` + c.tree.nodes[start].kind = `m` of nnkCurly: result = nnkIfStmt.newTree() for it in m.items: @@ -114,24 +273,27 @@ macro parseFormImpl(lang: static LangInfo) = if head.kind == nnkEmpty: # matches at the end of the sub-tree result = quote do: - if cursor.int == to.tree.nodes.len: + if cursor.int == c.tree.nodes.len: `next` else: `raiseErr`(line, col, "end of sub-tree expected, but got more nodes") else: let tags = nnkCurly.newTree() let name = lang.types[head.intVal].name - if lang.types[head.intVal].terminal: + case lang.types[head.intVal].kind + of tkTerminal: tags.add newLit(uint8(lang.types[head.intVal].ntag)) - else: + of tkRecord: + tags.add newLit(uint8(lang.types[head.intVal].rtag)) + of tkNonTerminal: for it in ntags(lang, lang.types[head.intVal]): tags.add newLit(uint8(it)) if m[1].intVal == 1: # single item? result = quote do: - if cursor.int < to.tree.nodes.len and - to.tree[cursor].kind in `tags`: - cursor = to.tree.next(cursor) + if cursor.int < c.tree.nodes.len and + c.tree[cursor].kind in `tags`: + cursor = c.tree.next(cursor) `next` else: `raiseErr`(line, col, "expected production of '" & `name` & "'") @@ -139,9 +301,9 @@ macro parseFormImpl(lang: static LangInfo) = # a list; may be comprised of zero or more items let bias = -m[1].intVal result = quote do: - for i in uint32(`bias`)..language pass. if def.kind notin {nnkProcDef, nnkFuncDef}: @@ -112,6 +121,30 @@ template defineAdapter(src: typedesc, dst: typedesc, name: untyped) = macro `->`(n: src, _: typedesc[dst]): dst {.used.} = newCall(ident(astToStr(name)), n) +template withCache(to: typedesc, inp, body: untyped): untyped = + ## If a mapping for `inp` -> `to` already exists, returns it, otherwise + ## evaluates `body` and remembers its result. + mixin table + let c = getTable[typeof(inp), to]() + var res: to + withValue c[], inp.id, val: + res = + when to is Value: to(index: val[]) + elif to is RecordRef: to(id: val[]) + elif to is Metavar: to(index: NodeIndex(val[])) + else: {.error.} + do: + # TODO: `body` should be wrapped in a lambda, so that `return` doesn't + # disables adding to the table + let val = if true: body else: default(to) + c[][inp.id] = + when to is Value: val.index + elif to is RecordRef: val.id + elif to is Metavar: val.index.uint32 + else: {.error.} + res = val + res + macro transformerImpl(sclass, dclass: static TypeClass, param, body: untyped): untyped = ## Refines the user-defined body of a processor into a real processor body. @@ -126,7 +159,7 @@ macro transformerImpl(sclass, dclass: static TypeClass, if result.kind != nnkStmtList: result = newStmtList(result) - if dclass == tcProduct: + if dclass in {tcProduction, tcRecord}: let to = ident"out.ast" result.insert 0, quote do: # inject a build macro overload that implicitly uses the @@ -134,11 +167,14 @@ macro transformerImpl(sclass, dclass: static TypeClass, template build(body: untyped): untyped {.used.} = build(`to`, typeof(result), body) - if sclass == dclass and sclass == tcProduct: + if sclass == dclass and sclass == tcProduction: if result[^1].kind == nnkCaseStmt: result[^1] = transformCase(result[^1]) else: error("trailing expression must be 'case'", result[^1]) + elif sclass == tcRecord: + result = newCall(bindSym"withCache", + newCall(ident"typeof", ident"result"), param, result) # else: leave the body as it is template checkType(lang, typ: untyped) = @@ -246,6 +282,8 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template slice[N](T: typedesc[Metavar[src, N]]): typedesc {.used.} = ChildSlice[T, Cursor] + template slice[N](T: typedesc[RecordRef[src, N]]): typedesc {.used.} = + ChildSlice[T, Cursor] template slice(T: typedesc[asts.Value[auto]]): typedesc {.used.} = ChildSlice[T, Cursor] @@ -253,6 +291,8 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) # XXX: consider renaming this template to `get` unpack(`input`.storage[], v.index, typeof(T)) + template get[N](r: RecordRef[src, N]): untyped {.used.} = + get(`input`, r) template equal[N](a, b: Metavar[src, N]): bool {.used.} = equal(`input`.tree, Cursor(a.index), Cursor(b.index)) @@ -262,12 +302,19 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = body.add quote do: template terminal(x: untyped): untyped {.used.} = `embed`(`output`.storage, x) - template build(n: typedesc[Metavar], body: untyped): untyped {.used.} = + template build[N](n: typedesc[Metavar[dst, N]], body: untyped): untyped {.used.} = + build(`output`, n, body) + template build[N](n: typedesc[RecordRef[dst, N]], body: untyped): untyped {.used.} = build(`output`, n, body) template match[N](sel: Metavar[dst, N], branches: varargs[untyped]): untyped {.used.} = match[dst, N](`output`.tree, IndCursor(sel.index), sel, branches) template slice[N](T: typedesc[Metavar[dst, N]]): typedesc {.used.} = ChildSlice[T, IndCursor] + template slice[N](T: typedesc[RecordRef[dst, N]]): typedesc {.used.} = + ChildSlice[T, IndCursor] + + template get[N](r: RecordRef[dst, N]): untyped {.used.} = + get(`output`, r) template equal[N](a, b: Metavar[dst, N]): bool {.used.} = equal(`output`.tree, IndCursor(a.index), IndCursor(b.index)) @@ -290,7 +337,7 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = body.add quote do: let pos = `call` # turn the AST with indirections into one without - `output`.tree = finish(`output`.tree, pos.index) + `output` = resolve(move `output`, pos.index) result = (move `output`, typeof(pos)(index: NodeIndex(0))) else: body.add quote do: @@ -327,6 +374,23 @@ template defineProcessors(dst: untyped) = # processors genProcessor(n.index, typeof(n).N) + proc `->`[X](r: RecordRef, T: typedesc[RecordRef[dst, X]]): T {.inject.} = + let tab = getTable[T, typeof(r)]() + # XXX: cannot use `withValue` because of symbol binding issues... + if r.id in tab[]: + T(id: tab[][r.id]) + else: + let s = addr pick(idef(dst), X, `out.ast`.records) + # reserve and remember a slot first, so that recursive records work + let id = s[].len.uint32 + s[].setLen(id + 1) + tab[][r.id] = id + + let rec {.cursor.} = get(r) + let tmp = transformRecord(rec, typeof(get(result))) + s[][id] = tmp + T(id: id) + proc `->`[T, C, U](s: ChildSlice[T, C], _: typedesc[U]): seq[U] {.inject, closure.} = # XXX: the explicit .closure annotation works around a closure inference # compiler bug @@ -334,6 +398,37 @@ template defineProcessors(dst: untyped) = for i, it in s.pairs: result[i] = it -> U +template wrapWithTables(lambda: untyped): untyped = + # the result of all record->* transformations is remembered, which requires + # a table for all used type pairs. This is tricky, as while what + # source*destination pairs there are is statically known, it's only known + # *after* the whole pass body has been type-checked, but the generated + # processors need access to their respective table early + var tabCounter {.global, compileTime.} = 0 + {.push checks: off, stacktrace: off.} + proc access(tab: int): ptr Table[uint32, uint32] {.closure.} + + proc getTable[A, B](): ptr Table[uint32, uint32] {.inject.} = + # the counter is only incremented (and a table is thus reserved) when + # the procedure is instantiated with new unique type parameters + const pos = tabCounter + static: inc tabCounter + access(pos) + + {.pop.} + + let tmp = lambda + + # now that the number of tables is known, statically allocate them and + # complete the `access` declaration + var tabs: array[tabCounter, Table[uint32, uint32]] + {.push checks: off, stacktrace: off.} + proc access(tab: int): ptr Table[uint32, uint32] {.closure.} = + addr tabs[tab] + {.pop.} + + tmp + macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = # create a forward declaration for each transformer: var preamble = newStmtList() @@ -371,7 +466,7 @@ macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) lambda.params = copyNimTree(def.params) - let call = newCall(lambda) + let call = newCall(newCall(bindSym"wrapWithTables", lambda)) # forward the original parameters to the lambda: for i in 1.. dst.`d`).index) - else: + of tkRecord: + let tag = dst.types[b.typ].rtag + quote do: + `append`(`output`, i, uint8(`tag`), (`got` -> dst.`d`).id) + of tkNonTerminal: quote do: `append`(`output`, i, RefTag, (`got` -> dst.`d`).index.uint32) @@ -130,18 +139,21 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, proc contains(lang: LangInfo, typ: LangType, search: int): bool = for it in typ.sub.items: result = it == search - if not result and not lang.types[it].terminal: + if not result and lang.types[it].kind == tkNonTerminal: result = contains(lang, typ, search) if result: break let smvar = ident(src.types[typ].mvar) let got = - case src.types[typ].terminal - of true: + case src.types[typ].kind + of tkTerminal: quote do: src.`smvar`(index: `input`[get(`input`, `cursor`)].val) - of false: + of tkRecord: + quote do: + src.`smvar`(id: `input`[get(`input`, `cursor`)].val) + of tkNonTerminal: quote do: src.`smvar`(index: get(`input`, `cursor`)) @@ -150,8 +162,8 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, # prefer a direct processor (i.e. 'a -> a') over 'a -> b' if dtyp != -1 and contains(dst, dst.types[dst.map[nterm]], dtyp): let dmvar = ident(dst.types[dtyp].mvar) - case dst.types[dtyp].terminal - of true: + case dst.types[dtyp].kind + of tkTerminal: # a new node needs to be allocated so that a reference to it can # be returned let tag = dst.types[dtyp].ntag.uint8 @@ -160,7 +172,16 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let `tmp` = `got` -> dst.`dmvar` `output`.nodes.add TreeNode[uint8](kind: `tag`, val: `tmp`.index) NodeIndex(`output`.nodes.high) - of false: + of tkRecord: + # a new node needs to be allocated so that a reference to it can + # be returned + let tag = dst.types[dtyp].rtag.uint8 + let tmp = genSym() + result = quote do: + let `tmp` = `got` -> dst.`dmvar` + `output`.nodes.add TreeNode[uint8](kind: `tag`, val: `tmp`.id) + NodeIndex(`output`.nodes.high) + of tkNonTerminal: result = quote do: (`got` -> dst.`dmvar`).index else: @@ -168,3 +189,20 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let dmvar = ident(dst.types[dst.map[nterm]].mvar) result = quote do: (`got` -> dst.`dmvar`).index + +template transformRecord*(rec: tuple, to: typedesc[tuple]): untyped = + ## Transforms the given record `rec` into a target-language record with + ## internal type `to`. + var dest: to + for dname, dval in fieldPairs(dest): + var found {.global, compileTime.} = false + for sname, sval in fieldPairs(rec): + when dname == sname: + dval = sval -> typeof(dval) + static: found = true + + # truncating is possible and allowed, but widening is not + when not found: + {.error: "source record is missing a field called '" & dname & "'".} + + dest diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 02beeb4b..c60dc5d5 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -6,38 +6,97 @@ # TODO: render and store the unexpected node/tree alongside their # corresponding error node -import std/[macros, tables] +import std/[intsets, macros, tables, typetraits] import experimental/[sexp] import nanopass/[asts, nplang] -proc unparse[N: static string, S](ast: Ast[auto, S], pos: var int): SexpNode +from nanopass/nppass import get -macro unparse(def: static LangInfo, nterm: static string, ast, pos: untyped) = +type + Ctx = object + ## The unparsing context object. + pos: int ## current node position + records: seq[IntSet] + ## for each record type, keeps track of the already-defined records + +proc nameToIndex[L; Name: static string](): int {.compileTime.} = + ## Turns a record type name to the index of the corresponding set in + ## `Ctx.records`. + var i = 0 + var tmp: L + # the index being stable across compilation is not necessary + for f in fields(tmp): + when f is RecordRef: + when typeof(f).N == Name: + if true: + return i + inc i + +proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode + +proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, + tup: tuple): SexpNode = + ## Unparses the nanopass record `tup`. The full record is only emitted for + ## the first occurrence - only the ID is emitted for all further occurrences. + if containsOrIncl(c.records[static nameToIndex[L, N]()], id): + result = newSList([newSSymbol(":record"), newSSymbol(N), newSInt(id)]) + else: + result = newSList() + result.add newSSymbol(":record-def") + result.add newSSymbol(N) + result.add newSInt(id) + for name, val in fieldPairs(tup): + let node = + when val is Value: + toSexp(unpack(ast.storage[], val.index, typeof(val).T)) + elif val is RecordRef: + unparse[typeof(val).N](ast, c, val.id.int, get(ast, val)) + elif val is Metavar: + let prev = c.pos + c.pos = val.index.int + let r = unparse[typeof(val).N](ast, c) + c.pos = prev + r + else: + {.error: "unreachable".} + + result.add newSList([newSSymbol(name), node]) + +macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = ## Unparses the non-terminal with name `nterm` from the given language at ## the current cursor position `pos`. let id = def.map[nterm] - var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`pos`].kind) + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`c`.pos].kind) proc genForType(def: LangInfo, typ: LangType): NimNode = - case typ.terminal - of true: + case typ.kind + of tkTerminal: let name = ident(typ.name) quote do: - inc `pos` - toSexp(unpack(`ast`.storage[], `ast`.tree.nodes[`pos` - 1].val, `name`)) - of false: + inc `c`.pos + toSexp(unpack(`ast`.storage[], `ast`.tree.nodes[`c`.pos - 1].val, `name`)) + of tkRecord: + let unparse = bindSym"unparse" + let mvar = ident(typ.mvar) + let tmp = genSym("id") + let name = typ.name + quote do: + let `tmp` = `ast`.tree.nodes[`c`.pos].val.int + inc `c`.pos + `unparse`[`name`](`ast`, `c`, `tmp`, `ast`.records.`mvar`[`tmp`]) + of tkNonTerminal: let unparse = bindSym"unparse" let name = typ.name quote do: - `unparse`[`name`](`ast`, `pos`) + `unparse`[`name`](`ast`, `c`) # form renderers: for it in def.types[id].forms.items: var body = nnkStmtList.newTree() let name = def.forms[it].name body.add quote do: - let len {.used.} = `ast`.tree.nodes[`pos`].val.int - inc `pos` + let len {.used.} = `ast`.tree.nodes[`c`.pos].val.int + inc `c`.pos result = newSList([newSSymbol(`name`)]) for i, e in def.forms[it].elems.pairs: let inner = genForType(def, def.types[e.typ]) @@ -51,13 +110,15 @@ macro unparse(def: static LangInfo, nterm: static string, ast, pos: untyped) = result.add `inner` caseStmt.add nnkOfBranch.newTree(newIntLitNode(def.forms[it].ntag), body) - # rendering for embedded terminals and non-terminals: + # rendering for embedded types: for it in def.types[id].sub.items: let body = nnkAsgn.newTree(ident"result", genForType(def, def.types[it])) - case def.types[it].terminal - of true: + case def.types[it].kind + of tkTerminal: caseStmt.add nnkOfBranch.newTree(newIntLitNode(def.types[it].ntag), body) - of false: + of tkRecord: + caseStmt.add nnkOfBranch.newTree(newIntLitNode(def.types[it].rtag), body) + of tkNonTerminal: let ofb = nnkOfBranch.newTree() for tag in ntags(def, def.types[it]).items: ofb.add newIntLitNode(tag) @@ -68,19 +129,20 @@ macro unparse(def: static LangInfo, nterm: static string, ast, pos: untyped) = # nodes as errors caseStmt.add nnkElse.newTree(quote do: result = newSList( - [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`pos`].kind)]) - `pos` = `ast`.tree.next(NodeIndex `pos`).int) + [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`c`.pos].kind)]) + `c`.pos = `ast`.tree.next(NodeIndex `c`.pos).int) result = caseStmt -proc unparse[N: static string, S](ast: Ast[auto, S], pos: var int): SexpNode = +proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode = ## Unparses the non-terminal at `pos`, returning the corresponding ## S-expression. Implemented outside the main macro to facilitate caching ## of the generic's instantiation. mixin idef - unparse(idef(typeof(ast).L), N, ast, pos) + unparse(idef(typeof(ast).L), N, ast, c) proc unparse*[L, S, N](ast: Ast[L, S], at: Metavar[L, N]): SexpNode = ## Unparses the production at the given position `at`, returning it as a ## self-contained S-expression. - var pos = at.index.int - unparse[N](ast, pos) + var c = Ctx(pos: at.index.int) + c.records.newSeq(tupleLen(L.meta.records)) + unparse[N](ast, c) From 2c59dd20584d77da7c59014cb28f6a237ee2f625 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 61/87] asts: use type aliases for tree and nodes This prepares for using a different tag type. --- nanopass/asts.nim | 43 +++++++++++++++++++++------------------- nanopass/nanopass.nim | 6 +++--- nanopass/npbuild.nim | 10 +++++----- nanopass/npmatch.nim | 5 ++--- nanopass/npparser.nim | 18 ++++++----------- nanopass/nptransform.nim | 15 +++++++------- 6 files changed, 46 insertions(+), 51 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 48ae4285..5bc268a3 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -10,10 +10,14 @@ export trees.`[]`, trees.next, trees.child, trees.len export trees.TreeNode type + Tag* = uint8 + AstNode* = TreeNode[Tag] + Tree* = PackedTree[Tag] + # note: the fields are exported so that the nanopass machinery can access # them. User code should, in most cases, not access the fields directly Ast*[L: object, Storage: object] = object - tree*: PackedTree[uint8] + tree*: Tree ## leaked implementation detail, don't use storage*: ref Storage ## leaked implementation detail, don't use @@ -42,7 +46,7 @@ type ChildSlice*[T: Metavar or RecordRef or Value, Cursor] = object ## A lightweight reference to a sequence of sibling nodes. The reference ## must not outlive the spawned-from tree. - tree: ptr PackedTree[uint8] + tree: ptr Tree start: Cursor len: uint32 @@ -55,18 +59,17 @@ const RefTag* = 128'u8 ## the node used internally for indirections -template isAtom*(x: uint8): bool = +template isAtom*(x: Tag): bool = ## The predicate required for using an uint8 as a ``PackedTree`` tag. x >= RefTag # ----- slice implementation ----- -proc slice*[T, C](tree: ptr PackedTree[uint8], start: C, len: uint32 - ): ChildSlice[T, C] = +proc slice*[T, C](tree: ptr Tree, start: C, len: uint32): ChildSlice[T, C] = ## Creates a reference to `len` sibling nodes starting at `start`. ChildSlice[T, C](tree: tree, start: start, len: len) -template load[T, C](tree: PackedTree[uint8], c: C): T = +template load[T, C](tree: Tree, c: C): T = mixin get, pos when T is Metavar: T(index: get(tree, c)) elif T is RecordRef: T(id: tree[pos(c)].val) @@ -109,43 +112,43 @@ proc high*(s: ChildSlice): int = int(s.len) - 1 # ----- internal cursor API ----- # the cursor interface consists of these routines: -# * ``advance(PackedTree[uint8], var Cursor)``: +# * ``advance(Tree, var Cursor)``: # moves the cursor to the sibling of the current node -# * ``get(PackedTree[uint8], Cursor): NodeIndex``: +# * ``get(Tree, Cursor): NodeIndex``: # returns the resolved index of the node the cursor points to # * ``pos(Cursor): NodeIndex``: # returns the unresolved index of the node the cursor points to -# * ``enter(PackedTree[uint8], var Cursor): Savepoint``: +# * ``enter(Tree, var Cursor): Savepoint``: # enters the subtree at the current cursor position -# * ``restore(PackedTree[uint8], var Cursor, Savepoint)``: +# * ``restore(Tree, var Cursor, Savepoint)``: # exits the current subtree # XXX: this should use static interfaces once supported by NimSkull {.push stacktrace: off, inline.} -proc advance*(tree: PackedTree[uint8], cr: var Cursor) {.inline.} = +proc advance*(tree: Tree, cr: var Cursor) {.inline.} = NodeIndex(cr) = next(tree, NodeIndex(cr)) -proc get*(tree: PackedTree[uint8], cr: Cursor): NodeIndex {.inline.} = +proc get*(tree: Tree, cr: Cursor): NodeIndex {.inline.} = NodeIndex cr template pos*(cr: Cursor): NodeIndex = NodeIndex cr -proc enter*(tree: PackedTree[uint8], cr: var Cursor): Cursor {.inline.} = +proc enter*(tree: Tree, cr: var Cursor): Cursor {.inline.} = # nothing to step into and thus no cursor to save result = cr cr = Cursor(tree.child(NodeIndex(cr), 0)) -template restore*(tree: PackedTree[uint8], cr: Cursor, saved: untyped) = +template restore*(tree: Tree, cr: Cursor, saved: untyped) = discard # nothing to restore # implementation for a cursor into a tree with indirections follows -proc advance*(tree: PackedTree[uint8], cr: var IndCursor) = +proc advance*(tree: Tree, cr: var IndCursor) = NodeIndex(cr) = next(tree, NodeIndex(cr)) -proc get*(tree: PackedTree[uint8], cr: IndCursor): NodeIndex = +proc get*(tree: Tree, cr: IndCursor): NodeIndex = if tree[NodeIndex(cr)].kind == 128: NodeIndex tree[NodeIndex(cr)].val else: @@ -156,14 +159,14 @@ template pos*(cr: IndCursor): NodeIndex = type Savepoint = tuple[origin: IndCursor, stepped: bool] -proc enter*(tree: PackedTree[uint8], cr: var IndCursor): Savepoint = +proc enter*(tree: Tree, cr: var IndCursor): Savepoint = result = (cr, tree[NodeIndex(cr)].kind == 128) if result.stepped: cr = IndCursor tree[NodeIndex(cr)].val else: cr = IndCursor tree.child(NodeIndex(cr), 0) -template restore*(tree: PackedTree[uint8], cr: var IndCursor, +template restore*(tree: Tree, cr: var IndCursor, saved: Savepoint) = if saved.stepped: cr = saved.origin @@ -174,7 +177,7 @@ template restore*(tree: PackedTree[uint8], cr: var IndCursor, # ------ additional tree operations -------- -proc equal*(tree: PackedTree[uint8], a, b: Cursor): bool = +proc equal*(tree: Tree, a, b: Cursor): bool = ## Compares the nodes/sub-trees at `a` and `b` for structural equality. if pos(a) == pos(b): return true @@ -193,7 +196,7 @@ proc equal*(tree: PackedTree[uint8], a, b: Cursor): bool = result = true -proc equal*(tree: PackedTree[uint8], a, b: IndCursor): bool = +proc equal*(tree: Tree, a, b: IndCursor): bool = ## Compares the nodes/sub-trees at `a` and `b` for structural equality. if pos(a) == pos(b): return true diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 25824d26..66cda213 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -27,12 +27,12 @@ macro defineLanguage*(name, base, body: untyped) = ## context. defineLanguageImpl(name, base, body) -proc resolve(ast: PackedTree[uint8], result: var PackedTree[uint8], n: NodeIndex) = +proc resolve(ast: Tree, result: var Tree, n: NodeIndex) = ## Copies the AST fragment starting at `n` to `result`, resolving all ## indirections in the process. template src: untyped = ast.nodes template dst: untyped = result.nodes - const size = sizeof(TreeNode[uint8]) + const size = sizeof(AstNode) template append(start, fin: uint32) = let pos = dst.len @@ -71,7 +71,7 @@ proc resolve(ast: PackedTree[uint8], result: var PackedTree[uint8], n: NodeIndex proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = ## Returns `ast` with all indirections in the AST resolved. - var output: PackedTree[uint8] + var output: Tree resolve(ast.tree, output, n) # also resolve the sub-trees referenced from records: for s in fields(ast.records): diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index e85b053e..5779d5b5 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -93,17 +93,17 @@ proc lengthError(len: int) {.noinline.} = fmt"no form is able to fit the expanded sequence (length = {len}") proc append[L, U](ast: var Ast[L, auto], x: Value[U]) = - ast.tree.nodes.add TreeNode[uint8]( + ast.tree.nodes.add AstNode( kind: typeof(lookup[U, L.meta.term_map]()).V, val: x.index) proc append[L](ast: var Ast[L, auto], x: RecordRef) = - ast.tree.nodes.add TreeNode[uint8]( + ast.tree.nodes.add AstNode( kind: typeof(lookup[typeof(x), L.meta.record_map]()).V, val: x.id) proc append[L](ast: var Ast[L, auto], x: Metavar) = - ast.tree.nodes.add TreeNode[uint8](kind: RefTag, val: uint32(x.index)) + ast.tree.nodes.add AstNode(kind: RefTag, val: uint32(x.index)) proc append[L](ast: var Ast[L, auto], x: openArray) = for it in x.items: @@ -453,13 +453,13 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = let id = candidates[0].tag.uint8 c.start = nil # no 'start' variable needed result.add genAst(ast, id, lenExpr) do: - ast.tree.nodes.add TreeNode[uint8](kind: id, val: uint32(lenExpr)) + ast.tree.nodes.add AstNode(kind: id, val: uint32(lenExpr)) else: # the tag is only known at the end c.start = genSym("start") result.add genAst(ast, start=c.start, lenExpr) do: let start = ast.tree.nodes.len - ast.tree.nodes.add TreeNode[uint8](kind: 0, val: uint32(lenExpr)) + ast.tree.nodes.add AstNode(kind: 0, val: uint32(lenExpr)) result.addAll emit(lang, c, tree, n, 0) of nnkBracket: diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 5e84ab31..5de642a7 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -1,7 +1,6 @@ ## Implements the high and low-level `match` macros. import std/[macros, intsets, strformat, tables] -import passes/trees import nanopass/[asts, helper, nplang] type @@ -816,7 +815,7 @@ proc matchImpl*(lang: LangInfo, src: int, name, ast, sel, rules: NimNode, result = generateForMatch(lang, name, ast, sel, optimize(total), els, config) macro matchImpl(lang: static LangInfo, nterm: static string, - name: typed, ast: PackedTree[uint8], cursor: untyped, + name: typed, ast: Tree, cursor: untyped, info: untyped, rules: varargs[untyped]): untyped = ## The internal implementation `match` dispatches to. # report an error for all missing form and type handling @@ -828,7 +827,7 @@ macro matchImpl(lang: static LangInfo, nterm: static string, copyLineInfo(cursor, info) # for better source locations result = matchImpl(lang, lang.map[nterm], name, ast, cursor, rules, config) -template match*[L; N: static](ast: PackedTree[uint8], cursor, info: untyped, +template match*[L; N: static](ast: Tree, cursor, info: untyped, branches: varargs[untyped]): untyped = ## Type-unsafe version of ``match``, meant for internal usage. bind matchImpl diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index 299176e5..7142b35d 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -80,9 +80,9 @@ macro genTerminalParser(lang: static LangInfo) = block: let val = tryParse(node, `typ`) if val.isSome: - return TreeNode[uint8](kind: `tag`, val: pack(lit, val.unsafeGet)) + return AstNode(kind: `tag`, val: pack(lit, val.unsafeGet)) -proc parseTerminal[L, S](lit: var S, node: SexpNode, line, col: int): TreeNode[uint8] = +proc parseTerminal[L, S](lit: var S, node: SexpNode, line, col: int): AstNode = ## Implements fallback parsing of terminals. mixin idef genTerminalParser(idef(L)) @@ -97,7 +97,7 @@ proc extract(c: var Ctx, to: var Metavar, pos: int) = let count = c.tree.nodes.len - pos c.staging.nodes.setLen(start + count) copyMem(addr c.staging.nodes[start], addr c.tree.nodes[pos], - sizeof(TreeNode[uint8]) * count) + sizeof(AstNode) * count) c.tree.nodes.shrink(pos) to.index = NodeIndex(start) @@ -152,16 +152,10 @@ macro parseFields(lang: static LangInfo, c: var Ctx, name: string) = var tup: typeof(c.records.mvar[0]) parseFieldsImpl(c, p, tup) c.records.mvar[slot] = tup - c.tree.nodes.add TreeNode[uint8]( - kind: uint8(tag), - val: uint32(slot) - ) + c.tree.nodes.add AstNode(kind: uint8(tag), val: uint32(slot)) else: c.maps[i].withValue id, val: - c.tree.nodes.add TreeNode[uint8]( - kind: uint8(tag), - val: val[] - ) + c.tree.nodes.add AstNode(kind: uint8(tag), val: val[]) do: raiseError(start[0], start[1], "record with ID " & $id & " is missing") @@ -203,7 +197,7 @@ proc rawParseForm[L, S](c: var Ctx[L, S], p: var SexpParser) = ## Parses and appends the elements for a form, without performing any ## grammar checks. let start = c.tree.nodes.len - c.tree.nodes.add TreeNode[uint8]() # sub-tree node + c.tree.nodes.add AstNode() # sub-tree node var len = 0 while p.currToken != tkParensRi: parse(c, p) diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 89d0276a..1941a4f2 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -21,13 +21,12 @@ proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = else: result = None -proc append(to: var PackedTree[uint8], i: var int, - tag: uint8, val: uint32) {.inline.} = - to.nodes[i] = TreeNode[uint8](kind: tag, val: val) +proc append(to: var Tree, i: var int, tag: uint8, val: uint32) {.inline.} = + to.nodes[i] = AstNode(kind: tag, val: val) inc i macro transform*(src, dst: static LangInfo, nterm: static string, - form: static int, input, output: PackedTree[uint8], + form: static int, input, output: Tree, cursor: untyped): untyped = ## Generates the transformation from the given source language form ## to a compatible target language production of the non-terminal with @@ -80,7 +79,7 @@ macro transform*(src, dst: static LangInfo, nterm: static string, var i = `output`.nodes.len # the node sequence has to be contiguous, so it's allocated upfront `output`.nodes.setLen(i + len + 1) - `output`.nodes[i] = TreeNode[uint8](kind: `id`, val: uint32(len)) + `output`.nodes[i] = AstNode(kind: `id`, val: uint32(len)) inc i # call the transformers and emit the nodes in one go: @@ -131,7 +130,7 @@ macro transform*(src, dst: static LangInfo, nterm: static string, result.add ident"root" macro transformType*(src, dst: static LangInfo, nterm: static string, - typ: static int, input, output: PackedTree[uint8], + typ: static int, input, output: Tree, cursor: untyped): untyped = ## Transforms the instance of type `typ` (may be either a terminal or non- ## terminal) at `cursor` to an AST fitting the destination non-terminal @@ -170,7 +169,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let tmp = genSym() result = quote do: let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add TreeNode[uint8](kind: `tag`, val: `tmp`.index) + `output`.nodes.add AstNode(kind: `tag`, val: `tmp`.index) NodeIndex(`output`.nodes.high) of tkRecord: # a new node needs to be allocated so that a reference to it can @@ -179,7 +178,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let tmp = genSym() result = quote do: let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add TreeNode[uint8](kind: `tag`, val: `tmp`.id) + `output`.nodes.add AstNode(kind: `tag`, val: `tmp`.id) NodeIndex(`output`.nodes.high) of tkNonTerminal: result = quote do: From 13629bd5b15f119f018505496f97f2912918bbe2 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 13 Jan 2026 22:56:57 +0000 Subject: [PATCH 62/87] asts: abstract over the internal node layout * use a routine for constructing nodes * use getters and setters for accessing a node's tag --- nanopass/asts.nim | 29 ++++++++++++++++++++++++----- nanopass/nanopass.nim | 4 ++-- nanopass/npbuild.nim | 18 ++++++++---------- nanopass/npmatch.nim | 2 +- nanopass/npparser.nim | 14 +++++++------- nanopass/nptransform.nim | 8 ++++---- nanopass/npunparser.nim | 4 ++-- 7 files changed, 48 insertions(+), 31 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 5bc268a3..0f716d50 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -59,10 +59,29 @@ const RefTag* = 128'u8 ## the node used internally for indirections +template tag*(n: AstNode): uint8 = + ## The node's tag. + n.kind + template isAtom*(x: Tag): bool = ## The predicate required for using an uint8 as a ``PackedTree`` tag. x >= RefTag +{.push stacktrace: off, profiler: off.} + +proc `tag=`*(n: var AstNode, tag: uint8) {.inline.} = + ## Sets the tag of `n` to `tag`. A low-level operation. + n.kind = tag + +proc `==`(a, b: AstNode): bool {.inline.} = + ## Compares the nodes for equality, ignoring source location info. + a.tag == b.tag and a.val == b.val + +{.pop.} + +template node*(tag: uint8, v: uint32): AstNode = + AstNode(kind: tag, val: v) + # ----- slice implementation ----- proc slice*[T, C](tree: ptr Tree, start: C, len: uint32): ChildSlice[T, C] = @@ -149,7 +168,7 @@ proc advance*(tree: Tree, cr: var IndCursor) = NodeIndex(cr) = next(tree, NodeIndex(cr)) proc get*(tree: Tree, cr: IndCursor): NodeIndex = - if tree[NodeIndex(cr)].kind == 128: + if tree[NodeIndex(cr)].tag == 128: NodeIndex tree[NodeIndex(cr)].val else: NodeIndex cr @@ -160,7 +179,7 @@ template pos*(cr: IndCursor): NodeIndex = type Savepoint = tuple[origin: IndCursor, stepped: bool] proc enter*(tree: Tree, cr: var IndCursor): Savepoint = - result = (cr, tree[NodeIndex(cr)].kind == 128) + result = (cr, tree[NodeIndex(cr)].tag == 128) if result.stepped: cr = IndCursor tree[NodeIndex(cr)].val else: @@ -209,18 +228,18 @@ proc equal*(tree: Tree, a, b: IndCursor): bool = let na = tree[pos(a)] let nb = tree[pos(b)] if na != nb: - if na.kind == RefTag: + if na.tag == RefTag: stack.add (a, b, i) i = 1 a = IndCursor(na.val) continue - elif nb.kind == RefTag: + elif nb.tag == RefTag: stack.add (a, b, i) i = 1 b = IndCursor(nb.val) continue return false - elif na.kind == RefTag: + elif na.tag == RefTag: stack.add (a, b, i) i = 1 a = IndCursor(na.val) diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 66cda213..a8bd321e 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -49,9 +49,9 @@ proc resolve(ast: Tree, result: var Tree, n: NodeIndex) = var (i, last) = stack[^1] let prev = i while i <= last: - if src[i].kind < RefTag: + if src[i].tag < RefTag: last += src[i].val - elif src[i].kind == RefTag: + elif src[i].tag == RefTag: if i > prev: # copy everything we got so far append(prev, i) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 5779d5b5..1f2e72f7 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -93,17 +93,15 @@ proc lengthError(len: int) {.noinline.} = fmt"no form is able to fit the expanded sequence (length = {len}") proc append[L, U](ast: var Ast[L, auto], x: Value[U]) = - ast.tree.nodes.add AstNode( - kind: typeof(lookup[U, L.meta.term_map]()).V, - val: x.index) + ast.tree.nodes.add node(typeof(lookup[U, L.meta.term_map]()).V, x.index) proc append[L](ast: var Ast[L, auto], x: RecordRef) = - ast.tree.nodes.add AstNode( - kind: typeof(lookup[typeof(x), L.meta.record_map]()).V, - val: x.id) + ast.tree.nodes.add node( + typeof(lookup[typeof(x), L.meta.record_map]()).V, + x.id) proc append[L](ast: var Ast[L, auto], x: Metavar) = - ast.tree.nodes.add AstNode(kind: RefTag, val: uint32(x.index)) + ast.tree.nodes.add node(RefTag, uint32(x.index)) proc append[L](ast: var Ast[L, auto], x: openArray) = for it in x.items: @@ -376,7 +374,7 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = else: let start = ctx.start let id = tup[2] - quote do: `ast`.tree.nodes[`start`].kind = `id` + quote do: `ast`.tree.nodes[`start`].tag = `id` let expanded = c.expanded if expanded != nil: @@ -453,13 +451,13 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = let id = candidates[0].tag.uint8 c.start = nil # no 'start' variable needed result.add genAst(ast, id, lenExpr) do: - ast.tree.nodes.add AstNode(kind: id, val: uint32(lenExpr)) + ast.tree.nodes.add node(id, uint32(lenExpr)) else: # the tag is only known at the end c.start = genSym("start") result.add genAst(ast, start=c.start, lenExpr) do: let start = ast.tree.nodes.len - ast.tree.nodes.add AstNode(kind: 0, val: uint32(lenExpr)) + ast.tree.nodes.add node(0, uint32(lenExpr)) result.addAll emit(lang, c, tree, n, 0) of nnkBracket: diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 5de642a7..b4deb5df 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -405,7 +405,7 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, let stackLen = stack.len let typ = e[0].intVal.int - var caseStmt = nnkCaseStmt.newTree(quote do: `ast`[pos(`cursor`)].kind) + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`[pos(`cursor`)].tag) var used = initIntSet() for i in 1.. dst.`dmvar` - `output`.nodes.add AstNode(kind: `tag`, val: `tmp`.index) + `output`.nodes.add node(`tag`, `tmp`.index) NodeIndex(`output`.nodes.high) of tkRecord: # a new node needs to be allocated so that a reference to it can @@ -178,7 +178,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let tmp = genSym() result = quote do: let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add AstNode(kind: `tag`, val: `tmp`.id) + `output`.nodes.add node(`tag`, `tmp`.id) NodeIndex(`output`.nodes.high) of tkNonTerminal: result = quote do: diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index c60dc5d5..0fb6b6b9 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -66,7 +66,7 @@ macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = ## Unparses the non-terminal with name `nterm` from the given language at ## the current cursor position `pos`. let id = def.map[nterm] - var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`c`.pos].kind) + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`c`.pos].tag) proc genForType(def: LangInfo, typ: LangType): NimNode = case typ.kind @@ -129,7 +129,7 @@ macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = # nodes as errors caseStmt.add nnkElse.newTree(quote do: result = newSList( - [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`c`.pos].kind)]) + [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`c`.pos].tag)]) `c`.pos = `ast`.tree.next(NodeIndex `c`.pos).int) result = caseStmt From 7b3631ebc4fc6b45baf060cb9732a728187ef92f Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 63/87] npmatch: update leftover `kind` field usage The tag is queried via `.tag` now. --- nanopass/npmatch.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index b4deb5df..2e52640c 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -478,9 +478,9 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, config.fillForm(lang, top[0][0].intVal.int, backup, info)) else: # complex case. The form ID is only known at run-time; dispatch over - # the entry-level node's kind for detecting the form + # the top-level node's tag let inner = nnkCaseStmt.newTree( - quote do: `ast`.nodes[pos(`backup`)].kind) + quote do: `ast`.nodes[pos(`backup`)].tag) for it in top[0].items: inner.add nnkOfBranch.newTree(it, config.fillForm(lang, it.intVal.int, backup, info)) From 32fc212d1c1153afc4826ffa71bea4b954f30c28 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 64/87] nanopass: fix `match` pattern matching The previous merging both rejected and accepted cases it shouldn't, which is now fixed. In addition, a comment documenting the current problems with matcher merging is added. --- nanopass/npmatch.nim | 88 +++++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 2e52640c..8779ed24 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -657,35 +657,79 @@ proc mergeInto(a, b: NimNode): NimNode = ## Merges the match expression `a` into the match expression `b`, using a ## simple top-down zip-like algorithm. proc canMerge(a, b: NimNode): bool = - if a[0] == b[0] and a[0].kind != nnkBracket: # same head? - true + case b.kind + of nnkCall: + # can only merge when the head is the same + a.kind == nnkCall and a[0] == b[0] and canMerge(a[^1], b[^1]) + of nnkCurly: + case a.kind + of nnkCall: a[0] == b[0] + of nnkCurly: true + else: false else: false - case b.kind - of nnkCurly: - # merge the source matchers into the target matchers - if a.kind == nnkCall: - # `a` is a matcher tha covers the full set of values for the type; just - # append it at the end - b.add a + # XXX: merging match expressions doesn't work as it should. Given the rules: + # ``` + # of If(x, x, x): ... + # of If(_, _, _): ... + # ``` + # the second rule should match everything that the `If(x, x, x)` + # doesn't handle, but given the desired ergonomics of the `match` macro + # and the limitations of NimSkull, the only way to achieve this right + # now would be to emit the body of the second rule multiple times. Since + # this approach can quickly result in significant code bloat, it's + # decided against, and the second rule therefore only receives + # `If(y, _, _)` + + proc merge(a, b: NimNode): NimNode = + case b.kind + of nnkCall: + b[^1] = mergeInto(a[^1], b[^1]) + b + of nnkCurly: + if a.kind == nnkCall: + # `a` is a matcher that covers the full set of values for the type; + # just append it at the end + b.add a + else: + # important: keep in mind that the order is significant + assert a.kind == nnkCurly + for i in 1.. Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 65/87] nppass: fix generated record transformers The type arguments to the table access were the wrong way around, leading to hand-written and generated transformers for the same types erroneously using different tables underneath. --- nanopass/nppass.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 7834bc8e..1beb3aba 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -375,7 +375,7 @@ template defineProcessors(dst: untyped) = genProcessor(n.index, typeof(n).N) proc `->`[X](r: RecordRef, T: typedesc[RecordRef[dst, X]]): T {.inject.} = - let tab = getTable[T, typeof(r)]() + let tab = getTable[typeof(r), T]() # XXX: cannot use `withValue` because of symbol binding issues... if r.id in tab[]: T(id: tab[][r.id]) From f58c1469800b27204dd5bbffc3e8d21bb74a9c36 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 66/87] nanopass: implement source location attributes Every production can now optionally have a source location attached to it. --- nanopass/asts.nim | 91 ++++++++++++++++++++++++++++++++++-- nanopass/npbuild.nim | 66 +++++++++++++++------------ nanopass/npparser.nim | 99 +++++++++++++++++++++++++++++++--------- nanopass/nppass.nim | 48 ++++++++++++++----- nanopass/nptransform.nim | 43 ++++++++++------- nanopass/npunparser.nim | 40 +++++++++++++++- 6 files changed, 304 insertions(+), 83 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 0f716d50..e913d629 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -10,10 +10,20 @@ export trees.`[]`, trees.next, trees.child, trees.len export trees.TreeNode type - Tag* = uint8 + SLocRef* = distinct range[0'u32 .. uint32((1 shl 24) - 1)] + ## Local reference to a `SourceLoc <#SourceLoc>`_. + + Tag* = distinct uint32 + ## 8-bit tag, 24-bit source location reference AstNode* = TreeNode[Tag] Tree* = PackedTree[Tag] + SourceLoc* = object + ## Describes a source location, which may span multiple lines and columns. + file*: uint32 ## ID of the packed file name + sline*, eline*: uint32 + scol*, ecol*: uint16 + # note: the fields are exported so that the nanopass machinery can access # them. User code should, in most cases, not access the fields directly Ast*[L: object, Storage: object] = object @@ -55,23 +65,37 @@ type IndCursor* = distinct NodeIndex ## A cursor into a tree with indirections. + LineCol* = tuple + line: uint32 + column: uint16 + const RefTag* = 128'u8 ## the node used internally for indirections + NoSLoc* = SLocRef(0) + ## represents the absence of a source location + +proc `==`*(a, b: SLocRef): bool {.borrow.} template tag*(n: AstNode): uint8 = ## The node's tag. - n.kind + # simply cut off the higher bits + cast[uint8](n.kind) + +template info*(n: AstNode): SLocRef = + ## The node's source location information reference. + cast[SLocRef](uint32(n.kind) shr 8) template isAtom*(x: Tag): bool = ## The predicate required for using an uint8 as a ``PackedTree`` tag. - x >= RefTag + cast[uint8](x) >= RefTag {.push stacktrace: off, profiler: off.} proc `tag=`*(n: var AstNode, tag: uint8) {.inline.} = ## Sets the tag of `n` to `tag`. A low-level operation. - n.kind = tag + # overwrite only the lower 8-bit + n.kind = Tag((uint32(n.kind) and 0xFFFFFF00'u32) or uint32(tag)) proc `==`(a, b: AstNode): bool {.inline.} = ## Compares the nodes for equality, ignoring source location info. @@ -80,7 +104,64 @@ proc `==`(a, b: AstNode): bool {.inline.} = {.pop.} template node*(tag: uint8, v: uint32): AstNode = - AstNode(kind: tag, val: v) + ## Construct a node with the given tag and value and no source location. + AstNode(kind: cast[Tag](tag), val: v) + +template node*(tag: uint8, loc: SLocRef, v: uint32): AstNode = + ## Construct a node with the given tag, source location, and value. + AstNode(kind: cast[Tag](uint32(tag) or (uint32(loc) shl 8)), val: v) + +# ----- source location management ----- + +template convert[T](v: Positive): T = + if v > high(typeof(T)).int: high(typeof(T)) + else: T(v) + +proc newSourceLoc*[S](s: var S, file: string, line, col: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + let rline = convert[uint32](line) + let rcol = convert[uint16](col) + let loc = SourceLoc( + file: pack(s, file), + sline: rline, eline: rline, + scol: rcol, ecol: rcol + ) + SLocRef(1 + pack(s, loc)) + +proc newSourceLoc*[S](s: var S, file: string, + sline, scol, eline, ecol: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + let loc = SourceLoc( + file: pack(s, file), + sline: convert[uint32](sline), + eline: convert[uint32](eline), + scol: convert[uint16](scol), + ecol: convert[uint16](ecol) + ) + SLocRef(1 + pack(s, loc)) + +proc newSourceLoc*(ast: var Ast, file: string, line, col: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + newSourceLoc(ast.storage[], file, line, col) + +proc newSourceLoc*(ast: var Ast, file: string, + sline, scol, eline, ecol: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + newSourceLoc(ast.storage[], file, sline, scol, eline, ecol) + +proc file*(ast: Ast, info: SLocRef): lent string {.inline.} = + ## The file for the given, valid `info`. + mixin unpack + assert info != NoSLoc + unpack(ast.storage[], + unpack(ast.storage[], uint32(info) - 1, SourceLoc).file, string) + +proc span*(ast: Ast, info: SLocRef): tuple[s, e: LineCol] = + ## The source span of the given, valid `info`. Both the start and end + ## are inclusive. + mixin unpack + let s = unpack(ast.storage[], uint32(info) - 1, SourceLoc) + ((s.sline, s.scol), (s.eline, s.ecol)) # ----- slice implementation ----- diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 1f2e72f7..c0d0d732 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -92,20 +92,25 @@ proc lengthError(len: int) {.noinline.} = raise ValueError.newException( fmt"no form is able to fit the expanded sequence (length = {len}") -proc append[L, U](ast: var Ast[L, auto], x: Value[U]) = - ast.tree.nodes.add node(typeof(lookup[U, L.meta.term_map]()).V, x.index) +proc append[L, U](ast: var Ast[L, auto], info: SLocRef, x: Value[U]) = + ast.tree.nodes.add node( + typeof(lookup[U, L.meta.term_map]()).V, + info, + x.index) -proc append[L](ast: var Ast[L, auto], x: RecordRef) = +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: RecordRef) = ast.tree.nodes.add node( typeof(lookup[typeof(x), L.meta.record_map]()).V, + info, x.id) -proc append[L](ast: var Ast[L, auto], x: Metavar) = +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: Metavar) = + # the ref itself doesn't need source location info ast.tree.nodes.add node(RefTag, uint32(x.index)) -proc append[L](ast: var Ast[L, auto], x: openArray) = +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: openArray) = for it in x.items: - append[L](ast, it) + append[L](ast, info, it) template coerce[S, T, U](s: S, val: U, _: typedesc[Value[T]]): Value[T] = mixin pack @@ -129,9 +134,9 @@ proc containsForm(lang: LangInfo, typ: LangType, form: int): bool = result = true break -proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode +proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode -proc buildRecord(lang: LangInfo, ast, e: NimNode): NimNode = +proc buildRecord(lang: LangInfo, ast, info, e: NimNode): NimNode = ## Translates a record construction form from the `build` language ## to NimSkull. assert e.kind == nnkObjConstr @@ -176,12 +181,12 @@ proc buildRecord(lang: LangInfo, ast, e: NimNode): NimNode = if val.kind == nnkSym: val # let the compiler report a type mismatch else: - buildRecord(lang, ast, val) + buildRecord(lang, ast, info, val) of tkNonTerminal: if val.kind == nnkSym: val # let the compiler report a type mismatch else: - buildForm(lang, ftyp, ast, val) + buildForm(lang, ftyp, ast, info, val) copyLineInfo(body, val) result.add nnkAsgn.newTree(newDotExpr(tmp, ident(fname)), body) @@ -190,7 +195,7 @@ proc buildRecord(lang: LangInfo, ast, e: NimNode): NimNode = `ast`.records.`mvar`.add(`tmp`) `ast`.L.`mvar`(id: `ast`.records.`mvar`.high.uint32) -proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = +proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = ## Emits a tree construction for the AST described by `e`, with the syntax ## from `lang`. @@ -216,7 +221,7 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = let append = bindSym"append" result = quote do: when matches(`src`, `expect`): - `append`(`ast`, `src`) + `append`(`ast`, `info`, `src`) else: `error` copyLineInfoForTree(result, src) @@ -450,14 +455,14 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = # simple case, the tag is known upfront let id = candidates[0].tag.uint8 c.start = nil # no 'start' variable needed - result.add genAst(ast, id, lenExpr) do: - ast.tree.nodes.add node(id, uint32(lenExpr)) + result.add genAst(ast, id, info, lenExpr) do: + ast.tree.nodes.add node(id, info, uint32(lenExpr)) else: # the tag is only known at the end c.start = genSym("start") - result.add genAst(ast, start=c.start, lenExpr) do: + result.add genAst(ast, start=c.start, info, lenExpr) do: let start = ast.tree.nodes.len - ast.tree.nodes.add node(0, uint32(lenExpr)) + ast.tree.nodes.add node(0, info, uint32(lenExpr)) result.addAll emit(lang, c, tree, n, 0) of nnkBracket: @@ -478,9 +483,9 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = result = quote do: when matches(`n`, `expect`) or `n` is `expect`.T: when `n` is `expect`.T: - `append`(`ast`, terminal(`n`)) + `append`(`ast`, `info`, terminal(`n`)) else: - `append`(`ast`, `n`) + `append`(`ast`, `info`, `n`) else: `error` of tkRecord, tkNonTerminal: @@ -507,7 +512,7 @@ proc buildForm(lang: LangInfo, typ: int, ast, e: NimNode): NimNode = # record constructions may create their own nodes, hence them # having to be hoisted result = genSym() - to.add newLetStmt(result, buildRecord(lang, ast, e)) + to.add newLetStmt(result, buildRecord(lang, ast, info, e)) of nnkBracket: result = e for i in 0.. real ID ## mappings + locs: seq[SLocRef] + ## stack of source locations + curSLoc: SLocRef + ## source location to use for parsed nodes # the core parser logic for a language is implemented in generic routines, # which themselves call internal macros; the external macro then only expands @@ -54,6 +58,20 @@ macro tags(lang: static LangInfo, typ: static int): set[uint8] = se.add newLit(uint8 it) se +proc eatString(p: var SexpParser): string = + if p.isTok(tkString): + result = captureCurrString(p) + discard getTok(p) + else: + raiseParseErr(p, "expected string") + +proc eatInt(p: var SexpParser): int = + if p.isTok(tkInt): + result = parseInt(currString(p)) + discard getTok(p) + else: + raiseParseErr(p, "expected integer") + proc raiseError(line, col: int, msg: string) {.noreturn.} = raise ValueError.newException("(" & $line & ", " & $col & ") " & msg) @@ -80,9 +98,10 @@ macro genTerminalParser(lang: static LangInfo) = block: let val = tryParse(node, `typ`) if val.isSome: - return node(`tag`, pack(lit, val.unsafeGet)) + return node(`tag`, c.curSLoc, pack(c.storage[], val.unsafeGet)) -proc parseTerminal[L, S](lit: var S, node: SexpNode, line, col: int): AstNode = +proc parseTerminal[L, S](c: var Ctx[L, S], node: SexpNode, + line, col: int): AstNode = ## Implements fallback parsing of terminals. mixin idef genTerminalParser(idef(L)) @@ -167,16 +186,8 @@ macro parseFields(lang: static LangInfo, c: var Ctx, name: string) = "there's no symbol type called '" & name & "'") ) -proc parseRecord[L, S](c: var Ctx[L, S], p: var SexpParser) = - ## Parses a record definition or reference from `p`. - assert p.currToken == tkKeyword - let isDef {.used.} = - case currString(p) - of ":record-def": true - of ":record": false - else: raiseParseErr(p, "expected ':record' or ':record-def'") - - discard getTok(p) +proc parseRecord[L, S](c: var Ctx[L, S], p: var SexpParser, isDef: bool) = + ## Parses a record definition (if `isDef` is true) or reference from `p`. space(p) let start {.used.} = (p.getLine(), p.getColumn()) if p.currToken != tkSymbol: @@ -184,12 +195,58 @@ proc parseRecord[L, S](c: var Ctx[L, S], p: var SexpParser) = let name = captureCurrString(p) discard getTok(p) space(p) - if p.currToken != tkInt: - raiseParseErr(p, "expected integer") - let id {.used.} = parseInt(p.currString) - discard getTok(p) + let id {.used.} = eatInt(p) parseFields(idef(L), c, name) + +proc parseMeta[L, S](c: var Ctx[L, S], p: var SexpParser) = + ## Parses a meta-expression. + assert p.currToken == tkKeyword + case currString(p) + of ":record-def": + discard getTok(p) + parseRecord(c, p, true) + of ":record": + discard getTok(p) + parseRecord(c, p, false) + of ":info": + # use the given source location for a tree + discard getTok(p) + c.locs.add c.curSLoc + space(p) + eat(p, tkParensLe) + let file = eatString(p) + space(p) + let startLine = eatInt(p) + space(p) + let startCol = eatInt(p) + space(p) + if p.currToken == tkParensRi: + # only a line-column pair + c.curSLoc = newSourceLoc(c.storage[], file, startLine, startCol) + else: + let endLine = eatInt(p) + space(p) + let endCol = eatInt(p) + space(p) + c.curSLoc = + newSourceLoc(c.storage[], file, startLine, startCol, endLine, endCol) + + eat(p, tkParensRi) + space(p) + parse(c, p) + c.curSLoc = c.locs.pop() + of ":no-info": + # use no source location for a tree + discard getTok(p) + space(p) + c.locs.add c.curSLoc + c.curSLoc = NoSLoc + parse(c, p) + c.curSLoc = c.locs.pop() + else: + raiseParseErr(p, "expected meta-expression") + space(p) eat(p, tkParensRi) @@ -197,7 +254,7 @@ proc rawParseForm[L, S](c: var Ctx[L, S], p: var SexpParser) = ## Parses and appends the elements for a form, without performing any ## grammar checks. let start = c.tree.nodes.len - c.tree.nodes.add AstNode() # sub-tree node + c.tree.nodes.add node(0, c.curSLoc, 0) # sub-tree node var len = 0 while p.currToken != tkParensRi: parse(c, p) @@ -358,13 +415,13 @@ macro parseFormImpl(lang: static LangInfo) = # emit the terminal parsing fallback result.add nnkElse.newTree( - genAst(c=ident"c", p=ident"p", lang=ident"L", name=ident"name") do: + genAst(c=ident"c", p=ident"p", name=ident"name") do: var node = newSList(newSSymbol(name)) while p.currToken != tkParensRi: node.add parseSexp(p) space(p) discard getTok(p) - c.tree.nodes.add parseTerminal[lang](c.storage[], node, line, col) + c.tree.nodes.add parseTerminal(c, node, line, col) ) proc parseForm[L, S](c: var Ctx[L, S], p: var SexpParser, @@ -387,13 +444,13 @@ proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) = space(p) parseForm(c, p, name, line, col) of tkKeyword: - parseRecord(c, p) + parseMeta(c, p) else: raiseParseErr(p, "expected a symbol") else: # parse an S-expression and have the user-provided parser figure it out let (line, col) = (p.getLine(), p.getColumn()) - c.tree.nodes.add parseTerminal[L](c.storage[], parseSexp(p), line, col) + c.tree.nodes.add parseTerminal(c, parseSexp(p), line, col) proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Metavar[L, N]]): (Ast[L, S], T) = ## Parses the S-expression-based AST representation from `p` into an `Ast`, diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 1beb3aba..0a379f3f 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -54,9 +54,12 @@ macro transformOutImpl(lang: static LangDef, name, def: untyped) = let ret = def.params[0] def.body = newStmtList(def.body) def.body.insert 0, quote do: - # inject a build overload that implicitly uses the output language + # for convenience, inject a `build` macro overload that uses the + # result type template build(body: untyped): untyped {.used.} = - build(`to`, `ret`, body) + build(`to`, `ret`, NoSLoc, body) + template build(info: SLocRef, body: untyped): untyped {.used.} = + build(`to`, `ret`, info, body) result = def @@ -160,12 +163,21 @@ macro transformerImpl(sclass, dclass: static TypeClass, result = newStmtList(result) if dclass in {tcProduction, tcRecord}: + # for convenience, inject a `build` macro overload that uses the + # result type let to = ident"out.ast" - result.insert 0, quote do: - # inject a build macro overload that implicitly uses the - # target non-terminal - template build(body: untyped): untyped {.used.} = - build(`to`, typeof(result), body) + if sclass == tcProduction: + # a source location is available + let info = genSym"info" + result.insert 0, quote do: + let `info` = info(`param`) + template build(body: untyped): untyped {.used.} = + build(`to`, typeof(result), `info`, body) + else: + # the user has to provide a source location + result.insert 0, quote do: + template build(info: SLocRef, body: untyped): untyped {.used.} = + build(`to`, typeof(result), info, body) if sclass == dclass and sclass == tcProduction: if result[^1].kind == nnkCaseStmt: @@ -293,6 +305,8 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = unpack(`input`.storage[], v.index, typeof(T)) template get[N](r: RecordRef[src, N]): untyped {.used.} = get(`input`, r) + template info[N](n: Metavar[src, N]): untyped {.used.} = + `input`.tree[n.index].info template equal[N](a, b: Metavar[src, N]): bool {.used.} = equal(`input`.tree, Cursor(a.index), Cursor(b.index)) @@ -302,10 +316,10 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = body.add quote do: template terminal(x: untyped): untyped {.used.} = `embed`(`output`.storage, x) - template build[N](n: typedesc[Metavar[dst, N]], body: untyped): untyped {.used.} = - build(`output`, n, body) - template build[N](n: typedesc[RecordRef[dst, N]], body: untyped): untyped {.used.} = - build(`output`, n, body) + template build[N](n: typedesc[Metavar[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = + build(`output`, n, info, body) + template build[N](n: typedesc[RecordRef[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = + build(`output`, n, info, body) template match[N](sel: Metavar[dst, N], branches: varargs[untyped]): untyped {.used.} = match[dst, N](`output`.tree, IndCursor(sel.index), sel, branches) template slice[N](T: typedesc[Metavar[dst, N]]): typedesc {.used.} = @@ -315,10 +329,22 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template get[N](r: RecordRef[dst, N]): untyped {.used.} = get(`output`, r) + template info[N](n: Metavar[dst, N]): untyped {.used.} = + `output`.tree[n.index].info template equal[N](a, b: Metavar[dst, N]): bool {.used.} = equal(`output`.tree, IndCursor(a.index), IndCursor(b.index)) + # the source location accessors are always available + if hasIn: + body.add quote do: + template file(at: SLocRef): lent string {.used.} = file(`input`, at) + template span(at: SLocRef): untyped {.used.} = span(`input`, at) + else: + body.add quote do: + template file(at: SLocRef): lent string {.used.} = file(`output`, at) + template span(at: SLocRef): untyped {.used.} = span(`output`, at) + if hasIn: # shadow the input tree with a cursor to prevent a costly copy when # it's captured by the closure diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 93e22112..180ba6f9 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -21,8 +21,9 @@ proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = else: result = None -proc append(to: var Tree, i: var int, tag: uint8, val: uint32) {.inline.} = - to.nodes[i] = node(tag, val) +proc append(to: var Tree, i: var int, tag: uint8, info: SLocRef, + val: uint32) {.inline.} = + to.nodes[i] = node(tag, info, val) inc i macro transform*(src, dst: static LangInfo, nterm: static string, @@ -74,30 +75,33 @@ macro transform*(src, dst: static LangInfo, nterm: static string, # add the root node: let body = quote do: let len = `input`.len(pos(`cursor`)) - discard enter(`input`, `cursor`) let root = `output`.nodes.len.NodeIndex var i = `output`.nodes.len # the node sequence has to be contiguous, so it's allocated upfront `output`.nodes.setLen(i + len + 1) - `output`.nodes[i] = node(`id`, uint32(len)) + `output`.nodes[i] = node(`id`, `input`[pos(`cursor`)].info, uint32(len)) inc i + discard enter(`input`, `cursor`) # call the transformers and emit the nodes in one go: for i, a in src.forms[form].elems.pairs: let b = dst.forms[target].elems[i] let s = ident(src.types[a.typ].mvar) let d = ident(dst.types[b.typ].mvar) + let pos = + case src.types[a.typ].kind + of tkTerminal, tkRecord: + quote do: pos(`cursor`) + of tkNonTerminal: + quote do: get(`input`, `cursor`) let got = case src.types[a.typ].kind of tkTerminal: - quote do: - src.`s`(index: `input`[pos(`cursor`)].val) + quote do: src.`s`(index: `input`[`pos`].val) of tkRecord: - quote do: - src.`s`(id: `input`[pos(`cursor`)].val) + quote do: src.`s`(id: `input`[`pos`].val) of tkNonTerminal: - quote do: - src.`s`(index: get(`input`, `cursor`)) + quote do: src.`s`(index: `pos`) let append = bindSym"append" let call = @@ -105,14 +109,17 @@ macro transform*(src, dst: static LangInfo, nterm: static string, of tkTerminal: let tag = dst.types[b.typ].ntag quote do: - `append`(`output`, i, uint8(`tag`), (`got` -> dst.`d`).index) + `append`(`output`, i, uint8(`tag`), `input`[`pos`].info, + (`got` -> dst.`d`).index) of tkRecord: let tag = dst.types[b.typ].rtag quote do: - `append`(`output`, i, uint8(`tag`), (`got` -> dst.`d`).id) + `append`(`output`, i, uint8(`tag`), `input`[`pos`].info, + (`got` -> dst.`d`).id) of tkNonTerminal: quote do: - `append`(`output`, i, RefTag, (`got` -> dst.`d`).index.uint32) + `append`(`output`, i, RefTag, NoSLoc, + (`got` -> dst.`d`).index.uint32) if a.repeat: let bias = src.forms[form].elems.len - 1 @@ -148,10 +155,10 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, case src.types[typ].kind of tkTerminal: quote do: - src.`smvar`(index: `input`[get(`input`, `cursor`)].val) + src.`smvar`(index: `input`[pos(`cursor`)].val) of tkRecord: quote do: - src.`smvar`(id: `input`[get(`input`, `cursor`)].val) + src.`smvar`(id: `input`[pos(`cursor`)].val) of tkNonTerminal: quote do: src.`smvar`(index: get(`input`, `cursor`)) @@ -168,8 +175,9 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let tag = dst.types[dtyp].ntag.uint8 let tmp = genSym() result = quote do: + let info = `input`[pos(`cursor`)].info let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add node(`tag`, `tmp`.index) + `output`.nodes.add node(`tag`, info, `tmp`.index) NodeIndex(`output`.nodes.high) of tkRecord: # a new node needs to be allocated so that a reference to it can @@ -177,8 +185,9 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, let tag = dst.types[dtyp].rtag.uint8 let tmp = genSym() result = quote do: + let info = `input`[pos(`cursor`)].info let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add node(`tag`, `tmp`.id) + `output`.nodes.add node(`tag`, info, `tmp`.id) NodeIndex(`output`.nodes.high) of tkNonTerminal: result = quote do: diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 0fb6b6b9..17542ee9 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -18,6 +18,8 @@ type pos: int ## current node position records: seq[IntSet] ## for each record type, keeps track of the already-defined records + curSLoc: SLocRef + ## currently active source location proc nameToIndex[L; Name: static string](): int {.compileTime.} = ## Turns a record type name to the index of the corresponding set in @@ -32,6 +34,32 @@ proc nameToIndex[L; Name: static string](): int {.compileTime.} = return i inc i +proc wrap(c: var Ctx, ast: Ast, info: SLocRef, n: sink SexpNode): SexpNode = + ## Wraps `n` in an ``:info`` or ``:no-info`` decorator expression where + ## necessary for `n` to use `info` as its source location. + if info == c.curSLoc: + n + elif info != NoSLoc: + let (start, fin) = span(ast, info) + if start == fin: + newSList([newSSymbol(":info"), + newSList([ + newSString(file(ast, info)), + newSInt(BiggestInt start.line), + newSInt(BiggestInt start.column)]), + n]) + else: + newSList([newSSymbol(":info"), + newSList([ + newSString(file(ast, info)), + newSInt(BiggestInt start.line), + newSInt(BiggestInt start.column), + newSInt(BiggestInt fin.line), + newSInt(BiggestInt fin.column)]), + n]) + else: + newSList([newSSymbol(":no-info"), n]) + proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, @@ -66,6 +94,12 @@ macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = ## Unparses the non-terminal with name `nterm` from the given language at ## the current cursor position `pos`. let id = def.map[nterm] + result = newStmtList() + result.add quote do: + let prev = `c`.curSLoc + let info = `ast`.tree.nodes[`c`.pos].info + `c`.curSLoc = info + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`c`.pos].tag) proc genForType(def: LangInfo, typ: LangType): NimNode = @@ -131,7 +165,11 @@ macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = result = newSList( [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`c`.pos].tag)]) `c`.pos = `ast`.tree.next(NodeIndex `c`.pos).int) - result = caseStmt + result.add caseStmt + let wrap = bindSym"wrap" + result.add quote do: + `c`.curSLoc = prev + result = `wrap`(`c`, `ast`, info, result) proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode = ## Unparses the non-terminal at `pos`, returning the corresponding From 1c1d0956f992ff4662a5d9ba2f9a4a497aa1efba Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 67/87] literals: implement `SourceLoc` storage --- passes/literals.nim | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/passes/literals.nim b/passes/literals.nim index 3dafe951..a533be14 100644 --- a/passes/literals.nim +++ b/passes/literals.nim @@ -1,12 +1,15 @@ ## Implements the storage for literal data embedded in ASTs. -# TODO: use bi-tables for the number and string values +import nanopass/asts + +# TODO: use bi-tables for the number, string, and source location values type Literals* = object ## Storage container for literal data, to be used with nanopass ASTs. numbers*: seq[uint64] ## a list of bit patterns strings*: seq[string] + locs*: seq[SourceLoc] const OverflowShift = 31 @@ -58,10 +61,12 @@ proc unpack*(s: Literals, id: uint32, _: typedesc[string]): lent string {.inline ## Returns the string stored under `id`. s.strings[id] -# TODO: remove the temporary overloads for objects again - -proc pack*[T: object](s: Literals, val: T): uint32 = - result = 0 +proc pack*(s: var Literals, val: SourceLoc): uint32 {.inline.} = + ## Adds `val` to the database and returns the ID to later look up `val` with. + result = uint32(s.locs.len) + s.locs.add val -proc unpack*[T: object](s: Literals, id: uint32, _: typedesc[T]): T = - discard +proc unpack*(s: Literals, id: uint32, _: typedesc[SourceLoc] + ): SourceLoc {.inline.} = + ## Returns the source location earlier stored under `id`. + s.locs[id] From da74df43c335b23903894f61c13cccb251941bc3 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 68/87] asts: rename `Metavar` to `Production` --- nanopass/asts.nim | 12 ++++------ nanopass/nanopass.nim | 2 +- nanopass/npbuild.nim | 6 ++--- nanopass/nplanggen.nim | 6 ++--- nanopass/npmatch.nim | 4 ++-- nanopass/npparser.nim | 9 +++---- nanopass/nppass.nim | 52 ++++++++++++++++++++--------------------- nanopass/nppatterns.nim | 2 +- nanopass/npunparser.nim | 4 ++-- 9 files changed, 48 insertions(+), 49 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index e913d629..5870f65a 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -34,11 +34,9 @@ type records*: typeof(L.meta.records) ## leaked implementation detail, don't use - Metavar*[L: object, N: static string] = object + Production*[L: object, N: static string] = object ## Represents a reference to an AST fragment that's a production of non- ## terminal `N` of language `L`. - # TODO: rename to NonTerminal (currently clashes with the type of the same - # name in `nanopass.nim`) index*: NodeIndex ## leaked implementation detail, don't use @@ -53,7 +51,7 @@ type id*: uint32 ## leaked implementation detail, don't use - ChildSlice*[T: Metavar or RecordRef or Value, Cursor] = object + ChildSlice*[T: Production or RecordRef or Value, Cursor] = object ## A lightweight reference to a sequence of sibling nodes. The reference ## must not outlive the spawned-from tree. tree: ptr Tree @@ -171,9 +169,9 @@ proc slice*[T, C](tree: ptr Tree, start: C, len: uint32): ChildSlice[T, C] = template load[T, C](tree: Tree, c: C): T = mixin get, pos - when T is Metavar: T(index: get(tree, c)) - elif T is RecordRef: T(id: tree[pos(c)].val) - else: T(index: tree[pos(c)].val) + when T is Production: T(index: get(tree, c)) + elif T is RecordRef: T(id: tree[pos(c)].val) + else: T(index: tree[pos(c)].val) iterator items*[T, C](s: ChildSlice[T, C]): T = mixin advance diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index a8bd321e..3aeed420 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -77,7 +77,7 @@ proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = for s in fields(ast.records): for tup in s.mitems: for it in fields(tup): - when it is Metavar: + when it is Production: let got = output.nodes.len resolve(ast.tree, output, it.index) it.index = NodeIndex(got) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index c0d0d732..58b6a7df 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -104,7 +104,7 @@ proc append[L](ast: var Ast[L, auto], info: SLocRef, x: RecordRef) = info, x.id) -proc append[L](ast: var Ast[L, auto], info: SLocRef, x: Metavar) = +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: Production) = # the ref itself doesn't need source location info ast.tree.nodes.add node(RefTag, uint32(x.index)) @@ -611,10 +611,10 @@ macro buildFirstPass(ast, mv, info, e: untyped): untyped = result.add quote do: `impl`(idef(`mv`.L), `mv`.N, `ast`, `tmp`, `e`) -template build*[L, N](ast: var Ast[L, auto], t: typedesc[Metavar[L, N]], +template build*[L, N](ast: var Ast[L, auto], t: typedesc[Production[L, N]], info: SLocRef, e: untyped): untyped = ## Evaluates the AST construction expression `e`, whose result must be a - ## production of `mv`, returning a `Metavar` pointing to the created AST + ## production of `mv`, returning a `Production` pointing to the created AST ## fragment. buildFirstPass(ast, t, info, e) diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim index a9f46090..a4aa5c0d 100644 --- a/nanopass/nplanggen.nim +++ b/nanopass/nplanggen.nim @@ -13,7 +13,7 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = let fields = nnkRecList.newTree() # the metavars are at top level of the type, for easy access by # the programmer - let mvar = bindSym"Metavar" + let prod = bindSym"Production" for name, it in def.terminals.pairs: for m in it.mvars.items: fields.add newIdentDefs(ident(m), @@ -22,7 +22,7 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = for m in it.mvars.items: fields.add newIdentDefs(ident(m), nnkBracketExpr.newTree( - mvar, + prod, ident(typName.strVal), newStrLitNode(name))) for name, it in def.records.pairs: @@ -55,7 +55,7 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = # in auto-complete suggestions let metaType = nnkTupleTy.newTree( newIdentDefs(ident"entry", - nnkBracketExpr.newTree(mvar, + nnkBracketExpr.newTree(prod, ident(typName.strVal), newStrLitNode(def.entry))), newIdentDefs(ident"nt", ntType)) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 8779ed24..18aedb66 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -877,7 +877,7 @@ template match*[L; N: static](ast: Tree, cursor, info: untyped, bind matchImpl matchImpl(idef(typeof(L)), N, typeof(L), ast, cursor, info, branches) -template match*[L, N](ast: Ast[L, auto], nt: Metavar[L, N], +template match*[L, N](ast: Ast[L, auto], p: Production[L, N], branches: varargs[untyped]): untyped = ## Provides a convenient way to destructure an AST. Meant to be used as ## follows: @@ -889,4 +889,4 @@ template match*[L, N](ast: Ast[L, auto], nt: Metavar[L, N], ## of ...: discard ## else: discard bind matchImpl - matchImpl(idef(typeof(L)), N, L, ast.tree, Cursor(nt.index), nt, branches) + matchImpl(idef(typeof(L)), N, L, ast.tree, Cursor(p.index), p, branches) diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index 0b70e0ef..eb786fb4 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -80,7 +80,7 @@ template check[L](c: Ctx[L, auto], pos: NodeIndex, line, col: int, ## Makes sure the production at `pos` is one of the non-terminal identified ## by `nterm`, raising an error if not. if c.tree[pos].tag notin tags(idef(typeof(L)), mapType[L, t]()): - when t is Metavar: + when t is Production: raiseError(line, col, "expected production of '" & t.N & "'") else: @@ -110,7 +110,7 @@ proc parseTerminal[L, S](c: var Ctx[L, S], node: SexpNode, proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) -proc extract(c: var Ctx, to: var Metavar, pos: int) = +proc extract(c: var Ctx, to: var Production, pos: int) = # move the sub-tree over to the out-of-band storage let start = c.staging.nodes.len let count = c.tree.nodes.len - pos @@ -452,7 +452,8 @@ proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) = let (line, col) = (p.getLine(), p.getColumn()) c.tree.nodes.add parseTerminal(c, parseSexp(p), line, col) -proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Metavar[L, N]]): (Ast[L, S], T) = +proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Production[L, N]] + ): (Ast[L, S], T) = ## Parses the S-expression-based AST representation from `p` into an `Ast`, ## returning the result, or - in case of an error - raising an exception. var c: Ctx[L, S] @@ -470,7 +471,7 @@ proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Metavar[L, N]]): (Ast[L, for recs in fields(c.records): for it in recs.mitems: for f in fields(it): - when f is Metavar: + when f is Production: f.index = NodeIndex(pos + ord(f.index)) result = ( diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 0a379f3f..e455f5c3 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -22,10 +22,10 @@ macro ctError(str: string, info: untyped) = nnkPragma.newTree(nnkExprColonExpr.newTree(ident"error", str)) template classify(x: typedesc): TypeClass = - when x is Value: tcValue - elif x is RecordRef: tcRecord - elif x is Metavar: tcProduction - else: tcValue + when x is Value: tcValue + elif x is RecordRef: tcRecord + elif x is Production: tcProduction + else: tcNone template embed(storage, arg: untyped): untyped = ## Implements terminal value construction. @@ -132,19 +132,19 @@ template withCache(to: typedesc, inp, body: untyped): untyped = var res: to withValue c[], inp.id, val: res = - when to is Value: to(index: val[]) - elif to is RecordRef: to(id: val[]) - elif to is Metavar: to(index: NodeIndex(val[])) - else: {.error.} + when to is Value: to(index: val[]) + elif to is RecordRef: to(id: val[]) + elif to is Production: to(index: NodeIndex(val[])) + else: {.error.} do: # TODO: `body` should be wrapped in a lambda, so that `return` doesn't # disables adding to the table let val = if true: body else: default(to) c[][inp.id] = - when to is Value: val.index - elif to is RecordRef: val.id - elif to is Metavar: val.index.uint32 - else: {.error.} + when to is Value: val.index + elif to is RecordRef: val.id + elif to is Production: val.index.uint32 + else: {.error.} res = val res @@ -289,10 +289,10 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = if hasIn: body.add quote do: - template match[N](sel: Metavar[src, N], branches: varargs[untyped]): untyped {.used.} = + template match[N](sel: Production[src, N], branches: varargs[untyped]): untyped {.used.} = match[src, N](`input`.tree, Cursor(sel.index), sel, branches) - template slice[N](T: typedesc[Metavar[src, N]]): typedesc {.used.} = + template slice[N](T: typedesc[Production[src, N]]): typedesc {.used.} = ChildSlice[T, Cursor] template slice[N](T: typedesc[RecordRef[src, N]]): typedesc {.used.} = ChildSlice[T, Cursor] @@ -305,10 +305,10 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = unpack(`input`.storage[], v.index, typeof(T)) template get[N](r: RecordRef[src, N]): untyped {.used.} = get(`input`, r) - template info[N](n: Metavar[src, N]): untyped {.used.} = + template info[N](n: Production[src, N]): untyped {.used.} = `input`.tree[n.index].info - template equal[N](a, b: Metavar[src, N]): bool {.used.} = + template equal[N](a, b: Production[src, N]): bool {.used.} = equal(`input`.tree, Cursor(a.index), Cursor(b.index)) if hasOut: @@ -316,23 +316,23 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = body.add quote do: template terminal(x: untyped): untyped {.used.} = `embed`(`output`.storage, x) - template build[N](n: typedesc[Metavar[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = + template build[N](n: typedesc[Production[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = build(`output`, n, info, body) template build[N](n: typedesc[RecordRef[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = build(`output`, n, info, body) - template match[N](sel: Metavar[dst, N], branches: varargs[untyped]): untyped {.used.} = + template match[N](sel: Production[dst, N], branches: varargs[untyped]): untyped {.used.} = match[dst, N](`output`.tree, IndCursor(sel.index), sel, branches) - template slice[N](T: typedesc[Metavar[dst, N]]): typedesc {.used.} = + template slice[N](T: typedesc[Production[dst, N]]): typedesc {.used.} = ChildSlice[T, IndCursor] template slice[N](T: typedesc[RecordRef[dst, N]]): typedesc {.used.} = ChildSlice[T, IndCursor] template get[N](r: RecordRef[dst, N]): untyped {.used.} = get(`output`, r) - template info[N](n: Metavar[dst, N]): untyped {.used.} = + template info[N](n: Production[dst, N]): untyped {.used.} = `output`.tree[n.index].info - template equal[N](a, b: Metavar[dst, N]): bool {.used.} = + template equal[N](a, b: Production[dst, N]): bool {.used.} = equal(`output`.tree, IndCursor(a.index), IndCursor(b.index)) # the source location accessors are always available @@ -394,7 +394,7 @@ template defineProcessors(dst: untyped) = template `->`[T](v: Value[T], _: typedesc[Value[T]]): Value[T] {.inject.} = v # nothing to do - proc `->`[X](n: Metavar, T: typedesc[Metavar[dst, X]]): T {.inject.} = + proc `->`[X](n: Production, T: typedesc[Production[dst, X]]): T {.inject.} = # note: the signature is overly broad so that overload resolution # prefers the more specific adapters created for the programmer-provided # processors @@ -528,11 +528,11 @@ macro outpassImpl(name, nterm: typedesc, def: untyped) = result = assemblePass(name, nil, def, call) -template nterm(x: typedesc[Metavar]): typedesc = x -template nterm(x: typedesc): typedesc = x.meta.entry +template nterm(x: typedesc[Production]): typedesc = x +template nterm(x: typedesc): typedesc = x.meta.entry -template lang(x: typedesc[Metavar]): typedesc = x.L -template lang(x: typedesc): typedesc = x +template lang(x: typedesc[Production]): typedesc = x.L +template lang(x: typedesc): typedesc = x macro inpass*(p: untyped) = ## Turns a procedure definition into a pass that takes arbitrary data as diff --git a/nanopass/nppatterns.nim b/nanopass/nppatterns.nim index 5920a8ef..0030520d 100644 --- a/nanopass/nppatterns.nim +++ b/nanopass/nppatterns.nim @@ -31,7 +31,7 @@ template matches*[T, U](x: T, _: typedesc[U]): bool = true else: matches(x, typeof(U.B)) - elif U is Metavar: + elif U is Production: when x is U: true else: diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 17542ee9..8971f897 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -79,7 +79,7 @@ proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, toSexp(unpack(ast.storage[], val.index, typeof(val).T)) elif val is RecordRef: unparse[typeof(val).N](ast, c, val.id.int, get(ast, val)) - elif val is Metavar: + elif val is Production: let prev = c.pos c.pos = val.index.int let r = unparse[typeof(val).N](ast, c) @@ -178,7 +178,7 @@ proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode = mixin idef unparse(idef(typeof(ast).L), N, ast, c) -proc unparse*[L, S, N](ast: Ast[L, S], at: Metavar[L, N]): SexpNode = +proc unparse*[L, S, N](ast: Ast[L, S], at: Production[L, N]): SexpNode = ## Unparses the production at the given position `at`, returning it as a ## self-contained S-expression. var c = Ctx(pos: at.index.int) From dfd47468f0dc49a080763f758d075c8461eb852b Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 69/87] asts: renamed `Value` field `index` to `id` The value stored by the field is more akin to an ID, hence the rename. --- nanopass/asts.nim | 4 ++-- nanopass/npbuild.nim | 6 +++--- nanopass/npmatch.nim | 2 +- nanopass/npparser.nim | 2 +- nanopass/nppass.nim | 8 ++++---- nanopass/nptransform.nim | 8 ++++---- nanopass/npunparser.nim | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 5870f65a..4a150e6d 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -43,7 +43,7 @@ type Value*[T] = object ## Represents a reference to a value with type `T` that's a terminal in ## an AST. - index*: uint32 + id*: uint32 ## leaked implementation detail, don't use RecordRef*[L: object, N: static string] = object @@ -171,7 +171,7 @@ template load[T, C](tree: Tree, c: C): T = mixin get, pos when T is Production: T(index: get(tree, c)) elif T is RecordRef: T(id: tree[pos(c)].val) - else: T(index: tree[pos(c)].val) + else: T(id: tree[pos(c)].val) iterator items*[T, C](s: ChildSlice[T, C]): T = mixin advance diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 58b6a7df..b0234225 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -96,7 +96,7 @@ proc append[L, U](ast: var Ast[L, auto], info: SLocRef, x: Value[U]) = ast.tree.nodes.add node( typeof(lookup[U, L.meta.term_map]()).V, info, - x.index) + x.id) proc append[L](ast: var Ast[L, auto], info: SLocRef, x: RecordRef) = ast.tree.nodes.add node( @@ -115,9 +115,9 @@ proc append[L](ast: var Ast[L, auto], info: SLocRef, x: openArray) = template coerce[S, T, U](s: S, val: U, _: typedesc[Value[T]]): Value[T] = mixin pack when T is U: - Value[T](index: pack(s, val)) # no coercion is necessary + Value[T](id: pack(s, val)) # no coercion is necessary else: - Value[T](index: pack(s, T(val))) # try a coercion, an error is fine + Value[T](id: pack(s, T(val))) # try a coercion, an error is fine {.pop.} diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 18aedb66..84f5113c 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -532,7 +532,7 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, else: case lang.types[typ.intVal].kind of tkTerminal: - quote do: `name`.`mvar`(index: `ast`[`pos`].val) + quote do: `name`.`mvar`(id: `ast`[`pos`].val) of tkRecord: quote do: `name`.`mvar`(id: `ast`[`pos`].val) of tkNonTerminal: diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index eb786fb4..845ae1ee 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -124,7 +124,7 @@ proc extract(c: var Ctx, to: var RecordRef, pos: int) = to.id = c.tree.nodes.pop().val proc extract(c: var Ctx, to: var Value, pos: int) = - to.index = c.tree.nodes.pop().val + to.id = c.tree.nodes.pop().val proc parseFieldsImpl[L](c: var Ctx[L, auto], p: var SexpParser, tup: var tuple) = diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index e455f5c3..38e75ed7 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -31,7 +31,7 @@ template embed(storage, arg: untyped): untyped = ## Implements terminal value construction. mixin pack let tmp = arg - Value[typeof(tmp)](index: pack(storage[], tmp)) + Value[typeof(tmp)](id: pack(storage[], tmp)) macro pick(lang: static LangInfo, name: static string, se: untyped): untyped = ## Returns the actual record identified by `r` in storage `storage`. @@ -132,7 +132,7 @@ template withCache(to: typedesc, inp, body: untyped): untyped = var res: to withValue c[], inp.id, val: res = - when to is Value: to(index: val[]) + when to is Value: to(id: val[]) elif to is RecordRef: to(id: val[]) elif to is Production: to(index: NodeIndex(val[])) else: {.error.} @@ -141,7 +141,7 @@ template withCache(to: typedesc, inp, body: untyped): untyped = # disables adding to the table let val = if true: body else: default(to) c[][inp.id] = - when to is Value: val.index + when to is Value: val.id elif to is RecordRef: val.id elif to is Production: val.index.uint32 else: {.error.} @@ -302,7 +302,7 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template val[T](v: nanopass.Value[T]): T {.used.} = # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) # XXX: consider renaming this template to `get` - unpack(`input`.storage[], v.index, typeof(T)) + unpack(`input`.storage[], v.id, typeof(T)) template get[N](r: RecordRef[src, N]): untyped {.used.} = get(`input`, r) template info[N](n: Production[src, N]): untyped {.used.} = diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index 180ba6f9..e9f55a31 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -97,7 +97,7 @@ macro transform*(src, dst: static LangInfo, nterm: static string, let got = case src.types[a.typ].kind of tkTerminal: - quote do: src.`s`(index: `input`[`pos`].val) + quote do: src.`s`(id: `input`[`pos`].val) of tkRecord: quote do: src.`s`(id: `input`[`pos`].val) of tkNonTerminal: @@ -110,7 +110,7 @@ macro transform*(src, dst: static LangInfo, nterm: static string, let tag = dst.types[b.typ].ntag quote do: `append`(`output`, i, uint8(`tag`), `input`[`pos`].info, - (`got` -> dst.`d`).index) + (`got` -> dst.`d`).id) of tkRecord: let tag = dst.types[b.typ].rtag quote do: @@ -155,7 +155,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, case src.types[typ].kind of tkTerminal: quote do: - src.`smvar`(index: `input`[pos(`cursor`)].val) + src.`smvar`(id: `input`[pos(`cursor`)].val) of tkRecord: quote do: src.`smvar`(id: `input`[pos(`cursor`)].val) @@ -177,7 +177,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, result = quote do: let info = `input`[pos(`cursor`)].info let `tmp` = `got` -> dst.`dmvar` - `output`.nodes.add node(`tag`, info, `tmp`.index) + `output`.nodes.add node(`tag`, info, `tmp`.id) NodeIndex(`output`.nodes.high) of tkRecord: # a new node needs to be allocated so that a reference to it can diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 8971f897..9ce1bfe7 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -76,7 +76,7 @@ proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, for name, val in fieldPairs(tup): let node = when val is Value: - toSexp(unpack(ast.storage[], val.index, typeof(val).T)) + toSexp(unpack(ast.storage[], val.id, typeof(val).T)) elif val is RecordRef: unparse[typeof(val).N](ast, c, val.id.int, get(ast, val)) elif val is Production: From 3b3fa6c8eb171b7d6632a06fb9950f9fbba1e02c Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 70/87] nanopass: move `resolve` to a separate module This allows keeping the routine private and not exposing it to importers of `nanopass`. --- nanopass/nanopass.nim | 59 ---------------------------------------- nanopass/nppass.nim | 10 ++++--- nanopass/npresolve.nim | 61 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 63 deletions(-) create mode 100644 nanopass/npresolve.nim diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 3aeed420..66d2e183 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -7,7 +7,6 @@ # * add a "compiler definition" macro import - passes/[trees], nanopass/[asts, nplanggen, npmatch, npbuild, npparser, nppass, nppatterns, npunparser] export asts @@ -26,61 +25,3 @@ macro defineLanguage*(name, base, body: untyped) = ## symbol with the given name. Extension doesn't imply a direction in this ## context. defineLanguageImpl(name, base, body) - -proc resolve(ast: Tree, result: var Tree, n: NodeIndex) = - ## Copies the AST fragment starting at `n` to `result`, resolving all - ## indirections in the process. - template src: untyped = ast.nodes - template dst: untyped = result.nodes - const size = sizeof(AstNode) - - template append(start, fin: uint32) = - let pos = dst.len - let num = int(fin - start) - dst.setLen(pos + num) - copyMem(addr dst[pos], addr src[start], num * size) - - # search for runs of contiguous nodes. When encountering an indirection, - # copy the run and move the source cursor to the indirection's - # target; repeat. - var stack = @[(uint32(n), uint32(n))] - while stack.len > 0: - block outer: - var (i, last) = stack[^1] - let prev = i - while i <= last: - if src[i].tag < RefTag: - last += src[i].val - elif src[i].tag == RefTag: - if i > prev: - # copy everything we got so far - append(prev, i) - - stack[^1] = (i + 1, last) - let next = src[i].val - stack.add (next, next) - break outer - - inc i - - if i > prev: - # copy the rest - append(prev, i) - - stack.shrink(stack.len - 1) - -proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = - ## Returns `ast` with all indirections in the AST resolved. - var output: Tree - resolve(ast.tree, output, n) - # also resolve the sub-trees referenced from records: - for s in fields(ast.records): - for tup in s.mitems: - for it in fields(tup): - when it is Production: - let got = output.nodes.len - resolve(ast.tree, output, it.index) - it.index = NodeIndex(got) - - ast.tree = output - result = ast diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 38e75ed7..e7f1b5b8 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -1,7 +1,7 @@ ## Implements the various pass macros. import std/[genasts, macros, packedsets, tables] -import nanopass/[asts, nplang, nplangdef, npmatch, nptransform] +import nanopass/[asts, nplang, nplangdef, npmatch, npresolve, nptransform] type TypeClass = enum tcNone, tcValue, tcRecord, tcProduction @@ -360,11 +360,13 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = else: body.add quote do: `output`.storage = new(`storageTy`) + + let resolve = bindSym"resolve" body.add quote do: let pos = `call` - # turn the AST with indirections into one without - `output` = resolve(move `output`, pos.index) - result = (move `output`, typeof(pos)(index: NodeIndex(0))) + # turn the AST with indirections into one without and return it + result = (`resolve`(move `output`, pos.index), + typeof(pos)(index: NodeIndex(0))) else: body.add quote do: result = `call` diff --git a/nanopass/npresolve.nim b/nanopass/npresolve.nim new file mode 100644 index 00000000..4f118b46 --- /dev/null +++ b/nanopass/npresolve.nim @@ -0,0 +1,61 @@ +## Implements the routines for post-processing ASTs produced by passes. + +import nanopass/[asts] + +proc resolve(ast: Tree, result: var Tree, n: NodeIndex) = + ## Copies the AST fragment starting at `n` to `result`, resolving all + ## indirections in the process. + template src: untyped = ast.nodes + template dst: untyped = result.nodes + const size = sizeof(AstNode) + + template append(start, fin: uint32) = + let pos = dst.len + let num = int(fin - start) + dst.setLen(pos + num) + copyMem(addr dst[pos], addr src[start], num * size) + + # search for runs of contiguous nodes. When encountering an indirection, + # copy the run and move the source cursor to the indirection's + # target; repeat. + var stack = @[(uint32(n), uint32(n))] + while stack.len > 0: + block outer: + var (i, last) = stack[^1] + let prev = i + while i <= last: + if src[i].tag < RefTag: + last += src[i].val + elif src[i].tag == RefTag: + if i > prev: + # copy everything we got so far + append(prev, i) + + stack[^1] = (i + 1, last) + let next = src[i].val + stack.add (next, next) + break outer + + inc i + + if i > prev: + # copy the rest + append(prev, i) + + stack.shrink(stack.len - 1) + +proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = + ## Returns `ast` with all indirections in the AST resolved. + var output: Tree + resolve(ast.tree, output, n) + # also resolve the sub-trees referenced from records: + for s in fields(ast.records): + for tup in s.mitems: + for it in fields(tup): + when it is Production: + let got = output.nodes.len + resolve(ast.tree, output, it.index) + it.index = NodeIndex(got) + + ast.tree = output + result = ast From 11771de62e9a13403e77ee386f660843bbe1ec0a Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 71/87] npbuild: don't use `copyLineInfoForTree` --- nanopass/npbuild.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index b0234225..77f9fc72 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -214,7 +214,8 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = result = quote do: {.error: "expected type fitting " & $`dst` & ", but got " & $typeof(`src`).} - copyLineInfoForTree(result, info) + # copy the line info to the pragma's operand + copyLineInfo(result[0][1], info) proc makeMatch(src, expect: NimNode): NimNode = let error = newMismatchError(src, expect, src) @@ -249,14 +250,14 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = let mvar = ident(typ.mvar) result = quote do: {.error: "expected '" & $`ast`.L.`mvar` & "', but got form".} - copyLineInfoForTree(result, n) + copyLineInfo(result[0][1], n) return of tkRecord: # expected a record, but the constructor can only be that of a form let mvar = ident(typ.mvar) result = quote do: {.error: "expected '" & $`ast`.L.`mvar` & "', but got form".} - copyLineInfoForTree(result, n) + copyLineInfo(result[0][1], n) return of tkNonTerminal: discard "all good" From f51902d8e97f497aaac34ab61b5e343af94bccb3 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 72/87] nplang: rename `tag` field of `Form` to `name` --- nanopass/nplang.nim | 2 +- nanopass/nplangdef.nim | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 4603a5b9..8350198a 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -96,7 +96,7 @@ proc buildLangInfo*(def: LangDef): LangInfo = # the subtype info for it in def.forms.items: result.forms.add SForm( - name: it.tag, + name: it.name, ntag: it.id, elems: mapIt(it.elems, (result.map[it.typ], it.repeat)) ) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 77f00a2c..dfa0f816 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -13,7 +13,7 @@ type Form* = object ## Semantic representation of a syntax form. - tag*: string # TODO: rename to name + name*: string id*: int ## the integer ID through which a tree node is identified as being an ## instance of the form @@ -97,7 +97,7 @@ template findIt[T](s: seq[T], predicate: untyped): untyped = r proc `$`(x: Form): string = - result = x.tag + result = x.name result.add "(" for i, it in x.elems.pairs: if i > 0: @@ -115,7 +115,7 @@ proc `==`(x, y: Elem): bool = proc `==`(x, y: Form): bool = ## Compares `x` and `y`, which must belong to the same language, for equality. - x.tag == y.tag and x.elems == y.elems + x.name == y.name and x.elems == y.elems proc checkName(target: LangDef, vars: Table[string, string], name: string, info: NimNode) = @@ -211,7 +211,7 @@ proc buildLanguage(add, sub: seq[NimNode], proc removeProd(def: var LangDef, n: NimNode, to: string) = proc find(def: LangDef, nt: NonTerminal, f: ParsedForm): int = for i, it in nt.forms.pairs: - if def.forms[it.semantic].tag == f.name and + if def.forms[it.semantic].name == f.name and it.vars.len == f.elems.len: block search: # compare the elements: @@ -384,7 +384,7 @@ proc buildLanguage(add, sub: seq[NimNode], proc addProd(def: var LangDef, n: NimNode, to: string) = proc addForm(def: var LangDef, p: ParsedForm): OrigForm = - var form = Form(tag: p.name, id: -1) # the ID is computed later + var form = Form(name: p.name, id: -1) # the ID is computed later for i, (name, repeat, info) in p.elems.pairs: if name notin vars: From 2baeeed40551b808f04ff59dc240ade42a3e16ee Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 73/87] nplanggen: move `entry` field to the top A preparation for `entry` becoming a non-terminal. --- nanopass/nplanggen.nim | 13 +++++++------ nanopass/nppass.nim | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim index a4aa5c0d..f2ee4883 100644 --- a/nanopass/nplanggen.nim +++ b/nanopass/nplanggen.nim @@ -32,6 +32,12 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = ident(typName.strVal), newStrLitNode(name))) + # add the entry non-terminal: + fields.add newIdentDefs(ident"entry", + nnkBracketExpr.newTree(prod, + ident(typName.strVal), + newStrLitNode(def.entry))) + let ntType = nnkTupleTy.newTree() let (csym, fsym) = (bindSym"PChoice", bindSym"PForm") # add the descriptions for the non-terminals @@ -53,12 +59,7 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = # everything meant for internal use is stored in an anonymous record in # the `meta` field, preventing name clashes and the fields showing up # in auto-complete suggestions - let metaType = nnkTupleTy.newTree( - newIdentDefs(ident"entry", - nnkBracketExpr.newTree(prod, - ident(typName.strVal), - newStrLitNode(def.entry))), - newIdentDefs(ident"nt", ntType)) + let metaType = nnkTupleTy.newTree() # create the terminal->tag map: let tup = nnkTupleConstr.newTree() diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index e7f1b5b8..ffaecfcb 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -531,7 +531,7 @@ macro outpassImpl(name, nterm: typedesc, def: untyped) = result = assemblePass(name, nil, def, call) template nterm(x: typedesc[Production]): typedesc = x -template nterm(x: typedesc): typedesc = x.meta.entry +template nterm(x: typedesc): typedesc = x.entry template lang(x: typedesc[Production]): typedesc = x.L template lang(x: typedesc): typedesc = x From 848094341d146cfd18a169d703ef94e37243853f Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 74/87] nplangdef: remove tag tracking from `LangDef` Tag tracking being part of the high-level `LangDef` is a layering violation, and the tag computation and storage is therefore removed from language definition processing. Instead, node tags are computed when constructing the `LangInfo` for a language, using tags derived from the entities' names so that tags for entities with the same name are equal across languages (which is of importance for some optimizations that might get used in the future). As a consequence, the language type creation has to be restructured, as creating the meta-type now requires access to the `LangInfo`. --- nanopass/nplang.nim | 34 ++++++++++-- nanopass/nplangdef.nim | 49 ++---------------- nanopass/nplanggen.nim | 114 ++++++++++++++++++++--------------------- 3 files changed, 89 insertions(+), 108 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 8350198a..77ea5509 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -1,8 +1,8 @@ ## Provides the query-focused types representing the language in the nanopass ## framework, as well as the routines for creating instances thereof. -import std/[sequtils, tables] -import nanopass/[nplangdef] +import std/[hashes, sequtils, tables] +import nanopass/[asts, nplangdef] type SForm* = object @@ -52,16 +52,40 @@ type Static*[V: static int] = distinct int ## Carrier for a compile-time known integer value. +proc add[T](s: var set[T], val: int): T = + ## Derives a value from `val` that's in range `T` and not yet present in `s`, + ## adding it to `s` and returning the value. + const Len = high(T) - low(T) + 1 + let v = val mod Len + result = v + low(T) + # increment the starting value until an unoccupied slot is found. This is + # similar to open-addressing in a hash table + let start = v + var pos = start + while result in s: + let next = (pos + 1) mod Len + if next == start: + raise ValueError.newException("no slots left") + pos = next + result = pos + low(T) + + s.incl(result) + proc buildLangInfo*(def: LangDef): LangInfo = ## Creates the pass-centric language representation for `def`. result.map = initTable[string, int](4) + var formTags: set[range[0 .. (int(RefTag) - 1)]] + ## tags for forms + var leafTags: set[(int(RefTag) + 1) .. 255] + ## tags for leaf nodes + for name, it in def.terminals.pairs: result.types.add LangType( name: name, mvar: it.mvars[0], kind: tkTerminal, - ntag: it.tag + ntag: add(leafTags, hash(name)) ) # add the name-to-type mappings: result.map[name] = high(result.types) @@ -85,7 +109,7 @@ proc buildLangInfo*(def: LangDef): LangInfo = name: name, mvar: it.mvars[0], kind: tkRecord, - rtag: it.tag + rtag: add(leafTags, hash(name)) ) # add the name-to-type mappings: result.map[name] = high(result.types) @@ -97,7 +121,7 @@ proc buildLangInfo*(def: LangDef): LangInfo = for it in def.forms.items: result.forms.add SForm( name: it.name, - ntag: it.id, + ntag: add(formTags, hash(it.name)), elems: mapIt(it.elems, (result.map[it.typ], it.repeat)) ) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index dfa0f816..7df0845b 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -1,7 +1,6 @@ ## Implements the language definition parsing and processing. import std/[macros, intsets, sets, strformat, tables] -from nanopass/asts import RefTag type # Core types capturing a defined language @@ -14,9 +13,6 @@ type Form* = object ## Semantic representation of a syntax form. name*: string - id*: int - ## the integer ID through which a tree node is identified as being an - ## instance of the form elems*: seq[Elem] OrigForm* = object @@ -29,9 +25,6 @@ type Terminal* = object mvars*: seq[string] ## the meta-variables for ranging over values of the type - tag*: int - ## the integer ID through which a tree node is identified as being - ## an instance of the terminal NonTerminal* = object mvars*: seq[string] @@ -44,9 +37,6 @@ type Record* = object mvars*: seq[string] ## the meta-variables for ranging over the record instances - tag*: int - ## the integer ID through which a tree node is identified as - ## storing a reference to an instance of the record fields*: seq[tuple[name, mvar, typ: string]] LangDef* = object @@ -82,10 +72,6 @@ type sub: seq[NimNode] add: seq[NimNode] -const - FirstTerminalTag* = RefTag + 1 - ## the start of the terminals' tag space - template findIt[T](s: seq[T], predicate: untyped): untyped = ## Version of ``find`` that allows providing an inline predicate, ## evaluated for every checked item. @@ -147,33 +133,6 @@ proc addForm(def: var LangDef, form: Form): int = def.forms.add form result = def.forms.high -proc computeNodeTags(def: var LangDef) = - ## Assigns node tags to forms and terminals. - var next = 0 - for it in def.forms.items: - next = max(it.id + 1, next) - # ^^ while simple, this does waste ID space - - for it in def.forms.mitems: - # TODO: report an error when the ID overflows the allowed range - if it.id == -1: - it.id = next - inc next - - next = int FirstTerminalTag - for it in def.terminals.values: - next = max(it.tag + 1, next) - - for it in def.terminals.mvalues: - if it.tag == -1: - it.tag = next - inc next - - for it in def.records.mvalues: - if it.tag == -1: - it.tag = next - inc next - proc buildLanguage(add, sub: seq[NimNode], def: seq[NonTerminalDef], records: seq[RecordDef], @@ -372,8 +331,7 @@ proc buildLanguage(add, sub: seq[NimNode], for it in add.items: let name = processTerminal(it) checkName(result, vars, name, it) - var tm = Terminal(tag: -1) - # the node tag is filled in later + var tm = Terminal() for i in 1.. 0 and name notin base.records: # it's a new record checkName(result, vars, name, it.name) - var rec = Record(tag: -1) # the tag is computed later + var rec = Record() for i in 1.. 0: + result.add newIdentDefs(ident"term_map", terminals) + if records.len > 0: + result.add newIdentDefs(ident"record_map", recMap) + + result.add newIdentDefs(ident"records", records) + +macro makeLanguageType(def: static LangDef, info: LangInfo, typName: untyped) = ## Creates the type representing the language defined by `def`. This is the ## type the nanopass-framework user passes around. ## @@ -38,64 +91,11 @@ macro makeLanguageType(def: static LangDef, typName: untyped) = ident(typName.strVal), newStrLitNode(def.entry))) - let ntType = nnkTupleTy.newTree() - let (csym, fsym) = (bindSym"PChoice", bindSym"PForm") - # add the descriptions for the non-terminals - for name, nt in def.nterminals.pairs: - let ln = ident(typName.strVal) - var p = ident"void" - for f in nt.forms.items: - let id = def.forms[f.semantic].id - p = quote do: - `csym`[`p`, `fsym`[`id`]] - - for v in nt.vars.items: - let id = ident(v) - p = quote do: - `csym`[`p`, `ln`.`id`] - - ntType.add newIdentDefs(ident(name), p) - # everything meant for internal use is stored in an anonymous record in # the `meta` field, preventing name clashes and the fields showing up # in auto-complete suggestions - let metaType = nnkTupleTy.newTree() - - # create the terminal->tag map: - let tup = nnkTupleConstr.newTree() - for name, it in def.terminals.pairs: - let n = it.tag - tup.add nnkTupleConstr.newTree( - ident(name), - nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(n))) - - metaType.add newIdentDefs(ident"term_map", tup) - - # create the record->tag map: - block: - let tup = nnkTupleConstr.newTree() - for it in def.records.values: - tup.add nnkTupleConstr.newTree( - newDotExpr(copyNimTree(typName), ident(it.mvars[0])), - nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(it.tag))) - - if tup.len > 0: - metaType.add newIdentDefs(ident"record_map", tup) - - # create the symbol storage type (a tuple of arrays-of-structs): - let st = nnkTupleTy.newTree() - for name, rec in def.records.pairs: - let tup = nnkTupleTy.newTree() - for (name, mvar, _) in rec.fields.items: - tup.add newIdentDefs(ident(name), newDotExpr(typName, ident(mvar))) - - # expose under the first meta-var there is for the type - st.add newIdentDefs(ident(rec.mvars[0]), - nnkBracketExpr.newTree(ident"seq", tup)) - - metaType.add newIdentDefs(ident"records", st) - - fields.add newIdentDefs(ident"meta", metaType) + fields.add newIdentDefs(ident"meta", + newCall(bindSym"makeMetaType", info, typName)) result = nnkTypeSection.newTree( nnkTypeDef.newTree( @@ -130,5 +130,5 @@ proc defineLanguageImpl*(name, base, body: NimNode): NimNode = const def = setup1 tmp = buildLangInfo(def) - makeLanguageType(def, name) + makeLanguageType(def, tmp, name) genHelpers(name, def, tmp) From e43025ee3f6411a59fdc3e2bd774194c930c1c58 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 75/87] nplangdef: implement proper duplicate production detection --- nanopass/nplangdef.nim | 46 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 7df0845b..2aee43ed 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -1,6 +1,6 @@ ## Implements the language definition parsing and processing. -import std/[macros, intsets, sets, strformat, tables] +import std/[macros, intsets, sets, strformat, strutils, tables] type # Core types capturing a defined language @@ -359,8 +359,7 @@ proc buildLanguage(add, sub: seq[NimNode], case n.kind of nnkCall: let got = def.addForm(parseForm(n)) - if def.nterminals[to].forms.findIt(it.semantic == got.semantic) != -1: - error(fmt"production is already part of '{to}'", n) + # duplicate productions are checked for later def.nterminals[to].forms.add got of nnkIdent: let name = n.strVal @@ -431,6 +430,47 @@ proc buildLanguage(add, sub: seq[NimNode], result.records[name] = record + # make sure all non-terminals are well-formed, which is possible only once + # all production additions were made + for name, nt in result.nterminals.pairs: + proc gather(def: LangDef, top, name: string, used: var IntSet, + included: var seq[string]) = + if name == top: + error(fmt"non-terminal '{top}' includes itself", info) + else: + if name notin included: + included.add name + + if name in def.nterminals: + for it in def.nterminals[name].vars.items: + gather(def, top, vars[it], used, included) + for it in def.nterminals[name].forms.items: + used.incl(it.semantic) + + var used = initIntSet() + var included = newSeq[string]() + + for v in nt.vars.items: + var gotUsed: IntSet + var gotIncluded: seq[string] + gather(result, name, vars[v], gotUsed, gotIncluded) + # add the gathered sets to the total sets: + for it in gotUsed.items: + if containsOrIncl(used, it): + error("duplicate production '$1' in non-terminal '$2'" % + [$result.forms[it], name], info) + + for it in gotIncluded.items: + if it in included: + error("duplicate production '$1' in non-terminal '$2'" % + [it, name], info) + included.add it + + for it in nt.forms.items: + if containsOrIncl(used, it.semantic): + error("duplicate production '$1' in non-terminal '$2'" % + [$result.forms[it.semantic], name], info) + # TODO: properly set the entry non-terminal result.entry = "module" From a16c5c25ecaf4d540161383c0eedd0b557649f13 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 76/87] nplanggen: fix errors pointing to the wrong location --- nanopass/nplangdef.nim | 12 +++++------- nanopass/nplanggen.nim | 14 +++++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 2aee43ed..f5383163 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -434,12 +434,11 @@ proc buildLanguage(add, sub: seq[NimNode], # all production additions were made for name, nt in result.nterminals.pairs: proc gather(def: LangDef, top, name: string, used: var IntSet, - included: var seq[string]) = + included: var HashSet[string]) = if name == top: error(fmt"non-terminal '{top}' includes itself", info) else: - if name notin included: - included.add name + included.incl(name) if name in def.nterminals: for it in def.nterminals[name].vars.items: @@ -448,11 +447,11 @@ proc buildLanguage(add, sub: seq[NimNode], used.incl(it.semantic) var used = initIntSet() - var included = newSeq[string]() + var included: HashSet[string] for v in nt.vars.items: var gotUsed: IntSet - var gotIncluded: seq[string] + var gotIncluded: HashSet[string] gather(result, name, vars[v], gotUsed, gotIncluded) # add the gathered sets to the total sets: for it in gotUsed.items: @@ -461,10 +460,9 @@ proc buildLanguage(add, sub: seq[NimNode], [$result.forms[it], name], info) for it in gotIncluded.items: - if it in included: + if containsOrIncl(included, it): error("duplicate production '$1' in non-terminal '$2'" % [it, name], info) - included.add it for it in nt.forms.items: if containsOrIncl(used, it.semantic): diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim index 6cb1f10d..8dcab37f 100644 --- a/nanopass/nplanggen.nim +++ b/nanopass/nplanggen.nim @@ -121,14 +121,18 @@ proc defineLanguageImpl*(name, base, body: NimNode): NimNode = if body[0].kind == nnkCommentStmt: body.del(0) - let setup1 = + # don't use genAst for creating the makeLanguage call, as it messes with the + # source location + let setup = if base.isNil: - genAst(body): makeLanguage(quote do: body) + newCall(bindSym"makeLanguage", newCall(bindSym"quote", body)) else: - genAst(body, base): makeLanguage(def(base), quote do: body) - result = genAst(setup1, name): + newCall(bindSym"makeLanguage", + newCall(ident"def", base), + newCall(bindSym"quote", body)) + result = genAst(setup, name): const - def = setup1 + def = setup tmp = buildLangInfo(def) makeLanguageType(def, tmp, name) genHelpers(name, def, tmp) From f8f876bf91bb05e40e65b4a17409fedc4e8040d6 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 77/87] nplangdef: implement entry point configuration --- nanopass/nplangdef.nim | 58 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index f5383163..d56bdc4f 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -105,7 +105,9 @@ proc `==`(x, y: Form): bool = proc checkName(target: LangDef, vars: Table[string, string], name: string, info: NimNode) = - if name in target.terminals: + if name == "entry": + error("cannot use name 'entry'; it's a reserved name", info) + elif name in target.terminals: error(fmt"terminal with name {name} already exists", info) elif name in target.records: error(fmt"record with name '{name}' already exists", info) @@ -136,6 +138,7 @@ proc addForm(def: var LangDef, form: Form): int = proc buildLanguage(add, sub: seq[NimNode], def: seq[NonTerminalDef], records: seq[RecordDef], + config: seq[NimNode], base: LangDef, info: NimNode): LangDef = ## The center-piece of language definition construction. Constructs a ## language definition by applying the diff for terminals (`add` and `sub`) @@ -327,6 +330,10 @@ proc buildLanguage(add, sub: seq[NimNode], result.records[name] = res + # inherit the entry point: + if base.entry != "" and base.entry in base.nterminals: + result.entry = base.entry + # ---- phase 2: make all additions for it in add.items: let name = processTerminal(it) @@ -469,15 +476,32 @@ proc buildLanguage(add, sub: seq[NimNode], error("duplicate production '$1' in non-terminal '$2'" % [$result.forms[it.semantic], name], info) - # TODO: properly set the entry non-terminal - result.entry = "module" + # process the extra configuration declarations: + for it in config.items: + it[0].expectKind nnkIdent + case it[0].strVal + of "entry": + it[1].expectKind nnkIdent + let entry = it[1].strVal + if entry notin result.nterminals: + error(fmt"no non-terminal with the name '{entry}' exists", it[1]) + result.entry = entry + else: + error("identifier must be 'entry'", it[0]) + + if result.entry == "": + if def.len == 0: + error("cannot infer entry non-terminal, as none exists", info) + # use the last-defined non-terminal as the entry point + result.entry = def[^1].name[0].strVal proc makeLanguage*(body: NimNode): LangDef = ## Creates a language definition from the ``defineLanguage`` DSL code. body.expectMinLen 1 - var add: seq[NimNode] - var def: seq[NonTerminalDef] + var terminals: seq[NimNode] + var nterminals: seq[NonTerminalDef] var records: seq[RecordDef] + var config: seq[NimNode] # second pass: process the productions proc extract(n: NimNode, list: var seq[NimNode]) = @@ -507,10 +531,13 @@ proc makeLanguage*(body: NimNode): LangDef = else: var nt = NonTerminalDef(name: it[1]) extract(it[2], nt.add) - def.add nt + nterminals.add nt continue of nnkCall: - add.add it + terminals.add it + continue + of nnkAsgn: + config.add it continue else: discard "report an error below" @@ -519,14 +546,16 @@ proc makeLanguage*(body: NimNode): LangDef = # to keep the implementation simple, a non-extension language is treated # internally as an empty language definition being extended - buildLanguage(add, @[], def, records, default(LangDef), body) + buildLanguage(terminals, @[], nterminals, records, config, default(LangDef), + body) proc makeLanguage*(base: LangDef, body: NimNode): LangDef = ## Creates a language definition from the ``defineLanguage`` DSL code ## and `base`. var add, sub: seq[NimNode] - var def: seq[NonTerminalDef] + var nterminals: seq[NonTerminalDef] var records: seq[RecordDef] + var config: seq[NimNode] var body = body if body.kind != nnkStmtList: @@ -576,7 +605,7 @@ proc makeLanguage*(base: LangDef, body: NimNode): LangDef = else: var nt = NonTerminalDef(name: it[1]) extract(it[2], nt.add, nt.sub) - def.add nt + nterminals.add nt handled = true of nnkPrefix: @@ -588,12 +617,15 @@ proc makeLanguage*(base: LangDef, body: NimNode): LangDef = handled = true of nnkCall: # non-terminal with no change in productions - def.add NonTerminalDef(name: it) + nterminals.add NonTerminalDef(name: it) + handled = true + of nnkAsgn: + config.add it handled = true else: discard if not handled: - error("expected `-a`, `+a`, or `a(...) ::= ...`", it[0]) + error("expected `-a`, `+a`, `a(...) ::= ...`, or `a = ...", it[0]) - buildLanguage(add, sub, def, records, base, body) + buildLanguage(add, sub, nterminals, records, config, base, body) From e84445a69d48a56ad2d960ca4e90aaebe1d2d8c5 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 78/87] nplangdef: disallow changing the meaning of type productions --- nanopass/nplang.nim | 2 +- nanopass/nplangdef.nim | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 77ea5509..71596fea 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -128,7 +128,7 @@ proc buildLangInfo*(def: LangDef): LangInfo = for name, it in def.nterminals.pairs: let id = result.map[name] for v in it.vars.items: - result.types[id].sub.add result.map[v] + result.types[id].sub.add result.map[v.mvar] for name, it in def.records.pairs: let id = result.map[name] diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index d56bdc4f..94bd1f8c 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -29,7 +29,7 @@ type NonTerminal* = object mvars*: seq[string] ## the meta-variables for ranging over the productions - vars*: seq[string] + vars*: seq[tuple[mvar, typ: string]] ## meta-variables used as productions forms*: seq[OrigForm] ## forms used as productions @@ -192,7 +192,7 @@ proc buildLanguage(add, sub: seq[NimNode], error(fmt"given form is not a production of '{to}'", n) def.nterminals[to].forms.delete(idx) of nnkIdent: - let idx = def.nterminals[to].vars.find(n.strVal) + let idx = def.nterminals[to].vars.findIt(it.mvar == n.strVal) if idx == -1: error(fmt"given form is not a production of '{to}'", n) def.nterminals[to].vars.delete(idx) @@ -310,8 +310,10 @@ proc buildLanguage(add, sub: seq[NimNode], # check the meta-vars: for v in res.vars.items: - if v notin vars: - error(fmt"cannot inherit '{name}'; '{v}' (used as a production of '{name}') was removed", + if v.mvar notin vars: + error(fmt"cannot inherit '{name}'; '{v.mvar}' was removed", info) + elif vars[v.mvar] != v.typ: + error(fmt"cannot inherit '{name}'; '{v.mvar}' changed its meaning", info) result.nterminals[name] = res @@ -372,7 +374,7 @@ proc buildLanguage(add, sub: seq[NimNode], let name = n.strVal if name notin vars: error(fmt"no meta-variable with name '{name}'", n) - def.nterminals[to].vars.add name + def.nterminals[to].vars.add (name, vars[name]) else: error(fmt"unexpected syntax: {n.kind}", n) @@ -449,7 +451,7 @@ proc buildLanguage(add, sub: seq[NimNode], if name in def.nterminals: for it in def.nterminals[name].vars.items: - gather(def, top, vars[it], used, included) + gather(def, top, it.typ, used, included) for it in def.nterminals[name].forms.items: used.incl(it.semantic) @@ -459,7 +461,7 @@ proc buildLanguage(add, sub: seq[NimNode], for v in nt.vars.items: var gotUsed: IntSet var gotIncluded: HashSet[string] - gather(result, name, vars[v], gotUsed, gotIncluded) + gather(result, name, v.typ, gotUsed, gotIncluded) # add the gathered sets to the total sets: for it in gotUsed.items: if containsOrIncl(used, it): From 69bc1608b8995bf16915fc09afa73c39d82a3f5b Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 79/87] nplanggen: reject problematic non-terminal compositions Parts of the framework's internals rely on there only being a single form in a non-terminal that matches for a sub-tree (as it makes the implementation a lot simpler), but there previously was nothing making sure this expectation actually holds - now there is. In addition, the parser in `npparser` is not able to handle lists correctly when which form to pick is still undecided when processing a list. These compositions are also disallowed now. --- nanopass/nplangdef.nim | 115 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 94bd1f8c..19c4f3b9 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -72,6 +72,11 @@ type sub: seq[NimNode] add: seq[NimNode] +type + # Extra types unrelated to the ones above + Relation = enum + Disjoint, Overlap, ProblematicPrefix, Same + template findIt[T](s: seq[T], predicate: untyped): untyped = ## Version of ``find`` that allows providing an inline predicate, ## evaluated for every checked item. @@ -103,6 +108,88 @@ proc `==`(x, y: Form): bool = ## Compares `x` and `y`, which must belong to the same language, for equality. x.name == y.name and x.elems == y.elems +proc compare(def: LangDef, a, b: Form): Relation = + ## Computes the relation between `a` and `b`. Commutative. + if a.name != b.name: + return Disjoint + + proc contains(def: LangDef, nt: NonTerminal, typ: string): bool = + result = false + for it in nt.vars.items: + if it.typ == typ or + (it.typ in def.nterminals and + contains(def, def.nterminals[it.typ], typ)): + result = true + break + + proc gather(def: LangDef, nt: NonTerminal): (IntSet, HashSet[string]) = + ## Gathers the form and type productions of `nt`, including + ## transitive ones. + proc aux(def: LangDef, nt: NonTerminal, forms: var IntSet, + types: var HashSet[string]) = + for it in nt.vars.items: + if it.typ in def.nterminals: + aux(def, def.nterminals[it.typ], forms, types) + + for it in nt.forms.items: + forms.incl(it.semantic) + + aux(def, nt, result[0], result[1]) + + proc overlaps(def: LangDef, a, b: string): bool = + ## Whether the types `a` and `b` overlap (i.e., share inhabitants). + if a in def.nterminals: + if b in def.nterminals: + let (formsA, typesA) = gather(def, def.nterminals[a]) + let (formsB, typesB) = gather(def, def.nterminals[b]) + not(disjoint(formsA, formsB)) or not(disjoint(typesA, typesB)) + else: + contains(def, def.nterminals[a], b) + elif b in def.nterminals: + contains(def, def.nterminals[b], a) + else: + a == b + + result = Same + var ai = 0 + var bi = 0 + while ai < a.elems.len and bi < b.elems.len: + let got = + if a.elems[ai].typ == b.elems[bi].typ: + result # Overlap cannot go back to Same + elif overlaps(def, a.elems[ai].typ, b.elems[bi].typ): + Overlap + else: + Disjoint + + if a.elems[ai].repeat == b.elems[bi].repeat: + result = got + if result == Disjoint: + break + inc ai + inc bi + elif a.elems[ai].repeat: + if got == Disjoint: + result = Overlap + inc ai + else: + result = ProblematicPrefix + break + elif b.elems[bi].repeat: + if got == Disjoint: + result = Overlap + inc bi + else: + result = ProblematicPrefix + break + else: + result = got + inc ai + inc bi + + if result == Overlap and (ai < a.elems.len or bi < b.elems.len): + result = Disjoint + proc checkName(target: LangDef, vars: Table[string, string], name: string, info: NimNode) = if name == "entry": @@ -439,8 +526,8 @@ proc buildLanguage(add, sub: seq[NimNode], result.records[name] = record - # make sure all non-terminals are well-formed, which is possible only once - # all production additions were made + # make sure all non-terminals are well-formed and exhibit some additional + # properties, which is possible only once all production additions were made for name, nt in result.nterminals.pairs: proc gather(def: LangDef, top, name: string, used: var IntSet, included: var HashSet[string]) = @@ -455,6 +542,26 @@ proc buildLanguage(add, sub: seq[NimNode], for it in def.nterminals[name].forms.items: used.incl(it.semantic) + proc checkRelation(def: LangDef, form: Form, against: IntSet) = + # languages where there are non-terminals with forms that: + # * share the same name and overlap up until and including a list + # * share inhabitants + # are significantly harder to work with, so the aforementioned cases are + # simply disallowed. Allowing two forms in the language that share + # inhabitants means that assigning a type to a form instance requires + # context (i.e., a non-terminal), but that's an okay concession + for f in against.items: + let got = compare(def, form, def.forms[f]) + case got + of Disjoint, Same: + discard "all good" + of Overlap: + error("the forms '$1' and '$2' overlap without being the same" % + [$form, $def.forms[f]], info) + of ProblematicPrefix: + error("the forms '$1' and '$2' are not disjoint at where a list is" % + [$form, $def.forms[f]], info) + var used = initIntSet() var included: HashSet[string] @@ -467,6 +574,8 @@ proc buildLanguage(add, sub: seq[NimNode], if containsOrIncl(used, it): error("duplicate production '$1' in non-terminal '$2'" % [$result.forms[it], name], info) + else: + checkRelation(result, result.forms[it], used) for it in gotIncluded.items: if containsOrIncl(included, it): @@ -477,6 +586,8 @@ proc buildLanguage(add, sub: seq[NimNode], if containsOrIncl(used, it.semantic): error("duplicate production '$1' in non-terminal '$2'" % [$result.forms[it.semantic], name], info) + else: + checkRelation(result, result.forms[it.semantic], used) # process the extra configuration declarations: for it in config.items: From 85448f6aecb7f8498a30132ce0e6f7afad34e95f Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 80/87] nppass: use template for definition of injected routines It gets rid of some indentation and also allows using proper bound symbols, instead of relying on the various macros being visible where the template are injected. --- nanopass/nppass.nim | 101 +++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index ffaecfcb..96fe3593 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -250,6 +250,61 @@ macro generatedImpl(def: untyped) = def.body = newCall(ident"->", def.params[1][0], copyNimTree(def.params[0])) result = def +template defineInWrappers(lang, input: untyped) = + ## Introduces the injected definitions for passes that receive an AST. + template match[N](sel: Production[lang, N], branches: varargs[untyped] + ): untyped {.used, inject.} = + match[lang, N](input.tree, Cursor(sel.index), sel, branches) + + template slice[N](T: typedesc[Production[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + template slice[N](T: typedesc[RecordRef[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + template slice(T: typedesc[asts.Value[auto]]): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + + template val[T](v: nanopass.Value[T]): T {.used, inject.} = + # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) + # XXX: consider renaming this template to `get` + unpack(input.storage[], v.id, typeof(T)) + template get[N](r: RecordRef[lang, N]): untyped {.used, inject.} = + get(input, r) + template info[N](n: Production[lang, N]): untyped {.used, inject.} = + input.tree[n.index].info + + template equal[N](a, b: Production[lang, N]): bool {.used, inject.} = + equal(input.tree, Cursor(a.index), Cursor(b.index)) + +template defineOutWrappers(lang, output: untyped) = + ## Introduces the injected definitions for passes that produce an AST. + template terminal(x: untyped): untyped {.used, inject.} = + embed(output.storage, x) + template build[N](n: typedesc[Production[lang, N]], info: SLocRef, + body: untyped): untyped {.used, inject.} = + build(output, n, info, body) + template build[N](n: typedesc[RecordRef[lang, N]], info: SLocRef, + body: untyped): untyped {.used, inject.} = + build(output, n, info, body) + template match[N](sel: Production[lang, N], branches: varargs[untyped] + ): untyped {.used, inject.} = + match[lang, N](output.tree, IndCursor(sel.index), sel, branches) + template slice[N](T: typedesc[Production[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, IndCursor] + template slice[N](T: typedesc[RecordRef[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, IndCursor] + + template get[N](r: RecordRef[lang, N]): untyped {.used, inject.} = + get(`output`, r) + template info[N](n: Production[lang, N]): untyped {.used, inject.} = + output.tree[n.index].info + + template equal[N](a, b: Production[lang, N]): bool {.used, inject.} = + equal(output.tree, IndCursor(a.index), IndCursor(b.index)) + proc assemblePass(src, dst, def, call: NimNode): NimNode = ## Assembles the final procedure definition for a pass. `def` is the original ## proc definition, `call` the call to the pass' implementation. @@ -288,52 +343,10 @@ proc assemblePass(src, dst, def, call: NimNode): NimNode = template dst: untyped {.used.} = `dst` if hasIn: - body.add quote do: - template match[N](sel: Production[src, N], branches: varargs[untyped]): untyped {.used.} = - match[src, N](`input`.tree, Cursor(sel.index), sel, branches) - - template slice[N](T: typedesc[Production[src, N]]): typedesc {.used.} = - ChildSlice[T, Cursor] - template slice[N](T: typedesc[RecordRef[src, N]]): typedesc {.used.} = - ChildSlice[T, Cursor] - template slice(T: typedesc[asts.Value[auto]]): typedesc {.used.} = - ChildSlice[T, Cursor] - - template val[T](v: nanopass.Value[T]): T {.used.} = - # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) - # XXX: consider renaming this template to `get` - unpack(`input`.storage[], v.id, typeof(T)) - template get[N](r: RecordRef[src, N]): untyped {.used.} = - get(`input`, r) - template info[N](n: Production[src, N]): untyped {.used.} = - `input`.tree[n.index].info - - template equal[N](a, b: Production[src, N]): bool {.used.} = - equal(`input`.tree, Cursor(a.index), Cursor(b.index)) + body.add newCall(bindSym"defineInWrappers", ident"src", input) if hasOut: - let embed = bindSym"embed" - body.add quote do: - template terminal(x: untyped): untyped {.used.} = - `embed`(`output`.storage, x) - template build[N](n: typedesc[Production[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = - build(`output`, n, info, body) - template build[N](n: typedesc[RecordRef[dst, N]], info: SLocRef, body: untyped): untyped {.used.} = - build(`output`, n, info, body) - template match[N](sel: Production[dst, N], branches: varargs[untyped]): untyped {.used.} = - match[dst, N](`output`.tree, IndCursor(sel.index), sel, branches) - template slice[N](T: typedesc[Production[dst, N]]): typedesc {.used.} = - ChildSlice[T, IndCursor] - template slice[N](T: typedesc[RecordRef[dst, N]]): typedesc {.used.} = - ChildSlice[T, IndCursor] - - template get[N](r: RecordRef[dst, N]): untyped {.used.} = - get(`output`, r) - template info[N](n: Production[dst, N]): untyped {.used.} = - `output`.tree[n.index].info - - template equal[N](a, b: Production[dst, N]): bool {.used.} = - equal(`output`.tree, IndCursor(a.index), IndCursor(b.index)) + body.add newCall(bindSym"defineOutWrappers", ident"dst", output) # the source location accessors are always available if hasIn: From 70fa15490ad75ec9b95c1a6943cec3e629688280 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 81/87] helper: remove `copyLineInfoForTree` --- nanopass/helper.nim | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nanopass/helper.nim b/nanopass/helper.nim index 45cd7b76..4fa7495e 100644 --- a/nanopass/helper.nim +++ b/nanopass/helper.nim @@ -2,13 +2,8 @@ import std/[macros] -proc copyLineInfoForTree*(n, info: NimNode) = - copyLineInfo(n, info) - for i in 0.. Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 82/87] npbuild, nppass: fix line info for `.error` pragmas The line info needs to be copied to the `nkExprColonExpr`, not to the string operand. --- nanopass/npbuild.nim | 8 ++++---- nanopass/nppass.nim | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 77f9fc72..803aff30 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -214,8 +214,8 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = result = quote do: {.error: "expected type fitting " & $`dst` & ", but got " & $typeof(`src`).} - # copy the line info to the pragma's operand - copyLineInfo(result[0][1], info) + # copy the line info to the expr-colon-expr + copyLineInfo(result[0], info) proc makeMatch(src, expect: NimNode): NimNode = let error = newMismatchError(src, expect, src) @@ -250,14 +250,14 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = let mvar = ident(typ.mvar) result = quote do: {.error: "expected '" & $`ast`.L.`mvar` & "', but got form".} - copyLineInfo(result[0][1], n) + copyLineInfo(result[0], n) return of tkRecord: # expected a record, but the constructor can only be that of a form let mvar = ident(typ.mvar) result = quote do: {.error: "expected '" & $`ast`.L.`mvar` & "', but got form".} - copyLineInfo(result[0][1], n) + copyLineInfo(result[0], n) return of tkNonTerminal: discard "all good" diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index 96fe3593..bb2bcb3f 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -18,8 +18,9 @@ macro isPartOf(lang: static LangInfo, lname, typ: untyped): bool = macro ctError(str: string, info: untyped) = ## Like the .error pragma, but with customizable source location information. - copyLineInfo(str, info) - nnkPragma.newTree(nnkExprColonExpr.newTree(ident"error", str)) + let it = nnkExprColonExpr.newTree(ident"error", str) + copyLineInfo(it, info) + nnkPragma.newTree(it) template classify(x: typedesc): TypeClass = when x is Value: tcValue From efd7ab1ea0809fff3eca809474c167b01c795a2a Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 83/87] nanopass: clean up & polish * rename some macros * fix and update some doc comments * improve some comment * reformat a few very long lines --- nanopass/asts.nim | 20 +++++++------ nanopass/nanopass.nim | 18 ++++++++---- nanopass/npbuild.nim | 35 ++++++++++++----------- nanopass/nplang.nim | 14 +++++---- nanopass/nplangdef.nim | 35 ++++++++++++++--------- nanopass/nplanggen.nim | 4 ++- nanopass/npmatch.nim | 62 +++++++++++++++++++++++----------------- nanopass/npparser.nim | 36 ++++++++++------------- nanopass/nppass.nim | 34 ++++++++++++---------- nanopass/nppatterns.nim | 2 +- nanopass/nptransform.nim | 19 ++++++------ nanopass/npunparser.nim | 9 +++--- 12 files changed, 160 insertions(+), 128 deletions(-) diff --git a/nanopass/asts.nim b/nanopass/asts.nim index 4a150e6d..c43d8b50 100644 --- a/nanopass/asts.nim +++ b/nanopass/asts.nim @@ -1,5 +1,6 @@ -## Implements the nanopass framework specific storage types for ASTs. The types -## are layered on top of `PackedTree `_. +## Implements the nanopass framework specific storage types for ASTs, as well +## as various fundamental, language-agnostic operations on ASTs. The types are +## layered on top of `PackedTree `_. import passes/trees @@ -35,8 +36,8 @@ type ## leaked implementation detail, don't use Production*[L: object, N: static string] = object - ## Represents a reference to an AST fragment that's a production of non- - ## terminal `N` of language `L`. + ## Represents a reference to a production of non-terminal `N` belonging to + ## language `L`. index*: NodeIndex ## leaked implementation detail, don't use @@ -59,7 +60,7 @@ type len: uint32 Cursor* = distinct NodeIndex - ## A cursor into a tree where without indirections. + ## A cursor into a tree without indirections. IndCursor* = distinct NodeIndex ## A cursor into a tree with indirections. @@ -81,11 +82,12 @@ template tag*(n: AstNode): uint8 = cast[uint8](n.kind) template info*(n: AstNode): SLocRef = - ## The node's source location information reference. + ## The node's source location information. cast[SLocRef](uint32(n.kind) shr 8) template isAtom*(x: Tag): bool = - ## The predicate required for using an uint8 as a ``PackedTree`` tag. + ## Whether `x` is the tag of a leaf node. + # the predicate is required for using an uint8 as a ``PackedTree`` tag cast[uint8](x) >= RefTag {.push stacktrace: off, profiler: off.} @@ -247,7 +249,7 @@ proc advance*(tree: Tree, cr: var IndCursor) = NodeIndex(cr) = next(tree, NodeIndex(cr)) proc get*(tree: Tree, cr: IndCursor): NodeIndex = - if tree[NodeIndex(cr)].tag == 128: + if tree[NodeIndex(cr)].tag == RefTag: NodeIndex tree[NodeIndex(cr)].val else: NodeIndex cr @@ -258,7 +260,7 @@ template pos*(cr: IndCursor): NodeIndex = type Savepoint = tuple[origin: IndCursor, stepped: bool] proc enter*(tree: Tree, cr: var IndCursor): Savepoint = - result = (cr, tree[NodeIndex(cr)].tag == 128) + result = (cr, tree[NodeIndex(cr)].tag == RefTag) if result.stepped: cr = IndCursor tree[NodeIndex(cr)].val else: diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim index 66d2e183..7d996bdd 100644 --- a/nanopass/nanopass.nim +++ b/nanopass/nanopass.nim @@ -1,13 +1,19 @@ ## Implements the nanopass framework, which is a collection of macro DSLs for ## defining intermediate languages (their syntax and grammar) and passes. - -# TODO: -# * implement types integration -# * implement meta-data support -# * add a "compiler definition" macro +## +## This is the entrypoint of the library. import - nanopass/[asts, nplanggen, npmatch, npbuild, npparser, nppass, nppatterns, npunparser] + nanopass/[ + asts, + nplanggen, + npmatch, + npbuild, + npparser, + nppass, + nppatterns, + npunparser + ] export asts export nppatterns.matches diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim index 803aff30..6db9bca8 100644 --- a/nanopass/npbuild.nim +++ b/nanopass/npbuild.nim @@ -1,7 +1,7 @@ -## Implements the `build` macro, for constructing abstract syntax trees. +## Implements the `build` macro, for constructing records and abstract +## syntax trees. import std/[genasts, macros, strformat, tables] -import passes/trees import nanopass/[asts, helper, nplang, nppatterns] type @@ -196,19 +196,20 @@ proc buildRecord(lang: LangInfo, ast, info, e: NimNode): NimNode = `ast`.L.`mvar`(id: `ast`.records.`mvar`.high.uint32) proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = - ## Emits a tree construction for the AST described by `e`, with the syntax - ## from `lang`. + ## Translates the form construction `e` in the context of `lang` from the + ## `build` language to NimSkull. `typ` is the non-terminal the form must + ## be a part of. - # the `build` macro is complex, as: + # the translation is complex, as: # * there may be multiple forms in a language that have the same name # * the interpolated operands' types are not known to the macro # * list expansion is allowed, at least a single one - # In effect, a sort of overload resolution has to be performed for picking - # which form the build syntax ultimately matches. Due to list expansion, - # this cannot always be known at compile-time, in which case disambiguation - # has to happen at *run-time*. In the abstract, the macro works by emitting - # a decision tree (using `when` statements) that selects the form based on - # the operands' types + # Overload resolution has to be performed for picking which concrete form + # the syntax actually represents. Due to list expansion, this cannot always + # be known at compile-time, in which case disambiguation has to happen at + # *run-time*. In the abstract, the macro works by emitting a decision tree + # (using `when` statements) that selects the form based on the + # operands' types proc newMismatchError(src, dst, info: NimNode): NimNode = result = quote do: @@ -225,7 +226,6 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = `append`(`ast`, `info`, `src`) else: `error` - copyLineInfoForTree(result, src) proc addAll(to, n: NimNode) = if n.kind == nnkStmtList: @@ -325,8 +325,8 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = return makeError("form doesn't match any of the productions expected here", n) - # `process` having a closure context is costly, so pass the necessary - # local state via an aggregate parameter to `emit` + # `process` having a closure context is costly, and therefore the necessary + # local state is passed via an aggregate parameter to `emit` type Context = tuple[start, expanded: NimNode] proc emit(lang: LangInfo, c: Context, t, n: NimNode, i: int): NimNode = @@ -481,6 +481,8 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = let expect = quote do: `ast`.L.`mvar` let error = newMismatchError(n, expect, n) let append = bindSym"append" + # the operand may either be a `Value` or match the actual value + # type directly result = quote do: when matches(`n`, `expect`) or `n` is `expect`.T: when `n` is `expect`.T: @@ -533,8 +535,9 @@ proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode = macro buildImpl(lang: static LangInfo, name: static string, ast, info, e: untyped): untyped = - ## Emits a tree construction for the AST described by `e`, with the syntax - ## from `lang`. + ## Translates the build expression `e`, constructing a form or record + ## fitting the type with name `name` in the context of `lang`, to a NimSkull + ## expression. let typ = lang.map[name] case lang.types[typ].kind of tkNonTerminal: buildForm(lang, typ, ast, info, e) diff --git a/nanopass/nplang.nim b/nanopass/nplang.nim index 71596fea..81359bc5 100644 --- a/nanopass/nplang.nim +++ b/nanopass/nplang.nim @@ -1,4 +1,4 @@ -## Provides the query-focused types representing the language in the nanopass +## Provides the query-focused types representing a language in the nanopass ## framework, as well as the routines for creating instances thereof. import std/[hashes, sequtils, tables] @@ -19,7 +19,7 @@ type tkNonTerminal LangType* = object - ## Terminals and non-terminals modeled as types. + ## Terminals, records, and non-terminals. name*: string mvar*: string ## name of a meta-variable that is used to range over the type @@ -38,10 +38,10 @@ type ## all production forms of the non-terminal LangInfo* = object - ## Representation of a language definition that stores the information in - ## a way that make it easier to work with for the DSL macros. - # important: for compilation speed, the AST representation of the data - # should be as short and concise as possible + ## Representation of a language that stores the information in a way that + ## make it easy to work with for the DSL macros. + # important: to keep compilation time low, the AST representation of the + # data should be as short and concise as possible types*: seq[LangType] map*: Table[string, int] ## maps type and meta-var names to the corresponding type @@ -150,6 +150,8 @@ proc ntags*(lang: LangInfo, typ: LangType): seq[int] = result.add ntags(lang, lang.types[it]) proc render*(lang: LangInfo, form: SForm): string = + ## Renders `form` into the textual representation of syntax resembling how + ## the form is defined. result.add form.name result.add "(" for i, it in form.elems.pairs: diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim index 19c4f3b9..fbe5d7d4 100644 --- a/nanopass/nplangdef.nim +++ b/nanopass/nplangdef.nim @@ -41,7 +41,7 @@ type LangDef* = object ## A checked and pre-processed language definition, carrying enough - ## source-level information necessary for implementing, e.g., inheritance. + ## source-level information necessary for implementing inheritance. terminals*: Table[string, Terminal] ## the terminals of the language nterminals*: Table[string, NonTerminal] @@ -93,6 +93,8 @@ proc `$`(x: Form): string = for i, it in x.elems.pairs: if i > 0: result.add ", " + if it.repeat: + result.add "..." result.add it.typ result.add ")" @@ -105,7 +107,8 @@ proc `==`(x, y: Elem): bool = x.typ == y.typ and x.repeat == y.repeat proc `==`(x, y: Form): bool = - ## Compares `x` and `y`, which must belong to the same language, for equality. + ## Compares `x` and `y`, which must belong to the same language, for + ## equality. x.name == y.name and x.elems == y.elems proc compare(def: LangDef, a, b: Form): Relation = @@ -201,7 +204,8 @@ proc checkName(target: LangDef, vars: Table[string, string], name: string, elif name in target.nterminals: error(fmt"non-terminal with name {name} already exists", info) elif name in vars: - error(fmt"'{name}' is already the name of a meta-variable for '{vars[name]}'", info) + error(fmt"'{name}' is already the name of a meta-variable for '{vars[name]}'", + info) proc parseForm(n: NimNode): ParsedForm = ## Parses a form in the context of a language definition. No semantic checks @@ -228,8 +232,9 @@ proc buildLanguage(add, sub: seq[NimNode], config: seq[NimNode], base: LangDef, info: NimNode): LangDef = ## The center-piece of language definition construction. Constructs a - ## language definition by applying the diff for terminals (`add` and `sub`) - ## and non- terminals (`def`) to `base`. `info` is used for error reporting. + ## language definition by applying the diff for terminals (`add` and `sub`), + ## records (`records`), and and non-terminals (`def`) to `base`. `info` is + ## used for error reporting. var base = base # ^^ base is modified in-place because it makes the implementation easier @@ -542,7 +547,8 @@ proc buildLanguage(add, sub: seq[NimNode], for it in def.nterminals[name].forms.items: used.incl(it.semantic) - proc checkRelation(def: LangDef, form: Form, against: IntSet) = + proc checkRelation(def: LangDef, form: Form, against: IntSet, + name: string) = # languages where there are non-terminals with forms that: # * share the same name and overlap up until and including a list # * share inhabitants @@ -556,11 +562,12 @@ proc buildLanguage(add, sub: seq[NimNode], of Disjoint, Same: discard "all good" of Overlap: - error("the forms '$1' and '$2' overlap without being the same" % - [$form, $def.forms[f]], info) + error("overlapping productions '$1' and '$2' in non-terminal '$3'" % + [$form, $def.forms[f], name], info) of ProblematicPrefix: - error("the forms '$1' and '$2' are not disjoint at where a list is" % - [$form, $def.forms[f]], info) + error(("productions '$1' and '$2' of non-terminal '$3' are not " & + "disjoint at where a list is") % + [$form, $def.forms[f], name], info) var used = initIntSet() var included: HashSet[string] @@ -569,13 +576,15 @@ proc buildLanguage(add, sub: seq[NimNode], var gotUsed: IntSet var gotIncluded: HashSet[string] gather(result, name, v.typ, gotUsed, gotIncluded) + # compute the form production relations against the existing ones first: + for it in gotUsed.items: + checkRelation(result, result.forms[it], used, name) + # add the gathered sets to the total sets: for it in gotUsed.items: if containsOrIncl(used, it): error("duplicate production '$1' in non-terminal '$2'" % [$result.forms[it], name], info) - else: - checkRelation(result, result.forms[it], used) for it in gotIncluded.items: if containsOrIncl(included, it): @@ -587,7 +596,7 @@ proc buildLanguage(add, sub: seq[NimNode], error("duplicate production '$1' in non-terminal '$2'" % [$result.forms[it.semantic], name], info) else: - checkRelation(result, result.forms[it.semantic], used) + checkRelation(result, result.forms[it.semantic], used, name) # process the extra configuration declarations: for it in config.items: diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim index 8dcab37f..a5b16688 100644 --- a/nanopass/nplanggen.nim +++ b/nanopass/nplanggen.nim @@ -64,13 +64,14 @@ macro makeLanguageType(def: static LangDef, info: LangInfo, typName: untyped) = ## The type also stores various information about the language that are ## needed by the pass-related macros, encoded as types. let fields = nnkRecList.newTree() - # the metavars are at top level of the type, for easy access by + # the metavars are at the top level of the type, for easy access by # the programmer let prod = bindSym"Production" for name, it in def.terminals.pairs: for m in it.mvars.items: fields.add newIdentDefs(ident(m), nnkBracketExpr.newTree(bindSym"Value", ident(name))) + for name, it in def.nterminals.pairs: for m in it.mvars.items: fields.add newIdentDefs(ident(m), @@ -78,6 +79,7 @@ macro makeLanguageType(def: static LangDef, info: LangInfo, typName: untyped) = prod, ident(typName.strVal), newStrLitNode(name))) + for name, it in def.records.pairs: for m in it.mvars.items: fields.add newIdentDefs(ident(m), diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim index 84f5113c..618a9707 100644 --- a/nanopass/npmatch.nim +++ b/nanopass/npmatch.nim @@ -1,11 +1,12 @@ -## Implements the high and low-level `match` macros. +## Implements the high and low-level `match` macros, which provide the pattern- +## matching-over-ASTs functionality. import std/[macros, intsets, strformat, tables] import nanopass/[asts, helper, nplang] type FillProc* = proc(lang: LangInfo, idx: int, n, info: NimNode): NimNode - ## Type for form or type fill callback. + ## Type for a form or type fill callback. ExpandConfig* = object fillForm*: FillProc ## called for filling-in the handling of forms. May be nil @@ -32,7 +33,7 @@ proc parseVar(n: NimNode): string = result = name[0..e] proc fits(lang: LangInfo, a, b: int): bool = - ## Computes whether type with id `a` can appear where a type with id `b` + ## Computes whether a type with id `a` can appear where a type with id `b` ## is expected. if a == b: result = true @@ -51,6 +52,7 @@ proc countTags(lang: LangInfo, typ: LangType): int = result += countTags(lang, lang.types[it]) proc containsForm(lang: LangInfo, typ: LangType, fid: int): bool = + ## Whether type `typ` contains the form with id `fid`. case typ.kind of tkNonTerminal: if fid in typ.forms: @@ -226,8 +228,8 @@ proc parsePattern(lang: LangInfo, n: NimNode): NimNode = case n[0].kind of nnkIdent: let id = ident(parseVar(n[0])) - # due to coalescing of match expressions, assigning an internal symbol - # to the temporary binding is not yet possible + # due to the later coalescing of match expressions, assigning an + # internal symbol to the temporary binding is not yet possible let call = nnkInfix.newTree(ident"->", newEmptyNode(), quote do: dst.`id`) copyLineInfo(call, n) # the matched type needs to be inferred @@ -259,7 +261,7 @@ proc parsePattern(lang: LangInfo, n: NimNode): NimNode = let tmp = parsePattern(lang, n[1]) if tmp[1].kind notin {nnkIntLit, nnkEmpty, nnkNilLit}: - error("only '...' and '...any' are allowed", n[1]) + error("only '...' and '...any' are allowed", n) # the '...' prefix is stripped from the expression result = makeTyped(tmp[0], nnkBracket.newTree(tmp[1]), n) of nnkIdent: @@ -273,7 +275,7 @@ proc parsePattern(lang: LangInfo, n: NimNode): NimNode = # must be a meta-variable let typ = parseVar(n) if typ notin lang.map: - error("no meta-variable with the give name exists", n) + error(fmt"no meta-variable called {typ} exists", n) result = makeTyped(n, newIntLitNode(lang.map[typ]), n) else: @@ -294,7 +296,12 @@ proc patternToString(n: NimNode; indent = 0): string = result.add " " result.add "}" of nnkCall: - result = repr(n[0]) + result = "" + case n[0].kind + of nnkIntLit, nnkPar, nnkBracket: + result = repr(n[0]) + else: + result = "" result.add "(" for i in 1.. 1: @@ -306,9 +313,7 @@ proc patternToString(n: NimNode; indent = 0): string = result.add ")" of nnkPar: result = "^" - result.add repr(n[0]) - result.add " " - result.add patternToString(n[1], indent) + result.add patternToString(n[0], indent) of nnkEmpty: result = "." of nnkStmtList: @@ -320,11 +325,13 @@ proc patternToString(n: NimNode; indent = 0): string = else: result = "" -proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, - config: ExpandConfig): NimNode = - ## Generates the NimSkull code for a match expression `expr`. `els` is either +proc matchToNimskull(lang: LangInfo, name, ast, sel, e, els: NimNode, + config: ExpandConfig): NimNode = + ## Translates the match expression `e` to NimSkull AST. `els` is either ## an `nnkElse` tree used for handling the rest of match, or nil, in which - ## case how to handle the rest of a match is dictated by `config`. + ## case how to handle the rest of a match is dictated by `config`. `name` is + ## the language type expression, `ast` the AST lvalue, and `sel` is the + ## initial cursor. let cursor = genSym("cursor") let backup = genSym("backup") @@ -336,7 +343,8 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, proc aux(lang: LangInfo, e, to: NimNode): NimNode = ## Does the actual work. - proc tm(lang: LangInfo, e, to: NimNode): NimNode = + proc translateMatch(lang: LangInfo, e, to: NimNode) = + ## Emits the cursor movement and binding creation for a match expression. if e[1].kind != nnkEmpty: # commit the current cursor to a local with the given name if e[0].kind == nnkBracket: @@ -369,11 +377,10 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, else: unreachable(e[0].kind) - result = aux(lang, e[^1], to) - case e.kind of nnkCall: - result = tm(lang, e, to) + translateMatch(lang, e, to) + result = aux(lang, e[^1], to) of nnkPar: # move the current cursor to the end of the subtree and pop it from # the stack @@ -439,9 +446,10 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, top = it handler.add newStmtList() - discard tm(lang, it, handler[^1]) + translateMatch(lang, it, handler[^1]) + discard aux(lang, it[^1], handler[^1]) caseStmt.add handler - # pop all leftover len entries + # pop all leftover stack entries stack.setLen(stackLen) let info = e[0] @@ -454,7 +462,7 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, if stack.len == 0 and not hasUsedElse: # the 'else' rule was never used. Add it to the top-level case - # statement such that a warning will be emitted + # statement, so that a warning will be emitted if caseStmt[^1].kind != nnkElse: caseStmt.add nnkElse.newTree(newCall(bindSym"unreachable")) caseStmt.add els @@ -504,7 +512,8 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, newIntLitNode(lang.forms[it].ntag), fillForm(lang, it, cursor, info)) - # fill in handling for not fully handled subtypes + # fill in handling for embedded types whose productions aren't + # all covered for it in lang.types[typ].sub.items: let br = genOfBranch(lang, lang.types[it], used) if br.len > 0: @@ -572,15 +581,16 @@ proc generateForMatch(lang: LangInfo, name, ast, sel, e, els: NimNode, proc patternToMatch(n: NimNode): tuple[head, tail: NimNode] = ## Translates a pattern into a match expression. A match expression has the - ## following structure: + ## following grammar: ## ## match ::= (nkCall (nkBracket ) ) ## | (nkCall (nkPar +) ) + ## | (nkCall (nkEmpty) ) ## | (nkCall ) ## name ::= | ## cont ::= ## | (nkPar ) # leave sub match - ## | (nkCurly +) # union matching + ## | (nkCurly +) # union matching ## | (nkTupleConstr (nkLetSection ...) ...) # binding ## | (nkStmtList ...) # custom tail logic var binds: NimNode @@ -856,7 +866,7 @@ proc matchImpl*(lang: LangInfo, src: int, name, ast, sel, rules: NimNode, total = nnkCurly.newTree(newIntLitNode(src)) assignSymbols(total) - result = generateForMatch(lang, name, ast, sel, optimize(total), els, config) + result = matchToNimskull(lang, name, ast, sel, optimize(total), els, config) macro matchImpl(lang: static LangInfo, nterm: static string, name: typed, ast: Tree, cursor: untyped, diff --git a/nanopass/npparser.nim b/nanopass/npparser.nim index 845ae1ee..8c673b9a 100644 --- a/nanopass/npparser.nim +++ b/nanopass/npparser.nim @@ -1,5 +1,5 @@ -## Implements the routines for parsing an S-expression-based AST -## representation into an AST. +## Implements the routines for parsing S-expression-based AST representation +## into ASTs. import std/[genasts, macros, strutils, tables, typetraits], @@ -25,13 +25,6 @@ type curSLoc: SLocRef ## source location to use for parsed nodes -# the core parser logic for a language is implemented in generic routines, -# which themselves call internal macros; the external macro then only expands -# to code calling said generic routines. The benefit: most of the logic for -# parsing an AST is only generated once per language, even when more than one -# parser is generated for a language. This is somewhat problematic for symbol -# binding (for the terminal parsers), however, given how generics work - macro mapTypeImpl(lang: static LangInfo, lname, typ: untyped): int = result = nnkWhenStmt.newTree() for i, it in lang.types.pairs: @@ -42,11 +35,11 @@ macro mapTypeImpl(lang: static LangInfo, lname, typ: untyped): int = result.add nnkElse.newTree(quote do: {.error: "unreachable".}) proc mapType[L, T](): int {.compileTime.} = - ## Maps the type `T` to the integer IDs its known under in `L`. + ## Maps the type `T` to the integer IDs identifying it in `L`. mapTypeImpl(idef(L), L, T) macro tags(lang: static LangInfo, typ: static int): set[uint8] = - ## Returns the node tags for the forms inhabiting `typ`. + ## Returns the node tags for the productions inhabiting `typ`. case lang.types[typ].kind of tkRecord: nnkCurly.newTree(newLit(uint8 lang.types[typ].rtag)) @@ -77,8 +70,8 @@ proc raiseError(line, col: int, msg: string) {.noreturn.} = template check[L](c: Ctx[L, auto], pos: NodeIndex, line, col: int, t: typedesc) = - ## Makes sure the production at `pos` is one of the non-terminal identified - ## by `nterm`, raising an error if not. + ## Makes sure the production at `pos` is one part of the non-terminal + ## identified by `nterm`, raising an error if not. if c.tree[pos].tag notin tags(idef(typeof(L)), mapType[L, t]()): when t is Production: raiseError(line, col, @@ -87,7 +80,7 @@ template check[L](c: Ctx[L, auto], pos: NodeIndex, line, col: int, raiseError(line, col, "expected '" & $t & "'") -macro genTerminalParser(lang: static LangInfo) = +macro parseTerminalImpl(lang: static LangInfo) = result = newStmtList() # emit the terminal handlers for it in lang.types.items: @@ -104,7 +97,7 @@ proc parseTerminal[L, S](c: var Ctx[L, S], node: SexpNode, line, col: int): AstNode = ## Implements fallback parsing of terminals. mixin idef - genTerminalParser(idef(L)) + parseTerminalImpl(idef(L)) raiseError(line, col, "'" & $node & "' is neither a valid language form nor terminal") @@ -140,7 +133,7 @@ proc parseFieldsImpl[L](c: var Ctx[L, auto], p: var SexpParser, # parse the field's value... parse(c, p) - # ...then make sure it's what it should be + # ...then make sure its type is correct check(c, NodeIndex(start), p.getLine(), p.getColumn(), typeof(it)) extract(c, it, start) @@ -148,7 +141,7 @@ proc parseFieldsImpl[L](c: var Ctx[L, auto], p: var SexpParser, eat(p, tkParensRi) macro parseFields(lang: static LangInfo, c: var Ctx, name: string) = - ## Selects the record type base on the dynamic value of `name`, parses + ## Selects the record type based on the dynamic value of `name`, parses ## the record's fields, registers the resulting record with `c`, and ## appends a record reference to the AST. result = nnkCaseStmt.newTree(name) @@ -370,8 +363,8 @@ macro parseFormImpl(lang: static LangInfo) = for name, forms in buckets.pairs: var m: NimNode # the match expression - # in order to produce some more helpful error message, the generated code is - # structured such that it also know *where* a grammar violation is, not + # in order to produce a more helpful error message, the generated code is + # structured such that it also knows *where* a grammar violation is, not # just that there is one for id in forms.items: @@ -431,7 +424,8 @@ proc parseForm[L, S](c: var Ctx[L, S], p: var SexpParser, parseFormImpl(idef(L)) proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) = - ## Parses a production from `p` at the current position. + ## Parses a production from `p`, raising an error when there's a syntax + ## error, or when the production is illformed. if p.currToken == tkParensLe: # could be both a form or terminal let (line, col) = (p.getLine(), p.getColumn()) @@ -463,7 +457,7 @@ proc parseAst*[S, L, N](p: var SexpParser, T: typedesc[Production[L, N]] parse(c, p) check(c, NodeIndex(0), line, col, T) - # append the staging buffer to the main buffer and update the reference + # append the staging buffer to the main buffer and update the references # in `c.records` let pos {.used.} = c.tree.nodes.len c.tree.nodes.add c.staging.nodes diff --git a/nanopass/nppass.nim b/nanopass/nppass.nim index bb2bcb3f..8efff6f9 100644 --- a/nanopass/nppass.nim +++ b/nanopass/nppass.nim @@ -28,8 +28,8 @@ template classify(x: typedesc): TypeClass = elif x is Production: tcProduction else: tcNone -template embed(storage, arg: untyped): untyped = - ## Implements terminal value construction. +template newTerminal(storage, arg: untyped): untyped = + ## Creates a ``Value`` storing `arg`. mixin pack let tmp = arg Value[typeof(tmp)](id: pack(storage[], tmp)) @@ -43,7 +43,7 @@ template get*[L; N](ast: Ast[L, auto], r: RecordRef[L, N]): untyped = pick(idef(typeof(L)), N, ast.records)[r.id] macro transformOutImpl(lang: static LangDef, name, def: untyped) = - ## Implements the transformation for processors in an *->language pass. + ## Turns a processor in an *->language pass into a real procedure. if def.kind notin {nnkProcDef, nnkFuncDef}: error(".transform must be applied to procedure definition", def) @@ -96,8 +96,9 @@ macro processorMatchImpl(lang: static LangInfo, src: static string, matchImpl(lang, lang.map[src], ident"src", input, sel, rules, config) -macro genProcessor(index, nterm: untyped): untyped = - ## Generates the body for a non-terminal processor. +macro transform(index, nterm: untyped): untyped = + ## Transforms the input language non-terminal with name `nterm` to a non- + ## terminal identified through the current result type. # simply emit an empty processorMatchImpl invocation. All branches will be # auto-generated # TODO: if none of the productions require a (direct or indirect) call to a @@ -197,7 +198,7 @@ template checkType(lang, typ: untyped) = ctError("type must belong to '" & $lang & "'", typ) macro transformInOutImpl(lang: static LangDef, name, def: untyped) = - ## Implements the transformation for language->language pass processors. + ## Turns the processor of a language->language pass into a real procedure. let name = name if def.kind notin {nnkProcDef, nnkFuncDef}: error(".transform must be applied to procedure definition", def) @@ -240,7 +241,8 @@ macro transformInOutImpl(lang: static LangDef, name, def: untyped) = result = def macro transformInImpl(lang: static LangDef, name, def: untyped) = - ## Implements the processing for transformers part of an input pass. + ## Turns the processor of a language->* pass into a real procedure. + # nothing to do result = def macro generatedImpl(def: untyped) = @@ -281,7 +283,7 @@ template defineInWrappers(lang, input: untyped) = template defineOutWrappers(lang, output: untyped) = ## Introduces the injected definitions for passes that produce an AST. template terminal(x: untyped): untyped {.used, inject.} = - embed(output.storage, x) + newTerminal(output.storage, x) template build[N](n: typedesc[Production[lang, N]], info: SLocRef, body: untyped): untyped {.used, inject.} = build(output, n, info, body) @@ -414,7 +416,7 @@ template defineProcessors(dst: untyped) = # note: the signature is overly broad so that overload resolution # prefers the more specific adapters created for the programmer-provided # processors - genProcessor(n.index, typeof(n).N) + transform(n.index, typeof(n).N) proc `->`[X](r: RecordRef, T: typedesc[RecordRef[dst, X]]): T {.inject.} = let tab = getTable[typeof(r), T]() @@ -554,10 +556,11 @@ macro inpass*(p: untyped) = ## Turns a procedure definition into a pass that takes arbitrary data as ## input and produces an AST for the specified language. ## The procedure's return type specifies the shape of the returned AST and - ## must must be the non-terminal of an IL. As a short-hand, just specifying - ## an IL is equivalent to specifying the IL's entry non-terminal. + ## must be the non-terminal of a language. As a short-hand, just specifying + ## a language is equivalent to specifying the language's entry non-terminal. ## - ## The return type of the transformed procedure is an AST. + ## The return type of the transformed procedure is an AST plus the specified + ## non-terminal. if p.kind != nnkProcDef: error(".inpass must be applied to a procedure definition", p) @@ -570,8 +573,8 @@ macro inpass*(p: untyped) = macro pass*(p: untyped) = ## Turns a procedure definition into a language->language pass, that is a - ## pass, that takes an AST (fragment) of language A and produces an AST of - ## language B. + ## pass, that takes an AST and non-terminal reference of language A and + ## produces an AST and a non-terminal reference of language B. if p.kind != nnkProcDef: error(".inpass must be applied to a procedure definition", p) @@ -586,7 +589,8 @@ macro pass*(p: untyped) = macro outpass*(p: untyped) = ## Turns a procedure definition into a language->* pass, that is, a pass - ## that takes an AST (fragment) of language A and produces a value. + ## that takes an AST and non-terminal reference of language A and produces + ## a value. if p.kind != nnkProcDef: error(".outpass must be applied to a procedure definition", p) diff --git a/nanopass/nppatterns.nim b/nanopass/nppatterns.nim index 0030520d..63628914 100644 --- a/nanopass/nppatterns.nim +++ b/nanopass/nppatterns.nim @@ -26,7 +26,7 @@ template matches*[T, U](x: T, _: typedesc[U]): bool = false elif U is PChoice: # why not just use `or`? Because that would unnecessarily expand the - # second `matches` invocation when the case first invocation was sucessful + # second `matches` invocation when the case first invocation was successful when matches(x, typeof(U.A)): true else: diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim index e9f55a31..5dc6d9c6 100644 --- a/nanopass/nptransform.nim +++ b/nanopass/nptransform.nim @@ -1,6 +1,7 @@ -## Implements the auto-generation of transformers for language forms. +## Implements the generation of transformers for language forms, records, +## and types. -import std/[macros, strformat, tables] +import std/[macros, strutils, tables] import passes/[trees] import nanopass/[asts, helper, nplang] @@ -29,9 +30,9 @@ proc append(to: var Tree, i: var int, tag: uint8, info: SLocRef, macro transform*(src, dst: static LangInfo, nterm: static string, form: static int, input, output: Tree, cursor: untyped): untyped = - ## Generates the transformation from the given source language form - ## to a compatible target language production of the non-terminal with - ## name `nterm`. + ## Transforms the input language form identified by `form` to a production + ## fitting fitting the target language non-terminal with name `nterm` and + ## appends the result to `output`. # find a target language form that's a production of the non-terminal and a # suitable morph target. Exact matches are preferred var target = -1 @@ -54,12 +55,10 @@ macro transform*(src, dst: static LangInfo, nterm: static string, morphability = m target = it - template formatValue(to: var string, x: SForm, prec: string) = - to.add render(src, x) - if morphability in {None, Ambiguous}: return - makeError(fmt"cannot generate transformer for '{src.forms[form]}'", cursor) + makeError("cannot generate transformer from '$1' to '$2'" % + [render(src, src.forms[form]), nterm], cursor) # important: the generated code being efficient is of major importance! Most # transformations will be auto-generated, and they should thus be as fast as @@ -140,7 +139,7 @@ macro transformType*(src, dst: static LangInfo, nterm: static string, typ: static int, input, output: Tree, cursor: untyped): untyped = ## Transforms the instance of type `typ` (may be either a terminal or non- - ## terminal) at `cursor` to an AST fitting the destination non-terminal + ## terminal) at `cursor` to a production fitting the destination non-terminal ## `nterm` and appends the result to `output`. proc contains(lang: LangInfo, typ: LangType, search: int): bool = for it in typ.sub.items: diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim index 9ce1bfe7..c5efad18 100644 --- a/nanopass/npunparser.nim +++ b/nanopass/npunparser.nim @@ -22,8 +22,8 @@ type ## currently active source location proc nameToIndex[L; Name: static string](): int {.compileTime.} = - ## Turns a record type name to the index of the corresponding set in - ## `Ctx.records`. + ## Returns the index of the set in `Ctx.records` corresponding to the given + ## record type name. var i = 0 var tmp: L # the index being stable across compilation is not necessary @@ -67,6 +67,7 @@ proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, ## Unparses the nanopass record `tup`. The full record is only emitted for ## the first occurrence - only the ID is emitted for all further occurrences. if containsOrIncl(c.records[static nameToIndex[L, N]()], id): + # the definition was emitted already, only emit a reference result = newSList([newSSymbol(":record"), newSSymbol(N), newSInt(id)]) else: result = newSList() @@ -159,8 +160,8 @@ macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = ofb.add body caseStmt.add ofb - # for robustness, and to make spotting of problems easier, render unexpected - # nodes as errors + # to be resilient against malformed input, and to make spotting of problems + # easier, render unexpected nodes as errors caseStmt.add nnkElse.newTree(quote do: result = newSList( [newSSymbol(":error"), newSInt(int `ast`.tree.nodes[`c`.pos].tag)]) From a60182b90c14b9bf1877cc9c0b945735fc6f6a47 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 84/87] compilerdef: adjust to nanopass changes --- passes/compilerdef.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/passes/compilerdef.nim b/passes/compilerdef.nim index c801f56a..a422d276 100644 --- a/passes/compilerdef.nim +++ b/passes/compilerdef.nim @@ -25,6 +25,6 @@ macro defineCompiler*(name, start, names: untyped) = prevAst = quote do: `tmp`[0] prevPos = quote do: `tmp`[1] result = quote do: - proc `name`(ast: Ast[`start`, Literals], it: `start`.meta.entry): auto = + proc `name`(ast: Ast[`start`, Literals], it: `start`.entry): auto = `body` result = (`prevAst`, `prevPos`) From 62e85e7400da6d50d4a88e1dddd11c1ba4236281 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 85/87] manual: adjust to the nanopass changes Also clean up and improve the manual in general. --- nanopass/manual.rst | 81 +++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/nanopass/manual.rst b/nanopass/manual.rst index 1a11e88f..1780973f 100644 --- a/nanopass/manual.rst +++ b/nanopass/manual.rst @@ -16,14 +16,33 @@ according to well-defined grammars. The idea is that passes focus on small, specific tasks, with tree traversal boilerplate generated automatically. -Concepts +Glossary -------- -* a *language* (in the context of the nanopass framework) is a formal grammar. -* a *terminal* is ... -* a *form* is a named schema for a term, made up of zero or more sub-terms -* a *non-terminal* is ... -* a *meta-variable* is a name ranging over a terminal or non-terminal. It may be viewed as an alias. +record + a heterogeneous map from names to values, whose shape (number of entries and + their names and types) is fixed + +terminal + in the context of the nanopass framework, a type that's defined outside a + language. This corresponds to a terminal in a formal grammar, hence the name + +form + a named schema for a term. Can also be viewed as a named tuple + +production + either a form, a record, or a terminal + +non-terminal + a set of productions + +meta-variable + ranges over the terms a type. In the nanopass framework, they're mostly + just short-hands for types + +language + a collection of terminals, records, and forms, plus the non-terminals for + connecting them together .. TODO rewrite this section. It should use the correct terminology while still being approachable to someone not overly familiar with formal grammars @@ -45,11 +64,11 @@ type is bound to the identifier. .. note:: - The type is not meant to be used as a type for values. Rather, it - acts as a namespace through which entities defined in the language are - accessed. + The type bound to the identifier is not meant to be used as a type for + values. Rather, it acts as a namespace through which entities defined in the + language are accessed. -The body consists of a sequence of terminal and non-terminal definitions. +The body consists of a sequence of terminal, record, and non-terminal definitions. .. code-block:: nim :test: "nim c $1" @@ -57,13 +76,17 @@ The body consists of a sequence of terminal and non-terminal definitions. import nanopass/nanopass defineLanguage L0: - int(i) # definition of a terminal - expr(e) ::= i # definition of a non-terminal + int(i) # definition of a terminal + rec(r) ::= (field: i) # definition of a record + expr(e) ::= i # definition of a non-terminal # this defines a language `L0` with: - # * a single terminal of type `int`, ranged over by meta-variable `i` + # * a terminal of type `int`, ranged over by meta-variable `i` + # * a record type with name `rec`, ranged over by meta-variable `r`. The + # record has a single field called `field`, whose values must be of + # type `int` # * a non-terminal with name `expr`, ranged over by meta-variable `e`. Where - # this non-terminal is expected, an `int` value is allowed + # this non-terminal is expected, there must be value of type `int` Meta-variables must be unique. @@ -99,7 +122,7 @@ regardless of the declarations' order. expr(e) ::= i int(i) -Each non-terminals must have a unique name. +No two non-terminals may have the same name. .. code-block:: nim :test: "nim c $1" @@ -112,9 +135,7 @@ Each non-terminals must have a unique name. expr(e) ::= i expr(b) ::= i # error: 'expr' name already in use -Non-terminals and meta-variables share a namespace, meaning that it's not -possible to give a name to a non-terminal already used for a meta-variable, -and vice versa. +All types share the same namespace. .. code-block:: nim :test: "nim c $1" @@ -124,10 +145,21 @@ and vice versa. defineLanguage L0: int(i) - i(e) ::= i # error: 'i' already in use + int(e) ::= i # a type with name 'int' already exists + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass -For terminals, the type expression must be an identifier, more complex -expressions are not allowed. + defineLanguage L0: + int(i) + int(e) ::= (field: i) # a type with name 'int' already exists + +Types and meta-variables also share a namespace, meaning that it's not +possible to give a name to a type already used for a meta-variable, +and vice versa. .. code-block:: nim :test: "nim c $1" @@ -136,10 +168,11 @@ expressions are not allowed. import nanopass/nanopass defineLanguage L0: - (ref int)(i) # error: not an identifier - expr(e) ::= i + int(i) + i(e) ::= i # error: 'i' already in use + -The identifier must also refer to a type that exists at the time +For terminal types, the name refer to a NimSkull type that exists at the time `defineLanguage` is expanded. .. code-block:: nim From 49a796c9fd96c3b6c1dc6cf582bd5693adcdbca9 Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 86/87] passes_legacy: remove obsolete parser definitions --- passes/passes_legacy.nim | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/passes/passes_legacy.nim b/passes/passes_legacy.nim index fd009062..b47cfb3d 100644 --- a/passes/passes_legacy.nim +++ b/passes/passes_legacy.nim @@ -1328,24 +1328,12 @@ proc toSexp(x: float): SexpNode = proc toSexp(x: string): SexpNode = newSString(x) -proc parseInput(p: var SexpParser): Lskully.m {.parser.} -proc parseInner(p: var SexpParser): L0.m {.parser.} -proc render(p: L6.m): SexpNode {.unparser.} -proc render(p: L5.m): SexpNode {.unparser.} -proc render(p: L4.m): SexpNode {.unparser.} -proc render(p: L3s2.m): SexpNode {.unparser.} -proc render(p: L3.m): SexpNode {.unparser.} -proc render(p: L2.m): SexpNode {.unparser.} -proc render(p: L1.m): SexpNode {.unparser.} -proc render(p: LPtr.m): SexpNode {.unparser.} -proc render(p: L0.m): SexpNode {.unparser.} - let f = openFileStream(getExecArgs()[0], fmRead) var p: SexpParser p.open(f) discard p.getTok() -let (ast, m) = parseInput(p) +let (ast, m) = parseAst[Literals](p, LSkully.m) f.close() echo "parsed" From 6b580b797e1686f0b931243633571dc447c0f0ae Mon Sep 17 00:00:00 2001 From: zerbina <100542850+zerbina@users.noreply.github.com> Date: Tue, 20 Jan 2026 22:10:02 +0000 Subject: [PATCH 87/87] passes_legacy: adjust to source locs --- passes/passes_legacy.nim | 136 +++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/passes/passes_legacy.nim b/passes/passes_legacy.nim index b47cfb3d..da2a401a 100644 --- a/passes/passes_legacy.nim +++ b/passes/passes_legacy.nim @@ -188,12 +188,12 @@ proc basicBlocks(ir: Lskully): L6 {.pass.} = proc target(x: src.tgt, map: Table[int64, int]): dst.tgt = match x: - of Goto(i): build dst.tgt, Goto(i(^map[i.val])) - of Unwind(): build dst.tgt, Unwind() + of Goto(i): build dst.tgt, x.info, Goto(i(^map[i.val])) + of Unwind(): build dst.tgt, x.info, Unwind() proc goto(x: src.go, map: Table[int64, int]): dst.go = match x: - of Goto(i): build dst.go, Goto(i(^map[i.val])) + of Goto(i): build dst.go, x.info, Goto(i(^map[i.val])) type BBlock = object isExcept: bool @@ -203,69 +203,69 @@ proc basicBlocks(ir: Lskully): L6 {.pass.} = proc blocks(x: src.st, bbs: var seq[dst.bb], bb: var BBlock, map: Table[int64, int]) = proc commitBlock(bbs: var seq[dst.bb], bb: var BBlock, ex: dst.ex) = if bb.isExcept: - bbs.add build(dst.bb, Except(^bb.params, ...bb.stmts, ex)) + bbs.add build(dst.bb, NoSLoc, Except(^bb.params, ...bb.stmts, ex)) else: - bbs.add build(dst.bb, Block(^bb.params, ...bb.stmts, ex)) + bbs.add build(dst.bb, NoSLoc, Block(^bb.params, ...bb.stmts, ex)) bb.stmts.shrink(0) proc startBlock(bb: var BBlock) = bb.isExcept = false - bb.params = build(dst.p, Params([])) + bb.params = build(dst.p, NoSLoc, Params([])) match x: of Stmts(...st): for it in st.items: blocks(it, bbs, bb, map) of Goto(i): - commitBlock bbs, bb, build(dst.ex, Goto(i(^map[i.val]))) + commitBlock bbs, bb, build(dst.ex, x.info, Goto(i(^map[i.val]))) of Join(i): # don't merge an empty entry block with the following block; the latter # might be a loop start if map[i.val] > bbs.len: - commitBlock bbs, bb, build(dst.ex, Goto(i(^map[i.val]))) + commitBlock bbs, bb, build(dst.ex, x.info, Goto(i(^map[i.val]))) startBlock(bb) of Except(_, lo): assert bb.stmts.len == 0, "control flow falls through into exception handler" bb.isExcept = true - bb.params = build(dst.p, Params([lo])) + bb.params = build(dst.p, x.info, Params([lo])) of Asgn([lv], [e]): - bb.stmts.add build(dst.st, Asgn(lv, e)) + bb.stmts.add build(dst.st, x.info, Asgn(lv, e)) of Store([t], [e0], [e1]): - bb.stmts.add build(dst.st, Store(t, e0, e1)) + bb.stmts.add build(dst.st, x.info, Store(t, e0, e1)) of Blit([e0], [e1], [e2]): - bb.stmts.add build(dst.st, Blit(e0, e1, e2)) + bb.stmts.add build(dst.st, x.info, Blit(e0, e1, e2)) of Clear([e0], [e1]): - bb.stmts.add build(dst.st, Clear(e0, e1)) + bb.stmts.add build(dst.st, x.info, Clear(e0, e1)) of Call(pr, ...[e]): - bb.stmts.add build(dst.st, Call(pr, ...e)) + bb.stmts.add build(dst.st, x.info, Call(pr, ...e)) of Call([t], [e0], ...[e1]): - bb.stmts.add build(dst.st, Call(t, e0, ...e1)) + bb.stmts.add build(dst.st, x.info, Call(t, e0, ...e1)) of Drop([e]): - bb.stmts.add build(dst.st, Drop(e)) + bb.stmts.add build(dst.st, x.info, Drop(e)) of CheckedCall([t], [e0], ...[e1], tgt): - commitBlock bbs, bb, build(dst.ex, CheckedCall(t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCall(t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) startBlock(bb) of CheckedCall(pr, ...[e], tgt): - commitBlock bbs, bb, build(dst.ex, CheckedCall(pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCall(pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) startBlock(bb) of CheckedCallAsgn(lo, [t], [e0], ...[e1], tgt): - commitBlock bbs, bb, build(dst.ex, CheckedCallAsgn(lo, t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCallAsgn(lo, t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) startBlock(bb) of CheckedCallAsgn(lo, pr, ...[e], tgt): - commitBlock bbs, bb, build(dst.ex, CheckedCallAsgn(lo, pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCallAsgn(lo, pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) startBlock(bb) of Return(): - commitBlock bbs, bb, build(dst.ex, Return()) + commitBlock bbs, bb, build(dst.ex, x.info, Return()) of Return([e]): - commitBlock bbs, bb, build(dst.ex, Return(e)) + commitBlock bbs, bb, build(dst.ex, x.info, Return(e)) of Raise([e], tgt): - commitBlock bbs, bb, build(dst.ex, Raise(e, ^target(tgt, map))) + commitBlock bbs, bb, build(dst.ex, x.info, Raise(e, ^target(tgt, map))) of Branch([e], go0, go1): - commitBlock bbs, bb, build(dst.ex, Branch(e, ^goto(go0, map), ^goto(go1, map))) + commitBlock bbs, bb, build(dst.ex, x.info, Branch(e, ^goto(go0, map), ^goto(go1, map))) of Unreachable(): - commitBlock bbs, bb, build(dst.ex, Unreachable()) + commitBlock bbs, bb, build(dst.ex, x.info, Unreachable()) of Loop(i): - commitBlock bbs, bb, build(dst.ex, Loop(i(^map[i.val]))) + commitBlock bbs, bb, build(dst.ex, x.info, Loop(i(^map[i.val]))) proc scanStmt(ir: src.st, map: var Table[int64, int], wasJoin: var bool, next: var int) = @@ -397,17 +397,17 @@ proc flattenPaths(ir: L5): L4 {.pass.} = proc filter(x: src.lv, args: var seq[dst.e]): (dst.ro, src.t) = match x: of Deref(t, [e]): - (build(dst.ro, Deref(^(t -> dst.t), e)), t) + (build(dst.ro, x.info, Deref(^(t -> dst.t), e)), t) of Field(lv, i): let (root, t) = filter(lv, args) - args.add build(dst.e, i) + args.add build(dst.e, x.info, i) (root, elemAt(t, i.val)) of At(lv, [e]): let (root, t) = filter(lv, args) args.add e (root, arrayElem(t)) of lo: - (build(dst.ro, lo), locals[ord lo.val]) + (build(dst.ro, x.info, lo), locals[ord lo.val]) proc typ(x: src.t): dst.t {.generated.} proc bblock(x: src.bb): dst.bb {.generated.} @@ -498,18 +498,18 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = proc getType(x: src.e): dst.t = match x: - of Le(_, _, _): build dst.t, UInt(i(1)) - of Lt(_, _, _): build dst.t, UInt(i(1)) - of Eq(_, _, _): build dst.t, UInt(i(1)) - of Not(_): build dst.t, UInt(i(1)) - of Addr(_): build dst.t, Ptr() - of Nil(): build dst.t, Ptr() - of Copy(g): discard g; build dst.t, Ptr() + of Le(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Lt(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Eq(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Not(_): build dst.t, NoSLoc, UInt(i(1)) + of Addr(_): build dst.t, NoSLoc, Ptr() + of Nil(): build dst.t, NoSLoc, Ptr() + of Copy(g): discard g; build dst.t, NoSLoc, Ptr() of Copy(Path([t], ...any)): t of Copy(lo): locals[ord lo.val] of Call(pr, ...any): typ(retType(signatures[ord pr.val])) of Call(t, ...any): typ(retType(t)) - of ProcVal(_): build dst.t, Ptr() + of ProcVal(_): build dst.t, NoSLoc, Ptr() of i: discard i; unreachable() of fl: discard fl; unreachable() of e: @@ -520,16 +520,16 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = proc operand(x: src.e): dst.e = if needsSave: match x: - of Addr([lv]): build dst.e, Addr(lv) - of Nil(): build dst.e, Nil() - of i: build dst.e, i - of fl: build dst.e, fl + of Addr([lv]): build dst.e, x.info, Addr(lv) + of Nil(): build dst.e, x.info, Nil() + of i: build dst.e, x.info, i + of fl: build dst.e, x.info, fl else: let tmp = newTemp(getType(x)) needsSave = false - stmts.add build(dst.st, Asgn(tmp, ^expr(x))) + stmts.add build(dst.st, x.info, Asgn(tmp, ^expr(x))) needsSave = true - build dst.e, Copy(tmp) + build dst.e, x.info, Copy(tmp) else: expr(x) @@ -569,21 +569,21 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = # ^^ the hoisted expression isn't affected by side effects match it: of Call(t, ...e1): - stmts.add build(dst.st, Asgn(tmp, Call(^typ(t), ...args(t, e1)))) + stmts.add build(dst.st, it.info, Asgn(tmp, Call(^typ(t), ...args(t, e1)))) of Call(pr, ...e1): - stmts.add build(dst.st, Asgn(tmp, Call(pr, ...args(signatures[ord pr.val], e1)))) + stmts.add build(dst.st, it.info, Asgn(tmp, Call(pr, ...args(signatures[ord pr.val], e1)))) else: - stmts.add build(dst.st, Asgn(tmp, ^expr(it))) + stmts.add build(dst.st, it.info, Asgn(tmp, ^expr(it))) needsSave = true - result[i] = build(dst.e, Addr(tmp)) + result[i] = build(dst.e, it.info, Addr(tmp)) elif needsSave: let pt = typ(pt) let tmp = newTemp(pt) needsSave = false # ^^ the hoisted expression isn't affected by side effects - stmts.add build(dst.st, Asgn(tmp, ^expr(it))) + stmts.add build(dst.st, it.info, Asgn(tmp, ^expr(it))) needsSave = true - result[i] = build(dst.e, Copy(tmp)) + result[i] = build(dst.e, it.info, Copy(tmp)) else: result[i] = expr(it) @@ -602,7 +602,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = if isAggregate(rt): let rt = typ(rt) let tmp = newTemp(rt) - stmts.add build(dst.st, Call(pr, [...s, Addr(tmp)])) + stmts.add build(dst.st, x.info, Call(pr, [...s, Addr(tmp)])) needsSave = true build Copy(tmp) else: @@ -614,7 +614,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = if isAggregate(t): let rt = typ(rt) let tmp = newTemp(rt) - stmts.add build(dst.st, Call(^typ(t), ^operand(e0), [...s, Addr(tmp)])) + stmts.add build(dst.st, x.info, Call(^typ(t), ^operand(e0), [...s, Addr(tmp)])) needsSave = true build Copy(tmp) else: @@ -651,7 +651,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = if isAggregate(retType(t)): let tmp = newTemp(typ retType(t)) # the temporary needs to be copied to the actual target upon landing - delayed[i.val] = build(dst.st, Asgn(lo, Copy(tmp))) + delayed[i.val] = build(dst.st, x.info, Asgn(lo, Copy(tmp))) build CheckedCall(^typ(t), callee, [...args, Addr(tmp)], Goto(i), tgt) else: build CheckedCallAsgn(lo, ^typ(t), callee, ...args, Goto(i), tgt) @@ -660,7 +660,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = if isAggregate(retType(sig)): let tmp = newTemp(typ retType(sig)) # the temporary needs to be copied to the actual target upon landing - delayed[i.val] = build(dst.st, Asgn(lo, Copy(tmp))) + delayed[i.val] = build(dst.st, x.info, Asgn(lo, Copy(tmp))) build CheckedCall(pr, [...args(sig, e), Addr(tmp)], Goto(i), tgt) else: build CheckedCallAsgn(lo, pr, ...args(sig, e), Goto(i), tgt) @@ -703,8 +703,8 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = if outParam.isSome: match ex: of Return([e0 -> e]): - stmts.insert build(dst.st, Store(^getType(e0), Copy(^outParam.unsafeGet), e)), start - build(dst.ex, Return()) + stmts.insert build(dst.st, e0.info, Store(^getType(e0), Copy(^outParam.unsafeGet), e)), start + build(dst.ex, ex.info, Return()) else: exit(ex) else: exit(ex) reverse(stmts.toOpenArray(start, stmts.high)) @@ -741,7 +741,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = origLocals = t1 locals = map(t1, typ) if isAggregate(retType(t0)): - outParam = some newTemp(build(dst.t, Ptr())) + outParam = some newTemp(build(dst.t, NoSLoc, Ptr())) else: outParam = none dst.lo var blocks = newSeq[dst.bb](bb.len) @@ -749,7 +749,7 @@ proc aggregateParams(ir: L4): L3s2 {.pass.} = blocks[i] = bblock(it, i) # turn all aggregate parameter locals into pointers for it in params.items: - locals[ord it] = build(dst.t, Ptr()) + locals[ord it] = build(dst.t, NoSLoc, Ptr()) build ProcDef(^typ(t0), Locals(...locals), List(...blocks)) proc aggregatesToBlob(ir: L3s2, ptrsize: uint): L3s1 {.pass.} = @@ -811,7 +811,7 @@ proc aggregatesToBlob(ir: L3s2, ptrsize: uint): L3s1 {.pass.} = (typ, result) = (; match root: of Deref(t, [e]): (t, e) - of lo: (locals[ord lo.val], build(dst.e, Addr(lo)))) + of lo: (locals[ord lo.val], build(dst.e, root.info, Addr(lo)))) var offset = 0'i64 for it in elems.items: @@ -824,15 +824,15 @@ proc aggregatesToBlob(ir: L3s2, ptrsize: uint): L3s1 {.pass.} = # an array access with a dynamic index if offset > 0: # add the static offset computed so far: - result = build(dst.e, Offset(result, i(offset), i(1))) + result = build(dst.e, it.info, Offset(result, i(offset), i(1))) offset = 0 typ = typeOfElem(typ, 0) # apply the dynamic array element offset: - result = build(dst.e, Offset(result, ^expr(it), i(^size(typ)))) + result = build(dst.e, it.info, Offset(result, ^expr(it), i(^size(typ)))) if offset > 0: - result = build(dst.e, Offset(result, i(offset), i(1))) + result = build(dst.e, result.info, Offset(result, i(offset), i(1))) proc lvalue(x: src.lv): dst.lv {.transform.} = case x @@ -996,7 +996,7 @@ proc localsToBlob(ir: L3s1, ptrSize: uint): L3 {.pass.} = if x.val in marker: let typ = locals[ord x.val] let tmp = newTemp(typ) - stmts.add build(dst.st, Store(typ, Addr(x), Copy(tmp))) + stmts.add build(dst.st, NoSLoc, Store(typ, Addr(x), Copy(tmp))) tmp else: x @@ -1031,7 +1031,7 @@ proc localsToBlob(ir: L3s1, ptrSize: uint): L3 {.pass.} = for i, it in t1.pairs: if Local(i) in marker and not isBlob(it): let (size, align) = sizeAndAlignment(it) - locals[i] = build(dst.t, Blob(size, align)) + locals[i] = build(dst.t, NoSLoc, Blob(size, align)) build ProcDef(^typ(t0), Locals(...locals), List(...bbs)) @@ -1061,7 +1061,7 @@ proc legalizeBlobOps(ir: L3): L2 {.pass.} = proc operand(x: src.e): dst.e = match x: - of Copy([lv0 -> lv]): build(dst.e, Addr(lv)) + of Copy([lv0 -> lv]): build(dst.e, x.info, Addr(lv)) of Load(_, [e]): e else: unreachable() @@ -1185,13 +1185,13 @@ proc stackAlloc(ir: L2): L1 {.pass.} = # the body stays as is, only the header needs to be modified build ProcDef(^typ(t), i(0), Locals(...map(t1, typ)), List(...map(bb, bblock))) else: - framePointer = build(dst.e, Copy(lo(^Local(nextId)))) + framePointer = build(dst.e, x.info, Copy(lo(^Local(nextId)))) var filtered = newSeq[dst.t](nextId + 1) for i, it in locals.pairs: if not it.onStack: filtered[it.offset] = typ(t1[i]) - filtered[^1] = build(dst.t, Ptr()) + filtered[^1] = build(dst.t, NoSLoc, Ptr()) var blocks = newSeq[dst.bb](bb.len) for i, it in bb.pairs: @@ -1199,7 +1199,7 @@ proc stackAlloc(ir: L2): L1 {.pass.} = # pass the frame pointer as an extra argument match it: of Block(Params(...lo), ...[st], [ex]): - blocks[i] = build(dst.bb, Block(Params([...map(lo, mapLocal), lo(nextid)]), ...st, ex)) + blocks[i] = build(dst.bb, it.info, Block(Params([...map(lo, mapLocal), lo(nextid)]), ...st, ex)) else: unreachable() else: @@ -1231,7 +1231,7 @@ proc inlineTypes(ir: L1): LPtr {.pass.} = match it: of ProcTy(...any): if pos != i: - types[i] = build(dst.t, tid(pos)) # needs a fixup + types[i] = build(dst.t, NoSLoc, tid(pos)) # needs a fixup inc pos else: types[i] = typ(it)