diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1b06cc60..ce66efff 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -168,3 +168,25 @@ jobs: env: TEST_FILE: tests/expr/t06_call_proc_with_tuple_return_type.test TEST_OUTPUT: "(TupleCons 100 200) : (TupleTy (IntTy) (IntTy))(Done 0)" + + docs: + name: "Test documentation" + runs-on: "ubuntu-latest" + + needs: context + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + filter: tree:0 + + - uses: nim-works/setup-nimskull@0.1.2 + with: + nimskull-version: "${{ needs.context.outputs.nimskull-version }}" + + - name: Build koch + run: nim c --outdir:bin koch.nim + + - name: Build and test documentation + run: bin/koch docs diff --git a/koch.nim b/koch.nim index 73840012..4fca1d3f 100644 --- a/koch.nim +++ b/koch.nim @@ -17,6 +17,7 @@ Commands: generate generates the various language-related modules build-defs verifies the language definitions and generates the textual representation for them + docs builds all documentation """ Programs = { "tester" : ("tools/tester.nim", true), @@ -32,6 +33,10 @@ Commands: DefaultGenerated = "generated" ## the default path for the generated modules + RstList = @[ + "nanopass/manual.rst" + ] + var nimExe = findExe("nim") verbose = true @@ -58,6 +63,14 @@ proc compile(file: sink string, name: string, extra: varargs[string]): bool = args.add file result = run(nimExe, args) +proc rstToHtml(file: sink string, dir: string, extra: varargs[string]): bool = + ## Runs RST-to-HTML conversion on `file`, using `dir` as the + ## output directory. + var args = @["rst2html", "--nimcache:build/docs/", "--outdir:" & dir] + args.add extra + args.add file + result = run(nimExe, args) + proc check(file: sink string, extra: varargs[string]): bool = ## Runs the ``check`` command on the given NimSkull `file`. var args = @["check"] @@ -184,6 +197,23 @@ proc buildDefs(args: string): bool = result = true +proc buildDocs(args: string): bool = + ## Handles building and testing the various documentation. + if args.len > 0: + return false + + let docroot = getCurrentDir() / "build" / "docs" + createDir(docroot) + + for it in RstList.items: + # the output is not relevant at the moment, so it's simply dumped into the + # artifacts directory + if not rstToHtml(it, docroot): + echo "Failure" + quit(1) + + result = true + proc showHelp(): bool = ## Shows the help text. echo HelpText @@ -213,6 +243,8 @@ while true: generate(opts.cmdLineRest) of "build-defs": buildDefs(opts.cmdLineRest) + of "docs": + buildDocs(opts.cmdLineRest) of "help": showHelp() else: diff --git a/nanopass/README.md b/nanopass/README.md new file mode 100644 index 00000000..95bfee28 --- /dev/null +++ b/nanopass/README.md @@ -0,0 +1,2 @@ +This directory contains the implementation of the nanopass framework used by +various parts of Phy. diff --git a/nanopass/asts.nim b/nanopass/asts.nim new file mode 100644 index 00000000..c43d8b50 --- /dev/null +++ b/nanopass/asts.nim @@ -0,0 +1,340 @@ +## Implements the nanopass framework specific storage types for ASTs, as well +## as various fundamental, language-agnostic operations on ASTs. The types are +## layered on top of `PackedTree `_. + +import passes/trees + +export trees.NodeIndex +# export some fundamental tree traversal and query operations, so that the +# nanopass implementation doesn't have to use bindSym everywhere +export trees.`[]`, trees.next, trees.child, trees.len +export trees.TreeNode + +type + SLocRef* = distinct range[0'u32 .. uint32((1 shl 24) - 1)] + ## Local reference to a `SourceLoc <#SourceLoc>`_. + + Tag* = distinct uint32 + ## 8-bit tag, 24-bit source location reference + AstNode* = TreeNode[Tag] + Tree* = PackedTree[Tag] + + SourceLoc* = object + ## Describes a source location, which may span multiple lines and columns. + file*: uint32 ## ID of the packed file name + sline*, eline*: uint32 + scol*, ecol*: uint16 + + # note: the fields are exported so that the nanopass machinery can access + # them. User code should, in most cases, not access the fields directly + Ast*[L: object, Storage: object] = object + tree*: Tree + ## leaked implementation detail, don't use + storage*: ref Storage + ## leaked implementation detail, don't use + records*: typeof(L.meta.records) + ## leaked implementation detail, don't use + + Production*[L: object, N: static string] = object + ## Represents a reference to a production of non-terminal `N` belonging to + ## language `L`. + index*: NodeIndex + ## leaked implementation detail, don't use + + Value*[T] = object + ## Represents a reference to a value with type `T` that's a terminal in + ## an AST. + id*: uint32 + ## leaked implementation detail, don't use + + RecordRef*[L: object, N: static string] = object + ## A reference to a record of a language. + id*: uint32 + ## leaked implementation detail, don't use + + ChildSlice*[T: Production or RecordRef or Value, Cursor] = object + ## A lightweight reference to a sequence of sibling nodes. The reference + ## must not outlive the spawned-from tree. + tree: ptr Tree + start: Cursor + len: uint32 + + Cursor* = distinct NodeIndex + ## A cursor into a tree without indirections. + IndCursor* = distinct NodeIndex + ## A cursor into a tree with indirections. + + LineCol* = tuple + line: uint32 + column: uint16 + +const + RefTag* = 128'u8 + ## the node used internally for indirections + NoSLoc* = SLocRef(0) + ## represents the absence of a source location + +proc `==`*(a, b: SLocRef): bool {.borrow.} + +template tag*(n: AstNode): uint8 = + ## The node's tag. + # simply cut off the higher bits + cast[uint8](n.kind) + +template info*(n: AstNode): SLocRef = + ## The node's source location information. + cast[SLocRef](uint32(n.kind) shr 8) + +template isAtom*(x: Tag): bool = + ## Whether `x` is the tag of a leaf node. + # the predicate is required for using an uint8 as a ``PackedTree`` tag + cast[uint8](x) >= RefTag + +{.push stacktrace: off, profiler: off.} + +proc `tag=`*(n: var AstNode, tag: uint8) {.inline.} = + ## Sets the tag of `n` to `tag`. A low-level operation. + # overwrite only the lower 8-bit + n.kind = Tag((uint32(n.kind) and 0xFFFFFF00'u32) or uint32(tag)) + +proc `==`(a, b: AstNode): bool {.inline.} = + ## Compares the nodes for equality, ignoring source location info. + a.tag == b.tag and a.val == b.val + +{.pop.} + +template node*(tag: uint8, v: uint32): AstNode = + ## Construct a node with the given tag and value and no source location. + AstNode(kind: cast[Tag](tag), val: v) + +template node*(tag: uint8, loc: SLocRef, v: uint32): AstNode = + ## Construct a node with the given tag, source location, and value. + AstNode(kind: cast[Tag](uint32(tag) or (uint32(loc) shl 8)), val: v) + +# ----- source location management ----- + +template convert[T](v: Positive): T = + if v > high(typeof(T)).int: high(typeof(T)) + else: T(v) + +proc newSourceLoc*[S](s: var S, file: string, line, col: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + let rline = convert[uint32](line) + let rcol = convert[uint16](col) + let loc = SourceLoc( + file: pack(s, file), + sline: rline, eline: rline, + scol: rcol, ecol: rcol + ) + SLocRef(1 + pack(s, loc)) + +proc newSourceLoc*[S](s: var S, file: string, + sline, scol, eline, ecol: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + let loc = SourceLoc( + file: pack(s, file), + sline: convert[uint32](sline), + eline: convert[uint32](eline), + scol: convert[uint16](scol), + ecol: convert[uint16](ecol) + ) + SLocRef(1 + pack(s, loc)) + +proc newSourceLoc*(ast: var Ast, file: string, line, col: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + newSourceLoc(ast.storage[], file, line, col) + +proc newSourceLoc*(ast: var Ast, file: string, + sline, scol, eline, ecol: Positive): SLocRef = + ## Creates a new source location and returns a reference to it. + newSourceLoc(ast.storage[], file, sline, scol, eline, ecol) + +proc file*(ast: Ast, info: SLocRef): lent string {.inline.} = + ## The file for the given, valid `info`. + mixin unpack + assert info != NoSLoc + unpack(ast.storage[], + unpack(ast.storage[], uint32(info) - 1, SourceLoc).file, string) + +proc span*(ast: Ast, info: SLocRef): tuple[s, e: LineCol] = + ## The source span of the given, valid `info`. Both the start and end + ## are inclusive. + mixin unpack + let s = unpack(ast.storage[], uint32(info) - 1, SourceLoc) + ((s.sline, s.scol), (s.eline, s.ecol)) + +# ----- slice implementation ----- + +proc slice*[T, C](tree: ptr Tree, start: C, len: uint32): ChildSlice[T, C] = + ## Creates a reference to `len` sibling nodes starting at `start`. + ChildSlice[T, C](tree: tree, start: start, len: len) + +template load[T, C](tree: Tree, c: C): T = + mixin get, pos + when T is Production: T(index: get(tree, c)) + elif T is RecordRef: T(id: tree[pos(c)].val) + else: T(id: tree[pos(c)].val) + +iterator items*[T, C](s: ChildSlice[T, C]): T = + mixin advance + var c = s.start + for _ in 0..= uint64(s.len): + raise IndexDefect.newException("index out of bounds") + else: + if uint64(i) >= uint64(s.len): + raise IndexDefect.newException("index out of bounds") + + var n = s.start + for _ in 0.. 0: + let n = tree[pos(a)] + if n != tree[pos(b)]: + return false + elif not isAtom(n.kind): + i += n.val + advance(tree, a) + advance(tree, b) + dec i + + result = true + +proc equal*(tree: Tree, a, b: IndCursor): bool = + ## Compares the nodes/sub-trees at `a` and `b` for structural equality. + if pos(a) == pos(b): + return true + + # needs a stack for bookkeeping + var (a, b) = (a, b) + var stack: seq[(IndCursor, IndCursor, uint32)] + var i = 1'u32 + while true: + let na = tree[pos(a)] + let nb = tree[pos(b)] + if na != nb: + if na.tag == RefTag: + stack.add (a, b, i) + i = 1 + a = IndCursor(na.val) + continue + elif nb.tag == RefTag: + stack.add (a, b, i) + i = 1 + b = IndCursor(nb.val) + continue + return false + elif na.tag == RefTag: + stack.add (a, b, i) + i = 1 + a = IndCursor(na.val) + b = IndCursor(nb.val) + continue + elif not isAtom(na.kind): + i += na.val + dec i + if i == 0: + if stack.len == 0: + break + (a, b, i) = stack.pop() + else: + advance(tree, a) + advance(tree, b) + + result = true diff --git a/nanopass/helper.nim b/nanopass/helper.nim new file mode 100644 index 00000000..4fa7495e --- /dev/null +++ b/nanopass/helper.nim @@ -0,0 +1,9 @@ +## Implements some helper and utility routines for working with NimNode AST. + +import std/[macros] + +proc makeError*(msg: string, info: NimNode): NimNode = + ## Creates an error statement reporting `msg` at the given source location. + let it = nnkExprColonExpr.newTree(ident"error", newStrLitNode(msg)) + copyLineInfo(it, info) + result = nnkPragma.newTree(it) diff --git a/nanopass/manual.rst b/nanopass/manual.rst new file mode 100644 index 00000000..1780973f --- /dev/null +++ b/nanopass/manual.rst @@ -0,0 +1,246 @@ + +Nanopass Framework +================== + +This document describes how to use the nanopass framework, what is allowed, and +what not. It only describes the framework's interface, not how things are +implemented internally. + +Nanopass Compiler Overview +-------------------------- + +A nanopass compiler is a compiler implemented as a series of narrow *passes*, +which *transform*, *analyze*, or otherwise produce abstract syntax trees (=AST) +according to well-defined grammars. + +The idea is that passes focus on small, specific tasks, with tree traversal +boilerplate generated automatically. + +Glossary +-------- + +record + a heterogeneous map from names to values, whose shape (number of entries and + their names and types) is fixed + +terminal + in the context of the nanopass framework, a type that's defined outside a + language. This corresponds to a terminal in a formal grammar, hence the name + +form + a named schema for a term. Can also be viewed as a named tuple + +production + either a form, a record, or a terminal + +non-terminal + a set of productions + +meta-variable + ranges over the terms a type. In the nanopass framework, they're mostly + just short-hands for types + +language + a collection of terminals, records, and forms, plus the non-terminals for + connecting them together + +.. TODO rewrite this section. It should use the correct terminology while still + being approachable to someone not overly familiar with formal grammars + +Usage +----- + +Language Definition +~~~~~~~~~~~~~~~~~~~ + +Languages are defined via the `defineLanguage` macro, which has two forms. + +From Scratch +~~~~~~~~~~~~ + +The first form `defineLanguage` form takes two arguments, an identifier and +a body, and defines a language from scratch. If the body is valid, a NimSkull +type is bound to the identifier. + +.. note:: + + The type bound to the identifier is not meant to be used as a type for + values. Rather, it acts as a namespace through which entities defined in the + language are accessed. + +The body consists of a sequence of terminal, record, and non-terminal definitions. + +.. code-block:: nim + :test: "nim c $1" + + import nanopass/nanopass + + defineLanguage L0: + int(i) # definition of a terminal + rec(r) ::= (field: i) # definition of a record + expr(e) ::= i # definition of a non-terminal + + # this defines a language `L0` with: + # * a terminal of type `int`, ranged over by meta-variable `i` + # * a record type with name `rec`, ranged over by meta-variable `r`. The + # record has a single field called `field`, whose values must be of + # type `int` + # * a non-terminal with name `expr`, ranged over by meta-variable `e`. Where + # this non-terminal is expected, there must be value of type `int` + +Meta-variables must be unique. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + float(i) # error: 'i' name already in use + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + integer(i) ::= i # error: 'i' name already in use + +All meta-variables defined in the body are accessible in all non-terminals, +regardless of the declarations' order. + +.. code-block:: nim + :test: "nim c $1" + + import nanopass/nanopass + + defineLanguage L0: + expr(e) ::= i + int(i) + +No two non-terminals may have the same name. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + expr(e) ::= i + expr(b) ::= i # error: 'expr' name already in use + +All types share the same namespace. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + int(e) ::= i # a type with name 'int' already exists + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + int(e) ::= (field: i) # a type with name 'int' already exists + +Types and meta-variables also share a namespace, meaning that it's not +possible to give a name to a type already used for a meta-variable, +and vice versa. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + int(i) + i(e) ::= i # error: 'i' already in use + + +For terminal types, the name refer to a NimSkull type that exists at the time +`defineLanguage` is expanded. + +.. code-block:: nim + :test: "nim c $1" + :status: 1 + + import nanopass/nanopass + + defineLanguage L0: + MyType(i) + expr(e) ::= i + + type MyType = object # too late, must be defined before the language + +.. TODO continue + +Define Language Via Difference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The second form of `defineLanguage` takes takes three arguments, two +identifiers and a body, and defines a language by applying the diff provided +by the body to the base language. If the result is valid, a NimSkull +type is bound to the identifier. + +The second argument must be the name of an in-scope identifier to which a +language definition was previously bound via `defineLanguage`. + +.. TODO continue + +AST +--- + +.. TODO continue + +Pattern Matching +---------------- + +The nanopass framework provides a facility for comprehending AST fragments +using pattern matching, via the `match` routine. + +.. TODO continue + +Passes +------ + + +.. TODO continue + +The `build` Form +~~~~~~~~~~~~~~~~ + +Passes that produce an AST have a special macro available within their body: +the `build` macro. It's used to create AST fragments in a statically type-safe +manner. + +.. TODO continue + +Value To Language +~~~~~~~~~~~~~~~~~ + +.. TODO continue + +Language To Language +~~~~~~~~~~~~~~~~~~~~ + +.. TODO continue + +Language To Value +~~~~~~~~~~~~~~~~~ + +.. TODO continue diff --git a/nanopass/nanopass.nim b/nanopass/nanopass.nim new file mode 100644 index 00000000..7d996bdd --- /dev/null +++ b/nanopass/nanopass.nim @@ -0,0 +1,33 @@ +## Implements the nanopass framework, which is a collection of macro DSLs for +## defining intermediate languages (their syntax and grammar) and passes. +## +## This is the entrypoint of the library. + +import + nanopass/[ + asts, + nplanggen, + npmatch, + npbuild, + npparser, + nppass, + nppatterns, + npunparser + ] + +export asts +export nppatterns.matches +export npbuild.build, npmatch.match, npunparser.unparse, npparser.parseAst +export nppass.pass, nppass.inpass, nppass.outpass +export nppass.get + +macro defineLanguage*(name, body: untyped) = + ## Creates a language definition and binds it to a const symbol with the + ## given name. + defineLanguageImpl(name, nil, body) + +macro defineLanguage*(name, base, body: untyped) = + ## Creates a language definition, extending `base`, and binds it to a const + ## symbol with the given name. Extension doesn't imply a direction in this + ## context. + defineLanguageImpl(name, base, body) diff --git a/nanopass/npbuild.nim b/nanopass/npbuild.nim new file mode 100644 index 00000000..6db9bca8 --- /dev/null +++ b/nanopass/npbuild.nim @@ -0,0 +1,629 @@ +## Implements the `build` macro, for constructing records and abstract +## syntax trees. + +import std/[genasts, macros, strformat, tables] +import nanopass/[asts, helper, nplang, nppatterns] + +type + Candidate = object + ## Represents a potential form candidate matching the input construction. + tag: int ## form tag + types: seq[int] + ## for each operand in the input construction, the required type + min: int ## minimum number of items a dynamic operand must have + max: int ## maximum number of items a dynamic operand can have + +proc toPrefix(cand: Candidate, start: int): NimNode = + ## Turns the type sequence starting at `start` into a prefix (to be used + ## in a prefix tree). + # note: for the sake of efficiency, the prefix tree is stored as an + # AST using a tiny custom grammar + result = nnkBracket.newTree() + for i in start.. c.max): + break + inc at + + tree.insert(at, tup) + of nnkBracket: + var i = i + var j = 0 + while j < tree.len - 1 and tree[j].intVal.int == c.types[i]: + inc i + inc j + + if j == tree.len - 1: + # the sequences are the same so far; merge the tails + insert(tree[j], c, i) + else: + # fork where the sequences start to differ + let old = nnkBracket.newTree(tree[(j + 1)..^1]) + tree.del(j + 1, tree.len - j - 1) + tree[j] = nnkCurly.newTree( + nnkPar.newTree(tree[j], old), + nnkPar.newTree(newIntLitNode(c.types[i]), toPrefix(c, i + 1))) + else: + unreachable() + +# ---------- helper routines ---------- + +{.push stacktrace: off.} + +proc lookup[E; M: tuple](): auto {.compileTime.} = + var x: M + for it in fields(x): + when E is typeof(it[0]): + return it[1] + +proc lengthError(len: int) {.noinline.} = + raise ValueError.newException( + fmt"no form is able to fit the expanded sequence (length = {len}") + +proc append[L, U](ast: var Ast[L, auto], info: SLocRef, x: Value[U]) = + ast.tree.nodes.add node( + typeof(lookup[U, L.meta.term_map]()).V, + info, + x.id) + +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: RecordRef) = + ast.tree.nodes.add node( + typeof(lookup[typeof(x), L.meta.record_map]()).V, + info, + x.id) + +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: Production) = + # the ref itself doesn't need source location info + ast.tree.nodes.add node(RefTag, uint32(x.index)) + +proc append[L](ast: var Ast[L, auto], info: SLocRef, x: openArray) = + for it in x.items: + append[L](ast, info, it) + +template coerce[S, T, U](s: S, val: U, _: typedesc[Value[T]]): Value[T] = + mixin pack + when T is U: + Value[T](id: pack(s, val)) # no coercion is necessary + else: + Value[T](id: pack(s, T(val))) # try a coercion, an error is fine + +{.pop.} + +# ---------- macro implementation ---------- + +proc containsForm(lang: LangInfo, typ: LangType, form: int): bool = + if form in typ.forms: + result = true + else: + result = false + for it in typ.sub.items: + if lang.types[it].kind == tkNonTerminal and + containsForm(lang, lang.types[it], form): + result = true + break + +proc buildForm(lang: LangInfo, typ: int, ast, info, e: NimNode): NimNode + +proc buildRecord(lang: LangInfo, ast, info, e: NimNode): NimNode = + ## Translates a record construction form from the `build` language + ## to NimSkull. + assert e.kind == nnkObjConstr + let name = e[0].strVal + let typ = lang.map.getOrDefault(name, -1) + if typ == -1: + return makeError(fmt"no meta-variable or record called '{name}' exists", + e[0]) + elif lang.types[typ].kind != tkRecord: + return makeError(fmt"type '{name}' is not a record", e[0]) + + let tmp = genSym("") + let mvar = ident(lang.types[typ].mvar) + result = newStmtList() + result.add newVarStmt(tmp, quote do: default(typeof(`ast`.records.`mvar`[0]))) + for i in 1.. happens when there's a static type error + err = got + else: + body.add emit(lang, c, it[1], n, i+1) + result.add nnkElifBranch.newTree(cond, body) + + if result.len == 0: + # none of the variants are viable + result = err + else: + result.add nnkElse.newTree( + makeError("not a valid production for this position", n[i])) + of nnkIfExpr: + proc genBody(ctx: Context, tup: NimNode): NimNode = + if ctx.start.isNil: + newStmtList() # the node tag is set already + else: + let start = ctx.start + let id = tup[2] + quote do: `ast`.tree.nodes[`start`].tag = `id` + + let expanded = c.expanded + if expanded != nil: + # only when there's a list to expand can more than one shape of a + # form statically match for a construction. Run-time disambiguation + # is required + result = nnkIfStmt.newTree() + for it in t.items: + var cond = ident"true" + if it[0].intVal == it[1].intVal: + # no bound check is needed + let m = it[0] + cond = quote do: len(`expanded`) == `m` + else: + if it[0].intVal > 0: + let val = it[0] + cond = nnkInfix.newTree(ident"and", cond, + quote do: len(`expanded`) >= `val`) + if it[1].intVal < high(int): + let val = it[1] + cond = nnkInfix.newTree(ident"and", cond, + quote do: len(`expanded`) <= `val`) + + result.add nnkElifExpr.newTree(cond, genBody(c, it)) + + if result.len == 1 and result[0][0].kind == nnkIdent: + # no check is necessary + result = result[0][1] + else: + result.add nnkElse.newTree( + genAst(expanded) do: lengthError(len(expanded))) + else: + result = genBody(c, t[0]) + of nnkIntLit: + result = process(lang, lang.types[t.intVal], n[i + 1]) + of nnkNilLit: + result = newStmtList() + else: + unreachable() + + proc appendLenExpr(to: var NimNode, n: NimNode, start: int) = + for i in start.. 0: + result.add ", " + if it.repeat: + result.add "..." + result.add lang.types[it.typ].mvar + result.add ")" diff --git a/nanopass/nplangdef.nim b/nanopass/nplangdef.nim new file mode 100644 index 00000000..fbe5d7d4 --- /dev/null +++ b/nanopass/nplangdef.nim @@ -0,0 +1,753 @@ +## Implements the language definition parsing and processing. + +import std/[macros, intsets, sets, strformat, strutils, tables] + +type + # Core types capturing a defined language + Elem* = object + ## Element of a form. + typ*: string + ## the actual type + repeat*: bool + + Form* = object + ## Semantic representation of a syntax form. + name*: string + elems*: seq[Elem] + + OrigForm* = object + ## Source-level-ish representation of a syntax form description. + vars*: seq[string] + ## the metavars-as-written for the form's elements + semantic*: int + ## index of the semantic representation + + Terminal* = object + mvars*: seq[string] + ## the meta-variables for ranging over values of the type + + NonTerminal* = object + mvars*: seq[string] + ## the meta-variables for ranging over the productions + vars*: seq[tuple[mvar, typ: string]] + ## meta-variables used as productions + forms*: seq[OrigForm] + ## forms used as productions + + Record* = object + mvars*: seq[string] + ## the meta-variables for ranging over the record instances + fields*: seq[tuple[name, mvar, typ: string]] + + LangDef* = object + ## A checked and pre-processed language definition, carrying enough + ## source-level information necessary for implementing inheritance. + terminals*: Table[string, Terminal] + ## the terminals of the language + nterminals*: Table[string, NonTerminal] + ## the non-terminals of the language + records*: Table[string, Record] + ## the records of the language + forms*: seq[Form] + ## all syntax forms present in the language + entry*: string + ## name of the non-terminal to use as the entry point + +type + # Intermediate types meant to bridge macro language to core types + ParsedForm = object + ## A syntax form description as parsed from NimNode AST. + name: string + elems: seq[tuple[mvar: string, repeat: bool, info: NimNode]] + + NonTerminalDef = object + ## Pre-processed non-terminal definition. + name: NimNode + sub: seq[NimNode] + add: seq[NimNode] + + RecordDef = object + ## Pre-processed record definition. + name: NimNode + sub: seq[NimNode] + add: seq[NimNode] + +type + # Extra types unrelated to the ones above + Relation = enum + Disjoint, Overlap, ProblematicPrefix, Same + +template findIt[T](s: seq[T], predicate: untyped): untyped = + ## Version of ``find`` that allows providing an inline predicate, + ## evaluated for every checked item. + var r = -1 + block: + for i, it {.inject.} in s.pairs: + if predicate: + r = i + r + +proc `$`(x: Form): string = + result = x.name + result.add "(" + for i, it in x.elems.pairs: + if i > 0: + result.add ", " + if it.repeat: + result.add "..." + result.add it.typ + result.add ")" + +proc `$`*(x: LangDef): string = + for it in x.forms.items: + result.add $it + result.add "\n" + +proc `==`(x, y: Elem): bool = + x.typ == y.typ and x.repeat == y.repeat + +proc `==`(x, y: Form): bool = + ## Compares `x` and `y`, which must belong to the same language, for + ## equality. + x.name == y.name and x.elems == y.elems + +proc compare(def: LangDef, a, b: Form): Relation = + ## Computes the relation between `a` and `b`. Commutative. + if a.name != b.name: + return Disjoint + + proc contains(def: LangDef, nt: NonTerminal, typ: string): bool = + result = false + for it in nt.vars.items: + if it.typ == typ or + (it.typ in def.nterminals and + contains(def, def.nterminals[it.typ], typ)): + result = true + break + + proc gather(def: LangDef, nt: NonTerminal): (IntSet, HashSet[string]) = + ## Gathers the form and type productions of `nt`, including + ## transitive ones. + proc aux(def: LangDef, nt: NonTerminal, forms: var IntSet, + types: var HashSet[string]) = + for it in nt.vars.items: + if it.typ in def.nterminals: + aux(def, def.nterminals[it.typ], forms, types) + + for it in nt.forms.items: + forms.incl(it.semantic) + + aux(def, nt, result[0], result[1]) + + proc overlaps(def: LangDef, a, b: string): bool = + ## Whether the types `a` and `b` overlap (i.e., share inhabitants). + if a in def.nterminals: + if b in def.nterminals: + let (formsA, typesA) = gather(def, def.nterminals[a]) + let (formsB, typesB) = gather(def, def.nterminals[b]) + not(disjoint(formsA, formsB)) or not(disjoint(typesA, typesB)) + else: + contains(def, def.nterminals[a], b) + elif b in def.nterminals: + contains(def, def.nterminals[b], a) + else: + a == b + + result = Same + var ai = 0 + var bi = 0 + while ai < a.elems.len and bi < b.elems.len: + let got = + if a.elems[ai].typ == b.elems[bi].typ: + result # Overlap cannot go back to Same + elif overlaps(def, a.elems[ai].typ, b.elems[bi].typ): + Overlap + else: + Disjoint + + if a.elems[ai].repeat == b.elems[bi].repeat: + result = got + if result == Disjoint: + break + inc ai + inc bi + elif a.elems[ai].repeat: + if got == Disjoint: + result = Overlap + inc ai + else: + result = ProblematicPrefix + break + elif b.elems[bi].repeat: + if got == Disjoint: + result = Overlap + inc bi + else: + result = ProblematicPrefix + break + else: + result = got + inc ai + inc bi + + if result == Overlap and (ai < a.elems.len or bi < b.elems.len): + result = Disjoint + +proc checkName(target: LangDef, vars: Table[string, string], name: string, + info: NimNode) = + if name == "entry": + error("cannot use name 'entry'; it's a reserved name", info) + elif name in target.terminals: + error(fmt"terminal with name {name} already exists", info) + elif name in target.records: + error(fmt"record with name '{name}' already exists", info) + elif name in target.nterminals: + error(fmt"non-terminal with name {name} already exists", info) + elif name in vars: + error(fmt"'{name}' is already the name of a meta-variable for '{vars[name]}'", + info) + +proc parseForm(n: NimNode): ParsedForm = + ## Parses a form in the context of a language definition. No semantic checks + ## take place. + n[0].expectKind nnkIdent + result.name = n[0].strVal + for i in 1.. 0: + if name notin base.nterminals: + error(fmt"base language has no non-terminal with name '{name}'", + it.name) + + for prod in it.sub.items: + removeProd(base, prod, name) + + if it.add.len == 0 and + base.nterminals[name].vars.len == 0 and + base.nterminals[name].forms.len == 0: + # the non-terminal is and will stay empty, remove it + base.nterminals.del(name) + + if name in base.nterminals: + # remove the old names: + base.nterminals[name].mvars.shrink(0) + + # apply the record field removals to the base language: + for it in records.items: + it.name.expectKind nnkCall + it.name[0].expectKind nnkIdent + let name = it.name[0].strVal + if it.sub.len > 0: + if name notin base.records: + error(fmt"base language has no record named '{name}'", it.name) + + var fields = move base.records[name].fields + for field in it.sub.items: + let fname = field[0] + fname.expectKind nnkIdent + block search: + for i in 0.. remove it + base.records.del(name) + else: + base.records[name].fields = fields + + if name in base.records: + # remove the old meta-variables: + base.records[name].mvars.shrink(0) + + # update the var list with the to-be-inherited meta-vars: + for name, it in base.terminals.pairs: + for n in it.mvars.items: + vars[n] = name + + for name, it in base.nterminals.pairs: + for n in it.mvars.items: + vars[n] = name + + for name, it in base.records.pairs: + for n in it.mvars.items: + vars[n] = name + + # only now set the new meta-variable names for inherited non-terminals: + for it in def.items: + let name = it.name[0].strVal + if name in base.nterminals: + for i in 1.. 0 and name notin base.nterminals: + # it's a new non-terminal + checkName(result, vars, name, it.name) + var nt = NonTerminal() + for i in 1.. 0 and name notin base.records: + # it's a new record + checkName(result, vars, name, it.name) + var rec = Record() + for i in 1.. 0: + for a in it.add.items: + addProd(result, a, name) + + # add the new record fields: + for it in records.items: + let name = it.name[0].strVal + if it.add.len > 0: + var record: Record + # temporarily pop the record from the table, so that it can be accessed + # more easily + discard result.records.pop(name, record) + + for field in it.add.items: + let fname = field[0] + let typ = field[1] + fname.expectKind nnkIdent + typ.expectKind nnkIdent + if findIt(record.fields, it.name == fname.strVal) != -1: + error(fmt"record '{name}' already has field named '{fname.strVal}'", + fname) + + # the type name may refer to a meta-variable, but this is resolved + # at a later point + if typ.strVal in vars: + record.fields.add (fname.strVal, typ.strVal, vars[typ.strVal]) + else: + error(fmt"no meta-var with name '{typ.strVal}' exists", typ) + + result.records[name] = record + + # make sure all non-terminals are well-formed and exhibit some additional + # properties, which is possible only once all production additions were made + for name, nt in result.nterminals.pairs: + proc gather(def: LangDef, top, name: string, used: var IntSet, + included: var HashSet[string]) = + if name == top: + error(fmt"non-terminal '{top}' includes itself", info) + else: + included.incl(name) + + if name in def.nterminals: + for it in def.nterminals[name].vars.items: + gather(def, top, it.typ, used, included) + for it in def.nterminals[name].forms.items: + used.incl(it.semantic) + + proc checkRelation(def: LangDef, form: Form, against: IntSet, + name: string) = + # languages where there are non-terminals with forms that: + # * share the same name and overlap up until and including a list + # * share inhabitants + # are significantly harder to work with, so the aforementioned cases are + # simply disallowed. Allowing two forms in the language that share + # inhabitants means that assigning a type to a form instance requires + # context (i.e., a non-terminal), but that's an okay concession + for f in against.items: + let got = compare(def, form, def.forms[f]) + case got + of Disjoint, Same: + discard "all good" + of Overlap: + error("overlapping productions '$1' and '$2' in non-terminal '$3'" % + [$form, $def.forms[f], name], info) + of ProblematicPrefix: + error(("productions '$1' and '$2' of non-terminal '$3' are not " & + "disjoint at where a list is") % + [$form, $def.forms[f], name], info) + + var used = initIntSet() + var included: HashSet[string] + + for v in nt.vars.items: + var gotUsed: IntSet + var gotIncluded: HashSet[string] + gather(result, name, v.typ, gotUsed, gotIncluded) + # compute the form production relations against the existing ones first: + for it in gotUsed.items: + checkRelation(result, result.forms[it], used, name) + + # add the gathered sets to the total sets: + for it in gotUsed.items: + if containsOrIncl(used, it): + error("duplicate production '$1' in non-terminal '$2'" % + [$result.forms[it], name], info) + + for it in gotIncluded.items: + if containsOrIncl(included, it): + error("duplicate production '$1' in non-terminal '$2'" % + [it, name], info) + + for it in nt.forms.items: + if containsOrIncl(used, it.semantic): + error("duplicate production '$1' in non-terminal '$2'" % + [$result.forms[it.semantic], name], info) + else: + checkRelation(result, result.forms[it.semantic], used, name) + + # process the extra configuration declarations: + for it in config.items: + it[0].expectKind nnkIdent + case it[0].strVal + of "entry": + it[1].expectKind nnkIdent + let entry = it[1].strVal + if entry notin result.nterminals: + error(fmt"no non-terminal with the name '{entry}' exists", it[1]) + result.entry = entry + else: + error("identifier must be 'entry'", it[0]) + + if result.entry == "": + if def.len == 0: + error("cannot infer entry non-terminal, as none exists", info) + # use the last-defined non-terminal as the entry point + result.entry = def[^1].name[0].strVal + +proc makeLanguage*(body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code. + body.expectMinLen 1 + var terminals: seq[NimNode] + var nterminals: seq[NonTerminalDef] + var records: seq[RecordDef] + var config: seq[NimNode] + + # second pass: process the productions + proc extract(n: NimNode, list: var seq[NimNode]) = + case n.kind + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], list) + extract(n[2], list) + else: + error("expected '|'", n[0]) + of nnkCall, nnkIdent: + list.add n + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + if it[2].kind == nnkTupleConstr: + var rec = RecordDef(name: it[1]) + for elem in it[2].items: + elem.expectKind nnkExprColonExpr + rec.add.add elem + records.add rec + else: + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add) + nterminals.add nt + continue + of nnkCall: + terminals.add it + continue + of nnkAsgn: + config.add it + continue + else: + discard "report an error below" + + error("items must be of the form `a in b`, or `a ::= ...`", it) + + # to keep the implementation simple, a non-extension language is treated + # internally as an empty language definition being extended + buildLanguage(terminals, @[], nterminals, records, config, default(LangDef), + body) + +proc makeLanguage*(base: LangDef, body: NimNode): LangDef = + ## Creates a language definition from the ``defineLanguage`` DSL code + ## and `base`. + var add, sub: seq[NimNode] + var nterminals: seq[NonTerminalDef] + var records: seq[RecordDef] + var config: seq[NimNode] + + var body = body + if body.kind != nnkStmtList: + body = newStmtList(body) + + proc extract(n: NimNode, add, sub: var seq[NimNode]) = + case n.kind + of nnkPrefix: + if n[0].eqIdent("+"): + add.add n[1] + elif n[0].eqIdent("-"): + sub.add n[1] + else: + error("expected `+` or `-`", n[0]) + of nnkInfix: + if n[0].eqIdent("|"): + n.expectLen 3 + extract(n[1], add, sub) + extract(n[2], add, sub) + else: + error("expected '|'", n[0]) + else: + error("unexpected syntax: " & $n.kind, n) + + for it in body.items: + var handled = false + case it.kind + of nnkInfix: + if it[0].eqIdent("::="): + it.expectLen 3 + if it[2].kind == nnkTupleConstr: + var rec = RecordDef(name: it[1]) + it[2].expectMinLen(1) + + # the + or - prefix is part of the name slot + for elem in it[2].items: + elem.expectKind nnkExprColonExpr + elem[0].expectKind nnkPrefix + if elem[0][0].eqIdent("+"): + rec.add.add nnkExprColonExpr.newTree(elem[0][1], elem[1]) + elif elem[0][0].eqIdent("-"): + rec.sub.add nnkExprColonExpr.newTree(elem[0][1], elem[1]) + else: + error("expected '+' or '-'", elem[0][0]) + + records.add rec + else: + var nt = NonTerminalDef(name: it[1]) + extract(it[2], nt.add, nt.sub) + nterminals.add nt + + handled = true + of nnkPrefix: + if it[0].eqIdent("-"): + sub.add it[1] + handled = true + elif it[0].eqIdent("+"): + add.add it[1] + handled = true + of nnkCall: + # non-terminal with no change in productions + nterminals.add NonTerminalDef(name: it) + handled = true + of nnkAsgn: + config.add it + handled = true + else: + discard + + if not handled: + error("expected `-a`, `+a`, `a(...) ::= ...`, or `a = ...", it[0]) + + buildLanguage(add, sub, nterminals, records, config, base, body) diff --git a/nanopass/nplanggen.nim b/nanopass/nplanggen.nim new file mode 100644 index 00000000..a5b16688 --- /dev/null +++ b/nanopass/nplanggen.nim @@ -0,0 +1,140 @@ +## Implements the macros and routines handling the generative part of +## language definition. + +import std/[genasts, macros, tables] +import nanopass/[asts, nplang, nplangdef, nppatterns] + +macro makeMetaType(def: static LangInfo, typname: untyped): typedesc = + ## Expands to the tuple type storing the various internal-only information + ## about a language. + result = nnkTupleTy.newTree() + + let (csym, fsym) = (bindSym"PChoice", bindSym"PForm") + let nonTerminals = nnkTupleTy.newTree() + let terminals = nnkTupleConstr.newTree() + let recMap = nnkTupleConstr.newTree() + let records = nnkTupleTy.newTree() + + for typ in def.types.items: + case typ.kind + of tkTerminal: + terminals.add nnkTupleConstr.newTree( + ident(typ.name), + nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(typ.ntag))) + of tkRecord: + recMap.add nnkTupleConstr.newTree( + newDotExpr(copyNimTree(typName), ident(typ.mvar)), + nnkBracketExpr.newTree(bindSym"Static", newIntLitNode(typ.rtag))) + + let tup = nnkTupleTy.newTree() + for (name, t) in typ.fields.items: + tup.add newIdentDefs(ident(name), + newDotExpr(typName, ident(def.types[t].mvar))) + + # expose under the first meta-var there is for the type + records.add newIdentDefs(ident(typ.mvar), + nnkBracketExpr.newTree(ident"seq", tup)) + of tkNonTerminal: + let ln = ident(typName.strVal) + var p = ident"void" + for f in typ.forms.items: + let id = def.forms[f].ntag + p = quote do: + `csym`[`p`, `fsym`[`id`]] + + for v in typ.sub.items: + let id = ident(def.types[v].mvar) + p = quote do: + `csym`[`p`, `ln`.`id`] + + nonTerminals.add newIdentDefs(ident(typ.name), p) + + result.add newIdentDefs(ident"nt", nonTerminals) + if terminals.len > 0: + result.add newIdentDefs(ident"term_map", terminals) + if records.len > 0: + result.add newIdentDefs(ident"record_map", recMap) + + result.add newIdentDefs(ident"records", records) + +macro makeLanguageType(def: static LangDef, info: LangInfo, typName: untyped) = + ## Creates the type representing the language defined by `def`. This is the + ## type the nanopass-framework user passes around. + ## + ## The type also stores various information about the language that are + ## needed by the pass-related macros, encoded as types. + let fields = nnkRecList.newTree() + # the metavars are at the top level of the type, for easy access by + # the programmer + let prod = bindSym"Production" + for name, it in def.terminals.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree(bindSym"Value", ident(name))) + + for name, it in def.nterminals.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree( + prod, + ident(typName.strVal), + newStrLitNode(name))) + + for name, it in def.records.pairs: + for m in it.mvars.items: + fields.add newIdentDefs(ident(m), + nnkBracketExpr.newTree(bindSym"RecordRef", + ident(typName.strVal), + newStrLitNode(name))) + + # add the entry non-terminal: + fields.add newIdentDefs(ident"entry", + nnkBracketExpr.newTree(prod, + ident(typName.strVal), + newStrLitNode(def.entry))) + + # everything meant for internal use is stored in an anonymous record in + # the `meta` field, preventing name clashes and the fields showing up + # in auto-complete suggestions + fields.add newIdentDefs(ident"meta", + newCall(bindSym"makeMetaType", info, typName)) + + result = nnkTypeSection.newTree( + nnkTypeDef.newTree( + typName, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + fields))) + +macro genHelpers(typ, a, b: untyped) = + ## Generates the helper templates `def` and `idef`, which are used for + ## retrieving the `LangDef` and `LangInfo` instance for a language type, + ## respectively. + let (def, info) = (bindSym"LangDef", bindSym"LangInfo") + quote do: + template def(_: typedesc[`typ`]): `def` = `a` + template idef(_: typedesc[`typ`]): `info` = `b` + +proc defineLanguageImpl*(name, base, body: NimNode): NimNode = + body.expectKind nnkStmtList + body.expectMinLen 1 + if body[0].kind == nnkCommentStmt: + body.del(0) + + # don't use genAst for creating the makeLanguage call, as it messes with the + # source location + let setup = + if base.isNil: + newCall(bindSym"makeLanguage", newCall(bindSym"quote", body)) + else: + newCall(bindSym"makeLanguage", + newCall(ident"def", base), + newCall(bindSym"quote", body)) + result = genAst(setup, name): + const + def = setup + tmp = buildLangInfo(def) + makeLanguageType(def, tmp, name) + genHelpers(name, def, tmp) diff --git a/nanopass/npmatch.nim b/nanopass/npmatch.nim new file mode 100644 index 00000000..618a9707 --- /dev/null +++ b/nanopass/npmatch.nim @@ -0,0 +1,902 @@ +## Implements the high and low-level `match` macros, which provide the pattern- +## matching-over-ASTs functionality. + +import std/[macros, intsets, strformat, tables] +import nanopass/[asts, helper, nplang] + +type + FillProc* = proc(lang: LangInfo, idx: int, n, info: NimNode): NimNode + ## Type for a form or type fill callback. + ExpandConfig* = object + fillForm*: FillProc + ## called for filling-in the handling of forms. May be nil + fillType*: FillProc + ## called for filling-in the handling of types. May be nil + +proc defaultFillForm(lang: LangInfo, idx: int, n, info: NimNode): NimNode = + makeError(fmt"missing rule for '{render(lang, lang.forms[idx])}'", info) + +proc defaultFillType(lang: LangInfo, idx: int, n, info: NimNode): NimNode = + makeError(fmt"missing rule for '{lang.types[idx].name}'", info) + +proc parseVar(n: NimNode): string = + n.expectKind nnkIdent + let name = n.strVal + var e = name.high + # to get the name of the var, trim trailing numbers and a single underscore + while e >= 0 and name[e] in {'0'..'9'}: + dec e + + if e >= 0 and name[e] == '_': + dec e + + result = name[0..e] + +proc fits(lang: LangInfo, a, b: int): bool = + ## Computes whether a type with id `a` can appear where a type with id `b` + ## is expected. + if a == b: + result = true + elif lang.types[b].kind == tkNonTerminal: + for it in lang.types[b].sub.items: + if fits(lang, a, it): + return true + +proc countTags(lang: LangInfo, typ: LangType): int = + result = typ.forms.len + for it in typ.sub.items: + case lang.types[it].kind + of tkTerminal, tkRecord: + result += 1 + of tkNonTerminal: + result += countTags(lang, lang.types[it]) + +proc containsForm(lang: LangInfo, typ: LangType, fid: int): bool = + ## Whether type `typ` contains the form with id `fid`. + case typ.kind + of tkNonTerminal: + if fid in typ.forms: + true + else: + for it in typ.sub.items: + if containsForm(lang, lang.types[it], fid): + return true + false + of tkTerminal, tkRecord: + false + +proc makeTyped(e, typ, info: NimNode): NimNode = + typ.copyLineInfo(info) # the type tree carries the source location + nnkExprColonExpr.newTree(e, typ) + +proc fitTo(lang: LangInfo, typ: int, pat: NimNode): NimNode = + ## Fits a non-'...' typed pattern to the given type, returning either the + ## fitted pattern or nil. + + proc canMerge(n, into: NimNode): bool = + assert n.kind == into.kind and n.kind == nnkCall + result = true + for i in 1..", newEmptyNode(), quote do: dst.`id`) + copyLineInfo(call, n) + # the matched type needs to be inferred + result = makeTyped( + nnkTupleConstr.newTree(nnkExprEqExpr.newTree(n[0], call)), + newNimNode(nnkNilLit), n) + of nnkInfix: + if n[0].len != 3 or not eqIdent(n[0][0], "->"): + error("expected '->' infix call", n[0]) + let src = parseVar(n[0][1]) + if src notin lang.map: + error(fmt"no meta-variable with name '{src}' exists", n[0][1]) + + let id = ident(parseVar(n[0][2])) + let call = nnkInfix.newTree(ident"->", n[0][1], quote do: dst.`id`) + copyLineInfo(call, n) + # bind the matched value to the first identifier and the result of + # the application to the second identifier + result = makeTyped( + nnkTupleConstr.newTree( + n[0][1], + nnkExprEqExpr.newTree(n[0][2], call)), + newIntLitNode(lang.map[src]), n) + else: + error("expected identifier or '->' infix call'", n[0]) + of nnkPrefix: + if not n[0].eqIdent("..."): + error("only `...` is allowed as a prefix", n[0]) + + let tmp = parsePattern(lang, n[1]) + if tmp[1].kind notin {nnkIntLit, nnkEmpty, nnkNilLit}: + error("only '...' and '...any' are allowed", n) + # the '...' prefix is stripped from the expression + result = makeTyped(tmp[0], nnkBracket.newTree(tmp[1]), n) + of nnkIdent: + if n.eqIdent("_"): + # placeholder, type may be inferred + result = makeTyped(n, newEmptyNode(), n) + elif n.eqIdent("any"): + # an alias for '_' + result = makeTyped(ident"_", newEmptyNode(), n) + else: + # must be a meta-variable + let typ = parseVar(n) + if typ notin lang.map: + error(fmt"no meta-variable called {typ} exists", n) + + result = makeTyped(n, newIntLitNode(lang.map[typ]), n) + else: + error("syntax error", n) + +proc patternToString(n: NimNode; indent = 0): string = + case n.kind + of nnkCurly: + result = "{" + result.add repr(n[0]) + for i in 1.." + result.add "(" + for i in 1.. 1: + result.add ", " + result.add repr(n[i]) + if n.len > 2: + result.add ", " + result.add patternToString(n[^1], indent) + result.add ")" + of nnkPar: + result = "^" + result.add patternToString(n[0], indent) + of nnkEmpty: + result = "." + of nnkStmtList: + result = "" + of nnkTupleConstr: + result = "(" + result.add repr(n[0]) + result.add ", )" + else: + result = "" + +proc matchToNimskull(lang: LangInfo, name, ast, sel, e, els: NimNode, + config: ExpandConfig): NimNode = + ## Translates the match expression `e` to NimSkull AST. `els` is either + ## an `nnkElse` tree used for handling the rest of match, or nil, in which + ## case how to handle the rest of a match is dictated by `config`. `name` is + ## the language type expression, `ast` the AST lvalue, and `sel` is the + ## initial cursor. + let cursor = genSym("cursor") + let backup = genSym("backup") + + var stack: seq[tuple[len, saved: NimNode]] ## cursor context stack + var top: NimNode + ## the topmost non-union match expression enclosing the currently + ## processed one + var hasUsedElse = false + + proc aux(lang: LangInfo, e, to: NimNode): NimNode = + ## Does the actual work. + proc translateMatch(lang: LangInfo, e, to: NimNode) = + ## Emits the cursor movement and binding creation for a match expression. + if e[1].kind != nnkEmpty: + # commit the current cursor to a local with the given name + if e[0].kind == nnkBracket: + let bias = e[2] + let len = stack[^1].len # can only be non-empty + to.add newLetStmt(e[1], quote do: (`cursor`, `len` - `bias`)) + else: + to.add newLetStmt(e[1], quote do: get(`ast`, `cursor`)) + + # only emit a cursor movement when the moved cursor is actually observed + if e[^1].kind in {nnkCall, nnkCurly, nnkPar}: + case e[0].kind + of nnkPar: + let nlen = genSym"len" + let saved = genSym"saved" + to.add quote do: + let `nlen` = `ast`[pos(`cursor`)].val + let `saved` = enter(`ast`, `cursor`) + + stack.add (nlen, saved) + of nnkEmpty, nnkIntLit: + to.add quote do: + advance(`ast`, `cursor`) + of nnkBracket: + let bias = e[2] + let len = stack[^1].len # can only be non-empty + to.add quote do: + for _ in 0 ..< (`len` - `bias`): + advance(`ast`, `cursor`) + else: + unreachable(e[0].kind) + + case e.kind + of nnkCall: + translateMatch(lang, e, to) + result = aux(lang, e[^1], to) + of nnkPar: + # move the current cursor to the end of the subtree and pop it from + # the stack + let (_, saved) = stack.pop() + to.add quote do: + restore(`ast`, `cursor`, `saved`) + result = aux(lang, e[0], to) + of nnkCurly: + # a dispatcher + proc genOfBranch(lang: LangInfo, typ: LangType, used: var IntSet, + allowEmpty=true): NimNode = + result = nnkOfBranch.newTree() + case typ.kind + of tkTerminal: + if not containsOrIncl(used, typ.ntag) or not allowEmpty: + result.add newIntLitNode(typ.ntag) + of tkRecord: + if not containsOrIncl(used, typ.rtag) or not allowEmpty: + result.add newIntLitNode(typ.rtag) + of tkNonTerminal: + let ntags = ntags(lang, typ) + for tag in ntags(lang, typ): + if not containsOrIncl(used, tag): + result.add newIntLitNode(tag) + + if not allowEmpty and result.len == 0: + # any node tag part of the non-terminal would do + result.add newIntLitNode(ntags[0]) + + let stackLen = stack.len + let typ = e[0].intVal.int + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`[pos(`cursor`)].tag) + var used = initIntSet() + for i in 1.. 0: # are there any uncovered values? + caseStmt.add els + hasUsedElse = true + + if stack.len == 0 and not hasUsedElse: + # the 'else' rule was never used. Add it to the top-level case + # statement, so that a warning will be emitted + if caseStmt[^1].kind != nnkElse: + caseStmt.add nnkElse.newTree(newCall(bindSym"unreachable")) + caseStmt.add els + elif stack.len > 0 and config.fillForm != nil: + # uncovered values in nested positions are handled by processing + # the whole production + assert top.kind == nnkCall and top[0].kind == nnkPar + var allCovered = true + for tag in ntags(lang, lang.types[typ]): + if tag notin used: + allCovered = false + break + + # important: the `backup` cursor has to be used here, as it still + # points to the original node, whereas `cursor` has been moved already + if allCovered: + discard "nothing to do" + elif top[0].len == 1: + # simple case. The form ID is known statically + caseStmt.add nnkElse.newTree( + config.fillForm(lang, top[0][0].intVal.int, backup, info)) + else: + # complex case. The form ID is only known at run-time; dispatch over + # the top-level node's tag + let inner = nnkCaseStmt.newTree( + quote do: `ast`.nodes[pos(`backup`)].tag) + for it in top[0].items: + inner.add nnkOfBranch.newTree(it, + config.fillForm(lang, it.intVal.int, backup, info)) + inner.add nnkElse.newTree(newCall(bindSym"unreachable")) + caseStmt.add nnkElse.newTree(inner) + else: + var + fillForm = config.fillForm + fillType = config.fillType + + # always report an error for nested productions when form-filling + # is unavailable + if fillForm.isNil or stack.len > 0: + fillForm = defaultFillForm + if fillType.isNil or stack.len > 0: + fillType = defaultFillType + + for it in lang.types[typ].forms.items: + if lang.forms[it].ntag notin used: + caseStmt.add nnkOfBranch.newTree( + newIntLitNode(lang.forms[it].ntag), + fillForm(lang, it, cursor, info)) + + # fill in handling for embedded types whose productions aren't + # all covered + for it in lang.types[typ].sub.items: + let br = genOfBranch(lang, lang.types[it], used) + if br.len > 0: + br.add fillType(lang, it, cursor, info) + caseStmt.add br + + if caseStmt[^1].kind != nnkElse: + # all allowed node tags are handled, the rest are known to + # be impossible + caseStmt.add nnkElse.newTree(newCall(bindSym"unreachable")) + + to.add caseStmt + # nothing may follow a dispatcher + result = nil + of nnkTupleConstr: + # in-place transform the identdefs into proper ones + + proc transformSource(lang: LangInfo, pos, typ, orig: NimNode): NimNode = + case typ.kind + of nnkIntLit: + # a single item binding + let mvar = ident(lang.types[typ.intVal].mvar) + if pos.kind == nnkIdent: + pos # the source is a bound identifier already + else: + case lang.types[typ.intVal].kind + of tkTerminal: + quote do: `name`.`mvar`(id: `ast`[`pos`].val) + of tkRecord: + quote do: `name`.`mvar`(id: `ast`[`pos`].val) + of tkNonTerminal: + quote do: `name`.`mvar`(index: `pos`) + of nnkBracket: + # a list binding + let mvar = ident(lang.types[typ[0].intVal].mvar) + if pos.kind == nnkIdent: + pos # the source is already a slice (bound to an identifier) + else: + quote do: + slice[`name`.`mvar`](addr `ast`, `pos`[0], uint32(`pos`[1])) + else: + unreachable() + + for it in e[0].items: + case it[2].kind + of nnkSym: + it[2] = transformSource(lang, it[2], it[1], it[2]) + else: + it[2][1] = transformSource(lang, it[2][1], it[1], it[2]) + it[1] = newEmptyNode() + + to.add e[0] + to.add e[1] + result = nil + of nnkStmtList: + to.add e + result = nil # statement lists are always trailing + else: + unreachable(e.kind) + + result = nnkStmtList.newTree() + result.add newVarStmt(cursor, sel) + result.add newVarStmt(backup, cursor) + discard aux(lang, e, result) + +proc patternToMatch(n: NimNode): tuple[head, tail: NimNode] = + ## Translates a pattern into a match expression. A match expression has the + ## following grammar: + ## + ## match ::= (nkCall (nkBracket ) ) + ## | (nkCall (nkPar +) ) + ## | (nkCall (nkEmpty) ) + ## | (nkCall ) + ## name ::= | + ## cont ::= + ## | (nkPar ) # leave sub match + ## | (nkCurly +) # union matching + ## | (nkTupleConstr (nkLetSection ...) ...) # binding + ## | (nkStmtList ...) # custom tail logic + var binds: NimNode + proc addBinding(got: NimNode) = + if binds.isNil: + binds = nnkLetSection.newTree() + binds.add got + + let empty = newEmptyNode() # save some allocations + + proc aux(n: NimNode, depth: int): tuple[head, tail: NimNode, depth: int] = + let e = n[0] + let typ = n[1] + case typ.kind + of nnkPar: + result.head = nnkCall.newTree(typ, empty) + result.tail = result.head + assert e.kind == nnkCall + var depth = depth + 1 + for i in 1.. list of identDefs + proc step(n: NimNode, depth: int) = + case n.kind + of nnkCall: + step(n[^1], depth + 1) + var defs: NimNode + if pop(map, depth, defs): + let s = genSym"pos" + n[1] = s + for it in defs.items: + if it[^1].kind in nnkCallKinds: + it[^1][1] = s + else: + it[^1] = s + of nnkCurly: + for i in 1.. real ID + ## mappings + locs: seq[SLocRef] + ## stack of source locations + curSLoc: SLocRef + ## source location to use for parsed nodes + +macro mapTypeImpl(lang: static LangInfo, lname, typ: untyped): int = + result = nnkWhenStmt.newTree() + for i, it in lang.types.pairs: + result.add nnkElifBranch.newTree( + newCall(ident"is", typ, newDotExpr(lname, ident(it.mvar))), + newIntLitNode(i)) + + result.add nnkElse.newTree(quote do: {.error: "unreachable".}) + +proc mapType[L, T](): int {.compileTime.} = + ## Maps the type `T` to the integer IDs identifying it in `L`. + mapTypeImpl(idef(L), L, T) + +macro tags(lang: static LangInfo, typ: static int): set[uint8] = + ## Returns the node tags for the productions inhabiting `typ`. + case lang.types[typ].kind + of tkRecord: + nnkCurly.newTree(newLit(uint8 lang.types[typ].rtag)) + of tkTerminal: + nnkCurly.newTree(newLit(uint8 lang.types[typ].ntag)) + of tkNonTerminal: + var se = nnkCurly.newTree() + for it in ntags(lang, lang.types[typ]): + se.add newLit(uint8 it) + se + +proc eatString(p: var SexpParser): string = + if p.isTok(tkString): + result = captureCurrString(p) + discard getTok(p) + else: + raiseParseErr(p, "expected string") + +proc eatInt(p: var SexpParser): int = + if p.isTok(tkInt): + result = parseInt(currString(p)) + discard getTok(p) + else: + raiseParseErr(p, "expected integer") + +proc raiseError(line, col: int, msg: string) {.noreturn.} = + raise ValueError.newException("(" & $line & ", " & $col & ") " & msg) + +template check[L](c: Ctx[L, auto], pos: NodeIndex, line, col: int, + t: typedesc) = + ## Makes sure the production at `pos` is one part of the non-terminal + ## identified by `nterm`, raising an error if not. + if c.tree[pos].tag notin tags(idef(typeof(L)), mapType[L, t]()): + when t is Production: + raiseError(line, col, + "expected production of '" & t.N & "'") + else: + raiseError(line, col, + "expected '" & $t & "'") + +macro parseTerminalImpl(lang: static LangInfo) = + result = newStmtList() + # emit the terminal handlers + for it in lang.types.items: + if it.kind == tkTerminal: + let typ = ident(it.name) + let tag = it.ntag.uint8 + result.add quote do: + block: + let val = tryParse(node, `typ`) + if val.isSome: + return node(`tag`, c.curSLoc, pack(c.storage[], val.unsafeGet)) + +proc parseTerminal[L, S](c: var Ctx[L, S], node: SexpNode, + line, col: int): AstNode = + ## Implements fallback parsing of terminals. + mixin idef + parseTerminalImpl(idef(L)) + raiseError(line, col, + "'" & $node & "' is neither a valid language form nor terminal") + +proc parse[L, S](c: var Ctx[L, S], p: var SexpParser) + +proc extract(c: var Ctx, to: var Production, pos: int) = + # move the sub-tree over to the out-of-band storage + let start = c.staging.nodes.len + let count = c.tree.nodes.len - pos + c.staging.nodes.setLen(start + count) + copyMem(addr c.staging.nodes[start], addr c.tree.nodes[pos], + sizeof(AstNode) * count) + c.tree.nodes.shrink(pos) + to.index = NodeIndex(start) + +proc extract(c: var Ctx, to: var RecordRef, pos: int) = + to.id = c.tree.nodes.pop().val + +proc extract(c: var Ctx, to: var Value, pos: int) = + to.id = c.tree.nodes.pop().val + +proc parseFieldsImpl[L](c: var Ctx[L, auto], p: var SexpParser, + tup: var tuple) = + ## Parses the body of a record-def and fills `tup` with the values. + for name, it in fieldPairs(tup): + let start = c.tree.nodes.len + space(p) + eat(p, tkParensLe) + if currString(p) != name: + raiseParseErr(p, "expected '" & name & "'") + discard getTok(p) + space(p) + + # parse the field's value... + parse(c, p) + # ...then make sure its type is correct + check(c, NodeIndex(start), p.getLine(), p.getColumn(), typeof(it)) + + extract(c, it, start) + space(p) + eat(p, tkParensRi) + +macro parseFields(lang: static LangInfo, c: var Ctx, name: string) = + ## Selects the record type based on the dynamic value of `name`, parses + ## the record's fields, registers the resulting record with `c`, and + ## appends a record reference to the AST. + result = nnkCaseStmt.newTree(name) + var i = 0 # index of the record type + for typ in lang.types.items: + if typ.kind == tkRecord: + let mvar = ident(typ.mvar) + let tag = typ.rtag + result.add nnkOfBranch.newTree(newStrLitNode(typ.name), + genAst(c, mvar, i, tag) do: + if isDef: + # records may be recursive, so reserve and remember a slot first + let slot = c.records.mvar.len + c.records.mvar.setLen(slot + 1) + if id in c.maps[i]: + raiseError(start[0], start[1], + "a record with the given ID has been defined already") + c.maps[i][id] = uint32(slot) + + var tup: typeof(c.records.mvar[0]) + parseFieldsImpl(c, p, tup) + c.records.mvar[slot] = tup + c.tree.nodes.add node(uint8(tag), uint32(slot)) + else: + c.maps[i].withValue id, val: + c.tree.nodes.add node(uint8(tag), val[]) + do: + raiseError(start[0], start[1], + "record with ID " & $id & " is missing") + ) + inc i + + result.add nnkElse.newTree(genAst(name) do: + raiseError(start[0], start[1], + "there's no symbol type called '" & name & "'") + ) + +proc parseRecord[L, S](c: var Ctx[L, S], p: var SexpParser, isDef: bool) = + ## Parses a record definition (if `isDef` is true) or reference from `p`. + space(p) + let start {.used.} = (p.getLine(), p.getColumn()) + if p.currToken != tkSymbol: + raiseParseErr(p, "expected symbol") + let name = captureCurrString(p) + discard getTok(p) + space(p) + let id {.used.} = eatInt(p) + + parseFields(idef(L), c, name) + +proc parseMeta[L, S](c: var Ctx[L, S], p: var SexpParser) = + ## Parses a meta-expression. + assert p.currToken == tkKeyword + case currString(p) + of ":record-def": + discard getTok(p) + parseRecord(c, p, true) + of ":record": + discard getTok(p) + parseRecord(c, p, false) + of ":info": + # use the given source location for a tree + discard getTok(p) + c.locs.add c.curSLoc + space(p) + eat(p, tkParensLe) + let file = eatString(p) + space(p) + let startLine = eatInt(p) + space(p) + let startCol = eatInt(p) + space(p) + if p.currToken == tkParensRi: + # only a line-column pair + c.curSLoc = newSourceLoc(c.storage[], file, startLine, startCol) + else: + let endLine = eatInt(p) + space(p) + let endCol = eatInt(p) + space(p) + c.curSLoc = + newSourceLoc(c.storage[], file, startLine, startCol, endLine, endCol) + + eat(p, tkParensRi) + space(p) + parse(c, p) + c.curSLoc = c.locs.pop() + of ":no-info": + # use no source location for a tree + discard getTok(p) + space(p) + c.locs.add c.curSLoc + c.curSLoc = NoSLoc + parse(c, p) + c.curSLoc = c.locs.pop() + else: + raiseParseErr(p, "expected meta-expression") + + space(p) + eat(p, tkParensRi) + +proc rawParseForm[L, S](c: var Ctx[L, S], p: var SexpParser) = + ## Parses and appends the elements for a form, without performing any + ## grammar checks. + let start = c.tree.nodes.len + c.tree.nodes.add node(0, c.curSLoc, 0) # sub-tree node + var len = 0 + while p.currToken != tkParensRi: + parse(c, p) + space(p) + inc len + + discard getTok(p) # eat the parens + c.tree.nodes[start].val = uint32(len) + # the tag is computed separately + +macro parseFormImpl(lang: static LangInfo) = + ## Expands to the form parser for `lang`. + + # to keep the implementation simple, subtrees are parsed without regard + # to grammar at first. Once a subtree is fully parsed, the grammar check + # takes place and - if the check succeeds - a form tag is assigned to + # the node + + proc mergeInto(m, into: NimNode): NimNode = + ## Merges match expression `m` into `into`. + proc mustTrail(n: NimNode): bool = + n[0].kind == nnkEmpty or n[1].intVal <= 0 + + case into.kind + of nnkCurly: + for i, it in into.pairs: + if it[0] == m[0]: # same head? + into[i] = mergeInto(m, into[i]) + return into + elif mustTrail(it) and not mustTrail(m): + into.insert i, m + return into + + into.add m + into + of nnkCall: + if m[0] == into[0]: + # same head + into[^1] = mergeInto(m[^1], into[^1]) + into + elif mustTrail(into): + # an 'else' always comes last + nnkCurly.newTree(m, into) + else: + nnkCurly.newTree(into, m) + else: + unreachable() + + proc translate(lang: LangInfo, m: NimNode): NimNode = + ## Translates the match expression from the mini-language to NimSkull code. + case m.kind + of nnkIntLit: + result = quote do: + c.tree.nodes[start].tag = `m` + of nnkCurly: + result = nnkIfStmt.newTree() + for it in m.items: + let sub = translate(lang, it) + if sub.kind == nnkIfStmt: + result.add sub[0] + else: + result.add nnkElse.newTree(sub) + of nnkCall: + let head = m[0] + let raiseErr = bindSym"raiseError" + let next = translate(lang, m[^1]) + if head.kind == nnkEmpty: + # matches at the end of the sub-tree + result = quote do: + if cursor.int == c.tree.nodes.len: + `next` + else: + `raiseErr`(line, col, "end of sub-tree expected, but got more nodes") + else: + let tags = nnkCurly.newTree() + let name = lang.types[head.intVal].name + case lang.types[head.intVal].kind + of tkTerminal: + tags.add newLit(uint8(lang.types[head.intVal].ntag)) + of tkRecord: + tags.add newLit(uint8(lang.types[head.intVal].rtag)) + of tkNonTerminal: + for it in ntags(lang, lang.types[head.intVal]): + tags.add newLit(uint8(it)) + + if m[1].intVal == 1: # single item? + result = quote do: + if cursor.int < c.tree.nodes.len and + c.tree[cursor].tag in `tags`: + cursor = c.tree.next(cursor) + `next` + else: + `raiseErr`(line, col, "expected production of '" & `name` & "'") + else: + # a list; may be comprised of zero or more items + let bias = -m[1].intVal + result = quote do: + for i in uint32(`bias`)..language pass into a real procedure. + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.body.kind == nnkEmpty: + # a forward declaration, bail + return def + + let to = ident"out.ast" + let ret = def.params[0] + def.body = newStmtList(def.body) + def.body.insert 0, quote do: + # for convenience, inject a `build` macro overload that uses the + # result type + template build(body: untyped): untyped {.used.} = + build(`to`, `ret`, NoSLoc, body) + template build(info: SLocRef, body: untyped): untyped {.used.} = + build(`to`, `ret`, info, body) + + result = def + +macro processorMatchImpl(lang: static LangInfo, src: static string, + sel: untyped, rules: varargs[untyped]): untyped = + ## Implements the transformation/expansion of a language->language + ## processor's trailing case statement/expression. + let input = newDotExpr(ident"in.ast", ident"tree") + let output = newDotExpr(ident"out.ast", ident"tree") + + proc fillForm(lang: LangInfo, form: int, n, info: NimNode): NimNode = + ## Generates a form transformer. + let sym = bindSym"transform" + result = quote do: + (typeof(result))( + index: `sym`(idef(src), idef(dst), typeof(result).N, `form`, + `input`, `output`, `n`)) + copyLineInfo(result[1][1][^1], info) + + proc fillType(lang: LangInfo, typ: int, n, info: NimNode): NimNode = + ## Generates a call to the type transformer for `typ`. + let sym = bindSym"transformType" + result = quote do: + (typeof(result))( + index: `sym`(idef(src), idef(dst), typeof(result).N, `typ`, + `input`, `output`, `n`)) + copyLineInfo(result[1][1][^1], info) + + let config = ExpandConfig( + fillForm: fillForm, + fillType: fillType, + ) + + matchImpl(lang, lang.map[src], ident"src", input, sel, rules, config) + +macro transform(index, nterm: untyped): untyped = + ## Transforms the input language non-terminal with name `nterm` to a non- + ## terminal identified through the current result type. + # simply emit an empty processorMatchImpl invocation. All branches will be + # auto-generated + # TODO: if none of the productions require a (direct or indirect) call to a + # custom processor, the source and target productions match, and the + # used tags map to the same ID in the source and target language, use a + # memcopy + let sym = bindSym"processorMatchImpl" + result = quote do: + `sym`(idef(src), `nterm`, Cursor(`index`)) + +proc hasPragma(def: NimNode, name: string): bool = + if def.pragma.kind == nnkPragma: + for it in def.pragma.items: + if it.eqIdent(name): + return true + +template defineAdapter(src: typedesc, dst: typedesc, name: untyped) = + ## Introduces a definition for the adapter from `->` to `name`. + # the `->` uses a macro instead of a template because: + # * it leaves the call expression to inherit the expansion site's + # line information + # * it only binds `name` to the actual symbol when the macro expands, meaning + # that the symbol is only marked as used when it really is, allowing + # "unused symbol" detection to still work + macro `->`(n: src, _: typedesc[dst]): dst {.used.} = + newCall(ident(astToStr(name)), n) + +template withCache(to: typedesc, inp, body: untyped): untyped = + ## If a mapping for `inp` -> `to` already exists, returns it, otherwise + ## evaluates `body` and remembers its result. + mixin table + let c = getTable[typeof(inp), to]() + var res: to + withValue c[], inp.id, val: + res = + when to is Value: to(id: val[]) + elif to is RecordRef: to(id: val[]) + elif to is Production: to(index: NodeIndex(val[])) + else: {.error.} + do: + # TODO: `body` should be wrapped in a lambda, so that `return` doesn't + # disables adding to the table + let val = if true: body else: default(to) + c[][inp.id] = + when to is Value: val.id + elif to is RecordRef: val.id + elif to is Production: val.index.uint32 + else: {.error.} + res = val + res + +macro transformerImpl(sclass, dclass: static TypeClass, + param, body: untyped): untyped = + ## Refines the user-defined body of a processor into a real processor body. + proc transformCase(n: NimNode): NimNode = + result = genAst(arg=n[0]): + processorMatchImpl(idef(src), typeof(arg).N, Cursor(arg.index)) + copyLineInfo(result, n) + for i in 1..language pass into a real procedure. + let name = name + if def.kind notin {nnkProcDef, nnkFuncDef}: + error(".transform must be applied to procedure definition", def) + + if def.params[0].kind == nnkEmpty: + error("a return type is required for a transfomer", def.name) + + var isManual = false + block: + let list = def.pragma + for i in 0..* pass into a real procedure. + # nothing to do + result = def + +macro generatedImpl(def: untyped) = + ## Implements the `.generated` pragma for language->language passes. + if def.body.kind != nnkEmpty: + error(".generated must be applied to forward declaration", def) + + def.body = newCall(ident"->", def.params[1][0], copyNimTree(def.params[0])) + result = def + +template defineInWrappers(lang, input: untyped) = + ## Introduces the injected definitions for passes that receive an AST. + template match[N](sel: Production[lang, N], branches: varargs[untyped] + ): untyped {.used, inject.} = + match[lang, N](input.tree, Cursor(sel.index), sel, branches) + + template slice[N](T: typedesc[Production[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + template slice[N](T: typedesc[RecordRef[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + template slice(T: typedesc[asts.Value[auto]]): typedesc {.used, inject.} = + ChildSlice[T, Cursor] + + template val[T](v: nanopass.Value[T]): T {.used, inject.} = + # TODO: return a `lent T` where ``unpack`` does too (this is tricky...) + # XXX: consider renaming this template to `get` + unpack(input.storage[], v.id, typeof(T)) + template get[N](r: RecordRef[lang, N]): untyped {.used, inject.} = + get(input, r) + template info[N](n: Production[lang, N]): untyped {.used, inject.} = + input.tree[n.index].info + + template equal[N](a, b: Production[lang, N]): bool {.used, inject.} = + equal(input.tree, Cursor(a.index), Cursor(b.index)) + +template defineOutWrappers(lang, output: untyped) = + ## Introduces the injected definitions for passes that produce an AST. + template terminal(x: untyped): untyped {.used, inject.} = + newTerminal(output.storage, x) + template build[N](n: typedesc[Production[lang, N]], info: SLocRef, + body: untyped): untyped {.used, inject.} = + build(output, n, info, body) + template build[N](n: typedesc[RecordRef[lang, N]], info: SLocRef, + body: untyped): untyped {.used, inject.} = + build(output, n, info, body) + template match[N](sel: Production[lang, N], branches: varargs[untyped] + ): untyped {.used, inject.} = + match[lang, N](output.tree, IndCursor(sel.index), sel, branches) + template slice[N](T: typedesc[Production[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, IndCursor] + template slice[N](T: typedesc[RecordRef[lang, N]] + ): typedesc {.used, inject.} = + ChildSlice[T, IndCursor] + + template get[N](r: RecordRef[lang, N]): untyped {.used, inject.} = + get(`output`, r) + template info[N](n: Production[lang, N]): untyped {.used, inject.} = + output.tree[n.index].info + + template equal[N](a, b: Production[lang, N]): bool {.used, inject.} = + equal(output.tree, IndCursor(a.index), IndCursor(b.index)) + +proc assemblePass(src, dst, def, call: NimNode): NimNode = + ## Assembles the final procedure definition for a pass. `def` is the original + ## proc definition, `call` the call to the pass' implementation. + let input = ident"in.ast" + let output = ident"out.ast" + let storageTy = ident"Literals" # TODO: don't hardcode + let hasIn = src != nil + let hasOut = dst != nil + + var body = newStmtList() + let (transformImpl, name) = + if hasIn and hasOut: + (bindSym"transformInOutImpl", dst) + elif hasIn: + (bindSym"transformInImpl", src) + elif hasOut: + (bindSym"transformOutImpl", dst) + else: + unreachable() + + body.add quote do: + template transform(p: untyped) {.used.} = + `transformImpl`(def(`name`), `name`, p) + + if hasIn and hasOut: + let impl = bindSym"generatedImpl" + body.add quote do: + template generated(p: untyped) {.used.} = `impl`(p) + + # alias the source and destination language with known names: + if hasIn: + body.add quote do: + template src: untyped {.used.} = `src` + if hasOut: + body.add quote do: + template dst: untyped {.used.} = `dst` + + if hasIn: + body.add newCall(bindSym"defineInWrappers", ident"src", input) + + if hasOut: + body.add newCall(bindSym"defineOutWrappers", ident"dst", output) + + # the source location accessors are always available + if hasIn: + body.add quote do: + template file(at: SLocRef): lent string {.used.} = file(`input`, at) + template span(at: SLocRef): untyped {.used.} = span(`input`, at) + else: + body.add quote do: + template file(at: SLocRef): lent string {.used.} = file(`output`, at) + template span(at: SLocRef): untyped {.used.} = span(`output`, at) + + if hasIn: + # shadow the input tree with a cursor to prevent a costly copy when + # it's captured by the closure + body.add quote do: + let `input` {.cursor.} = `input` + if hasOut: + body.add quote do: + var `output` = Ast[dst, `storageTy`]() + if hasIn: + # re-use the storage from the input + body.add quote do: + `output`.storage = `input`.storage + else: + body.add quote do: + `output`.storage = new(`storageTy`) + + let resolve = bindSym"resolve" + body.add quote do: + let pos = `call` + # turn the AST with indirections into one without and return it + result = (`resolve`(move `output`, pos.index), + typeof(pos)(index: NodeIndex(0))) + else: + body.add quote do: + result = `call` + + def.body = body + # patch the signature: + if hasIn: + def.params.insert(1, + newIdentDefs(input, + nnkBracketExpr.newTree(ident"Ast", src, storageTy))) + if hasOut: + def.params[0] = + nnkTupleConstr.newTree( + nnkBracketExpr.newTree(ident"Ast", dst, storageTy), + def.params[0]) + + result = def + +template defineProcessors(dst: untyped) = + ## Helper template for the pass macro implementation. Defines the implicit + ## `->` routines that will be invoked be default unless overridden. + template `->`[T, U](x: T, _: typedesc[U]): U {.inject.} = + # the fallback processor called when nothing else matches + ctError("cannot generate transformer from '" & $typeof(T) & + " to '" & $typeof(U) & "'", x) + + template `->`[T](v: Value[T], _: typedesc[Value[T]]): Value[T] {.inject.} = + v # nothing to do + + proc `->`[X](n: Production, T: typedesc[Production[dst, X]]): T {.inject.} = + # note: the signature is overly broad so that overload resolution + # prefers the more specific adapters created for the programmer-provided + # processors + transform(n.index, typeof(n).N) + + proc `->`[X](r: RecordRef, T: typedesc[RecordRef[dst, X]]): T {.inject.} = + let tab = getTable[typeof(r), T]() + # XXX: cannot use `withValue` because of symbol binding issues... + if r.id in tab[]: + T(id: tab[][r.id]) + else: + let s = addr pick(idef(dst), X, `out.ast`.records) + # reserve and remember a slot first, so that recursive records work + let id = s[].len.uint32 + s[].setLen(id + 1) + tab[][r.id] = id + + let rec {.cursor.} = get(r) + let tmp = transformRecord(rec, typeof(get(result))) + s[][id] = tmp + T(id: id) + + proc `->`[T, C, U](s: ChildSlice[T, C], _: typedesc[U]): seq[U] {.inject, closure.} = + # XXX: the explicit .closure annotation works around a closure inference + # compiler bug + result = newSeq[U](s.len) + for i, it in s.pairs: + result[i] = it -> U + +template wrapWithTables(lambda: untyped): untyped = + # the result of all record->* transformations is remembered, which requires + # a table for all used type pairs. This is tricky, as while what + # source*destination pairs there are is statically known, it's only known + # *after* the whole pass body has been type-checked, but the generated + # processors need access to their respective table early + var tabCounter {.global, compileTime.} = 0 + {.push checks: off, stacktrace: off.} + proc access(tab: int): ptr Table[uint32, uint32] {.closure.} + + proc getTable[A, B](): ptr Table[uint32, uint32] {.inject.} = + # the counter is only incremented (and a table is thus reserved) when + # the procedure is instantiated with new unique type parameters + const pos = tabCounter + static: inc tabCounter + access(pos) + + {.pop.} + + let tmp = lambda + + # now that the number of tables is known, statically allocate them and + # complete the `access` declaration + var tabs: array[tabCounter, Table[uint32, uint32]] + {.push checks: off, stacktrace: off.} + proc access(tab: int): ptr Table[uint32, uint32] {.closure.} = + addr tabs[tab] + {.pop.} + + tmp + +macro passImpl(src, dst, srcnterm, dstnterm: typedesc, def: untyped) = + # create a forward declaration for each transformer: + var preamble = newStmtList() + var i = 0 + while i < def.body.len: + let it = def.body[i] + # not ideal, as it breaks some custom macro/template pragmas, but without + # resorting to typed macros, there's no other way to do this transform + if it.kind == nnkProcDef and it.hasPragma("transform"): + if it.body.kind == nnkEmpty: + # remove the user-defined forward declaration + def.body.del(i) + dec i # undo the following `inc` + else: + let backup = it.body + it.body = newEmptyNode() + preamble.add copyNimTree(it) + it.body = backup + inc i + + # add the generic processor procedure, which all processor invocations + # for processors not supplied by the programmer will end up using + preamble.add newCall(bindSym"defineProcessors", dst) + + # if the body doesn't end in an expression, add a call to the + # entry processor + if def.body[^1].kind == nnkProcDef: + # ^^ a heuristic, but should work okay enough + def.body.add newCall(ident"->", def.params[1][0], dstnterm) + + def.body = newStmtList(preamble, def.body) + def.params[0] = dstnterm + def.params[1][^2] = srcnterm + + let lambda = newProc(newEmptyNode(), body=def.body, procType=nnkProcDef) + lambda.params = copyNimTree(def.params) + + let call = newCall(newCall(bindSym"wrapWithTables", lambda)) + # forward the original parameters to the lambda: + for i in 1..language pass, that is a + ## pass, that takes an AST and non-terminal reference of language A and + ## produces an AST and a non-terminal reference of language B. + if p.kind != nnkProcDef: + error(".inpass must be applied to a procedure definition", p) + + var target = p.params[0] + if target.kind == nnkEmpty: + error("a return type is required, but none is provided", p.params[0]) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(input = p.params[1][^2], target, p): + passImpl(lang(input), lang(target), nterm(input), nterm(target), p) + +macro outpass*(p: untyped) = + ## Turns a procedure definition into a language->* pass, that is, a pass + ## that takes an AST and non-terminal reference of language A and produces + ## a value. + if p.kind != nnkProcDef: + error(".outpass must be applied to a procedure definition", p) + + let ret = p.params[0] + if ret.kind == nnkEmpty: + error("a return type is required, but none is provided", ret) + if p.params.len == 1: + error("the input parameter is missing", p.params) + + result = genAst(input = p.params[1][^2], p): + outpassImpl(lang(input), nterm(input), p) diff --git a/nanopass/nppatterns.nim b/nanopass/nppatterns.nim new file mode 100644 index 00000000..63628914 --- /dev/null +++ b/nanopass/nppatterns.nim @@ -0,0 +1,40 @@ +## Implements the type-based pattern matching used by the code generated by +## parts of the nanopass framework. + +import std/[macros] + +type + PChoice*[A, B] = object + ## Predicate: T is `A` or `B`. + + PArray*[A] = object + ## Predicate: T is array-like with element `A`. + +macro dot(e, fname: untyped): untyped = + ## ``dot(x, "abc")`` -> ``x.abc``. + macro dotAux(fname: static string, e: untyped): untyped = + newDotExpr(e, ident(fname)) + + newCall(bindSym"dotAux", fname, e) + +template matches*[T, U](x: T, _: typedesc[U]): bool = + ## Implements type-based pattern matching, used by the nanopass macros. + when U is PArray: + when T is seq: + matches(default(typeof(x[0])), typeof(U.A)) + else: + false + elif U is PChoice: + # why not just use `or`? Because that would unnecessarily expand the + # second `matches` invocation when the case first invocation was successful + when matches(x, typeof(U.A)): + true + else: + matches(x, typeof(U.B)) + elif U is Production: + when x is U: + true + else: + matches(x, dot(U.L.meta.nt, U.N)) + else: + x is U diff --git a/nanopass/npresolve.nim b/nanopass/npresolve.nim new file mode 100644 index 00000000..4f118b46 --- /dev/null +++ b/nanopass/npresolve.nim @@ -0,0 +1,61 @@ +## Implements the routines for post-processing ASTs produced by passes. + +import nanopass/[asts] + +proc resolve(ast: Tree, result: var Tree, n: NodeIndex) = + ## Copies the AST fragment starting at `n` to `result`, resolving all + ## indirections in the process. + template src: untyped = ast.nodes + template dst: untyped = result.nodes + const size = sizeof(AstNode) + + template append(start, fin: uint32) = + let pos = dst.len + let num = int(fin - start) + dst.setLen(pos + num) + copyMem(addr dst[pos], addr src[start], num * size) + + # search for runs of contiguous nodes. When encountering an indirection, + # copy the run and move the source cursor to the indirection's + # target; repeat. + var stack = @[(uint32(n), uint32(n))] + while stack.len > 0: + block outer: + var (i, last) = stack[^1] + let prev = i + while i <= last: + if src[i].tag < RefTag: + last += src[i].val + elif src[i].tag == RefTag: + if i > prev: + # copy everything we got so far + append(prev, i) + + stack[^1] = (i + 1, last) + let next = src[i].val + stack.add (next, next) + break outer + + inc i + + if i > prev: + # copy the rest + append(prev, i) + + stack.shrink(stack.len - 1) + +proc resolve*[L, S](ast: sink Ast[L, S], n: NodeIndex): Ast[L, S] = + ## Returns `ast` with all indirections in the AST resolved. + var output: Tree + resolve(ast.tree, output, n) + # also resolve the sub-trees referenced from records: + for s in fields(ast.records): + for tup in s.mitems: + for it in fields(tup): + when it is Production: + let got = output.nodes.len + resolve(ast.tree, output, it.index) + it.index = NodeIndex(got) + + ast.tree = output + result = ast diff --git a/nanopass/nptransform.nim b/nanopass/nptransform.nim new file mode 100644 index 00000000..5dc6d9c6 --- /dev/null +++ b/nanopass/nptransform.nim @@ -0,0 +1,215 @@ +## Implements the generation of transformers for language forms, records, +## and types. + +import std/[macros, strutils, tables] +import passes/[trees] +import nanopass/[asts, helper, nplang] + +type + Morphability = enum + None, Ambiguous, Inexact, Exact + +proc canMorph(src, dst: LangInfo, a, b: SForm): Morphability = + ## Computes whether form `a` from `src` can be morphed into `b` from `dst`. + if a.elems.len == b.elems.len and a.name == b.name: + result = Exact + for i, it in a.elems.pairs: + if it.repeat != b.elems[i].repeat: + result = None + break + elif src.types[it.typ].name != dst.types[b.elems[i].typ].name: + result = Inexact + else: + result = None + +proc append(to: var Tree, i: var int, tag: uint8, info: SLocRef, + val: uint32) {.inline.} = + to.nodes[i] = node(tag, info, val) + inc i + +macro transform*(src, dst: static LangInfo, nterm: static string, + form: static int, input, output: Tree, + cursor: untyped): untyped = + ## Transforms the input language form identified by `form` to a production + ## fitting fitting the target language non-terminal with name `nterm` and + ## appends the result to `output`. + # find a target language form that's a production of the non-terminal and a + # suitable morph target. Exact matches are preferred + var target = -1 + var morphability = None + for it in dst.types[dst.map[nterm]].forms.items: + let m = canMorph(src, dst, src.forms[form], dst.forms[it]) + case m + of None, Ambiguous: + discard "nothing to do" + of Inexact: + case morphability + of Inexact: + morphability = Ambiguous + of Exact, Ambiguous: + discard "keep as is" + of None: + morphability = Inexact + target = it + of Exact: + morphability = m + target = it + + if morphability in {None, Ambiguous}: + return + makeError("cannot generate transformer from '$1' to '$2'" % + [render(src, src.forms[form]), nterm], cursor) + + # important: the generated code being efficient is of major importance! Most + # transformations will be auto-generated, and they should thus be as fast as + # possible + # TODO: if none of the child nodes require a processor call, memcopy the + # whole sub-tree + # TODO: (bigger refactor) pass the cursor to the transformers as a var + # parameter, which would eliminate unnecessary tree seeking when + # there's many calls to fully auto-generated non-terminal processors + + let id = dst.forms[target].ntag.uint8 + result = newStmtList() + # add the root node: + let body = quote do: + let len = `input`.len(pos(`cursor`)) + let root = `output`.nodes.len.NodeIndex + var i = `output`.nodes.len + # the node sequence has to be contiguous, so it's allocated upfront + `output`.nodes.setLen(i + len + 1) + `output`.nodes[i] = node(`id`, `input`[pos(`cursor`)].info, uint32(len)) + inc i + discard enter(`input`, `cursor`) + + # call the transformers and emit the nodes in one go: + for i, a in src.forms[form].elems.pairs: + let b = dst.forms[target].elems[i] + let s = ident(src.types[a.typ].mvar) + let d = ident(dst.types[b.typ].mvar) + let pos = + case src.types[a.typ].kind + of tkTerminal, tkRecord: + quote do: pos(`cursor`) + of tkNonTerminal: + quote do: get(`input`, `cursor`) + let got = + case src.types[a.typ].kind + of tkTerminal: + quote do: src.`s`(id: `input`[`pos`].val) + of tkRecord: + quote do: src.`s`(id: `input`[`pos`].val) + of tkNonTerminal: + quote do: src.`s`(index: `pos`) + + let append = bindSym"append" + let call = + case dst.types[b.typ].kind + of tkTerminal: + let tag = dst.types[b.typ].ntag + quote do: + `append`(`output`, i, uint8(`tag`), `input`[`pos`].info, + (`got` -> dst.`d`).id) + of tkRecord: + let tag = dst.types[b.typ].rtag + quote do: + `append`(`output`, i, uint8(`tag`), `input`[`pos`].info, + (`got` -> dst.`d`).id) + of tkNonTerminal: + quote do: + `append`(`output`, i, RefTag, NoSLoc, + (`got` -> dst.`d`).index.uint32) + + if a.repeat: + let bias = src.forms[form].elems.len - 1 + body.add quote do: + for _ in 0..<(len - `bias`): + `call` + advance(`input`, `cursor`) + else: + body.add quote do: + `call` + advance(`input`, `cursor`) + + result.add body + # the callsite takes care of fitting the index to the right type + result.add ident"root" + +macro transformType*(src, dst: static LangInfo, nterm: static string, + typ: static int, input, output: Tree, + cursor: untyped): untyped = + ## Transforms the instance of type `typ` (may be either a terminal or non- + ## terminal) at `cursor` to a production fitting the destination non-terminal + ## `nterm` and appends the result to `output`. + proc contains(lang: LangInfo, typ: LangType, search: int): bool = + for it in typ.sub.items: + result = it == search + if not result and lang.types[it].kind == tkNonTerminal: + result = contains(lang, typ, search) + if result: + break + + let smvar = ident(src.types[typ].mvar) + let got = + case src.types[typ].kind + of tkTerminal: + quote do: + src.`smvar`(id: `input`[pos(`cursor`)].val) + of tkRecord: + quote do: + src.`smvar`(id: `input`[pos(`cursor`)].val) + of tkNonTerminal: + quote do: + src.`smvar`(index: get(`input`, `cursor`)) + + let dtyp = dst.map.getOrDefault(src.types[typ].name, -1) + + # prefer a direct processor (i.e. 'a -> a') over 'a -> b' + if dtyp != -1 and contains(dst, dst.types[dst.map[nterm]], dtyp): + let dmvar = ident(dst.types[dtyp].mvar) + case dst.types[dtyp].kind + of tkTerminal: + # a new node needs to be allocated so that a reference to it can + # be returned + let tag = dst.types[dtyp].ntag.uint8 + let tmp = genSym() + result = quote do: + let info = `input`[pos(`cursor`)].info + let `tmp` = `got` -> dst.`dmvar` + `output`.nodes.add node(`tag`, info, `tmp`.id) + NodeIndex(`output`.nodes.high) + of tkRecord: + # a new node needs to be allocated so that a reference to it can + # be returned + let tag = dst.types[dtyp].rtag.uint8 + let tmp = genSym() + result = quote do: + let info = `input`[pos(`cursor`)].info + let `tmp` = `got` -> dst.`dmvar` + `output`.nodes.add node(`tag`, info, `tmp`.id) + NodeIndex(`output`.nodes.high) + of tkNonTerminal: + result = quote do: + (`got` -> dst.`dmvar`).index + else: + # no direct processor is possible; use an indirect processor + let dmvar = ident(dst.types[dst.map[nterm]].mvar) + result = quote do: + (`got` -> dst.`dmvar`).index + +template transformRecord*(rec: tuple, to: typedesc[tuple]): untyped = + ## Transforms the given record `rec` into a target-language record with + ## internal type `to`. + var dest: to + for dname, dval in fieldPairs(dest): + var found {.global, compileTime.} = false + for sname, sval in fieldPairs(rec): + when dname == sname: + dval = sval -> typeof(dval) + static: found = true + + # truncating is possible and allowed, but widening is not + when not found: + {.error: "source record is missing a field called '" & dname & "'".} + + dest diff --git a/nanopass/npunparser.nim b/nanopass/npunparser.nim new file mode 100644 index 00000000..c5efad18 --- /dev/null +++ b/nanopass/npunparser.nim @@ -0,0 +1,187 @@ +## Implements the routines for unparsing an ASTs back into S-expressions. + +# TODO: instead of assembling a S-expression directly, the unparser should +# emit a stream of S-expression tokens (ideally as an iterator, but's +# that not really possible today, at least in an efficient manner) +# TODO: render and store the unexpected node/tree alongside their +# corresponding error node + +import std/[intsets, macros, tables, typetraits] +import experimental/[sexp] +import nanopass/[asts, nplang] + +from nanopass/nppass import get + +type + Ctx = object + ## The unparsing context object. + pos: int ## current node position + records: seq[IntSet] + ## for each record type, keeps track of the already-defined records + curSLoc: SLocRef + ## currently active source location + +proc nameToIndex[L; Name: static string](): int {.compileTime.} = + ## Returns the index of the set in `Ctx.records` corresponding to the given + ## record type name. + var i = 0 + var tmp: L + # the index being stable across compilation is not necessary + for f in fields(tmp): + when f is RecordRef: + when typeof(f).N == Name: + if true: + return i + inc i + +proc wrap(c: var Ctx, ast: Ast, info: SLocRef, n: sink SexpNode): SexpNode = + ## Wraps `n` in an ``:info`` or ``:no-info`` decorator expression where + ## necessary for `n` to use `info` as its source location. + if info == c.curSLoc: + n + elif info != NoSLoc: + let (start, fin) = span(ast, info) + if start == fin: + newSList([newSSymbol(":info"), + newSList([ + newSString(file(ast, info)), + newSInt(BiggestInt start.line), + newSInt(BiggestInt start.column)]), + n]) + else: + newSList([newSSymbol(":info"), + newSList([ + newSString(file(ast, info)), + newSInt(BiggestInt start.line), + newSInt(BiggestInt start.column), + newSInt(BiggestInt fin.line), + newSInt(BiggestInt fin.column)]), + n]) + else: + newSList([newSSymbol(":no-info"), n]) + +proc unparse[N: static string, S](ast: Ast[auto, S], c: var Ctx): SexpNode + +proc unparse[N: static string, L](ast: Ast[L, auto], c: var Ctx, id: int, + tup: tuple): SexpNode = + ## Unparses the nanopass record `tup`. The full record is only emitted for + ## the first occurrence - only the ID is emitted for all further occurrences. + if containsOrIncl(c.records[static nameToIndex[L, N]()], id): + # the definition was emitted already, only emit a reference + result = newSList([newSSymbol(":record"), newSSymbol(N), newSInt(id)]) + else: + result = newSList() + result.add newSSymbol(":record-def") + result.add newSSymbol(N) + result.add newSInt(id) + for name, val in fieldPairs(tup): + let node = + when val is Value: + toSexp(unpack(ast.storage[], val.id, typeof(val).T)) + elif val is RecordRef: + unparse[typeof(val).N](ast, c, val.id.int, get(ast, val)) + elif val is Production: + let prev = c.pos + c.pos = val.index.int + let r = unparse[typeof(val).N](ast, c) + c.pos = prev + r + else: + {.error: "unreachable".} + + result.add newSList([newSSymbol(name), node]) + +macro unparse(def: static LangInfo, nterm: static string, ast, c: untyped) = + ## Unparses the non-terminal with name `nterm` from the given language at + ## the current cursor position `pos`. + let id = def.map[nterm] + result = newStmtList() + result.add quote do: + let prev = `c`.curSLoc + let info = `ast`.tree.nodes[`c`.pos].info + `c`.curSLoc = info + + var caseStmt = nnkCaseStmt.newTree(quote do: `ast`.tree.nodes[`c`.pos].tag) + + proc genForType(def: LangInfo, typ: LangType): NimNode = + case typ.kind + of tkTerminal: + let name = ident(typ.name) + quote do: + inc `c`.pos + toSexp(unpack(`ast`.storage[], `ast`.tree.nodes[`c`.pos - 1].val, `name`)) + of tkRecord: + let unparse = bindSym"unparse" + let mvar = ident(typ.mvar) + let tmp = genSym("id") + let name = typ.name + quote do: + let `tmp` = `ast`.tree.nodes[`c`.pos].val.int + inc `c`.pos + `unparse`[`name`](`ast`, `c`, `tmp`, `ast`.records.`mvar`[`tmp`]) + of tkNonTerminal: + let unparse = bindSym"unparse" + let name = typ.name + quote do: + `unparse`[`name`](`ast`, `c`) + + # form renderers: + for it in def.types[id].forms.items: + var body = nnkStmtList.newTree() + let name = def.forms[it].name + body.add quote do: + let len {.used.} = `ast`.tree.nodes[`c`.pos].val.int + inc `c`.pos + result = newSList([newSSymbol(`name`)]) + for i, e in def.forms[it].elems.pairs: + let inner = genForType(def, def.types[e.typ]) + if e.repeat: + let start = def.forms[it].elems.len - 1 + body.add quote do: + for _ in `start`..`_ +## macro, for defining nanopass compilers, which are compositions of passes. + +import std/macros + +macro defineCompiler*(name, start, names: untyped) = + ## Generates a compiler procedures, running the provided passes in + ## sequence. Interim implementation. + var prevAst = ident"ast" + var prevPos = ident"it" + let body = newStmtList() + for it in names.items: + let tmp = genSym() + if it.kind == nnkIdent: + let name = it.strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, quote do: `it`(`prevAst`, `prevPos`)) + else: + let g = it + g.insert 1, prevPos + g.insert 1, prevAst + let name = it[0].strVal + body.add quote do: echo "-- ", `name` + body.add newLetStmt(tmp, g) + prevAst = quote do: `tmp`[0] + prevPos = quote do: `tmp`[1] + result = quote do: + proc `name`(ast: Ast[`start`, Literals], it: `start`.entry): auto = + `body` + result = (`prevAst`, `prevPos`) diff --git a/passes/literals.nim b/passes/literals.nim new file mode 100644 index 00000000..a533be14 --- /dev/null +++ b/passes/literals.nim @@ -0,0 +1,72 @@ +## Implements the storage for literal data embedded in ASTs. + +import nanopass/asts + +# TODO: use bi-tables for the number, string, and source location values + +type + Literals* = object + ## Storage container for literal data, to be used with nanopass ASTs. + numbers*: seq[uint64] ## a list of bit patterns + strings*: seq[string] + locs*: seq[SourceLoc] + +const + OverflowShift = 31 + OverflowBit = 1'u32 shl OverflowShift + ## use the most significant bit to flag whether a value is larger than + ## `max(int32)` and overflows into `PackedTree.numbers` + +template castAny[T, U](x: U): T = + ## A `cast` that doesn't warn when casting into the origin type. + when T is U: x + else: cast[T](x) + +proc pack*(s: var Literals, val: SomeInteger): uint32 {.inline.} = + ## Adds the bit-representation of `val` to `s` and returns an ID to refer + ## to the value with later. + when sizeof(val) < 4: + # always fits into the packed representation + result = cast[uint32](val) + else: + if (val shr OverflowShift) != 0: + # overflows the packed value range + result = uint32(s.numbers.len) or OverflowBit + when val is SomeSignedInt: + s.numbers.add castAny[uint64](int64(val)) # sign extend + else: + s.numbers.add castAny[uint64](val) + else: + result = castAny[uint32](val) # fits the packed range + +proc pack*(s: var Literals, val: float): uint32 {.inline.} = + ## Adds the bit-representation of `val` to `s` and returns an ID to refer + ## to the value with later. + result = uint32(s.numbers.len) + s.numbers.add cast[uint64](val) + +proc pack*(s: var Literals, val: string): uint32 {.inline.} = + ## Adds the `val` to `s` and returns an ID to refer to the value with later. + result = uint32(s.strings.len) + s.strings.add val + +proc unpack*[T: SomeNumber](s: Literals, id: uint32, _: typedesc[T]): T {.inline.} = + ## Returns the bit-representation stored under `id` interpreted as `T`. + if (id and OverflowBit) != 0: + castAny[T](s.numbers[id and not OverflowBit]) + else: + castAny[T](id) + +proc unpack*(s: Literals, id: uint32, _: typedesc[string]): lent string {.inline.} = + ## Returns the string stored under `id`. + s.strings[id] + +proc pack*(s: var Literals, val: SourceLoc): uint32 {.inline.} = + ## Adds `val` to the database and returns the ID to later look up `val` with. + result = uint32(s.locs.len) + s.locs.add val + +proc unpack*(s: Literals, id: uint32, _: typedesc[SourceLoc] + ): SourceLoc {.inline.} = + ## Returns the source location earlier stored under `id`. + s.locs[id] diff --git a/passes/passes.nim b/passes/passes.nim new file mode 100644 index 00000000..3b3f46c8 --- /dev/null +++ b/passes/passes.nim @@ -0,0 +1,976 @@ +## The home of all intermediate languages and passes, representing the core of +## the compiler. + +import + std/[ + streams, + strformat, + strutils, + tables + ], + experimental/[ + sexp_parse, + sexp + ], + nanopass/nanopass, + passes/[literals, compilerdef], + phy/[reporting, default_reporting] + +import passes/trees except Literals + +type + Symbol = object + Ident = distinct string + +defineLanguage Lsrc: + int(n) + float(fl) + string(str) + Ident(x) + + rec_field(rf) ::= Field(x, e) + field_decl(f) ::= Field(x, t) + + pattern(p) ::= As(x, t) + mrule(mr) ::= Rule(p, e) + expr(e) ::= n | fl | x | str | + ArrayCons(...e) | + TupleCons(...e) | + RecordCons(rf, ...rf) | + Seq(t, ...e) | + Seq(str) | + Call(e, ...e) | + FieldAccess(e, n) | + FieldAccess(e, x) | + At(e, e) | + As(e, t) | + And(e, e) | + Or(e, e) | + If(e, e) | + If(e, e, e) | + While(e, e) | + Return(e) | + Unreachable() | + Exprs(...e, e) | + Asgn(e, e) | + Decl(x, e) | + Match(e, mr, ...mr) + typ(t) ::= x | VoidTy() | UnitTy() | BoolTy() | CharTy() | IntTy() | FloatTy() | + ArrayTy(n, t) | + SeqTy(t) | + TupleTy(t, ...t) | + RecordTy(f, ...f) | + UnionTy(t, ...t) | + ProcTy(t, ...t) + + param_decl(pd) ::= ParamDecl(x, t) + params(pa) ::= Params(...pd) + decl(d) ::= ProcDecl(x, t, pa, e) | TypeDecl(x, t) + module(m) ::= Module(...d) + +defineLanguage L1, Lsrc: + ## Language without And and Or. + expr(e) ::= -And(e, e) | -Or(e, e) + +defineLanguage L2, L1: + ## Language without single-branch `If`. + expr(e) ::= -If(e, e) + +defineLanguage L3, L2: + ## Language that replaces Decl with Let. + expr(e) ::= -Decl(x, e) | +Let(x, e, e) + +defineLanguage L4, L3: + ## Language with symbols instead of raw identifiers. + +Symbol(s) + expr(e) ::= -x | +s | + -Let(x, e, e) | +Let(s, e, e) + typ(t) ::= -x | +s + pattern(p) ::= -As(x, t) | +As(t, s) # the element order is swapped + + param_decl(pd) ::= -ParamDecl(x, t) | +ParamDecl(s, t) + decl(d) ::= -ProcDecl(x, t, pa, e) | +ProcDecl(s, t, pa, e) | + -TypeDecl(x, t) | +TypeDecl(s, t) + +defineLanguage L5, L4: + ## Language with type information. + # expressions that always have the same type don't use a type tag + expr(e) ::= -ArrayCons(...e) | +ArrayCons(t, ...e) | + -TupleCons(...e) | +TupleCons(t, e, ...e) | + -RecordCons(rf, ...rf) | +RecordCons(t, rf, ...rf) | + -Seq(str) | +Seq(t, str) | + -Call(e, ...e) | +Call(t, e, ...e) | + -FieldAccess(e, n) | +FieldAccess(t, e, n) | + -FieldAccess(e, x) | +FieldAccess(t, e, x) | + -At(e, e) | +At(t, e, e) | + -As(e, t) | +As(t, e, t) | + -If(e, e, e) | +If(t, e, e, e) | + -Exprs(...e, e) | +Exprs(t, ...e, e) | + -Match(e, mr, ...mr) | +Match(t, e, mr, ...mr) | + +Unit() | + +Prim(t, str, ...e) # primitive calls (i.e., magic operations) + +defineLanguage Lnoas, L5: + ## Language with more specific forms instead of the generic `As`. + expr(e) ::= -As(t, e, t) | +Inj(t, e, t) + +defineLanguage L6, Lnoas: + ## Language with the `Match` from removed. + pattern(p) ::= -As(t, s) + mrule(mr) ::= -Rule(p, e) + expr(e) ::= -Match(t, e, mr, ...mr) | + +Is(t, e, t) | + +Unpack(t, e) + +defineLanguage L7, L6: + ## Language with no tagged unions, only untagged unions. + expr(e) ::= -Is(t, e, t) | -Unpack(t, e) + +defineLanguage L8, L7: + ## Language without records. + rec_field(rf) ::= -Field(x, e) + field_decl(f) ::= -Field(x, t) + expr(e) ::= -RecordCons(t, rf, ...rf) | -FieldAccess(t, e, x) + typ(t) ::= -RecordTy(f, ...f) + +defineLanguage L9, L8: + ## Language where copies are explicit and all aggregate access has a named + ## local as the root. + lvalue(lv) ::= + +s | + +At(t, lv, e) | + +FieldAccess(t, lv, n) + expr(e) ::= + -s | + -At(t, e, e) | + -FieldAccess(t, e, n) | + +lv | + +Copy(t, lv) + +defineLanguage Lseq, L9: + ## Language with no built-in sequence values nor types. + # a "box" is a single-owner single value container + expr(e) ::= -Seq(t, ...e) | -Seq(t, str) | +Box(t, e) | +Unbox(t, e) + typ(t) ::= -SeqTy(t) | +BoxTy(t) + +defineLanguage Lnocopy, Lseq: + ## Language without `Copy` form. + expr(e) ::= -Copy(t, lv) + +defineLanguage Lnobox, Lnocopy: + expr(e) ::= -Box(t, e) | -Unbox(t, e) + typ(t) ::= -BoxTy(t) | +PtrTy() + +defineLanguage Lnocons, Lnobox: + ## Language without aggregate constructors. + init(ie) ::= +e | +Undef() + expr(e) ::= -Let(s, e, e) | +Let(s, ie, e) | + -ArrayCons(t, ...e) | + -TupleCons(t, e, ...e,) + +defineLanguage Lnoletwithval, Lnocons: + ## Language without Let initializers. + init(ie) ::= -e | -Undef() + expr(e) ::= -Let(s, ie, e) | +Let(s, e) + +defineLanguage Lstmt, Lnoletwithval: + ## Language with a statement/expression separation. + expr(e) ::= -Asgn(e, e) | + -Unreachable() | + -Return(e) | + -While(e, e) | + -Exprs(t, ...e, e) | + +Exprs(t, ...st, e) + stmt(st) ::= +Asgn(e, e) | + +Unreachable() | + +Return(e) | + +While(e, st) | + +Pass() | # statement without effect + +Call(t, e, ...e) | + +If(e, st, st) | +If(e, st) | + +Let(s, st) | + +Stmts(...st, st) + decl(d) ::= -ProcDecl(s, t, pa, e) | +ProcDecl(s, t, pa, st) + +defineLanguage L10, Lstmt: + ## Language with no If expressions. + expr(e) ::= -If(t, e, e, e) + +defineLanguage L11, L10: + ## Language with only blob and scalar types. + lvalue(lv) ::= -s | + -FieldAccess(t, lv, n) | + -At(t, lv, e) + expr(e) ::= -lv | + +s | + +Offset(t, e, e, n) | + +Load(t, e) | + +Addr(s) + stmt(st) ::= -Asgn(e, e) | + +Asgn(s, e) | + +Store(e, e) + +defineLanguage L12, L11: + ## Language with only simple callee expressions. + expr(e) ::= -Call(t, e, ...e) | +Call(t, s, ...e) + stmt(st) ::= -Call(t, e, ...e) | +Call(t, s, ...e) + +proc pack(s: var Literals, val: Ident): uint32 {.inline.} = + pack(s, string(val)) + +proc unpack(s: Literals, id: uint32, _: typedesc[Ident]): lent Ident {.inline.} = + Ident unpack(s, id, string) + +proc eatString(s: var SexpParser): string = + result = s.captureCurrString() + discard s.getTok() + +proc expect(s: SexpParser, kind: TTokKind) = + if s.currToken != kind: + # TODO: report a proper syntax error + raise ValueError.newException("expected " & $kind & ", got " & $s.currToken) + +proc eat(s: var SexpParser, kind: TTokKind) = + s.expect(kind) + discard s.getTok() + +proc parse(s: var SexpParser): Lsrc {.inpass.} = + ## Parses the source language from an S-expression stream. + # note: the implementation is incomplete and incorrect (there's also no + # formal grammar backing it) + proc expr(s: var SexpParser): dst.e {.transform.} + proc typ(s: var SexpParser): dst.t {.transform.} + + proc ident(s: var SexpParser): dst.x {.transform.} = + s.space() + s.expect(tkSymbol) + terminal Ident(s.eatString()) + + proc call(s: var SexpParser): dst.e {.transform.} = + # note: does not handle the parenthesis + let callee = expr(s) + var args = newSeq[dst.e]() + s.space() + while s.currToken != tkParensRi: + args.add expr(s) + s.space() + + build Call(callee, args) + + proc pattern(s: var SexpParser): dst.p {.transform.} = + s.eat(tkParensLe) + s.expect(tkSymbol) + let name = s.eatString() + s.space() + let t = typ(s) + s.eat(tkParensRi) + build As(x(^Ident(name)), t) + + proc rule(s: var SexpParser): dst.mr {.transform.} = + s.eat(tkParensLe) + s.eat(tkSymbol) + # TODO: make sure the symbol is "rule" + s.space() + let p = pattern(s) + s.space() + let body = expr(s) + s.eat(tkParensRi) + build Rule(p, body) + + proc expr(s: var SexpParser): dst.e {.transform.} = + s.space() + case s.currToken + of tkParensLe: + let k = s.getTok() + if k == tkSymbol: + case s.currString + of "if": + discard s.getTok() + result = build If(^expr(s), ^expr(s), ^expr(s)) + of "while": + discard s.getTok() + result = build While(^expr(s), ^expr(s)) + of "decl": + discard s.getTok() + result = build Decl(^ident(s), ^expr(s)) + of "and": + discard s.getTok() + result = build And(^expr(s), ^expr(s)) + of "or": + discard s.getTok() + result = build Or(^expr(s), ^expr(s)) + of "as": + discard s.getTok() + result = build As(^expr(s), ^typ(s)) + of "exprs": + discard s.getTok() + s.space() + var elems: seq[dst.e] + while s.currToken != tkParensRi: + elems.add expr(s) + s.space() + + let last = elems.pop() + result = build Exprs(elems, last) + of "match": + discard s.getTok() + let e = expr(s) + s.space() + let first = rule(s) + s.space() + var next: seq[dst.mr] + while s.currToken != tkParensRi: + next.add rule(s) + s.space() + result = build Match(e, first, next) + else: + # TODO: implement the remaining keywords + result = call(s) + else: + result = call(s) + + s.eat(tkParensRi) + of tkString: + result = build(dst.e, str(^s.eatString())) + of tkSymbol: + result = build x(^Ident(s.eatString())) + of tkInt: + result = build n(^parseInt(s.eatString())) + of tkFloat: + result = build fl(^parseFloat(s.eatString())) + else: + raise ValueError.newException($s.currToken) + # syntaxError(s, "expected expression, but got " & ) + + proc typ(s: var SexpParser): dst.t {.transform.} = + s.space() + s.eat(tkParensLe) + s.expect(tkSymbol) + let str = s.eatString() + case str + of "bool": result = build BoolTy() + of "int": result = build IntTy() + of "float": result = build FloatTy() + of "union": + let first = typ(s) + s.space() + var e: seq[dst.t] + while s.currToken != tkParensRi: + e.add typ(s) + s.space() + result = build UnionTy(first, e) + else: + result = build x(str) + s.eat(tkParensRi) + + proc param(s: var SexpParser): dst.pd {.transform.} = + s.eat(tkParensLe) + result = build ParamDecl(^ident(s), ^typ(s)) + s.space() + s.eat(tkParensRi) + + proc params(s: var SexpParser): dst.pa {.transform.} = + var p: seq[dst.pd] + s.space() + s.eat(tkParensLe) + while s.currToken != tkParensRi: + p.add param(s) + s.space() + s.eat(tkParensRi) + build Params(p) + + proc top(s: var SexpParser): dst.d {.transform.} = + s.eat(tkParensLe) + s.expect(tkSymbol) + case s.eatString() + of "proc": + result = build ProcDecl(^ident(s), ^typ(s), ^params(s), ^expr(s)) + of "type": + result = build TypeDecl(^ident(s), ^typ(s)) + else: + # TODO: proper syntax error + raise ValueError.newException("") + s.eat(tkParensRi) + + proc module(s: var SexpParser): dst.m {.transform.} = + s.space() + var decls: seq[dst.d] + while s.currToken != tkEof: + decls.add top(s) + s.space() + build Module(decls) + + module(s) + +proc removeAndOr(x: Lsrc): L1 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of And([e0], [e1]): build If(e0, e1, x(^Ident("false"))) + of Or([e0], [e1]): build If(e0, x(^Ident("true")), e1) + +proc removeSingleIf(x: L1): L2 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of If([e0], [e1]): build If(e0, e1, TupleCons([])) + +proc declToLet(x: L2): L3 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of Decl(x, [e]): + build Let(x, e, TupleCons([])) + of Exprs(e0, [e1]): + var r = e1 + var got: seq[dst.e] + for i in countdown(e0.high, 0): + match e0[i]: + of Decl(x, e): + r = + if got.len == 0: build Let(x, ^expr(e), r) + else: build Let(x, ^expr(e), Exprs(got, r)) + else: + got.insert expr(e0[i]) + + if got.len == 0: build Exprs(got, r) + else: r + +proc symbolize(x: L3, r: ref ReportContext[string]): L4 {.pass.} = + ## Binds all identifiers to symbols. + var ctx: Table[string, Value[Symbol]] + proc error(msg: string) = + r.error(msg) + + proc add(name: sink string): Value[Symbol] = + if name in ctx: + error(fmt"'{name}' has a symbol bound already") + result = terminal Symbol() + ctx[name] = result + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Let(x, e0, e1): + let a = expr(e0) + let s = add(x.val.string) + let b = expr(e1) + # the symbol goes out of scope at the end of the Let + ctx.del(x.val.string) + build Let(s, a, b) + of x: + if x.val.string in ctx: + build ^ctx[x.val.string] + else: + error(fmt"undeclared identifier: '{x.val.string}'") + build s(^Symbol()) # error correction + + proc typ(n: src.t): dst.t {.transform.} = + case n + of x: + if x.val.string in ctx: + build ^ctx[x.val.string] + else: + error(fmt"undeclared identifier: '{x.val.string}'") + build s(^Symbol()) # error correction + + proc rule(n: src.mr): dst.mr {.transform.} = + case n + of Rule(p, e): + match p: + of As(x, t): + let s = add(x.val.string) + let e2 = expr(e) + ctx.del(x.val.string) + build Rule(As(^typ(t), s), e2) + + proc param(n: src.pd): dst.pd {.transform.} = + case n + of ParamDecl(x, [t]): + build ParamDecl(^add(x.val.string), t) + + proc params(n: src.pa): dst.pa {.generated.} + + proc decl(n: src.d): dst.d {.transform.} = + case n + of ProcDecl(x, t, pa, e): + let s = add(x.val.string) + build ProcDecl(s, ^typ(t), ^params(pa), ^expr(e)) + of TypeDecl(x, [t]): + build TypeDecl(^add(x.val.string), t) + +proc typeCheck(x: L4): L5 {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + # XXX: temporary implementation, just so that something exists + case n + of ArrayCons([e]): + build ArrayCons(UnitTy(), e) + of TupleCons([e]): + if e.len == 0: build Unit() + else: build TupleCons(UnitTy(), ^e[0], e) # FIXME + of RecordCons([rf0], [rf1]): + build RecordCons(UnitTy(), rf0, rf1) + of Seq(str): + build Seq(SeqTy(CharTy()), str) + of Seq([t], [e]): + build Seq(SeqTy(t), e) + of Call([e0], [e1]): + build Call(UnitTy(), e0, e1) + of FieldAccess([e], n): + build FieldAccess(UnitTy(), e, n) + of FieldAccess([e], x): + build FieldAccess(UnitTy(), e, x) + of At([e0], [e1]): + build At(UnitTy(), e0, e1) + of If([e0], [e1], [e2]): + build If(UnitTy(), e0, e1, e2) + of As([e], [t]): + build As(t, e, t) + of Exprs([e0], [e1]): + build Exprs(UnitTy(), e0, e1) + of Match([e], [mr0], [mr1]): + build Match(UnitTy(), e, mr0, mr1) + +proc specializeAs(x: L5): Lnoas {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of As([t0], [e], [t1]): + # TODO: handle the non-union-constructor meaning of `As` (i.e., turn + # into an `Exprs`) + build Inj(t0, e, t1) + +proc lowerMatch(x: Lnoas): L6 {.pass.} = + ## Replaces `Match` forms with chains of if expressions. + proc expr(n: src.e): dst.e {.transform.} = + case n + of Match([t], [e], mr0, mr1): + proc wrap(tmp: dst.s, r: src.mr, last: dst.e): dst.e = + match r: + of Rule(p, e): + match p: + of As([t2], s): + build If(t, Is(BoolTy(), tmp, t2), + Let(s, Unpack(t2, tmp), ^expr(e)), last) + + let tmp = terminal Symbol() + var last = build Unit() + # TODO: ^^ use `Unreachable`. Presently doesn't work due to a pass + # ordering issue + for i in countdown(mr1.high, 0): + last = wrap(tmp, mr1[i], last) + build Let(tmp, e, ^wrap(tmp, mr0, last)) + + +proc nounion(x: L6): L7 {.pass.} = + # TODO: use a proper name + proc same(a, b: src.t): bool = + # TODO: implement this somehow. Structural tree equality should likely + # just be provided by the framework + false + + proc computeTag(union, elem: src.t): int = + match union: + of UnionTy(t0, t1): + if same(t0, elem): + 0 + else: + var r = 0 + # for i, it in t1.pairs: + # if same(it.index, elem.index): + # r = i + # break + r + else: + unreachable() + + proc getType(n: src.e): src.t = + # TODO: figure out some way to use '_' for elements (to discard them) + match n: + of Call(t, e0, e1): t + of If(t, e0, e1, e2): t + # TODO: complete + else: unreachable() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Is([t], e, t1): + # replace with a tag comparison + let u = getType(e) + build Prim(t, str(^"eq"), + [FieldAccess(IntTy(), ^expr(e), n(^0)), n(^computeTag(u, t1))]) + of Unpack([t], [e]): + # XXX: how to best get access to the lowered type? + build FieldAccess(t, FieldAccess(t, e, n(^1)), n(^0)) + of Inj(t0, [e], t1): + build TupleCons(^typ(t0), n(^computeTag(t0, t1)), [e]) + + proc typ(n: src.t): dst.t {.transform.} = + case n + of UnionTy([t0], [t1]): + # becomes a tag + union + build TupleTy(IntTy(), [UnionTy(t0, t1)]) + +proc norecord(x: L7): L8 {.pass.} = + # TODO: use a proper name + proc fdef(n: src.rf): dst.e {.transform.} = + case n + of Field(x, [e]): e + + proc fdef(n: src.f): dst.t {.transform.} = + case n + of Field(x, [t]): t + + proc expr(n: src.e): dst.e {.transform.} = + case n + of RecordCons([t], [e0], [e1]): + # TODO: sort the fields while ensuring correct evaluation order + build TupleCons(t, e0, e1) + of FieldAccess([t], [e], x): + build FieldAccess(t, e, n(^0)) # TODO: use the correct index + + proc typ(n: src.t): dst.t {.transform.} = + case n + of RecordTy([t0], [t1]): + # TODO: sort the fields, so that `RecordTy(IntTy(), FloatTy())` and + # `RecordTy(FloatTy(), IntTy())` both produce the same + build TupleTy(t0, t1) + +proc bridge(x: L8): Lnocons {.pass.} = + ## Pass to bridge between some ILs for which the intermediate lowering + ## passes are still missing. + # TODO: implement the intermediate passes + proc typ(n: src.t): dst.t {.transform.} = + case n + of SeqTy(t): build BoolTy() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Seq(t, e): build Unit() + of Seq(t, str): build Unit() + of Let(s, [e1], [e2]): build Let(s, e1, e2) + of ArrayCons(t, e): build Unit() + of TupleCons(t, e0, e1): build Unit() + of FieldAccess(t, e, n): build Unit() + of At(t, e0, e1): build Unit() + +proc simplifyLet(x: Lnocons): Lnoletwithval {.pass.} = + proc expr(n: src.e): dst.e {.transform.} = + case n + of Let(s, ie, [e]): + match ie: + of Undef(): build Let(s, e) + of e1: build Let(s, Exprs(UnitTy(), [Asgn(s, ^expr(e1))], e)) + +proc exprToStmt(x: Lnoletwithval): Lstmt {.pass.} = + proc etos(n: src.e): dst.st {.transform.} = + case n + # TODO: auto-generate transforms such as, e.g., `Let(s, [st]) -> Let(s, st)` + of Let(s, [st]): build Let(s, st) + of Unit(): build Pass() # a trailing Unit expression becomes Pass + of If(t, [e], [st], e2): + match e2: + of Unit(): build If(e, st) + else: build If(e, st, ^etos(e2)) + of Exprs(t, [st0], [st1]): + build Stmts(st0, st1) + of While([e], [st]): + build While(e, st) + # TODO: also auto-generate the below + of Asgn([e0], [e1]): build Asgn(e0, e1) + of Return([e]): build Return(e) + of Unreachable(): build Unreachable() + else: + # TODO: maybe raise a proper run-time error? It would allow for better + # error messages and thus easier development + unreachable() + + proc expr(n: src.e): dst.e {.transform.} = + case n + of Exprs([t], [st], [e]): build Exprs(t, st, e) + of Asgn([e0], [e1]): + build Exprs(UnitTy(), [Asgn(e0, e1)], Unit()) + of While([e], [st]): + build Exprs(UnitTy(), [While(e, st)], Unit()) + # TODO: auto-generate the exception raising? Right now, a static error is + # reported, but - for convenience - the nanopass framework could + # also just leave reporting an error to the runtime (Scheme's + # nanopass framework does it that way, for what it's worth) + of Return(e): + raise ValueError.newException("not an expr") + of Unreachable(): + raise ValueError.newException("not an expr") + + proc decl(n: src.d): dst.d {.transform.} = + # TODO: move this to a separate pass and handle void bodies correctly + case n + of ProcDecl(s, [t], [pa], [e]): build ProcDecl(s, t, pa, Return(e)) + +proc ifExprToStmt(x: Lstmt): L10 {.pass.} = + # TODO: needs to happen earlier + proc expr(n: src.e): dst.e {.transform.} = + case n + of If([t], [e0], [e1], [e2]): + let s = terminal Symbol() + build Let(s, Exprs(t, [If(e0, Asgn(s, e1), Asgn(s, e2))], s)) + +proc blobify(x: L10): L11 {.pass.} = + ## Turns all aggregate types into blob types. + proc lval(n: src.lv): (dst.t, dst.e) = + match n: + of s: + # TODO: use the correct expression type + (build(dst.t, UnitTy()), build(dst.e, Addr(s))) + of At([t], lv, e): + let (elem, x) = lval(lv) + # TODO: use the correct stride (taken from the `elem`) + (t, build(dst.e, Offset(PtrTy(), x, ^expr(e), n(^1)))) + of FieldAccess([t], lv, n): + let (elem, x) = lval(lv) + # TODO: use the correct offset + (t, build(dst.e, Offset(PtrTy(), x, n, n(^1)))) + + proc expr(n: src.e): dst.e {.transform.} = + case n + of lv: + let (t, e) = lval(lv) + build Load(t, e) + + proc stmt(n: src.st): dst.st {.transform.} = + case n + of Asgn(e0, [e1]): + match e0: + of lv: build Store(^(lval(lv)[1]), e1) + else: unreachable() + +proc calleeToSymbol(x: L11): L12 {.pass.} = + ## Removes all complex callee expressions. + proc expr(n: src.e): dst.e {.transform.} = + case n + of Call([t], e0, [e1]): + match e0: + of s: build Call(t, s, e1) + else: + let tmp = terminal Symbol() + build Let(tmp, Exprs(t, [Asgn(tmp, ^expr(e0))], Call(t, tmp, e1))) + + proc stmt(n: src.st): dst.st {.transform.} = + case n + of Call([t], e0, [e1]): + match e0: + of s: build Call(t, s, e1) + else: + let tmp = terminal Symbol() + build Let(tmp, Stmts([Asgn(tmp, ^expr(e0))], Call(t, tmp, e1))) + +proc toSexp(x: SomeInteger): SexpNode = newSInt(x) +proc toSexp(x: float): SexpNode = newSFloat(x) +proc toSexp(x: string): SexpNode = newSString(x) +proc toSexp(x: Ident): SexpNode = newSSymbol(x.string) + +proc render*(x: Lsrc.m): SexpNode {.renderer.} + +# TODO: fix the symbol binding in the pass DSL such that `vmenv` can be +# imported at the top (at the moment, ambiguity errors ensue) +import vm/[vmmodules, vmspec, vmenv] + +type TypeKind = enum + tkInt, tkFloat, tkPtr, tkBlob + +# XXX: doesn't work yet +#[ +proc genvm(x: L12): VmModule {.outpass.} = + ## VM code generator. Generates a VM module from `x`. + type Assembler = object + code: seq[Instr] + locals: seq[tuple[typ: TypeKind, used: bool]] + map: Table[Value[Symbol], uint32] + + proc instr(c: var Assembler, op: Opcode) = + c.code.add Instr(InstrType(op)) + proc instr(c: var Assembler, op: Opcode, a: int32) = + c.code.add Instr(InstrType(op) or (a.InstrType shl instrAShift)) + proc instr(c: var Assembler, op: Opcode, b: uint16) = + c.code.add Instr(InstrType(op) or (b.InstrType shl instrBShift)) + + proc label(c: Assembler): uint32 = c.code.len.uint32 + proc jump(c: var Assembler, op: Opcode): uint32 = + c.code.add Instr(InstrType(op)) + result = c.code.high.uint32 + proc join(c: var Assembler, pos: uint32) = + c.code[pos] = Instr(InstrType(c.code[pos].opcode)) + proc jump(c: var Assembler, op: Opcode, target: uint32) = + c.code.add Instr(InstrType(op) or (InstrType(c.code.len.uint32 - target) shl instrAShift)) + + proc allocLocal(c: var Assembler, typ: TypeKind): int32 = + for i, it in c.locals.mpairs: + if it.typ == typ and not it.used: + it.used = true + return i.int32 + + c.locals.add (typ, true) + result = c.locals.high.int32 + + proc alloc(c: var Assembler, x: Symbol): int32 = + result = c.allocLocal: + case x.typ.kind + of tkFloat, tkInt: x.typ.kind + else: tkInt + + if x.typ.kind == tkBlob: + # allocate a stack slot + c.instr(opcStackAlloc, x.typ.size) + c.instr(opcPopLocal, result) + + proc free(c: var Assembler, loc: int32, x: Symbol) = + c.locals[loc].used = false + if x.typ.kind == tkBlob: + c.instr(opcStackFree, x.typ.size) + + proc expr(n: src.e, c: var Assembler) = + match n: + of Prim(t, str, e): + case str.val + of "+": + expr(e[0], c) + expr(e[1], c) + c.gen(opcAddInt) + of "<": + expr(e[0], c) + expr(e[1], c) + c.gen(opcLtInt) + of "<=": + expr(e[0], c) + expr(e[1], c) + c.gen(opcLeInt) + of "==": + expr(e[0], c) + expr(e[1], c) + c.gen(opcEqInt) + # of AddChecked(t, e0, e1, x): + # expr(e0, c) + # expr(e1, c) + # c.instr(opcAddChck) + # c.instr(opcPopLocal, x.get) + of Addr(x): + c.instr(opcGetLocal, x) + of Let(s, e): + let x = c.alloc(s) + expr(e, c) + c.free(x) + of Load(t, e): + expr(e) + c.gen(opcLdInt32) + of x: + c.gen(opcGetLocal, c.map[x]) + of Call(t, s, e): + for it in e.items: + expr(it) + c.gen(opcCall, s, x) + of Exprs(st, e): + for it in st.items: + stmt(it, c) + expr(e, c) + + proc stmt(n: src.st, c: var Assembler) = + match n: + of Let(s, st): + let x = c.alloc(s.val) + stmt(st, c) + c.free(x, s.val) + of If(e, st0, st1): + expr(e, c) + let x = c.jump(opcBranch) + stmt(st0, c) + let y = c.jump(opcJmp) + c.join(x) + stmt(st1, c) + c.join(y) + of If(e, st): + expr(e, c) + let x = c.jump(opcBranch) + stmt(st, c) + c.join(x) + of While(e, st): + let lab = c.label() + expr(e, c) + let x = c.jump(opcBranch) + stmt(st, c) + c.jump(opcJmp, lab) + c.join(x) + of Asgn(s, e): + expr(e, c) + c.instr(opcPopLocal, c.map[s]) + of Store(e0, e1): + expr(e0, c) + expr(e1, c) + c.instr(opcWrInt16) + # of MemCopy(e0, e1, e2): + # expr(e0) + # expr(e1) + # expr(e2) + # c.gen(opcMemCopy) + of Return(e): + expr(e, c) + c.instr(opcRet) + of Call(t, s, e1): + for it in e1.items: + expr(it, c) + if s.val.typ.kind == tkPtr: + c.instr(opcGetLocal, c.map[s]) + c.instr(opcIndCall, e1.len) + else: + c.instr(opcCall, s.val, e1.len) + of Unreachable(): + c.instr(opcUnreachable) + + proc prc(name: src.x, ret: src.t, params: src.pa, body: src.st, m: var VmModule) = + var c = Assembler() + let p = unpack(params) + for i, it in p.pairs: + let loc = c.alloc(it.typ) + c.locals[it] = loc + c.instr(opcPopLocal, loc) + stmt(body, c) + # TODO: append to the module + + proc module(n: src.m, m: var VmModule): string = + let procs = unpack(n) + for it in procs.items: + let (x, t, pa, s) = unpack(it) + prc(x, t, pa, s, m) + + module(x, result) +]# + +# proc render(x: L4.m): SexpNode {.renderer.} + +# some logic for testing: +var rep = initDefaultReporter[string]() +defineCompiler compile, [ + removeAndOr, + removeSingleIf, + declToLet, + symbolize(rep), + typeCheck, + specializeAs, + lowerMatch, + nounion, + norecord, + bridge, + simplifyLet, + exprToStmt, + ifExprToStmt, + blobify, + calleeToSymbol] + +var s: SexpParser +# s.open(newStringStream(""" +# (proc a (bool) () (exprs (decl x true) (decl y (and x 2)) (and (or 1 2) (and 3 4)))) +# """)) +s.open(newStringStream(""" +(proc a (bool) ((x (int))) (== x 2)) +(proc b (bool) ((x (float))) (== x 2)) +(proc test (bool) ((x (union (int) (float)))) (match x (rule (y (int)) (a x)) (rule (z (float)) (b z)))) +(proc main (bool) () (test (as 1 (union (int) (float))))) +""")) +discard s.getTok() +let m = parse(s) +s.close() + +let got = compile(m) diff --git a/passes/passes_legacy.nim b/passes/passes_legacy.nim new file mode 100644 index 00000000..da2a401a --- /dev/null +++ b/passes/passes_legacy.nim @@ -0,0 +1,1353 @@ + +import nanopass/nanopass +import passes/literals +import std/[tables, intsets, options, algorithm] + +type + Local = distinct uint32 + Global = distinct uint32 + Proc = distinct uint32 + Type = distinct uint32 + +defineLanguage Lskully: + ## Language for the code output by skully. + int64(i) + float64(fl) + string(str) + Local(lo) + Global(g) + Proc(pr) # proc is a reserved word + Type(tid) + + field(f) ::= Field(i, t) + typ(t) ::= tid | Int(i) | UInt(i) | Float(i) | Ptr() | Void() | + Record(i, i, f, ...f) | Union(i, i, t, ...t) | Array(i, i, i, t) | + ProcTy(t, ...t) + + lvalue(lv) ::= lo | g | Deref(t, e) | Field(lv, i) | At(lv, e) + expr(e) ::= i | fl | ProcVal(i) | + Call(t, e, ...e) | Call(pr, ...e) | + Conv(t, t, e) | Reinterp(t, t, e) | Trunc(t, t, e) | + Zext(t, t, e) | Sext(t, t, e) | + Demote(t, t, e) | Promote(t, t, e) | + Nil() | + Load(t, e) | Copy(lv) | Addr(lv) | + Add(t, e, e) | Sub(t, e, e) | Mul(t, e, e) | Div(t, e, e) | Mod(t, e, e) | + BitNot(t, e) | BitAnd(t, e, e) | BitOr(t, e, e) | BitXor(t, e, e) | + AddChck(t, e, e, lo) | SubChck(t, e, e, lo) | MulChck(t, e, e, lo) | + Shl(t, e, e) | Shr(t, e, e) | + Neg(t, e) | + Not(e) | + Eq(t, e, e) | Le(t, e, e) | Lt(t, e, e) + + goto(go) ::= Goto(i) + + target(tgt) ::= Goto(i) | Unwind() + stmt(st) ::= Stmts(st, ...st) | Goto(i) | Join(i) | + CheckedCall(t, e, ...e, tgt) | + CheckedCall(pr, ...e, tgt) | + CheckedCallAsgn(lo, t, e, ...e, tgt) | + CheckedCallAsgn(lo, pr, ...e, tgt) | + Call(pr, ...e) | Call(t, e, ...e) | + Return() | Return(e) | + Raise(e, tgt) | + Branch(e, go, go) | + Blit(e, e, e) | Clear(e, e) | + Store(t, e, e) | Asgn(lv, e) | Except(i, lo) | + Unreachable() | + Loop(i) | + Drop(e) + + params(p) ::= Params(...lo) + locals(locs) ::= Locals(...t) + + init(ini) ::= Data(t, str) | Data(t) + + # the v + decl(d) ::= Import(t, str) | + ProcDef(t, p, locs, st) | + GlobalDef(t, i) | GlobalDef(t, fl) | + GlobalLoc(t, ini) + exprt(exp) ::= Export(str, g) | Export(str, pr) # export is a reserved word + + # + tdecls(td) ::= TypeDefs(...t) + gdecls(gd) ::= GlobalDefs(...d) + pdecls(pd) ::= ProcDefs(...d) + exports(es) ::= List(...exp) + + module(m) ::= Module(td, gd, pd, es) + +defineLanguage L6, Lskully: + ## Language with basic blocks. + bblock(bb) ::= +Block(p, ...st, ex) | +Except(p, ...st, ex) + stmt(st) ::= -Return() | + -Return(e) | + -Goto(i) | + -Raise(e, tgt) | + -Loop(i) | + -Unreachable() | + -Join(i) | + -Except(i, lo) | + -CheckedCall(t, e, ...e, tgt) | + -CheckedCall(pr, ...e, tgt) | + -CheckedCallAsgn(lo, t, e, ...e, tgt) | + -CheckedCallAsgn(lo, pr, ...e, tgt) | + -Branch(e, go, go) | + -Stmts(st, ...st) + exit(ex) ::= +Return() | +Return(e) | +Goto(i) | +Raise(e, tgt) | +Branch(e, go, go) | + +CheckedCall(t, e, ...e, go, tgt) | + +CheckedCall(pr, ...e, go, tgt) | + +CheckedCallAsgn(lo, t, e, ...e, go, tgt) | + +CheckedCallAsgn(lo, pr, ...e, go, tgt) | + +Unreachable() | + +Loop(i) + blocks(bl) ::= +List(bb, ...bb) + decl(d) ::= -ProcDef(t, p, locs, st) | +ProcDef(t, locs, bl) + +defineLanguage L5, L6: + ## Language without mutable global locations. + init(ini) ::= -Data(t) | -Data(t, str) | +Data(i, i) | +Data(i, str) + decl(d) ::= -GlobalLoc(t, ini) | +GlobalDef(t, ini) + lvalue(lv) ::= -g + expr(e) ::= +Copy(g) + +defineLanguage L4, L5: + ## Language with combined, flat paths. + root(ro) ::= +lo | +Deref(t, e) + lvalue(lv) ::= -Field(lv, i) | -At(lv, e) | -Deref(t, e) | + +Path(t, ro, e, ...e) + +defineLanguage L3s2, L4: + ## Language without aggregate parameters. + +defineLanguage L3s1, L3s2: + ## Language without aggregate types. + expr(e) ::= +Offset(e, e, e) + typ(t) ::= +Blob(i, i) | + -Record(i, i, f, ...f) | + -Array(i, i, i, t) | + -Union(i, i, t, ...t) + lvalue(lv) ::= -Path(t, ro, e, ...e) + +defineLanguage L3, L3s1: + ## Language without primitive locals that can have their address taken. + +defineLanguage L2, L3: + ## Language without blob assignments. + +defineLanguage L1, L2: + ## Language with explicit stack management and only primitive types. + typ(t) ::= -Blob(i, i) + expr(e) ::= -Addr(lv) + decl(d) ::= -ProcDef(t, locs, bl) | + +ProcDef(t, i, locs, bl) + +defineLanguage LPtr, L1: + ## Language where only proc types are identified. + +defineLanguage L0, LPtr: + ## Language without pointer types and related operations. + typ(t) ::= -Ptr() + expr(e) ::= -Nil() | -Offset(e, e, e) + +# terminal adapter routines: +template defineTerminal(typ: typedesc) {.dirty.} = + template unpack(lit: Literals, id: uint32, _: typedesc[typ]): typ = + typ(id) + template pack(lit: var Literals, val: typ): uint32 = + uint32(val) + +defineTerminal(Local) +defineTerminal(Global) +defineTerminal(Proc) +defineTerminal(Type) + +template map[T, U](x: ChildSlice[T, auto], p: proc(x: T): U): seq[U] = + let cs = x + var s = newSeq[U](cs.len) + for i, it in x.pairs: + s[i] = p(it) + s + +template mapIt[T](x: ChildSlice[T, auto], body: untyped): untyped = + let cs = x + var s = newSeq[ + typeof( + block: + var it {.inject.}: T + body + it) + ](cs.len) + for i, it {.inject.} in x.pairs: + s[i] = body + s + +proc basicBlocks(ir: Lskully): L6 {.pass.} = + ## Transforms the bodies of procedures into a basic-block-oriented structure. + + proc target(x: src.tgt, map: Table[int64, int]): dst.tgt = + match x: + of Goto(i): build dst.tgt, x.info, Goto(i(^map[i.val])) + of Unwind(): build dst.tgt, x.info, Unwind() + + proc goto(x: src.go, map: Table[int64, int]): dst.go = + match x: + of Goto(i): build dst.go, x.info, Goto(i(^map[i.val])) + + type BBlock = object + isExcept: bool + params: dst.p + stmts: seq[dst.st] + + proc blocks(x: src.st, bbs: var seq[dst.bb], bb: var BBlock, map: Table[int64, int]) = + proc commitBlock(bbs: var seq[dst.bb], bb: var BBlock, ex: dst.ex) = + if bb.isExcept: + bbs.add build(dst.bb, NoSLoc, Except(^bb.params, ...bb.stmts, ex)) + else: + bbs.add build(dst.bb, NoSLoc, Block(^bb.params, ...bb.stmts, ex)) + bb.stmts.shrink(0) + + proc startBlock(bb: var BBlock) = + bb.isExcept = false + bb.params = build(dst.p, NoSLoc, Params([])) + + match x: + of Stmts(...st): + for it in st.items: + blocks(it, bbs, bb, map) + of Goto(i): + commitBlock bbs, bb, build(dst.ex, x.info, Goto(i(^map[i.val]))) + of Join(i): + # don't merge an empty entry block with the following block; the latter + # might be a loop start + if map[i.val] > bbs.len: + commitBlock bbs, bb, build(dst.ex, x.info, Goto(i(^map[i.val]))) + startBlock(bb) + of Except(_, lo): + assert bb.stmts.len == 0, "control flow falls through into exception handler" + bb.isExcept = true + bb.params = build(dst.p, x.info, Params([lo])) + of Asgn([lv], [e]): + bb.stmts.add build(dst.st, x.info, Asgn(lv, e)) + of Store([t], [e0], [e1]): + bb.stmts.add build(dst.st, x.info, Store(t, e0, e1)) + of Blit([e0], [e1], [e2]): + bb.stmts.add build(dst.st, x.info, Blit(e0, e1, e2)) + of Clear([e0], [e1]): + bb.stmts.add build(dst.st, x.info, Clear(e0, e1)) + of Call(pr, ...[e]): + bb.stmts.add build(dst.st, x.info, Call(pr, ...e)) + of Call([t], [e0], ...[e1]): + bb.stmts.add build(dst.st, x.info, Call(t, e0, ...e1)) + of Drop([e]): + bb.stmts.add build(dst.st, x.info, Drop(e)) + of CheckedCall([t], [e0], ...[e1], tgt): + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCall(t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCall(pr, ...[e], tgt): + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCall(pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCallAsgn(lo, [t], [e0], ...[e1], tgt): + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCallAsgn(lo, t, e0, ...e1, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of CheckedCallAsgn(lo, pr, ...[e], tgt): + commitBlock bbs, bb, build(dst.ex, x.info, CheckedCallAsgn(lo, pr, ...e, Goto(i(^(bbs.len+1))), ^target(tgt, map))) + startBlock(bb) + of Return(): + commitBlock bbs, bb, build(dst.ex, x.info, Return()) + of Return([e]): + commitBlock bbs, bb, build(dst.ex, x.info, Return(e)) + of Raise([e], tgt): + commitBlock bbs, bb, build(dst.ex, x.info, Raise(e, ^target(tgt, map))) + of Branch([e], go0, go1): + commitBlock bbs, bb, build(dst.ex, x.info, Branch(e, ^goto(go0, map), ^goto(go1, map))) + of Unreachable(): + commitBlock bbs, bb, build(dst.ex, x.info, Unreachable()) + of Loop(i): + commitBlock bbs, bb, build(dst.ex, x.info, Loop(i(^map[i.val]))) + + proc scanStmt(ir: src.st, map: var Table[int64, int], wasJoin: var bool, + next: var int) = + match ir: + of Stmts(...st): + for it in st.items: + scanStmt(it, map, wasJoin, next) + of Join(i): + if wasJoin: + # merge joins immediately following each other + map[i.val] = next - 1 + else: + map[i.val] = next + inc next + wasJoin = false + of Except(i, _): + map[i.val] = next + inc next + wasJoin = false + of CheckedCall(...any): + inc next + wasJoin = true + of CheckedCallAsgn(...any): + inc next + wasJoin = true + else: + wasJoin = false + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], [p], [locs], st): + var map: Table[int64, int] + var wasJoin = false + var next = 1 + scanStmt(st, map, wasJoin, next) + var bbs: seq[dst.bb] + var bb = BBlock(isExcept: false, params: p) + blocks(st, bbs, bb, map) + build ProcDef(t, locs, List(...bbs)) + +proc globalsToPointer(ir: L6, ptrsize: int): L5 {.pass.} = + ## Turns global locations into pointer globals. Read access is turned + ## into loads, write access into stores, and taking the address into copies + ## of the pointer global. + let (types, globals) = (; + match ir: + of Module(TypeDefs(...t), GlobalDefs(...d), ...any): + var list = newSeq[src.t](d.len) + for i, it in d.pairs: + match it: + of GlobalLoc(t, _): list[i] = t + of GlobalDef(t, _): list[i] = t + else: unreachable() + (t, list)) + + proc gtype(g: Global): dst.t = + globals[ord g] -> dst.t + + proc sizeAndAlignment(x: L6.t): tuple[size, align: Value[int64]] = + match x: + of tid: + sizeAndAlignment(types[ord tid.val]) + of Union(i1, i2, ...any): (i1, i2) + of Record(i1, i2, ...any): (i1, i2) + of Array(i1, i2, ...any): (i1, i2) + of Int(i): (i, i) + of UInt(i): (i, i) + of Float(i): (i, i) + of Ptr(): (terminal int64(ptrsize), terminal int64(ptrsize)) + else: + unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(g): + build Load(^gtype(g.val), Copy(g)) + of Addr(g): + # the address of the location is now stored in the global + build Copy(g) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of g: + build Deref(^gtype(g.val), Copy(g)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(g, [e]): + build Store(^gtype(g.val), Copy(g), e) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of GlobalLoc(t, Data(_)): + let (size, align) = sizeAndAlignment(t) + build GlobalDef(Ptr(), Data(align, size)) + of GlobalLoc(t, Data(_, str)): + let (_, align) = sizeAndAlignment(t) + build GlobalDef(Ptr(), Data(align, str)) + +proc flattenPaths(ir: L5): L4 {.pass.} = + ## Turns `At` and `Field` expressions into flat `Path` expressions, for easier + ## processing by later passes. + var locals: slice(src.t) + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + + proc arrayElem(x: src.t): src.t = + match x: + of tid: + arrayElem(types[ord tid.val]) + of Array(_, _, _, t): + t + else: + unreachable() + + proc elemAt(x: src.t, i: int64): src.t = + match x: + of tid: + elemAt(types[ord tid.val], i) + of Record(_, _, ...f): + match f[i]: + of Field(_, t): t + of Union(_, _, ...t): + t[i] + else: + unreachable() + + proc filter(x: src.lv, args: var seq[dst.e]): (dst.ro, src.t) = + match x: + of Deref(t, [e]): + (build(dst.ro, x.info, Deref(^(t -> dst.t), e)), t) + of Field(lv, i): + let (root, t) = filter(lv, args) + args.add build(dst.e, x.info, i) + (root, elemAt(t, i.val)) + of At(lv, [e]): + let (root, t) = filter(lv, args) + args.add e + (root, arrayElem(t)) + of lo: + (build(dst.ro, x.info, lo), locals[ord lo.val]) + + proc typ(x: src.t): dst.t {.generated.} + proc bblock(x: src.bb): dst.bb {.generated.} + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of lo: build lo + of _: + var args: seq[dst.e] + let (root, t) = filter(x, args) + assert args.len > 0 + build Path(^typ(t), root, ...args) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc aggregateParams(ir: L4): L3s2 {.pass.} = + ## Turns all aggregate parameters into pointer parameters and replaces returns + ## of aggregate with out parameters. + + # The pass' implementation is fairly involved. Only lvalue expressions can have + # their address taken, meaning that rvalue argument expressions of aggregate + # type have to be assigned to a temporary first, which then has its address + # passed to the procedure. + # + # This leads to problem: an expression can have side effects, and reordering + # it with other (impure) expressions might alter the program's meaning! + # + # Therefore, all (impure) expressions happening before an expression that is + # turned into an assignment also have to be committed to temporaries. To + # implement this, the operands of all operations are iterated from + # *right to left*. + proc resolve(types: slice(src.t), t: src.t): src.t = + match t: + of tid: types[ord tid.val] + else: t + + let (types, signatures) = (; + match ir: + of Module(TypeDefs(...t), _, ProcDefs(...d), ...any): + var sigs = newSeq[src.t](d.len) + for i, it in d.pairs: + # the types for procedures are looked up, resolved, and cached, which + # greatly speeds up later type lookup + match it: + of ProcDef(t1, ...any): sigs[i] = resolve(t, t1) + of Import(t1, _): sigs[i] = resolve(t, t1) + else: unreachable() + (t, sigs)) + + var origLocals: slice(src.t) + var locals: seq[dst.t] + var params: PackedSet[Local] + var stmts: seq[dst.st] ## accumulator for statements + var needsSave: bool + + proc isAggregate(x: src.t): bool = + match x: + of tid: isAggregate(types[ord tid.val]) + of Record(...any): true + of Array(...any): true + of Union(...any): true + else: false + + proc retType(x: src.t): src.t = + match x: + of tid: retType(types[ord tid.val]) + of ProcTy(t, ...any): t + else: unreachable() + + proc paramAt(x: src.t, pos: int): src.t = + match x: + of tid: + paramAt(types[ord tid.val], pos) + of ProcTy(_, ...t): + t[pos] + else: unreachable() + + proc newTemp(t: dst.t): dst.lo = + result = terminal(Local(locals.len)) + locals.add(t) + + proc typ(x: src.t): dst.t {.transform.} + proc expr(x: src.e): dst.e {.transform.} + + proc getType(x: src.e): dst.t = + match x: + of Le(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Lt(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Eq(_, _, _): build dst.t, NoSLoc, UInt(i(1)) + of Not(_): build dst.t, NoSLoc, UInt(i(1)) + of Addr(_): build dst.t, NoSLoc, Ptr() + of Nil(): build dst.t, NoSLoc, Ptr() + of Copy(g): discard g; build dst.t, NoSLoc, Ptr() + of Copy(Path([t], ...any)): t + of Copy(lo): locals[ord lo.val] + of Call(pr, ...any): typ(retType(signatures[ord pr.val])) + of Call(t, ...any): typ(retType(t)) + of ProcVal(_): build dst.t, NoSLoc, Ptr() + of i: discard i; unreachable() + of fl: discard fl; unreachable() + of e: + match e: + of any(t, ...any): typ(t) + else: unreachable() + + proc operand(x: src.e): dst.e = + if needsSave: + match x: + of Addr([lv]): build dst.e, x.info, Addr(lv) + of Nil(): build dst.e, x.info, Nil() + of i: build dst.e, x.info, i + of fl: build dst.e, x.info, fl + else: + let tmp = newTemp(getType(x)) + needsSave = false + stmts.add build(dst.st, x.info, Asgn(tmp, ^expr(x))) + needsSave = true + build dst.e, x.info, Copy(tmp) + else: + expr(x) + + proc operands(x: slice(src.e)): seq[dst.e] = + # process in reverse + result.newSeq(x.len) + result[^1] = expr(x[x.high]) + for i in countdown(x.high - 1, 0): + result[i] = operand(x[i]) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of Path([t], ro, ...e): + let e1 = operands(e) + build Path(t, ^root(ro), ...e1) + + proc root(x: src.ro): dst.ro {.transform.} = + case x + of Deref([t], e): + build Deref(t, ^operand(e)) + of lo: + if lo.val in params: + build Deref(^locals[ord lo.val], Copy(lo)) + else: + build lo + + proc args(sig: src.t, e: slice(src.e)): seq[dst.e] = + result = newSeq[dst.e](e.len) + # process in reverse + for i in countdown(e.high, 0): + let it = e[i] + let pt = paramAt(sig, i) + if isAggregate(pt): + let pt = typ(pt) + let tmp = newTemp(pt) + needsSave = false + # ^^ the hoisted expression isn't affected by side effects + match it: + of Call(t, ...e1): + stmts.add build(dst.st, it.info, Asgn(tmp, Call(^typ(t), ...args(t, e1)))) + of Call(pr, ...e1): + stmts.add build(dst.st, it.info, Asgn(tmp, Call(pr, ...args(signatures[ord pr.val], e1)))) + else: + stmts.add build(dst.st, it.info, Asgn(tmp, ^expr(it))) + needsSave = true + result[i] = build(dst.e, it.info, Addr(tmp)) + elif needsSave: + let pt = typ(pt) + let tmp = newTemp(pt) + needsSave = false + # ^^ the hoisted expression isn't affected by side effects + stmts.add build(dst.st, it.info, Asgn(tmp, ^expr(it))) + needsSave = true + result[i] = build(dst.e, it.info, Copy(tmp)) + else: + result[i] = expr(it) + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Addr(lo): + if lo.val in params: + build Copy(lo) + else: + build Addr(lo) + of Call(pr, ...e): + let t = signatures[ord pr.val] + let s = args(t, e) + + let rt = retType(t) + if isAggregate(rt): + let rt = typ(rt) + let tmp = newTemp(rt) + stmts.add build(dst.st, x.info, Call(pr, [...s, Addr(tmp)])) + needsSave = true + build Copy(tmp) + else: + build Call(pr, ...s) + of Call(t, e0, ...e1): + # note: e0 needs to be translated last, to keep the reverse processing order + let s = args(t, e1) + let rt = retType(t) + if isAggregate(t): + let rt = typ(rt) + let tmp = newTemp(rt) + stmts.add build(dst.st, x.info, Call(^typ(t), ^operand(e0), [...s, Addr(tmp)])) + needsSave = true + build Copy(tmp) + else: + build Call(^typ(t), ^operand(e0), ...s) + of Copy(lo): + if lo.val in params: + build Load(^locals[ord lo.val], Copy(lo)) + else: + build Copy(lo) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Call(t, e0, ...e1): + let args = args(t, e1) + build Call(^typ(t), ^operand(e0), ...args) + of Call(pr, ...e): + build Call(pr, ...args(signatures[ord pr.val], e)) + of Blit(e0, e1, e2): + let e2_1 = expr(e2) + let e1_1 = operand(e1) + let e0_1 = operand(e0) + build Blit(e0_1, e1_1, e2_1) + of Clear(e0, e1): + let e1_1 = expr(e1) + build Clear(^operand(e0), e1_1) + + var outParam: Option[Value[Local]] + var delayed: Table[int64, dst.st] + proc exit(x: src.ex): dst.ex {.transform.} = + case x + of CheckedCallAsgn(lo, t, e0, ...e1, Goto(i), [tgt]): + let args = args(t, e1) + let callee = operand(e0) + if isAggregate(retType(t)): + let tmp = newTemp(typ retType(t)) + # the temporary needs to be copied to the actual target upon landing + delayed[i.val] = build(dst.st, x.info, Asgn(lo, Copy(tmp))) + build CheckedCall(^typ(t), callee, [...args, Addr(tmp)], Goto(i), tgt) + else: + build CheckedCallAsgn(lo, ^typ(t), callee, ...args, Goto(i), tgt) + of CheckedCallAsgn(lo, pr, ...e, Goto(i), [tgt]): + let sig = signatures[ord pr.val] + if isAggregate(retType(sig)): + let tmp = newTemp(typ retType(sig)) + # the temporary needs to be copied to the actual target upon landing + delayed[i.val] = build(dst.st, x.info, Asgn(lo, Copy(tmp))) + build CheckedCall(pr, [...args(sig, e), Addr(tmp)], Goto(i), tgt) + else: + build CheckedCallAsgn(lo, pr, ...args(sig, e), Goto(i), tgt) + of CheckedCall(pr, ...e, [go], [tgt]): + build CheckedCall(pr, ...args(signatures[ord pr.val], e), go, tgt) + of CheckedCall(t, e0, ...e1, [go], [tgt]): + let args = args(t, e1) + build CheckedCall(^typ(t), ^operand(e0), ...args, go, tgt) + + proc bblock(x: src.bb, pos: int): dst.bb {.transform.} = + proc aux(lo: slice(src.lo), st0: slice(src.st), ex: src.ex, + isExcept: bool, pos: int): dst.bb = + # TODO: allow using child slices directly in expansion positions + var bparams: seq[dst.lo] + for it in lo.items: + if pos == 0 and isAggregate(origLocals[ord it.val]): + params.incl(it.val) + bparams.add it + + if pos == 0 and outParam.isSome: + bparams.add outParam.unsafeGet + + stmts.shrink(0) + var extra: dst.st + if delayed.pop(pos, extra): + stmts.add extra + + for it in st0.items: + needsSave = false + let start = stmts.len + let st = stmt(it) + # the new statements, if any, were added in reverse; correct the order + reverse(stmts.toOpenArray(start, stmts.high)) + stmts.add st + + needsSave = false + + let start = stmts.len + let exit = + if outParam.isSome: + match ex: + of Return([e0 -> e]): + stmts.insert build(dst.st, e0.info, Store(^getType(e0), Copy(^outParam.unsafeGet), e)), start + build(dst.ex, ex.info, Return()) + else: exit(ex) + else: exit(ex) + reverse(stmts.toOpenArray(start, stmts.high)) + + if isExcept: + build Except(Params(...bparams), ...stmts, exit) + else: + build Block(Params(...bparams), ...stmts, exit) + + case x + of Block(Params(...lo), ...st, ex): + aux(lo, st, ex, false, pos) + of Except(Params(...lo), ...st, ex): + aux(lo, st, ex, true, pos) + + proc typ(x: src.t): dst.t {.transform.} = + case x + of ProcTy(t0, ...t1): + var got = newSeq[dst.t](t1.len) + for i, it in t1.pairs: + got[i] = + if isAggregate(it): build Ptr() + else: typ(it) + if isAggregate(t0): + build ProcTy(Void(), [...got, Ptr()]) + else: + build ProcTy(^typ(t0), ...got) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t0, Locals(...t1), List(...bb)): + params.clear() + delayed.clear() + origLocals = t1 + locals = map(t1, typ) + if isAggregate(retType(t0)): + outParam = some newTemp(build(dst.t, NoSLoc, Ptr())) + else: + outParam = none dst.lo + var blocks = newSeq[dst.bb](bb.len) + for i, it in bb.pairs: + blocks[i] = bblock(it, i) + # turn all aggregate parameter locals into pointers + for it in params.items: + locals[ord it] = build(dst.t, NoSLoc, Ptr()) + build ProcDef(^typ(t0), Locals(...locals), List(...blocks)) + +proc aggregatesToBlob(ir: L3s2, ptrsize: uint): L3s1 {.pass.} = + ## Turns all aggregate types into blob types. Path expression are turned into + ## address value arithmetic. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + var locals: slice(src.t) + + proc size(x: src.t): int64 = + match x: + of tid: size(types[ord tid.val]) + of Record(i, ...any): i.val + of Union(i, ...any): i.val + of Array(i, ...any): i.val + of Int(i): i.val + of UInt(i): i.val + of Float(i): i.val + of Ptr(): int64(ptrsize) + else: unreachable() + + proc elementOffset(x: src.t, elem: int64): int64 = + match x: + of tid: + elementOffset(types[ord tid.val], elem) + of Record(_, _, ...f): + match f[elem]: + of Field(i, _): i.val + of Union(...any): + 0 + of Array(_, _, _, t): + size(t) * elem + else: + unreachable() + + proc typeOfElem(x: src.t, elem: int64): src.t = + match x: + of tid: + typeOfElem(types[ord tid.val], elem) + of Record(_, _, ...f): + match f[elem]: + of Field(_, t): t + of Union(_, _, ...t): + t[elem] + of Array(_, _, _, t): + t + else: + unreachable() + + proc typ(x: src.t): dst.t {.transform.} = + case x + of Record(i0, i1, ...any): build Blob(i0, i1) + of Array(i0, i1, ...any): build Blob(i0, i1) + of Union(i0, i1, ...any): build Blob(i0, i1) + + proc path(root: src.ro, elems: slice(src.e)): dst.e = + var typ: src.t + (typ, result) = (; + match root: + of Deref(t, [e]): (t, e) + of lo: (locals[ord lo.val], build(dst.e, root.info, Addr(lo)))) + + var offset = 0'i64 + for it in elems.items: + match it: + of i: + # a static field or array access + offset += elementOffset(typ, i.val) + typ = typeOfElem(typ, i.val) + else: + # an array access with a dynamic index + if offset > 0: + # add the static offset computed so far: + result = build(dst.e, it.info, Offset(result, i(offset), i(1))) + offset = 0 + + typ = typeOfElem(typ, 0) + # apply the dynamic array element offset: + result = build(dst.e, it.info, Offset(result, ^expr(it), i(^size(typ)))) + + if offset > 0: + result = build(dst.e, result.info, Offset(result, i(offset), i(1))) + + proc lvalue(x: src.lv): dst.lv {.transform.} = + case x + of Path(...any): unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(lv): + match lv: + of Path([t], ro, ...e): + build Load(t, ^path(ro, e)) + else: + build Copy(^lvalue(lv)) + of Addr(lv): + match lv: + of Path(_, ro, ...e): + path(ro, e) # the path becomes an address value + else: + build Addr(^lvalue(lv)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lv, [e]): + match lv: + of Path([t], ro, ...e1): + build Store(t, ^path(ro, e1), e) + else: + build Asgn(^lvalue(lv), e) + + proc bblock(x: src.bb): dst.bb {.generated.} + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc localsToBlob(ir: L3s1, ptrSize: uint): L3 {.pass.} = + ## Turns all non-blob locals that have their address taken into blob locals. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t) + var marker: PackedSet[Local] + var locals: seq[dst.t] + + proc scan(x: src.bb) = + ## Scans the basic-block for address-of operations and registers all + ## locals that have their address taken in marker. + # this is terrible code, for the largest part consisting of just + # traversal boilerplate. While generation of traversal logic would address + # this issue, the code also exemplifies why having an AST with many + # irregular forms is a bad idea + proc scan(x: src.e) = + match x: + of Addr(lo): + # the expression the pass is actually interested in + marker.incl lo.val + # traversal boilerplate follows + of any(t, ...e): # call and binary operator forms + discard t + for it in e.items: + scan(it) + of Call(pr, ...e): + discard pr + for it in e.items: + scan(it) + of any(t0, t1, e): # the conversion forms + discard t0; discard t1; scan(e) + of any(...e): + for it in e.items: + scan(it) + of any(t, e0, e1, lo): # the check forms + discard t; discard lo; scan(e0); scan(e1) + of Copy(_): discard + of ProcVal(_): discard + of Nil(): discard + of i: discard i + of fl: discard fl + + proc scan(x: src.st) = + # boilerplate... + match x: + of Call(_, ...e): + for it in e.items: + scan(it) + of Asgn(_, e): + scan(e) + of Store(_, e0, e1): + scan(e0); scan(e1) + of any(...e): + for it in e.items: + scan(it) + + proc scan(x: src.ex) = + # boilerplate... + match x: + of CheckedCallAsgn(_, _, ...e, _, _): + for it in e.items: + scan(it) + of CheckedCall(_, ...e, _, _): + for it in e.items: + scan(it) + of Raise(e, _): + scan(e) + of Branch(e, _, _): + scan(e) + of Return(e): + scan(e) + of Goto(_): discard + of Loop(_): discard + of any(): discard + + match x: + of Block(_, ...st, ex): + for it in st.items: scan(it) + scan(ex) + of Except(_, ...st, ex): + for it in st.items: scan(it) + scan(ex) + + proc sizeAndAlignment(x: src.t): (Value[int64], Value[int64]) = + match x: + of tid: sizeAndAlignment(types[ord tid.val]) + of Int(i): (i, i) + of UInt(i): (i, i) + of Float(i): (i, i) + of Ptr(): (terminal(int64(ptrSize)), terminal(int64(ptrSize))) + else: unreachable() + + proc isBlob(x: src.t): bool = + match x: + of tid: isBlob(types[ord tid.val]) + of Blob(_, _): true + else: false + + proc typ(x: src.t): dst.t {.transform.} = + case x + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Copy(lo): + if lo.val in marker: + build Load(^locals[ord lo.val], Addr(lo)) + else: + build Copy(lo) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lo, [e]): + if lo.val in marker: + build Store(^locals[ord lo.val], Addr(lo), e) + else: + build Asgn(lo, e) + + proc newTemp(t: dst.t): dst.lo = + result = terminal Local(locals.len) + locals.add t + + proc bblock(x: src.bb): dst.bb {.transform.} = + proc update(x: src.lo, stmts: var seq[dst.st]): dst.lo = + if x.val in marker: + let typ = locals[ord x.val] + let tmp = newTemp(typ) + stmts.add build(dst.st, NoSLoc, Store(typ, Addr(x), Copy(tmp))) + tmp + else: + x + + # if a block parameter (which can only be of primitive type at this + # point) has its address taken, the parameter must be turned back into a + # primitive type. In effect, this means that the real parameter is copied + # to a stack location on block entry that the rest of the procedure + # then uses in place of the real parameter + case x + of Block(Params(...lo), ...[st], [ex]): + var stmts: seq[dst.st] + let params = mapIt(lo, update(it, stmts)) + build Block(Params(...params), [...stmts, ...st], ex) + of Except(Params(...lo), ...[st], [ex]): + var stmts: seq[dst.st] + let params = mapIt(lo, update(it, stmts)) + build Except(Params(...params), [...stmts, ...st], ex) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t0, Locals(...t1), List(...bb)): + marker.clear() + locals = map(t1, typ) + # gather the locals that have their address taken: + for it in bb.items: + scan(it) + + let bbs = map(bb, bblock) + # ^^ potentially modifies the list of locals + + for i, it in t1.pairs: + if Local(i) in marker and not isBlob(it): + let (size, align) = sizeAndAlignment(it) + locals[i] = build(dst.t, NoSLoc, Blob(size, align)) + + build ProcDef(^typ(t0), Locals(...locals), List(...bbs)) + +proc legalizeBlobOps(ir: L3): L2 {.pass.} = + ## Turns loads, stores, and assignments of blob types into blit copies. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t + ) + var locals: slice(src.t) + + proc size(x: src.t): int64 = + match x: + of Blob(i, _): i.val.int64 + of tid: size(types[ord tid.val]) + else: unreachable() + + proc isBlob(x: src.t): bool = + match x: + of Blob(...any): true + of tid: isBlob(types[ord tid.val]) + else: false + + proc expr(x: src.e): dst.e {.generated.} + proc typ(x: src.t): dst.t {.generated.} + proc bblock(x: src.bb): dst.bb {.generated.} + + proc operand(x: src.e): dst.e = + match x: + of Copy([lv0 -> lv]): build(dst.e, x.info, Addr(lv)) + of Load(_, [e]): e + else: unreachable() + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Store(t, [e0], e1): + if isBlob(t): + build Blit(e0, ^operand(e1), i(^size(t))) + else: + build Store(^typ(t), e0, ^expr(e1)) + of Asgn(lo, e): + let t = locals[ord lo.val] + if isBlob(t): + build Blit(Addr(lo), ^operand(e), i(^size(t))) + else: + build Asgn(lo, ^expr(e)) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef([t], Locals(...t1), List(...bb)): + locals = t1 + build ProcDef(t, Locals(...map(t1, typ)), List(...map(bb, bblock))) + +proc stackAlloc(ir: L2): L1 {.pass.} = + ## Simple stack allocation pass. Adds a frame pointer local to all procedures + ## that use stack memory and turns blob locals into stack locations. + let types = (; + match ir: + of Module(TypeDefs(...t), ...any): t + ) + var locals: seq[tuple[onStack: bool, offset: uint32]] + var framePointer: dst.e + + func isBlob(x: src.t): bool = + match x: + of tid: isBlob(types[ord tid.val]) + of Blob(_, _): true + else: false + + func sizeAndAlign(x: src.t): (uint32, uint32) = + match x: + of tid: sizeAndAlign(types[ord tid.val]) + of Blob(i0, i1): (i0.val.uint32, i1.val.uint32) + else: unreachable() + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Addr(lo): + assert locals[ord lo.val].onStack, $ord(lo.val) + if locals[ord lo.val].offset == 0: + # no addition is necessary + framePointer + else: + # XXX: it could make sense to try merging the offset computation into + # an enclosing one, where possible + build Offset(framePointer, i(^locals[ord lo.val].offset), i(1)) + of Copy(lo): + assert not locals[ord lo.val].onStack + build Copy(lo(^locals[ord lo.val].offset)) + + proc stmt(x: src.st): dst.st {.transform.} = + case x + of Asgn(lo, [e]): + assert not locals[ord lo.val].onStack + build Asgn(lo(^locals[ord lo.val].offset), e) + + proc exit(x: src.ex): dst.ex {.transform.} = + case x + of CheckedCallAsgn(lo, [t0 -> t], ...[e], [go], [tgt]): + # XXX: very inefficient + build CheckedCallAsgn(lo(^locals[ord lo.val].offset), t, ...e, go, tgt) + of CheckedCallAsgn(lo, pr, ...[e], [go], [tgt]): + build CheckedCallAsgn(lo(^locals[ord lo.val].offset), pr, ...e, go, tgt) + + proc bblock(x: src.bb): dst.bb {.generated.} + + proc typ(x: src.t): dst.t {.transform.} = + case x + of Blob(_, _): build Ptr() + + proc mapLocal(x: src.lo): dst.lo = + assert not locals[ord x.val].onStack + terminal Local(locals[ord x.val].offset) + + proc params(x: src.p): dst.p {.transform.} = + case x + of Params(...lo): + var lo1 = newSeq[dst.lo](lo.len) + for i, it in lo.pairs: + lo1[i] = mapLocal(it) + build Params(...lo1) + + proc decl(x: src.d): dst.d {.transform.} = + case x + of ProcDef(t, Locals(...t1), List(...bb)): + locals.setLen(t1.len) + var + nextId = 0'u32 + stackOffset = 0'u32 + + # assign a stack location to every blob local. At the moment, blob locals + # are put into consecutive stack location, regardless of whether they have + # disjoint lifetimes or not + for i, it in t1.pairs: + if isBlob(it): + let (size, align) = sizeAndAlign(it) + doAssert align <= 8, "over-aligned locals are currently not supported" + # align the stack offset: + stackOffset = (stackOffset + (align - 1)) and not (align - 1) + locals[i] = (true, stackOffset) + stackOffset += size # reserve the needed space + else: + locals[i] = (false, nextId) + inc nextId + + # make the frame size a multiple of 8, so that the start of stack frame is + # always on an 8 byte boundary + stackOffset = (stackOffset + 7) and not 7'u32 + + if stackOffset == 0: + # the body stays as is, only the header needs to be modified + build ProcDef(^typ(t), i(0), Locals(...map(t1, typ)), List(...map(bb, bblock))) + else: + framePointer = build(dst.e, x.info, Copy(lo(^Local(nextId)))) + var filtered = newSeq[dst.t](nextId + 1) + for i, it in locals.pairs: + if not it.onStack: + filtered[it.offset] = typ(t1[i]) + + filtered[^1] = build(dst.t, NoSLoc, Ptr()) + + var blocks = newSeq[dst.bb](bb.len) + for i, it in bb.pairs: + if i == 0: + # pass the frame pointer as an extra argument + match it: + of Block(Params(...lo), ...[st], [ex]): + blocks[i] = build(dst.bb, it.info, Block(Params([...map(lo, mapLocal), lo(nextid)]), ...st, ex)) + else: + unreachable() + else: + blocks[i] = bblock(it) + + build ProcDef(^typ(t), i(stackOffset), Locals(...filtered), List(...blocks)) + +proc inlineTypes(ir: L1): LPtr {.pass.} = + ## Removes `Blob` types and turns all identified numeric types into inline + ## types. + var types: Table[int, dst.t] + + proc typ(x: src.t): dst.t {.transform.} = + case x + of tid: + var r: dst.t + types.withValue ord(tid.val), val: + r = val[] + do: + r = build tid + r + + proc typedefs(x: src.td): dst.td {.transform.} = + case x + of TypeDefs(...t): + var pos = 0 + # build the lookup table for types to inline: + for i, it in t.pairs: + match it: + of ProcTy(...any): + if pos != i: + types[i] = build(dst.t, NoSLoc, tid(pos)) # needs a fixup + inc pos + else: + types[i] = typ(it) + + # inline the types and remove them from the list of typedefs: + var outtypes = newSeq[dst.t](pos) + pos = 0 + for it in t.items: + match it: + of ProcTy(...any): + outtypes[pos] = typ(it) + inc pos + else: + discard "drop the type" + + build TypeDefs(...outtypes) + +proc ptrToInt(ir: LPtr, ptrsize: Positive): L0 {.pass.} = + ## Turns pointer types into unsigned integers. + proc typ(x: src.t): dst.t {.transform.} = + case x + of Ptr(): build UInt(i(ptrsize)) + + proc expr(x: src.e): dst.e {.transform.} = + case x + of Nil(): build i(0) # represented as zero + of Offset([e0], [e1], e2): + match e2: + of i: + if i.val == 1: + return build(Add(UInt(i(ptrsize)), e0, e1)) + else: + discard "nothing to do" + + # the index is scaled + build Add(UInt(i(ptrsize)), e0, Mul(UInt(i(ptrsize)), e1, ^expr(e2))) + of Reinterp(Ptr(), UInt(i), [e0]): + if i.val == 8: e0 # drop the bitcast + else: build Reinterp(UInt(i(ptrsize)), UInt(i), e0) + of Reinterp(UInt(i), Ptr(), [e0]): + if i.val == 8: e0 # drop the bitcast + else: build Reinterp(UInt(i), UInt(i(ptrsize)), e0) + +# ----- compiler implementation ------ + +import experimental/[sexp, sexp_parse] +import passes/compilerdef +import std/[os, streams, strutils] + +proc tryParse[T: Global or Proc or Local or Type](n: SexpNode, _: typedesc[T]): Option[T] = + if n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == $T and n[1].kind == SInt: + some(T(n[1].num)) + else: + none(T) + +proc tryParse(n: SexpNode, _: typedesc[string]): Option[string] = + if n.kind == SString: + some(n.str) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "StringVal" and n[1].kind == SString: + some(n[1].str) + else: + none(string) + +proc tryParse(n: SexpNode, _: typedesc[int64]): Option[int64] = + if n.kind == SInt: + some(int64(n.num)) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "IntVal" and n[1].kind == SInt: + some(int64(n[1].num)) + else: + none(int64) + +proc tryParse(n: SexpNode, _: typedesc[float]): Option[float] = + if n.kind == SFloat: + some(n.fnum) + elif n.kind == SList and n.len == 2 and n[0].kind == SSymbol and n[0].symbol == "FloatVal": + if n[1].kind == SFloat: + some(n[1].fnum) + elif n[1].kind == SSymbol: + some(parseFloat(n[1].symbol)) + else: + none(float) + elif n.kind == SSymbol: + some(parseFloat(n.symbol)) + else: + none(float) + +proc toSexp[T: Local or Global or Proc or Type](x: T): SexpNode = + newSList([newSSymbol($T), newSInt(ord x)]) + +proc toSexp(x: int64): SexpNode = + newSInt(int x) +proc toSexp(x: float): SexpNode = + newSFloat(x) +proc toSexp(x: string): SexpNode = + newSString(x) + +let f = openFileStream(getExecArgs()[0], fmRead) +var p: SexpParser +p.open(f) +discard p.getTok() + +let (ast, m) = parseAst[Literals](p, LSkully.m) +f.close() +echo "parsed" + +defineCompiler compile, LSkully, [ + basicBlocks, + globalsToPointer(8), + flattenPaths, + aggregateParams, + aggregatesToBlob(8), + localsToBlob(8), + legalizeBlobOps, + stackAlloc, + inlineTypes, + ptrToInt(8) +] + +discard compile(ast, m)