Replace hardcoded pass sequence with a scheduler

Passes are now tagged with information about what they do, and are
automatically placed into the correct order. The scheduling algorithm
is designed such that the same input will always produce the same
output, so that the order that passes tests is the same order used in
production. This can be disabled for testing purposes.

pyproject.toml

@@ -29,6 +29,7 @@ regex = ">=2018.08.29"
 [tool.poetry.dev-dependencies]
 pytest = ">=4.2"
 pytest-cov = ">=2.6.1"
+pytest-repeat = "^0.7.0"
 flake8 = "^3.7.4"
 coverage = "^4.5.2"
 hypothesis = "^4.5.0"

src/jeff65/__init__.py

@@ -44,15 +44,13 @@ def install(cls):
 BraceMessage.install()
 
 
+debugopts = {"log_debug": False, "unstable_pass_schedule": False}
+
+
 def main(argv=None):
     parser = argparse.ArgumentParser()
-
     parser.add_argument(
-        "--debug",
-        help="run in debug mode",
-        dest="debug",
-        action="store_true",
-        default=False,
+        "-Z", help="enable debug settings", dest="debugopt", action="append"
     )
     parser.add_argument(
         "-v",
@@ -80,7 +78,18 @@ def main(argv=None):
     objdump_parser.set_defaults(func=cmd_objdump)
 
     args = parser.parse_args(argv)
-    if args.debug:
+    if args.debugopt is not None:
+        unknown_opts = set(args.debugopt) - debugopts.keys()
+        if len(unknown_opts) > 0:
+            optlist = ", ".join(unknown_opts)
+            print(f"Unknown debugging options {optlist}", file=sys.stderr)
+            print(f"Allowed options are:", file=sys.stderr)
+            for opt in sorted(debugopts.keys()):
+                print(f"    {opt}", file=sys.stderr)
+            sys.exit(1)
+        debugopts.update({opt: True for opt in args.debugopt})
+
+    if debugopts["log_debug"]:
         logging.basicConfig(level=logging.DEBUG)
     elif args.verbose:
         logging.basicConfig(level=logging.INFO)

src/jeff65/ast.py

@@ -119,6 +119,21 @@ def select(self, *attrs):
         return current
 
 
+class NullPass:
+    """Base class for null passes.
+
+    Inserting null passes is helpful when writing annotations for the
+    scheduler; in particular, it allows a set of tags to be reduced to a single
+    tag.
+    """
+
+    def transform_enter(self, t, node):
+        return node
+
+    def transform_exit(self, t, node):
+        return node
+
+
 class TranslationPass:
     """Base class for translation passes."""
 

src/jeff65/gold/compiler.py

@@ -17,7 +17,7 @@
 import logging
 import sys
 from . import grammar
-from .. import ast, blum, parsing
+from .. import ast, blum, parsing, scheduler
 from .passes import asm, binding, lower, resolve, simplify, typepasses
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ def parse(fileobj, name):
 def translate(unit):
     # parse will close the file for us
     obj = parse(open_unit(unit), name=unit.name)
-    for p in passes:
+    for p in scheduler.create_schedule(passes):
         obj = obj.transform(p())
         logger.debug(__("Pass {}:\n{:p}", p.__name__, obj))
 

src/jeff65/gold/passes/asm.py

@@ -30,6 +30,10 @@ def asmrun(fmt, *args):
 
 @pattern.transform(pattern.Order.Any)
 class AssembleWithRelocations:
+    introduces = {"asmrun"}
+    uses = set()
+    deletes = {"lda", "jmp", "rts", "sta"}
+
     @pattern.match(
         ast.AstNode(
             "lda",
@@ -74,6 +78,10 @@ def rts(self):
 
 @pattern.transform(pattern.Order.Ascending)
 class FlattenSymbol:
+    introduces = {"fun_symbol"}
+    uses = set()
+    deletes = {"fun", "asmrun"}
+
     @pattern.match(
         ast.AstNode(
             "block",

src/jeff65/gold/passes/binding.py

@@ -87,6 +87,10 @@ class ShadowNames(ScopedPass):
     constructing types.
     """
 
+    introduces = {"binding:shadows"}
+    uses = set()
+    deletes = set()
+
     def exit_constant(self, node):
         self.bind_name(node.attrs["name"], True)
         return node
@@ -95,12 +99,20 @@ def exit_constant(self, node):
 class BindNamesToTypes(ScopedPass):
     """Binds names to types. These are later overridden by the storage."""
 
+    introduces = {"binding:types"}
+    uses = {"constant", "binding:typenames"}
+    deletes = set()
+
     def exit_constant(self, node):
         self.bind_name(node.attrs["name"], node.attrs["type"])
         return node
 
 
 class EvaluateConstants(ScopedPass):
+    introduces = {"binding:constants"}
+    uses = {"resolved:functions"}
+    deletes = {"constant"}
+
     def __init__(self):
         super().__init__()
         self.evaluating = False
@@ -125,6 +137,10 @@ def exit_call(self, node):
 
 
 class ResolveConstants(ScopedPass):
+    introduces = {"resolved:constants"}
+    uses = {"binding:constants", "binding:types"}
+    deletes = set()
+
     def exit_identifier(self, node):
         value = self.look_up_constant(node.attrs["name"])
         if not value:

src/jeff65/gold/passes/lower.py

@@ -21,6 +21,10 @@
 
 @pattern.transform(pattern.Order.Any)
 class LowerAssignment:
+    introduces = {"lda"}
+    uses = {"binding:types", "resolved:storage"}
+    deletes = {"set"}
+
     @pattern.match(
         ast.AstNode(
             "block",
@@ -47,6 +51,10 @@ def lower_set(self, span, ty, lvalue, rvalue, nxt):
 
 
 class LowerFunctions(ast.TranslationPass):
+    introduces = {"rts"}
+    uses = {"fun"}
+    deletes = set()
+
     def exit_fun(self, node):
         children = node.select("body", "stmt")
         children.append(asm.rts(node.span))

src/jeff65/gold/passes/resolve.py

@@ -22,6 +22,10 @@
 
 @pattern.transform(pattern.Order.Descending)
 class ResolveStorage:
+    introduces = {"resolved:storage", "absolute_storage", "immediate_storage"}
+    uses = {"resolved:constants", "binding:types"}
+    deletes = {"deref", "numeric"}
+
     @pattern.match(
         ast.AstNode(
             "deref",
@@ -48,6 +52,10 @@ def numeric_to_immediate(self, value):
 class ResolveUnits(binding.ScopedPass):
     """Resolves external units identified in 'use' statements."""
 
+    introduces = {"binding:unit"}
+    uses = set()
+    deletes = {"use"}
+
     builtin_units = {"mem": mem.MemUnit()}
 
     def exit_use(self, node):
@@ -65,6 +73,10 @@ def exit_toplevel(self, node):
 class ResolveMembers(binding.ScopedPass):
     """Resolves members to functions."""
 
+    introduces = {"resolved:functions"}
+    uses = {"binding:unit"}
+    deletes = {"member_access"}
+
     def exit_member_access(self, node):
         member = node.attrs["member"]
         name = node.attrs["namespace"].attrs["name"]

src/jeff65/gold/passes/typepasses.py

@@ -20,6 +20,10 @@
 
 
 class ConstructTypes(ast.TranslationPass):
+    introduces = {"binding:typenames"}
+    uses = set()
+    deletes = {"ref"}
+
     builtin_types = {
         "u8": types.u8,
         "u16": types.u16,
@@ -42,6 +46,10 @@ def enter_type_ref(self, node):
 
 
 class PropagateTypes(binding.ScopedPass):
+    introduces = {"binding:types"}
+    uses = {"binding:constants", "binding:typenames"}
+    deletes = set()
+
     def enter_identifier(self, node):
         t = self.look_up_name(node.attrs["name"])
         return node.update_attrs({"type": t})

src/jeff65/graph.py

@@ -0,0 +1,102 @@
+# jeff65 graph manipulation
+# Copyright (C) 2019  jeff65 maintainers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import attr
+import random
+
+
+class TopologyError(Exception):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+@attr.s(slots=True, cmp=False)
+class Node:
+    # N.B. we use a list of links to guarantee stable order; a set would result
+    # in different output each run.
+    value = attr.ib()
+    links = attr.ib(factory=list, converter=list)
+
+
+class Graph:
+    def __init__(self, nodes=None, stable=True):
+        # Keep track of order; by traversing nodes in a consistent order, we
+        # can ensure consistent output (if we need to).
+
+        self._nodelist = [] if nodes is None else list(nodes)
+        self._nodes = set(self._nodelist)
+        self._stable = stable
+        self._sorted = None
+
+    @property
+    def nodes(self):
+        return set(self._nodes)
+
+    def __iter__(self):
+        return iter(self._nodelist)
+
+    def add_node(self, node):
+        if node not in self.nodes:
+            self._nodelist.append(node)
+            self._nodes.add(node)
+        for n in node.links:
+            self.add_node(n)
+
+    def add_edge(self, start, end):
+        if start not in self._nodes:
+            raise TopologyError("start node not in graph")
+        if end not in self._nodes:
+            raise TopologyError("end node not in graph")
+        start.links.append(end)
+        self._sorted = None
+
+    def _sort(self):
+        # DFS topological sort per Tarjan (1976). All nodes start out in the
+        # white set. We perform a recursive DFS, marking nodes grey on the way
+        # down and black on the way up, emitting them as we mark them black.
+
+        # If we end up having to work with graphs so big that we end up blowing
+        # out the stack, consider switching to the Kahn (1962) algorithm, which
+        # is non-recursive, but more irritating to implement as it requires
+        # making a copy of the graph to modify destructively.
+
+        white = list(self._nodelist)
+        if not self._stable:
+            random.shuffle(white)
+
+        black = set()
+        grey = set()
+
+        def darken(n):
+            if n in black:
+                return
+            if n in grey:
+                raise TopologyError("Not a DAG")
+
+            grey.add(n)
+            for m in n.links:
+                yield from darken(m)
+            grey.remove(n)
+            black.add(n)
+            yield n
+
+        while len(white) > 0:
+            yield from darken(white.pop())
+
+    def sorted(self):
+        if self._sorted is None:
+            self._sorted = list(self._sort())
+        return self._sorted

src/jeff65/pattern.py

@@ -99,7 +99,11 @@ def dummy_handler(self, t, node):
 def transform_handler(self, t, node):
     for predicate, template in self.ptpairs:
         captures = {}
-        if predicate._match(node, captures):
+        try:
+            m = predicate._match(node, captures)
+        except Exception as e:
+            raise MatchError(f"Error in matcher decorating {template}") from e
+        if m:
             f = template.__get__(self, type)
             n = f(**captures)
             if isinstance(n, ast.AstNode) and n.span is None: