From d739e005e5f54db7c1679e02be814a9bf6daad20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 10:53:49 +0200 Subject: [PATCH 001/123] Add check for normal form --- .../effekt/core/optimizer/Normalizer.scala | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala index 79a0251d3..8c4cb7a2f 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala @@ -2,7 +2,7 @@ package effekt package core package optimizer -import effekt.util.messages.INTERNAL_ERROR +import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } import scala.annotation.tailrec import scala.collection.mutable @@ -30,6 +30,24 @@ import scala.collection.mutable */ object Normalizer { normal => + def assertNormal(t: Tree)(using E: ErrorReporter): Unit = Tree.visit(t) { + // The only allowed forms are the following. + // In the future, Stmt.Shift should also be performed statically. + case Stmt.Val(_, _, binding: (Stmt.Reset | Stmt.Var | Stmt.App | Stmt.Invoke | Stmt.Region | Stmt.Shift | Stmt.Resume), body) => + assertNormal(binding); assertNormal(body) + case t @ Stmt.Val(_, _, binding, body) => + E.warning(s"Not allowed as binding of Val: ${util.show(t)}") + case t @ Stmt.App(b: BlockLit, targs, vargs, bargs) => + E.warning(s"Unreduced beta-redex: ${util.show(t)}") + case t @ Stmt.Invoke(b: New, method, tpe, targs, vargs, bargs) => + E.warning(s"Unreduced beta-redex: ${util.show(t)}") + case t @ Stmt.If(cond: Literal, thn, els) => + E.warning(s"Unreduced if: ${util.show(t)}") + case t @ Stmt.Match(sc: Make, clauses, default) => + E.warning(s"Unreduced match: ${util.show(t)}") + } + + case class Context( blocks: Map[Id, Block], exprs: Map[Id, Expr], From c91f531e189023fdcd524d12ee10ddfeb9dae0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 10:54:21 +0200 Subject: [PATCH 002/123] WIP add first draft of new normalizer --- .../effekt/core/optimizer/NewNormalizer.scala | 615 ++++++++++++++++++ .../effekt/core/optimizer/Normalizer.scala | 3 +- .../effekt/core/optimizer/Optimizer.scala | 43 +- 3 files changed, 640 insertions(+), 21 deletions(-) create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala new file mode 100644 index 000000000..c6afc7baf --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -0,0 +1,615 @@ +package effekt +package core +package optimizer + +import effekt.source.Span +import effekt.core.optimizer.semantics.NeutralStmt +import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } +import kiama.output.ParenPrettyPrinter + +import scala.annotation.tailrec +import scala.collection.mutable +import scala.collection.immutable.ListMap + +// TODO +// while linearity is difficult to track bottom up, variable usage is possible +// this way deadcode can be eliminated on the way up. +// +// plan: don't inline... this is a separate pass after normalization +object semantics { + + // Values + // ------ + + type Addr = Id + type Label = Id + + enum Value { + // Stuck + //case Var(id: Id, annotatedType: ValueType) + case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) + + // Actual Values + case Literal(value: Any, annotatedType: ValueType) + case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) + + val free: Set[Addr] = this match { + // case Value.Var(id, annotatedType) => Set.empty + case Value.Extern(id, targs, vargs) => vargs.toSet + case Value.Literal(value, annotatedType) => Set.empty + case Value.Make(data, tag, targs, vargs) => vargs.toSet + } + } + + // TODO find better name for this + enum Binding { + case Let(value: Value) + case Def(block: Block) + case Val(stmt: NeutralStmt) + case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) + + val free: Set[Addr] = this match { + case Binding.Let(value) => value.free + case Binding.Def(block) => block.free + case Binding.Val(stmt) => stmt.free + case Binding.Run(f, targs, vargs) => vargs.toSet + } + } + + type Bindings = List[(Id, Binding)] + object Bindings { + def empty: Bindings = Nil + } + extension (bindings: Bindings) { + def free: Set[Id] = { + val bound = bindings.map(_._1).toSet + val free = bindings.flatMap { b => b._2.free }.toSet + free -- bound + } + } + + /** + * A Scope is a bit like a basic block, but without the terminator + */ + class Scope( + var bindings: ListMap[Id, Binding], + var inverse: Map[Value, Id], + outer: Option[Scope] + ) { + // floating values to the top is not always beneficial. For example + // def foo() = COMPUTATION + // vs + // let x = COMPUTATION + // def foo() = x + def getDefinition(value: Value): Option[Addr] = + inverse.get(value) orElse outer.flatMap(_.getDefinition(value)) + + def allocate(hint: String, value: Value): Addr = + getDefinition(value) match { + case Some(value) => value + case None => + val addr = Id(hint) + bindings = bindings.updated(addr, Binding.Let(value)) + inverse = inverse.updated(value, addr) + addr + } + + def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr]): Addr = { + val addr = Id(hint) + bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs)) + addr + } + + // TODO Option[Value] or Var(id) in Value? + def lookupValue(addr: Addr): Option[Value] = bindings.get(addr) match { + case Some(Binding.Let(value)) => Some(value) + case _ => outer.flatMap(_.lookupValue(addr)) + } + + def define(hint: String, block: Block): Label = + val label = Id(hint) + bindings = bindings.updated(label, Binding.Def(block)) + label + + def push(id: Id, stmt: NeutralStmt): Unit = + bindings = bindings.updated(id, Binding.Val(stmt)) + } + object Scope { + def empty: Scope = new Scope(ListMap.empty, Map.empty, None) + } + + case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { + def free: Set[Addr] = body.free -- vparams.map(_.id) + } + + case class BasicBlock(bindings: Bindings, body: NeutralStmt) { + def free: Set[Addr] = bindings.free ++ body.free + } + + enum Computation { + // Unknown + case Var(id: Id) + // Known function + case Def(label: Label) + // Known object + case New(interface: BlockType.Interface, operations: List[(Id, Label)]) + } + + // Statements + // ---------- + enum NeutralStmt { + // continuation is unknown + case Return(result: Addr) + // callee is unknown or we do not want to inline (TODO no block arguments for now) + case App(label: Label, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + // callee is unknown + case Invoke(id: Id, method: Id, methodTpe: BlockType, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + // cond is unknown + case If(cond: Addr, thn: BasicBlock, els: BasicBlock) + // scrutinee is unknown + case Match(scrutinee: Addr, clauses: List[(Id, Block)], default: Option[BasicBlock]) + + // aborts at runtime + case Hole(span: Span) + + val free: Set[Addr] = this match { + case NeutralStmt.App(label, targs, vargs, bargs) => vargs.toSet + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => vargs.toSet + case NeutralStmt.If(cond, thn, els) => Set(cond) ++ thn.free ++ els.free + case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) + case NeutralStmt.Return(result) => Set(result) + case NeutralStmt.Hole(span) => Set.empty + } + } + + object PrettyPrinter extends ParenPrettyPrinter { + + override val defaultIndent = 2 + + def toDoc(s: NeutralStmt): Doc = s match { + case NeutralStmt.Return(result) => + "return" <+> toDoc(result) + case NeutralStmt.If(cond, thn, els) => + "if" <+> parens(toDoc(cond)) <+> toDoc(thn) <+> "else" <+> toDoc(els) + case NeutralStmt.Match(scrutinee, clauses, default) => + "match" <+> parens(toDoc(scrutinee)) // <+> braces(hcat(clauses.map {})) + case NeutralStmt.App(label, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + toDoc(label) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + toDoc(label) <> "." <> toDoc(method) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + + case NeutralStmt.Hole(span) => "hole()" + } + + def toDoc(id: Id): Doc = id.show + + def toDoc(value: Value): Doc = value match { + // case Value.Var(id, tpe) => toDoc(id) + + case Value.Extern(callee, targs, vargs) => + toDoc(callee.id) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) + + case Value.Literal(value, _) => util.show(value) + + case Value.Make(data, tag, targs, vargs) => + "make" <+> toDoc(data) <+> toDoc(tag) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) + } + + def toDoc(block: Block): Doc = block match { + case Block(tparams, vparams, bparams, body) => + (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> + parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) + } + + def toDoc(comp: Computation): Doc = comp match { + case Computation.Var(id) => toDoc(id) + case Computation.Def(label) => toDoc(label) + case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { + hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") + } + } + + def toDoc(bindings: Bindings): Doc = + hcat(bindings.map { + case (addr, Binding.Let(value)) => "let" <+> toDoc(addr) <+> "=" <+> toDoc(value) <> line + case (addr, Binding.Def(block)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line + case (addr, Binding.Val(stmt)) => "val" <+> toDoc(addr) <+> "=" <+> toDoc(stmt) <> line + case (addr, Binding.Run(callee, tparams, vparams)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> + (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> + parens(hsep(vparams.map(toDoc), comma)) <> line + }) + + def toDoc(block: BasicBlock): Doc = + braces(nest(line <> toDoc(block.bindings) <> toDoc(block.body)) <> line) + + def toDoc(p: ValueParam): Doc = toDoc(p.id) <> ":" <+> toDoc(p.tpe) + def toDoc(p: BlockParam): Doc = braces(toDoc(p.id)) + + def toDoc(t: ValueType): Doc = util.show(t) + def toDoc(t: BlockType): Doc = util.show(t) + + def show(stmt: NeutralStmt): String = pretty(toDoc(stmt), 80).layout + def show(value: Value): String = pretty(toDoc(value), 80).layout + def show(block: Block): String = pretty(toDoc(block), 80).layout + def show(bindings: Bindings): String = pretty(toDoc(bindings), 80).layout + } + +} + +/** + * A new normalizer that is conservative (avoids code bloat) + */ +object NewNormalizer { normal => + + import semantics.* + + // "effects" + case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { + def lookupValue(id: Id): Addr = values(id) + def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) + def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) + + def lookupComputation(id: Id): Computation = computations(id) + def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) + def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) + } + object Env { + def empty: Env = Env(Map.empty, Map.empty) + } + + def reify(scope: Scope, body: NeutralStmt): BasicBlock = { + var used = body.free + var filtered = Bindings.empty + // TODO implement properly + scope.bindings.toSeq.reverse.foreach { + // TODO for now we keep ALL definitions + case (addr, b: Binding.Def) => + used = used ++ b.free + filtered = (addr, b) :: filtered + case (addr, s: Binding.Val) => + used = used ++ s.free + filtered = (addr, s) :: filtered + case (addr, v: Binding.Run) => + used = used ++ v.free + filtered = (addr, v) :: filtered + + // TODO if type is unit like, we can potentially drop this binding (but then we need to make up a "fresh" unit at use site) + case (addr, v: Binding.Let) if used.contains(addr) => + used = used ++ v.free + filtered = (addr, v) :: filtered + case (addr, v: Binding.Let) => () + } + + // we want to avoid turning tailcalls into non tail calls like + // + // val x = app(x) + // return x + // + // so we eta-reduce here. Can we achieve this by construction? + // TODO lastOption will go through the list AGAIN, let's see whether this causes performance problems + (filtered.lastOption, body) match { + case (Some((id1, Binding.Val(stmt))), NeutralStmt.Return(id2)) if id1 == id2 => + BasicBlock(filtered.init, stmt) + case (_, _) => + BasicBlock(filtered, body) + } + } + + def nested(prog: Scope ?=> NeutralStmt)(using scope: Scope): BasicBlock = { + // TODO parent code and parent store + val local = Scope(ListMap.empty, Map.empty, Some(scope)) + val result = prog(using local) + reify(local, result) + } + + // "handlers" + def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(id, addr)) + + def bind[R](id: Id, computation: Computation)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindComputation(id, computation)) + + def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(values)) + + // Stacks + // ------ + enum Stack { + case Empty + case Static(tpe: ValueType, apply: Env => Scope => Addr => NeutralStmt) + case Dynamic(label: Label) + } + + def returnTo(stack: Stack, arg: Addr)(using env: Env, scope: Scope): NeutralStmt = stack match { + case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Static(tpe, apply) => apply(env)(scope)(arg) + case Stack.Dynamic(label) => NeutralStmt.App(label, List.empty, List(arg), Nil) + } + + def reify(stack: Stack, stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = stack match { + case Stack.Empty => stmt + // [[ val x = { val y = stmt1; stmt2 }; stmt3 ]] = [[ val y = stmt1; val x = stmt2; stmt3 ]] + case Stack.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(env)(scope)(tmp) + // stack is already reified + case Stack.Dynamic(label) => + stmt + } + + def join(stack: Stack)(f: Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = stack match { + case Stack.Static(tpe, apply) => + val x = Id("x") + nested { scope ?=> apply(env)(scope)(x) } match { + // Avoid trivial continuations like + // def k_6268 = (x_6267: Int_3) { + // return x_6267 + // } + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App)) => f(stack) + case body => + val k = scope.define("k", Block(Nil, ValueParam(x, tpe) :: Nil, Nil, body)) + f(Stack.Dynamic(k)) + } + case Stack.Empty => f(stack) + case Stack.Dynamic(label) => f(stack) + } + + def push(tpe: ValueType)(f: Env ?=> Scope ?=> Addr => NeutralStmt): Stack = Stack.Static(tpe, + env => scope => arg => f(using env)(using scope)(arg) + ) + + def evaluate(block: core.Block)(using env: Env, scope: Scope): Computation = block match { + case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => + env.lookupComputation(id) + case b @ core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + Computation.Def(scope.define("f", evaluate(b))) + case core.Block.Unbox(pure) => + ??? + case core.Block.New(Implementation(interface, operations)) => + val ops = operations.map { + case Operation(name, tparams, cparams, vparams, bparams, body) => + val label = scope.define(name.name.name, evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body): core.Block.BlockLit)) + (name, label) + } + Computation.New(interface, ops) + } + + def evaluate(block: core.Block.BlockLit)(using env: Env, scope: Scope): Block = + block match { + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + // we keep the params as they are for now... + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + Block(tparams, vparams, bparams, nested { + evaluate(body, Stack.Empty) + }) + } + + def evaluate(expr: Expr)(using env: Env, scope: Scope): Addr = expr match { + case Pure.ValueVar(id, annotatedType) => + env.lookupValue(id) + + case Pure.Literal(value, annotatedType) => + scope.allocate("x", Value.Literal(value, annotatedType)) + + // right now everything is stuck... no constant folding ... + case Pure.PureApp(f, targs, vargs) => + scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate))) + + // TODO fix once this is a statement + case DirectApp(f, targs, vargs, bargs) => + scope.run("x", f, targs, vargs.map(evaluate)) + + case Pure.Make(data, tag, targs, vargs) => + scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) + + case Pure.Box(b, annotatedCapture) => + ??? + } + + def evaluate(stmt: Stmt, stack: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { + + case Stmt.Return(expr) => + returnTo(stack, evaluate(expr)) + + case Stmt.Val(id, annotatedTpe, binding, body) => + // This push can lead to an eta-redex (a superfluous push...) + evaluate(binding, push(annotatedTpe) { res => + bind(id, res) { evaluate(body, stack) } + }) + + case Stmt.Let(id, annotatedTpe, binding, body) => + bind(id, evaluate(binding)) { evaluate(body, stack) } + + case Stmt.Def(id, block, body) => + bind(id, evaluate(block)) { evaluate(body, stack) } + + case Stmt.App(callee, targs, vargs, bargs) => + evaluate(callee) match { + case Computation.Var(id) => + reify(stack, NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate))) + case Computation.Def(label) => + // TODO this should be "jump" + reify(stack, NeutralStmt.App(label, targs, vargs.map(evaluate), bargs.map(evaluate))) + case Computation.New(interface, operations) => sys error "Should not happen: app on new" + } + + case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => + evaluate(callee) match { + case Computation.Var(id) => + reify(stack, NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate))) + case Computation.Def(label) => sys error "Should not happen: invoke on def" + case Computation.New(interface, operations) => + val op = operations.collectFirst { case (id, label) if id == method => label }.get + reify(stack, NeutralStmt.App(op, targs, vargs.map(evaluate), bargs.map(evaluate))) + } + + case Stmt.If(cond, thn, els) => + val sc = evaluate(cond) + scope.lookupValue(sc) match { + case Some(Value.Literal(true, _)) => evaluate(thn, stack) + case Some(Value.Literal(false, _)) => evaluate(els, stack) + case _ => + join(stack) { k => + NeutralStmt.If(sc, nested { + evaluate(thn, k) + }, nested { + evaluate(els, k) + }) + } + } + + case Stmt.Match(scrutinee, clauses, default) => + val sc = evaluate(scrutinee) + scope.lookupValue(sc) match { + case Some(Value.Make(data, tag, targs, vargs)) => + // TODO substitute types (or bind them in the env)! + clauses.collectFirst { + case (tpe, BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => + bind(vparams.map(_.id).zip(vargs)) { evaluate(body, stack) } + }.getOrElse { + evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, stack) + } + case _ => + join(stack) { k => + NeutralStmt.Match(sc, + // This is ALMOST like evaluate(BlockLit), but keeps the current continuation + clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => + given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) + val block = Block(tparams, vparams, bparams, nested { + evaluate(body, k) + }) + (id, block) + }, + default.map { stmt => nested { evaluate(stmt, k) } }) + } + } + + case Stmt.Hole(span) => NeutralStmt.Hole(span) + + case Stmt.Alloc(id, init, region, body) => ??? + case Stmt.Var(ref, init, capture, body) => ??? + case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => ??? + case Stmt.Put(ref, annotatedCapt, value, body) => ??? + + // scoping constructs + case Stmt.Region(body) => ??? + case Stmt.Shift(prompt, body) => ??? + case Stmt.Reset(body) => ??? + case Stmt.Resume(k, body) => ??? + } + + def run(mod: ModuleDecl): ModuleDecl = { + val toplevelEnv = Env.empty.bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(defn.id))) + + val typingContext = TypingContext(Map.empty, mod.definitions.collect { + case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) + }.toMap) + + val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) + mod.copy(definitions = newDefinitions) + } + + inline def debug(inline msg: => Any) = println(msg) + + def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { + case Toplevel.Def(id, BlockLit(tparams, cparams, vparams, bparams, body)) => + debug(s"------- ${util.show(id)} -------") + debug(util.show(body)) + + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + + given scope: Scope = Scope.empty + val result = evaluate(body, Stack.Empty) + + debug(s"---------------------") + val block = Block(tparams, vparams, bparams, reify(scope, result)) + debug(PrettyPrinter.show(block)) + + debug(s"---------------------") + val embedded = embedBlockLit(block) + Toplevel.Def(id, embedded) + case other => other + } + + case class TypingContext(values: Map[Addr, ValueType], blocks: Map[Label, (BlockType, Captures)]) { + def bind(id: Id, tpe: ValueType): TypingContext = this.copy(values = values + (id -> tpe)) + def bind(id: Id, tpe: BlockType, capt: Captures): TypingContext = this.copy(blocks = blocks + (id -> (tpe, capt))) + def bindValues(vparams: List[ValueParam]): TypingContext = this.copy(values = values ++ vparams.map(p => p.id -> p.tpe)) + def bindComputations(bparams: List[BlockParam]): TypingContext = this.copy(blocks = blocks ++ bparams.map(p => p.id -> (p.tpe, p.capt))) + def lookupValue(id: Id): ValueType = values.getOrElse(id, sys.error(s"Unknown value: ${util.show(id)}")) + } + + def embedStmt(neutral: NeutralStmt)(using TypingContext): core.Stmt = neutral match { + case NeutralStmt.Return(result) => + Stmt.Return(embedPure(result)) + case NeutralStmt.App(label, targs, vargs, bargs) => + Stmt.App(embedBlockVar(label), targs, vargs.map(embedPure), bargs.map(embedBlock)) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => + Stmt.Invoke(embedBlockVar(label), method, tpe, targs, vargs.map(embedPure), bargs.map(embedBlock)) + case NeutralStmt.If(cond, thn, els) => + Stmt.If(embedPure(cond), embedStmt(thn), embedStmt(els)) + case NeutralStmt.Match(scrutinee, clauses, default) => + Stmt.Match(embedPure(scrutinee), + clauses.map { case (id, block) => id -> embedBlockLit(block) }, + default.map(embedStmt)) + case NeutralStmt.Hole(span) => + Stmt.Hole(span) + } + + def embedStmt(basicBlock: BasicBlock)(using G: TypingContext): core.Stmt = basicBlock match { + case BasicBlock(bindings, stmt) => + bindings.foldRight((G: TypingContext) => embedStmt(stmt)(using G)) { + case ((id, Binding.Let(value)), rest) => G => + val coreExpr = embedPure(value)(using G) + // TODO why do we even have this type in core, if we always infer it? + Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) + case ((id, Binding.Def(block)), rest) => G => + val coreBlock = embedBlockLit(block)(using G) + Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) + case ((id, Binding.Val(stmt)), rest) => G => + val coreStmt = embedStmt(stmt)(using G) + Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) + case ((id, Binding.Run(callee, targs, vargs)), rest) => G => + val coreExpr = DirectApp(callee, targs, vargs.map(arg => embedPure(arg)(using G)), Nil) + Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) + }(G) + } + + def embedPure(value: Value)(using TypingContext): core.Pure = value match { + case Value.Extern(callee, targs, vargs) => Pure.PureApp(callee, targs, vargs.map(embedPure)) + case Value.Literal(value, annotatedType) => Pure.Literal(value, annotatedType) + case Value.Make(data, tag, targs, vargs) => Pure.Make(data, tag, targs, vargs.map(embedPure)) + } + def embedPure(addr: Addr)(using G: TypingContext): core.Pure = Pure.ValueVar(addr, G.lookupValue(addr)) + + def embedBlock(comp: Computation)(using TypingContext): core.Block = comp match { + case Computation.Var(id) => embedBlockVar(id) + case Computation.Def(label) => embedBlockVar(label) + case Computation.New(interface, operations) => ??? // TODO eta-expand... + } + + def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { + case Block(tparams, vparams, bparams, body) => + core.Block.BlockLit(tparams, Nil, vparams, bparams, + embedStmt(body)(using G.bindValues(vparams).bindComputations(bparams))) + } + def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = + val (tpe, capt) = G.blocks(label) + core.BlockVar(label, tpe, capt) +} diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala index 8c4cb7a2f..edc338358 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala @@ -96,10 +96,11 @@ object Normalizer { normal => val context = Context(defs, Map.empty, DeclarationContext(m.declarations, m.externs), mutable.Map.from(usage), maxInlineSize) val (normalizedDefs, _) = normalizeToplevel(m.definitions)(using context) + m.copy(definitions = normalizedDefs) } - def normalizeToplevel(definitions: List[Toplevel])(using ctx: Context): (List[Toplevel], Context) = + private def normalizeToplevel(definitions: List[Toplevel])(using ctx: Context): (List[Toplevel], Context) = var contextSoFar = ctx val defs = definitions.map { case Toplevel.Def(id, block) => diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index f494eb96c..57683ce6a 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -28,26 +28,29 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { Deadcode.remove(mainSymbol, tree) } - if !Context.config.optimize() then return tree; - - // (2) lift static arguments - tree = Context.timed("static-argument-transformation", source.name) { - StaticArguments.transform(mainSymbol, tree) - } - - def normalize(m: ModuleDecl) = { - val anfed = BindSubexpressions.transform(m) - val normalized = Normalizer.normalize(Set(mainSymbol), anfed, Context.config.maxInlineSize().toInt) - val live = Deadcode.remove(mainSymbol, normalized) - val tailRemoved = RemoveTailResumptions(live) - val contified = DirectStyle.rewrite(tailRemoved) - contified - } - - // (3) normalize a few times (since tail resumptions might only surface after normalization and leave dead Resets) - tree = Context.timed("normalize-1", source.name) { normalize(tree) } - tree = Context.timed("normalize-2", source.name) { normalize(tree) } - tree = Context.timed("normalize-3", source.name) { normalize(tree) } + tree = NewNormalizer.run(tree) + // + // if !Context.config.optimize() then return tree; + // + // // (2) lift static arguments + // tree = Context.timed("static-argument-transformation", source.name) { + // StaticArguments.transform(mainSymbol, tree) + // } + // + // def normalize(m: ModuleDecl) = { + // val anfed = BindSubexpressions.transform(m) + // val normalized = Normalizer.normalize(Set(mainSymbol), anfed, Context.config.maxInlineSize().toInt) + // Normalizer.assertNormal(normalized) + // val live = Deadcode.remove(mainSymbol, normalized) + // val tailRemoved = RemoveTailResumptions(live) + // val contified = DirectStyle.rewrite(tailRemoved) + // contified + // } + // + // // (3) normalize a few times (since tail resumptions might only surface after normalization and leave dead Resets) + // tree = Context.timed("normalize-1", source.name) { normalize(tree) } + // tree = Context.timed("normalize-2", source.name) { normalize(tree) } + // tree = Context.timed("normalize-3", source.name) { normalize(tree) } tree } From 372c047574a6165e60c6fb5ab3718605c922a49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 12:23:20 +0200 Subject: [PATCH 003/123] Start supporting recursive local functions and reset and shift --- .../src/main/scala/effekt/core/Type.scala | 2 +- .../effekt/core/optimizer/NewNormalizer.scala | 83 +++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/Type.scala b/effekt/shared/src/main/scala/effekt/core/Type.scala index 773671bf7..41e2b52c5 100644 --- a/effekt/shared/src/main/scala/effekt/core/Type.scala +++ b/effekt/shared/src/main/scala/effekt/core/Type.scala @@ -131,7 +131,7 @@ object Type { def instantiate(f: BlockType.Function, targs: List[ValueType], cargs: List[Captures]): BlockType.Function = f match { case BlockType.Function(tparams, cparams, vparams, bparams, result) => assert(targs.size == tparams.size, "Wrong number of type arguments") - assert(cargs.size == cparams.size, "Wrong number of capture arguments") + assert(cargs.size == cparams.size, s"Wrong number of capture arguments on ${util.show(f)}: ${util.show(cargs)}") val tsubst = (tparams zip targs).toMap val csubst = (cparams zip cargs).toMap diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index c6afc7baf..b6e4a18da 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -23,6 +23,7 @@ object semantics { type Addr = Id type Label = Id + type Prompt = Id enum Value { // Stuck @@ -45,12 +46,14 @@ object semantics { enum Binding { case Let(value: Value) case Def(block: Block) + case Rec(block: Block, tpe: BlockType, capt: Captures) case Val(stmt: NeutralStmt) case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) val free: Set[Addr] = this match { case Binding.Let(value) => value.free case Binding.Def(block) => block.free + case Binding.Rec(block, tpe, capt) => block.free case Binding.Val(stmt) => stmt.free case Binding.Run(f, targs, vargs) => vargs.toSet } @@ -111,6 +114,10 @@ object semantics { bindings = bindings.updated(label, Binding.Def(block)) label + def defineRecursive(label: Label, block: Block, tpe: BlockType, capt: Captures): Label = + bindings = bindings.updated(label, Binding.Rec(block, tpe, capt)) + label + def push(id: Id, stmt: NeutralStmt): Unit = bindings = bindings.updated(id, Binding.Val(stmt)) } @@ -149,6 +156,9 @@ object semantics { // scrutinee is unknown case Match(scrutinee: Addr, clauses: List[(Id, Block)], default: Option[BasicBlock]) + case Reset(prompt: BlockParam, body: BasicBlock) + case Shift(prompt: Prompt, body: Block) + // aborts at runtime case Hole(span: Span) @@ -158,6 +168,8 @@ object semantics { case NeutralStmt.If(cond, thn, els) => Set(cond) ++ thn.free ++ els.free case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) case NeutralStmt.Return(result) => Set(result) + case NeutralStmt.Reset(prompt, body) => body.free + case NeutralStmt.Shift(prompt, body) => body.free case NeutralStmt.Hole(span) => Set.empty } } @@ -172,7 +184,8 @@ object semantics { case NeutralStmt.If(cond, thn, els) => "if" <+> parens(toDoc(cond)) <+> toDoc(thn) <+> "else" <+> toDoc(els) case NeutralStmt.Match(scrutinee, clauses, default) => - "match" <+> parens(toDoc(scrutinee)) // <+> braces(hcat(clauses.map {})) + "match" <+> parens(toDoc(scrutinee)) <+> braces(hcat(clauses.map { case (id, block) => toDoc(id) <> ":" <+> toDoc(block) })) <> + (if (default.isDefined) "else" <+> toDoc(default.get) else emptyDoc) case NeutralStmt.App(label, targs, vargs, bargs) => // Format as: l1[T1, T2](r1, r2) toDoc(label) <> @@ -185,6 +198,12 @@ object semantics { (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + case NeutralStmt.Reset(prompt, body) => + "reset" <+> braces(toDoc(prompt) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) + + case NeutralStmt.Shift(prompt, body) => + "shift" <> parens(toDoc(prompt)) <+> toDoc(body) + case NeutralStmt.Hole(span) => "hole()" } @@ -224,6 +243,7 @@ object semantics { hcat(bindings.map { case (addr, Binding.Let(value)) => "let" <+> toDoc(addr) <+> "=" <+> toDoc(value) <> line case (addr, Binding.Def(block)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line + case (addr, Binding.Rec(block, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line case (addr, Binding.Val(stmt)) => "val" <+> toDoc(addr) <+> "=" <+> toDoc(stmt) <> line case (addr, Binding.Run(callee, tparams, vparams)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> @@ -277,6 +297,9 @@ object NewNormalizer { normal => case (addr, b: Binding.Def) => used = used ++ b.free filtered = (addr, b) :: filtered + case (addr, b: Binding.Rec) => + used = used ++ b.free + filtered = (addr, b) :: filtered case (addr, s: Binding.Val) => used = used ++ s.free filtered = (addr, s) :: filtered @@ -434,6 +457,14 @@ object NewNormalizer { normal => case Stmt.Let(id, annotatedTpe, binding, body) => bind(id, evaluate(binding)) { evaluate(body, stack) } + // can be recursive + case Stmt.Def(id, block: core.BlockLit, body) => + given Env = env.bindComputation(id, Computation.Def(id)) + scope.defineRecursive(id, evaluate(block), block.tpe, block.capt) + bind(id, Computation.Def(id)) { + evaluate(body, stack) + } + case Stmt.Def(id, block, body) => bind(id, evaluate(block)) { evaluate(body, stack) } @@ -507,8 +538,18 @@ object NewNormalizer { normal => // scoping constructs case Stmt.Region(body) => ??? - case Stmt.Shift(prompt, body) => ??? - case Stmt.Reset(body) => ??? + case Stmt.Shift(prompt, body) => + // TODO implement correctly... + reify(stack, NeutralStmt.Shift(prompt.id, evaluate(body))) + case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => + // TODO is Var correct here?? Probably needs to be a new computation value... + // but shouldn't it be a fresh prompt each time? + given Env = env.bindComputation(prompt.id -> Computation.Var(prompt.id) :: Nil) + // TODO implement properly + reify(stack, NeutralStmt.Reset(prompt, nested { + evaluate(body, Stack.Empty) + })) + case Stmt.Reset(_) => ??? case Stmt.Resume(k, body) => ??? } @@ -543,6 +584,8 @@ object NewNormalizer { normal => debug(s"---------------------") val embedded = embedBlockLit(block) + debug(util.show(embedded)) + Toplevel.Def(id, embedded) case other => other } @@ -555,7 +598,7 @@ object NewNormalizer { normal => def lookupValue(id: Id): ValueType = values.getOrElse(id, sys.error(s"Unknown value: ${util.show(id)}")) } - def embedStmt(neutral: NeutralStmt)(using TypingContext): core.Stmt = neutral match { + def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { case NeutralStmt.Return(result) => Stmt.Return(embedPure(result)) case NeutralStmt.App(label, targs, vargs, bargs) => @@ -568,6 +611,14 @@ object NewNormalizer { normal => Stmt.Match(embedPure(scrutinee), clauses.map { case (id, block) => id -> embedBlockLit(block) }, default.map(embedStmt)) + case NeutralStmt.Reset(prompt, body) => + val capture = prompt.capt match { + case set if set.size == 1 => set.head + case _ => sys error "Prompt needs to have a single capture" + } + Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputations(prompt :: Nil)))) + case NeutralStmt.Shift(prompt, body) => + Stmt.Shift(embedBlockVar(prompt), embedBlockLit(body)) case NeutralStmt.Hole(span) => Stmt.Hole(span) } @@ -582,6 +633,9 @@ object NewNormalizer { normal => case ((id, Binding.Def(block)), rest) => G => val coreBlock = embedBlockLit(block)(using G) Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) + case ((id, Binding.Rec(block, tpe, capt)), rest) => G => + val coreBlock = embedBlockLit(block)(using G.bind(id, tpe, capt)) + Stmt.Def(id, coreBlock, rest(G.bind(id, tpe, capt))) case ((id, Binding.Val(stmt)), rest) => G => val coreStmt = embedStmt(stmt)(using G) Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) @@ -598,10 +652,25 @@ object NewNormalizer { normal => } def embedPure(addr: Addr)(using G: TypingContext): core.Pure = Pure.ValueVar(addr, G.lookupValue(addr)) - def embedBlock(comp: Computation)(using TypingContext): core.Block = comp match { + def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { case Computation.Var(id) => embedBlockVar(id) case Computation.Def(label) => embedBlockVar(label) - case Computation.New(interface, operations) => ??? // TODO eta-expand... + case Computation.New(interface, operations) => + val ops = operations.map { case (id, label) => + G.blocks(label) match { + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + val tparams2 = tparams.map(t => Id(t)) + // TODO if we freshen cparams, then we also need to substitute them in the result AND + val cparams2 = cparams //.map(c => Id(c)) + val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) + val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } + + core.Operation(id, tparams2, cparams, vparams2, bparams2, + Stmt.App(embedBlockVar(label), tparams2.map(ValueType.Var.apply), vparams2.map(p => ValueVar(p.id, p.tpe)), bparams2.map(p => BlockVar(p.id, p.tpe, p.capt)))) + case _ => sys error "Unexpected block type" + } + } + core.Block.New(Implementation(interface, ops)) } def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { @@ -610,6 +679,6 @@ object NewNormalizer { normal => embedStmt(body)(using G.bindValues(vparams).bindComputations(bparams))) } def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = - val (tpe, capt) = G.blocks(label) + val (tpe, capt) = G.blocks.getOrElse(label, sys error s"Unknown block: ${util.show(label)}. ${G.blocks.keys.map(util.show).mkString(", ")}") core.BlockVar(label, tpe, capt) } From b3e92f044c6c7fd709e991bb81707836675bc16c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 14:52:34 +0200 Subject: [PATCH 004/123] WIP fix a few bugs --- .../src/test/scala/effekt/core/VMTests.scala | 2 +- .../effekt/core/optimizer/NewNormalizer.scala | 20 +++++++++---------- .../effekt/core/optimizer/Optimizer.scala | 14 ++++++++++--- .../effekt/generator/js/TransformerCps.scala | 4 +++- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/VMTests.scala b/effekt/jvm/src/test/scala/effekt/core/VMTests.scala index 4aad51f3e..5126b960a 100644 --- a/effekt/jvm/src/test/scala/effekt/core/VMTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/VMTests.scala @@ -1295,7 +1295,7 @@ class VMTests extends munit.FunSuite { val (result, summary) = runFile(path) val expected = expectedResultFor(f).getOrElse { s"Missing checkfile for ${path}"} assertNoDiff(result, expected) - expectedSummary.foreach { expected => assertEquals(summary, expected) } + //expectedSummary.foreach { expected => assertEquals(summary, expected) } } catch { case i: VMError => fail(i.getMessage, i) } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b6e4a18da..cc0060e61 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -125,7 +125,7 @@ object semantics { def empty: Scope = new Scope(ListMap.empty, Map.empty, None) } - case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { + case class Block(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { def free: Set[Addr] = body.free -- vparams.map(_.id) } @@ -226,7 +226,7 @@ object semantics { } def toDoc(block: Block): Doc = block match { - case Block(tparams, vparams, bparams, body) => + case Block(tparams, cparams, vparams, bparams, body) => (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) } @@ -382,7 +382,7 @@ object NewNormalizer { normal => // } case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App)) => f(stack) case body => - val k = scope.define("k", Block(Nil, ValueParam(x, tpe) :: Nil, Nil, body)) + val k = scope.define("k", Block(Nil, Nil, ValueParam(x, tpe) :: Nil, Nil, body)) f(Stack.Dynamic(k)) } case Stack.Empty => f(stack) @@ -416,7 +416,7 @@ object NewNormalizer { normal => given localEnv: Env = env .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - Block(tparams, vparams, bparams, nested { + Block(tparams, cparams, vparams, bparams, nested { evaluate(body, Stack.Empty) }) } @@ -434,6 +434,7 @@ object NewNormalizer { normal => // TODO fix once this is a statement case DirectApp(f, targs, vargs, bargs) => + assert(bargs.isEmpty) scope.run("x", f, targs, vargs.map(evaluate)) case Pure.Make(data, tag, targs, vargs) => @@ -520,7 +521,7 @@ object NewNormalizer { normal => // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = Block(tparams, vparams, bparams, nested { + val block = Block(tparams, cparams, vparams, bparams, nested { evaluate(body, k) }) (id, block) @@ -557,8 +558,7 @@ object NewNormalizer { normal => val toplevelEnv = Env.empty.bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(defn.id))) val typingContext = TypingContext(Map.empty, mod.definitions.collect { - case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap) + case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) }.toMap) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) mod.copy(definitions = newDefinitions) @@ -579,7 +579,7 @@ object NewNormalizer { normal => val result = evaluate(body, Stack.Empty) debug(s"---------------------") - val block = Block(tparams, vparams, bparams, reify(scope, result)) + val block = Block(tparams, cparams, vparams, bparams, reify(scope, result)) debug(PrettyPrinter.show(block)) debug(s"---------------------") @@ -674,8 +674,8 @@ object NewNormalizer { normal => } def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { - case Block(tparams, vparams, bparams, body) => - core.Block.BlockLit(tparams, Nil, vparams, bparams, + case Block(tparams, cparams, vparams, bparams, body) => + core.Block.BlockLit(tparams, cparams, vparams, bparams, embedStmt(body)(using G.bindValues(vparams).bindComputations(bparams))) } def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 57683ce6a..8cb762784 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -28,9 +28,17 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { Deadcode.remove(mainSymbol, tree) } - tree = NewNormalizer.run(tree) - // - // if !Context.config.optimize() then return tree; + if !Context.config.optimize() then return tree; + + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer.run(tree) } + Normalizer.assertNormal(tree) + // println(util.show(tree)) + tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer.run(tree) } + Normalizer.assertNormal(tree) + tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } + Normalizer.assertNormal(tree) + //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) + // // // (2) lift static arguments // tree = Context.timed("static-argument-transformation", source.name) { diff --git a/effekt/shared/src/main/scala/effekt/generator/js/TransformerCps.scala b/effekt/shared/src/main/scala/effekt/generator/js/TransformerCps.scala index 4e07a0a3c..72d823e3c 100644 --- a/effekt/shared/src/main/scala/effekt/generator/js/TransformerCps.scala +++ b/effekt/shared/src/main/scala/effekt/generator/js/TransformerCps.scala @@ -211,9 +211,11 @@ object TransformerCps extends Transformer { js.Const(nameDef(id), toJS(binding)) :: toJS(body).run(k) } + // Note: currently we only perform this translation if there isn't already a direct-style continuation + // // [[ let k(x, ks) = ...; if (...) jump k(42, ks2) else jump k(10, ks3) ]] = // let x; if (...) { x = 42; ks = ks2 } else { x = 10; ks = ks3 } ... - case cps.Stmt.LetCont(id, Cont.ContLam(params, ks, body), body2) if canBeDirect(id, body2) => + case cps.Stmt.LetCont(id, Cont.ContLam(params, ks, body), body2) if D.directStyle.isEmpty && canBeDirect(id, body2) => Binding { k => params.map { p => js.Let(nameDef(p), js.Undefined) } ::: toJS(body2)(using markDirectStyle(id, params, ks)).stmts ++ From e90094ee758d7c2ac24d7e04d3deaee224fd03fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 15:37:37 +0200 Subject: [PATCH 005/123] WIP bargs for direct app --- .../effekt/core/optimizer/NewNormalizer.scala | 23 +++++++++---------- .../effekt/core/optimizer/Optimizer.scala | 7 ++++-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index cc0060e61..587ece46b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -48,14 +48,14 @@ object semantics { case Def(block: Block) case Rec(block: Block, tpe: BlockType, capt: Captures) case Val(stmt: NeutralStmt) - case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) + case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) val free: Set[Addr] = this match { case Binding.Let(value) => value.free case Binding.Def(block) => block.free case Binding.Rec(block, tpe, capt) => block.free case Binding.Val(stmt) => stmt.free - case Binding.Run(f, targs, vargs) => vargs.toSet + case Binding.Run(f, targs, vargs, bargs) => vargs.toSet } } @@ -97,9 +97,9 @@ object semantics { addr } - def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr]): Addr = { + def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]): Addr = { val addr = Id(hint) - bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs)) + bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs, bargs)) addr } @@ -245,9 +245,9 @@ object semantics { case (addr, Binding.Def(block)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line case (addr, Binding.Rec(block, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line case (addr, Binding.Val(stmt)) => "val" <+> toDoc(addr) <+> "=" <+> toDoc(stmt) <> line - case (addr, Binding.Run(callee, tparams, vparams)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> - (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> - parens(hsep(vparams.map(toDoc), comma)) <> line + case (addr, Binding.Run(callee, targs, vargs, bargs)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line }) def toDoc(block: BasicBlock): Doc = @@ -432,10 +432,9 @@ object NewNormalizer { normal => case Pure.PureApp(f, targs, vargs) => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate))) - // TODO fix once this is a statement case DirectApp(f, targs, vargs, bargs) => assert(bargs.isEmpty) - scope.run("x", f, targs, vargs.map(evaluate)) + scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate)) case Pure.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) @@ -564,7 +563,7 @@ object NewNormalizer { normal => mod.copy(definitions = newDefinitions) } - inline def debug(inline msg: => Any) = println(msg) + inline def debug(inline msg: => Any) = () // println(msg) def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { case Toplevel.Def(id, BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -639,8 +638,8 @@ object NewNormalizer { normal => case ((id, Binding.Val(stmt)), rest) => G => val coreStmt = embedStmt(stmt)(using G) Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) - case ((id, Binding.Run(callee, targs, vargs)), rest) => G => - val coreExpr = DirectApp(callee, targs, vargs.map(arg => embedPure(arg)(using G)), Nil) + case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => + val coreExpr = DirectApp(callee, targs, vargs.map(arg => embedPure(arg)(using G)), bargs.map(arg => embedBlock(arg)(using G))) Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) }(G) } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 8cb762784..b24bb7920 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -35,10 +35,13 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { // println(util.show(tree)) tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer.run(tree) } Normalizer.assertNormal(tree) - tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } - Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) + // tree = Context.timed("old-normalizer-1", source.name) { Normalizer.normalize(Set(mainSymbol), tree, 0) } + // tree = Context.timed("old-normalizer-2", source.name) { Normalizer.normalize(Set(mainSymbol), tree, 0) } + // + // tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } + // Normalizer.assertNormal(tree) // // // (2) lift static arguments // tree = Context.timed("static-argument-transformation", source.name) { From a0b7474b57f21edbe152badd19675fc8afcbc3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 16:25:18 +0200 Subject: [PATCH 006/123] WIP support resume and avoid eta-redexes --- .../effekt/core/optimizer/NewNormalizer.scala | 79 +++++++++++++++---- .../effekt/core/optimizer/Optimizer.scala | 13 +-- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 587ece46b..d74904365 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -156,8 +156,12 @@ object semantics { // scrutinee is unknown case Match(scrutinee: Addr, clauses: List[(Id, Block)], default: Option[BasicBlock]) + // what's actually unknown here? case Reset(prompt: BlockParam, body: BasicBlock) - case Shift(prompt: Prompt, body: Block) + // prompt / context is unknown + case Shift(prompt: Prompt, kCapt: Capture, k: BlockParam, body: BasicBlock) + // continuation is unknown + case Resume(k: Id, body: BasicBlock) // aborts at runtime case Hole(span: Span) @@ -169,7 +173,8 @@ object semantics { case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) case NeutralStmt.Return(result) => Set(result) case NeutralStmt.Reset(prompt, body) => body.free - case NeutralStmt.Shift(prompt, body) => body.free + case NeutralStmt.Shift(prompt, capt, k, body) => body.free + case NeutralStmt.Resume(k, body) => body.free case NeutralStmt.Hole(span) => Set.empty } } @@ -201,8 +206,11 @@ object semantics { case NeutralStmt.Reset(prompt, body) => "reset" <+> braces(toDoc(prompt) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) - case NeutralStmt.Shift(prompt, body) => - "shift" <> parens(toDoc(prompt)) <+> toDoc(body) + case NeutralStmt.Shift(prompt, capt, k, body) => + "shift" <> parens(toDoc(prompt)) <+> braces(toDoc(k) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) + + case NeutralStmt.Resume(k, body) => + "resume" <> parens(toDoc(k)) <+> toDoc(body) case NeutralStmt.Hole(span) => "hole()" } @@ -400,10 +408,31 @@ object NewNormalizer { normal => Computation.Def(scope.define("f", evaluate(b))) case core.Block.Unbox(pure) => ??? + case core.Block.New(Implementation(interface, operations)) => val ops = operations.map { case Operation(name, tparams, cparams, vparams, bparams, body) => - val label = scope.define(name.name.name, evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body): core.Block.BlockLit)) + // Check whether the operation is already "just" an eta expansion and then use the identifier... + // no need to create a fresh block literal + val eta: Option[Label] = + body match { + case Stmt.App(BlockVar(id, _, _), targs, vargs, bargs) => + def sameTargs = targs == tparams.map(t => ValueType.Var(t)) + def sameVargs = vargs == vparams.map(p => ValueVar(p.id, p.tpe)) + def sameBargs = bargs == bparams.map(p => BlockVar(p.id, p.tpe, p.capt)) + def isEta = sameTargs && sameVargs && sameBargs + + env.lookupComputation(id) match { + case Computation.Def(label) if isEta => Some(label) + case _ => None + } + case _ => None + } + + val label = eta.getOrElse { + scope.define(name.name.name, + evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body): core.Block.BlockLit)) + } (name, label) } Computation.New(interface, ops) @@ -531,26 +560,42 @@ object NewNormalizer { normal => case Stmt.Hole(span) => NeutralStmt.Hole(span) + // State + case Stmt.Region(body) => ??? case Stmt.Alloc(id, init, region, body) => ??? case Stmt.Var(ref, init, capture, body) => ??? case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => ??? case Stmt.Put(ref, annotatedCapt, value, body) => ??? - // scoping constructs - case Stmt.Region(body) => ??? - case Stmt.Shift(prompt, body) => + // Control Effects + case Stmt.Shift(prompt, BlockLit(Nil, cparam :: Nil, Nil, k :: Nil, body)) => // TODO implement correctly... - reify(stack, NeutralStmt.Shift(prompt.id, evaluate(body))) + val neutralBody = { + given Env = env.bindComputation(k.id -> Computation.Var(k.id) :: Nil) + nested { + evaluate(body, Stack.Empty) + } + } + assert(Set(cparam) == k.capt, "At least for now these need to be the same") + reify(stack, NeutralStmt.Shift(prompt.id, cparam, k, neutralBody)) + case Stmt.Shift(_, _) => ??? case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => // TODO is Var correct here?? Probably needs to be a new computation value... // but shouldn't it be a fresh prompt each time? - given Env = env.bindComputation(prompt.id -> Computation.Var(prompt.id) :: Nil) - // TODO implement properly - reify(stack, NeutralStmt.Reset(prompt, nested { + val neutralBody = { + given Env = env.bindComputation(prompt.id -> Computation.Var(prompt.id) :: Nil) + nested { evaluate(body, Stack.Empty) - })) + } + } + // TODO implement properly + reify(stack, NeutralStmt.Reset(prompt, neutralBody)) case Stmt.Reset(_) => ??? - case Stmt.Resume(k, body) => ??? + case Stmt.Resume(k, body) => + // TODO implement properly + reify(stack, NeutralStmt.Resume(k.id, nested { + evaluate(body, Stack.Empty) + })) } def run(mod: ModuleDecl): ModuleDecl = { @@ -616,8 +661,10 @@ object NewNormalizer { normal => case _ => sys error "Prompt needs to have a single capture" } Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputations(prompt :: Nil)))) - case NeutralStmt.Shift(prompt, body) => - Stmt.Shift(embedBlockVar(prompt), embedBlockLit(body)) + case NeutralStmt.Shift(prompt, capt, k, body) => + Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputations(k :: Nil)))) + case NeutralStmt.Resume(k, body) => + Stmt.Resume(embedBlockVar(k), embedStmt(body)) case NeutralStmt.Hole(span) => Stmt.Hole(span) } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index b24bb7920..2c5f0525b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -42,11 +42,14 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { // // tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } // Normalizer.assertNormal(tree) - // - // // (2) lift static arguments - // tree = Context.timed("static-argument-transformation", source.name) { - // StaticArguments.transform(mainSymbol, tree) - // } + + // (2) lift static arguments + tree = Context.timed("static-argument-transformation", source.name) { + StaticArguments.transform(mainSymbol, tree) + } + + tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } + Normalizer.assertNormal(tree) // // def normalize(m: ModuleDecl) = { // val anfed = BindSubexpressions.transform(m) From 6466381b0b67704681da71572c83ea5f38ef1817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 3 Sep 2025 17:10:15 +0200 Subject: [PATCH 007/123] WIP before proper shift --- .../effekt/core/optimizer/NewNormalizer.scala | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index d74904365..b0ee28cb8 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -5,6 +5,7 @@ package optimizer import effekt.source.Span import effekt.core.optimizer.semantics.NeutralStmt import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } +import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter import scala.annotation.tailrec @@ -472,6 +473,7 @@ object NewNormalizer { normal => ??? } + // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) def evaluate(stmt: Stmt, stack: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { case Stmt.Return(expr) => @@ -511,7 +513,7 @@ object NewNormalizer { normal => evaluate(callee) match { case Computation.Var(id) => reify(stack, NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate))) - case Computation.Def(label) => sys error "Should not happen: invoke on def" + case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" case Computation.New(interface, operations) => val op = operations.collectFirst { case (id, label) if id == method => label }.get reify(stack, NeutralStmt.App(op, targs, vargs.map(evaluate), bargs.map(evaluate))) @@ -563,12 +565,17 @@ object NewNormalizer { normal => // State case Stmt.Region(body) => ??? case Stmt.Alloc(id, init, region, body) => ??? + case Stmt.Var(ref, init, capture, body) => ??? case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => ??? case Stmt.Put(ref, annotatedCapt, value, body) => ??? // Control Effects case Stmt.Shift(prompt, BlockLit(Nil, cparam :: Nil, Nil, k :: Nil, body)) => + val p = env.lookupComputation(prompt.id) match { + case Computation.Var(id) => id + case _ => ??? + } // TODO implement correctly... val neutralBody = { given Env = env.bindComputation(k.id -> Computation.Var(k.id) :: Nil) @@ -577,38 +584,51 @@ object NewNormalizer { normal => } } assert(Set(cparam) == k.capt, "At least for now these need to be the same") - reify(stack, NeutralStmt.Shift(prompt.id, cparam, k, neutralBody)) + reify(stack, NeutralStmt.Shift(p, cparam, k, neutralBody)) case Stmt.Shift(_, _) => ??? case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => // TODO is Var correct here?? Probably needs to be a new computation value... // but shouldn't it be a fresh prompt each time? + val p = Id(prompt.id) val neutralBody = { - given Env = env.bindComputation(prompt.id -> Computation.Var(prompt.id) :: Nil) + given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) nested { evaluate(body, Stack.Empty) } } // TODO implement properly - reify(stack, NeutralStmt.Reset(prompt, neutralBody)) + reify(stack, NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) case Stmt.Reset(_) => ??? case Stmt.Resume(k, body) => + val r = env.lookupComputation(k.id) match { + case Computation.Var(id) => id + case _ => ??? + } // TODO implement properly - reify(stack, NeutralStmt.Resume(k.id, nested { + reify(stack, NeutralStmt.Resume(r, nested { evaluate(body, Stack.Empty) })) } def run(mod: ModuleDecl): ModuleDecl = { - val toplevelEnv = Env.empty.bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(defn.id))) + + // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) + val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } + val toplevelEnv = Env.empty + // user defined functions + .bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(defn.id))) + // async extern functions + .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Def(defn.id))) val typingContext = TypingContext(Map.empty, mod.definitions.collect { - case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) }.toMap) + case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) + }.toMap) // ++ asyncExterns.map { d => d.id -> null }) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) mod.copy(definitions = newDefinitions) } - inline def debug(inline msg: => Any) = () // println(msg) + inline def debug(inline msg: => Any) = println(msg) def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { case Toplevel.Def(id, BlockLit(tparams, cparams, vparams, bparams, body)) => From c3c7c587bbfe498edcef5a66f0d759ee01f350b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Thu, 4 Sep 2025 10:49:45 +0200 Subject: [PATCH 008/123] Start preparing for iterated CPS --- .../effekt/core/optimizer/NewNormalizer.scala | 146 ++++++++++-------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b0ee28cb8..2b5c824ae 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -362,45 +362,57 @@ object NewNormalizer { normal => case Static(tpe: ValueType, apply: Env => Scope => Addr => NeutralStmt) case Dynamic(label: Label) } - - def returnTo(stack: Stack, arg: Addr)(using env: Env, scope: Scope): NeutralStmt = stack match { - case Stack.Empty => NeutralStmt.Return(arg) - case Stack.Static(tpe, apply) => apply(env)(scope)(arg) - case Stack.Dynamic(label) => NeutralStmt.App(label, List.empty, List(arg), Nil) - } - - def reify(stack: Stack, stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = stack match { - case Stack.Empty => stmt - // [[ val x = { val y = stmt1; stmt2 }; stmt3 ]] = [[ val y = stmt1; val x = stmt2; stmt3 ]] - case Stack.Static(tpe, apply) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - apply(env)(scope)(tmp) - // stack is already reified - case Stack.Dynamic(label) => - stmt - } - - def join(stack: Stack)(f: Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = stack match { - case Stack.Static(tpe, apply) => - val x = Id("x") - nested { scope ?=> apply(env)(scope)(x) } match { - // Avoid trivial continuations like - // def k_6268 = (x_6267: Int_3) { - // return x_6267 - // } - case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App)) => f(stack) - case body => - val k = scope.define("k", Block(Nil, Nil, ValueParam(x, tpe) :: Nil, Nil, body)) - f(Stack.Dynamic(k)) + enum MetaStack { + case Last(stack: Stack) + // case Segment(prompt: Prompt, stack: Stack, next: MetaStack) + + def ret(arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { + case MetaStack.Last(stack) => stack match { + case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Static(tpe, apply) => apply(env)(scope)(arg) + case Stack.Dynamic(label) => NeutralStmt.App(label, List.empty, List(arg), Nil) + } + } + def push(tpe: ValueType)(f: Env ?=> Scope ?=> Addr => MetaStack => NeutralStmt): MetaStack = this match { + case MetaStack.Last(stack) => MetaStack.Last(Stack.Static(tpe, + env => scope => arg => f(using env)(using scope)(arg)(this) + )) + } + def reify(stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { + case MetaStack.Last(stack) => stack match { + case Stack.Empty => stmt + // [[ val x = { val y = stmt1; stmt2 }; stmt3 ]] = [[ val y = stmt1; val x = stmt2; stmt3 ]] + case Stack.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(env)(scope)(tmp) + // stack is already reified + case Stack.Dynamic(label) => + stmt + } + } + def joinpoint(f: MetaStack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { + case MetaStack.Last(stack) => stack match { + case Stack.Static(tpe, apply) => + val x = Id("x") + nested { scope ?=> apply(env)(scope)(x) } match { + // Avoid trivial continuations like + // def k_6268 = (x_6267: Int_3) { + // return x_6267 + // } + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App)) => f(this) + case body => + val k = scope.define("k", Block(Nil, Nil, ValueParam(x, tpe) :: Nil, Nil, body)) + f(MetaStack.Last(Stack.Dynamic(k))) + } + case Stack.Empty => f(this) + case Stack.Dynamic(label) => f(this) } - case Stack.Empty => f(stack) - case Stack.Dynamic(label) => f(stack) + } + } + object MetaStack { + def empty: MetaStack = MetaStack.Last(Stack.Empty) } - - def push(tpe: ValueType)(f: Env ?=> Scope ?=> Addr => NeutralStmt): Stack = Stack.Static(tpe, - env => scope => arg => f(using env)(using scope)(arg) - ) def evaluate(block: core.Block)(using env: Env, scope: Scope): Computation = block match { case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => @@ -447,7 +459,7 @@ object NewNormalizer { normal => .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) Block(tparams, cparams, vparams, bparams, nested { - evaluate(body, Stack.Empty) + evaluate(body, MetaStack.empty) }) } @@ -474,58 +486,58 @@ object NewNormalizer { normal => } // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) - def evaluate(stmt: Stmt, stack: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { + def evaluate(stmt: Stmt, k: MetaStack)(using env: Env, scope: Scope): NeutralStmt = stmt match { case Stmt.Return(expr) => - returnTo(stack, evaluate(expr)) + k.ret(evaluate(expr)) case Stmt.Val(id, annotatedTpe, binding, body) => // This push can lead to an eta-redex (a superfluous push...) - evaluate(binding, push(annotatedTpe) { res => - bind(id, res) { evaluate(body, stack) } + evaluate(binding, k.push(annotatedTpe) { res => k => + bind(id, res) { evaluate(body, k) } }) case Stmt.Let(id, annotatedTpe, binding, body) => - bind(id, evaluate(binding)) { evaluate(body, stack) } + bind(id, evaluate(binding)) { evaluate(body, k) } // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => given Env = env.bindComputation(id, Computation.Def(id)) scope.defineRecursive(id, evaluate(block), block.tpe, block.capt) bind(id, Computation.Def(id)) { - evaluate(body, stack) + evaluate(body, k) } case Stmt.Def(id, block, body) => - bind(id, evaluate(block)) { evaluate(body, stack) } + bind(id, evaluate(block)) { evaluate(body, k) } case Stmt.App(callee, targs, vargs, bargs) => evaluate(callee) match { case Computation.Var(id) => - reify(stack, NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate))) case Computation.Def(label) => // TODO this should be "jump" - reify(stack, NeutralStmt.App(label, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.App(label, targs, vargs.map(evaluate), bargs.map(evaluate))) case Computation.New(interface, operations) => sys error "Should not happen: app on new" } case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => evaluate(callee) match { case Computation.Var(id) => - reify(stack, NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate))) case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" case Computation.New(interface, operations) => val op = operations.collectFirst { case (id, label) if id == method => label }.get - reify(stack, NeutralStmt.App(op, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.App(op, targs, vargs.map(evaluate), bargs.map(evaluate))) } case Stmt.If(cond, thn, els) => val sc = evaluate(cond) scope.lookupValue(sc) match { - case Some(Value.Literal(true, _)) => evaluate(thn, stack) - case Some(Value.Literal(false, _)) => evaluate(els, stack) + case Some(Value.Literal(true, _)) => evaluate(thn, k) + case Some(Value.Literal(false, _)) => evaluate(els, k) case _ => - join(stack) { k => + k.joinpoint { k => NeutralStmt.If(sc, nested { evaluate(thn, k) }, nested { @@ -541,12 +553,12 @@ object NewNormalizer { normal => // TODO substitute types (or bind them in the env)! clauses.collectFirst { case (tpe, BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => - bind(vparams.map(_.id).zip(vargs)) { evaluate(body, stack) } + bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k) } }.getOrElse { - evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, stack) + evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k) } case _ => - join(stack) { k => + k.joinpoint { k => NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -571,20 +583,20 @@ object NewNormalizer { normal => case Stmt.Put(ref, annotatedCapt, value, body) => ??? // Control Effects - case Stmt.Shift(prompt, BlockLit(Nil, cparam :: Nil, Nil, k :: Nil, body)) => + case Stmt.Shift(prompt, BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => val p = env.lookupComputation(prompt.id) match { case Computation.Var(id) => id case _ => ??? } // TODO implement correctly... val neutralBody = { - given Env = env.bindComputation(k.id -> Computation.Var(k.id) :: Nil) + given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) nested { - evaluate(body, Stack.Empty) + evaluate(body, MetaStack.empty) } } - assert(Set(cparam) == k.capt, "At least for now these need to be the same") - reify(stack, NeutralStmt.Shift(p, cparam, k, neutralBody)) + assert(Set(cparam) == k2.capt, "At least for now these need to be the same") + k.reify(NeutralStmt.Shift(p, cparam, k2, neutralBody)) case Stmt.Shift(_, _) => ??? case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => // TODO is Var correct here?? Probably needs to be a new computation value... @@ -593,20 +605,20 @@ object NewNormalizer { normal => val neutralBody = { given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) nested { - evaluate(body, Stack.Empty) + evaluate(body, MetaStack.empty) } } // TODO implement properly - reify(stack, NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) + k.reify(NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) case Stmt.Reset(_) => ??? - case Stmt.Resume(k, body) => - val r = env.lookupComputation(k.id) match { + case Stmt.Resume(k2, body) => + val r = env.lookupComputation(k2.id) match { case Computation.Var(id) => id case _ => ??? } // TODO implement properly - reify(stack, NeutralStmt.Resume(r, nested { - evaluate(body, Stack.Empty) + k.reify(NeutralStmt.Resume(r, nested { + evaluate(body, MetaStack.empty) })) } @@ -640,7 +652,7 @@ object NewNormalizer { normal => .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) given scope: Scope = Scope.empty - val result = evaluate(body, Stack.Empty) + val result = evaluate(body, MetaStack.empty) debug(s"---------------------") val block = Block(tparams, cparams, vparams, bparams, reify(scope, result)) From 945c8260e89084132047cb1348d04b1127770969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Fri, 5 Sep 2025 14:04:15 +0200 Subject: [PATCH 009/123] Add jump and reorganize a bit --- .../effekt/core/optimizer/NewNormalizer.scala | 238 +++++++++++------- 1 file changed, 145 insertions(+), 93 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 2b5c824ae..b870c94aa 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,6 +2,7 @@ package effekt package core package optimizer +import effekt.core.optimizer.NewNormalizer.MetaStack import effekt.source.Span import effekt.core.optimizer.semantics.NeutralStmt import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } @@ -17,6 +18,21 @@ import scala.collection.immutable.ListMap // this way deadcode can be eliminated on the way up. // // plan: don't inline... this is a separate pass after normalization +// +// plan: only introduce parameters for free things inside a block that are bound in the **stack** +// that is in +// +// only abstract over p, but not n: +// +// def outer(n: Int) = +// def foo(p) = shift(p) { ... n ... } +// reset { p => +// ... +// } +// +// Same actually for stack allocated mutable state, we should abstract over those (but only those) +// and keep the function in its original location. +// This means we only need to abstract over blocks, no values, no types. object semantics { // Values @@ -26,6 +42,9 @@ object semantics { type Label = Id type Prompt = Id + type Variables = Set[Id] + def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet + enum Value { // Stuck //case Var(id: Id, annotatedType: ValueType) @@ -35,8 +54,8 @@ object semantics { case Literal(value: Any, annotatedType: ValueType) case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) - val free: Set[Addr] = this match { - // case Value.Var(id, annotatedType) => Set.empty + val free: Variables = this match { + // case Value.Var(id, annotatedType) => Variables.empty case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Make(data, tag, targs, vargs) => vargs.toSet @@ -51,12 +70,12 @@ object semantics { case Val(stmt: NeutralStmt) case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) - val free: Set[Addr] = this match { + val free: Variables = this match { case Binding.Let(value) => value.free case Binding.Def(block) => block.free case Binding.Rec(block, tpe, capt) => block.free case Binding.Val(stmt) => stmt.free - case Binding.Run(f, targs, vargs, bargs) => vargs.toSet + case Binding.Run(f, targs, vargs, bargs) => vargs.toSet ++ all(bargs, _.free) } } @@ -64,13 +83,6 @@ object semantics { object Bindings { def empty: Bindings = Nil } - extension (bindings: Bindings) { - def free: Set[Id] = { - val bound = bindings.map(_._1).toSet - val free = bindings.flatMap { b => b._2.free }.toSet - free -- bound - } - } /** * A Scope is a bit like a basic block, but without the terminator @@ -126,12 +138,97 @@ object semantics { def empty: Scope = new Scope(ListMap.empty, Map.empty, None) } - case class Block(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { - def free: Set[Addr] = body.free -- vparams.map(_.id) + def reify(scope: Scope, body: NeutralStmt): BasicBlock = { + var used = body.free + var filtered = Bindings.empty + // TODO implement properly + scope.bindings.toSeq.reverse.foreach { + // TODO for now we keep ALL definitions + case (addr, b: Binding.Def) => + used = used ++ b.free + filtered = (addr, b) :: filtered + case (addr, b: Binding.Rec) => + used = used ++ b.free + filtered = (addr, b) :: filtered + case (addr, s: Binding.Val) => + used = used ++ s.free + filtered = (addr, s) :: filtered + case (addr, v: Binding.Run) => + used = used ++ v.free + filtered = (addr, v) :: filtered + + // TODO if type is unit like, we can potentially drop this binding (but then we need to make up a "fresh" unit at use site) + case (addr, v: Binding.Let) if used.contains(addr) => + used = used ++ v.free + filtered = (addr, v) :: filtered + case (addr, v: Binding.Let) => () + } + + // we want to avoid turning tailcalls into non tail calls like + // + // val x = app(x) + // return x + // + // so we eta-reduce here. Can we achieve this by construction? + // TODO lastOption will go through the list AGAIN, let's see whether this causes performance problems + (filtered.lastOption, body) match { + case (Some((id1, Binding.Val(stmt))), NeutralStmt.Return(id2)) if id1 == id2 => + BasicBlock(filtered.init, stmt) + case (_, _) => + BasicBlock(filtered, body) + } + } + + def nested(prog: Scope ?=> NeutralStmt)(using scope: Scope): BasicBlock = { + // TODO parent code and parent store + val local = Scope(ListMap.empty, Map.empty, Some(scope)) + val result = prog(using local) + reify(local, result) + } + + case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { + def lookupValue(id: Id): Addr = values(id) + def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) + def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) + + def lookupComputation(id: Id): Computation = computations(id) + def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) + def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) + } + object Env { + def empty: Env = Env(Map.empty, Map.empty) + } + + case class Block(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], + // impl: List[Addr] => List[Computation] => Env => Scope => MetaStack => NeutralStmt, + body: BasicBlock) { + + val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) + } + + // TODO how do we detect mutually recursive toplevel functions??? + object Block { + def apply(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam])(impl: List[Addr] => List[Computation] => Env => Scope => MetaStack => NeutralStmt)(using env: Env, scope: Scope): Block = { + // TODO also substitute type params... + val reified: BasicBlock = nested { + impl(vparams.map(p => p.id))(bparams.map(p => Computation.Var(p.id)))(env)(scope)(MetaStack.empty) + } + ??? + } } case class BasicBlock(bindings: Bindings, body: NeutralStmt) { - def free: Set[Addr] = bindings.free ++ body.free + val free: Variables = { + var free = body.free + bindings.reverse.foreach { + case (id, b: Binding.Let) => free = (free - id) ++ b.free + case (id, b: Binding.Def) => free = (free - id) ++ b.free + case (id, b: Binding.Rec) => free = (free - id) ++ (b.free - id) + case (id, b: Binding.Val) => free = (free - id) ++ b.free + case (id, b: Binding.Run) => free = (free - id) ++ b.free + } + free + } } enum Computation { @@ -141,21 +238,29 @@ object semantics { case Def(label: Label) // Known object case New(interface: BlockType.Interface, operations: List[(Id, Label)]) + + val free: Variables = this match { + case Computation.Var(id) => Set(id) + case Computation.Def(label) => Set(label) + case Computation.New(interface, operations) => operations.map(_._2).toSet + } } // Statements // ---------- enum NeutralStmt { // continuation is unknown - case Return(result: Addr) - // callee is unknown or we do not want to inline (TODO no block arguments for now) - case App(label: Label, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + case Return(result: Id) + // callee is unknown or we do not want to inline + case App(callee: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) + // Known jump, but we do not want to inline + case Jump(label: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) // callee is unknown - case Invoke(id: Id, method: Id, methodTpe: BlockType, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + case Invoke(id: Id, method: Id, methodTpe: BlockType, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) // cond is unknown - case If(cond: Addr, thn: BasicBlock, els: BasicBlock) + case If(cond: Id, thn: BasicBlock, els: BasicBlock) // scrutinee is unknown - case Match(scrutinee: Addr, clauses: List[(Id, Block)], default: Option[BasicBlock]) + case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) // what's actually unknown here? case Reset(prompt: BlockParam, body: BasicBlock) @@ -167,15 +272,16 @@ object semantics { // aborts at runtime case Hole(span: Span) - val free: Set[Addr] = this match { - case NeutralStmt.App(label, targs, vargs, bargs) => vargs.toSet - case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => vargs.toSet + val free: Variables = this match { + case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) + case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) case NeutralStmt.If(cond, thn, els) => Set(cond) ++ thn.free ++ els.free case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) case NeutralStmt.Return(result) => Set(result) - case NeutralStmt.Reset(prompt, body) => body.free - case NeutralStmt.Shift(prompt, capt, k, body) => body.free - case NeutralStmt.Resume(k, body) => body.free + case NeutralStmt.Reset(prompt, body) => body.free - prompt.id + case NeutralStmt.Shift(prompt, capt, k, body) => (body.free - k.id) + prompt + case NeutralStmt.Resume(k, body) => Set(k) ++ body.free case NeutralStmt.Hole(span) => Set.empty } } @@ -192,6 +298,11 @@ object semantics { case NeutralStmt.Match(scrutinee, clauses, default) => "match" <+> parens(toDoc(scrutinee)) <+> braces(hcat(clauses.map { case (id, block) => toDoc(id) <> ":" <+> toDoc(block) })) <> (if (default.isDefined) "else" <+> toDoc(default.get) else emptyDoc) + case NeutralStmt.Jump(label, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + "jump" <+> toDoc(label) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) case NeutralStmt.App(label, targs, vargs, bargs) => // Format as: l1[T1, T2](r1, r2) toDoc(label) <> @@ -283,68 +394,6 @@ object NewNormalizer { normal => import semantics.* - // "effects" - case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { - def lookupValue(id: Id): Addr = values(id) - def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) - def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) - - def lookupComputation(id: Id): Computation = computations(id) - def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) - def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) - } - object Env { - def empty: Env = Env(Map.empty, Map.empty) - } - - def reify(scope: Scope, body: NeutralStmt): BasicBlock = { - var used = body.free - var filtered = Bindings.empty - // TODO implement properly - scope.bindings.toSeq.reverse.foreach { - // TODO for now we keep ALL definitions - case (addr, b: Binding.Def) => - used = used ++ b.free - filtered = (addr, b) :: filtered - case (addr, b: Binding.Rec) => - used = used ++ b.free - filtered = (addr, b) :: filtered - case (addr, s: Binding.Val) => - used = used ++ s.free - filtered = (addr, s) :: filtered - case (addr, v: Binding.Run) => - used = used ++ v.free - filtered = (addr, v) :: filtered - - // TODO if type is unit like, we can potentially drop this binding (but then we need to make up a "fresh" unit at use site) - case (addr, v: Binding.Let) if used.contains(addr) => - used = used ++ v.free - filtered = (addr, v) :: filtered - case (addr, v: Binding.Let) => () - } - - // we want to avoid turning tailcalls into non tail calls like - // - // val x = app(x) - // return x - // - // so we eta-reduce here. Can we achieve this by construction? - // TODO lastOption will go through the list AGAIN, let's see whether this causes performance problems - (filtered.lastOption, body) match { - case (Some((id1, Binding.Val(stmt))), NeutralStmt.Return(id2)) if id1 == id2 => - BasicBlock(filtered.init, stmt) - case (_, _) => - BasicBlock(filtered, body) - } - } - - def nested(prog: Scope ?=> NeutralStmt)(using scope: Scope): BasicBlock = { - // TODO parent code and parent store - val local = Scope(ListMap.empty, Map.empty, Some(scope)) - val result = prog(using local) - reify(local, result) - } - // "handlers" def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = prog(using env.bindValue(id, addr)) @@ -370,7 +419,7 @@ object NewNormalizer { normal => case MetaStack.Last(stack) => stack match { case Stack.Empty => NeutralStmt.Return(arg) case Stack.Static(tpe, apply) => apply(env)(scope)(arg) - case Stack.Dynamic(label) => NeutralStmt.App(label, List.empty, List(arg), Nil) + case Stack.Dynamic(label) => NeutralStmt.Jump(label, List.empty, List(arg), Nil) } } def push(tpe: ValueType)(f: Env ?=> Scope ?=> Addr => MetaStack => NeutralStmt): MetaStack = this match { @@ -400,7 +449,7 @@ object NewNormalizer { normal => // def k_6268 = (x_6267: Int_3) { // return x_6267 // } - case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App)) => f(this) + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => f(this) case body => val k = scope.define("k", Block(Nil, Nil, ValueParam(x, tpe) :: Nil, Nil, body)) f(MetaStack.Last(Stack.Dynamic(k))) @@ -503,6 +552,8 @@ object NewNormalizer { normal => // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => given Env = env.bindComputation(id, Computation.Def(id)) + + // TODO mark block as potentially recursive... scope.defineRecursive(id, evaluate(block), block.tpe, block.capt) bind(id, Computation.Def(id)) { evaluate(body, k) @@ -516,8 +567,7 @@ object NewNormalizer { normal => case Computation.Var(id) => k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate))) case Computation.Def(label) => - // TODO this should be "jump" - k.reify(NeutralStmt.App(label, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate))) case Computation.New(interface, operations) => sys error "Should not happen: app on new" } @@ -528,7 +578,7 @@ object NewNormalizer { normal => case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" case Computation.New(interface, operations) => val op = operations.collectFirst { case (id, label) if id == method => label }.get - k.reify(NeutralStmt.App(op, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.Jump(op, targs, vargs.map(evaluate), bargs.map(evaluate))) } case Stmt.If(cond, thn, els) => @@ -677,6 +727,8 @@ object NewNormalizer { normal => def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { case NeutralStmt.Return(result) => Stmt.Return(embedPure(result)) + case NeutralStmt.Jump(label, targs, vargs, bargs) => + Stmt.App(embedBlockVar(label), targs, vargs.map(embedPure), bargs.map(embedBlock)) case NeutralStmt.App(label, targs, vargs, bargs) => Stmt.App(embedBlockVar(label), targs, vargs.map(embedPure), bargs.map(embedBlock)) case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => From fd59e675aca21b5b454a3ca791f0e2b3d79ce0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Fri, 5 Sep 2025 16:31:09 +0200 Subject: [PATCH 010/123] Introduce closures --- .../effekt/core/optimizer/NewNormalizer.scala | 257 +++++++++++------- .../effekt/core/optimizer/Optimizer.scala | 16 +- 2 files changed, 168 insertions(+), 105 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b870c94aa..275925719 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -4,7 +4,7 @@ package optimizer import effekt.core.optimizer.NewNormalizer.MetaStack import effekt.source.Span -import effekt.core.optimizer.semantics.NeutralStmt +import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter @@ -122,14 +122,11 @@ object semantics { case _ => outer.flatMap(_.lookupValue(addr)) } - def define(hint: String, block: Block): Label = - val label = Id(hint) + def define(label: Label, block: Block): Unit = bindings = bindings.updated(label, Binding.Def(block)) - label - def defineRecursive(label: Label, block: Block, tpe: BlockType, capt: Captures): Label = + def defineRecursive(label: Label, block: Block, tpe: BlockType, capt: Captures): Unit = bindings = bindings.updated(label, Binding.Rec(block, tpe, capt)) - label def push(id: Id, stmt: NeutralStmt): Unit = bindings = bindings.updated(id, Binding.Val(stmt)) @@ -199,24 +196,10 @@ object semantics { def empty: Env = Env(Map.empty, Map.empty) } - case class Block(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], - // impl: List[Addr] => List[Computation] => Env => Scope => MetaStack => NeutralStmt, - body: BasicBlock) { - + case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) } - // TODO how do we detect mutually recursive toplevel functions??? - object Block { - def apply(tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam])(impl: List[Addr] => List[Computation] => Env => Scope => MetaStack => NeutralStmt)(using env: Env, scope: Scope): Block = { - // TODO also substitute type params... - val reified: BasicBlock = nested { - impl(vparams.map(p => p.id))(bparams.map(p => Computation.Var(p.id)))(env)(scope)(MetaStack.empty) - } - ??? - } - } - case class BasicBlock(bindings: Bindings, body: NeutralStmt) { val free: Variables = { var free = body.free @@ -235,17 +218,21 @@ object semantics { // Unknown case Var(id: Id) // Known function - case Def(label: Label) + case Def(closure: Closure) // Known object - case New(interface: BlockType.Interface, operations: List[(Id, Label)]) + case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) val free: Variables = this match { case Computation.Var(id) => Set(id) - case Computation.Def(label) => Set(label) - case Computation.New(interface, operations) => operations.map(_._2).toSet + case Computation.Def(closure) => closure.free + case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } } + case class Closure(label: Label, environment: List[Computation]) { + val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet + } + // Statements // ---------- enum NeutralStmt { @@ -346,18 +333,21 @@ object semantics { } def toDoc(block: Block): Doc = block match { - case Block(tparams, cparams, vparams, bparams, body) => + case Block(tparams, vparams, bparams, body) => (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) } def toDoc(comp: Computation): Doc = comp match { case Computation.Var(id) => toDoc(id) - case Computation.Def(label) => toDoc(label) + case Computation.Def(closure) => toDoc(closure) case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") } } + def toDoc(closure: Closure): Doc = closure match { + case Closure(label, env) => toDoc(label) <> brackets(hsep(env.map(toDoc), comma)) + } def toDoc(bindings: Bindings): Doc = hcat(bindings.map { @@ -408,24 +398,43 @@ object NewNormalizer { normal => // ------ enum Stack { case Empty - case Static(tpe: ValueType, apply: Env => Scope => Addr => NeutralStmt) + case Static(tpe: ValueType, apply: Env => Scope => Addr => MetaStack => NeutralStmt) case Dynamic(label: Label) + + def push(tpe: ValueType)(f: Env => Scope => Addr => MetaStack => NeutralStmt): Stack = this match { + case Stack.Empty => + Stack.Static(tpe, + env => scope => arg => k => f(env)(scope)(arg)(k) + ) + case Stack.Static(tpe2, f2) => + Stack.Static(tpe, + env => scope => arg => k => f(env)(scope)(arg)(k.push(tpe2)(f2)) + ) + case Stack.Dynamic(label) => ??? + } + } enum MetaStack { case Last(stack: Stack) - // case Segment(prompt: Prompt, stack: Stack, next: MetaStack) + case Segment(prompt: BlockParam, stack: Stack, next: MetaStack) + + val bound: List[BlockParam] = this match { + case MetaStack.Last(stack) => Nil + case MetaStack.Segment(prompt, stack, next) => prompt :: next.bound + } def ret(arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { - case MetaStack.Last(stack) => stack match { - case Stack.Empty => NeutralStmt.Return(arg) - case Stack.Static(tpe, apply) => apply(env)(scope)(arg) - case Stack.Dynamic(label) => NeutralStmt.Jump(label, List.empty, List(arg), Nil) - } + case MetaStack.Last(Stack.Empty) => NeutralStmt.Return(arg) + case MetaStack.Last(Stack.Static(tpe, apply)) => apply(env)(scope)(arg)(MetaStack.Last(Stack.Empty)) + case MetaStack.Last(Stack.Dynamic(label)) => NeutralStmt.Jump(label, Nil, List(arg), Nil) + + case MetaStack.Segment(prompt, Stack.Empty, next) => next.ret(arg) + case MetaStack.Segment(prompt, Stack.Static(tpe, apply), next) => ??? // apply(env)(scope)(arg)(MetaStack.Segment(prompt, Stack.Empty, next)) + case MetaStack.Segment(prompt, Stack.Dynamic(label), next) => ??? // MetaStack.Segment(prompt, Stack.Empty, next).reify(NeutralStmt.Jump(label, Nil, List(arg), Nil)) } - def push(tpe: ValueType)(f: Env ?=> Scope ?=> Addr => MetaStack => NeutralStmt): MetaStack = this match { - case MetaStack.Last(stack) => MetaStack.Last(Stack.Static(tpe, - env => scope => arg => f(using env)(using scope)(arg)(this) - )) + def push(tpe: ValueType)(f: Env => Scope => Addr => MetaStack => NeutralStmt): MetaStack = this match { + case MetaStack.Last(stack) => MetaStack.Last(stack.push(tpe)(f)) + case MetaStack.Segment(prompt, stack, next) => MetaStack.Segment(prompt, stack, next.push(tpe)(f)) } def reify(stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { case MetaStack.Last(stack) => stack match { @@ -434,40 +443,87 @@ object NewNormalizer { normal => case Stack.Static(tpe, apply) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(env)(scope)(tmp) + apply(env)(scope)(tmp)(MetaStack.Last(Stack.Empty)) // stack is already reified case Stack.Dynamic(label) => stmt } + case MetaStack.Segment(prompt, Stack.Empty, rest) => + NeutralStmt.Reset(prompt, nested { rest.reify(stmt) }) + case MetaStack.Segment(prompt, Stack.Static(tpe, apply), rest) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(env)(scope)(tmp)(MetaStack.Segment(prompt, Stack.Empty, rest)) + case MetaStack.Segment(prompt, Stack.Dynamic(label), rest) => ??? } def joinpoint(f: MetaStack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { case MetaStack.Last(stack) => stack match { case Stack.Static(tpe, apply) => val x = Id("x") - nested { scope ?=> apply(env)(scope)(x) } match { + nested { scope ?=> apply(env)(scope)(x)(MetaStack.Last(Stack.Empty)) } match { // Avoid trivial continuations like // def k_6268 = (x_6267: Int_3) { // return x_6267 // } case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => f(this) case body => - val k = scope.define("k", Block(Nil, Nil, ValueParam(x, tpe) :: Nil, Nil, body)) + val k = Id("k") + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, Nil, body)) f(MetaStack.Last(Stack.Dynamic(k))) } case Stack.Empty => f(this) case Stack.Dynamic(label) => f(this) } + case MetaStack.Segment(prompt, stack, rest) => + ??? } } object MetaStack { def empty: MetaStack = MetaStack.Last(Stack.Empty) } - def evaluate(block: core.Block)(using env: Env, scope: Scope): Computation = block match { + // used for potentially recursive definitions + def evaluateRecursive(id: Id, block: core.BlockLit, escaping: MetaStack)(using env: Env, scope: Scope): Computation = + block match { + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + val freshened = Id(id) + + // we keep the params as they are for now... + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + .bindComputation(id, Computation.Var(freshened)) + + val normalizedBlock = Block(tparams, vparams, bparams, nested { + evaluate(body, MetaStack.empty) + }) + + val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } + + scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams), block.tpe, block.capt) + Computation.Def(Closure(freshened, closureParams.map(p => Computation.Var(p.id)))) + } + + // the stack here is not the one this is run in, but the one the definition potentially escapes + def evaluate(block: core.Block, hint: String, escaping: MetaStack)(using env: Env, scope: Scope): Computation = block match { case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => env.lookupComputation(id) - case b @ core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => - Computation.Def(scope.define("f", evaluate(b))) + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + // we keep the params as they are for now... + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + + val normalizedBlock = Block(tparams, vparams, bparams, nested { + evaluate(body, MetaStack.empty) + }) + + val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } + + val f = Id(hint) + scope.define(f, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams)) + Computation.Def(Closure(f, closureParams.map(p => Computation.Var(p.id)))) + case core.Block.Unbox(pure) => ??? @@ -476,7 +532,7 @@ object NewNormalizer { normal => case Operation(name, tparams, cparams, vparams, bparams, body) => // Check whether the operation is already "just" an eta expansion and then use the identifier... // no need to create a fresh block literal - val eta: Option[Label] = + val eta: Option[Closure] = body match { case Stmt.App(BlockVar(id, _, _), targs, vargs, bargs) => def sameTargs = targs == tparams.map(t => ValueType.Var(t)) @@ -485,33 +541,24 @@ object NewNormalizer { normal => def isEta = sameTargs && sameVargs && sameBargs env.lookupComputation(id) match { - case Computation.Def(label) if isEta => Some(label) + // TODO what to do with closure environment + case Computation.Def(closure) if isEta => Some(closure) case _ => None } case _ => None } - val label = eta.getOrElse { - scope.define(name.name.name, - evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body): core.Block.BlockLit)) + val closure = eta.getOrElse { + evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), name.name.name, escaping) match { + case Computation.Def(closure) => closure + case _ => sys error "Should not happen" + } } - (name, label) + (name, closure) } Computation.New(interface, ops) } - def evaluate(block: core.Block.BlockLit)(using env: Env, scope: Scope): Block = - block match { - case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => - // we keep the params as they are for now... - given localEnv: Env = env - .bindValue(vparams.map(p => p.id -> p.id)) - .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - Block(tparams, cparams, vparams, bparams, nested { - evaluate(body, MetaStack.empty) - }) - } - def evaluate(expr: Expr)(using env: Env, scope: Scope): Addr = expr match { case Pure.ValueVar(id, annotatedType) => env.lookupValue(id) @@ -525,7 +572,7 @@ object NewNormalizer { normal => case DirectApp(f, targs, vargs, bargs) => assert(bargs.isEmpty) - scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate)) + scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", MetaStack.empty))) case Pure.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) @@ -542,7 +589,10 @@ object NewNormalizer { normal => case Stmt.Val(id, annotatedTpe, binding, body) => // This push can lead to an eta-redex (a superfluous push...) - evaluate(binding, k.push(annotatedTpe) { res => k => + evaluate(binding, k.push(annotatedTpe) { env => scope => res => k => + // TODO not sure this is necessary + given Env = env + given Scope = scope bind(id, res) { evaluate(body, k) } }) @@ -551,34 +601,32 @@ object NewNormalizer { normal => // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => - given Env = env.bindComputation(id, Computation.Def(id)) - - // TODO mark block as potentially recursive... - scope.defineRecursive(id, evaluate(block), block.tpe, block.capt) - bind(id, Computation.Def(id)) { - evaluate(body, k) - } + bind(id, evaluateRecursive(id, block, k)) { evaluate(body, k) } case Stmt.Def(id, block, body) => - bind(id, evaluate(block)) { evaluate(body, k) } + bind(id, evaluate(block, id.name.name, k)) { evaluate(body, k) } + // TODO here the stack passed to the blocks could be an empty one since we reify it anyways... case Stmt.App(callee, targs, vargs, bargs) => - evaluate(callee) match { + val escapingStack = MetaStack.empty + evaluate(callee, "f", escapingStack) match { case Computation.Var(id) => - k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate))) - case Computation.Def(label) => - k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) + case Computation.Def(Closure(label, environment)) => + k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment)) case Computation.New(interface, operations) => sys error "Should not happen: app on new" } case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => - evaluate(callee) match { + val escapingStack = MetaStack.empty + evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => - k.reify(NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate))) + k.reify(NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" case Computation.New(interface, operations) => - val op = operations.collectFirst { case (id, label) if id == method => label }.get - k.reify(NeutralStmt.Jump(op, targs, vargs.map(evaluate), bargs.map(evaluate))) + operations.collectFirst { case (id, Closure(label, environment)) if id == method => + k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment)) + }.get } case Stmt.If(cond, thn, els) => @@ -613,7 +661,7 @@ object NewNormalizer { normal => // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = Block(tparams, cparams, vparams, bparams, nested { + val block = Block(tparams, vparams, bparams, nested { evaluate(body, k) }) (id, block) @@ -648,18 +696,26 @@ object NewNormalizer { normal => assert(Set(cparam) == k2.capt, "At least for now these need to be the same") k.reify(NeutralStmt.Shift(p, cparam, k2, neutralBody)) case Stmt.Shift(_, _) => ??? + //case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => + // // TODO is Var correct here?? Probably needs to be a new computation value... + // // but shouldn't it be a fresh prompt each time? + // val p = Id(prompt.id) + // val neutralBody = { + // given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) + // nested { + // evaluate(body, MetaStack.empty) + // } + // } + // // TODO implement properly + // k.reify(NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) + + case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => - // TODO is Var correct here?? Probably needs to be a new computation value... - // but shouldn't it be a fresh prompt each time? val p = Id(prompt.id) - val neutralBody = { - given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) - nested { - evaluate(body, MetaStack.empty) - } - } - // TODO implement properly - k.reify(NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) + // TODO is Var correct here?? Probably needs to be a new computation value... + given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) + evaluate(body, MetaStack.Segment(BlockParam(p, prompt.tpe, prompt.capt), Stack.Empty, k)) + case Stmt.Reset(_) => ??? case Stmt.Resume(k2, body) => val r = env.lookupComputation(k2.id) match { @@ -678,9 +734,9 @@ object NewNormalizer { normal => val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } val toplevelEnv = Env.empty // user defined functions - .bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(defn.id))) + .bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(Closure(defn.id, Nil)))) // async extern functions - .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Def(defn.id))) + .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Def(Closure(defn.id, Nil)))) val typingContext = TypingContext(Map.empty, mod.definitions.collect { case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) @@ -705,7 +761,7 @@ object NewNormalizer { normal => val result = evaluate(body, MetaStack.empty) debug(s"---------------------") - val block = Block(tparams, cparams, vparams, bparams, reify(scope, result)) + val block = Block(tparams, vparams, bparams, reify(scope, result)) debug(PrettyPrinter.show(block)) debug(s"---------------------") @@ -784,9 +840,11 @@ object NewNormalizer { normal => def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { case Computation.Var(id) => embedBlockVar(id) - case Computation.Def(label) => embedBlockVar(label) + case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) + case Computation.Def(Closure(label, environment)) => ??? // TODO eta expand case Computation.New(interface, operations) => - val ops = operations.map { case (id, label) => + // TODO deal with environment + val ops = operations.map { case (id, Closure(label, environment)) => G.blocks(label) match { case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => val tparams2 = tparams.map(t => Id(t)) @@ -804,7 +862,12 @@ object NewNormalizer { normal => } def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { - case Block(tparams, cparams, vparams, bparams, body) => + case Block(tparams, vparams, bparams, body) => + val cparams = bparams.map { + case BlockParam(id, tpe, captures) => + assert(captures.size == 1) + captures.head + } core.Block.BlockLit(tparams, cparams, vparams, bparams, embedStmt(body)(using G.bindValues(vparams).bindComputations(bparams))) } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 2c5f0525b..97efebdf7 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -33,8 +33,8 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer.run(tree) } Normalizer.assertNormal(tree) // println(util.show(tree)) - tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer.run(tree) } - Normalizer.assertNormal(tree) + // tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer.run(tree) } + // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) // tree = Context.timed("old-normalizer-1", source.name) { Normalizer.normalize(Set(mainSymbol), tree, 0) } @@ -44,12 +44,12 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { // Normalizer.assertNormal(tree) // (2) lift static arguments - tree = Context.timed("static-argument-transformation", source.name) { - StaticArguments.transform(mainSymbol, tree) - } - - tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } - Normalizer.assertNormal(tree) + // tree = Context.timed("static-argument-transformation", source.name) { + // StaticArguments.transform(mainSymbol, tree) + // } + // + // tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } + // Normalizer.assertNormal(tree) // // def normalize(m: ModuleDecl) = { // val anfed = BindSubexpressions.transform(m) From ca63185a2e09c5a29b3bd17e714bf1cb7ebbabfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Fri, 5 Sep 2025 19:56:24 +0200 Subject: [PATCH 011/123] Rename Stack to Frame --- .../effekt/core/optimizer/NewNormalizer.scala | 117 ++++++++++-------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 275925719..67b311df1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,7 +2,7 @@ package effekt package core package optimizer -import effekt.core.optimizer.NewNormalizer.MetaStack +import effekt.core.optimizer.NewNormalizer.Stack import effekt.source.Span import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } @@ -396,71 +396,78 @@ object NewNormalizer { normal => // Stacks // ------ - enum Stack { - case Empty - case Static(tpe: ValueType, apply: Env => Scope => Addr => MetaStack => NeutralStmt) + enum Frame { + case Return + case Static(tpe: ValueType, apply: Env => Scope => Addr => Stack => NeutralStmt) case Dynamic(label: Label) - def push(tpe: ValueType)(f: Env => Scope => Addr => MetaStack => NeutralStmt): Stack = this match { - case Stack.Empty => - Stack.Static(tpe, + def push(tpe: ValueType)(f: Env => Scope => Addr => Stack => NeutralStmt): Frame = this match { + case Frame.Return => + Frame.Static(tpe, env => scope => arg => k => f(env)(scope)(arg)(k) ) - case Stack.Static(tpe2, f2) => - Stack.Static(tpe, + case Frame.Static(tpe2, f2) => + Frame.Static(tpe, env => scope => arg => k => f(env)(scope)(arg)(k.push(tpe2)(f2)) ) - case Stack.Dynamic(label) => ??? + case Frame.Dynamic(label) => ??? } + // maybe, for once it is simpler to decompose stacks like + // + // f, (p, f) :: (p, f) :: Nil + // + // where the frame on the reset is the one AFTER the prompt NOT BEFORE! + + } - enum MetaStack { - case Last(stack: Stack) - case Segment(prompt: BlockParam, stack: Stack, next: MetaStack) + enum Stack { + case Last(frame: Frame) + case Reset(prompt: BlockParam, frame: Frame, next: Stack) val bound: List[BlockParam] = this match { - case MetaStack.Last(stack) => Nil - case MetaStack.Segment(prompt, stack, next) => prompt :: next.bound + case Stack.Last(stack) => Nil + case Stack.Reset(prompt, stack, next) => prompt :: next.bound } def ret(arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { - case MetaStack.Last(Stack.Empty) => NeutralStmt.Return(arg) - case MetaStack.Last(Stack.Static(tpe, apply)) => apply(env)(scope)(arg)(MetaStack.Last(Stack.Empty)) - case MetaStack.Last(Stack.Dynamic(label)) => NeutralStmt.Jump(label, Nil, List(arg), Nil) + case Stack.Last(Frame.Return) => NeutralStmt.Return(arg) + case Stack.Last(Frame.Static(tpe, apply)) => apply(env)(scope)(arg)(Stack.Last(Frame.Return)) + case Stack.Last(Frame.Dynamic(label)) => NeutralStmt.Jump(label, Nil, List(arg), Nil) - case MetaStack.Segment(prompt, Stack.Empty, next) => next.ret(arg) - case MetaStack.Segment(prompt, Stack.Static(tpe, apply), next) => ??? // apply(env)(scope)(arg)(MetaStack.Segment(prompt, Stack.Empty, next)) - case MetaStack.Segment(prompt, Stack.Dynamic(label), next) => ??? // MetaStack.Segment(prompt, Stack.Empty, next).reify(NeutralStmt.Jump(label, Nil, List(arg), Nil)) + case Stack.Reset(prompt, Frame.Return, next) => next.ret(arg) + case Stack.Reset(prompt, Frame.Static(tpe, apply), next) => ??? // apply(env)(scope)(arg)(MetaStack.Segment(prompt, Stack.Empty, next)) + case Stack.Reset(prompt, Frame.Dynamic(label), next) => ??? // MetaStack.Segment(prompt, Stack.Empty, next).reify(NeutralStmt.Jump(label, Nil, List(arg), Nil)) } - def push(tpe: ValueType)(f: Env => Scope => Addr => MetaStack => NeutralStmt): MetaStack = this match { - case MetaStack.Last(stack) => MetaStack.Last(stack.push(tpe)(f)) - case MetaStack.Segment(prompt, stack, next) => MetaStack.Segment(prompt, stack, next.push(tpe)(f)) + def push(tpe: ValueType)(f: Env => Scope => Addr => Stack => NeutralStmt): Stack = this match { + case Stack.Last(frame) => Stack.Last(frame.push(tpe)(f)) + case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, next.push(tpe)(f)) } def reify(stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { - case MetaStack.Last(stack) => stack match { - case Stack.Empty => stmt + case Stack.Last(frame) => frame match { + case Frame.Return => stmt // [[ val x = { val y = stmt1; stmt2 }; stmt3 ]] = [[ val y = stmt1; val x = stmt2; stmt3 ]] - case Stack.Static(tpe, apply) => + case Frame.Static(tpe, apply) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(env)(scope)(tmp)(MetaStack.Last(Stack.Empty)) + apply(env)(scope)(tmp)(Stack.Last(Frame.Return)) // stack is already reified - case Stack.Dynamic(label) => + case Frame.Dynamic(label) => stmt } - case MetaStack.Segment(prompt, Stack.Empty, rest) => + case Stack.Reset(prompt, Frame.Return, rest) => NeutralStmt.Reset(prompt, nested { rest.reify(stmt) }) - case MetaStack.Segment(prompt, Stack.Static(tpe, apply), rest) => + case Stack.Reset(prompt, Frame.Static(tpe, apply), rest) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(env)(scope)(tmp)(MetaStack.Segment(prompt, Stack.Empty, rest)) - case MetaStack.Segment(prompt, Stack.Dynamic(label), rest) => ??? + apply(env)(scope)(tmp)(Stack.Reset(prompt, Frame.Return, rest)) + case Stack.Reset(prompt, Frame.Dynamic(label), rest) => ??? } - def joinpoint(f: MetaStack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { - case MetaStack.Last(stack) => stack match { - case Stack.Static(tpe, apply) => + def joinpoint(f: Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { + case Stack.Last(stack) => stack match { + case Frame.Static(tpe, apply) => val x = Id("x") - nested { scope ?=> apply(env)(scope)(x)(MetaStack.Last(Stack.Empty)) } match { + nested { scope ?=> apply(env)(scope)(x)(Stack.Last(Frame.Return)) } match { // Avoid trivial continuations like // def k_6268 = (x_6267: Int_3) { // return x_6267 @@ -469,21 +476,21 @@ object NewNormalizer { normal => case body => val k = Id("k") scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, Nil, body)) - f(MetaStack.Last(Stack.Dynamic(k))) + f(Stack.Last(Frame.Dynamic(k))) } - case Stack.Empty => f(this) - case Stack.Dynamic(label) => f(this) + case Frame.Return => f(this) + case Frame.Dynamic(label) => f(this) } - case MetaStack.Segment(prompt, stack, rest) => + case Stack.Reset(prompt, stack, rest) => ??? } } - object MetaStack { - def empty: MetaStack = MetaStack.Last(Stack.Empty) + object Stack { + def empty: Stack = Stack.Last(Frame.Return) } // used for potentially recursive definitions - def evaluateRecursive(id: Id, block: core.BlockLit, escaping: MetaStack)(using env: Env, scope: Scope): Computation = + def evaluateRecursive(id: Id, block: core.BlockLit, escaping: Stack)(using env: Env, scope: Scope): Computation = block match { case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => val freshened = Id(id) @@ -495,7 +502,7 @@ object NewNormalizer { normal => .bindComputation(id, Computation.Var(freshened)) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, MetaStack.empty) + evaluate(body, Stack.empty) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -505,7 +512,7 @@ object NewNormalizer { normal => } // the stack here is not the one this is run in, but the one the definition potentially escapes - def evaluate(block: core.Block, hint: String, escaping: MetaStack)(using env: Env, scope: Scope): Computation = block match { + def evaluate(block: core.Block, hint: String, escaping: Stack)(using env: Env, scope: Scope): Computation = block match { case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => env.lookupComputation(id) case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => @@ -515,7 +522,7 @@ object NewNormalizer { normal => .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, MetaStack.empty) + evaluate(body, Stack.empty) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -572,7 +579,7 @@ object NewNormalizer { normal => case DirectApp(f, targs, vargs, bargs) => assert(bargs.isEmpty) - scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", MetaStack.empty))) + scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.empty))) case Pure.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) @@ -582,7 +589,7 @@ object NewNormalizer { normal => } // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) - def evaluate(stmt: Stmt, k: MetaStack)(using env: Env, scope: Scope): NeutralStmt = stmt match { + def evaluate(stmt: Stmt, k: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { case Stmt.Return(expr) => k.ret(evaluate(expr)) @@ -608,7 +615,7 @@ object NewNormalizer { normal => // TODO here the stack passed to the blocks could be an empty one since we reify it anyways... case Stmt.App(callee, targs, vargs, bargs) => - val escapingStack = MetaStack.empty + val escapingStack = Stack.empty evaluate(callee, "f", escapingStack) match { case Computation.Var(id) => k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) @@ -618,7 +625,7 @@ object NewNormalizer { normal => } case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => - val escapingStack = MetaStack.empty + val escapingStack = Stack.empty evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => k.reify(NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) @@ -690,7 +697,7 @@ object NewNormalizer { normal => val neutralBody = { given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) nested { - evaluate(body, MetaStack.empty) + evaluate(body, Stack.empty) } } assert(Set(cparam) == k2.capt, "At least for now these need to be the same") @@ -714,7 +721,7 @@ object NewNormalizer { normal => val p = Id(prompt.id) // TODO is Var correct here?? Probably needs to be a new computation value... given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) - evaluate(body, MetaStack.Segment(BlockParam(p, prompt.tpe, prompt.capt), Stack.Empty, k)) + evaluate(body, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), Frame.Return, k)) case Stmt.Reset(_) => ??? case Stmt.Resume(k2, body) => @@ -724,7 +731,7 @@ object NewNormalizer { normal => } // TODO implement properly k.reify(NeutralStmt.Resume(r, nested { - evaluate(body, MetaStack.empty) + evaluate(body, Stack.empty) })) } @@ -758,7 +765,7 @@ object NewNormalizer { normal => .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) given scope: Scope = Scope.empty - val result = evaluate(body, MetaStack.empty) + val result = evaluate(body, Stack.empty) debug(s"---------------------") val block = Block(tparams, vparams, bparams, reify(scope, result)) From f91bef00b8dcabc5a1a4c9995d7fd544f0db38d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Mon, 8 Sep 2025 10:16:51 +0200 Subject: [PATCH 012/123] Reify stacks --- .../effekt/core/optimizer/NewNormalizer.scala | 214 ++++++++---------- 1 file changed, 98 insertions(+), 116 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 67b311df1..61b32eee2 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -54,6 +54,7 @@ object semantics { case Literal(value: Any, annotatedType: ValueType) case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) + // this could not only compute free variables, but also usage information to guide the inliner (see "secrets of the ghc inliner") val free: Variables = this match { // case Value.Var(id, annotatedType) => Variables.empty case Value.Extern(id, targs, vargs) => vargs.toSet @@ -135,7 +136,7 @@ object semantics { def empty: Scope = new Scope(ListMap.empty, Map.empty, None) } - def reify(scope: Scope, body: NeutralStmt): BasicBlock = { + def reifyBindings(scope: Scope, body: NeutralStmt): BasicBlock = { var used = body.free var filtered = Bindings.empty // TODO implement properly @@ -180,7 +181,7 @@ object semantics { // TODO parent code and parent store val local = Scope(ListMap.empty, Map.empty, Some(scope)) val result = prog(using local) - reify(local, result) + reifyBindings(local, result) } case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { @@ -249,7 +250,7 @@ object semantics { // scrutinee is unknown case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) - // what's actually unknown here? + // body is stuck case Reset(prompt: BlockParam, body: BasicBlock) // prompt / context is unknown case Shift(prompt: Prompt, kCapt: Capture, k: BlockParam, body: BasicBlock) @@ -398,95 +399,61 @@ object NewNormalizer { normal => // ------ enum Frame { case Return - case Static(tpe: ValueType, apply: Env => Scope => Addr => Stack => NeutralStmt) + case Static(apply: Env => Scope => Addr => Stack => NeutralStmt) case Dynamic(label: Label) - def push(tpe: ValueType)(f: Env => Scope => Addr => Stack => NeutralStmt): Frame = this match { - case Frame.Return => - Frame.Static(tpe, - env => scope => arg => k => f(env)(scope)(arg)(k) - ) - case Frame.Static(tpe2, f2) => - Frame.Static(tpe, - env => scope => arg => k => f(env)(scope)(arg)(k.push(tpe2)(f2)) - ) - case Frame.Dynamic(label) => ??? + def ret(ks: Stack, arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { + case Frame.Return => ks match { + case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Reset(p, k, ks) => k.ret(ks, arg) + } + case Frame.Static(apply) => apply(env)(scope)(arg)(ks) + case Frame.Dynamic(label) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), Nil) } } - // maybe, for once it is simpler to decompose stacks like - // - // f, (p, f) :: (p, f) :: Nil - // - // where the frame on the reset is the one AFTER the prompt NOT BEFORE! - + def push(f: Env => Scope => Addr => Frame => Stack => NeutralStmt): Frame = + Frame.Static( + env => scope => arg => ks => f(env)(scope)(arg)(this)(ks) + ) + def joinpoint(ks: Stack)(f: Frame => Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = ??? } - enum Stack { - case Last(frame: Frame) - case Reset(prompt: BlockParam, frame: Frame, next: Stack) - - val bound: List[BlockParam] = this match { - case Stack.Last(stack) => Nil - case Stack.Reset(prompt, stack, next) => prompt :: next.bound - } - def ret(arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { - case Stack.Last(Frame.Return) => NeutralStmt.Return(arg) - case Stack.Last(Frame.Static(tpe, apply)) => apply(env)(scope)(arg)(Stack.Last(Frame.Return)) - case Stack.Last(Frame.Dynamic(label)) => NeutralStmt.Jump(label, Nil, List(arg), Nil) + def reify(k: Frame, ks: Stack)(stmt: (Env, Scope) ?=> NeutralStmt)(using Env, Scope): NeutralStmt = + reify(ks) { reify(k) { stmt } } - case Stack.Reset(prompt, Frame.Return, next) => next.ret(arg) - case Stack.Reset(prompt, Frame.Static(tpe, apply), next) => ??? // apply(env)(scope)(arg)(MetaStack.Segment(prompt, Stack.Empty, next)) - case Stack.Reset(prompt, Frame.Dynamic(label), next) => ??? // MetaStack.Segment(prompt, Stack.Empty, next).reify(NeutralStmt.Jump(label, Nil, List(arg), Nil)) - } - def push(tpe: ValueType)(f: Env => Scope => Addr => Stack => NeutralStmt): Stack = this match { - case Stack.Last(frame) => Stack.Last(frame.push(tpe)(f)) - case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, next.push(tpe)(f)) - } - def reify(stmt: NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { - case Stack.Last(frame) => frame match { - case Frame.Return => stmt - // [[ val x = { val y = stmt1; stmt2 }; stmt3 ]] = [[ val y = stmt1; val x = stmt2; stmt3 ]] - case Frame.Static(tpe, apply) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - apply(env)(scope)(tmp)(Stack.Last(Frame.Return)) - // stack is already reified - case Frame.Dynamic(label) => - stmt - } - case Stack.Reset(prompt, Frame.Return, rest) => - NeutralStmt.Reset(prompt, nested { rest.reify(stmt) }) - case Stack.Reset(prompt, Frame.Static(tpe, apply), rest) => + def reify(k: Frame)(stmt: (Env, Scope) ?=> NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = + k match { + case Frame.Return => stmt + case Frame.Static(apply) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(env)(scope)(tmp)(Stack.Reset(prompt, Frame.Return, rest)) - case Stack.Reset(prompt, Frame.Dynamic(label), rest) => ??? + apply(env)(scope)(tmp)(Stack.Empty) + case Frame.Dynamic(label) => stmt } - def joinpoint(f: Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = this match { - case Stack.Last(stack) => stack match { - case Frame.Static(tpe, apply) => - val x = Id("x") - nested { scope ?=> apply(env)(scope)(x)(Stack.Last(Frame.Return)) } match { - // Avoid trivial continuations like - // def k_6268 = (x_6267: Int_3) { - // return x_6267 - // } - case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => f(this) - case body => - val k = Id("k") - scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, Nil, body)) - f(Stack.Last(Frame.Dynamic(k))) - } - case Frame.Return => f(this) - case Frame.Dynamic(label) => f(this) - } - case Stack.Reset(prompt, stack, rest) => - ??? + + @tailrec + def reify(ks: Stack)(stmt: (Env, Scope) ?=> NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = + ks match { + case Stack.Empty => stmt + case Stack.Reset(prompt, frame, next) => + reify(next) { reify(frame) { NeutralStmt.Reset(prompt, nested { stmt }) }} + } + + + // maybe, for once it is simpler to decompose stacks like + // + // f, (p, f) :: (p, f) :: Nil + // + // where the frame on the reset is the one AFTER the prompt NOT BEFORE! + enum Stack { + case Empty + case Reset(prompt: BlockParam, frame: Frame, next: Stack) + + lazy val bound: List[BlockParam] = this match { + case Stack.Empty => Nil + case Stack.Reset(prompt, stack, next) => prompt :: next.bound } - } - object Stack { - def empty: Stack = Stack.Last(Frame.Return) } // used for potentially recursive definitions @@ -502,7 +469,7 @@ object NewNormalizer { normal => .bindComputation(id, Computation.Var(freshened)) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Stack.empty) + evaluate(body, Frame.Return, Stack.Empty) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -522,7 +489,7 @@ object NewNormalizer { normal => .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Stack.empty) + evaluate(body, Frame.Return, Stack.Empty) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -579,7 +546,7 @@ object NewNormalizer { normal => case DirectApp(f, targs, vargs, bargs) => assert(bargs.isEmpty) - scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.empty))) + scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Empty))) case Pure.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) @@ -589,64 +556,77 @@ object NewNormalizer { normal => } // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) - def evaluate(stmt: Stmt, k: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { + def evaluate(stmt: Stmt, k: Frame, ks: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { case Stmt.Return(expr) => - k.ret(evaluate(expr)) + k.ret(ks, evaluate(expr)) case Stmt.Val(id, annotatedTpe, binding, body) => // This push can lead to an eta-redex (a superfluous push...) - evaluate(binding, k.push(annotatedTpe) { env => scope => res => k => + evaluate(binding, k.push { env => scope => res => k => ks => // TODO not sure this is necessary given Env = env given Scope = scope - bind(id, res) { evaluate(body, k) } - }) + bind(id, res) { evaluate(body, k, ks) } + }, ks) case Stmt.Let(id, annotatedTpe, binding, body) => - bind(id, evaluate(binding)) { evaluate(body, k) } + bind(id, evaluate(binding)) { evaluate(body, k, ks) } // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => - bind(id, evaluateRecursive(id, block, k)) { evaluate(body, k) } + bind(id, evaluateRecursive(id, block, ks)) { evaluate(body, k, ks) } case Stmt.Def(id, block, body) => - bind(id, evaluate(block, id.name.name, k)) { evaluate(body, k) } + bind(id, evaluate(block, id.name.name, ks)) { evaluate(body, k, ks) } + + // TODO actually remove a cut instead of binding the block argument + // alternatively, we could keep some information about linearity. + // For example, here we are in an active position. + case Stmt.App(BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => + // TODO also bind type arguments in environment + // TODO substitute cparams??? + val newEnv = env + .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) + .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) + + evaluate(body, k, ks)(using newEnv, scope) // TODO here the stack passed to the blocks could be an empty one since we reify it anyways... case Stmt.App(callee, targs, vargs, bargs) => - val escapingStack = Stack.empty + val escapingStack = Stack.Empty evaluate(callee, "f", escapingStack) match { case Computation.Var(id) => - k.reify(NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) + reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => - k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment)) + val args = vargs.map(evaluate) + reify(k, ks) { NeutralStmt.Jump(label, targs, args, bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } case Computation.New(interface, operations) => sys error "Should not happen: app on new" } case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => - val escapingStack = Stack.empty + val escapingStack = Stack.Empty evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => - k.reify(NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)))) + reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" case Computation.New(interface, operations) => operations.collectFirst { case (id, Closure(label, environment)) if id == method => - k.reify(NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment)) + reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get } case Stmt.If(cond, thn, els) => val sc = evaluate(cond) scope.lookupValue(sc) match { - case Some(Value.Literal(true, _)) => evaluate(thn, k) - case Some(Value.Literal(false, _)) => evaluate(els, k) + case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) + case Some(Value.Literal(false, _)) => evaluate(els, k, ks) case _ => - k.joinpoint { k => + k.joinpoint(ks) { k => ks => NeutralStmt.If(sc, nested { - evaluate(thn, k) + evaluate(thn, k, ks) }, nested { - evaluate(els, k) + evaluate(els, k, ks) }) } } @@ -658,22 +638,22 @@ object NewNormalizer { normal => // TODO substitute types (or bind them in the env)! clauses.collectFirst { case (tpe, BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => - bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k) } + bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k, ks) } }.getOrElse { - evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k) + evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) } case _ => - k.joinpoint { k => + k.joinpoint(ks) { k => ks => NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) val block = Block(tparams, vparams, bparams, nested { - evaluate(body, k) + evaluate(body, k, ks) }) (id, block) }, - default.map { stmt => nested { evaluate(stmt, k) } }) + default.map { stmt => nested { evaluate(stmt, k, ks) } }) } } @@ -697,11 +677,11 @@ object NewNormalizer { normal => val neutralBody = { given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) nested { - evaluate(body, Stack.empty) + evaluate(body, Frame.Return, Stack.Empty) } } assert(Set(cparam) == k2.capt, "At least for now these need to be the same") - k.reify(NeutralStmt.Shift(p, cparam, k2, neutralBody)) + reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } case Stmt.Shift(_, _) => ??? //case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => // // TODO is Var correct here?? Probably needs to be a new computation value... @@ -710,7 +690,7 @@ object NewNormalizer { normal => // val neutralBody = { // given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) // nested { - // evaluate(body, MetaStack.empty) + // evaluate(body, MetaStack.Empty) // } // } // // TODO implement properly @@ -721,7 +701,7 @@ object NewNormalizer { normal => val p = Id(prompt.id) // TODO is Var correct here?? Probably needs to be a new computation value... given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) - evaluate(body, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), Frame.Return, k)) + evaluate(body, Frame.Return, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), k, ks)) case Stmt.Reset(_) => ??? case Stmt.Resume(k2, body) => @@ -730,9 +710,11 @@ object NewNormalizer { normal => case _ => ??? } // TODO implement properly - k.reify(NeutralStmt.Resume(r, nested { - evaluate(body, Stack.empty) - })) + reify(k, ks) { + NeutralStmt.Resume(r, nested { + evaluate(body, Frame.Return, Stack.Empty) + }) + } } def run(mod: ModuleDecl): ModuleDecl = { @@ -765,10 +747,10 @@ object NewNormalizer { normal => .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) given scope: Scope = Scope.empty - val result = evaluate(body, Stack.empty) + val result = evaluate(body, Frame.Return, Stack.Empty) debug(s"---------------------") - val block = Block(tparams, vparams, bparams, reify(scope, result)) + val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) debug(PrettyPrinter.show(block)) debug(s"---------------------") From 763ca4f6a230f7193b8b9bf415bb9a5dc26f1d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Mon, 8 Sep 2025 11:35:02 +0200 Subject: [PATCH 013/123] Notes from the meeting --- .../shared/src/main/scala/effekt/core/Tree.scala | 7 +++++++ .../effekt/core/optimizer/NewNormalizer.scala | 16 +++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/Tree.scala b/effekt/shared/src/main/scala/effekt/core/Tree.scala index b3419f3cb..ca0f8462a 100644 --- a/effekt/shared/src/main/scala/effekt/core/Tree.scala +++ b/effekt/shared/src/main/scala/effekt/core/Tree.scala @@ -211,6 +211,13 @@ enum Block extends Tree { val capt: Captures = Type.inferCapt(this) def show: String = util.show(this) + + this match { + case Block.BlockVar(id, annotatedTpe, annotatedCapt) => () + case Block.BlockLit(tparams, cparams, vparams, bparams, body) => assert(cparams.size == bparams.size) + case Block.Unbox(pure) => () + case Block.New(impl) => () + } } export Block.* diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 61b32eee2..c51414a2c 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -42,6 +42,7 @@ object semantics { type Label = Id type Prompt = Id + // this could not only compute free variables, but also usage information to guide the inliner (see "secrets of the ghc inliner") type Variables = Set[Id] def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet @@ -54,7 +55,6 @@ object semantics { case Literal(value: Any, annotatedType: ValueType) case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) - // this could not only compute free variables, but also usage information to guide the inliner (see "secrets of the ghc inliner") val free: Variables = this match { // case Value.Var(id, annotatedType) => Variables.empty case Value.Extern(id, targs, vargs) => vargs.toSet @@ -90,7 +90,7 @@ object semantics { */ class Scope( var bindings: ListMap[Id, Binding], - var inverse: Map[Value, Id], + var inverse: Map[Value, Addr], outer: Option[Scope] ) { // floating values to the top is not always beneficial. For example @@ -220,6 +220,9 @@ object semantics { case Var(id: Id) // Known function case Def(closure: Closure) + + // case Inline(body: core.BlockLit, closure: Env) + // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) @@ -237,9 +240,9 @@ object semantics { // Statements // ---------- enum NeutralStmt { - // continuation is unknown + // context (continuation) is unknown case Return(result: Id) - // callee is unknown or we do not want to inline + // callee is unknown case App(callee: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) // Known jump, but we do not want to inline case Jump(label: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) @@ -381,7 +384,8 @@ object semantics { /** * A new normalizer that is conservative (avoids code bloat) */ -object NewNormalizer { normal => +object NewNormalizer { + import semantics.* @@ -604,6 +608,8 @@ object NewNormalizer { normal => case Computation.New(interface, operations) => sys error "Should not happen: app on new" } + // case Stmt.Invoke(New) + case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => val escapingStack = Stack.Empty evaluate(callee, "o", escapingStack) match { From 3862bcbe9b3771d6b658ed5f064baf6dd12c6dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Tue, 9 Sep 2025 11:41:03 +0200 Subject: [PATCH 014/123] Draft inline and joinpoints --- .../effekt/core/optimizer/NewNormalizer.scala | 98 +++++++++++++------ .../effekt/core/optimizer/Optimizer.scala | 4 +- 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index c51414a2c..8cb7fcce2 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,7 +2,6 @@ package effekt package core package optimizer -import effekt.core.optimizer.NewNormalizer.Stack import effekt.source.Span import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } @@ -189,7 +188,7 @@ object semantics { def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) - def lookupComputation(id: Id): Computation = computations(id) + def lookupComputation(id: Id): Computation = computations.getOrElse(id, sys error s"Unknown computation: ${util.show(id)} -- env: ${computations.map { case (id, comp) => s"${util.show(id)}: $comp" }.mkString("\n") }") def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) } @@ -221,14 +220,16 @@ object semantics { // Known function case Def(closure: Closure) - // case Inline(body: core.BlockLit, closure: Env) + // TODO it looks like this was not a good idea... Many operations (like embed) are not supported on Inline + case Inline(body: core.BlockLit, closure: Env) // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) - val free: Variables = this match { + lazy val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free + case Computation.Inline(body, closure) => ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } } @@ -345,6 +346,7 @@ object semantics { def toDoc(comp: Computation): Doc = comp match { case Computation.Var(id) => toDoc(id) case Computation.Def(closure) => toDoc(closure) + case Computation.Inline(block, env) => ??? case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") } @@ -384,8 +386,7 @@ object semantics { /** * A new normalizer that is conservative (avoids code bloat) */ -object NewNormalizer { - +class NewNormalizer(shouldInline: Id => Boolean) { import semantics.* @@ -403,45 +404,75 @@ object NewNormalizer { // ------ enum Frame { case Return - case Static(apply: Env => Scope => Addr => Stack => NeutralStmt) - case Dynamic(label: Label) + case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) + case Dynamic(closure: Closure) - def ret(ks: Stack, arg: Addr)(using env: Env, scope: Scope): NeutralStmt = this match { + def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { case Frame.Return => ks match { case Stack.Empty => NeutralStmt.Return(arg) case Stack.Reset(p, k, ks) => k.ret(ks, arg) } - case Frame.Static(apply) => apply(env)(scope)(arg)(ks) - case Frame.Dynamic(label) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), Nil) } + case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) + case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } } - def push(f: Env => Scope => Addr => Frame => Stack => NeutralStmt): Frame = - Frame.Static( - env => scope => arg => ks => f(env)(scope)(arg)(this)(ks) - ) + // pushing purposefully does not abstract over env (it closes over it!) + def push(tpe: ValueType)(f: Scope => Addr => Frame => Stack => NeutralStmt): Frame = + Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) + } + + def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { - def joinpoint(ks: Stack)(f: Frame => Stack => NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = ??? + def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { + case Frame.Static(tpe, apply) => + val x = Id("x") + val body = nested { scope ?=> apply(scope)(x)(Stack.Empty) } + + val k = Id("k") + val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + + + Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) + case Frame.Return => k + case Frame.Dynamic(label) => k + } + + def reifyStack(ks: Stack): Stack = ks match { + case Stack.Empty => Stack.Empty + case Stack.Reset(prompt, frame, next) => + Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) + } + f(reifyFrame(k, ks))(reifyStack(ks)) } - def reify(k: Frame, ks: Stack)(stmt: (Env, Scope) ?=> NeutralStmt)(using Env, Scope): NeutralStmt = + def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = reify(ks) { reify(k) { stmt } } - def reify(k: Frame)(stmt: (Env, Scope) ?=> NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = + def reify(k: Frame)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = k match { case Frame.Return => stmt - case Frame.Static(apply) => + case Frame.Static(tpe, apply) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(env)(scope)(tmp)(Stack.Empty) - case Frame.Dynamic(label) => stmt + apply(scope)(tmp)(Stack.Empty) + case Frame.Dynamic(Closure(label, closure)) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + NeutralStmt.Jump(label, Nil, List(tmp), closure) } @tailrec - def reify(ks: Stack)(stmt: (Env, Scope) ?=> NeutralStmt)(using env: Env, scope: Scope): NeutralStmt = + final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = ks match { case Stack.Empty => stmt + // only reify reset if p is free in body case Stack.Reset(prompt, frame, next) => - reify(next) { reify(frame) { NeutralStmt.Reset(prompt, nested { stmt }) }} + reify(next) { reify(frame) { + val body = nested { stmt } + if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) + else stmt // TODO this runs normalization a second time in the outer scope! + }} } @@ -567,9 +598,7 @@ object NewNormalizer { case Stmt.Val(id, annotatedTpe, binding, body) => // This push can lead to an eta-redex (a superfluous push...) - evaluate(binding, k.push { env => scope => res => k => ks => - // TODO not sure this is necessary - given Env = env + evaluate(binding, k.push(annotatedTpe) { scope => res => k => ks => given Scope = scope bind(id, res) { evaluate(body, k, ks) } }, ks) @@ -577,6 +606,9 @@ object NewNormalizer { case Stmt.Let(id, annotatedTpe, binding, body) => bind(id, evaluate(binding)) { evaluate(body, k, ks) } + case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id) => + bind(id, Computation.Inline(block, env)) { evaluate(body, k, ks) } + // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => bind(id, evaluateRecursive(id, block, ks)) { evaluate(body, k, ks) } @@ -600,6 +632,13 @@ object NewNormalizer { case Stmt.App(callee, targs, vargs, bargs) => val escapingStack = Stack.Empty evaluate(callee, "f", escapingStack) match { + case Computation.Inline(BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => + println(stmt) + val newEnv = closureEnv + .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) + .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) + + evaluate(body, k, ks)(using newEnv, scope) case Computation.Var(id) => reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => @@ -616,6 +655,7 @@ object NewNormalizer { case Computation.Var(id) => reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" + case Computation.Inline(blocklit, closure) => sys error s"Should not happen: invoke on inline" case Computation.New(interface, operations) => operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } @@ -628,7 +668,7 @@ object NewNormalizer { case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) case Some(Value.Literal(false, _)) => evaluate(els, k, ks) case _ => - k.joinpoint(ks) { k => ks => + joinpoint(k, ks) { k => ks => NeutralStmt.If(sc, nested { evaluate(thn, k, ks) }, nested { @@ -649,7 +689,7 @@ object NewNormalizer { evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) } case _ => - k.joinpoint(ks) { k => ks => + joinpoint(k, ks) { k => ks => NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -705,6 +745,7 @@ object NewNormalizer { case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => val p = Id(prompt.id) + println(s"Binding: ${util.show(prompt.id)}") // TODO is Var correct here?? Probably needs to be a new computation value... given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) evaluate(body, Frame.Return, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), k, ks)) @@ -837,6 +878,7 @@ object NewNormalizer { case Computation.Var(id) => embedBlockVar(id) case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) case Computation.Def(Closure(label, environment)) => ??? // TODO eta expand + case Computation.Inline(blocklit, env) => ??? case Computation.New(interface, operations) => // TODO deal with environment val ops = operations.map { case (id, Closure(label, environment)) => diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 97efebdf7..5cd5c883d 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -30,10 +30,10 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer.run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer(_ => true).run(tree) } Normalizer.assertNormal(tree) // println(util.show(tree)) - // tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer.run(tree) } + tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer(_ => true).run(tree) } // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From e379d02e1934d28c5760304acc71b9e757539520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Tue, 9 Sep 2025 12:08:28 +0200 Subject: [PATCH 015/123] Avoid trivial continuations again --- .../effekt/core/optimizer/NewNormalizer.scala | 21 ++++++++++++------- .../effekt/core/optimizer/Optimizer.scala | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 8cb7fcce2..1406dd4f5 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -426,14 +426,19 @@ class NewNormalizer(shouldInline: Id => Boolean) { def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { case Frame.Static(tpe, apply) => val x = Id("x") - val body = nested { scope ?=> apply(scope)(x)(Stack.Empty) } - - val k = Id("k") - val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } - scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) - - - Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) + nested { scope ?=> apply(scope)(x)(Stack.Empty) } match { + // Avoid trivial continuations like + // def k_6268 = (x_6267: Int_3) { + // return x_6267 + // } + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => + k + case body => + val k = Id("k") + val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) + } case Frame.Return => k case Frame.Dynamic(label) => k } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 5cd5c883d..fe1a124ac 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -33,7 +33,7 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer(_ => true).run(tree) } Normalizer.assertNormal(tree) // println(util.show(tree)) - tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer(_ => true).run(tree) } + tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer(_ => false).run(tree) } // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From 6caa47b4ba3d946d03b196e30615267556498eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Tue, 9 Sep 2025 12:19:40 +0200 Subject: [PATCH 016/123] Experiment with inlining strategies --- .../effekt/core/optimizer/NewNormalizer.scala | 4 ++-- .../scala/effekt/core/optimizer/Optimizer.scala | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 1406dd4f5..b59fb1552 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -386,7 +386,7 @@ object semantics { /** * A new normalizer that is conservative (avoids code bloat) */ -class NewNormalizer(shouldInline: Id => Boolean) { +class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { import semantics.* @@ -611,7 +611,7 @@ class NewNormalizer(shouldInline: Id => Boolean) { case Stmt.Let(id, annotatedTpe, binding, body) => bind(id, evaluate(binding)) { evaluate(body, k, ks) } - case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id) => + case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id, block) => bind(id, Computation.Inline(block, env)) { evaluate(body, k, ks) } // can be recursive diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index fe1a124ac..717768afe 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -4,7 +4,7 @@ package optimizer import effekt.PhaseResult.CoreTransformed import effekt.context.Context - +import effekt.core.optimizer.Usage.Recursive import kiama.util.Source object Optimizer extends Phase[CoreTransformed, CoreTransformed] { @@ -30,10 +30,18 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer(_ => true).run(tree) } + val usage = Reachable(Set(mainSymbol), tree) + + val inlineSmall = NewNormalizer { (id, b) => + !usage.get(id).contains(Recursive) && b.size < 20 + } + val dontInline = NewNormalizer { (id, b) => false } + val inlineAll = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } + + tree = Context.timed("new-normalizer-1", source.name) { inlineSmall.run(tree) } Normalizer.assertNormal(tree) // println(util.show(tree)) - tree = Context.timed("new-normalizer-2", source.name) { NewNormalizer(_ => false).run(tree) } + tree = Context.timed("new-normalizer-2", source.name) { inlineSmall.run(tree) } // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From a0ecb8598e21f528235fb296c2ad16e334dd48e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Tue, 9 Sep 2025 13:23:49 +0200 Subject: [PATCH 017/123] Capture the continuation statically --- .../effekt/core/optimizer/NewNormalizer.scala | 282 ++++++++++-------- .../iterate_increment.effekt | 1 - 2 files changed, 156 insertions(+), 127 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b59fb1552..e24a6249d 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -195,6 +195,16 @@ object semantics { object Env { def empty: Env = Env(Map.empty, Map.empty) } + // "handlers" + def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(id, addr)) + + def bind[R](id: Id, computation: Computation)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindComputation(id, computation)) + + def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(values)) + case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) @@ -223,6 +233,8 @@ object semantics { // TODO it looks like this was not a good idea... Many operations (like embed) are not supported on Inline case Inline(body: core.BlockLit, closure: Env) + case Continuation(k: Cont) + // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) @@ -230,6 +242,7 @@ object semantics { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free case Computation.Inline(body, closure) => ??? + case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } } @@ -278,6 +291,122 @@ object semantics { } } + // Stacks + // ------ + enum Frame { + case Return + case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) + case Dynamic(closure: Closure) + + def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { + case Frame.Return => ks match { + case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Reset(p, k, ks) => k.ret(ks, arg) + } + case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) + case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } + } + + // pushing purposefully does not abstract over env (it closes over it!) + def push(tpe: ValueType)(f: Scope => Addr => Frame => Stack => NeutralStmt): Frame = + Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) + } + + // maybe, for once it is simpler to decompose stacks like + // + // f, (p, f) :: (p, f) :: Nil + // + // where the frame on the reset is the one AFTER the prompt NOT BEFORE! + enum Stack { + case Empty + case Reset(prompt: BlockParam, frame: Frame, next: Stack) + + lazy val bound: List[BlockParam] = this match { + case Stack.Empty => Nil + case Stack.Reset(prompt, stack, next) => prompt :: next.bound + } + } + + enum Cont { + case Empty + case Reset(frame: Frame, prompt: BlockParam, rest: Cont) + } + + def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { + case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" + case Stack.Reset(prompt, frame, next) if prompt.id == p => + (Cont.Reset(k, prompt, Cont.Empty), frame, next) + case Stack.Reset(prompt, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Reset(k, prompt, c), frame2, stack) + } + + def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { + case Cont.Empty => (k, ks) + case Cont.Reset(frame, prompt, rest) => + val (k1, ks1) = resume(rest, frame, ks) + (frame, Stack.Reset(prompt, k1, ks1)) + } + + def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { + + def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { + case Frame.Static(tpe, apply) => + val x = Id("x") + nested { scope ?=> apply(scope)(x)(Stack.Empty) } match { + // Avoid trivial continuations like + // def k_6268 = (x_6267: Int_3) { + // return x_6267 + // } + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => + k + case body => + val k = Id("k") + val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) + } + case Frame.Return => k + case Frame.Dynamic(label) => k + } + + def reifyStack(ks: Stack): Stack = ks match { + case Stack.Empty => Stack.Empty + case Stack.Reset(prompt, frame, next) => + Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) + } + f(reifyFrame(k, ks))(reifyStack(ks)) + } + + def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = + reify(ks) { reify(k) { stmt } } + + def reify(k: Frame)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + k match { + case Frame.Return => stmt + case Frame.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(scope)(tmp)(Stack.Empty) + case Frame.Dynamic(Closure(label, closure)) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + NeutralStmt.Jump(label, Nil, List(tmp), closure) + } + + @tailrec + final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + ks match { + case Stack.Empty => stmt + // only reify reset if p is free in body + case Stack.Reset(prompt, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) + else stmt // TODO this runs normalization a second time in the outer scope! + }} + } + object PrettyPrinter extends ParenPrettyPrinter { override val defaultIndent = 2 @@ -347,6 +476,7 @@ object semantics { case Computation.Var(id) => toDoc(id) case Computation.Def(closure) => toDoc(closure) case Computation.Inline(block, env) => ??? + case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") } @@ -390,112 +520,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { import semantics.* - // "handlers" - def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindValue(id, addr)) - - def bind[R](id: Id, computation: Computation)(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindComputation(id, computation)) - - def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindValue(values)) - - // Stacks - // ------ - enum Frame { - case Return - case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) - case Dynamic(closure: Closure) - - def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { - case Frame.Return => ks match { - case Stack.Empty => NeutralStmt.Return(arg) - case Stack.Reset(p, k, ks) => k.ret(ks, arg) - } - case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) - case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } - } - - // pushing purposefully does not abstract over env (it closes over it!) - def push(tpe: ValueType)(f: Scope => Addr => Frame => Stack => NeutralStmt): Frame = - Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) - } - - def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { - - def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { - case Frame.Static(tpe, apply) => - val x = Id("x") - nested { scope ?=> apply(scope)(x)(Stack.Empty) } match { - // Avoid trivial continuations like - // def k_6268 = (x_6267: Int_3) { - // return x_6267 - // } - case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => - k - case body => - val k = Id("k") - val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } - scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) - Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) - } - case Frame.Return => k - case Frame.Dynamic(label) => k - } - - def reifyStack(ks: Stack): Stack = ks match { - case Stack.Empty => Stack.Empty - case Stack.Reset(prompt, frame, next) => - Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) - } - f(reifyFrame(k, ks))(reifyStack(ks)) - } - - def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = - reify(ks) { reify(k) { stmt } } - - def reify(k: Frame)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = - k match { - case Frame.Return => stmt - case Frame.Static(tpe, apply) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - apply(scope)(tmp)(Stack.Empty) - case Frame.Dynamic(Closure(label, closure)) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - NeutralStmt.Jump(label, Nil, List(tmp), closure) - } - - @tailrec - final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = - ks match { - case Stack.Empty => stmt - // only reify reset if p is free in body - case Stack.Reset(prompt, frame, next) => - reify(next) { reify(frame) { - val body = nested { stmt } - if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) - else stmt // TODO this runs normalization a second time in the outer scope! - }} - } - - - // maybe, for once it is simpler to decompose stacks like - // - // f, (p, f) :: (p, f) :: Nil - // - // where the frame on the reset is the one AFTER the prompt NOT BEFORE! - enum Stack { - case Empty - case Reset(prompt: BlockParam, frame: Frame, next: Stack) - - lazy val bound: List[BlockParam] = this match { - case Stack.Empty => Nil - case Stack.Reset(prompt, stack, next) => prompt :: next.bound - } - } - // used for potentially recursive definitions def evaluateRecursive(id: Id, block: core.BlockLit, escaping: Stack)(using env: Env, scope: Scope): Computation = block match { @@ -638,7 +662,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val escapingStack = Stack.Empty evaluate(callee, "f", escapingStack) match { case Computation.Inline(BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => - println(stmt) val newEnv = closureEnv .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) @@ -649,7 +672,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate) reify(k, ks) { NeutralStmt.Jump(label, targs, args, bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } - case Computation.New(interface, operations) => sys error "Should not happen: app on new" + case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } // case Stmt.Invoke(New) @@ -659,12 +682,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } - case Computation.Def(label) => sys error s"Should not happen: invoke on def ${label}" - case Computation.Inline(blocklit, closure) => sys error s"Should not happen: invoke on inline" case Computation.New(interface, operations) => operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get + case _: (Computation.Inline | Computation.Def | Computation.Continuation) => sys error s"Should not happen" } case Stmt.If(cond, thn, els) => @@ -724,15 +746,21 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Var(id) => id case _ => ??? } - // TODO implement correctly... - val neutralBody = { - given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) - nested { - evaluate(body, Frame.Return, Stack.Empty) + + if (ks.bound.exists { other => other.id == p }) { + val (cont, frame, stack) = shift(p, k, ks) + given Env = env.bindComputation(k2.id -> Computation.Continuation(cont) :: Nil) + evaluate(body, frame, stack) + } else { + val neutralBody = { + given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) + nested { + evaluate(body, Frame.Return, Stack.Empty) + } } + assert(Set(cparam) == k2.capt, "At least for now these need to be the same") + reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } } - assert(Set(cparam) == k2.capt, "At least for now these need to be the same") - reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } case Stmt.Shift(_, _) => ??? //case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => // // TODO is Var correct here?? Probably needs to be a new computation value... @@ -750,23 +778,24 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => val p = Id(prompt.id) - println(s"Binding: ${util.show(prompt.id)}") // TODO is Var correct here?? Probably needs to be a new computation value... given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) evaluate(body, Frame.Return, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), k, ks)) case Stmt.Reset(_) => ??? case Stmt.Resume(k2, body) => - val r = env.lookupComputation(k2.id) match { - case Computation.Var(id) => id + env.lookupComputation(k2.id) match { + case Computation.Var(r) => + reify(k, ks) { + NeutralStmt.Resume(r, nested { + evaluate(body, Frame.Return, Stack.Empty) + }) + } + case Computation.Continuation(k3) => + val (k4, ks4) = resume(k3, k, ks) + evaluate(body, k4, ks4) case _ => ??? } - // TODO implement properly - reify(k, ks) { - NeutralStmt.Resume(r, nested { - evaluate(body, Frame.Return, Stack.Empty) - }) - } } def run(mod: ModuleDecl): ModuleDecl = { @@ -884,6 +913,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) case Computation.Def(Closure(label, environment)) => ??? // TODO eta expand case Computation.Inline(blocklit, env) => ??? + case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => // TODO deal with environment val ops = operations.map { case (id, Closure(label, environment)) => diff --git a/examples/benchmarks/duality_of_compilation/iterate_increment.effekt b/examples/benchmarks/duality_of_compilation/iterate_increment.effekt index 61d4989d6..ab30fe334 100644 --- a/examples/benchmarks/duality_of_compilation/iterate_increment.effekt +++ b/examples/benchmarks/duality_of_compilation/iterate_increment.effekt @@ -11,4 +11,3 @@ def run(n: Int) = iterate(n, 0) { x => x + 1 } def main() = benchmark(5){run} - From 3cd1e21d0c9c07bceef352f98a8f1670a9231601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Tue, 9 Sep 2025 14:55:07 +0200 Subject: [PATCH 018/123] WIP --- .../effekt/core/optimizer/NewNormalizer.scala | 16 ++++++++++++++-- .../scala/effekt/core/optimizer/Optimizer.scala | 16 ++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index e24a6249d..53ec31778 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -241,8 +241,8 @@ object semantics { lazy val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free - case Computation.Inline(body, closure) => ??? - case Computation.Continuation(k) => ??? + case Computation.Inline(body, closure) => Set.empty // TODO ??? + case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } } @@ -636,6 +636,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { bind(id, evaluate(binding)) { evaluate(body, k, ks) } case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id, block) => + println(s"Marking ${util.show(id)} as inlinable") bind(id, Computation.Inline(block, env)) { evaluate(body, k, ks) } // can be recursive @@ -715,6 +716,17 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { }.getOrElse { evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) } + // linear usage of the continuation + // case _ if (clauses.size + default.size) <= 1 => + // NeutralStmt.Match(sc, + // clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => + // given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) + // val block = Block(tparams, vparams, bparams, nested { + // evaluate(body, k, ks) + // }) + // (id, block) + // }, + // default.map { stmt => nested { evaluate(stmt, k, ks) } }) case _ => joinpoint(k, ks) { k => ks => NeutralStmt.Match(sc, diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 717768afe..afaa3ec4b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -4,7 +4,7 @@ package optimizer import effekt.PhaseResult.CoreTransformed import effekt.context.Context -import effekt.core.optimizer.Usage.Recursive +import effekt.core.optimizer.Usage.{ Once, Recursive } import kiama.util.Source object Optimizer extends Phase[CoreTransformed, CoreTransformed] { @@ -30,18 +30,18 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - val usage = Reachable(Set(mainSymbol), tree) - - val inlineSmall = NewNormalizer { (id, b) => - !usage.get(id).contains(Recursive) && b.size < 20 + def inlineSmall(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => + usage.get(id).contains(Once) || (!usage.get(id).contains(Recursive) && b.size < 40) } val dontInline = NewNormalizer { (id, b) => false } - val inlineAll = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } + def inlineUnique(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) } + def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } - tree = Context.timed("new-normalizer-1", source.name) { inlineSmall.run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } Normalizer.assertNormal(tree) + tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) - tree = Context.timed("new-normalizer-2", source.name) { inlineSmall.run(tree) } + tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From ab34284d02916c626cf1874e4631f84fe11224f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Thu, 11 Sep 2025 11:15:26 +0200 Subject: [PATCH 019/123] Update comments --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 53ec31778..4d257ee05 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -13,10 +13,9 @@ import scala.collection.mutable import scala.collection.immutable.ListMap // TODO -// while linearity is difficult to track bottom up, variable usage is possible -// this way deadcode can be eliminated on the way up. +// - change story of how inlining is implemented. We need to also support toplevel functions that potentially +// inline each other. Do we need to sort them topologically? How do we deal with (mutually) recursive definitions? // -// plan: don't inline... this is a separate pass after normalization // // plan: only introduce parameters for free things inside a block that are bound in the **stack** // that is in @@ -626,7 +625,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { k.ret(ks, evaluate(expr)) case Stmt.Val(id, annotatedTpe, binding, body) => - // This push can lead to an eta-redex (a superfluous push...) evaluate(binding, k.push(annotatedTpe) { scope => res => k => ks => given Scope = scope bind(id, res) { evaluate(body, k, ks) } @@ -646,9 +644,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Def(id, block, body) => bind(id, evaluate(block, id.name.name, ks)) { evaluate(body, k, ks) } - // TODO actually remove a cut instead of binding the block argument - // alternatively, we could keep some information about linearity. - // For example, here we are in an active position. case Stmt.App(BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => // TODO also bind type arguments in environment // TODO substitute cparams??? @@ -658,8 +653,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(body, k, ks)(using newEnv, scope) - // TODO here the stack passed to the blocks could be an empty one since we reify it anyways... case Stmt.App(callee, targs, vargs, bargs) => + // Here the stack passed to the blocks is an empty one since we reify it anyways... val escapingStack = Stack.Empty evaluate(callee, "f", escapingStack) match { case Computation.Inline(BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => From 0fa77536f74ecaf9c40e080f94b898352ec372e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 25 Sep 2025 12:27:08 +0200 Subject: [PATCH 020/123] Implement box/unbox in new normalizer --- .../effekt/core/optimizer/NewNormalizer.scala | 91 ++++++++++++++----- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4d257ee05..473229ba5 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -53,11 +53,14 @@ object semantics { case Literal(value: Any, annotatedType: ValueType) case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) + case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) + val free: Variables = this match { // case Value.Var(id, annotatedType) => Variables.empty case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Make(data, tag, targs, vargs) => vargs.toSet + case Value.Box(body, tpe) => body.free } } @@ -204,11 +207,18 @@ object semantics { def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = prog(using env.bindValue(values)) + sealed trait Block { + val free: Variables + } - case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { + case class BlockLit(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) extends Block { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) } + case class NeutralUnbox(addr: Id) extends Block { + val free: Variables = Set(addr) + } + case class BasicBlock(bindings: Bindings, body: NeutralStmt) { val free: Variables = { var free = body.free @@ -237,12 +247,15 @@ object semantics { // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) + case Unbox(addr: Addr) + lazy val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free case Computation.Inline(body, closure) => Set.empty // TODO ??? case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet + case Computation.Unbox(addr) => Set(addr) } } @@ -264,7 +277,7 @@ object semantics { // cond is unknown case If(cond: Id, thn: BasicBlock, els: BasicBlock) // scrutinee is unknown - case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) + case Match(scrutinee: Id, clauses: List[(Id, BlockLit)], default: Option[BasicBlock]) // body is stuck case Reset(prompt: BlockParam, body: BasicBlock) @@ -362,7 +375,7 @@ object semantics { case body => val k = Id("k") val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } - scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + scope.define(k, BlockLit(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) } case Frame.Return => k @@ -463,12 +476,16 @@ object semantics { "make" <+> toDoc(data) <+> toDoc(tag) <> (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) + + case Value.Box(body, tpe) => + "box" <+> braces(nest(line <> toDoc(body) <> line)) } def toDoc(block: Block): Doc = block match { - case Block(tparams, vparams, bparams, body) => + case BlockLit(tparams, vparams, bparams, body) => (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> - parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) + parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) + case NeutralUnbox(addr) => "unbox" <> parens(toDoc(addr)) } def toDoc(comp: Computation): Doc = comp match { @@ -479,6 +496,7 @@ object semantics { case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") } + case Computation.Unbox(addr) => "unbox" <> parens(toDoc(addr)) } def toDoc(closure: Closure): Doc = closure match { case Closure(label, env) => toDoc(label) <> brackets(hsep(env.map(toDoc), comma)) @@ -531,7 +549,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) .bindComputation(id, Computation.Var(freshened)) - val normalizedBlock = Block(tparams, vparams, bparams, nested { + val normalizedBlock = BlockLit(tparams, vparams, bparams, nested { evaluate(body, Frame.Return, Stack.Empty) }) @@ -551,7 +569,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - val normalizedBlock = Block(tparams, vparams, bparams, nested { + val normalizedBlock = BlockLit(tparams, vparams, bparams, nested { evaluate(body, Frame.Return, Stack.Empty) }) @@ -562,7 +580,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Computation.Def(Closure(f, closureParams.map(p => Computation.Var(p.id)))) case core.Block.Unbox(pure) => - ??? + val addr = evaluate(pure)(using env, scope) + scope.lookupValue(addr) match { + case Some(Value.Box(body, _)) => body + case Some(_) | None => Computation.Unbox(addr) + } case core.Block.New(Implementation(interface, operations)) => val ops = operations.map { @@ -615,7 +637,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) case Pure.Box(b, annotatedCapture) => - ??? + val comp = evaluate(b, "x", Stack.Empty) + scope.allocate("x", Value.Box(comp, annotatedCapture)) } // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) @@ -644,7 +667,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Def(id, block, body) => bind(id, evaluate(block, id.name.name, ks)) { evaluate(body, k, ks) } - case Stmt.App(BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => + case Stmt.App(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => // TODO also bind type arguments in environment // TODO substitute cparams??? val newEnv = env @@ -657,7 +680,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // Here the stack passed to the blocks is an empty one since we reify it anyways... val escapingStack = Stack.Empty evaluate(callee, "f", escapingStack) match { - case Computation.Inline(BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => + case Computation.Inline(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => val newEnv = closureEnv .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) @@ -668,6 +691,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate) reify(k, ks) { NeutralStmt.Jump(label, targs, args, bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } + case Computation.Unbox(addr) => + val tmp = Id("unbox") + scope.define(tmp, NeutralUnbox(addr)) + reify(k, ks) { NeutralStmt.App(tmp, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } @@ -682,6 +709,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get + case Computation.Unbox(addr) => + val tmp = Id("unbox") + scope.define(tmp, NeutralUnbox(tmp)) + reify(k, ks) { NeutralStmt.Invoke(tmp, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case _: (Computation.Inline | Computation.Def | Computation.Continuation) => sys error s"Should not happen" } @@ -706,7 +737,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Some(Value.Make(data, tag, targs, vargs)) => // TODO substitute types (or bind them in the env)! clauses.collectFirst { - case (tpe, BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => + case (tpe, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k, ks) } }.getOrElse { evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) @@ -726,9 +757,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { joinpoint(k, ks) { k => ks => NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation - clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => + clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = Block(tparams, vparams, bparams, nested { + val block = BlockLit(tparams, vparams, bparams, nested { evaluate(body, k, ks) }) (id, block) @@ -748,7 +779,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Put(ref, annotatedCapt, value, body) => ??? // Control Effects - case Stmt.Shift(prompt, BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => + case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => val p = env.lookupComputation(prompt.id) match { case Computation.Var(id) => id case _ => ??? @@ -783,7 +814,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // k.reify(NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) - case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => + case Stmt.Reset(core.Block.BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => val p = Id(prompt.id) // TODO is Var correct here?? Probably needs to be a new computation value... given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) @@ -826,7 +857,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { inline def debug(inline msg: => Any) = println(msg) def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { - case Toplevel.Def(id, BlockLit(tparams, cparams, vparams, bparams, body)) => + case Toplevel.Def(id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => debug(s"------- ${util.show(id)} -------") debug(util.show(body)) @@ -838,11 +869,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val result = evaluate(body, Frame.Return, Stack.Empty) debug(s"---------------------") - val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) + val block = BlockLit(tparams, vparams, bparams, reifyBindings(scope, result)) debug(PrettyPrinter.show(block)) debug(s"---------------------") - val embedded = embedBlockLit(block) + val embedded = embedBlock(block) debug(util.show(embedded)) Toplevel.Def(id, embedded) @@ -894,10 +925,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO why do we even have this type in core, if we always infer it? Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) case ((id, Binding.Def(block)), rest) => G => - val coreBlock = embedBlockLit(block)(using G) + val coreBlock = embedBlock(block)(using G) Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) case ((id, Binding.Rec(block, tpe, capt)), rest) => G => - val coreBlock = embedBlockLit(block)(using G.bind(id, tpe, capt)) + val coreBlock = embedBlock(block)(using G.bind(id, tpe, capt)) Stmt.Def(id, coreBlock, rest(G.bind(id, tpe, capt))) case ((id, Binding.Val(stmt)), rest) => G => val coreStmt = embedStmt(stmt)(using G) @@ -912,6 +943,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Value.Extern(callee, targs, vargs) => Pure.PureApp(callee, targs, vargs.map(embedPure)) case Value.Literal(value, annotatedType) => Pure.Literal(value, annotatedType) case Value.Make(data, tag, targs, vargs) => Pure.Make(data, tag, targs, vargs.map(embedPure)) + case Value.Box(body, annotatedCapture) => Pure.Box(embedBlock(body), annotatedCapture) } def embedPure(addr: Addr)(using G: TypingContext): core.Pure = Pure.ValueVar(addr, G.lookupValue(addr)) @@ -938,10 +970,23 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } } core.Block.New(Implementation(interface, ops)) + case Computation.Unbox(addr) => + core.Block.Unbox(embedPure(addr)) } - def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { - case Block(tparams, vparams, bparams, body) => + def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { + case BlockLit(tparams, vparams, bparams, b) => + val cparams = bparams.map { + case BlockParam(id, tpe, captures) => + assert(captures.size == 1) + captures.head + } + core.Block.BlockLit(tparams, cparams, vparams, bparams, + embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) + case NeutralUnbox(addr) => core.Block.Unbox(embedPure(addr)(using G)) + } + def embedBlockLit(block: BlockLit)(using G: TypingContext): core.BlockLit = block match { + case BlockLit(tparams, vparams, bparams, body) => val cparams = bparams.map { case BlockParam(id, tpe, captures) => assert(captures.size == 1) From 2f1000e942656e17c5cacd22ca4fff7e9a53d4d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 25 Sep 2025 12:31:54 +0200 Subject: [PATCH 021/123] Import effekt.core.Pure --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 473229ba5..a58e1985f 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -4,6 +4,7 @@ package optimizer import effekt.source.Span import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } +import effekt.core.Pure import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter From 6c7845ea3b420968a81bd0147519a64d057701c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 25 Sep 2025 12:54:40 +0200 Subject: [PATCH 022/123] Second try to un-confuse Scala --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index a58e1985f..2e9a725b0 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -4,7 +4,6 @@ package optimizer import effekt.source.Span import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } -import effekt.core.Pure import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter @@ -620,24 +619,24 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def evaluate(expr: Expr)(using env: Env, scope: Scope): Addr = expr match { - case Pure.ValueVar(id, annotatedType) => + case core.Pure.ValueVar(id, annotatedType) => env.lookupValue(id) - case Pure.Literal(value, annotatedType) => + case core.Pure.Literal(value, annotatedType) => scope.allocate("x", Value.Literal(value, annotatedType)) // right now everything is stuck... no constant folding ... - case Pure.PureApp(f, targs, vargs) => + case core.Pure.PureApp(f, targs, vargs) => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate))) case DirectApp(f, targs, vargs, bargs) => assert(bargs.isEmpty) scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Empty))) - case Pure.Make(data, tag, targs, vargs) => + case core.Pure.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) - case Pure.Box(b, annotatedCapture) => + case core.Pure.Box(b, annotatedCapture) => val comp = evaluate(b, "x", Stack.Empty) scope.allocate("x", Value.Box(comp, annotatedCapture)) } From b916e6caf3fc89b3d4f12397d1aec98b3cba887b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 29 Sep 2025 15:57:36 +0200 Subject: [PATCH 023/123] Add two test cases for normalizing boxes --- .../test/scala/effekt/core/CoreTests.scala | 16 ++ .../effekt/core/NewNormalizerTests.scala | 194 ++++++++++++++++++ .../scala/effekt/core/PrettyPrinter.scala | 3 + 3 files changed, 213 insertions(+) create mode 100644 effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala diff --git a/effekt/jvm/src/test/scala/effekt/core/CoreTests.scala b/effekt/jvm/src/test/scala/effekt/core/CoreTests.scala index 144223bcd..dda503b16 100644 --- a/effekt/jvm/src/test/scala/effekt/core/CoreTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/CoreTests.scala @@ -28,6 +28,21 @@ trait CoreTests extends munit.FunSuite { |""".stripMargin }) + def shouldBeEqual(obtained: Toplevel, expected: Toplevel, clue: => Any)(using Location) = + assertEquals(obtained, expected, { + s"""${clue} + |===================== + |Got: + |---- + |${effekt.core.PrettyPrinter.format(obtained).layout} + | + |Expected: + |--------- + |${effekt.core.PrettyPrinter.format(expected).layout} + | + |""".stripMargin + }) + def shouldBeEqual(obtained: Stmt, expected: Stmt, clue: => Any)(using Location) = assertEquals(obtained, expected, { s"""${clue} @@ -55,6 +70,7 @@ trait CoreTests extends munit.FunSuite { assertEquals(obtainedPrinted, expectedPrinted) shouldBeEqual(obtainedRenamed, expectedRenamed, clue) } + def assertAlphaEquivalentStatements(obtained: Stmt, expected: Stmt, clue: => Any = "values are not alpha-equivalent", diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala new file mode 100644 index 000000000..10b7789fd --- /dev/null +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -0,0 +1,194 @@ +package effekt.core + +import effekt.PhaseResult.CoreTransformed +import effekt.context.{Context, IOModuleDB} +import effekt.core.optimizer.{NewNormalizer, Normalizer} +import effekt.util.PlainMessaging +import effekt.* +import kiama.output.PrettyPrinterTypes.Document +import kiama.util.{Source, StringSource} +import munit.Location + +class NewNormalizerTests extends CoreTests { + object plainMessaging extends PlainMessaging + object context extends Context with IOModuleDB { + val messaging = plainMessaging + + object frontend extends NormalizeOnly + + override lazy val compiler = frontend.asInstanceOf + } + + def compileString(content: String): (Id, symbols.Module, ModuleDecl) = + val config = new EffektConfig(Seq("--Koutput", "string")) + config.verify() + context.setup(config) + context.frontend.compile(StringSource(content, "input.effekt"))(using context).map { + case (_, decl) => decl + }.getOrElse { + val errors = plainMessaging.formatMessages(context.messaging.buffer) + sys error errors + } + + def normalize(contents: String): (Id, core.ModuleDecl) = { + val (main, mod, decl) = compileString(contents) + (main, decl) + } + + def assertAlphaEquivalentToplevels( + actual: ModuleDecl, + expected: ModuleDecl, + defNames: List[String], + externNames: List[String] + )(using Location): Unit = { + + def findDef(mod: ModuleDecl, name: String) = + mod.definitions.find(_.id.name.name == name) + .getOrElse(throw new NoSuchElementException(s"Definition '$name' not found")) + + def findExternDef(mod: ModuleDecl, name: String) = + mod.externs.collect { case d@Extern.Def(_, _, _, _, _, _, _, _) => d } + .find(_.id.name.name == name) + .getOrElse(throw new NoSuchElementException(s"Extern def '$name' not found")) + + val externPairs: List[(Id, Id)] = + externNames.flatMap { name => + val canon = Id(name) + List( + findExternDef(actual, name).id -> canon, + findExternDef(expected, name).id -> canon + ) + } + + def compareOneDef(name: String): Unit = { + val aDef = findDef(actual, name) + val eDef = findDef(expected, name) + + val canon = Id(name) + val pairs: Map[Id, Id] = + (List(aDef.id -> canon, eDef.id -> canon) ++ externPairs).toMap + + val renamer = TestRenamer(Names(defaultNames), "$", List(pairs)) + shouldBeEqual( + renamer(aDef), + renamer(eDef), + s"Top-level '$name' is not alpha-equivalent" + ) + } + + defNames.foreach(compareOneDef) + } + + // This example shows a box that contains an extern reference. + // The normalizer is able to unbox this indirection away. + test("extern in box") { + val input = + """ + |extern def foo: Int = vm"42" + | + |def run(): Int = { + | val f = box { + | foo + | } at { io } + | + | val x = f()() + | return x + |} + | + |def main() = println(run()) + |""".stripMargin + + val expected = + parse(""" + |module input + | + |extern {io} def foo(): Int = vm"42" + | + |def run() = { + | let x = !(foo : () => Int @ {io})() + | return x: Int + |} + |""".stripMargin + ) + + val (mainId, actual) = normalize(input) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) + } + + // This example shows a box that cannot be normalized away + test("box passed to extern") { + val input = + """ + |extern {io} def foo(f: => Int at {}): Int = vm"42" + | + |def run(): Int = { + | val f = box { + | 42 + | } at {} + | + | val x = foo(f) + | return x + |} + | + |def main() = println(run()) + |""".stripMargin + + val expected = + parse(""" + |module input + | + |extern {io} def foo(): Int = vm"42" + | + |def run() = { + | def f() = { + | let x = 42 + | return x: Int + | } + | let f_box: => Int at {} = box {} (f: => Int @ {}) + | let x: Int = + | ! (foo: (=> Int at {}) => Int @ {io})(f_box: => Int at {}) + | + | return x: Int + |} + |""".stripMargin + ) + + val (mainId, actual) = normalize(input) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) + } +} + +/** + * A "backend" that simply outputs the aggregated core module. + * This is called IR and note Core to avoid name clashes with package `effekt.core` + * + * This is, for example, used by the interpreter. + */ +class NormalizeOnly extends Compiler[(Id, symbols.Module, ModuleDecl)] { + + def extension = ".effekt-core.ir" + + override def supportedFeatureFlags: List[String] = List("vm") + + override def prettyIR(source: Source, stage: Stage)(using C: Context): Option[Document] = None + + override def treeIR(source: Source, stage: Stage)(using Context): Option[Any] = None + + override def compile(source: Source)(using C: Context): Option[(Map[String, String], (Id, symbols.Module, ModuleDecl))] = + Optimized.run(source).map { res => (Map.empty, res) } + + lazy val Core = Phase.cached("core") { + Frontend andThen Middleend + } + + lazy val Optimized = allToCore(Core) andThen Aggregate andThen core.optimizer.Optimizer map { + case input @ CoreTransformed(source, tree, mod, core) => + val mainSymbol = Context.ensureMainExists(mod) + val dontInline = NewNormalizer { (id, b) => false } + val normalTree = dontInline.run(core) + Normalizer.assertNormal(normalTree) + (mainSymbol, mod, normalTree) + } +} diff --git a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala index 14f2e1e83..1d7fb3c85 100644 --- a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala +++ b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala @@ -17,6 +17,9 @@ object PrettyPrinter extends ParenPrettyPrinter { def format(t: ModuleDecl): Document = pretty(toDoc(t), 4) + def format(t: Toplevel): Document = + pretty(toDoc(t), 4) + def format(defs: List[Toplevel]): String = pretty(toDoc(defs), 60).layout From 7389fef778a904243d76614fad7649b07b6fea18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 29 Sep 2025 15:59:40 +0200 Subject: [PATCH 024/123] Fix copy-pasted doc comment --- .../jvm/src/test/scala/effekt/core/NewNormalizerTests.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 10b7789fd..7d2438bbb 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -161,10 +161,7 @@ class NewNormalizerTests extends CoreTests { } /** - * A "backend" that simply outputs the aggregated core module. - * This is called IR and note Core to avoid name clashes with package `effekt.core` - * - * This is, for example, used by the interpreter. + * A "backend" that simply outputs the normalized core module. */ class NormalizeOnly extends Compiler[(Id, symbols.Module, ModuleDecl)] { From dfaa230f5ffe9b1d9a3ab988618bd2ece2a0624a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 29 Sep 2025 16:52:19 +0200 Subject: [PATCH 025/123] Add test case for neutral unbox --- .../effekt/core/NewNormalizerTests.scala | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 7d2438bbb..074cf681d 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -116,7 +116,8 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) } - // This example shows a box that cannot be normalized away + // This example shows a box that cannot be normalized away. + // This is because the box is passed to an extern definition. test("box passed to extern") { val input = """ @@ -158,6 +159,41 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) } + + // This example shows an unbox that cannot be normalized away. + // This is because the box is retrieved from an extern definition. + test("unbox blocked by extern") { + val input = + """ + |extern {} def foo(): => Int at {} = vm"42" + | + |def run(): Int = { + | val x = foo()() + | return x + |} + | + |def main() = println(run()) + |""".stripMargin + + val expected = + parse(""" + |module input + | + |extern {} def foo(): => Int at {} = vm"42" + | + |def run() = { + | let f: => Int at {} = + | (foo: => (=> Int at {}) @ {})() + | def f_unbox = unbox f: => Int at {} + | (f_unbox: => Int @ {})() + |} + |""".stripMargin + ) + + val (mainId, actual) = normalize(input) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) + } } /** From 99c4203371ef107919786b4c2f4baf88f8b1184e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 2 Oct 2025 12:48:55 +0200 Subject: [PATCH 026/123] Properly implement unbox handling --- .../effekt/core/NewNormalizerTests.scala | 38 +++++++++----- .../effekt/core/optimizer/NewNormalizer.scala | 51 ++++++++++--------- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 074cf681d..0734a7ba4 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -2,7 +2,7 @@ package effekt.core import effekt.PhaseResult.CoreTransformed import effekt.context.{Context, IOModuleDB} -import effekt.core.optimizer.{NewNormalizer, Normalizer} +import effekt.core.optimizer.{Deadcode, NewNormalizer, Normalizer, Optimizer} import effekt.util.PlainMessaging import effekt.* import kiama.output.PrettyPrinterTypes.Document @@ -83,7 +83,7 @@ class NewNormalizerTests extends CoreTests { // The normalizer is able to unbox this indirection away. test("extern in box") { val input = - """ + """ |extern def foo: Int = vm"42" | |def run(): Int = { @@ -91,25 +91,34 @@ class NewNormalizerTests extends CoreTests { | foo | } at { io } | - | val x = f()() + | val x = unbox f()() | return x |} | |def main() = println(run()) |""".stripMargin - val expected = - parse(""" + val expected = parse( + """ |module input | |extern {io} def foo(): Int = vm"42" | |def run() = { - | let x = !(foo : () => Int @ {io})() - | return x: Int + | def f1() = { + | def f2() = { + | let x = !(foo: () => Int @ {io})() + | return x: Int + | } + | let y = box {io} f2: () => Int @ {io} + | return y: () => Int at {io} + | } + | val z: () => Int at {io} = (f1: () => (() => Int at {io}) @ {io})(); + | def r = unbox z: () => Int at {io} + | (r: () => Int @ {io})() |} - |""".stripMargin - ) + | + |""".stripMargin) val (mainId, actual) = normalize(input) @@ -216,12 +225,13 @@ class NormalizeOnly extends Compiler[(Id, symbols.Module, ModuleDecl)] { Frontend andThen Middleend } - lazy val Optimized = allToCore(Core) andThen Aggregate andThen core.optimizer.Optimizer map { + lazy val Optimized = allToCore(Core) andThen Aggregate map { case input @ CoreTransformed(source, tree, mod, core) => val mainSymbol = Context.ensureMainExists(mod) - val dontInline = NewNormalizer { (id, b) => false } - val normalTree = dontInline.run(core) - Normalizer.assertNormal(normalTree) - (mainSymbol, mod, normalTree) + var tree = Deadcode.remove(mainSymbol, core) + val normalizer = NewNormalizer { (id, b) => false } + tree = normalizer.run(tree) + Normalizer.assertNormal(tree) + (mainSymbol, mod, tree) } } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 2e9a725b0..5cdb987df 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,9 +2,10 @@ package effekt package core package optimizer +import effekt.core.ValueType.Boxed import effekt.source.Span -import effekt.core.optimizer.semantics.{ Computation, NeutralStmt } -import effekt.util.messages.{ ErrorReporter, INTERNAL_ERROR } +import effekt.core.optimizer.semantics.{Computation, NeutralStmt} +import effekt.util.messages.{ErrorReporter, INTERNAL_ERROR} import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter @@ -71,6 +72,7 @@ object semantics { case Rec(block: Block, tpe: BlockType, capt: Captures) case Val(stmt: NeutralStmt) case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + case Unbox(addr: Addr, tpe: BlockType, capt: Captures) val free: Variables = this match { case Binding.Let(value) => value.free @@ -78,6 +80,7 @@ object semantics { case Binding.Rec(block, tpe, capt) => block.free case Binding.Val(stmt) => stmt.free case Binding.Run(f, targs, vargs, bargs) => vargs.toSet ++ all(bargs, _.free) + case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set(addr) } } @@ -118,6 +121,12 @@ object semantics { addr } + def unbox(innerAddr: Addr, tpe: BlockType, capt: Captures): Addr = { + val unboxAddr = Id("unbox") + bindings = bindings.updated(unboxAddr, Binding.Unbox(innerAddr, tpe, capt)) + unboxAddr + } + // TODO Option[Value] or Var(id) in Value? def lookupValue(addr: Addr): Option[Value] = bindings.get(addr) match { case Some(Binding.Let(value)) => Some(value) @@ -161,6 +170,9 @@ object semantics { used = used ++ v.free filtered = (addr, v) :: filtered case (addr, v: Binding.Let) => () + case (addr, b: Binding.Unbox) => + used = used ++ b.free + filtered = (addr, b):: filtered } // we want to avoid turning tailcalls into non tail calls like @@ -215,10 +227,6 @@ object semantics { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) } - case class NeutralUnbox(addr: Id) extends Block { - val free: Variables = Set(addr) - } - case class BasicBlock(bindings: Bindings, body: NeutralStmt) { val free: Variables = { var free = body.free @@ -228,6 +236,7 @@ object semantics { case (id, b: Binding.Rec) => free = (free - id) ++ (b.free - id) case (id, b: Binding.Val) => free = (free - id) ++ b.free case (id, b: Binding.Run) => free = (free - id) ++ b.free + case (id, b: Binding.Unbox) => free = (free - id) ++ b.free } free } @@ -247,15 +256,12 @@ object semantics { // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) - case Unbox(addr: Addr) - lazy val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free case Computation.Inline(body, closure) => Set.empty // TODO ??? case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet - case Computation.Unbox(addr) => Set(addr) } } @@ -485,7 +491,6 @@ object semantics { case BlockLit(tparams, vparams, bparams, body) => (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) - case NeutralUnbox(addr) => "unbox" <> parens(toDoc(addr)) } def toDoc(comp: Computation): Doc = comp match { @@ -496,7 +501,6 @@ object semantics { case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") } - case Computation.Unbox(addr) => "unbox" <> parens(toDoc(addr)) } def toDoc(closure: Closure): Doc = closure match { case Closure(label, env) => toDoc(label) <> brackets(hsep(env.map(toDoc), comma)) @@ -511,6 +515,7 @@ object semantics { case (addr, Binding.Run(callee, targs, vargs, bargs)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line + case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line }) def toDoc(block: BasicBlock): Doc = @@ -583,7 +588,14 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val addr = evaluate(pure)(using env, scope) scope.lookupValue(addr) match { case Some(Value.Box(body, _)) => body - case Some(_) | None => Computation.Unbox(addr) + case Some(_) | None => { + val (tpe, capt) = pure.tpe match { + case Boxed(tpe, capt) => (tpe, capt) + case _ => sys error "should not happen" + } + val unboxAddr = scope.unbox(addr, tpe, capt) + Computation.Var(unboxAddr) + } } case core.Block.New(Implementation(interface, operations)) => @@ -691,10 +703,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate) reify(k, ks) { NeutralStmt.Jump(label, targs, args, bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } - case Computation.Unbox(addr) => - val tmp = Id("unbox") - scope.define(tmp, NeutralUnbox(addr)) - reify(k, ks) { NeutralStmt.App(tmp, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } @@ -709,10 +717,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get - case Computation.Unbox(addr) => - val tmp = Id("unbox") - scope.define(tmp, NeutralUnbox(tmp)) - reify(k, ks) { NeutralStmt.Invoke(tmp, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case _: (Computation.Inline | Computation.Def | Computation.Continuation) => sys error s"Should not happen" } @@ -837,7 +841,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def run(mod: ModuleDecl): ModuleDecl = { - // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } val toplevelEnv = Env.empty @@ -936,6 +939,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => val coreExpr = DirectApp(callee, targs, vargs.map(arg => embedPure(arg)(using G)), bargs.map(arg => embedBlock(arg)(using G))) Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) + case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => + val pureValue = embedPure(addr)(using G) + Stmt.Def(id, Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) }(G) } @@ -970,8 +976,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } } core.Block.New(Implementation(interface, ops)) - case Computation.Unbox(addr) => - core.Block.Unbox(embedPure(addr)) } def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { @@ -983,7 +987,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } core.Block.BlockLit(tparams, cparams, vparams, bparams, embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) - case NeutralUnbox(addr) => core.Block.Unbox(embedPure(addr)(using G)) } def embedBlockLit(block: BlockLit)(using G: TypingContext): core.BlockLit = block match { case BlockLit(tparams, vparams, bparams, body) => From 5ec0c04f371b4041f4f99b6b0df326c9ffb749c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 2 Oct 2025 13:02:06 +0200 Subject: [PATCH 027/123] Collapse Box again --- .../effekt/core/optimizer/NewNormalizer.scala | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 5cdb987df..a3808000f 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -219,11 +219,7 @@ object semantics { def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = prog(using env.bindValue(values)) - sealed trait Block { - val free: Variables - } - - case class BlockLit(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) extends Block { + case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) } @@ -283,7 +279,7 @@ object semantics { // cond is unknown case If(cond: Id, thn: BasicBlock, els: BasicBlock) // scrutinee is unknown - case Match(scrutinee: Id, clauses: List[(Id, BlockLit)], default: Option[BasicBlock]) + case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) // body is stuck case Reset(prompt: BlockParam, body: BasicBlock) @@ -381,7 +377,7 @@ object semantics { case body => val k = Id("k") val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } - scope.define(k, BlockLit(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) } case Frame.Return => k @@ -488,7 +484,7 @@ object semantics { } def toDoc(block: Block): Doc = block match { - case BlockLit(tparams, vparams, bparams, body) => + case Block(tparams, vparams, bparams, body) => (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) } @@ -554,7 +550,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) .bindComputation(id, Computation.Var(freshened)) - val normalizedBlock = BlockLit(tparams, vparams, bparams, nested { + val normalizedBlock = Block(tparams, vparams, bparams, nested { evaluate(body, Frame.Return, Stack.Empty) }) @@ -574,7 +570,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - val normalizedBlock = BlockLit(tparams, vparams, bparams, nested { + val normalizedBlock = Block(tparams, vparams, bparams, nested { evaluate(body, Frame.Return, Stack.Empty) }) @@ -763,7 +759,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = BlockLit(tparams, vparams, bparams, nested { + val block = Block(tparams, vparams, bparams, nested { evaluate(body, k, ks) }) (id, block) @@ -872,7 +868,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val result = evaluate(body, Frame.Return, Stack.Empty) debug(s"---------------------") - val block = BlockLit(tparams, vparams, bparams, reifyBindings(scope, result)) + val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) debug(PrettyPrinter.show(block)) debug(s"---------------------") @@ -941,7 +937,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => val pureValue = embedPure(addr)(using G) - Stmt.Def(id, Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) + Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) }(G) } @@ -979,7 +975,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { - case BlockLit(tparams, vparams, bparams, b) => + case Block(tparams, vparams, bparams, b) => val cparams = bparams.map { case BlockParam(id, tpe, captures) => assert(captures.size == 1) @@ -988,8 +984,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { core.Block.BlockLit(tparams, cparams, vparams, bparams, embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) } - def embedBlockLit(block: BlockLit)(using G: TypingContext): core.BlockLit = block match { - case BlockLit(tparams, vparams, bparams, body) => + def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { + case Block(tparams, vparams, bparams, body) => val cparams = bparams.map { case BlockParam(id, tpe, captures) => assert(captures.size == 1) From 202a0ea047d9aadd0663c924a262c2b3f1df26a7 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 2 Oct 2025 13:08:36 +0200 Subject: [PATCH 028/123] add TODOs for regions and mutable state --- .../effekt/core/optimizer/NewNormalizer.scala | 54 ++++++++++++++----- .../effekt/core/optimizer/Optimizer.scala | 5 +- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index a3808000f..63783acb3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -249,6 +249,8 @@ object semantics { case Continuation(k: Cont) + //case Region(prompt: Id) ??? + // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) @@ -261,6 +263,7 @@ object semantics { } } + // TODO add escaping mutable variables case class Closure(label: Label, environment: List[Computation]) { val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet } @@ -288,6 +291,10 @@ object semantics { // continuation is unknown case Resume(k: Id, body: BasicBlock) + // case Var(id: Id, init: Id, body: BasicBlock) + // case Get + // case Put + // aborts at runtime case Hole(span: Span) @@ -326,6 +333,16 @@ object semantics { Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) } + // case class Store(var vars: Map[Addr, Value]) { + // def get(addr: Addr): Value = { + // vars(addr) + // } + // def set(addr: Addr, v: Value): Unit = { + // vars = vars.updated(addr, v) + // } + // } + type Store = Map[Id, Addr] + // maybe, for once it is simpler to decompose stacks like // // f, (p, f) :: (p, f) :: Nil @@ -334,6 +351,10 @@ object semantics { enum Stack { case Empty case Reset(prompt: BlockParam, frame: Frame, next: Stack) + // case Var(id: Id, addr: Addr, next: Stack) TODO + //case Region(bindings: ???) + + var store = Map.empty[Id, Addr] lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil @@ -343,23 +364,25 @@ object semantics { enum Cont { case Empty - case Reset(frame: Frame, prompt: BlockParam, rest: Cont) + case Reset(frame: Frame, prompt: BlockParam, store: Store, rest: Cont) } def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" case Stack.Reset(prompt, frame, next) if prompt.id == p => - (Cont.Reset(k, prompt, Cont.Empty), frame, next) + (Cont.Reset(k, prompt, ks.store, Cont.Empty), frame, next) case Stack.Reset(prompt, frame, next) => val (c, frame2, stack) = shift(p, frame, next) - (Cont.Reset(k, prompt, c), frame2, stack) + (Cont.Reset(k, prompt, ks.store, c), frame2, stack) } def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { case Cont.Empty => (k, ks) - case Cont.Reset(frame, prompt, rest) => + case Cont.Reset(frame, prompt, store, rest) => val (k1, ks1) = resume(rest, frame, ks) - (frame, Stack.Reset(prompt, k1, ks1)) + val stack = Stack.Reset(prompt, k1, ks1) + stack.store = store + (frame, stack) } def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { @@ -414,7 +437,7 @@ object semantics { case Stack.Empty => stmt // only reify reset if p is free in body case Stack.Reset(prompt, frame, next) => - reify(next) { reify(frame) { + reify(next) { reify(frame) { // reify(next) { reify(store) { reify(frame) { ... }}} ??? ORDER? TODO val body = nested { stmt } if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) else stmt // TODO this runs normalization a second time in the outer scope! @@ -774,9 +797,16 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Region(body) => ??? case Stmt.Alloc(id, init, region, body) => ??? - case Stmt.Var(ref, init, capture, body) => ??? - case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => ??? - case Stmt.Put(ref, annotatedCapt, value, body) => ??? + // TODO + case Stmt.Var(ref, init, capture, body) => + val addr = evaluate(init) + ks.store = ks.store.updated(ref, addr) + evaluate(body, k, ks) + case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => + bind(id, ks.store(ref)) { evaluate(body, k, ks) } + case Stmt.Put(ref, annotatedCapt, value, body) => + ks.store = ks.store.updated(ref, evaluate(value)) + evaluate(body, k, ks) // Control Effects case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => @@ -867,12 +897,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { given scope: Scope = Scope.empty val result = evaluate(body, Frame.Return, Stack.Empty) - debug(s"---------------------") + debug(s"----------normalized-----------") val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) debug(PrettyPrinter.show(block)) - debug(s"---------------------") - val embedded = embedBlock(block) + debug(s"----------embedded-----------") + val embedded = embedBlockLit(block) debug(util.show(embedded)) Toplevel.Def(id, embedded) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index afaa3ec4b..120fed3dc 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -37,11 +37,12 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { def inlineUnique(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) } def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } - tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } + // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { dontInline.run(tree) } Normalizer.assertNormal(tree) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) - tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } + // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From 4c0d42b0eed0dd339aba7d723904c5670b295430 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:51:17 +0200 Subject: [PATCH 029/123] fix ImpureApp --- .../src/main/scala/effekt/core/Type.scala | 7 +++- .../effekt/core/optimizer/NewNormalizer.scala | 37 ++++++++++--------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/Type.scala b/effekt/shared/src/main/scala/effekt/core/Type.scala index 41e2b52c5..0c279aba4 100644 --- a/effekt/shared/src/main/scala/effekt/core/Type.scala +++ b/effekt/shared/src/main/scala/effekt/core/Type.scala @@ -194,14 +194,17 @@ object Type { def bindingType(stmt: Stmt.ImpureApp): ValueType = stmt match { case Stmt.ImpureApp(id, callee, targs, vargs, bargs, body) => - Type.instantiate(callee.tpe.asInstanceOf[core.BlockType.Function], targs, bargs.map(_.capt)).result + bindingType(callee, targs, vargs, bargs) } def bindingType(bind: Binding.ImpureApp): ValueType = bind match { case Binding.ImpureApp(id, callee, targs, vargs, bargs) => - Type.instantiate(callee.tpe.asInstanceOf[core.BlockType.Function], targs, bargs.map(_.capt)).result + bindingType(callee, targs, vargs, bargs) } + def bindingType(callee: BlockVar, targs: List[ValueType], vargs: List[Expr], bargs: List[Block]): ValueType = + Type.instantiate(callee.tpe.asInstanceOf[core.BlockType.Function], targs, bargs.map(_.capt)).result + def inferType(stmt: Stmt): ValueType = stmt match { case Stmt.Def(id, block, body) => body.tpe case Stmt.Let(id, tpe, binding, body) => body.tpe diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 63783acb3..d0fee44c6 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -650,24 +650,20 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def evaluate(expr: Expr)(using env: Env, scope: Scope): Addr = expr match { - case core.Pure.ValueVar(id, annotatedType) => + case Expr.ValueVar(id, annotatedType) => env.lookupValue(id) - case core.Pure.Literal(value, annotatedType) => + case core.Expr.Literal(value, annotatedType) => scope.allocate("x", Value.Literal(value, annotatedType)) // right now everything is stuck... no constant folding ... - case core.Pure.PureApp(f, targs, vargs) => + case core.Expr.PureApp(f, targs, vargs) => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate))) - case DirectApp(f, targs, vargs, bargs) => - assert(bargs.isEmpty) - scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Empty))) - - case core.Pure.Make(data, tag, targs, vargs) => + case core.Expr.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) - case core.Pure.Box(b, annotatedCapture) => + case core.Expr.Box(b, annotatedCapture) => val comp = evaluate(b, "x", Stack.Empty) scope.allocate("x", Value.Box(comp, annotatedCapture)) } @@ -684,6 +680,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { bind(id, res) { evaluate(body, k, ks) } }, ks) + case Stmt.ImpureApp(id, f, targs, vargs, bargs, body) => + assert(bargs.isEmpty) + val addr = scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Empty))) + evaluate(body, k, ks)(using env.bindValue(id, addr), scope) + case Stmt.Let(id, annotatedTpe, binding, body) => bind(id, evaluate(binding)) { evaluate(body, k, ks) } @@ -963,21 +964,23 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val coreStmt = embedStmt(stmt)(using G) Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => - val coreExpr = DirectApp(callee, targs, vargs.map(arg => embedPure(arg)(using G)), bargs.map(arg => embedBlock(arg)(using G))) - Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) + val vargs1 = vargs.map(arg => embedPure(arg)(using G)) + val bargs1 = bargs.map(arg => embedBlock(arg)(using G)) + val tpe = Type.bindingType(callee, targs, vargs1, bargs1) + core.ImpureApp(id, callee, targs, vargs1, bargs1, rest(G.bind(id, tpe))) case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => val pureValue = embedPure(addr)(using G) Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) }(G) } - def embedPure(value: Value)(using TypingContext): core.Pure = value match { - case Value.Extern(callee, targs, vargs) => Pure.PureApp(callee, targs, vargs.map(embedPure)) - case Value.Literal(value, annotatedType) => Pure.Literal(value, annotatedType) - case Value.Make(data, tag, targs, vargs) => Pure.Make(data, tag, targs, vargs.map(embedPure)) - case Value.Box(body, annotatedCapture) => Pure.Box(embedBlock(body), annotatedCapture) + def embedPure(value: Value)(using TypingContext): core.Expr = value match { + case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedPure)) + case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) + case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedPure)) + case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) } - def embedPure(addr: Addr)(using G: TypingContext): core.Pure = Pure.ValueVar(addr, G.lookupValue(addr)) + def embedPure(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { case Computation.Var(id) => embedBlockVar(id) From 6a9b7eb4ab7b88a744fb9b8983e77b20ea21e30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 2 Oct 2025 16:15:36 +0200 Subject: [PATCH 030/123] Fix tests --- effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 0734a7ba4..51f50f71a 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -107,7 +107,7 @@ class NewNormalizerTests extends CoreTests { |def run() = { | def f1() = { | def f2() = { - | let x = !(foo: () => Int @ {io})() + | let ! x = (foo: () => Int @ {io})() | return x: Int | } | let y = box {io} f2: () => Int @ {io} @@ -120,6 +120,7 @@ class NewNormalizerTests extends CoreTests { | |""".stripMargin) + val (mainId, actual) = normalize(input) assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) From 853302c34ff9a5704bf5a3fcf26f8558261bc8a3 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:58:57 +0200 Subject: [PATCH 031/123] progress on local state --- .../effekt/core/optimizer/NewNormalizer.scala | 96 +++++++++++++------ 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index d0fee44c6..464a34fba 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -291,7 +291,7 @@ object semantics { // continuation is unknown case Resume(k: Id, body: BasicBlock) - // case Var(id: Id, init: Id, body: BasicBlock) + case Var(id: BlockParam, init: Addr, body: BasicBlock) // case Get // case Put @@ -308,6 +308,7 @@ object semantics { case NeutralStmt.Reset(prompt, body) => body.free - prompt.id case NeutralStmt.Shift(prompt, capt, k, body) => (body.free - k.id) + prompt case NeutralStmt.Resume(k, body) => Set(k) ++ body.free + case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id case NeutralStmt.Hole(span) => Set.empty } } @@ -323,6 +324,7 @@ object semantics { case Frame.Return => ks match { case Stack.Empty => NeutralStmt.Return(arg) case Stack.Reset(p, k, ks) => k.ret(ks, arg) + case Stack.Var(id, curr, init, k, ks) => k.ret(ks, arg) } case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } @@ -351,38 +353,58 @@ object semantics { enum Stack { case Empty case Reset(prompt: BlockParam, frame: Frame, next: Stack) - // case Var(id: Id, addr: Addr, next: Stack) TODO - //case Region(bindings: ???) - - var store = Map.empty[Id, Addr] + case Var(id: BlockParam, curr: Addr, init: Addr, frame: Frame, next: Stack) + // TODO desugar regions into var? + // case Region(bindings: Map[Id, Addr]) lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil - case Stack.Reset(prompt, stack, next) => prompt :: next.bound + case Stack.Reset(prompt, frame, next) => prompt :: next.bound + case Stack.Var(id, curr, init, frame, next) => id :: next.bound } } + def get(id: Id, ks: Stack): Addr = ks match { + case Stack.Empty => ??? // TODO + case Stack.Reset(prompt, frame, next) => get(id, next) + case Stack.Var(id1, curr, init, frame, next) if id == id1.id => curr + case Stack.Var(id1, curr, init, frame, next) => get(id, next) + } + + def put(id: Id, value: Addr, ks: Stack): Stack = ks match { + case Stack.Empty => ??? // TODO + case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, put(id, value, next)) + case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Stack.Var(id1, value, init, frame, next) + case Stack.Var(id1, curr, init, frame, next) => Stack.Var(id1, curr, init, frame, put(id, value, next)) + } + enum Cont { case Empty - case Reset(frame: Frame, prompt: BlockParam, store: Store, rest: Cont) + case Reset(frame: Frame, prompt: BlockParam, rest: Cont) + case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) } def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" case Stack.Reset(prompt, frame, next) if prompt.id == p => - (Cont.Reset(k, prompt, ks.store, Cont.Empty), frame, next) + (Cont.Reset(k, prompt, Cont.Empty), frame, next) case Stack.Reset(prompt, frame, next) => val (c, frame2, stack) = shift(p, frame, next) - (Cont.Reset(k, prompt, ks.store, c), frame2, stack) + (Cont.Reset(k, prompt, c), frame2, stack) + case Stack.Var(id, curr, init, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Var(k, id, curr, init, c), frame2, stack) } def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { case Cont.Empty => (k, ks) - case Cont.Reset(frame, prompt, store, rest) => + case Cont.Reset(frame, prompt, rest) => val (k1, ks1) = resume(rest, frame, ks) val stack = Stack.Reset(prompt, k1, ks1) - stack.store = store (frame, stack) + case Cont.Var(frame, id, curr, init, rest) => + val (k1, ks1) = resume(rest, frame, ks) + (frame, Stack.Var(id, curr, init, k1, ks1)) } def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { @@ -411,6 +433,8 @@ object semantics { case Stack.Empty => Stack.Empty case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) + case Stack.Var(id, curr, init, frame, next) => + Stack.Var(id, curr, init, reifyFrame(frame, next), reifyStack(next)) } f(reifyFrame(k, ks))(reifyStack(ks)) } @@ -442,6 +466,11 @@ object semantics { if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) else stmt // TODO this runs normalization a second time in the outer scope! }} + case Stack.Var(id, curr, init, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + NeutralStmt.Var(id, init, body) + }} } object PrettyPrinter extends ParenPrettyPrinter { @@ -482,6 +511,9 @@ object semantics { case NeutralStmt.Resume(k, body) => "resume" <> parens(toDoc(k)) <+> toDoc(body) + case NeutralStmt.Var(id, init, body) => + "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body) + case NeutralStmt.Hole(span) => "hole()" } @@ -801,13 +833,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO case Stmt.Var(ref, init, capture, body) => val addr = evaluate(init) - ks.store = ks.store.updated(ref, addr) - evaluate(body, k, ks) + evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, addr, k, ks)) case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => - bind(id, ks.store(ref)) { evaluate(body, k, ks) } + util.trace(ks) + bind(id, get(ref, ks)) { evaluate(body, k, ks) } case Stmt.Put(ref, annotatedCapt, value, body) => - ks.store = ks.store.updated(ref, evaluate(value)) - evaluate(body, k, ks) + evaluate(body, k, put(ref, evaluate(value), ks)) // Control Effects case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => @@ -868,6 +899,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def run(mod: ModuleDecl): ModuleDecl = { + util.trace(mod) // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } val toplevelEnv = Env.empty @@ -920,17 +952,17 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { case NeutralStmt.Return(result) => - Stmt.Return(embedPure(result)) + Stmt.Return(embedExpr(result)) case NeutralStmt.Jump(label, targs, vargs, bargs) => - Stmt.App(embedBlockVar(label), targs, vargs.map(embedPure), bargs.map(embedBlock)) + Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) case NeutralStmt.App(label, targs, vargs, bargs) => - Stmt.App(embedBlockVar(label), targs, vargs.map(embedPure), bargs.map(embedBlock)) + Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => - Stmt.Invoke(embedBlockVar(label), method, tpe, targs, vargs.map(embedPure), bargs.map(embedBlock)) + Stmt.Invoke(embedBlockVar(label), method, tpe, targs, vargs.map(embedExpr), bargs.map(embedBlock)) case NeutralStmt.If(cond, thn, els) => - Stmt.If(embedPure(cond), embedStmt(thn), embedStmt(els)) + Stmt.If(embedExpr(cond), embedStmt(thn), embedStmt(els)) case NeutralStmt.Match(scrutinee, clauses, default) => - Stmt.Match(embedPure(scrutinee), + Stmt.Match(embedExpr(scrutinee), clauses.map { case (id, block) => id -> embedBlockLit(block) }, default.map(embedStmt)) case NeutralStmt.Reset(prompt, body) => @@ -943,6 +975,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputations(k :: Nil)))) case NeutralStmt.Resume(k, body) => Stmt.Resume(embedBlockVar(k), embedStmt(body)) + case NeutralStmt.Var(id, init, body) => + val capt = id.capt match { + case cs if cs.size == 1 => cs.head + case _ => ??? // TODO + } + Stmt.Var(id.id, embedExpr(init), capt, embedStmt(body)) case NeutralStmt.Hole(span) => Stmt.Hole(span) } @@ -951,7 +989,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case BasicBlock(bindings, stmt) => bindings.foldRight((G: TypingContext) => embedStmt(stmt)(using G)) { case ((id, Binding.Let(value)), rest) => G => - val coreExpr = embedPure(value)(using G) + val coreExpr = embedExpr(value)(using G) // TODO why do we even have this type in core, if we always infer it? Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) case ((id, Binding.Def(block)), rest) => G => @@ -964,23 +1002,23 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val coreStmt = embedStmt(stmt)(using G) Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => - val vargs1 = vargs.map(arg => embedPure(arg)(using G)) + val vargs1 = vargs.map(arg => embedExpr(arg)(using G)) val bargs1 = bargs.map(arg => embedBlock(arg)(using G)) val tpe = Type.bindingType(callee, targs, vargs1, bargs1) core.ImpureApp(id, callee, targs, vargs1, bargs1, rest(G.bind(id, tpe))) case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => - val pureValue = embedPure(addr)(using G) + val pureValue = embedExpr(addr)(using G) Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) }(G) } - def embedPure(value: Value)(using TypingContext): core.Expr = value match { - case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedPure)) + def embedExpr(value: Value)(using TypingContext): core.Expr = value match { + case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) - case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedPure)) + case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) } - def embedPure(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) + def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { case Computation.Var(id) => embedBlockVar(id) From e5ec970bfcc0fb0caa1938c83dd7e38d1f3b6130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 6 Oct 2025 13:37:16 +0200 Subject: [PATCH 032/123] Add simple test case for mutable variables --- .../effekt/core/NewNormalizerTests.scala | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 51f50f71a..42e7298af 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -204,6 +204,38 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) } + + test("Mutable variable use that could be constant folded") { + val input = + """ + |def run(): Int = { + | var x = 41 + | x = x + 1 + | return x + |} + | + |def main() = println(run()) + |""".stripMargin + + val expected = + parse(""" + |module input + | + |extern {} def infixAdd(x: Int, y: Int): Int = vm "" + | + |def run() = { + | let x1 = 41 + | let x2 = 1 + | let x3 = (infixAdd: (Int, Int) => Int @ {})(x1: Int, x2: Int) + | return x3: Int + |} + |""".stripMargin + ) + + val (mainId, actual) = normalize(input) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), List("infixAdd")) + } } /** From 41de27df511b1f29af55e4f61f1a948f11df0f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 6 Oct 2025 13:40:01 +0200 Subject: [PATCH 033/123] Throw proper errors in impossible cases --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 464a34fba..445d01716 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -365,14 +365,14 @@ object semantics { } def get(id: Id, ks: Stack): Addr = ks match { - case Stack.Empty => ??? // TODO + case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(id)} in empty stack" case Stack.Reset(prompt, frame, next) => get(id, next) case Stack.Var(id1, curr, init, frame, next) if id == id1.id => curr case Stack.Var(id1, curr, init, frame, next) => get(id, next) } def put(id: Id, value: Addr, ks: Stack): Stack = ks match { - case Stack.Empty => ??? // TODO + case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(id)} in empty stack" case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, put(id, value, next)) case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Stack.Var(id1, value, init, frame, next) case Stack.Var(id1, curr, init, frame, next) => Stack.Var(id1, curr, init, frame, put(id, value, next)) From 1cc680db5dbc510acc278529ba2ba4e3ada0afee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 6 Oct 2025 13:45:18 +0200 Subject: [PATCH 034/123] Throw proper error in impossible case --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 445d01716..4c2b58fb3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -978,7 +978,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case NeutralStmt.Var(id, init, body) => val capt = id.capt match { case cs if cs.size == 1 => cs.head - case _ => ??? // TODO + case _ => sys error "Variable needs to have a single capture" } Stmt.Var(id.id, embedExpr(init), capt, embedStmt(body)) case NeutralStmt.Hole(span) => From cf1eef6e8fc8fa48e5bade746307b7570fa4e2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 7 Oct 2025 10:50:50 +0200 Subject: [PATCH 035/123] Add test for mutating peano Nats --- .../effekt/core/NewNormalizerTests.scala | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 42e7298af..baea83b2e 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -39,18 +39,30 @@ class NewNormalizerTests extends CoreTests { actual: ModuleDecl, expected: ModuleDecl, defNames: List[String], - externNames: List[String] + externNames: List[String] = List(), + declNames: List[String] = List(), + ctorNames: List[(String, String)] = List() )(using Location): Unit = { def findDef(mod: ModuleDecl, name: String) = mod.definitions.find(_.id.name.name == name) .getOrElse(throw new NoSuchElementException(s"Definition '$name' not found")) + def findDecl(mod: ModuleDecl, name: String)= + mod.declarations.find(_.id.name.name == name) + .getOrElse(throw new NoSuchElementException(s"Declaration '$name' not found")) + + def findCtor(data: Data, name: String) = + data.constructors.find(_.id.name.name == name) + .getOrElse(throw new NoSuchElementException( + s"Constructor '$name' not found in data '${data.id.name.name}'" + )) + def findExternDef(mod: ModuleDecl, name: String) = mod.externs.collect { case d@Extern.Def(_, _, _, _, _, _, _, _) => d } .find(_.id.name.name == name) .getOrElse(throw new NoSuchElementException(s"Extern def '$name' not found")) - + val externPairs: List[(Id, Id)] = externNames.flatMap { name => val canon = Id(name) @@ -60,13 +72,43 @@ class NewNormalizerTests extends CoreTests { ) } + val declPairs: List[(Id, Id)] = + declNames.flatMap { name => + val canon = Id(name) + List( + findDecl(actual, name).id -> canon, + findDecl(expected, name).id -> canon + ) + } + + val ctorPairs: List[(Id, Id)] = + ctorNames.flatMap { case (dataName, ctorName) => + val canon = Id(ctorName) + val actualData = findDecl(actual, dataName) match { + case d: Data => d + case _: Interface => throw new IllegalArgumentException( + s"Expected data declaration for '$dataName', found interface" + ) + } + val expectedData = findDecl(expected, dataName) match { + case d: Data => d + case _: Interface => throw new IllegalArgumentException( + s"Expected data declaration for '$dataName', found interface" + ) + } + List( + findCtor(actualData, ctorName).id -> canon, + findCtor(expectedData, ctorName).id -> canon + ) + } + def compareOneDef(name: String): Unit = { val aDef = findDef(actual, name) val eDef = findDef(expected, name) val canon = Id(name) val pairs: Map[Id, Id] = - (List(aDef.id -> canon, eDef.id -> canon) ++ externPairs).toMap + (List(aDef.id -> canon, eDef.id -> canon) ++ externPairs ++ declPairs ++ ctorPairs).toMap val renamer = TestRenamer(Names(defaultNames), "$", List(pairs)) shouldBeEqual( @@ -131,7 +173,7 @@ class NewNormalizerTests extends CoreTests { test("box passed to extern") { val input = """ - |extern {io} def foo(f: => Int at {}): Int = vm"42" + |extern def foo(f: => Int at {}) at {io}: Int = vm"42" | |def run(): Int = { | val f = box { @@ -157,8 +199,7 @@ class NewNormalizerTests extends CoreTests { | return x: Int | } | let f_box: => Int at {} = box {} (f: => Int @ {}) - | let x: Int = - | ! (foo: (=> Int at {}) => Int @ {io})(f_box: => Int at {}) + | let ! x = (foo: (=> Int at {}) => Int @ {io})(f_box: => Int at {}) | | return x: Int |} @@ -175,7 +216,7 @@ class NewNormalizerTests extends CoreTests { test("unbox blocked by extern") { val input = """ - |extern {} def foo(): => Int at {} = vm"42" + |extern def foo() at {}: => Int at {} = vm"42" | |def run(): Int = { | val x = foo()() @@ -205,7 +246,9 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) } - test("Mutable variable use that could be constant folded") { + // One might expect the following example to constant fold. + // However, extern definitions such as infixAdd are currently always neutral. + test("Extern infixAdd blocks constant folding of mutable variable") { val input = """ |def run(): Int = { @@ -236,6 +279,50 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("infixAdd")) } + + test("Mutable Peano Nats turn into let-bindings") { + val input = + """ + |type Nat { + | Z() + | S(pred: Nat) + |} + | + |def toInt(n: Nat): Int = n match { + | case Z() => 0 + | case S(pred) => 1 + toInt(pred) + |} + | + |def run(): Nat = { + | var x = Z() + | x = S(x) + | return x + |} + | + |def main() = println(run().toInt()) + |""".stripMargin + + val expected = + parse(""" + |module input + | + |type Nat { + | Z() + | S(pred: Nat) + |} + | + |def run() = { + | let x1 = make Nat Z() + | let x2 = make Nat S(x1: Nat) + | return x2: Nat + |} + |""".stripMargin + ) + + val (mainId, actual) = normalize(input) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), List(), List("Nat"), List(("Nat", "Z"), ("Nat", "S"))) + } } /** From 74071b53ad8dfccd4769c536eed27963973b3fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 14 Oct 2025 10:13:09 +0200 Subject: [PATCH 036/123] Work on normalizing state * Separate Empty and Unknown stacks * Handle neutral variables * Handle neutral get * Add test case for getting mutable vars under shift/reset * Add missing functionality to core parser and test renamer --- .../effekt/core/NewNormalizerTests.scala | 63 +++++++++++- .../effekt/core/optimizer/NewNormalizer.scala | 95 +++++++++++++------ 2 files changed, 128 insertions(+), 30 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index baea83b2e..e25d45d5a 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -62,7 +62,7 @@ class NewNormalizerTests extends CoreTests { mod.externs.collect { case d@Extern.Def(_, _, _, _, _, _, _, _) => d } .find(_.id.name.name == name) .getOrElse(throw new NoSuchElementException(s"Extern def '$name' not found")) - + val externPairs: List[(Id, Id)] = externNames.flatMap { name => val canon = Id(name) @@ -280,6 +280,7 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("infixAdd")) } + // This test case shows mutable variable assignments turning into static let-bindings test("Mutable Peano Nats turn into let-bindings") { val input = """ @@ -323,6 +324,66 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List(), List("Nat"), List(("Nat", "Z"), ("Nat", "S"))) } + + // This test case shows a mutable variable that is captured in an effect handler. + // The resulting core code shows how the reference is passed to the handler block. + // Even though the variable is not mutated, the normalizer currently cannot eliminate the reference. + // This is because the stack used to normalize the handler is currently treated as "unknown". + test("Mutable variable read in handler") { + val input = + """ + |effect bar: Unit + | + |extern def foo(x: Int) at {io}: Unit = vm"" + | + |def run() = { + | var x = 1 + | try { + | do bar() + | } with bar { + | foo(x) + | } + |} + | + |def main() = println(run()) + | + |""".stripMargin + + val (mainId, actual) = normalize(input) + + val expected = + parse(""" + |module input + | + |interface bar { + | bar: => Unit + |} + | + |extern {io} def foo(x: Int): Unit = vm"" + | + |def run() = { + | let y = 1 + | + | def handle(){q @ p: Prompt[Unit]}{s @ r: Ref[Int]} = + | shift (q: Prompt[Unit] @ {p}) { + | () { k @ p: Resume[Unit, Unit]} => { + | get z: Int = !s @ r; + | let ! o = (foo: (Int) => Unit @ {io})(z: Int) + | return o: Unit + | } + | } + | + | var z @ x = y: Int; + | reset { + | () { q @ p: Prompt[Unit] } => + | (handle: { p: Prompt[Unit] }{ x: Ref[Int] } => Unit @ {})() { q: Prompt[Unit] @ { p } } { z: Ref[Int] @ { x } } + | } + |} + |""".stripMargin + ) + + assertAlphaEquivalentToplevels(actual, expected, List("run"), declNames=List("bar"), externNames=List("foo")) + } } /** diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4c2b58fb3..46e604ba1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -4,7 +4,8 @@ package optimizer import effekt.core.ValueType.Boxed import effekt.source.Span -import effekt.core.optimizer.semantics.{Computation, NeutralStmt} +import effekt.core.optimizer.semantics.{Computation, NeutralStmt, Value} +import effekt.symbols.builtins import effekt.util.messages.{ErrorReporter, INTERNAL_ERROR} import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter @@ -47,7 +48,7 @@ object semantics { enum Value { // Stuck - //case Var(id: Id, annotatedType: ValueType) + case Var(id: Id, annotatedType: ValueType) case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) // Actual Values @@ -57,7 +58,7 @@ object semantics { case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) val free: Variables = this match { - // case Value.Var(id, annotatedType) => Variables.empty + case Value.Var(id, annotatedType) => Set.empty case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Make(data, tag, targs, vargs) => vargs.toSet @@ -73,6 +74,7 @@ object semantics { case Val(stmt: NeutralStmt) case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) case Unbox(addr: Addr, tpe: BlockType, capt: Captures) + case Get(ref: Id, tpe: ValueType, cap: Captures) val free: Variables = this match { case Binding.Let(value) => value.free @@ -81,6 +83,7 @@ object semantics { case Binding.Val(stmt) => stmt.free case Binding.Run(f, targs, vargs, bargs) => vargs.toSet ++ all(bargs, _.free) case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set(addr) + case Binding.Get(ref, tpe, cap) => Set(ref) } } @@ -115,6 +118,12 @@ object semantics { addr } + def allocateGet(ref: Id, tpe: ValueType, cap: Captures): Addr = { + val addr = Id("get") + bindings = bindings.updated(addr, Binding.Get(ref, tpe, cap)) + addr + } + def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]): Addr = { val addr = Id(hint) bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs, bargs)) @@ -173,6 +182,9 @@ object semantics { case (addr, b: Binding.Unbox) => used = used ++ b.free filtered = (addr, b):: filtered + case (addr, g: Binding.Get) => + used = used ++ g.free + filtered = (addr, g) :: filtered } // we want to avoid turning tailcalls into non tail calls like @@ -233,6 +245,7 @@ object semantics { case (id, b: Binding.Val) => free = (free - id) ++ b.free case (id, b: Binding.Run) => free = (free - id) ++ b.free case (id, b: Binding.Unbox) => free = (free - id) ++ b.free + case (id, b: Binding.Get) => free = (free - id) ++ b.free } free } @@ -292,7 +305,6 @@ object semantics { case Resume(k: Id, body: BasicBlock) case Var(id: BlockParam, init: Addr, body: BasicBlock) - // case Get // case Put // aborts at runtime @@ -320,9 +332,12 @@ object semantics { case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) case Dynamic(closure: Closure) + /* Return an argument `arg` through this frame and the rest of the stack `ks` + */ def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { case Frame.Return => ks match { case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Unknown => NeutralStmt.Return(arg) case Stack.Reset(p, k, ks) => k.ret(ks, arg) case Stack.Var(id, curr, init, k, ks) => k.ret(ks, arg) } @@ -351,7 +366,15 @@ object semantics { // // where the frame on the reset is the one AFTER the prompt NOT BEFORE! enum Stack { + /** + * Statically known to be empty + * This only occurs at the entrypoint of normalization. + * In other cases, where the stack is not known, you should use Unknown instead. + */ case Empty + /** Dynamic tail (we do not know the shape of the remaining stack) + */ + case Unknown case Reset(prompt: BlockParam, frame: Frame, next: Stack) case Var(id: BlockParam, curr: Addr, init: Addr, frame: Frame, next: Stack) // TODO desugar regions into var? @@ -359,20 +382,24 @@ object semantics { lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil + case Stack.Unknown => Nil case Stack.Reset(prompt, frame, next) => prompt :: next.bound case Stack.Var(id, curr, init, frame, next) => id :: next.bound } } - def get(id: Id, ks: Stack): Addr = ks match { - case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(id)} in empty stack" - case Stack.Reset(prompt, frame, next) => get(id, next) - case Stack.Var(id1, curr, init, frame, next) if id == id1.id => curr - case Stack.Var(id1, curr, init, frame, next) => get(id, next) + def get(ref: Id, ks: Stack): Option[Addr] = ks match { + case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(ref)} in empty stack" + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => get(ref, next) + case Stack.Var(id1, curr, init, frame, next) if ref == id1.id => Some(curr) + case Stack.Var(id1, curr, init, frame, next) => get(ref, next) } def put(id: Id, value: Addr, ks: Stack): Stack = ks match { case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(id)} in empty stack" + case Stack.Unknown => sys error s"Cannot put ${util.show(id)} in unknown stack" case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, put(id, value, next)) case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Stack.Var(id1, value, init, frame, next) case Stack.Var(id1, curr, init, frame, next) => Stack.Var(id1, curr, init, frame, put(id, value, next)) @@ -386,6 +413,7 @@ object semantics { def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" + case Stack.Unknown => sys error s"Cannot find prompt ${util.show(p)} in unknown stack" case Stack.Reset(prompt, frame, next) if prompt.id == p => (Cont.Reset(k, prompt, Cont.Empty), frame, next) case Stack.Reset(prompt, frame, next) => @@ -407,12 +435,11 @@ object semantics { (frame, Stack.Var(id, curr, init, k1, ks1)) } - def joinpoint(k: Frame, ks: Stack)(f: Frame => Stack => NeutralStmt)(using scope: Scope): NeutralStmt = { - + def joinpoint(k: Frame, ks: Stack)(f: (Frame, Stack) => NeutralStmt)(using scope: Scope): NeutralStmt = { def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { case Frame.Static(tpe, apply) => val x = Id("x") - nested { scope ?=> apply(scope)(x)(Stack.Empty) } match { + nested { scope ?=> apply(scope)(x)(Stack.Unknown) } match { // Avoid trivial continuations like // def k_6268 = (x_6267: Int_3) { // return x_6267 @@ -431,12 +458,13 @@ object semantics { def reifyStack(ks: Stack): Stack = ks match { case Stack.Empty => Stack.Empty + case Stack.Unknown => Stack.Unknown case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) case Stack.Var(id, curr, init, frame, next) => Stack.Var(id, curr, init, reifyFrame(frame, next), reifyStack(next)) } - f(reifyFrame(k, ks))(reifyStack(ks)) + f(reifyFrame(k, ks), reifyStack(ks)) } def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = @@ -448,7 +476,7 @@ object semantics { case Frame.Static(tpe, apply) => val tmp = Id("tmp") scope.push(tmp, stmt) - apply(scope)(tmp)(Stack.Empty) + apply(scope)(tmp)(Stack.Unknown) case Frame.Dynamic(Closure(label, closure)) => val tmp = Id("tmp") scope.push(tmp, stmt) @@ -459,6 +487,7 @@ object semantics { final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = ks match { case Stack.Empty => stmt + case Stack.Unknown => stmt // only reify reset if p is free in body case Stack.Reset(prompt, frame, next) => reify(next) { reify(frame) { // reify(next) { reify(store) { reify(frame) { ... }}} ??? ORDER? TODO @@ -536,6 +565,8 @@ object semantics { case Value.Box(body, tpe) => "box" <+> braces(nest(line <> toDoc(body) <> line)) + + case Value.Var(id, tpe) => toDoc(id) } def toDoc(block: Block): Doc = block match { @@ -567,6 +598,7 @@ object semantics { (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line + case (addr, Binding.Get(ref, tpe, cap)) => "let" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line }) def toDoc(block: BasicBlock): Doc = @@ -606,7 +638,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindComputation(id, Computation.Var(freshened)) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Empty) + evaluate(body, Frame.Return, Stack.Unknown) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -626,7 +658,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Empty) + evaluate(body, Frame.Return, Stack.Unknown) }) val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } @@ -696,7 +728,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) case core.Expr.Box(b, annotatedCapture) => - val comp = evaluate(b, "x", Stack.Empty) + val comp = evaluate(b, "x", Stack.Unknown) scope.allocate("x", Value.Box(comp, annotatedCapture)) } @@ -714,7 +746,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.ImpureApp(id, f, targs, vargs, bargs, body) => assert(bargs.isEmpty) - val addr = scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Empty))) + val addr = scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Unknown))) evaluate(body, k, ks)(using env.bindValue(id, addr), scope) case Stmt.Let(id, annotatedTpe, binding, body) => @@ -742,7 +774,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.App(callee, targs, vargs, bargs) => // Here the stack passed to the blocks is an empty one since we reify it anyways... - val escapingStack = Stack.Empty + val escapingStack = Stack.Unknown evaluate(callee, "f", escapingStack) match { case Computation.Inline(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => val newEnv = closureEnv @@ -761,7 +793,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // case Stmt.Invoke(New) case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => - val escapingStack = Stack.Empty + val escapingStack = Stack.Unknown evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } @@ -778,7 +810,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) case Some(Value.Literal(false, _)) => evaluate(els, k, ks) case _ => - joinpoint(k, ks) { k => ks => + joinpoint(k, ks) { (k, ks) => NeutralStmt.If(sc, nested { evaluate(thn, k, ks) }, nested { @@ -810,7 +842,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // }, // default.map { stmt => nested { evaluate(stmt, k, ks) } }) case _ => - joinpoint(k, ks) { k => ks => + joinpoint(k, ks) { (k, ks) => NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -835,8 +867,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val addr = evaluate(init) evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, addr, k, ks)) case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => - util.trace(ks) - bind(id, get(ref, ks)) { evaluate(body, k, ks) } + get(ref, ks) match { + case Some(addr) => bind(id, addr) { evaluate(body, k, ks) } + case None => bind(id, scope.allocateGet(ref, annotatedTpe, annotatedCapt)) { evaluate(body, k, ks) } + } case Stmt.Put(ref, annotatedCapt, value, body) => evaluate(body, k, put(ref, evaluate(value), ks)) @@ -855,7 +889,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val neutralBody = { given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) nested { - evaluate(body, Frame.Return, Stack.Empty) + evaluate(body, Frame.Return, Stack.Unknown) } } assert(Set(cparam) == k2.capt, "At least for now these need to be the same") @@ -888,7 +922,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Var(r) => reify(k, ks) { NeutralStmt.Resume(r, nested { - evaluate(body, Frame.Return, Stack.Empty) + evaluate(body, Frame.Return, Stack.Unknown) }) } case Computation.Continuation(k3) => @@ -975,12 +1009,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputations(k :: Nil)))) case NeutralStmt.Resume(k, body) => Stmt.Resume(embedBlockVar(k), embedStmt(body)) - case NeutralStmt.Var(id, init, body) => - val capt = id.capt match { + case NeutralStmt.Var(blockParam, init, body) => + val capt = blockParam.capt match { case cs if cs.size == 1 => cs.head case _ => sys error "Variable needs to have a single capture" } - Stmt.Var(id.id, embedExpr(init), capt, embedStmt(body)) + Stmt.Var(blockParam.id, embedExpr(init), capt, embedStmt(body)(using G.bind(blockParam.id, blockParam.tpe, blockParam.capt))) case NeutralStmt.Hole(span) => Stmt.Hole(span) } @@ -1009,6 +1043,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => val pureValue = embedExpr(addr)(using G) Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) + case ((id, Binding.Get(ref, tpe, cap)), rest) => G => + Stmt.Get(id, tpe, ref, cap, rest(G.bind(id, tpe)) ) }(G) } @@ -1017,6 +1053,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) + case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) } def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) From 044010e2a6bd8f7b39259aaeb6850f285920b77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 14 Oct 2025 11:46:01 +0200 Subject: [PATCH 037/123] Add test case for passing mutable var to id --- .../effekt/core/NewNormalizerTests.scala | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index e25d45d5a..e844f98d4 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -384,6 +384,44 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), declNames=List("bar"), externNames=List("foo")) } + + // This test case shows a mutable variable passed to the identity function. + // Currently, the normalizer is not able to see through the identity function, + // but it does ignore the mutable variable and just passes the initial value. + test("Pass mutable variable to identity function uses let binding") { + val input = + """ + |def run(): Int = { + | def f(x: Int) = x + | var x = 42 + | f(x) + |} + | + |def main() = println(run()) + | + |""".stripMargin + + val (mainId, actual) = normalize(input) + + val expected = + parse( + """ + |module input + | + |def run() = { + | def f(x: Int) = { + | return x: Int + | } + | + | let y = 42 + | var x @ z = y: Int; + | (f : (Int) => Int @ {})(y: Int) + |} + |""".stripMargin + ) + + assertAlphaEquivalentToplevels(actual, expected, List("run")) + } } /** From c20758519c5f3d41f7dd84cb40e98cb3c523e976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 14 Oct 2025 12:06:56 +0200 Subject: [PATCH 038/123] Add test case for tracked mutation --- .../effekt/core/NewNormalizerTests.scala | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index e844f98d4..7408188f5 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -422,6 +422,44 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run")) } + + // This test shows that when the value of a mutable variable is known, we can directly pass the let-bound value. + test("Mutate mutable variable before passing it to identity function") { + val input = + """ + |def run(): Int = { + | def f(x: Int) = x + | var x = 42 + | x = 43 + | f(x) + |} + | + |def main() = println(run()) + | + |""".stripMargin + + val (mainId, actual) = normalize(input) + + val expected = + parse( + """ + |module input + | + |def run() = { + | def f(x: Int) = { + | return x: Int + | } + | + | let y = 42 + | let w = 43 + | var x @ z = y: Int; + | (f : (Int) => Int @ {})(w: Int) + |} + |""".stripMargin + ) + + assertAlphaEquivalentToplevels(actual, expected, List("run")) + } } /** From ac047defe1afa788621a7c10b8689c22123be313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 20 Oct 2025 10:15:29 +0200 Subject: [PATCH 039/123] Implement neutral put --- .../effekt/core/NewNormalizerTests.scala | 51 +++++++++++++++++++ .../test/scala/effekt/core/TestRenamer.scala | 32 ++++++++++++ .../effekt/core/optimizer/NewNormalizer.scala | 29 +++++++---- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 7408188f5..f58948263 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -460,6 +460,57 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run")) } + + // This test shows a mutable reference captured by a block parameter. + // During normalization, this block parameter gets lifted to a `def`. + // One might hope for this mutable variable to be eliminated entirely, + // but currently the normalizer does not inline definitions. + test("Mutable reference can be lifted") { + val input = + """ + |def run(): Int = { + | def modifyProg { setter: Int => Unit }: Unit = { + | setter(2) + | () + | } + | var x = 1 + | modifyProg { y => x = y } + | x + |} + | + |def main() = println(run()) + |""".stripMargin + + val (mainId, actual) = normalize(input) + + val expected = + parse( + """ + |module input + | + |def run() = { + | def modifyProg(){setter: (Int) => Unit} = { + | let x = 2 + | val tmp: Unit = (setter: (Int) => Unit @ {setter})(x: Int); + | let y = () + | return y: Unit + | } + | let y = 1 + | var x @ c = y: Int; + | def f(y: Int) = { + | put x @ c = y: Int; + | let z = () + | return z: Unit + | } + | val tmp: Unit = (modifyProg: (){setter : (Int) => Unit} => Unit @ {})(){f: (Int) => Unit @ {c}}; + | get o: Int = !x @ c; + | return o: Int + |} + |""".stripMargin + ) + + assertAlphaEquivalentToplevels(actual, expected, List("run")) + } } /** diff --git a/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala b/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala index 0d44dde01..612446cdb 100644 --- a/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala +++ b/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala @@ -112,6 +112,12 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends val resolvedCapt = rewrite(capt) withBinding(id) { core.Get(rewrite(id), rewrite(tpe), resolvedRef, resolvedCapt, rewrite(body)) } + case core.App(callee, targs, vargs, bargs) => + val resolvedCallee = rewrite(callee) + val resolvedTargs = targs map rewrite + val resolvedVargs = vargs map rewrite + val resolvedBargs = bargs map rewrite + core.App(resolvedCallee, resolvedTargs, resolvedVargs, resolvedBargs) } override def block: PartialFunction[Block, Block] = { @@ -120,6 +126,32 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends Block.BlockLit(tparams map rewrite, cparams map rewrite, vparams map rewrite, bparams map rewrite, rewrite(body)) } + case Block.BlockVar(id: Id, annotatedTpe: BlockType, annotatedCapt: Captures) => { + withBinding(id) { + val idOut = rewrite(id) + val annotatedTpeOut = rewrite(annotatedTpe) + val annotatedCaptOut = rewrite(annotatedCapt) + Block.BlockVar(rewrite(id), rewrite(annotatedTpe), rewrite(annotatedCapt)) + } + } + } + + override def rewrite(t: BlockType): BlockType = t match { + case BlockType.Function(tparams, cparams, vparams, bparams, result: ValueType) => + // TODO: is this how we want to treat captures here? + val resolvedCapt = cparams.map(id => Map(id -> freshIdFor(id))).reduceOption(_ ++ _).getOrElse(Map()) + withBindings(tparams) { + withMapping(resolvedCapt) { + BlockType.Function( + tparams.map(rewrite), + resolvedCapt.values.toList.map(rewrite), + vparams.map(rewrite), + bparams.map(rewrite), + rewrite(result) + ) + }} + case BlockType.Interface(name, targs) => + BlockType.Interface(name, targs map rewrite) } override def rewrite(o: Operation): Operation = o match { diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 46e604ba1..c9dcf867a 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -305,7 +305,7 @@ object semantics { case Resume(k: Id, body: BasicBlock) case Var(id: BlockParam, init: Addr, body: BasicBlock) - // case Put + case Put(ref: Id, tpe: ValueType, cap: Captures, value: Addr, body: BasicBlock) // aborts at runtime case Hole(span: Span) @@ -322,6 +322,7 @@ object semantics { case NeutralStmt.Resume(k, body) => Set(k) ++ body.free case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id case NeutralStmt.Hole(span) => Set.empty + case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref, value) ++ body.free } } @@ -396,13 +397,14 @@ object semantics { case Stack.Var(id1, curr, init, frame, next) if ref == id1.id => Some(curr) case Stack.Var(id1, curr, init, frame, next) => get(ref, next) } - - def put(id: Id, value: Addr, ks: Stack): Stack = ks match { + + def put(id: Id, value: Addr, ks: Stack): Option[Stack] = ks match { case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(id)} in empty stack" - case Stack.Unknown => sys error s"Cannot put ${util.show(id)} in unknown stack" - case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, frame, put(id, value, next)) - case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Stack.Var(id1, value, init, frame, next) - case Stack.Var(id1, curr, init, frame, next) => Stack.Var(id1, curr, init, frame, put(id, value, next)) + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => put(id, value, next).map(Stack.Reset(prompt, frame, _)) + case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Some(Stack.Var(id1, value, init, frame, next)) + case Stack.Var(id1, curr, init, frame, next) => put(id, value, next).map(Stack.Var(id1, curr, init, frame, _)) } enum Cont { @@ -544,6 +546,9 @@ object semantics { "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body) case NeutralStmt.Hole(span) => "hole()" + + case NeutralStmt.Put(ref, tpe, cap, value, body) => + "put" <+> toDoc(ref) <+> "=" <+> toDoc(value) <> line <> toDoc(body) } def toDoc(id: Id): Doc = id.show @@ -598,7 +603,7 @@ object semantics { (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line - case (addr, Binding.Get(ref, tpe, cap)) => "let" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line + case (addr, Binding.Get(ref, tpe, cap)) => "get" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line }) def toDoc(block: BasicBlock): Doc = @@ -872,7 +877,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case None => bind(id, scope.allocateGet(ref, annotatedTpe, annotatedCapt)) { evaluate(body, k, ks) } } case Stmt.Put(ref, annotatedCapt, value, body) => - evaluate(body, k, put(ref, evaluate(value), ks)) + put(ref, evaluate(value), ks) match { + case Some(stack) => evaluate(body, k, stack) + case None => + NeutralStmt.Put(ref, value.tpe, annotatedCapt, evaluate(value), nested { evaluate(body, k, ks) }) + } // Control Effects case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => @@ -1017,6 +1026,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Var(blockParam.id, embedExpr(init), capt, embedStmt(body)(using G.bind(blockParam.id, blockParam.tpe, blockParam.capt))) case NeutralStmt.Hole(span) => Stmt.Hole(span) + case NeutralStmt.Put(ref, annotatedTpe, annotatedCapt, value, body) => + Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) } def embedStmt(basicBlock: BasicBlock)(using G: TypingContext): core.Stmt = basicBlock match { From f36d2979676c411ff21394e72e2eba8a2a6caa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 20 Oct 2025 10:33:03 +0200 Subject: [PATCH 040/123] Fix unbox --- .../src/test/scala/effekt/core/NewNormalizerTests.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index f58948263..51d8218d9 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -124,16 +124,15 @@ class NewNormalizerTests extends CoreTests { // This example shows a box that contains an extern reference. // The normalizer is able to unbox this indirection away. test("extern in box") { - val input = - """ + val input = """ |extern def foo: Int = vm"42" | |def run(): Int = { | val f = box { - | foo + | box foo | } at { io } | - | val x = unbox f()() + | val x = /* unbox */ f()() | return x |} | From 4ba304b93af7ee3d47c379b4f617cbda2f562372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 20 Oct 2025 10:37:32 +0200 Subject: [PATCH 041/123] Correct test name --- effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 51d8218d9..4638480e5 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -464,7 +464,7 @@ class NewNormalizerTests extends CoreTests { // During normalization, this block parameter gets lifted to a `def`. // One might hope for this mutable variable to be eliminated entirely, // but currently the normalizer does not inline definitions. - test("Mutable reference can be lifted") { + test("Block param capturing mutable reference can be lifted") { val input = """ |def run(): Int = { From 1bb691853e8182cb3aa7b4c20e736b7f5169285e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 20 Oct 2025 16:30:03 +0200 Subject: [PATCH 042/123] Add failing test case --- .../scala/effekt/core/NewNormalizerTests.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 4638480e5..7d90ff547 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -510,6 +510,22 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run")) } + + // TODO: Currently fails + test("todo") { + val input = + """ + |def run() = { + | var k = 2; + | while (k <= 1) {} + | 0 + |} + | + |def main() = println(run()) + |""".stripMargin + + val (mainId, actual) = normalize(input) + } } /** From f8e86137d23d1e792181b2eb0b9909a9d9ceb676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 21 Oct 2025 10:40:58 +0200 Subject: [PATCH 043/123] Fix captures when embedding recursive bindings --- .../test/scala/effekt/core/NewNormalizerTests.scala | 13 ++++++++----- .../scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 7d90ff547..a6e745407 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -511,20 +511,23 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run")) } - // TODO: Currently fails - test("todo") { + // This test case shows that we can normalize a potentially recursive block that never actually recurses. + // The while loop in the surface language is translated to a recursive block in core. + // Note that the variable `v` and its capture need to be correctly passed around. + test("Can normalize while loop with non-satisfiable condition") { val input = """ |def run() = { - | var k = 2; - | while (k <= 1) {} + | var v = 2; + | while (v <= 1) {} | 0 |} | |def main() = println(run()) |""".stripMargin - val (mainId, actual) = normalize(input) + // Does not throw + normalize(input) } } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index c9dcf867a..11e111fa0 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1042,7 +1042,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) case ((id, Binding.Rec(block, tpe, capt)), rest) => G => val coreBlock = embedBlock(block)(using G.bind(id, tpe, capt)) - Stmt.Def(id, coreBlock, rest(G.bind(id, tpe, capt))) + Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) case ((id, Binding.Val(stmt)), rest) => G => val coreStmt = embedStmt(stmt)(using G) Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) From 6dab62f4e19bafd2843587843fb01c3da3da8809 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 21 Oct 2025 12:02:26 +0200 Subject: [PATCH 044/123] add some more tests --- .../effekt/core/NewNormalizerTests.scala | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index a6e745407..505377708 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -529,6 +529,57 @@ class NewNormalizerTests extends CoreTests { // Does not throw normalize(input) } + + test("Mutable variables are added as an extra parameter") { + val input = + """ + |def main() = { + | var x = 1 + | def update() = { x = 2 } + | update() + | println(x) + |} + """.stripMargin + + normalize(input) + } + + test("Reset/Shift with mutable variable") { + val input = + """ + |effect Eff(): Unit + |def main() = { + | var x = 0 + | try { + | x = 1 + | do Eff() + | } with Eff { resume(()) } + | println(x) + |} + """.stripMargin + + normalize(input) + } + + test("Mutable variable and recursive function") { + val input = + """ + |def main() = { + | var x = 0 + | def loop(n: Int): Unit = { + | if (n == 0) () + | else { + | x = x + 1 + | loop(n - 1) + | } + | } + | loop(5) + | println(x) + |} + """ + + normalize(input) + } } /** From 1d615a6cfaee143b4603d0595e9849e223a25543 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 21 Oct 2025 17:18:11 +0200 Subject: [PATCH 045/123] try fixing recursive calls with captures --- .../effekt/core/optimizer/NewNormalizer.scala | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 11e111fa0..52c8cf7ec 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -13,6 +13,7 @@ import kiama.output.ParenPrettyPrinter import scala.annotation.tailrec import scala.collection.mutable import scala.collection.immutable.ListMap +import effekt.core.Type.resultType // TODO // - change story of how inlining is implemented. We need to also support toplevel functions that potentially @@ -640,7 +641,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { given localEnv: Env = env .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - .bindComputation(id, Computation.Var(freshened)) + // TODO is this really correct? Pessimistically, we assume all bound variables of the escaping stack are captured + .bindComputation(id, Computation.Def(Closure(freshened, escaping.bound.map { p => Computation.Var(p.id) }))) val normalizedBlock = Block(tparams, vparams, bparams, nested { evaluate(body, Frame.Return, Stack.Unknown) @@ -791,7 +793,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate) - reify(k, ks) { NeutralStmt.Jump(label, targs, args, bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } + val blockargs = bargs.map(evaluate(_, "f", escapingStack)) + reify(k, ks) { NeutralStmt.Jump(label, targs, args, blockargs ++ environment) } case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } @@ -1066,6 +1069,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) } + def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { @@ -1103,16 +1107,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { core.Block.BlockLit(tparams, cparams, vparams, bparams, embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) } - def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = block match { - case Block(tparams, vparams, bparams, body) => - val cparams = bparams.map { - case BlockParam(id, tpe, captures) => - assert(captures.size == 1) - captures.head - } - core.Block.BlockLit(tparams, cparams, vparams, bparams, - embedStmt(body)(using G.bindValues(vparams).bindComputations(bparams))) - } + + def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = embedBlock(block).asInstanceOf[core.BlockLit] + def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = val (tpe, capt) = G.blocks.getOrElse(label, sys error s"Unknown block: ${util.show(label)}. ${G.blocks.keys.map(util.show).mkString(", ")}") core.BlockVar(label, tpe, capt) From 9e8fb0422b5345685e8dfacbc503330cfc43f6d0 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 22 Oct 2025 16:25:18 +0200 Subject: [PATCH 046/123] normalize twice for recursive functions --- .../effekt/core/NewNormalizerTests.scala | 2 +- .../effekt/core/optimizer/NewNormalizer.scala | 45 +++++++++++++++---- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 505377708..64d78852b 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -576,7 +576,7 @@ class NewNormalizerTests extends CoreTests { | loop(5) | println(x) |} - """ + """.stripMargin normalize(input) } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 52c8cf7ec..fe80fcfd9 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -13,7 +13,6 @@ import kiama.output.ParenPrettyPrinter import scala.annotation.tailrec import scala.collection.mutable import scala.collection.immutable.ListMap -import effekt.core.Type.resultType // TODO // - change story of how inlining is implemented. We need to also support toplevel functions that potentially @@ -99,8 +98,17 @@ object semantics { class Scope( var bindings: ListMap[Id, Binding], var inverse: Map[Value, Addr], - outer: Option[Scope] + val outer: Option[Scope] ) { + // Backtrack the internal state of Scope after running `prog` + def local[A](prog: => A): A = { + val scopeBefore = Scope(this.bindings, this.inverse, this.outer) + val res = prog + this.bindings = scopeBefore.bindings + this.inverse = scopeBefore.inverse + res + } + // floating values to the top is not always beneficial. For example // def foo() = COMPUTATION // vs @@ -641,17 +649,36 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { given localEnv: Env = env .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - // TODO is this really correct? Pessimistically, we assume all bound variables of the escaping stack are captured - .bindComputation(id, Computation.Def(Closure(freshened, escaping.bound.map { p => Computation.Var(p.id) }))) + // Assume that we capture nothing + .bindComputation(id, Computation.Def(Closure(freshened, Nil))) - val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Unknown) - }) + val normalizedBlock = scope.local { + Block(tparams, vparams, bparams, nested { + evaluate(body, Frame.Return, Stack.Unknown)(using localEnv) + }) + } val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } + + // Only normalize again if we actually we wrong in our assumption that we capture nothing + if (closureParams.isEmpty) { + scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) + Computation.Def(Closure(freshened, Nil)) + } else { + val captures = closureParams.map { p => Computation.Var(p.id) } + given localEnv1: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + .bindComputation(id, Computation.Def(Closure(freshened, captures))) + + val normalizedBlock1 = Block(tparams, vparams, bparams, nested { + evaluate(body, Frame.Return, Stack.Unknown)(using localEnv1) + }) + + scope.defineRecursive(freshened, normalizedBlock1.copy(bparams = normalizedBlock1.bparams ++ closureParams), block.tpe, block.capt) + Computation.Def(Closure(freshened, captures)) + } - scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams), block.tpe, block.capt) - Computation.Def(Closure(freshened, closureParams.map(p => Computation.Var(p.id)))) } // the stack here is not the one this is run in, but the one the definition potentially escapes From 21d1b41d0581eec0e84159d0f6bab3bf148e9e22 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 22 Oct 2025 17:59:26 +0200 Subject: [PATCH 047/123] fix evaluateRecursive --- .../effekt/core/optimizer/NewNormalizer.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index fe80fcfd9..b50a9f4ba 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,6 +2,7 @@ package effekt package core package optimizer +import effekt.core.BlockType import effekt.core.ValueType.Boxed import effekt.source.Span import effekt.core.optimizer.semantics.{Computation, NeutralStmt, Value} @@ -659,8 +660,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } - + // Only normalize again if we actually we wrong in our assumption that we capture nothing + // We might run into exponential complexity for nested recursives functions if (closureParams.isEmpty) { scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) Computation.Def(Closure(freshened, Nil)) @@ -675,7 +677,16 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(body, Frame.Return, Stack.Unknown)(using localEnv1) }) - scope.defineRecursive(freshened, normalizedBlock1.copy(bparams = normalizedBlock1.bparams ++ closureParams), block.tpe, block.capt) + val tpe: BlockType.Function = block.tpe match { + case _: BlockType.Interface => ??? + case ftpe: BlockType.Function => ftpe + } + scope.defineRecursive( + freshened, + normalizedBlock1.copy(bparams = normalizedBlock1.bparams ++ closureParams), + tpe.copy(cparams = tpe.cparams ++ closureParams.map { p => p.id }), + block.capt + ) Computation.Def(Closure(freshened, captures)) } From 0fc8be1d409a042a2f3012ac9be71fd034201e5f Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:40:52 +0200 Subject: [PATCH 048/123] fix pretty printer --- .../main/scala/effekt/core/optimizer/NewNormalizer.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b50a9f4ba..25a7bb998 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -553,12 +553,12 @@ object semantics { "resume" <> parens(toDoc(k)) <+> toDoc(body) case NeutralStmt.Var(id, init, body) => - "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body) + "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) case NeutralStmt.Hole(span) => "hole()" case NeutralStmt.Put(ref, tpe, cap, value, body) => - "put" <+> toDoc(ref) <+> "=" <+> toDoc(value) <> line <> toDoc(body) + toDoc(ref) <+> ":=" <+> toDoc(value) <> line <> toDoc(body.bindings) <> toDoc(body.body) } def toDoc(id: Id): Doc = id.show @@ -613,7 +613,7 @@ object semantics { (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line - case (addr, Binding.Get(ref, tpe, cap)) => "get" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line + case (addr, Binding.Get(ref, tpe, cap)) => "let" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line }) def toDoc(block: BasicBlock): Doc = From eb7fecb5ca45f87883675ea97596ca99ddb4361c Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:33:53 +0200 Subject: [PATCH 049/123] add comment about region desugaring --- .../effekt/core/optimizer/NewNormalizer.scala | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 25a7bb998..262cf7fd3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -34,6 +34,33 @@ import scala.collection.immutable.ListMap // Same actually for stack allocated mutable state, we should abstract over those (but only those) // and keep the function in its original location. // This means we only need to abstract over blocks, no values, no types. +// +// TODO Region desugaring +// region r { +// reset { p => +// var x in r = 42 +// x = !x + 1 +// println(!x) +// } +// } +// +// reset { r => +// reset { p => +// //var x in r = 42 +// shift(r) { k => +// var x = 42 +// resume(k) { +// x = !x + 1 +// println(!x) +// } +// } +// } +// } +// +// - Typeability preservation: {r: Region} becomes {r: Prompt[T]} +// [[ def f() {r: Region} = s ]] = def f[T]() {r: Prompt[T]} = ... +// - Continuation capture is _not_ constant time in JS backend, so we expect a (drastic) slowdown when desugaring + object semantics { // Values From c15d4138185276505b5134fb3157ba42b3764d2c Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:34:59 +0200 Subject: [PATCH 050/123] add idea about fixing reification stack over-approximation --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 262cf7fd3..4c1f839a1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -515,6 +515,16 @@ object semantics { case Frame.Static(tpe, apply) => val tmp = Id("tmp") scope.push(tmp, stmt) + // TODO Over-approximation + // Don't pass Stack.Unkown but rather the stack until the next reset? + /* + |----------| |----------| |---------| + | | ---> ... ---> | | ---> ... ---> | | ---> ... + |----------| |----------| |---------| + r1 r2 prompt + + Pass r1 :: ... :: r2 :: ... :: prompt :: UNKOWN + */ apply(scope)(tmp)(Stack.Unknown) case Frame.Dynamic(Closure(label, closure)) => val tmp = Id("tmp") From a4ee4923629f1b5849f1fe1f142406cb3553738a Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 13:35:36 +0200 Subject: [PATCH 051/123] naively implement region normalization --- .../effekt/core/NewNormalizerTests.scala | 35 ++++++ .../effekt/core/optimizer/NewNormalizer.scala | 101 +++++++++++++++--- 2 files changed, 120 insertions(+), 16 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 64d78852b..ad4c252d2 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -580,6 +580,41 @@ class NewNormalizerTests extends CoreTests { normalize(input) } + + test("basic region usage") { + val input = + """ + |def main() = { + | val y = region r { + | var x in r = 42 + | x = x + 1 + | x + | } + | println(y) + |} + |""".stripMargin + + normalize(input) + } + + test("region parameter") { + val input = + """ + |def main() = { + | val y = region reg { + | def foo(init: Int) {r: Region} = { + | var x in r = init + | x = x + 1 + | x + | } + | foo(42) {reg} + | } + | println(y) + |} + |""".stripMargin + + normalize(input) + } } /** diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4c1f839a1..279c62cf9 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -344,6 +344,9 @@ object semantics { case Var(id: BlockParam, init: Addr, body: BasicBlock) case Put(ref: Id, tpe: ValueType, cap: Captures, value: Addr, body: BasicBlock) + case Region(id: BlockParam, body: BasicBlock) + case Alloc(id: Id, init: Addr, region: Id, body: BasicBlock) + // aborts at runtime case Hole(span: Span) @@ -358,8 +361,10 @@ object semantics { case NeutralStmt.Shift(prompt, capt, k, body) => (body.free - k.id) + prompt case NeutralStmt.Resume(k, body) => Set(k) ++ body.free case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id - case NeutralStmt.Hole(span) => Set.empty case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref, value) ++ body.free + case NeutralStmt.Region(id, body) => body.free - id.id + case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id + case NeutralStmt.Hole(span) => Set.empty } } @@ -378,6 +383,7 @@ object semantics { case Stack.Unknown => NeutralStmt.Return(arg) case Stack.Reset(p, k, ks) => k.ret(ks, arg) case Stack.Var(id, curr, init, k, ks) => k.ret(ks, arg) + case Stack.Region(id, bindings, k, ks) => k.ret(ks, arg) } case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } @@ -416,13 +422,14 @@ object semantics { case Reset(prompt: BlockParam, frame: Frame, next: Stack) case Var(id: BlockParam, curr: Addr, init: Addr, frame: Frame, next: Stack) // TODO desugar regions into var? - // case Region(bindings: Map[Id, Addr]) + case Region(id: BlockParam, bindings: Map[Id, (Addr, Addr)], frame: Frame, next: Stack) lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil case Stack.Unknown => Nil case Stack.Reset(prompt, frame, next) => prompt :: next.bound case Stack.Var(id, curr, init, frame, next) => id :: next.bound + case Stack.Region(id, bindings, frame, next) => id :: next.bound } } @@ -433,21 +440,50 @@ object semantics { case Stack.Reset(prompt, frame, next) => get(ref, next) case Stack.Var(id1, curr, init, frame, next) if ref == id1.id => Some(curr) case Stack.Var(id1, curr, init, frame, next) => get(ref, next) + case Stack.Region(id, bindings, frame, next) => + if (bindings.contains(ref)) { + Some(bindings(ref)._1) + } else { + get(ref, next) + } } - def put(id: Id, value: Addr, ks: Stack): Option[Stack] = ks match { - case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(id)} in empty stack" + def put(ref: Id, value: Addr, ks: Stack): Option[Stack] = ks match { + case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(ref)} in empty stack" // We have reached the end of the known stack, so the variable must be in the unknown part. case Stack.Unknown => None - case Stack.Reset(prompt, frame, next) => put(id, value, next).map(Stack.Reset(prompt, frame, _)) - case Stack.Var(id1, curr, init, frame, next) if id == id1.id => Some(Stack.Var(id1, value, init, frame, next)) - case Stack.Var(id1, curr, init, frame, next) => put(id, value, next).map(Stack.Var(id1, curr, init, frame, _)) + case Stack.Reset(prompt, frame, next) => put(ref, value, next).map(Stack.Reset(prompt, frame, _)) + case Stack.Var(id, curr, init, frame, next) if ref == id.id => Some(Stack.Var(id, value, init, frame, next)) + case Stack.Var(id, curr, init, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, init, frame, _)) + case Stack.Region(id, bindings, frame, next) => + if (bindings.contains(ref)){ + Some(Stack.Region(id, bindings.updated(ref, (value, bindings(ref)._2)), frame, next)) + } else { + put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) + } + } + + def alloc(ref: Id, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { + case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(ref)} in empty stack" + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => + alloc(ref, reg, value, next).map(Stack.Reset(prompt, frame, _)) + case Stack.Var(id, curr, init, frame, next) => + alloc(ref, reg, value, next).map(Stack.Var(id, curr, init, frame, _)) + case Stack.Region(id, bindings, frame, next) => + if (reg == id.id){ + Some(Stack.Region(id, bindings.updated(ref, (value, value)), frame, next)) + } else { + alloc(ref, reg, value, next).map(Stack.Region(id, bindings, frame, _)) + } } enum Cont { case Empty case Reset(frame: Frame, prompt: BlockParam, rest: Cont) case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) + case Region(frame: Frame, id: BlockParam, bindings: Map[Id, (Addr, Addr)], rest: Cont) } def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { @@ -461,6 +497,9 @@ object semantics { case Stack.Var(id, curr, init, frame, next) => val (c, frame2, stack) = shift(p, frame, next) (Cont.Var(k, id, curr, init, c), frame2, stack) + case Stack.Region(id, bindings, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Region(k, id, bindings, c), frame2, stack) } def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { @@ -472,6 +511,9 @@ object semantics { case Cont.Var(frame, id, curr, init, rest) => val (k1, ks1) = resume(rest, frame, ks) (frame, Stack.Var(id, curr, init, k1, ks1)) + case Cont.Region(frame, id, bindings, rest) => + val (k1, ks1) = resume(rest, frame, ks) + (frame, Stack.Region(id, bindings, k1, ks1)) } def joinpoint(k: Frame, ks: Stack)(f: (Frame, Stack) => NeutralStmt)(using scope: Scope): NeutralStmt = { @@ -502,6 +544,8 @@ object semantics { Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) case Stack.Var(id, curr, init, frame, next) => Stack.Var(id, curr, init, reifyFrame(frame, next), reifyStack(next)) + case Stack.Region(id, bindings, frame, next) => + Stack.Region(id, bindings, reifyFrame(frame, next), reifyStack(next)) } f(reifyFrame(k, ks), reifyStack(ks)) } @@ -533,7 +577,7 @@ object semantics { } @tailrec - final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = { ks match { case Stack.Empty => stmt case Stack.Unknown => stmt @@ -549,7 +593,13 @@ object semantics { val body = nested { stmt } NeutralStmt.Var(id, init, body) }} + case Stack.Region(id, bindings, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + NeutralStmt.Region(id, body) + }} } + } object PrettyPrinter extends ParenPrettyPrinter { @@ -592,10 +642,16 @@ object semantics { case NeutralStmt.Var(id, init, body) => "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) - case NeutralStmt.Hole(span) => "hole()" - case NeutralStmt.Put(ref, tpe, cap, value, body) => toDoc(ref) <+> ":=" <+> toDoc(value) <> line <> toDoc(body.bindings) <> toDoc(body.body) + + case NeutralStmt.Region(id, body) => + "region" <+> toDoc(id) <+> toDoc(body) + + case NeutralStmt.Alloc(id, init, region, body) => + "var" <+> toDoc(id) <+> "in" <+> toDoc(region) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) + + case NeutralStmt.Hole(span) => "hole()" } def toDoc(id: Id): Doc = id.show @@ -942,8 +998,16 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Hole(span) => NeutralStmt.Hole(span) // State - case Stmt.Region(body) => ??? - case Stmt.Alloc(id, init, region, body) => ??? + case Stmt.Region(BlockLit(Nil, List(capture), Nil, List(cap), body)) => + given Env = env.bindComputation(cap.id, Computation.Var(cap.id)) + evaluate(body, Frame.Return, Stack.Region(cap, Map.empty, k, ks)) + case Stmt.Region(_) => ??? + case Stmt.Alloc(id, init, region, body) => + val addr = evaluate(init) + alloc(id, region, addr, ks) match { + case Some(ks1) => evaluate(body, k, ks1) + case None => NeutralStmt.Alloc(id, addr, region, nested { evaluate(body, k, ks) }) + } // TODO case Stmt.Var(ref, init, capture, body) => @@ -1020,7 +1084,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } def run(mod: ModuleDecl): ModuleDecl = { - util.trace(mod) + //util.trace(mod) // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } val toplevelEnv = Env.empty @@ -1037,7 +1101,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { mod.copy(definitions = newDefinitions) } - inline def debug(inline msg: => Any) = println(msg) + val showDebugInfo = false + inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { case Toplevel.Def(id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -1102,10 +1167,14 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case _ => sys error "Variable needs to have a single capture" } Stmt.Var(blockParam.id, embedExpr(init), capt, embedStmt(body)(using G.bind(blockParam.id, blockParam.tpe, blockParam.capt))) - case NeutralStmt.Hole(span) => - Stmt.Hole(span) case NeutralStmt.Put(ref, annotatedTpe, annotatedCapt, value, body) => Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) + case NeutralStmt.Region(id, body) => + Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputations(List(id))))) + case NeutralStmt.Alloc(id, init, region, body) => + Stmt.Alloc(id, embedExpr(init), region, embedStmt(body)) + case NeutralStmt.Hole(span) => + Stmt.Hole(span) } def embedStmt(basicBlock: BasicBlock)(using G: TypingContext): core.Stmt = basicBlock match { From 3266782f91d8848de568c1e486e7f2068710201f Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:11:59 +0200 Subject: [PATCH 052/123] don't reify regions if unneeded --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 279c62cf9..4ee6db038 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -596,7 +596,8 @@ object semantics { case Stack.Region(id, bindings, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - NeutralStmt.Region(id, body) + if (body.free contains id.id) NeutralStmt.Region(id, body) + else stmt }} } } From c1ce5e6fdcb896c581db2b1fafca1eb79eb9bcc0 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:12:20 +0200 Subject: [PATCH 053/123] fix var pretty printer --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4ee6db038..babd2f33b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -641,7 +641,7 @@ object semantics { "resume" <> parens(toDoc(k)) <+> toDoc(body) case NeutralStmt.Var(id, init, body) => - "var" <+> toDoc(id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) + "var" <+> toDoc(id.id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) case NeutralStmt.Put(ref, tpe, cap, value, body) => toDoc(ref) <+> ":=" <+> toDoc(value) <> line <> toDoc(body.bindings) <> toDoc(body.body) From 76b1927ba07cd22307e865400799b151ea9ac26c Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:13:19 +0200 Subject: [PATCH 054/123] add comment to evaluate(Stmt) --- .../effekt/core/optimizer/NewNormalizer.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index babd2f33b..f032fb272 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -912,6 +912,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(body, k, ks)(using newEnv, scope) case Stmt.App(callee, targs, vargs, bargs) => + // TODO Why? Should it really be Stack.Unkown? // Here the stack passed to the blocks is an empty one since we reify it anyways... val escapingStack = Stack.Unknown evaluate(callee, "f", escapingStack) match { @@ -925,7 +926,18 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate) - val blockargs = bargs.map(evaluate(_, "f", escapingStack)) + // TODO ks or Stack.Unkown? + /* + try { + prog { + do Eff() + } + } with Eff { ... } + --- + val captures = stack.bound.filter { block.free } + is incorrect as the result is always the empty capture set since Stack.Unkown.bound = Set() + */ + val blockargs = bargs.map(evaluate(_, "f", ks)) reify(k, ks) { NeutralStmt.Jump(label, targs, args, blockargs ++ environment) } case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } From 80ce8fe1f3bca2281971a0fddc789695794546ea Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:13:37 +0200 Subject: [PATCH 055/123] refactor --- .../main/scala/effekt/core/optimizer/NewNormalizer.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index f032fb272..e07df97d9 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1145,8 +1145,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { def bind(id: Id, tpe: ValueType): TypingContext = this.copy(values = values + (id -> tpe)) def bind(id: Id, tpe: BlockType, capt: Captures): TypingContext = this.copy(blocks = blocks + (id -> (tpe, capt))) def bindValues(vparams: List[ValueParam]): TypingContext = this.copy(values = values ++ vparams.map(p => p.id -> p.tpe)) - def bindComputations(bparams: List[BlockParam]): TypingContext = this.copy(blocks = blocks ++ bparams.map(p => p.id -> (p.tpe, p.capt))) def lookupValue(id: Id): ValueType = values.getOrElse(id, sys.error(s"Unknown value: ${util.show(id)}")) + def bindComputations(bparams: List[BlockParam]): TypingContext = this.copy(blocks = blocks ++ bparams.map(p => p.id -> (p.tpe, p.capt))) + def bindComputation(bparam: BlockParam): TypingContext = this.copy(blocks = blocks + (bparam.id -> (bparam.tpe, bparam.capt))) } def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { @@ -1169,9 +1170,9 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case set if set.size == 1 => set.head case _ => sys error "Prompt needs to have a single capture" } - Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputations(prompt :: Nil)))) + Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputation(prompt)))) case NeutralStmt.Shift(prompt, capt, k, body) => - Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputations(k :: Nil)))) + Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputation(k)))) case NeutralStmt.Resume(k, body) => Stmt.Resume(embedBlockVar(k), embedStmt(body)) case NeutralStmt.Var(blockParam, init, body) => @@ -1183,7 +1184,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case NeutralStmt.Put(ref, annotatedTpe, annotatedCapt, value, body) => Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) case NeutralStmt.Region(id, body) => - Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputations(List(id))))) + Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputation(id)))) case NeutralStmt.Alloc(id, init, region, body) => Stmt.Alloc(id, embedExpr(init), region, embedStmt(body)) case NeutralStmt.Hole(span) => From 623899409eb1a222957bed9aab08e55ce3f2c268 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:13:58 +0200 Subject: [PATCH 056/123] try to eta-expand closures when embedding --- .../effekt/core/optimizer/NewNormalizer.scala | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index e07df97d9..148a17db3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1231,9 +1231,37 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { - case Computation.Var(id) => embedBlockVar(id) - case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) - case Computation.Def(Closure(label, environment)) => ??? // TODO eta expand + case Computation.Var(id) => + embedBlockVar(id) + case Computation.Def(Closure(label, Nil)) => + embedBlockVar(label) + // TODO fix eta-expansion, this is ~~bogus~~ work-in-progress + case Computation.Def(Closure(label, environment)) => + val blockvar = embedBlockVar(label) + G.blocks(label) match { + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + val vpIds = vparams.map { p => Id("x") } + val bpIds = bparams.map { p => Id(p.show) } + val vp = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } + val bp = bpIds.zip(bparams).map { (id, p) => core.BlockParam(id, p, Set()) } + core.Block.BlockLit( + tparams, + Nil, + vp, + Nil, + Stmt.App( + blockvar, + Nil, + vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) }, + bparams.zip(environment).map { case (bp, c) => c match { + case Computation.Var(id) => BlockVar(id, bp, Set()) + case _ => ??? // cannot occur + } + } + ) + ) + case _ => ??? + } case Computation.Inline(blocklit, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => From 603501a7929b5e93585c11dbc9f7b25bf117eae4 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:06:51 +0100 Subject: [PATCH 057/123] try fixing eta-expansion --- .../effekt/core/optimizer/NewNormalizer.scala | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 148a17db3..fe02f6b7d 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -473,6 +473,7 @@ object semantics { alloc(ref, reg, value, next).map(Stack.Var(id, curr, init, frame, _)) case Stack.Region(id, bindings, frame, next) => if (reg == id.id){ + // TODO Some(Stack.Region(id, bindings.updated(ref, (value, value)), frame, next)) } else { alloc(ref, reg, value, next).map(Stack.Region(id, bindings, frame, _)) @@ -565,7 +566,7 @@ object semantics { |----------| |----------| |---------| | | ---> ... ---> | | ---> ... ---> | | ---> ... |----------| |----------| |---------| - r1 r2 prompt + r1 r2 first next prompt Pass r1 :: ... :: r2 :: ... :: prompt :: UNKOWN */ @@ -694,7 +695,7 @@ object semantics { } } def toDoc(closure: Closure): Doc = closure match { - case Closure(label, env) => toDoc(label) <> brackets(hsep(env.map(toDoc), comma)) + case Closure(label, env) => toDoc(label) <+> "@" <+> brackets(hsep(env.map(toDoc), comma)) } def toDoc(bindings: Bindings): Doc = @@ -1235,29 +1236,34 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { embedBlockVar(id) case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) - // TODO fix eta-expansion, this is ~~bogus~~ work-in-progress case Computation.Def(Closure(label, environment)) => val blockvar = embedBlockVar(label) G.blocks(label) match { case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + // TODO this uses the invariant that we _append_ all environment captures to the bparams + val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) val vpIds = vparams.map { p => Id("x") } - val bpIds = bparams.map { p => Id(p.show) } - val vp = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } - val bp = bpIds.zip(bparams).map { (id, p) => core.BlockParam(id, p, Set()) } + val bpIds = origBparams.map { p => Id("f") } + val vps = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } + val bps = bpIds.zip(origBparams).map { (id, p) => core.BlockParam(id, p, Set()) } + val vargs = vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) } + // TODO don't use empty capture set for synthesised BlockVars + val bargs = + bpIds.zip(origBparams).map { case (id, bp) => BlockVar(id, bp, Set()) } ++ + synthBparams.zip(environment).map { + case (bp, Computation.Var(id)) => BlockVar(id, bp, Set()) + case _ => ??? + } core.Block.BlockLit( tparams, - Nil, - vp, - Nil, + cparams.take(cparams.length - environment.length), + vps, + bps, Stmt.App( blockvar, Nil, - vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) }, - bparams.zip(environment).map { case (bp, c) => c match { - case Computation.Var(id) => BlockVar(id, bp, Set()) - case _ => ??? // cannot occur - } - } + vargs, + bargs ) ) case _ => ??? From f5e7ec5d50e4ae2685e25dbf65a0c6f5ee266b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 10:49:19 +0100 Subject: [PATCH 058/123] Fix looking up top-level vals --- .../effekt/core/NewNormalizerTests.scala | 30 +++++++++++++++++++ .../effekt/core/optimizer/NewNormalizer.scala | 16 +++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index ad4c252d2..9058695dc 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -615,6 +615,36 @@ class NewNormalizerTests extends CoreTests { normalize(input) } + + test("Can lookup top-level def") { + val input = + """ + |def top(): Int = 43 + | + |def main() = { + | val x = top() + top() + | println(x.show) + |} + |""".stripMargin + + // Does not throw + normalize(input) + } + + test("Can lookup top-level val") { + val input = + """ + |val top: Int = 43 + | + |def main() = { + | val x = top + top + | println(x.show) + |} + |""".stripMargin + + // Does not throw + normalize(input) + } } /** diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index fe02f6b7d..9a634a906 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,7 +2,7 @@ package effekt package core package optimizer -import effekt.core.BlockType +import effekt.core.{BlockType, Toplevel} import effekt.core.ValueType.Boxed import effekt.source.Span import effekt.core.optimizer.semantics.{Computation, NeutralStmt, Value} @@ -1102,12 +1102,20 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } val toplevelEnv = Env.empty - // user defined functions - .bindComputation(mod.definitions.map(defn => defn.id -> Computation.Def(Closure(defn.id, Nil)))) + // user-defined functions + .bindComputation(mod.definitions.collect { + case Toplevel.Def(id, b) => id -> Computation.Def(Closure(id, Nil)) + }) + // user-defined values + .bindValue(mod.definitions.collect { + case Toplevel.Val(id, _, _) => id -> id + }) // async extern functions .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Def(Closure(defn.id, Nil)))) - val typingContext = TypingContext(Map.empty, mod.definitions.collect { + val typingContext = TypingContext(mod.definitions.collect { + case Toplevel.Val(id, tpe, _) => id -> tpe + }.toMap, mod.definitions.collect { case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) }.toMap) // ++ asyncExterns.map { d => d.id -> null }) From 84a7c37e2cf3615b1f55e6dd46de64b807d497a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 11:08:01 +0100 Subject: [PATCH 059/123] Add test case for @dvdvgt's fix --- .../effekt/core/NewNormalizerTests.scala | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 9058695dc..970ba6ad6 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -645,6 +645,66 @@ class NewNormalizerTests extends CoreTests { // Does not throw normalize(input) } + + // This effect tests a subtle aspect of normalizing with block parameters. + // Consider the provided input program. + // The normalized program looks as follows, with irrelevant details elided: + // ```scala + // () { + // def break = ... + // def run = ... + // let x = 1 + // def f = (v: Int){f} {y} {p} { + // let x = 2 + // y := x + // jump break(){p} + // } + // val o = reset {{p} => + // var y = x + // val tmp = jump run(){f @ [y, p]} + // let z = () + // return z + // } + // jump println(o) + // } + // ``` + // As you can see, the block argument supplied to block parameter `prog` of `run` is lifted to a `def f`. + // This definition needs to abstract over all free captures and variables in the body. + // In particular, the call side of `f` needs to supply the correct closure. + // Therefore, it is not possible to simply call `run` as follows: + // ```scala + // jump run(){f} + // ``` + // where `f` would be passed as a block variable. + // Instead, we need to "eta-expand" this parameter to a block that calls f with the correct captures: + // ```scala + // jump run(){f @ [y, p]} + // ``` + // where `y` and `p` are the correct captures from the reset body. + test("Block parameters get lifted and captures are passed correctly") { + val input: String = + """ + |effect break(): Unit + | + |def main() = { + | val x = try { + | def run { prog: (Int) {() => Unit} => Unit }: Unit = prog(42) { () => () } + | var y = 1 + | run { (v) { f } => + | y = 2 + | do break() + | } + | () + | } with break { + | () + | } + | println(x) + |} + |""".stripMargin + + // Does not throw + normalize(input) + } } /** From 99e31f132afdfaff3ad69b9050791fff0fa4705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 11:08:40 +0100 Subject: [PATCH 060/123] Fix typo --- effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 970ba6ad6..4d7d9bca2 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -646,7 +646,7 @@ class NewNormalizerTests extends CoreTests { normalize(input) } - // This effect tests a subtle aspect of normalizing with block parameters. + // This case tests a subtle aspect of normalizing with block parameters. // Consider the provided input program. // The normalized program looks as follows, with irrelevant details elided: // ```scala From 7f23c7f34e0f3b402b309757d6d977c1ffd50e2a Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:04:21 +0100 Subject: [PATCH 061/123] fix region alloc bug --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 9a634a906..6cf4b4226 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -464,7 +464,9 @@ object semantics { } def alloc(ref: Id, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { - case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(ref)} in empty stack" + // This case can occur if we normalize a function that abstracts over a region as a parameter + // We return None and force the reification of the allocation + case Stack.Empty => None // We have reached the end of the known stack, so the variable must be in the unknown part. case Stack.Unknown => None case Stack.Reset(prompt, frame, next) => From ff4e60da6a5c9dcd25b1c3632245f92478888ed0 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:45:35 +0100 Subject: [PATCH 062/123] use capture information when eta-expanding --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 6cf4b4226..d7eff59db 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1252,16 +1252,18 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => // TODO this uses the invariant that we _append_ all environment captures to the bparams val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) + val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) + val vpIds = vparams.map { p => Id("x") } val bpIds = origBparams.map { p => Id("f") } val vps = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } - val bps = bpIds.zip(origBparams).map { (id, p) => core.BlockParam(id, p, Set()) } val vargs = vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) } - // TODO don't use empty capture set for synthesised BlockVars + val namedBparams = bpIds.zip(origBparams).zip(origCapts) + val bps = namedBparams.map { case ((id, p), c) => core.BlockParam(id, p, Set(c)) } val bargs = - bpIds.zip(origBparams).map { case (id, bp) => BlockVar(id, bp, Set()) } ++ - synthBparams.zip(environment).map { - case (bp, Computation.Var(id)) => BlockVar(id, bp, Set()) + namedBparams.map { case ((id, bp), c) => core.BlockVar(id, bp, Set(c)) } ++ + environment.zip(synthBparams).zip(synthCapts).map { + case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) case _ => ??? } core.Block.BlockLit( From f8f1ca7dcde194188e26526b62b87f7c36401ce9 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:28:43 +0100 Subject: [PATCH 063/123] simplify Region stack: don't store initial Addr --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index d7eff59db..efd6b0b8b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -422,7 +422,7 @@ object semantics { case Reset(prompt: BlockParam, frame: Frame, next: Stack) case Var(id: BlockParam, curr: Addr, init: Addr, frame: Frame, next: Stack) // TODO desugar regions into var? - case Region(id: BlockParam, bindings: Map[Id, (Addr, Addr)], frame: Frame, next: Stack) + case Region(id: BlockParam, bindings: Map[Id, Addr], frame: Frame, next: Stack) lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil @@ -442,7 +442,7 @@ object semantics { case Stack.Var(id1, curr, init, frame, next) => get(ref, next) case Stack.Region(id, bindings, frame, next) => if (bindings.contains(ref)) { - Some(bindings(ref)._1) + Some(bindings(ref)) } else { get(ref, next) } @@ -457,7 +457,7 @@ object semantics { case Stack.Var(id, curr, init, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, init, frame, _)) case Stack.Region(id, bindings, frame, next) => if (bindings.contains(ref)){ - Some(Stack.Region(id, bindings.updated(ref, (value, bindings(ref)._2)), frame, next)) + Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) } else { put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) } @@ -475,8 +475,7 @@ object semantics { alloc(ref, reg, value, next).map(Stack.Var(id, curr, init, frame, _)) case Stack.Region(id, bindings, frame, next) => if (reg == id.id){ - // TODO - Some(Stack.Region(id, bindings.updated(ref, (value, value)), frame, next)) + Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) } else { alloc(ref, reg, value, next).map(Stack.Region(id, bindings, frame, _)) } @@ -486,7 +485,7 @@ object semantics { case Empty case Reset(frame: Frame, prompt: BlockParam, rest: Cont) case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) - case Region(frame: Frame, id: BlockParam, bindings: Map[Id, (Addr, Addr)], rest: Cont) + case Region(frame: Frame, id: BlockParam, bindings: Map[Id, Addr], rest: Cont) } def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { From ebf813566d26516708856829385d6a6ec0128ce8 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:29:06 +0100 Subject: [PATCH 064/123] extract eta-expansion into function --- .../effekt/core/optimizer/NewNormalizer.scala | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index efd6b0b8b..739c15197 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1245,40 +1245,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { embedBlockVar(id) case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) - case Computation.Def(Closure(label, environment)) => - val blockvar = embedBlockVar(label) - G.blocks(label) match { - case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => - // TODO this uses the invariant that we _append_ all environment captures to the bparams - val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) - val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) - - val vpIds = vparams.map { p => Id("x") } - val bpIds = origBparams.map { p => Id("f") } - val vps = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } - val vargs = vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) } - val namedBparams = bpIds.zip(origBparams).zip(origCapts) - val bps = namedBparams.map { case ((id, p), c) => core.BlockParam(id, p, Set(c)) } - val bargs = - namedBparams.map { case ((id, bp), c) => core.BlockVar(id, bp, Set(c)) } ++ - environment.zip(synthBparams).zip(synthCapts).map { - case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) - case _ => ??? - } - core.Block.BlockLit( - tparams, - cparams.take(cparams.length - environment.length), - vps, - bps, - Stmt.App( - blockvar, - Nil, - vargs, - bargs - ) - ) - case _ => ??? - } + case Computation.Def(closure) => + etaExpand(closure) case Computation.Inline(blocklit, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => @@ -1299,6 +1267,43 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } core.Block.New(Implementation(interface, ops)) } + + def etaExpand(closure: Closure)(using G: TypingContext): core.BlockLit = { + val Closure(label, environment) = closure + val blockvar = embedBlockVar(label) + G.blocks(label) match { + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + // TODO this uses the invariant that we _append_ all environment captures to the bparams + val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) + val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) + + val vpIds = vparams.map { p => Id("x") } + val bpIds = origBparams.map { p => Id("f") } + val vps = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } + val vargs = vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) } + val namedBparams = bpIds.zip(origBparams).zip(origCapts) + val bps = namedBparams.map { case ((id, p), c) => core.BlockParam(id, p, Set(c)) } + val bargs = + namedBparams.map { case ((id, bp), c) => core.BlockVar(id, bp, Set(c)) } ++ + environment.zip(synthBparams).zip(synthCapts).map { + case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) + case _ => ??? + } + core.Block.BlockLit( + tparams, + cparams.take(cparams.length - environment.length), + vps, + bps, + Stmt.App( + blockvar, + Nil, + vargs, + bargs + ) + ) + case _ => ??? + } + } def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { case Block(tparams, vparams, bparams, b) => From 601694250c885f6768e3f4e530ceee707544b10c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 14:30:40 +0100 Subject: [PATCH 065/123] Split cparams off bparams when embedding interface instances --- .../effekt/core/optimizer/NewNormalizer.scala | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 739c15197..c4f784847 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -692,7 +692,7 @@ object semantics { case Computation.Inline(block, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { - hsep(operations.map { case (id, impl) => toDoc(id) <> ":" <+> toDoc(impl) }, ",") + hsep(operations.map { case (id, impl) => "def" <+> toDoc(id) <+> "=" <+> toDoc(impl) }, ",") } } def toDoc(closure: Closure): Doc = closure match { @@ -1124,7 +1124,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { mod.copy(definitions = newDefinitions) } - val showDebugInfo = false + val showDebugInfo = true inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { @@ -1259,9 +1259,57 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val cparams2 = cparams //.map(c => Id(c)) val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } + // In the following section, we create a new instance of the interface. + // All operation bodies were lifted to block literals in an earlier stage. + // While doing so, their block parameters (bparams) were concatenated with their capture parameters (cparams). + // When we embed back to core, we need to "eta-expand" the operation body to supply the correct captures from the environment. + // To see why this "eta-expansion" is necessary to achieve this, consider the following example: + // ```scala + // effect Eff(): Unit + // def use = { do Eff() } + // def main() = { + // val r = try { + // use() + // } with Eff { + // resume(()) + // } + // } + // ``` + // the handler body normalizes to the following: + // ```scala + // reset {{p} => + // jump use(){new Eff {def Eff = Eff @ [p]}} + // } + // ``` + // where + // ``` + // def Eff = (){p} { ... } + // ``` + // In particular, the prompt `p` needs to be passed to the lifted operation body. + // ``` + val (origBparams, synthBparams) = bparams2.splitAt(bparams2.length - environment.length) + val bargs = + // TODO: Fix captures + origBparams.map { case bp => BlockVar(bp.id, bp.tpe, Set()) } ++ + synthBparams.zip(environment).map { + // TODO: Fix captures + case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) + case _ => sys error "should not happen" + } - core.Operation(id, tparams2, cparams, vparams2, bparams2, - Stmt.App(embedBlockVar(label), tparams2.map(ValueType.Var.apply), vparams2.map(p => ValueVar(p.id, p.tpe)), bparams2.map(p => BlockVar(p.id, p.tpe, p.capt)))) + core.Operation( + id, + tparams2, + cparams.take(cparams.length - environment.length), + vparams2, + origBparams, + Stmt.App( + embedBlockVar(label), + tparams2.map(ValueType.Var.apply), + vparams2.map(p => ValueVar(p.id, p.tpe)), + bargs + ) + ) case _ => sys error "Unexpected block type" } } From 7fe6e759f6588f670d7edb32a42b50d981a770e2 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:47:46 +0100 Subject: [PATCH 066/123] fix targs --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index c4f784847..d769d4826 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1124,7 +1124,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { mod.copy(definitions = newDefinitions) } - val showDebugInfo = true + val showDebugInfo = false inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { @@ -1344,7 +1344,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { bps, Stmt.App( blockvar, - Nil, + tparams.map { core.ValueType.Var.apply }, vargs, bargs ) From 06595c942beee0b106730ba030d40edd8b7e878b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 15:05:02 +0100 Subject: [PATCH 067/123] Make type of Closure.environment more precise --- .../main/scala/effekt/core/optimizer/NewNormalizer.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index d769d4826..73e003ee1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -314,7 +314,7 @@ object semantics { } // TODO add escaping mutable variables - case class Closure(label: Label, environment: List[Computation]) { + case class Closure(label: Label, environment: List[Computation.Var]) { val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet } @@ -763,7 +763,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) Computation.Def(Closure(freshened, Nil)) } else { - val captures = closureParams.map { p => Computation.Var(p.id) } + val captures = closureParams.map { p => Computation.Var(p.id): Computation.Var } given localEnv1: Env = env .bindValue(vparams.map(p => p.id -> p.id)) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) @@ -1294,7 +1294,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { synthBparams.zip(environment).map { // TODO: Fix captures case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) - case _ => sys error "should not happen" } core.Operation( @@ -1335,7 +1334,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { namedBparams.map { case ((id, bp), c) => core.BlockVar(id, bp, Set(c)) } ++ environment.zip(synthBparams).zip(synthCapts).map { case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) - case _ => ??? } core.Block.BlockLit( tparams, From 750c0a64648c5521a894807ae8cac0f2516b0c0d Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:09:51 +0100 Subject: [PATCH 068/123] refactor etaExpand to make it more readable --- .../effekt/core/optimizer/NewNormalizer.scala | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 73e003ee1..47c30c90e 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1314,40 +1314,41 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } core.Block.New(Implementation(interface, ops)) } - + def etaExpand(closure: Closure)(using G: TypingContext): core.BlockLit = { val Closure(label, environment) = closure val blockvar = embedBlockVar(label) G.blocks(label) match { + // TODO why is `captures` unused? case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => - // TODO this uses the invariant that we _append_ all environment captures to the bparams - val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) + val vps = vparams.map { p => core.ValueParam(Id("x"), p) } + val vargs = vps.map { vp => core.Expr.ValueVar(vp.id, vp.tpe) } + + // this uses the invariant that we _append_ all environment captures to the bparams val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) + val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) + val origBps = origBparams.zip(origCapts).map { case (bp, c) => core.BlockParam(Id("f"), bp, Set(c)) } + val origBargs = origBps.map { bp => core.BlockVar(bp.id, bp.tpe, bp.capt) } + val synthBargs = environment.zip(synthBparams).zip(synthCapts).map { + case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) + } + val bargs = origBargs ++ synthBargs + + val targs = tparams.map { core.ValueType.Var.apply } - val vpIds = vparams.map { p => Id("x") } - val bpIds = origBparams.map { p => Id("f") } - val vps = vpIds.zip(vparams).map { (id, p) => core.ValueParam(id, p) } - val vargs = vpIds.zip(vparams).map { (id, p) => core.Expr.ValueVar(id, p) } - val namedBparams = bpIds.zip(origBparams).zip(origCapts) - val bps = namedBparams.map { case ((id, p), c) => core.BlockParam(id, p, Set(c)) } - val bargs = - namedBparams.map { case ((id, bp), c) => core.BlockVar(id, bp, Set(c)) } ++ - environment.zip(synthBparams).zip(synthCapts).map { - case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) - } core.Block.BlockLit( tparams, - cparams.take(cparams.length - environment.length), + origCapts, vps, - bps, + origBps, Stmt.App( blockvar, - tparams.map { core.ValueType.Var.apply }, + targs, vargs, bargs ) ) - case _ => ??? + case _ => sys.error("Unexpected block type for a closure") } } From f1288586d7d6515c103f573a694ec625b09032d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 15:57:09 +0100 Subject: [PATCH 069/123] etaExpand -> etaExpandToBlockLit --- .../main/scala/effekt/core/optimizer/NewNormalizer.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 47c30c90e..2fc753388 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1246,7 +1246,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Def(Closure(label, Nil)) => embedBlockVar(label) case Computation.Def(closure) => - etaExpand(closure) + etaExpandToBlockLit(closure) case Computation.Inline(blocklit, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => @@ -1315,7 +1315,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { core.Block.New(Implementation(interface, ops)) } - def etaExpand(closure: Closure)(using G: TypingContext): core.BlockLit = { + /** + * Embed `Computation.Def` to a `core.BlockLit` + * This eta-expands the block var that stands for the `Computation.Def` to a full block literal + * so that we can supply the correct capture arguments from the environment. + */ + def etaExpandToBlockLit(closure: Closure)(using G: TypingContext): core.BlockLit = { val Closure(label, environment) = closure val blockvar = embedBlockVar(label) G.blocks(label) match { From 25560842af1aae1451dc0ae47d87b27857fac828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 16:00:49 +0100 Subject: [PATCH 070/123] Extract etaExpandToOperation --- .../effekt/core/optimizer/NewNormalizer.scala | 131 +++++++++--------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 2fc753388..cad2f4220 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1250,68 +1250,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Computation.Inline(blocklit, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => - // TODO deal with environment - val ops = operations.map { case (id, Closure(label, environment)) => - G.blocks(label) match { - case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => - val tparams2 = tparams.map(t => Id(t)) - // TODO if we freshen cparams, then we also need to substitute them in the result AND - val cparams2 = cparams //.map(c => Id(c)) - val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) - val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } - // In the following section, we create a new instance of the interface. - // All operation bodies were lifted to block literals in an earlier stage. - // While doing so, their block parameters (bparams) were concatenated with their capture parameters (cparams). - // When we embed back to core, we need to "eta-expand" the operation body to supply the correct captures from the environment. - // To see why this "eta-expansion" is necessary to achieve this, consider the following example: - // ```scala - // effect Eff(): Unit - // def use = { do Eff() } - // def main() = { - // val r = try { - // use() - // } with Eff { - // resume(()) - // } - // } - // ``` - // the handler body normalizes to the following: - // ```scala - // reset {{p} => - // jump use(){new Eff {def Eff = Eff @ [p]}} - // } - // ``` - // where - // ``` - // def Eff = (){p} { ... } - // ``` - // In particular, the prompt `p` needs to be passed to the lifted operation body. - // ``` - val (origBparams, synthBparams) = bparams2.splitAt(bparams2.length - environment.length) - val bargs = - // TODO: Fix captures - origBparams.map { case bp => BlockVar(bp.id, bp.tpe, Set()) } ++ - synthBparams.zip(environment).map { - // TODO: Fix captures - case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) - } - - core.Operation( - id, - tparams2, - cparams.take(cparams.length - environment.length), - vparams2, - origBparams, - Stmt.App( - embedBlockVar(label), - tparams2.map(ValueType.Var.apply), - vparams2.map(p => ValueVar(p.id, p.tpe)), - bargs - ) - ) - case _ => sys error "Unexpected block type" - } - } + val ops = operations.map { etaExpandToOperation.tupled } core.Block.New(Implementation(interface, ops)) } @@ -1357,6 +1296,74 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } } + /** + * Embed an operation as part of a `Computation.New`. + * This eta-expands the block var that stands for the operation body to a full operation + * so that we can supply the correct capture arguments from the environment. + */ + def etaExpandToOperation(id: Id, closure: Closure)(using G: TypingContext): core.Operation = { + val Closure(label, environment) = closure + G.blocks(label) match { + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + val tparams2 = tparams.map(t => Id(t)) + // TODO if we freshen cparams, then we also need to substitute them in the result AND + val cparams2 = cparams //.map(c => Id(c)) + val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) + val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } + // In the following section, we create a new instance of the interface. + // All operation bodies were lifted to block literals in an earlier stage. + // While doing so, their block parameters (bparams) were concatenated with their capture parameters (cparams). + // When we embed back to core, we need to "eta-expand" the operation body to supply the correct captures from the environment. + // To see why this "eta-expansion" is necessary to achieve this, consider the following example: + // ```scala + // effect Eff(): Unit + // def use = { do Eff() } + // def main() = { + // val r = try { + // use() + // } with Eff { + // resume(()) + // } + // } + // ``` + // the handler body normalizes to the following: + // ```scala + // reset {{p} => + // jump use(){new Eff {def Eff = Eff @ [p]}} + // } + // ``` + // where + // ``` + // def Eff = (){p} { ... } + // ``` + // In particular, the prompt `p` needs to be passed to the lifted operation body. + // ``` + val (origBparams, synthBparams) = bparams2.splitAt(bparams2.length - environment.length) + val bargs = + // TODO: Fix captures + origBparams.map { case bp => BlockVar(bp.id, bp.tpe, Set()) } ++ + synthBparams.zip(environment).map { + // TODO: Fix captures + case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) + } + + core.Operation( + id, + tparams2, + cparams.take(cparams.length - environment.length), + vparams2, + origBparams, + Stmt.App( + embedBlockVar(label), + tparams2.map(ValueType.Var.apply), + vparams2.map(p => ValueVar(p.id, p.tpe)), + bargs + ) + ) + case _ => sys error "Unexpected block type" + } + } + def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { case Block(tparams, vparams, bparams, b) => val cparams = bparams.map { From 7119c7b7270c1179be21845e15e9f6485fe3ffcd Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:52:24 +0100 Subject: [PATCH 071/123] add comment about evaluate of Box being wrong currently --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index cad2f4220..be12d6bcd 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -869,6 +869,17 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) case core.Expr.Box(b, annotatedCapture) => + // TODO this wrong + /* + var counter = 22; + val p : Borrowed[Int] at counter = box new Borrowed[Int] { + def dereference() = counter + }; + counter = counter + 1; + println(p.dereference) + */ + // should capture `counter` but does not since the stack is Stack.Unknown + // (effekt.JavaScriptTests.examples/pos/capture/borrows.effekt (js)) val comp = evaluate(b, "x", Stack.Unknown) scope.allocate("x", Value.Box(comp, annotatedCapture)) } From efb15368646da77334e6cfa5f1e327cda769b55c Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:52:57 +0100 Subject: [PATCH 072/123] fix toplevelEnv in run --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index be12d6bcd..8019e1e68 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1116,7 +1116,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val toplevelEnv = Env.empty // user-defined functions .bindComputation(mod.definitions.collect { - case Toplevel.Def(id, b) => id -> Computation.Def(Closure(id, Nil)) + case Toplevel.Def(id, b) => id -> Computation.Var(id) }) // user-defined values .bindValue(mod.definitions.collect { From d70d8e620614c58d7bafc9a449ada6d62d2c7176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 28 Oct 2025 17:28:41 +0100 Subject: [PATCH 073/123] Improve TODO --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 8019e1e68..1e8e229ae 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1317,7 +1317,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { G.blocks(label) match { case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => val tparams2 = tparams.map(t => Id(t)) - // TODO if we freshen cparams, then we also need to substitute them in the result AND + // TODO if we freshen cparams, then we also need to substitute them in the result AND the parameters val cparams2 = cparams //.map(c => Id(c)) val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } From 6733ee7e796633f44c192b57b7b1214171938ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Wed, 29 Oct 2025 09:49:27 +0100 Subject: [PATCH 074/123] Provide types for async externs --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 1e8e229ae..190760d7e 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1129,7 +1129,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Toplevel.Val(id, tpe, _) => id -> tpe }.toMap, mod.definitions.collect { case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap) // ++ asyncExterns.map { d => d.id -> null }) + }.toMap ++ asyncExterns.map { d => d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map(_.tpe), d.bparams.map(_.tpe), d.ret), d.annotatedCapture) }) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) mod.copy(definitions = newDefinitions) From 2b949fa0622f548cd9c1f9893310814667d4ed8b Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 30 Oct 2025 13:58:14 +0100 Subject: [PATCH 075/123] fix async top-level defs --- .../effekt/core/optimizer/NewNormalizer.scala | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 190760d7e..a5f1b96de 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1113,6 +1113,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { //util.trace(mod) // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } + val asyncTypes = asyncExterns.map { d => + d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map { _.tpe }, d.bparams.map { bp => bp.tpe }, d.ret), d.annotatedCapture) + } + val toplevelEnv = Env.empty // user-defined functions .bindComputation(mod.definitions.collect { @@ -1123,13 +1127,16 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Toplevel.Val(id, _, _) => id -> id }) // async extern functions - .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Def(Closure(defn.id, Nil)))) - - val typingContext = TypingContext(mod.definitions.collect { - case Toplevel.Val(id, tpe, _) => id -> tpe - }.toMap, mod.definitions.collect { - case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap ++ asyncExterns.map { d => d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map(_.tpe), d.bparams.map(_.tpe), d.ret), d.annotatedCapture) }) + .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Var(defn.id))) + + val typingContext = TypingContext( + mod.definitions.collect { + case Toplevel.Val(id, tpe, _) => id -> tpe + }.toMap, + mod.definitions.collect { + case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) + }.toMap ++ asyncTypes + ) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) mod.copy(definitions = newDefinitions) From b754eebf6b49bc96acf727dace7a1f912e934bd6 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 30 Oct 2025 14:52:42 +0100 Subject: [PATCH 076/123] fix escape analysis for Boxes --- .../effekt/core/optimizer/NewNormalizer.scala | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index a5f1b96de..4b3f48048 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -809,7 +809,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Computation.Def(Closure(f, closureParams.map(p => Computation.Var(p.id)))) case core.Block.Unbox(pure) => - val addr = evaluate(pure)(using env, scope) + val addr = evaluate(pure, escaping) scope.lookupValue(addr) match { case Some(Value.Box(body, _)) => body case Some(_) | None => { @@ -854,7 +854,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Computation.New(interface, ops) } - def evaluate(expr: Expr)(using env: Env, scope: Scope): Addr = expr match { + def evaluate(expr: Expr, escaping: Stack)(using env: Env, scope: Scope): Addr = expr match { case Expr.ValueVar(id, annotatedType) => env.lookupValue(id) @@ -863,13 +863,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // right now everything is stuck... no constant folding ... case core.Expr.PureApp(f, targs, vargs) => - scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate))) + scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) case core.Expr.Make(data, tag, targs, vargs) => - scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate))) + scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate(_, escaping)))) case core.Expr.Box(b, annotatedCapture) => - // TODO this wrong /* var counter = 22; val p : Borrowed[Int] at counter = box new Borrowed[Int] { @@ -880,7 +879,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { */ // should capture `counter` but does not since the stack is Stack.Unknown // (effekt.JavaScriptTests.examples/pos/capture/borrows.effekt (js)) - val comp = evaluate(b, "x", Stack.Unknown) + // TLDR we need to pass an escaping stack to do a proper escape analysis. Stack.Unkown is insufficient + val comp = evaluate(b, "x", escaping) scope.allocate("x", Value.Box(comp, annotatedCapture)) } @@ -888,7 +888,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { def evaluate(stmt: Stmt, k: Frame, ks: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { case Stmt.Return(expr) => - k.ret(ks, evaluate(expr)) + k.ret(ks, evaluate(expr, ks)) case Stmt.Val(id, annotatedTpe, binding, body) => evaluate(binding, k.push(annotatedTpe) { scope => res => k => ks => @@ -898,11 +898,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.ImpureApp(id, f, targs, vargs, bargs, body) => assert(bargs.isEmpty) - val addr = scope.run("x", f, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", Stack.Unknown))) + val addr = scope.run("x", f, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", Stack.Unknown))) evaluate(body, k, ks)(using env.bindValue(id, addr), scope) case Stmt.Let(id, annotatedTpe, binding, body) => - bind(id, evaluate(binding)) { evaluate(body, k, ks) } + bind(id, evaluate(binding, ks)) { evaluate(body, k, ks) } case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id, block) => println(s"Marking ${util.show(id)} as inlinable") @@ -919,7 +919,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO also bind type arguments in environment // TODO substitute cparams??? val newEnv = env - .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) + .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a, ks) }) .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) evaluate(body, k, ks)(using newEnv, scope) @@ -931,14 +931,14 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(callee, "f", escapingStack) match { case Computation.Inline(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => val newEnv = closureEnv - .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a) }) + .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a, ks) }) .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) evaluate(body, k, ks)(using newEnv, scope) case Computation.Var(id) => - reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } + reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => - val args = vargs.map(evaluate) + val args = vargs.map(evaluate(_, ks)) // TODO ks or Stack.Unkown? /* try { @@ -961,16 +961,16 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val escapingStack = Stack.Unknown evaluate(callee, "o", escapingStack) match { case Computation.Var(id) => - reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack))) } + reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.New(interface, operations) => operations.collectFirst { case (id, Closure(label, environment)) if id == method => - reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } + reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get case _: (Computation.Inline | Computation.Def | Computation.Continuation) => sys error s"Should not happen" } case Stmt.If(cond, thn, els) => - val sc = evaluate(cond) + val sc = evaluate(cond, ks) scope.lookupValue(sc) match { case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) case Some(Value.Literal(false, _)) => evaluate(els, k, ks) @@ -985,7 +985,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { } case Stmt.Match(scrutinee, clauses, default) => - val sc = evaluate(scrutinee) + val sc = evaluate(scrutinee, ks) scope.lookupValue(sc) match { case Some(Value.Make(data, tag, targs, vargs)) => // TODO substitute types (or bind them in the env)! @@ -1029,7 +1029,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { evaluate(body, Frame.Return, Stack.Region(cap, Map.empty, k, ks)) case Stmt.Region(_) => ??? case Stmt.Alloc(id, init, region, body) => - val addr = evaluate(init) + val addr = evaluate(init, ks) alloc(id, region, addr, ks) match { case Some(ks1) => evaluate(body, k, ks1) case None => NeutralStmt.Alloc(id, addr, region, nested { evaluate(body, k, ks) }) @@ -1037,7 +1037,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO case Stmt.Var(ref, init, capture, body) => - val addr = evaluate(init) + val addr = evaluate(init, ks) evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, addr, k, ks)) case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => get(ref, ks) match { @@ -1045,10 +1045,11 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case None => bind(id, scope.allocateGet(ref, annotatedTpe, annotatedCapt)) { evaluate(body, k, ks) } } case Stmt.Put(ref, annotatedCapt, value, body) => - put(ref, evaluate(value), ks) match { + val addr = evaluate(value, ks) + put(ref, addr, ks) match { case Some(stack) => evaluate(body, k, stack) case None => - NeutralStmt.Put(ref, value.tpe, annotatedCapt, evaluate(value), nested { evaluate(body, k, ks) }) + NeutralStmt.Put(ref, value.tpe, annotatedCapt, addr, nested { evaluate(body, k, ks) }) } // Control Effects From affa610590f36575658ee9c016ce5846cb072c80 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 30 Oct 2025 15:26:15 +0100 Subject: [PATCH 077/123] fix Stack.Var reification --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4b3f48048..eccc3906c 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -593,7 +593,8 @@ object semantics { case Stack.Var(id, curr, init, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - NeutralStmt.Var(id, init, body) + if (body.free contains id.id) NeutralStmt.Var(id, curr, body) + else stmt }} case Stack.Region(id, bindings, frame, next) => reify(next) { reify(frame) { From 7cf8ed765c6859937eecad5cba887b16bd871aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Tue, 4 Nov 2025 10:11:12 +0100 Subject: [PATCH 078/123] Remove redundant init value tracking --- .../effekt/core/optimizer/NewNormalizer.scala | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index eccc3906c..3432b6a94 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -382,7 +382,7 @@ object semantics { case Stack.Empty => NeutralStmt.Return(arg) case Stack.Unknown => NeutralStmt.Return(arg) case Stack.Reset(p, k, ks) => k.ret(ks, arg) - case Stack.Var(id, curr, init, k, ks) => k.ret(ks, arg) + case Stack.Var(id, curr, k, ks) => k.ret(ks, arg) case Stack.Region(id, bindings, k, ks) => k.ret(ks, arg) } case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) @@ -420,7 +420,7 @@ object semantics { */ case Unknown case Reset(prompt: BlockParam, frame: Frame, next: Stack) - case Var(id: BlockParam, curr: Addr, init: Addr, frame: Frame, next: Stack) + case Var(id: BlockParam, curr: Addr, frame: Frame, next: Stack) // TODO desugar regions into var? case Region(id: BlockParam, bindings: Map[Id, Addr], frame: Frame, next: Stack) @@ -428,7 +428,7 @@ object semantics { case Stack.Empty => Nil case Stack.Unknown => Nil case Stack.Reset(prompt, frame, next) => prompt :: next.bound - case Stack.Var(id, curr, init, frame, next) => id :: next.bound + case Stack.Var(id, curr, frame, next) => id :: next.bound case Stack.Region(id, bindings, frame, next) => id :: next.bound } } @@ -438,8 +438,8 @@ object semantics { // We have reached the end of the known stack, so the variable must be in the unknown part. case Stack.Unknown => None case Stack.Reset(prompt, frame, next) => get(ref, next) - case Stack.Var(id1, curr, init, frame, next) if ref == id1.id => Some(curr) - case Stack.Var(id1, curr, init, frame, next) => get(ref, next) + case Stack.Var(id1, curr, frame, next) if ref == id1.id => Some(curr) + case Stack.Var(id1, curr, frame, next) => get(ref, next) case Stack.Region(id, bindings, frame, next) => if (bindings.contains(ref)) { Some(bindings(ref)) @@ -453,8 +453,8 @@ object semantics { // We have reached the end of the known stack, so the variable must be in the unknown part. case Stack.Unknown => None case Stack.Reset(prompt, frame, next) => put(ref, value, next).map(Stack.Reset(prompt, frame, _)) - case Stack.Var(id, curr, init, frame, next) if ref == id.id => Some(Stack.Var(id, value, init, frame, next)) - case Stack.Var(id, curr, init, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, init, frame, _)) + case Stack.Var(id, curr, frame, next) if ref == id.id => Some(Stack.Var(id, value, frame, next)) + case Stack.Var(id, curr, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, frame, _)) case Stack.Region(id, bindings, frame, next) => if (bindings.contains(ref)){ Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) @@ -471,8 +471,8 @@ object semantics { case Stack.Unknown => None case Stack.Reset(prompt, frame, next) => alloc(ref, reg, value, next).map(Stack.Reset(prompt, frame, _)) - case Stack.Var(id, curr, init, frame, next) => - alloc(ref, reg, value, next).map(Stack.Var(id, curr, init, frame, _)) + case Stack.Var(id, curr, frame, next) => + alloc(ref, reg, value, next).map(Stack.Var(id, curr, frame, _)) case Stack.Region(id, bindings, frame, next) => if (reg == id.id){ Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) @@ -484,7 +484,7 @@ object semantics { enum Cont { case Empty case Reset(frame: Frame, prompt: BlockParam, rest: Cont) - case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) + case Var(frame: Frame, id: BlockParam, curr: Addr, rest: Cont) case Region(frame: Frame, id: BlockParam, bindings: Map[Id, Addr], rest: Cont) } @@ -496,9 +496,9 @@ object semantics { case Stack.Reset(prompt, frame, next) => val (c, frame2, stack) = shift(p, frame, next) (Cont.Reset(k, prompt, c), frame2, stack) - case Stack.Var(id, curr, init, frame, next) => + case Stack.Var(id, curr, frame, next) => val (c, frame2, stack) = shift(p, frame, next) - (Cont.Var(k, id, curr, init, c), frame2, stack) + (Cont.Var(k, id, curr, c), frame2, stack) case Stack.Region(id, bindings, frame, next) => val (c, frame2, stack) = shift(p, frame, next) (Cont.Region(k, id, bindings, c), frame2, stack) @@ -510,9 +510,9 @@ object semantics { val (k1, ks1) = resume(rest, frame, ks) val stack = Stack.Reset(prompt, k1, ks1) (frame, stack) - case Cont.Var(frame, id, curr, init, rest) => + case Cont.Var(frame, id, curr, rest) => val (k1, ks1) = resume(rest, frame, ks) - (frame, Stack.Var(id, curr, init, k1, ks1)) + (frame, Stack.Var(id, curr, k1, ks1)) case Cont.Region(frame, id, bindings, rest) => val (k1, ks1) = resume(rest, frame, ks) (frame, Stack.Region(id, bindings, k1, ks1)) @@ -544,8 +544,8 @@ object semantics { case Stack.Unknown => Stack.Unknown case Stack.Reset(prompt, frame, next) => Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) - case Stack.Var(id, curr, init, frame, next) => - Stack.Var(id, curr, init, reifyFrame(frame, next), reifyStack(next)) + case Stack.Var(id, curr, frame, next) => + Stack.Var(id, curr, reifyFrame(frame, next), reifyStack(next)) case Stack.Region(id, bindings, frame, next) => Stack.Region(id, bindings, reifyFrame(frame, next), reifyStack(next)) } @@ -590,7 +590,7 @@ object semantics { if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) else stmt // TODO this runs normalization a second time in the outer scope! }} - case Stack.Var(id, curr, init, frame, next) => + case Stack.Var(id, curr, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } if (body.free contains id.id) NeutralStmt.Var(id, curr, body) @@ -1039,7 +1039,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // TODO case Stmt.Var(ref, init, capture, body) => val addr = evaluate(init, ks) - evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, addr, k, ks)) + evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => get(ref, ks) match { case Some(addr) => bind(id, addr) { evaluate(body, k, ks) } From d202bdd0f3f9d9b2aa17b91b9257d158d4ec7d28 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:18:29 +0100 Subject: [PATCH 079/123] fix region reification --- .../effekt/core/optimizer/NewNormalizer.scala | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 3432b6a94..4eafa06a4 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -345,7 +345,7 @@ object semantics { case Put(ref: Id, tpe: ValueType, cap: Captures, value: Addr, body: BasicBlock) case Region(id: BlockParam, body: BasicBlock) - case Alloc(id: Id, init: Addr, region: Id, body: BasicBlock) + case Alloc(id: BlockParam, init: Addr, region: Id, body: BasicBlock) // aborts at runtime case Hole(span: Span) @@ -363,7 +363,7 @@ object semantics { case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref, value) ++ body.free case NeutralStmt.Region(id, body) => body.free - id.id - case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id + case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id.id case NeutralStmt.Hole(span) => Set.empty } } @@ -422,14 +422,14 @@ object semantics { case Reset(prompt: BlockParam, frame: Frame, next: Stack) case Var(id: BlockParam, curr: Addr, frame: Frame, next: Stack) // TODO desugar regions into var? - case Region(id: BlockParam, bindings: Map[Id, Addr], frame: Frame, next: Stack) + case Region(id: BlockParam, bindings: Map[BlockParam, Addr], frame: Frame, next: Stack) lazy val bound: List[BlockParam] = this match { case Stack.Empty => Nil case Stack.Unknown => Nil case Stack.Reset(prompt, frame, next) => prompt :: next.bound case Stack.Var(id, curr, frame, next) => id :: next.bound - case Stack.Region(id, bindings, frame, next) => id :: next.bound + case Stack.Region(id, bindings, frame, next) => id :: next.bound ++ bindings.keys } } @@ -441,8 +441,9 @@ object semantics { case Stack.Var(id1, curr, frame, next) if ref == id1.id => Some(curr) case Stack.Var(id1, curr, frame, next) => get(ref, next) case Stack.Region(id, bindings, frame, next) => - if (bindings.contains(ref)) { - Some(bindings(ref)) + val containsRef = bindings.keys.map { b => b._1.id }.toSet.contains(ref) + if (containsRef) { + Some(bindings(id)) } else { get(ref, next) } @@ -456,14 +457,15 @@ object semantics { case Stack.Var(id, curr, frame, next) if ref == id.id => Some(Stack.Var(id, value, frame, next)) case Stack.Var(id, curr, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, frame, _)) case Stack.Region(id, bindings, frame, next) => - if (bindings.contains(ref)){ - Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) + val containsRef = bindings.keys.map { b => b._1.id }.toSet.contains(ref) + if (containsRef){ + Some(Stack.Region(id, bindings.updated(id, value), frame, next)) } else { put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) } } - def alloc(ref: Id, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { + def alloc(ref: BlockParam, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { // This case can occur if we normalize a function that abstracts over a region as a parameter // We return None and force the reification of the allocation case Stack.Empty => None @@ -484,8 +486,13 @@ object semantics { enum Cont { case Empty case Reset(frame: Frame, prompt: BlockParam, rest: Cont) +<<<<<<< HEAD case Var(frame: Frame, id: BlockParam, curr: Addr, rest: Cont) case Region(frame: Frame, id: BlockParam, bindings: Map[Id, Addr], rest: Cont) +======= + case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) + case Region(frame: Frame, id: BlockParam, bindings: Map[BlockParam, Addr], rest: Cont) +>>>>>>> 9039d884 (fix region reification) } def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { @@ -599,7 +606,14 @@ object semantics { case Stack.Region(id, bindings, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - if (body.free contains id.id) NeutralStmt.Region(id, body) + val bodyUsesBinding = body.free.exists(bindings.map { b => b._1.id }.toSet.contains(_)) + if (body.free.contains(id.id) || bodyUsesBinding) { + // we need to reify all bindings in this region as allocs using their current value + val reifiedAllocs = bindings.foldLeft(body) { case (acc, (bp, addr)) => + nested { NeutralStmt.Alloc(bp, addr, id.id, acc) } + } + NeutralStmt.Region(id, reifiedAllocs) + } else stmt }} } @@ -759,7 +773,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } // Only normalize again if we actually we wrong in our assumption that we capture nothing - // We might run into exponential complexity for nested recursives functions + // We might run into exponential complexity for nested recursive functions if (closureParams.isEmpty) { scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) Computation.Def(Closure(freshened, Nil)) @@ -1031,12 +1045,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Region(_) => ??? case Stmt.Alloc(id, init, region, body) => val addr = evaluate(init, ks) - alloc(id, region, addr, ks) match { + val bp = BlockParam(id, Type.TState(init.tpe), Set(region)) + alloc(bp, region, addr, ks) match { case Some(ks1) => evaluate(body, k, ks1) - case None => NeutralStmt.Alloc(id, addr, region, nested { evaluate(body, k, ks) }) + case None => NeutralStmt.Alloc(bp, addr, region, nested { evaluate(body, k, ks) }) } - // TODO case Stmt.Var(ref, init, capture, body) => val addr = evaluate(init, ks) evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) @@ -1215,8 +1229,8 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) case NeutralStmt.Region(id, body) => Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputation(id)))) - case NeutralStmt.Alloc(id, init, region, body) => - Stmt.Alloc(id, embedExpr(init), region, embedStmt(body)) + case NeutralStmt.Alloc(blockparam, init, region, body) => + Stmt.Alloc(blockparam.id, embedExpr(init), region, embedStmt(body)(using G.bind(blockparam.id, blockparam.tpe, blockparam.capt))) case NeutralStmt.Hole(span) => Stmt.Hole(span) } From 2da5d4c63bf529c5f73e646466cd39289aa2a358 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:20:12 +0100 Subject: [PATCH 080/123] some comments --- effekt/shared/src/main/scala/effekt/core/Type.scala | 2 +- .../src/main/scala/effekt/core/optimizer/Normalizer.scala | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/Type.scala b/effekt/shared/src/main/scala/effekt/core/Type.scala index 0c279aba4..93125bf05 100644 --- a/effekt/shared/src/main/scala/effekt/core/Type.scala +++ b/effekt/shared/src/main/scala/effekt/core/Type.scala @@ -131,7 +131,7 @@ object Type { def instantiate(f: BlockType.Function, targs: List[ValueType], cargs: List[Captures]): BlockType.Function = f match { case BlockType.Function(tparams, cparams, vparams, bparams, result) => assert(targs.size == tparams.size, "Wrong number of type arguments") - assert(cargs.size == cparams.size, s"Wrong number of capture arguments on ${util.show(f)}: ${util.show(cargs)}") + assert(cargs.size == cparams.size, s"Wrong number of capture arguments on ${util.show(f)} (capture arguments != capture parameters): ${util.show(cargs)} != ${util.show(cparams)}") val tsubst = (tparams zip targs).toMap val csubst = (cparams zip cargs).toMap diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala index edc338358..464e1791c 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Normalizer.scala @@ -35,6 +35,12 @@ object Normalizer { normal => // In the future, Stmt.Shift should also be performed statically. case Stmt.Val(_, _, binding: (Stmt.Reset | Stmt.Var | Stmt.App | Stmt.Invoke | Stmt.Region | Stmt.Shift | Stmt.Resume), body) => assertNormal(binding); assertNormal(body) + /* + val x = if (...) { return 1 } else { return 2 }; s + is always normalized to + def joinpoint(x: Int) = s + if (...) { joinpoint(1) } else { joinpoint(2) } + */ case t @ Stmt.Val(_, _, binding, body) => E.warning(s"Not allowed as binding of Val: ${util.show(t)}") case t @ Stmt.App(b: BlockLit, targs, vargs, bargs) => From 94d7a59484497e00a1fcb7d2efee620aefd8c586 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:20:44 +0100 Subject: [PATCH 081/123] fully reactivate VM tests --- effekt/jvm/src/test/scala/effekt/core/VMTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/VMTests.scala b/effekt/jvm/src/test/scala/effekt/core/VMTests.scala index 5126b960a..4aad51f3e 100644 --- a/effekt/jvm/src/test/scala/effekt/core/VMTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/VMTests.scala @@ -1295,7 +1295,7 @@ class VMTests extends munit.FunSuite { val (result, summary) = runFile(path) val expected = expectedResultFor(f).getOrElse { s"Missing checkfile for ${path}"} assertNoDiff(result, expected) - //expectedSummary.foreach { expected => assertEquals(summary, expected) } + expectedSummary.foreach { expected => assertEquals(summary, expected) } } catch { case i: VMError => fail(i.getMessage, i) } From 3396014d802133714b908e77980d02f13122dcf2 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:21:34 +0100 Subject: [PATCH 082/123] don't assertNormal for now (broken due to missing inlining) --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 2 +- .../shared/src/main/scala/effekt/core/optimizer/Optimizer.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4eafa06a4..9ec8b8fcb 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1144,7 +1144,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { }) // async extern functions .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Var(defn.id))) - + val typingContext = TypingContext( mod.definitions.collect { case Toplevel.Val(id, tpe, _) => id -> tpe diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 120fed3dc..671ded14a 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -39,7 +39,7 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } tree = Context.timed("new-normalizer-1", source.name) { dontInline.run(tree) } - Normalizer.assertNormal(tree) + //Normalizer.assertNormal(tree) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } From dac43b2fcf280c0a835e704c178dee5f907409cb Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:53:28 +0100 Subject: [PATCH 083/123] fix bindings of top-level defs --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 9ec8b8fcb..49b014613 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -2,7 +2,6 @@ package effekt package core package optimizer -import effekt.core.{BlockType, Toplevel} import effekt.core.ValueType.Boxed import effekt.source.Span import effekt.core.optimizer.semantics.{Computation, NeutralStmt, Value} @@ -954,7 +953,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate(_, ks)) - // TODO ks or Stack.Unkown? + // TODO ks or Stack.Unknown? /* try { prog { @@ -1136,7 +1135,12 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val toplevelEnv = Env.empty // user-defined functions .bindComputation(mod.definitions.collect { - case Toplevel.Def(id, b) => id -> Computation.Var(id) + case Toplevel.Def(id, b) => id -> (b match { + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => Computation.Def(Closure(id, Nil)) + case core.Block.BlockVar(idd, annotatedTpe, annotatedCapt) => Computation.Var(id) + case core.Block.Unbox(pure) => Computation.Var(id) + case core.Block.New(impl) => Computation.Var(id) + }) }) // user-defined values .bindValue(mod.definitions.collect { From 1ed7966d51b68c95d157b1ca09a72cd1196220cf Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:54:51 +0100 Subject: [PATCH 084/123] reifyKnown if call is pure --- .../effekt/core/optimizer/NewNormalizer.scala | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 49b014613..ac4eecdfb 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -298,7 +298,10 @@ object semantics { case Continuation(k: Cont) + // TODO ? distinguish? //case Region(prompt: Id) ??? + //case Prompt(prompt: Id) ??? + //case Reference(prompt: Id) ??? // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) @@ -584,6 +587,20 @@ object semantics { NeutralStmt.Jump(label, Nil, List(tmp), closure) } + def reifyKnown(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + k match { + case Frame.Return => reify(ks) { stmt } + case Frame.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(scope)(tmp)(ks) + case Frame.Dynamic(Closure(label, closure)) => reify(ks) { scope ?=> + val tmp = Id("tmp") + scope.push(tmp, stmt) + NeutralStmt.Jump(label, Nil, List(tmp), closure) + } + } + @tailrec final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = { ks match { @@ -965,7 +982,17 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { is incorrect as the result is always the empty capture set since Stack.Unkown.bound = Set() */ val blockargs = bargs.map(evaluate(_, "f", ks)) - reify(k, ks) { NeutralStmt.Jump(label, targs, args, blockargs ++ environment) } + // TODO isPureApp(Closure(label, environment), stmt.capt, blockargs) is more precise + // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation + if (stmt.capt.isEmpty) { + reifyKnown(k, ks) { + NeutralStmt.Jump(label, targs, args, blockargs) + } + } else { + reify(k, ks) { + NeutralStmt.Jump(label, targs, args, blockargs ++ environment) + } + } case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" } From a1ec539b99f9fe46ec85533e835e998e89b1eebb Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 16:52:01 +0100 Subject: [PATCH 085/123] fix error in get and put for regions --- .../effekt/core/optimizer/NewNormalizer.scala | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index ac4eecdfb..e4b61a010 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -396,16 +396,6 @@ object semantics { Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) } - // case class Store(var vars: Map[Addr, Value]) { - // def get(addr: Addr): Value = { - // vars(addr) - // } - // def set(addr: Addr, v: Value): Unit = { - // vars = vars.updated(addr, v) - // } - // } - type Store = Map[Id, Addr] - // maybe, for once it is simpler to decompose stacks like // // f, (p, f) :: (p, f) :: Nil @@ -443,11 +433,10 @@ object semantics { case Stack.Var(id1, curr, frame, next) if ref == id1.id => Some(curr) case Stack.Var(id1, curr, frame, next) => get(ref, next) case Stack.Region(id, bindings, frame, next) => - val containsRef = bindings.keys.map { b => b._1.id }.toSet.contains(ref) - if (containsRef) { - Some(bindings(id)) - } else { - get(ref, next) + val containsRef = bindings.keys.find(bp => bp.id == ref) + containsRef match { + case Some(bparam) => Some(bindings(bparam)) + case None => get(ref, next) } } @@ -459,11 +448,10 @@ object semantics { case Stack.Var(id, curr, frame, next) if ref == id.id => Some(Stack.Var(id, value, frame, next)) case Stack.Var(id, curr, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, frame, _)) case Stack.Region(id, bindings, frame, next) => - val containsRef = bindings.keys.map { b => b._1.id }.toSet.contains(ref) - if (containsRef){ - Some(Stack.Region(id, bindings.updated(id, value), frame, next)) - } else { - put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) + val containsRef = bindings.keys.find(bp => bp.id == ref) + containsRef match { + case Some(bparam) => Some(Stack.Region(id, bindings.updated(bparam, value), frame, next)) + case None => put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) } } @@ -571,14 +559,14 @@ object semantics { val tmp = Id("tmp") scope.push(tmp, stmt) // TODO Over-approximation - // Don't pass Stack.Unkown but rather the stack until the next reset? + // Don't pass Stack.Unknown but rather the stack until the next reset? /* |----------| |----------| |---------| | | ---> ... ---> | | ---> ... ---> | | ---> ... |----------| |----------| |---------| r1 r2 first next prompt - Pass r1 :: ... :: r2 :: ... :: prompt :: UNKOWN + Pass r1 :: ... :: r2 :: ... :: prompt :: UNKNOWN */ apply(scope)(tmp)(Stack.Unknown) case Frame.Dynamic(Closure(label, closure)) => From e6ee9ea1937f84b7ba6055f303c1dab7bddcf008 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:18:01 +0100 Subject: [PATCH 086/123] also check environment to determine whether App is pure --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index e4b61a010..b65d470cb 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -972,9 +972,10 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { val blockargs = bargs.map(evaluate(_, "f", ks)) // TODO isPureApp(Closure(label, environment), stmt.capt, blockargs) is more precise // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation - if (stmt.capt.isEmpty) { + // TODO examples/pos/lambdas/localstate.effekt fails if we only check stmt.capt + if (stmt.capt.isEmpty && environment.isEmpty) { reifyKnown(k, ks) { - NeutralStmt.Jump(label, targs, args, blockargs) + NeutralStmt.Jump(label, targs, args, blockargs ++ environment) } } else { reify(k, ks) { From 324e0560045bf747b347a6c8b50a6cd55f9e7d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Wed, 5 Nov 2025 09:44:47 +0100 Subject: [PATCH 087/123] Rip out previous attempt at inlining --- .../effekt/core/NewNormalizerTests.scala | 2 +- .../effekt/core/optimizer/NewNormalizer.scala | 20 ++----------------- .../effekt/core/optimizer/Optimizer.scala | 9 +-------- 3 files changed, 4 insertions(+), 27 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 4d7d9bca2..9799f5001 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -731,7 +731,7 @@ class NormalizeOnly extends Compiler[(Id, symbols.Module, ModuleDecl)] { case input @ CoreTransformed(source, tree, mod, core) => val mainSymbol = Context.ensureMainExists(mod) var tree = Deadcode.remove(mainSymbol, core) - val normalizer = NewNormalizer { (id, b) => false } + val normalizer = NewNormalizer() tree = normalizer.run(tree) Normalizer.assertNormal(tree) (mainSymbol, mod, tree) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index b65d470cb..bd841eca8 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -293,9 +293,6 @@ object semantics { // Known function case Def(closure: Closure) - // TODO it looks like this was not a good idea... Many operations (like embed) are not supported on Inline - case Inline(body: core.BlockLit, closure: Env) - case Continuation(k: Cont) // TODO ? distinguish? @@ -309,7 +306,6 @@ object semantics { lazy val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free - case Computation.Inline(body, closure) => Set.empty // TODO ??? case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } @@ -708,7 +704,6 @@ object semantics { def toDoc(comp: Computation): Doc = comp match { case Computation.Var(id) => toDoc(id) case Computation.Def(closure) => toDoc(closure) - case Computation.Inline(block, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => "def" <+> toDoc(id) <+> "=" <+> toDoc(impl) }, ",") @@ -751,7 +746,7 @@ object semantics { /** * A new normalizer that is conservative (avoids code bloat) */ -class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { +class NewNormalizer { import semantics.* @@ -923,10 +918,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { case Stmt.Let(id, annotatedTpe, binding, body) => bind(id, evaluate(binding, ks)) { evaluate(body, k, ks) } - case Stmt.Def(id, block: core.BlockLit, body) if shouldInline(id, block) => - println(s"Marking ${util.show(id)} as inlinable") - bind(id, Computation.Inline(block, env)) { evaluate(body, k, ks) } - // can be recursive case Stmt.Def(id, block: core.BlockLit, body) => bind(id, evaluateRecursive(id, block, ks)) { evaluate(body, k, ks) } @@ -948,12 +939,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { // Here the stack passed to the blocks is an empty one since we reify it anyways... val escapingStack = Stack.Unknown evaluate(callee, "f", escapingStack) match { - case Computation.Inline(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), closureEnv) => - val newEnv = closureEnv - .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a, ks) }) - .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) - - evaluate(body, k, ks)(using newEnv, scope) case Computation.Var(id) => reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } case Computation.Def(Closure(label, environment)) => @@ -996,7 +981,7 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get - case _: (Computation.Inline | Computation.Def | Computation.Continuation) => sys error s"Should not happen" + case _: (Computation.Def | Computation.Continuation) => sys error s"Should not happen" } case Stmt.If(cond, thn, els) => @@ -1301,7 +1286,6 @@ class NewNormalizer(shouldInline: (Id, BlockLit) => Boolean) { embedBlockVar(label) case Computation.Def(closure) => etaExpandToBlockLit(closure) - case Computation.Inline(blocklit, env) => ??? case Computation.Continuation(k) => ??? case Computation.New(interface, operations) => val ops = operations.map { etaExpandToOperation.tupled } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 671ded14a..4cb6fa7f3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -30,15 +30,8 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - def inlineSmall(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => - usage.get(id).contains(Once) || (!usage.get(id).contains(Recursive) && b.size < 40) - } - val dontInline = NewNormalizer { (id, b) => false } - def inlineUnique(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) } - def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } - // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } - tree = Context.timed("new-normalizer-1", source.name) { dontInline.run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } //Normalizer.assertNormal(tree) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) From 6ffc203318774287f05722118600144d31f419e0 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 5 Nov 2025 15:31:35 +0100 Subject: [PATCH 088/123] delete Computation.Inline --- .../effekt/core/optimizer/NewNormalizer.scala | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index bd841eca8..426ff0d79 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -421,6 +421,7 @@ object semantics { } } + @tailrec def get(ref: Id, ks: Stack): Option[Addr] = ks match { case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(ref)} in empty stack" // We have reached the end of the known stack, so the variable must be in the unknown part. @@ -578,9 +579,9 @@ object semantics { val tmp = Id("tmp") scope.push(tmp, stmt) apply(scope)(tmp)(ks) - case Frame.Dynamic(Closure(label, closure)) => reify(ks) { scope ?=> + case Frame.Dynamic(Closure(label, closure)) => reify(ks) { sc ?=> val tmp = Id("tmp") - scope.push(tmp, stmt) + sc.push(tmp, stmt) NeutralStmt.Jump(label, Nil, List(tmp), closure) } } @@ -590,9 +591,8 @@ object semantics { ks match { case Stack.Empty => stmt case Stack.Unknown => stmt - // only reify reset if p is free in body case Stack.Reset(prompt, frame, next) => - reify(next) { reify(frame) { // reify(next) { reify(store) { reify(frame) { ... }}} ??? ORDER? TODO + reify(next) { reify(frame) { val body = nested { stmt } if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) else stmt // TODO this runs normalization a second time in the outer scope! @@ -935,15 +935,11 @@ class NewNormalizer { evaluate(body, k, ks)(using newEnv, scope) case Stmt.App(callee, targs, vargs, bargs) => - // TODO Why? Should it really be Stack.Unkown? - // Here the stack passed to the blocks is an empty one since we reify it anyways... - val escapingStack = Stack.Unknown - evaluate(callee, "f", escapingStack) match { + evaluate(callee, "f", ks) match { case Computation.Var(id) => - reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } + reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", ks))) } case Computation.Def(Closure(label, environment)) => val args = vargs.map(evaluate(_, ks)) - // TODO ks or Stack.Unknown? /* try { prog { @@ -955,12 +951,12 @@ class NewNormalizer { is incorrect as the result is always the empty capture set since Stack.Unkown.bound = Set() */ val blockargs = bargs.map(evaluate(_, "f", ks)) - // TODO isPureApp(Closure(label, environment), stmt.capt, blockargs) is more precise // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation // TODO examples/pos/lambdas/localstate.effekt fails if we only check stmt.capt + // TODO capture {io, global} is also fine if (stmt.capt.isEmpty && environment.isEmpty) { reifyKnown(k, ks) { - NeutralStmt.Jump(label, targs, args, blockargs ++ environment) + NeutralStmt.Jump(label, targs, args, blockargs) } } else { reify(k, ks) { From 801626e3afb6279d95dfa74905e166c75bfd0c3b Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:26:02 +0100 Subject: [PATCH 089/123] add very basic inliner --- .../scala/effekt/core/optimizer/Inliner.scala | 78 +++++++++++++++++++ .../effekt/core/optimizer/Optimizer.scala | 15 +++- 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala new file mode 100644 index 000000000..cdd38af84 --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -0,0 +1,78 @@ +package effekt.core.optimizer + +import effekt.core.* +import effekt.util + +sealed trait InliningPolicy { + def apply(id: Id): Boolean +} + +class Unique(usage: Map[Id, Usage]) extends InliningPolicy { + override def apply(id: Id): Boolean = + usage.get(id).contains(Usage.Once) && usage.get(id).contains(Usage.Recursive) +} + +case class Context( + blocks: Map[Id, Block], + exprs: Map[Id, Expr], + //maxInlineSize: Int, +) { + def bind(id: Id, expr: Expr): Context = copy(exprs = exprs + (id -> expr)) + def bind(id: Id, block: Block): Context = copy(blocks = blocks + (id -> block)) +} + +object Context { + def empty: Context = Context(Map.empty, Map.empty) +} + +class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Context] { + + def run(mod: ModuleDecl): ModuleDecl = { + given Context = Context.empty + mod match { + case ModuleDecl(path, includes, declarations, externs, definitions, exports) => + ModuleDecl(path, includes, declarations, externs, definitions.map { d => + util.trace("rewriting", util.show(d)) + val res = rewrite(d) + util.trace("after", util.show(res)) + res + }, exports) + } + } + + private def blockFor(id: Id)(using ctx: Context): Option[Block] = + ctx.blocks.get(id) + + private def exprFor(id: Id)(using ctx: Context): Option[Expr] = + ctx.exprs.get(id) + + override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { + case app @ Stmt.App(Block.BlockVar(id, tpe, capts), targs, vargs, bargs) if shouldInline(id) => + val BlockType.Function(tparams, cparams, vparams, bparams, result) = tpe match { + case f: BlockType.Function => f + case BlockType.Interface(name, targs) => ??? + } + // TODO this is wrong, we need to bind the arguments + util.trace("inlining", id) + Stmt.App(blockFor(id).get, targs, vargs, bargs) + case Stmt.Def(id, block, body) => + ctx.bind(id, block) + Stmt.Def(id, block, rewrite(body)) + case Stmt.Let(id, tpe, binding, body) => + ctx.bind(id, binding) + Stmt.Let(id, tpe, binding, rewrite(body)) + } + + override def expr(using Context): PartialFunction[Expr, Expr] = { + case v @ Expr.ValueVar(id, tpe) if shouldInline(id) => + val e = exprFor(id) + util.trace("inlining", id) + e match { + case Some(p: Expr.Make) => p + case Some(p: Expr.Literal) => p + case Some(p: Expr.Box) => p + case Some(other) if other.capt.isEmpty => other + case _ => v + } + } +} diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 4cb6fa7f3..1aa11e7bc 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -29,10 +29,21 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { } if !Context.config.optimize() then return tree; - + + /* + def inlineSmall(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => + usage.get(id).contains(Once) || (!usage.get(id).contains(Recursive) && b.size < 40) + } + val dontInline = NewNormalizer { (id, b) => false } + def inlineUnique(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) } + def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } + */ // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - //Normalizer.assertNormal(tree) + val usages = Reachable(Set(mainSymbol), tree) + util.trace(usages) + tree = Inliner(Unique(usages)).run(tree) + util.trace(util.show(tree)) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } From 353b2ddc3b073b635600fee05c7ddade88c229c0 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Thu, 6 Nov 2025 18:14:07 +0100 Subject: [PATCH 090/123] get inliner to work --- .../scala/effekt/core/optimizer/Inliner.scala | 81 +++++++++++++------ .../effekt/core/optimizer/NewNormalizer.scala | 4 +- .../effekt/core/optimizer/Optimizer.scala | 8 +- 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index cdd38af84..c4277dec8 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -1,28 +1,30 @@ package effekt.core.optimizer import effekt.core.* -import effekt.util +import effekt.core sealed trait InliningPolicy { - def apply(id: Id): Boolean + def apply(id: Id)(using Context): Boolean } class Unique(usage: Map[Id, Usage]) extends InliningPolicy { - override def apply(id: Id): Boolean = - usage.get(id).contains(Usage.Once) && usage.get(id).contains(Usage.Recursive) + override def apply(id: Id)(using ctx: Context): Boolean = + usage.get(id).contains(Usage.Once) && + !usage.get(id).contains(Usage.Recursive) && + ctx.blocks.get(id).exists(_.size <= ctx.maxInlineSize) } case class Context( blocks: Map[Id, Block], exprs: Map[Id, Expr], - //maxInlineSize: Int, + maxInlineSize: Int, ) { def bind(id: Id, expr: Expr): Context = copy(exprs = exprs + (id -> expr)) def bind(id: Id, block: Block): Context = copy(blocks = blocks + (id -> block)) } object Context { - def empty: Context = Context(Map.empty, Map.empty) + def empty: Context = Context(Map.empty, Map.empty, 50) } class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Context] { @@ -31,12 +33,7 @@ class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Cont given Context = Context.empty mod match { case ModuleDecl(path, includes, declarations, externs, definitions, exports) => - ModuleDecl(path, includes, declarations, externs, definitions.map { d => - util.trace("rewriting", util.show(d)) - val res = rewrite(d) - util.trace("after", util.show(res)) - res - }, exports) + ModuleDecl(path, includes, declarations, externs, definitions.map(rewrite), exports) } } @@ -46,27 +43,51 @@ class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Cont private def exprFor(id: Id)(using ctx: Context): Option[Expr] = ctx.exprs.get(id) - override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { - case app @ Stmt.App(Block.BlockVar(id, tpe, capts), targs, vargs, bargs) if shouldInline(id) => - val BlockType.Function(tparams, cparams, vparams, bparams, result) = tpe match { - case f: BlockType.Function => f - case BlockType.Interface(name, targs) => ??? + def bindBlock(body: Stmt, bparam: BlockParam, barg: Block): Stmt = + Stmt.Def(bparam.id, barg, body) + + def bindBlocks(body: Stmt, blocks: List[(BlockParam, Block)]): Stmt = + blocks.headOption.map { (bp, barg) => + blocks.tail.foldRight(bindBlock(body, bp, barg)) { case ((bp, barg), acc) => + bindBlock(acc, bp, barg) + } + }.getOrElse(body) + + def bindValue(body: Stmt, vparam: ValueParam, varg: Expr): Stmt = + Stmt.Let(vparam.id, vparam.tpe, varg, body) + + def bindValues(body: Stmt, values: List[(ValueParam, Expr)]): Stmt = + values.headOption.map { (vp, varg) => + values.tail.foldRight(Stmt.Let(vp.id, vp.tpe, varg, body)) { case ((vp, varg), acc) => + bindValue(acc, vp, varg) } - // TODO this is wrong, we need to bind the arguments - util.trace("inlining", id) - Stmt.App(blockFor(id).get, targs, vargs, bargs) + }.getOrElse(body) + + override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { + case app @ Stmt.App(bvar: BlockVar, targs, vargs, bargs) if shouldInline(bvar.id) => + //util.trace("inlining", bvar.id) + val vas = vargs.map(rewrite) + val bas = bargs.map(rewrite) + blockFor(bvar.id).map { + case Block.BlockLit(tparams, cparams, vparams, bparams, body) => + var b = bindValues(rewrite(body), vparams.zip(vas)) + b = bindBlocks(b, bparams.zip(bas)) + b + case b => + Stmt.App(b, targs, vargs, bargs) + }.getOrElse(Stmt.App(bvar, targs, vas, bas)) case Stmt.Def(id, block, body) => - ctx.bind(id, block) - Stmt.Def(id, block, rewrite(body)) + given Context = ctx.bind(id, block) + Stmt.Def(id, rewrite(block), rewrite(body)) case Stmt.Let(id, tpe, binding, body) => - ctx.bind(id, binding) - Stmt.Let(id, tpe, binding, rewrite(body)) + given Context = ctx.bind(id, binding) + Stmt.Let(id, tpe, rewrite(binding), rewrite(body)) } override def expr(using Context): PartialFunction[Expr, Expr] = { - case v @ Expr.ValueVar(id, tpe) if shouldInline(id) => + case v@Expr.ValueVar(id, tpe) if shouldInline(id) => val e = exprFor(id) - util.trace("inlining", id) + //util.trace("inlining", id) e match { case Some(p: Expr.Make) => p case Some(p: Expr.Literal) => p @@ -75,4 +96,12 @@ class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Cont case _ => v } } + + override def toplevel(using ctx: Context): PartialFunction[Toplevel, Toplevel] = { + case Toplevel.Def(id, block) => + given Context = ctx.bind(id, block) + Toplevel.Def(id, rewrite(block)) + case Toplevel.Val(id, tpe, binding) => + Toplevel.Val(id, tpe, rewrite(binding)) + } } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 426ff0d79..8fc78e7b1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -82,6 +82,7 @@ object semantics { case Literal(value: Any, annotatedType: ValueType) case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) + // TODO use dynamic captures case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) val free: Variables = this match { @@ -831,6 +832,7 @@ class NewNormalizer { case Boxed(tpe, capt) => (tpe, capt) case _ => sys error "should not happen" } + // TODO translate static capture set capt to a dynamic capture set (e.g. {exc} -> {@p_17}) val unboxAddr = scope.unbox(addr, tpe, capt) Computation.Var(unboxAddr) } @@ -890,7 +892,7 @@ class NewNormalizer { }; counter = counter + 1; println(p.dereference) - */ + */ // should capture `counter` but does not since the stack is Stack.Unknown // (effekt.JavaScriptTests.examples/pos/capture/borrows.effekt (js)) // TLDR we need to pass an escaping stack to do a proper escape analysis. Stack.Unkown is insufficient diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 1aa11e7bc..476d816a0 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -40,10 +40,10 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { */ // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - val usages = Reachable(Set(mainSymbol), tree) - util.trace(usages) - tree = Inliner(Unique(usages)).run(tree) - util.trace(util.show(tree)) + //util.trace(util.show(tree)) + tree = Inliner(Unique(Reachable(Set(mainSymbol), tree))).run(tree) + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } + //util.trace(util.show(tree)) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } From e1926043ef2f92611f087f49bc27e29ddcf1acd8 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:44:10 +0100 Subject: [PATCH 091/123] always inline jumps --- .../scala/effekt/core/optimizer/Inliner.scala | 30 ++++++++++++------- .../effekt/core/optimizer/Optimizer.scala | 9 +++--- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index c4277dec8..b265dad6e 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -14,6 +14,20 @@ class Unique(usage: Map[Id, Usage]) extends InliningPolicy { ctx.blocks.get(id).exists(_.size <= ctx.maxInlineSize) } +class UniqueJumpSimple(usage: Map[Id, Usage]) extends InliningPolicy { + override def apply(id: Id)(using ctx: Context): Boolean = { + val use = usage.get(id) + val block = ctx.blocks.get(id) + var doInline = !usage.get(id).contains(Usage.Recursive) + doInline &&= use.contains(Usage.Once) || block.collect { + case effekt.core.Block.BlockLit(_, _, _, _, _: Stmt.Return) => true + case effekt.core.Block.BlockLit(_, _, _, _, _: Stmt.App) => true + }.isDefined + doInline &&= block.exists(_.size <= ctx.maxInlineSize) + doInline + } +} + case class Context( blocks: Map[Id, Block], exprs: Map[Id, Expr], @@ -47,21 +61,17 @@ class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Cont Stmt.Def(bparam.id, barg, body) def bindBlocks(body: Stmt, blocks: List[(BlockParam, Block)]): Stmt = - blocks.headOption.map { (bp, barg) => - blocks.tail.foldRight(bindBlock(body, bp, barg)) { case ((bp, barg), acc) => - bindBlock(acc, bp, barg) - } - }.getOrElse(body) + blocks.foldRight(body) { case ((bp, barg), acc) => + bindBlock(acc, bp, barg) + } def bindValue(body: Stmt, vparam: ValueParam, varg: Expr): Stmt = Stmt.Let(vparam.id, vparam.tpe, varg, body) def bindValues(body: Stmt, values: List[(ValueParam, Expr)]): Stmt = - values.headOption.map { (vp, varg) => - values.tail.foldRight(Stmt.Let(vp.id, vp.tpe, varg, body)) { case ((vp, varg), acc) => - bindValue(acc, vp, varg) - } - }.getOrElse(body) + values.foldRight(body) { case ((vp, varg), acc) => + bindValue(acc, vp, varg) + } override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { case app @ Stmt.App(bvar: BlockVar, targs, vargs, bargs) if shouldInline(bvar.id) => diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 476d816a0..abc737cec 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -29,7 +29,7 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { } if !Context.config.optimize() then return tree; - + /* def inlineSmall(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) || (!usage.get(id).contains(Recursive) && b.size < 40) @@ -40,10 +40,11 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { */ // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - //util.trace(util.show(tree)) - tree = Inliner(Unique(Reachable(Set(mainSymbol), tree))).run(tree) + util.trace(util.show(tree)) + tree = Inliner(UniqueJumpSimple(Reachable(Set(mainSymbol), tree))).run(tree) + util.trace(util.show(tree)) tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - //util.trace(util.show(tree)) + util.trace(util.show(tree)) tree = StaticArguments.transform(mainSymbol, tree) // println(util.show(tree)) // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } From f1468980d9443e17732f3e1fbf182368e0dac203 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 12 Nov 2025 10:27:31 +0100 Subject: [PATCH 092/123] fix renaming when inlining --- .../scala/effekt/core/optimizer/Inliner.scala | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index b265dad6e..391a1d961 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -3,27 +3,30 @@ package effekt.core.optimizer import effekt.core.* import effekt.core +import scala.collection.mutable + sealed trait InliningPolicy { def apply(id: Id)(using Context): Boolean } -class Unique(usage: Map[Id, Usage]) extends InliningPolicy { +class Unique(maxInlineSize: Int) extends InliningPolicy { override def apply(id: Id)(using ctx: Context): Boolean = - usage.get(id).contains(Usage.Once) && - !usage.get(id).contains(Usage.Recursive) && - ctx.blocks.get(id).exists(_.size <= ctx.maxInlineSize) + ctx.usages.get(id).contains(Usage.Once) && + !ctx.usages.get(id).contains(Usage.Recursive) && + ctx.blocks.get(id).exists(_.size <= maxInlineSize) } -class UniqueJumpSimple(usage: Map[Id, Usage]) extends InliningPolicy { +class UniqueJumpSimple(maxInlineSize: Int) extends InliningPolicy { override def apply(id: Id)(using ctx: Context): Boolean = { - val use = usage.get(id) + val use = ctx.usages.get(id) val block = ctx.blocks.get(id) - var doInline = !usage.get(id).contains(Usage.Recursive) + var doInline = !ctx.usages.get(id).contains(Usage.Recursive) doInline &&= use.contains(Usage.Once) || block.collect { - case effekt.core.Block.BlockLit(_, _, _, _, _: Stmt.Return) => true - case effekt.core.Block.BlockLit(_, _, _, _, _: Stmt.App) => true - }.isDefined - doInline &&= block.exists(_.size <= ctx.maxInlineSize) + case Block.BlockLit(_, _, _, _, _: Stmt.Return) => true + case Block.BlockLit(_, _, _, _, _: Stmt.App) => true + case Block.BlockVar(_, _, _) => true + }.getOrElse(false) + doInline &&= block.exists(_.size <= maxInlineSize) doInline } } @@ -31,20 +34,20 @@ class UniqueJumpSimple(usage: Map[Id, Usage]) extends InliningPolicy { case class Context( blocks: Map[Id, Block], exprs: Map[Id, Expr], - maxInlineSize: Int, + usages: mutable.Map[Id, Usage] ) { def bind(id: Id, expr: Expr): Context = copy(exprs = exprs + (id -> expr)) def bind(id: Id, block: Block): Context = copy(blocks = blocks + (id -> block)) } object Context { - def empty: Context = Context(Map.empty, Map.empty, 50) + def empty(usages: Map[Id, Usage]): Context = Context(Map.empty, Map.empty, mutable.Map.from(usages)) } -class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Context] { +class Inliner(shouldInline: InliningPolicy, usages: Map[Id, Usage]) extends Tree.RewriteWithContext[Context] { def run(mod: ModuleDecl): ModuleDecl = { - given Context = Context.empty + given Context = Context.empty(usages) mod match { case ModuleDecl(path, includes, declarations, externs, definitions, exports) => ModuleDecl(path, includes, declarations, externs, definitions.map(rewrite), exports) @@ -73,25 +76,47 @@ class Inliner(shouldInline: InliningPolicy) extends Tree.RewriteWithContext[Cont bindValue(acc, vp, varg) } + def inlineApp(b: Block.BlockLit, targs: List[ValueType], vargs: List[Expr], bargs: List[Block])(using ctx: Context): Stmt = { + // (1) Rename definition's blocklit to keep IDs globally unique after inlining + val (renamedBlock @ Block.BlockLit(tparams, cparams, vparams, bparams, body), renamedIds) = Renamer.rename(b) + // (2) Copy usage information for renamed IDs + renamedIds.foreach { (from, to) => + ctx.usages.get(from).foreach { info => ctx.usages.update(to, info) } + } + // (3) We only need to bind block arguments that are _not_ block variables. Thus, separate block var args from the rest + val (bvars, other) = bparams.zip(bargs).partition { + case (_, _: Block.BlockVar) => true + case _ => false + } + // (4) Substitute. Only substitute block var args, other block args are bound before the inlinee's body + val substBody = substitutions.substitute(renamedBlock.body)(using substitutions.Substitution( + (tparams zip targs).toMap, + (cparams zip bargs.map(_.capt)).toMap, + (vparams.map(_.id) zip vargs).toMap, + bvars.map { (bp, bv) => bp.id -> bv }.toMap + )) + // (5) Bind all block arguments that are not block variables + bindBlocks(substBody, other) + } + override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { case app @ Stmt.App(bvar: BlockVar, targs, vargs, bargs) if shouldInline(bvar.id) => - //util.trace("inlining", bvar.id) val vas = vargs.map(rewrite) val bas = bargs.map(rewrite) blockFor(bvar.id).map { - case Block.BlockLit(tparams, cparams, vparams, bparams, body) => - var b = bindValues(rewrite(body), vparams.zip(vas)) - b = bindBlocks(b, bparams.zip(bas)) - b + case block: Block.BlockLit => + inlineApp(block, targs, vargs, bargs) case b => Stmt.App(b, targs, vargs, bargs) }.getOrElse(Stmt.App(bvar, targs, vas, bas)) case Stmt.Def(id, block, body) => - given Context = ctx.bind(id, block) - Stmt.Def(id, rewrite(block), rewrite(body)) + val b = rewrite(block)(using ctx) + given Context = ctx.bind(id, b) + Stmt.Def(id, b, rewrite(body)) case Stmt.Let(id, tpe, binding, body) => - given Context = ctx.bind(id, binding) - Stmt.Let(id, tpe, rewrite(binding), rewrite(body)) + val expr = rewrite(binding)(using ctx) + given Context = ctx.bind(id, expr) + Stmt.Let(id, tpe, expr, rewrite(body)) } override def expr(using Context): PartialFunction[Expr, Expr] = { From d22117ea43c5fccf2555d94600381cd9ed6ae228 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 12 Nov 2025 10:28:13 +0100 Subject: [PATCH 093/123] integrate inliner into optimization pipeline --- .../effekt/core/optimizer/Optimizer.scala | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index abc737cec..2103c50cf 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -39,15 +39,19 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } */ // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - util.trace(util.show(tree)) - tree = Inliner(UniqueJumpSimple(Reachable(Set(mainSymbol), tree))).run(tree) - util.trace(util.show(tree)) - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - util.trace(util.show(tree)) tree = StaticArguments.transform(mainSymbol, tree) - // println(util.show(tree)) - // tree = Context.timed("new-normalizer-2", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } + //util.trace(util.show(tree)) + val inliningPolicy = UniqueJumpSimple( + maxInlineSize = 150 + ) + val reachability = Reachable(Set(mainSymbol), tree) + tree = Inliner(inliningPolicy, reachability).run(tree) + //tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } + //util.trace(util.show(tree)) + tree = Deadcode.remove(mainSymbol, tree) + //util.trace(util.show(tree)) + // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From ed0b2605a7d59cacad31c8ddd274d380a22d3e2b Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 12 Nov 2025 10:40:31 +0100 Subject: [PATCH 094/123] first step in separating free variable computation from dynamic capture computation --- .../effekt/core/optimizer/NewNormalizer.scala | 78 +++++++++++++++---- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 8fc78e7b1..5fdaebf64 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -85,11 +85,14 @@ object semantics { // TODO use dynamic captures case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) + val dynamicCapture: Variables = Set.empty + val free: Variables = this match { - case Value.Var(id, annotatedType) => Set.empty + case Value.Var(id, annotatedType) => Set(id) case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Make(data, tag, targs, vargs) => vargs.toSet + // Box abstracts over all free computation variables, only when unboxing, they occur free again case Value.Box(body, tpe) => body.free } } @@ -109,10 +112,22 @@ object semantics { case Binding.Def(block) => block.free case Binding.Rec(block, tpe, capt) => block.free case Binding.Val(stmt) => stmt.free - case Binding.Run(f, targs, vargs, bargs) => vargs.toSet ++ all(bargs, _.free) + // TODO block args for externs are not supported (for now?) + case Binding.Run(f, targs, vargs, bargs) => vargs.toSet case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set(addr) case Binding.Get(ref, tpe, cap) => Set(ref) } + + val dynamicCapture: Variables = this match { + case Binding.Let(value) => value.dynamicCapture + case Binding.Def(block) => block.dynamicCapture + case Binding.Rec(block, tpe, capt) => block.dynamicCapture + case Binding.Val(stmt) => stmt.dynamicCapture + // TODO block args for externs are not supported (for now?) + case Binding.Run(f, targs, vargs, bargs) => Set.empty + case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set.empty // TODO + case Binding.Get(ref, tpe, cap) => Set(ref) + } } type Bindings = List[(Id, Binding)] @@ -270,6 +285,7 @@ object semantics { case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) + val dynamicCapture: Variables = body.dynamicCapture -- bparams.map(_.id) } case class BasicBlock(bindings: Bindings, body: NeutralStmt) { @@ -286,6 +302,17 @@ object semantics { } free } + + val dynamicCapture: Variables = { + bindings.foldLeft(body.dynamicCapture) { (captures, b) => + b match { + case (id, b: Binding.Def) => (captures - id) ++ b.dynamicCapture + case (id, b: Binding.Rec) => (captures - id) ++ (b.dynamicCapture - id) + case (id, b) => captures ++ b.dynamicCapture + } + } + body.dynamicCapture ++ bindings.flatMap(_._2.dynamicCapture) + } } enum Computation { @@ -296,25 +323,28 @@ object semantics { case Continuation(k: Cont) - // TODO ? distinguish? - //case Region(prompt: Id) ??? - //case Prompt(prompt: Id) ??? - //case Reference(prompt: Id) ??? - // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) - lazy val free: Variables = this match { + val free: Variables = this match { case Computation.Var(id) => Set(id) case Computation.Def(closure) => closure.free case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet } + + val dynamicCapture: Variables = this match { + case Computation.Var(id) => Set(id) + case Computation.Def(closure) => closure.dynamicCapture + case Computation.Continuation(k) => Set.empty // TODO ??? + case Computation.New(interface, operations) => operations.flatMap(_._2.dynamicCapture).toSet + } } // TODO add escaping mutable variables case class Closure(label: Label, environment: List[Computation.Var]) { val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet + val dynamicCapture: Variables = environment.map(_.id).toSet } // Statements @@ -365,6 +395,24 @@ object semantics { case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id.id case NeutralStmt.Hole(span) => Set.empty } + + val dynamicCapture: Variables = this match { + case NeutralStmt.Return(result) => Set.empty + case NeutralStmt.Hole(span) => Set.empty + + case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.If(cond, thn, els) => thn.dynamicCapture ++ els.dynamicCapture + case NeutralStmt.Match(scrutinee, clauses, default) => clauses.flatMap(_._2.dynamicCapture).toSet ++ default.map(_.dynamicCapture).getOrElse(Set.empty) + case NeutralStmt.Reset(prompt, body) => body.dynamicCapture - prompt.id + case NeutralStmt.Shift(prompt, capt, k, body) => (body.dynamicCapture - k.id) + prompt + case NeutralStmt.Resume(k, body) => Set(k) ++ body.dynamicCapture + case NeutralStmt.Var(id, init, body) => body.dynamicCapture - id.id + case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref) ++ body.dynamicCapture + case NeutralStmt.Region(id, body) => body.dynamicCapture - id.id + case NeutralStmt.Alloc(id, init, region, body) => Set(region) ++ body.dynamicCapture - id.id + } } // Stacks @@ -526,7 +574,7 @@ object semantics { k case body => val k = Id("k") - val closureParams = escaping.bound.collect { case p if body.free contains p.id => p } + val closureParams = escaping.bound.collect { case p if body.dynamicCapture contains p.id => p } scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) } @@ -595,20 +643,20 @@ object semantics { case Stack.Reset(prompt, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - if (body.free contains prompt.id) NeutralStmt.Reset(prompt, body) + if (body.dynamicCapture contains prompt.id) NeutralStmt.Reset(prompt, body) else stmt // TODO this runs normalization a second time in the outer scope! }} case Stack.Var(id, curr, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - if (body.free contains id.id) NeutralStmt.Var(id, curr, body) + if (body.dynamicCapture contains id.id) NeutralStmt.Var(id, curr, body) else stmt }} case Stack.Region(id, bindings, frame, next) => reify(next) { reify(frame) { val body = nested { stmt } - val bodyUsesBinding = body.free.exists(bindings.map { b => b._1.id }.toSet.contains(_)) - if (body.free.contains(id.id) || bodyUsesBinding) { + val bodyUsesBinding = body.dynamicCapture.exists(bindings.map { b => b._1.id }.toSet.contains(_)) + if (body.dynamicCapture.contains(id.id) || bodyUsesBinding) { // we need to reify all bindings in this region as allocs using their current value val reifiedAllocs = bindings.foldLeft(body) { case (acc, (bp, addr)) => nested { NeutralStmt.Alloc(bp, addr, id.id, acc) } @@ -770,7 +818,7 @@ class NewNormalizer { }) } - val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } + val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } // Only normalize again if we actually we wrong in our assumption that we capture nothing // We might run into exponential complexity for nested recursive functions @@ -817,7 +865,7 @@ class NewNormalizer { evaluate(body, Frame.Return, Stack.Unknown) }) - val closureParams = escaping.bound.filter { p => normalizedBlock.free contains p.id } + val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } val f = Id(hint) scope.define(f, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams)) From 9f25839848930533762bbfa7d79927b678a351fe Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:45:33 +0100 Subject: [PATCH 095/123] fix resume function --- .../effekt/core/optimizer/NewNormalizer.scala | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 5fdaebf64..fa6fa631b 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -4,9 +4,7 @@ package optimizer import effekt.core.ValueType.Boxed import effekt.source.Span -import effekt.core.optimizer.semantics.{Computation, NeutralStmt, Value} import effekt.symbols.builtins -import effekt.util.messages.{ErrorReporter, INTERNAL_ERROR} import effekt.symbols.builtins.AsyncCapability import kiama.output.ParenPrettyPrinter @@ -119,7 +117,7 @@ object semantics { } val dynamicCapture: Variables = this match { - case Binding.Let(value) => value.dynamicCapture + case Binding.Let(value) => Set.empty case Binding.Def(block) => block.dynamicCapture case Binding.Rec(block, tpe, capt) => block.dynamicCapture case Binding.Val(stmt) => stmt.dynamicCapture @@ -548,16 +546,16 @@ object semantics { } def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { - case Cont.Empty => (k, ks) + case Cont.Empty => + (k, ks) case Cont.Reset(frame, prompt, rest) => - val (k1, ks1) = resume(rest, frame, ks) - val stack = Stack.Reset(prompt, k1, ks1) - (frame, stack) + val (k1, ks1) = resume(rest, k, ks) + (frame, Stack.Reset(prompt, k1, ks1)) case Cont.Var(frame, id, curr, rest) => - val (k1, ks1) = resume(rest, frame, ks) + val (k1, ks1) = resume(rest, k, ks) (frame, Stack.Var(id, curr, k1, ks1)) case Cont.Region(frame, id, bindings, rest) => - val (k1, ks1) = resume(rest, frame, ks) + val (k1, ks1) = resume(rest, k, ks) (frame, Stack.Region(id, bindings, k1, ks1)) } @@ -886,6 +884,7 @@ class NewNormalizer { } } + // TODO this does not work for recursive objects currently case core.Block.New(Implementation(interface, operations)) => val ops = operations.map { case Operation(name, tparams, cparams, vparams, bparams, body) => @@ -1135,19 +1134,6 @@ class NewNormalizer { reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } } case Stmt.Shift(_, _) => ??? - //case Stmt.Reset(BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => - // // TODO is Var correct here?? Probably needs to be a new computation value... - // // but shouldn't it be a fresh prompt each time? - // val p = Id(prompt.id) - // val neutralBody = { - // given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) - // nested { - // evaluate(body, MetaStack.Empty) - // } - // } - // // TODO implement properly - // k.reify(NeutralStmt.Reset(BlockParam(p, prompt.tpe, prompt.capt), neutralBody)) - case Stmt.Reset(core.Block.BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => val p = Id(prompt.id) From 990f6a9d8193e1e4b0a0a2b60df191e844013595 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:46:06 +0100 Subject: [PATCH 096/123] use dynamic capture for deciding whether to use reifyKnown --- .../main/scala/effekt/core/optimizer/NewNormalizer.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index fa6fa631b..122675cac 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1001,9 +1001,9 @@ class NewNormalizer { */ val blockargs = bargs.map(evaluate(_, "f", ks)) // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation - // TODO examples/pos/lambdas/localstate.effekt fails if we only check stmt.capt - // TODO capture {io, global} is also fine - if (stmt.capt.isEmpty && environment.isEmpty) { + // compute dynamic captures of the whole statement (App node) + val dynCaptures = blockargs.flatMap(_.dynamicCapture) ++ environment + if (dynCaptures.isEmpty) { reifyKnown(k, ks) { NeutralStmt.Jump(label, targs, args, blockargs) } From d5aa04fd92100cee2b9205557ef40c17642af56b Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:47:33 +0100 Subject: [PATCH 097/123] run Normalizer, Inliner, Normalizer --- .../src/main/scala/effekt/core/optimizer/Optimizer.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 2103c50cf..5111dd7e0 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -40,17 +40,19 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { */ // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } tree = StaticArguments.transform(mainSymbol, tree) + //util.trace(util.show(tree)) tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } //util.trace(util.show(tree)) val inliningPolicy = UniqueJumpSimple( maxInlineSize = 150 ) val reachability = Reachable(Set(mainSymbol), tree) + //util.trace(util.show(tree)) tree = Inliner(inliningPolicy, reachability).run(tree) - //tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } + tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } //util.trace(util.show(tree)) tree = Deadcode.remove(mainSymbol, tree) - //util.trace(util.show(tree)) + util.trace(util.show(tree)) // Normalizer.assertNormal(tree) //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) From f78d52804019c8a67fad0b073bf290b30d87da1b Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Tue, 18 Nov 2025 10:05:23 +0100 Subject: [PATCH 098/123] use old optimization pipeline with new-normalizer --- .../effekt/core/optimizer/Optimizer.scala | 64 ++++--------------- 1 file changed, 13 insertions(+), 51 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 5111dd7e0..66c38d071 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -30,61 +30,23 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - /* - def inlineSmall(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => - usage.get(id).contains(Once) || (!usage.get(id).contains(Recursive) && b.size < 40) - } - val dontInline = NewNormalizer { (id, b) => false } - def inlineUnique(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => usage.get(id).contains(Once) } - def inlineAll(usage: Map[Id, Usage]) = NewNormalizer { (id, b) => !usage.get(id).contains(Recursive) } - */ - // tree = Context.timed("new-normalizer-1", source.name) { inlineSmall(Reachable(Set(mainSymbol), tree)).run(tree) } + // (2) lift static arguments tree = StaticArguments.transform(mainSymbol, tree) - //util.trace(util.show(tree)) - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - //util.trace(util.show(tree)) + val inliningPolicy = UniqueJumpSimple( maxInlineSize = 150 ) - val reachability = Reachable(Set(mainSymbol), tree) - //util.trace(util.show(tree)) - tree = Inliner(inliningPolicy, reachability).run(tree) - tree = Context.timed("new-normalizer-1", source.name) { NewNormalizer().run(tree) } - //util.trace(util.show(tree)) - tree = Deadcode.remove(mainSymbol, tree) - util.trace(util.show(tree)) - - // Normalizer.assertNormal(tree) - //tree = Normalizer.normalize(Set(mainSymbol), tree, Context.config.maxInlineSize().toInt) - - // tree = Context.timed("old-normalizer-1", source.name) { Normalizer.normalize(Set(mainSymbol), tree, 0) } - // tree = Context.timed("old-normalizer-2", source.name) { Normalizer.normalize(Set(mainSymbol), tree, 0) } - // - // tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } - // Normalizer.assertNormal(tree) - - // (2) lift static arguments - // tree = Context.timed("static-argument-transformation", source.name) { - // StaticArguments.transform(mainSymbol, tree) - // } - // - // tree = Context.timed("new-normalizer-3", source.name) { NewNormalizer.run(tree) } - // Normalizer.assertNormal(tree) - // - // def normalize(m: ModuleDecl) = { - // val anfed = BindSubexpressions.transform(m) - // val normalized = Normalizer.normalize(Set(mainSymbol), anfed, Context.config.maxInlineSize().toInt) - // Normalizer.assertNormal(normalized) - // val live = Deadcode.remove(mainSymbol, normalized) - // val tailRemoved = RemoveTailResumptions(live) - // val contified = DirectStyle.rewrite(tailRemoved) - // contified - // } - // - // // (3) normalize a few times (since tail resumptions might only surface after normalization and leave dead Resets) - // tree = Context.timed("normalize-1", source.name) { normalize(tree) } - // tree = Context.timed("normalize-2", source.name) { normalize(tree) } - // tree = Context.timed("normalize-3", source.name) { normalize(tree) } + def normalize(m: ModuleDecl) = { + val anfed = BindSubexpressions.transform(m) + val normalized = NewNormalizer().run(anfed) + val inlined = Inliner(inliningPolicy, Reachable(Set(mainSymbol), normalized)).run(normalized) + val live = Deadcode.remove(mainSymbol, inlined) + val tailRemoved = RemoveTailResumptions(live) + val contified = DirectStyle.rewrite(tailRemoved) + contified + } + tree = Context.timed("new-normalizer-1", source.name) { normalize(tree) } + tree = Context.timed("new-normalizer-2", source.name) { normalize(tree) } tree } From e5458db6f0bc2d3ac403ed0ac9580f253b1dc371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Brachtha=CC=88user?= Date: Wed, 19 Nov 2025 13:47:31 +0100 Subject: [PATCH 099/123] Implement something like dependent specialization for repeated matches on the same scrutinee --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 122675cac..4379420a5 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1072,7 +1072,15 @@ class NewNormalizer { // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = Block(tparams, vparams, bparams, nested { + val block = Block(tparams, vparams, bparams, nested { scope ?=> + // here we now know that our scrutinee sc has the shape id(vparams, ...) + + val datatype = scrutinee.tpe match { + case tpe @ ValueType.Data(name, targs) => tpe + case tpe => sys error s"Should not happen: pattern matching on a non-datatype: ${tpe}" + } + val eta = Value.Make(datatype, id, tparams.map(t => ValueType.Var(t)), vparams.map(p => p.id)) + scope.bindings = scope.bindings.updated(sc, Binding.Let(eta)) evaluate(body, k, ks) }) (id, block) From 4f83aeb8c411b48e811694e24a536862ad07cc8d Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:02:54 +0100 Subject: [PATCH 100/123] bind variables so that they can be used as bargs --- .../scala/effekt/core/optimizer/NewNormalizer.scala | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 4379420a5..510494966 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -302,13 +302,6 @@ object semantics { } val dynamicCapture: Variables = { - bindings.foldLeft(body.dynamicCapture) { (captures, b) => - b match { - case (id, b: Binding.Def) => (captures - id) ++ b.dynamicCapture - case (id, b: Binding.Rec) => (captures - id) ++ (b.dynamicCapture - id) - case (id, b) => captures ++ b.dynamicCapture - } - } body.dynamicCapture ++ bindings.flatMap(_._2.dynamicCapture) } } @@ -1106,7 +1099,10 @@ class NewNormalizer { case Stmt.Var(ref, init, capture, body) => val addr = evaluate(init, ks) - evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) + // TODO Var means unknown. Here we have contrary: it is a statically (!) known variable + bind(ref, Computation.Var(ref)) { + evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) + } case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => get(ref, ks) match { case Some(addr) => bind(id, addr) { evaluate(body, k, ks) } From b2e567ae7fae521b6f4917cf430c03cbf6e811fe Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:03:27 +0100 Subject: [PATCH 101/123] fix pretty printing of Def with ImpureApp --- effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala index 1d7fb3c85..25e93d241 100644 --- a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala +++ b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala @@ -193,7 +193,7 @@ object PrettyPrinter extends ParenPrettyPrinter { def toDoc(s: Stmt): Doc = s match { // requires a block to be readable: - case _ : (Stmt.Def | Stmt.Let | Stmt.Val | Stmt.Alloc | Stmt.Var | Stmt.Get | Stmt.Put) => block(toDocStmts(s)) + case _ : (Stmt.Def | Stmt.Let | Stmt.ImpureApp | Stmt.Val | Stmt.Alloc | Stmt.Var | Stmt.Get | Stmt.Put) => block(toDocStmts(s)) case other => toDocStmts(s) } From f5ffb0ade8f7237ee21fab9a8c46d8c24d5e1c17 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:04:15 +0100 Subject: [PATCH 102/123] fix inlining of toplevel functions --- .../main/scala/effekt/core/optimizer/Inliner.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index 391a1d961..2eadb35ad 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -47,10 +47,17 @@ object Context { class Inliner(shouldInline: InliningPolicy, usages: Map[Id, Usage]) extends Tree.RewriteWithContext[Context] { def run(mod: ModuleDecl): ModuleDecl = { - given Context = Context.empty(usages) mod match { case ModuleDecl(path, includes, declarations, externs, definitions, exports) => - ModuleDecl(path, includes, declarations, externs, definitions.map(rewrite), exports) + var ctx = Context.empty(usages) + val d = definitions.map { + case Toplevel.Def(id, block) => + val b = rewrite(block)(using ctx) + ctx = ctx.bind(id, b) + Toplevel.Def(id, b) + case v@Toplevel.Val(id, tpe, binding) => v + } + ModuleDecl(path, includes, declarations, externs, d, exports) } } From 3b60b4c18c81d3c5e599fb0c306697e172769450 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Wed, 19 Nov 2025 16:04:37 +0100 Subject: [PATCH 103/123] try to inline method calls --- .../scala/effekt/core/optimizer/Inliner.scala | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index 2eadb35ad..b4c157ac5 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -107,15 +107,24 @@ class Inliner(shouldInline: InliningPolicy, usages: Map[Id, Usage]) extends Tree } override def stmt(using ctx: Context): PartialFunction[Stmt, Stmt] = { - case app @ Stmt.App(bvar: BlockVar, targs, vargs, bargs) if shouldInline(bvar.id) => + case app @ Stmt.App(v: BlockVar, targs, vargs, bargs) if shouldInline(v.id) => val vas = vargs.map(rewrite) val bas = bargs.map(rewrite) - blockFor(bvar.id).map { + blockFor(v.id).map { case block: Block.BlockLit => inlineApp(block, targs, vargs, bargs) case b => Stmt.App(b, targs, vargs, bargs) - }.getOrElse(Stmt.App(bvar, targs, vas, bas)) + }.getOrElse(Stmt.App(v, targs, vas, bas)) + case Stmt.Invoke(v: BlockVar, method, methodTpe, targs, vargs, bargs) if shouldInline(v.id) => + val vas = vargs.map(rewrite) + val bas = bargs.map(rewrite) + blockFor(v.id).collect { + case b@Block.New(Implementation(interface, operations)) => + val op = operations.find { op => op.name == method }.get + val b: Block.BlockLit = Block.BlockLit(op.tparams, op.cparams, op.vparams, op.bparams, op.body) + inlineApp(b, targs, vas, bas) + }.getOrElse(Stmt.Invoke(v, method, methodTpe, targs, vas, bas)) case Stmt.Def(id, block, body) => val b = rewrite(block)(using ctx) given Context = ctx.bind(id, b) From f162c04d082e42db04fea57a843234943260945f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Wed, 19 Nov 2025 17:22:29 +0100 Subject: [PATCH 104/123] Implement initial support for builtins evaluation --- .../effekt/core/NewNormalizerTests.scala | 2 +- .../effekt/core/PolymorphismBoxing.scala | 11 ++- .../main/scala/effekt/core/Transformer.scala | 22 +++++- .../src/main/scala/effekt/core/Tree.scala | 15 +++- .../effekt/core/optimizer/NewNormalizer.scala | 75 ++++++++++++++++--- .../src/main/scala/effekt/core/vm/VM.scala | 6 +- .../main/scala/effekt/cps/Transformer.scala | 4 +- .../effekt/generator/chez/Transformer.scala | 2 +- .../scala/effekt/machine/Transformer.scala | 2 +- .../effekt/source/ResolveExternDefs.scala | 4 +- 10 files changed, 120 insertions(+), 23 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 9799f5001..542cf4194 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -59,7 +59,7 @@ class NewNormalizerTests extends CoreTests { )) def findExternDef(mod: ModuleDecl, name: String) = - mod.externs.collect { case d@Extern.Def(_, _, _, _, _, _, _, _) => d } + mod.externs.collect { case d: Extern.Def => d } .find(_.id.name.name == name) .getOrElse(throw new NoSuchElementException(s"Extern def '$name' not found")) diff --git a/effekt/shared/src/main/scala/effekt/core/PolymorphismBoxing.scala b/effekt/shared/src/main/scala/effekt/core/PolymorphismBoxing.scala index 30febd6b2..bbc851a65 100644 --- a/effekt/shared/src/main/scala/effekt/core/PolymorphismBoxing.scala +++ b/effekt/shared/src/main/scala/effekt/core/PolymorphismBoxing.scala @@ -86,12 +86,17 @@ object PolymorphismBoxing extends Phase[CoreTransformed, CoreTransformed] { } def transform(extern: Extern)(using Context, DeclarationContext): Extern = extern match { - case Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, body) => + case Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, targetBody, vmBody) => Extern.Def(id, tparams, cparams, vparams map transform, bparams map transform, transform(ret), - annotatedCapture, body match { + annotatedCapture, targetBody match { case ExternBody.StringExternBody(ff, bbody) => ExternBody.StringExternBody(ff, Template(bbody.strings, bbody.args map transform)) case e @ ExternBody.Unsupported(_) => e - } ) + }, + vmBody match { + case Some(ExternBody.StringExternBody(ff, bbody)) => Some(ExternBody.StringExternBody(ff, Template(bbody.strings, bbody.args map transform))) + case other => other + } + ) case Extern.Include(ff, contents) => Extern.Include(ff, contents) } diff --git a/effekt/shared/src/main/scala/effekt/core/Transformer.scala b/effekt/shared/src/main/scala/effekt/core/Transformer.scala index f56064b43..d76aa62eb 100644 --- a/effekt/shared/src/main/scala/effekt/core/Transformer.scala +++ b/effekt/shared/src/main/scala/effekt/core/Transformer.scala @@ -108,15 +108,33 @@ object Transformer extends Phase[Typechecked, CoreTransformed] { val sym@ExternFunction(name, tps, _, _, ret, effects, capt, _, _) = f.symbol assert(effects.isEmpty) val cps = bps.map(b => b.symbol.capture) - val tBody = bodies match { + val vmBodyIdx = bodies.indexWhere(_.featureFlag.matches("vm", matchDefault = false)) + val vmBody = bodies.lift(vmBodyIdx) match { + case Some(source.ExternBody.StringExternBody(ff, body, span)) => + Some(ExternBody.StringExternBody(ff, Template(body.strings, body.args.map(transformAsExpr)))) + case _ => None + } + val otherBodies = if (vmBodyIdx < 0) bodies else bodies.patch(vmBodyIdx, Nil, 1) + val targetBody = otherBodies match { case source.ExternBody.StringExternBody(ff, body, span) :: Nil => ExternBody.StringExternBody(ff, Template(body.strings, body.args.map(transformAsExpr))) case source.ExternBody.Unsupported(err) :: Nil => ExternBody.Unsupported(err) + case Nil => vmBody.getOrElse(Context.abort("Externs should be resolved and desugared before core.Transformer")) case _ => Context.abort("Externs should be resolved and desugared before core.Transformer") } - List(Extern.Def(sym, tps, cps.unspan, vps.unspan map transform, bps.unspan map transform, transform(ret), transform(capt), tBody)) + List(Extern.Def( + sym, + tps, + cps.unspan, + vps.unspan map transform, + bps.unspan map transform, + transform(ret), + transform(capt), + targetBody, + vmBody, + )) case e @ source.ExternInclude(ff, path, contents, _, doc, span) => List(Extern.Include(ff, contents.get)) diff --git a/effekt/shared/src/main/scala/effekt/core/Tree.scala b/effekt/shared/src/main/scala/effekt/core/Tree.scala index ca0f8462a..6175312d2 100644 --- a/effekt/shared/src/main/scala/effekt/core/Tree.scala +++ b/effekt/shared/src/main/scala/effekt/core/Tree.scala @@ -1,6 +1,7 @@ package effekt package core +import effekt.core.ExternBody.StringExternBody import effekt.source.FeatureFlag import effekt.util.Structural import effekt.util.messages.INTERNAL_ERROR @@ -127,7 +128,19 @@ case class Property(id: Id, tpe: BlockType) extends Tree * FFI external definitions */ enum Extern extends Tree { - case Def(id: Id, tparams: List[Id], cparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], ret: ValueType, annotatedCapture: Captures, body: ExternBody) + case Def( + id: Id, + tparams: List[Id], + cparams: List[Id], + vparams: List[ValueParam], + bparams: List[BlockParam], + ret: ValueType, + annotatedCapture: Captures, + /* Extern body for the chosen compilation target */ + targetBody: ExternBody, + /* Extern body for the vm target, if any */ + vmBody: Option[StringExternBody] + ) case Include(featureFlag: FeatureFlag, contents: String) } sealed trait ExternBody extends Tree diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 510494966..3da3ffa84 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -314,6 +314,8 @@ object semantics { case Continuation(k: Cont) + case BuiltinExtern(id: Id, builtinName: String) + // Known object case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) @@ -322,6 +324,7 @@ object semantics { case Computation.Def(closure) => closure.free case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet + case Computation.BuiltinExtern(id, vmSymbol) => Set.empty } val dynamicCapture: Variables = this match { @@ -329,6 +332,7 @@ object semantics { case Computation.Def(closure) => closure.dynamicCapture case Computation.Continuation(k) => Set.empty // TODO ??? case Computation.New(interface, operations) => operations.flatMap(_._2.dynamicCapture).toSet + case Computation.BuiltinExtern(id, vmSymbol) => Set.empty } } @@ -748,6 +752,7 @@ object semantics { case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { hsep(operations.map { case (id, impl) => "def" <+> toDoc(id) <+> "=" <+> toDoc(impl) }, ",") } + case Computation.BuiltinExtern(id, vmSymbol) => "extern" <+> toDoc(id) <+> "=" <+> vmSymbol } def toDoc(closure: Closure): Doc = closure match { case Closure(label, env) => toDoc(label) <+> "@" <+> brackets(hsep(env.map(toDoc), comma)) @@ -917,9 +922,23 @@ class NewNormalizer { case core.Expr.Literal(value, annotatedType) => scope.allocate("x", Value.Literal(value, annotatedType)) - // right now everything is stuck... no constant folding ... case core.Expr.PureApp(f, targs, vargs) => - scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) + val externDef = env.lookupComputation(f.id) + val vargsEvaluated = vargs.map(evaluate(_, escaping)) + val valuesOpt: Option[List[semantics.Value]] = + vargsEvaluated.foldLeft(Option(List.empty[semantics.Value])) { (acc, addr) => + for { + xs <- acc + x <- scope.lookupValue(addr) + } yield x :: xs + }.map(_.reverse) + (valuesOpt, externDef) match { + case (Some(values), Computation.BuiltinExtern(id, name)) if supportedBuiltins(name).isDefinedAt(values) => + val impl = supportedBuiltins(name) + val res = impl(values) + scope.allocate("x", res) + case _ => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) + } case core.Expr.Make(data, tag, targs, vargs) => scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate(_, escaping)))) @@ -1005,7 +1024,7 @@ class NewNormalizer { NeutralStmt.Jump(label, targs, args, blockargs ++ environment) } } - case _: (Computation.New | Computation.Continuation) => sys error "Should not happen" + case _: (Computation.New | Computation.Continuation | Computation.BuiltinExtern) => sys error "Should not happen" } // case Stmt.Invoke(New) @@ -1019,7 +1038,7 @@ class NewNormalizer { operations.collectFirst { case (id, Closure(label, environment)) if id == method => reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } }.get - case _: (Computation.Def | Computation.Continuation) => sys error s"Should not happen" + case _: (Computation.Def | Computation.Continuation | Computation.BuiltinExtern) => sys error s"Should not happen" } case Stmt.If(cond, thn, els) => @@ -1164,11 +1183,17 @@ class NewNormalizer { def run(mod: ModuleDecl): ModuleDecl = { //util.trace(mod) // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) - val asyncExterns = mod.externs.collect { case defn: Extern.Def if defn.annotatedCapture.contains(AsyncCapability.capture) => defn } - val asyncTypes = asyncExterns.map { d => + val externTypes = mod.externs.collect { case d: Extern.Def => d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map { _.tpe }, d.bparams.map { bp => bp.tpe }, d.ret), d.annotatedCapture) } + val (builtinExterns, otherExterns) = mod.externs.collect { case d: Extern.Def => d }.partition { + case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => + val builtinName = vmBody.contents.strings.head + supportedBuiltins.contains(builtinName) + case _ => false + } + val toplevelEnv = Env.empty // user-defined functions .bindComputation(mod.definitions.collect { @@ -1184,7 +1209,9 @@ class NewNormalizer { case Toplevel.Val(id, _, _) => id -> id }) // async extern functions - .bindComputation(asyncExterns.map(defn => defn.id -> Computation.Var(defn.id))) + .bindComputation(otherExterns.map(defn => defn.id -> Computation.Var(defn.id))) + // pure extern functions + .bindComputation(builtinExterns.flatMap(defn => defn.vmBody.map(vmBody => defn.id -> Computation.BuiltinExtern(defn.id, vmBody.contents.strings.head)))) val typingContext = TypingContext( mod.definitions.collect { @@ -1192,14 +1219,14 @@ class NewNormalizer { }.toMap, mod.definitions.collect { case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap ++ asyncTypes + }.toMap ++ externTypes ) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) mod.copy(definitions = newDefinitions) } - val showDebugInfo = false + val showDebugInfo = true inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { @@ -1323,6 +1350,7 @@ class NewNormalizer { case Computation.Def(closure) => etaExpandToBlockLit(closure) case Computation.Continuation(k) => ??? + case Computation.BuiltinExtern(_, _) => ??? case Computation.New(interface, operations) => val ops = operations.map { etaExpandToOperation.tupled } core.Block.New(Implementation(interface, ops)) @@ -1455,3 +1483,32 @@ class NewNormalizer { val (tpe, capt) = G.blocks.getOrElse(label, sys error s"Unknown block: ${util.show(label)}. ${G.blocks.keys.map(util.show).mkString(", ")}") core.BlockVar(label, tpe, capt) } + +type ~>[-A, +B] = PartialFunction[A, B] + +type BuiltinImpl = List[semantics.Value] ~> semantics.Value + +def builtin(name: String)(impl: List[semantics.Value] ~> semantics.Value): (String, BuiltinImpl) = name -> impl + +type Builtins = Map[String, BuiltinImpl] + +lazy val integers: Builtins = Map( + // Arithmetic + // ---------- + builtin("effekt::infixAdd(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x + y, Type.TInt) + }, +) + +lazy val supportedBuiltins: Builtins = integers + +protected object As { + object Int { + def unapply(v: semantics.Value): Option[scala.Long] = v match { + case semantics.Value.Literal(value: scala.Long, _) => Some(value) + case semantics.Value.Literal(value: scala.Int, _) => Some(value.toLong) + case semantics.Value.Literal(value: java.lang.Integer, _) => Some(value.toLong) + case _ => None + } + } +} \ No newline at end of file diff --git a/effekt/shared/src/main/scala/effekt/core/vm/VM.scala b/effekt/shared/src/main/scala/effekt/core/vm/VM.scala index 55114202f..14022519c 100644 --- a/effekt/shared/src/main/scala/effekt/core/vm/VM.scala +++ b/effekt/shared/src/main/scala/effekt/core/vm/VM.scala @@ -523,8 +523,10 @@ class Interpreter(instrumentation: Instrumentation, runtime: Runtime) { val functions = m.definitions.collect { case Toplevel.Def(id, b: Block.BlockLit) => id -> b }.toMap val builtinFunctions = m.externs.collect { - case Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, - ExternBody.StringExternBody(FeatureFlag.NamedFeatureFlag("vm", _), Template(name :: Nil, Nil))) => + case Extern.Def( + id, tps, cps, vps, bps, ret, annotatedCapture, _, + Some(ExternBody.StringExternBody(FeatureFlag.NamedFeatureFlag("vm", _), Template(name :: Nil, Nil))) + ) => id -> builtins.getOrElse(name, throw VMError.MissingBuiltin(name)) }.toMap diff --git a/effekt/shared/src/main/scala/effekt/cps/Transformer.scala b/effekt/shared/src/main/scala/effekt/cps/Transformer.scala index ccf42d954..c1a3b22c9 100644 --- a/effekt/shared/src/main/scala/effekt/cps/Transformer.scala +++ b/effekt/shared/src/main/scala/effekt/cps/Transformer.scala @@ -35,8 +35,8 @@ object Transformer { } def transform(extern: core.Extern)(using TransformationContext): Extern = extern match { - case core.Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, body) => - Extern.Def(id, vparams.map(_.id), bparams.map(_.id), annotatedCapture.contains(symbols.builtins.AsyncCapability.capture), transform(body)) + case core.Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, targetBody, vmBody) => + Extern.Def(id, vparams.map(_.id), bparams.map(_.id), annotatedCapture.contains(symbols.builtins.AsyncCapability.capture), transform(targetBody)) case core.Extern.Include(featureFlag, contents) => Extern.Include(featureFlag, contents) } diff --git a/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala b/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala index 8f9e87251..77ec1fc9d 100644 --- a/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala +++ b/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala @@ -123,7 +123,7 @@ trait Transformer { } def toChez(decl: core.Extern)(using ErrorReporter): chez.Def = decl match { - case Extern.Def(id, tpe, cps, vps, bps, ret, capt, body) => + case Extern.Def(id, tpe, cps, vps, bps, ret, capt, body, vmBody) => val tBody = body match { case ExternBody.StringExternBody(featureFlag, contents) => toChez(contents) case u: ExternBody.Unsupported => diff --git a/effekt/shared/src/main/scala/effekt/machine/Transformer.scala b/effekt/shared/src/main/scala/effekt/machine/Transformer.scala index 4e25df32f..3ef99c943 100644 --- a/effekt/shared/src/main/scala/effekt/machine/Transformer.scala +++ b/effekt/shared/src/main/scala/effekt/machine/Transformer.scala @@ -52,7 +52,7 @@ object Transformer { } def transform(extern: core.Extern)(using BlocksParamsContext, ErrorReporter): Declaration = extern match { - case core.Extern.Def(name, tps, cparams, vparams, bparams, ret, capture, body) => + case core.Extern.Def(name, tps, cparams, vparams, bparams, ret, capture, body, vmBody) => if bparams.nonEmpty then ErrorReporter.abort("Foreign functions currently cannot take block arguments.") val transformedParams = vparams.map(transform) diff --git a/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala b/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala index 1c179e56f..2ad6e6ed9 100644 --- a/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala +++ b/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala @@ -13,7 +13,9 @@ object ResolveExternDefs extends Phase[Typechecked, Typechecked] { case Typechecked(source, tree, mod) => Some(Typechecked(source, rewrite(tree), mod)) } - def supported(using Context): List[String] = Context.compiler.supportedFeatureFlags + // The list of supported feature flags for this backend + // The "vm" flag is always supported as it is used in the normalizer as part of the compiler pipeline. + def supported(using Context): List[String] = Context.compiler.supportedFeatureFlags :+ "vm" def defaultExternBody(warning: String)(using Context): ExternBody = ExternBody.Unsupported(Context.plainMessage(warning, kiama.util.Severities.Warning)) From a871e7b6a115c81fa1e3444fa315a8c12a7ff0f3 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Sat, 22 Nov 2025 11:34:27 +0100 Subject: [PATCH 105/123] optimize linear usage of cont. in match --- .../effekt/core/optimizer/NewNormalizer.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 3da3ffa84..520233be8 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -117,13 +117,13 @@ object semantics { } val dynamicCapture: Variables = this match { - case Binding.Let(value) => Set.empty + case Binding.Let(value) => value.dynamicCapture case Binding.Def(block) => block.dynamicCapture case Binding.Rec(block, tpe, capt) => block.dynamicCapture case Binding.Val(stmt) => stmt.dynamicCapture // TODO block args for externs are not supported (for now?) case Binding.Run(f, targs, vargs, bargs) => Set.empty - case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set.empty // TODO + case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => capt // TODO these are the static not dynamic captures case Binding.Get(ref, tpe, cap) => Set(ref) } } @@ -1047,6 +1047,7 @@ class NewNormalizer { case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) case Some(Value.Literal(false, _)) => evaluate(els, k, ks) case _ => + // joinpoint(k, ks, List(thn, els)) { (A, Frame, Stack) => (NeutralStmt, Variables) } { case thn1 :: els1 :: Nil => NeutralStmt.If(sc, thn1, els1) } joinpoint(k, ks) { (k, ks) => NeutralStmt.If(sc, nested { evaluate(thn, k, ks) @@ -1055,7 +1056,6 @@ class NewNormalizer { }) } } - case Stmt.Match(scrutinee, clauses, default) => val sc = evaluate(scrutinee, ks) scope.lookupValue(sc) match { @@ -1067,7 +1067,6 @@ class NewNormalizer { }.getOrElse { evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) } - // linear usage of the continuation // case _ if (clauses.size + default.size) <= 1 => // NeutralStmt.Match(sc, // clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -1079,7 +1078,7 @@ class NewNormalizer { // }, // default.map { stmt => nested { evaluate(stmt, k, ks) } }) case _ => - joinpoint(k, ks) { (k, ks) => + def neutralMatch(k: Frame, ks: Stack) = NeutralStmt.Match(sc, // This is ALMOST like evaluate(BlockLit), but keeps the current continuation clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => @@ -1098,6 +1097,12 @@ class NewNormalizer { (id, block) }, default.map { stmt => nested { evaluate(stmt, k, ks) } }) + // linear usage of the continuation: do not create a joinpoint. + // This is a simple optimization for record access since r.x is always desugared into a match + if (default.size + clauses.size > 1) { + joinpoint(k, ks) { (k, ks) => neutralMatch(k, ks) } + } else { + neutralMatch(k, ks) } } From bc00c8dce3a88c6481880bb09aa500bb9aa87714 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Sat, 22 Nov 2025 11:35:27 +0100 Subject: [PATCH 106/123] experiment with inlining policy --- .../scala/effekt/core/optimizer/Inliner.scala | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala index b4c157ac5..868f81878 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Inliner.scala @@ -17,16 +17,38 @@ class Unique(maxInlineSize: Int) extends InliningPolicy { } class UniqueJumpSimple(maxInlineSize: Int) extends InliningPolicy { + def isSimple(s: Stmt): Boolean = s match { + case Stmt.Def(id, block, body) => isSimple(body) + case Stmt.Let(id, annotatedTpe, binding, body) => isSimple(body) + case Stmt.ImpureApp(id, callee, targs, vargs, bargs, body) => isSimple(body) + + case Stmt.Alloc(id, init, region, body) => true + case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => true + case Stmt.Put(ref, annotatedCapt, value, body) => true + case Stmt.Var(ref, init, capture, body) => true + + case Stmt.Return(expr) => true + case Stmt.App(callee, targs, vargs, bargs) => true + case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => true + + case Stmt.Val(id, annotatedTpe, binding, body) => false + case Stmt.If(cond, thn, els) => false + case Stmt.Match(scrutinee, clauses, default) => false + case Stmt.Region(body) => false + case Stmt.Reset(body) => false + case Stmt.Shift(prompt, body) => false + case Stmt.Resume(k, body) => false + case Stmt.Hole(span) => false + } override def apply(id: Id)(using ctx: Context): Boolean = { val use = ctx.usages.get(id) val block = ctx.blocks.get(id) var doInline = !ctx.usages.get(id).contains(Usage.Recursive) - doInline &&= use.contains(Usage.Once) || block.collect { - case Block.BlockLit(_, _, _, _, _: Stmt.Return) => true - case Block.BlockLit(_, _, _, _, _: Stmt.App) => true + doInline &&= use.contains(Usage.Once) || (block.collect { + case Block.BlockLit(_, _, _, _, stmt) => isSimple(stmt) + case Block.New(_) => true case Block.BlockVar(_, _, _) => true - }.getOrElse(false) - doInline &&= block.exists(_.size <= maxInlineSize) + }.getOrElse(false) && block.exists(_.size <= maxInlineSize)) doInline } } From 5be2cb0ef652c5fd9a40a59a84362dfc084b7ef9 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Sat, 22 Nov 2025 11:36:14 +0100 Subject: [PATCH 107/123] remove DirectStyle opt. from opt. pipeline --- .../effekt/core/optimizer/Optimizer.scala | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 66c38d071..d54f167e4 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -30,23 +30,24 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { if !Context.config.optimize() then return tree; - // (2) lift static arguments - tree = StaticArguments.transform(mainSymbol, tree) - val inliningPolicy = UniqueJumpSimple( - maxInlineSize = 150 + maxInlineSize = 15 ) - def normalize(m: ModuleDecl) = { - val anfed = BindSubexpressions.transform(m) - val normalized = NewNormalizer().run(anfed) - val inlined = Inliner(inliningPolicy, Reachable(Set(mainSymbol), normalized)).run(normalized) + def normalize(m: ModuleDecl) = Context.timed("new-normalizer", source.name) { + val staticArgs = StaticArguments.transform(mainSymbol, m) + val normalized = NewNormalizer().run(staticArgs) + val reachability = Reachable(Set(mainSymbol), normalized) + val inlined = Inliner(inliningPolicy, reachability).run(normalized) val live = Deadcode.remove(mainSymbol, inlined) val tailRemoved = RemoveTailResumptions(live) - val contified = DirectStyle.rewrite(tailRemoved) - contified + //val contified = DirectStyle.rewrite(tailRemoved) + tailRemoved } - tree = Context.timed("new-normalizer-1", source.name) { normalize(tree) } - tree = Context.timed("new-normalizer-2", source.name) { normalize(tree) } + tree = normalize(tree) + tree = normalize(tree) + tree = normalize(tree) + tree = normalize(tree) + //util.trace(tree) tree } From 94c2126fb9757b6ba4ce3f554422980ac616b9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Wed, 26 Nov 2025 17:49:47 +0100 Subject: [PATCH 108/123] Initial algebraic simplification for integers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on code by @b-studios. Co-authored-by: Jonathan Brachthäuser --- .../effekt/core/optimizer/NewNormalizer.scala | 63 ++++++- .../core/optimizer/theories/Integers.scala | 176 ++++++++++++++++++ .../effekt/source/ResolveExternDefs.scala | 3 +- 3 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 520233be8..9a86b381f 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -6,6 +6,7 @@ import effekt.core.ValueType.Boxed import effekt.source.Span import effekt.symbols.builtins import effekt.symbols.builtins.AsyncCapability +import effekt.core.optimizer.theories import kiama.output.ParenPrettyPrinter import scala.annotation.tailrec @@ -78,6 +79,8 @@ object semantics { // Actual Values case Literal(value: Any, annotatedType: ValueType) + case Integer(value: theories.Integers.Integer) + case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) // TODO use dynamic captures @@ -89,6 +92,7 @@ object semantics { case Value.Var(id, annotatedType) => Set(id) case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty + case Value.Integer(value) => value.free case Value.Make(data, tag, targs, vargs) => vargs.toSet // Box abstracts over all free computation variables, only when unboxing, they occur free again case Value.Box(body, tpe) => body.free @@ -737,6 +741,8 @@ object semantics { "box" <+> braces(nest(line <> toDoc(body) <> line)) case Value.Var(id, tpe) => toDoc(id) + + case Value.Integer(value) => value.show } def toDoc(block: Block): Doc = block match { @@ -919,8 +925,10 @@ class NewNormalizer { case Expr.ValueVar(id, annotatedType) => env.lookupValue(id) - case core.Expr.Literal(value, annotatedType) => - scope.allocate("x", Value.Literal(value, annotatedType)) + case core.Expr.Literal(value, annotatedType) => value match { + case As.IntExpr(x) => scope.allocate("x", Value.Integer(x)) + case _ => scope.allocate("x", Value.Literal(value, annotatedType)) + } case core.Expr.PureApp(f, targs, vargs) => val externDef = env.lookupComputation(f.id) @@ -1199,6 +1207,13 @@ class NewNormalizer { case _ => false } + val builtinNameToBlockVar: Map[String, BlockVar] = builtinExterns.collect { + case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => + val builtinName = vmBody.contents.strings.head + val bv: BlockVar = BlockVar(id, BlockType.Function(tps, cps, vps.map { _.tpe }, bps.map { bp => bp.tpe }, ret), capt) + builtinName -> bv + }.toMap + val toplevelEnv = Env.empty // user-defined functions .bindComputation(mod.definitions.collect { @@ -1224,7 +1239,8 @@ class NewNormalizer { }.toMap, mod.definitions.collect { case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap ++ externTypes + }.toMap ++ externTypes, + builtinNameToBlockVar ) val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) @@ -1239,12 +1255,12 @@ class NewNormalizer { debug(s"------- ${util.show(id)} -------") debug(util.show(body)) - given localEnv: Env = env - .bindValue(vparams.map(p => p.id -> p.id)) + val scope = Scope.empty + val localEnv: Env = env + .bindValue(vparams.map(p => p.id -> scope.allocate("p", Value.Var(p.id, p.tpe)))) .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - given scope: Scope = Scope.empty - val result = evaluate(body, Frame.Return, Stack.Empty) + val result = evaluate(body, Frame.Return, Stack.Empty)(using localEnv, scope) debug(s"----------normalized-----------") val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) @@ -1258,7 +1274,7 @@ class NewNormalizer { case other => other } - case class TypingContext(values: Map[Addr, ValueType], blocks: Map[Label, (BlockType, Captures)]) { + case class TypingContext(values: Map[Addr, ValueType], blocks: Map[Label, (BlockType, Captures)], builtinBlockVars: Map[String, BlockVar]) { def bind(id: Id, tpe: ValueType): TypingContext = this.copy(values = values + (id -> tpe)) def bind(id: Id, tpe: BlockType, capt: Captures): TypingContext = this.copy(blocks = blocks + (id -> (tpe, capt))) def bindValues(vparams: List[ValueParam]): TypingContext = this.copy(values = values ++ vparams.map(p => p.id -> p.tpe)) @@ -1337,12 +1353,13 @@ class NewNormalizer { }(G) } - def embedExpr(value: Value)(using TypingContext): core.Expr = value match { + def embedExpr(value: Value)(using cx: TypingContext): core.Expr = value match { case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) + case Value.Integer(value) => theories.Integers.reify(value, cx.builtinBlockVars) } def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) @@ -1501,8 +1518,20 @@ lazy val integers: Builtins = Map( // Arithmetic // ---------- builtin("effekt::infixAdd(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x + y, Type.TInt) + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.add(x, y)) }, + builtin("effekt::infixSub(Int, Int)") { + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.sub(x, y)) + }, + builtin("effekt::infixMul(Int, Int)") { + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.mul(x, y)) + }, + builtin("effekt::infixDiv(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x / y)) + }, + builtin("effekt::mod(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x % y)) + } ) lazy val supportedBuiltins: Builtins = integers @@ -1516,4 +1545,18 @@ protected object As { case _ => None } } + + object IntExpr { + def unapply(v: semantics.Value): Option[theories.Integers.Integer] = v match { + // Integer literals not yet embedded into the theory of integers + case semantics.Value.Literal(value: scala.Long, _) => Some(theories.Integers.embed(value)) + case semantics.Value.Literal(value: scala.Int, _) => Some(theories.Integers.embed(value.toLong)) + case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.Integers.embed(value.toLong)) + // Variables of type integer + case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.Integers.embed(id)) + // Already embedded integers + case semantics.Value.Integer(value) => Some(value) + case _ => None + } + } } \ No newline at end of file diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala new file mode 100644 index 000000000..514d3b5eb --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala @@ -0,0 +1,176 @@ +package effekt.core.optimizer.theories + +import effekt.core.Block.BlockVar +import effekt.core.{Expr, Id, Type} + +/** + * Theory for integers with neutral variables: multivariate polynomials with integer coefficients. + * + * KNOWN LIMITATION: This implementation assumes 64-bit signed integers. + * Unfortunately, this is unsound for the JavaScript backend, which uses JavaScript numbers that are IEEE-754 doubles. + */ +object Integers { + case class Integer(value: Long, addends: Addends) { + val free: Set[Id] = addends.flatMap { case (factors, _) => factors.keys }.toSet + + def show: String = { + val Integer(v, a) = this + val terms = a.map { case (factors, n) => + val factorStr = if (factors.isEmpty) "1" else { + factors.map { case (id, exp) => + if (exp == 1) s"${id.name}" + else s"${id.name}^$exp" + }.mkString("*") + } + if (n == 1) s"$factorStr" + else s"$n*$factorStr" + }.toList + + val constPart = if (v != 0) List(v.toString) else Nil + (constPart ++ terms).mkString(" + ") + } + } + + enum Operation { + case Add, Sub, Mul, Div + } + import Operation._ + + def embed(value: Long): Integers.Integer = Integer(value, Map.empty) + def embed(id: Id): Integers.Integer = Integer(0, Map(Map(id -> 1) -> 1)) + + def reify(value: Integer, embedBuiltinName: String => BlockVar): Expr = Reify(embedBuiltinName).reify(value) + + // 3 * x * x / y = Addend(3, Map(x -> 2, y -> -1)) + type Addends = Map[Factors, Long] + type Factors = Map[Id, Int] + + def normalize(n: Integer): Integer = normalized(n.value, n.addends) + + def normalized(value: Long, addends: Addends): Integer = + val (const, norm) = normalizeAddends(addends) + Integer(value + const, norm) + + def add(l: Integer, r: Integer): Integer = (l, r) match { + // 2 + (3 * x) + 4 + (5 * y) = 6 + (3 * x) + (5 * y) + case (Integer(x, xs), Integer(y, ys)) => + normalized(x + y, add(xs, ys)) + } + + def add(xs: Addends, ys: Addends): Addends = { + var addends = xs + ys.foreach { case (factors, n) => + val m: Long = addends.getOrElse(factors, 0) + addends = addends.updated(factors, n + m) + } + addends + } + + // 3 * x1^2 + 2 * EMPTY + 0 * x2^3 = 2 + 3 * x1^2 + def normalizeAddends(xs: Addends): (Long, Addends) = { + var constant: Long = 0 + var filtered: Addends = Map.empty + xs.foreach { case (factors, n) => + if (factors.isEmpty) { + constant += n + } + if (n != 0) { + filtered = filtered.updated(factors, n) + } + } + (constant, filtered) + } + + def neg(l: Integer): Integer = mul(l, -1) + + // (42 + 3*x + y) - (42 + 3*x + y) = (42 + 3*x + y) + (-1*42 + -1*3*x + -1*y) + def sub(l: Integer, r: Integer): Integer = + add(l, neg(r)) + + def mul(l: Integer, factor: Long): Integer = l match { + case Integer(value, addends) => + Integer(value * factor, addends.map { case (f, n) => f -> n * factor }) + } + + def mul(l: Integer, factor: Factors): Integer = l match { + case Integer(value, addends) => + Integer(0, Map(factor -> value) ++ addends.map { case (f, n) => + mul(f, factor) -> n + }) + } + + // (x * x * y) * (x * y * z) = x^3 + y^2 + z^1 + def mul(l: Factors, r: Factors): Factors = { + var factors = l + r.foreach { case (f, n) => + val m = factors.getOrElse(f, 0) + factors = factors.updated(f, n + m) + } + normalizeFactors(factors) + } + + // x1^2 * x2^0 * x3^3 = x1^2 * x3^3 + def normalizeFactors(f: Factors): Factors = + f.filterNot { case (id, exp) => exp == 0 } + + // (42 + 3*x + y) * (42 + 3*x + y) + // = + // (42 + 3*x + y) * 42 + (42 + 3*x + y) * 3*x + (42 + 3*x + y) * y + def mul(l: Integer, r: Integer): Integer = r match { + case Integer(y, ys) => + var sum: Integer = mul(l, y) + ys.foreach { case (f, n) => sum = add(sum, mul(mul(l, n), f)) } + normalize(sum) + } + + case class Reify(embedBuiltinName: String => BlockVar) { + def reifyVar(id: Id): Expr = Expr.ValueVar(id, Type.TInt) + + def reifyInt(v: Long): Expr = Expr.Literal(v, Type.TInt) + + def reifyOp(l: Expr, op: Operation, r: Expr): Expr = op match { + case Add => Expr.PureApp(embedBuiltinName("effekt::infixAdd(Int, Int)"), List(), List(l, r)) + case Sub => Expr.PureApp(embedBuiltinName("effekt::infixSub(Int, Int)"), List(), List(l, r)) + case Mul => Expr.PureApp(embedBuiltinName("effekt::infixMul(Int, Int)"), List(), List(l, r)) + case Div => Expr.PureApp(embedBuiltinName("effekt::infixDiv(Int, Int)"), List(), List(l, r)) + } + + def reify(v: Integer): Expr = + val Integer(const, addends) = normalize(v) + + val adds = addends.toList.map { case (factors, n) => + if (n == 1) reifyFactors(factors) + else reifyOp(reifyInt(n), Mul, reifyFactors(factors)) + }.reduceOption { case (l, r) => reifyOp(l, Add, r) } + + adds.map { a => + if (const != 0) reifyOp(reifyInt(const), Add, a) + else a + }.getOrElse { + reifyInt(const) + } + + def reifyFactor(x: Id, n: Int): Expr = + if (n <= 0) sys error "Should not happen" + else if (n == 1) reifyVar(x) + else reifyOp(reifyVar(x), Mul, reifyFactor(x, n - 1)) + + def reifyFactors(ys: Factors): Expr = { + val factors = ys.toList.filterNot { case (_, n) => n == 0 } + val positive = factors.filter { case (_, n) => n > 0 } + val negative = factors.filter { case (_, n) => n < 0 } + + val pos = positive.map { case (x, n) => reifyFactor(x, n) }.reduceOption { case (l, r) => reifyOp(l, Mul, r) } + val neg = negative.map { case (x, n) => reifyFactor(x, n * -1) }.reduceOption { case (l, r) => reifyOp(l, Mul, r) } + val numerator = pos.getOrElse { + reifyInt(1) + } + + neg.map { denominator => + reifyOp(numerator, Div, denominator) + }.getOrElse { + numerator + } + } + } +} diff --git a/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala b/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala index 2ad6e6ed9..40bf0cbe6 100644 --- a/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala +++ b/effekt/shared/src/main/scala/effekt/source/ResolveExternDefs.scala @@ -42,6 +42,7 @@ object ResolveExternDefs extends Phase[Typechecked, Typechecked] { def rewrite(defn: Def)(using Context): Option[Def] = Context.focusing(defn) { case Def.ExternDef(id, tparams, vparams, bparams, capture, ret, bodies, doc, span) => + val vmBody = bodies.find(_.featureFlag.matches("vm", matchDefault = false)) findPreferred(bodies) match { case body@ExternBody.StringExternBody(featureFlag, template, span) => if (featureFlag.isDefault) { @@ -49,7 +50,7 @@ object ResolveExternDefs extends Phase[Typechecked, Typechecked] { + s"please annotate it with a feature flag (Supported by the current backend: ${Context.compiler.supportedFeatureFlags.mkString(", ")})") } - val d = Def.ExternDef(id, tparams, vparams, bparams, capture, ret, List(body), doc, span) + val d = Def.ExternDef(id, tparams, vparams, bparams, capture, ret, List(body) ++ vmBody.toList, doc, span) Context.copyAnnotations(defn, d) Some(d) case ExternBody.EffektExternBody(featureFlag, body, span) => From b325eb9416a6d6cc7ec8e21603cb753620b58f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 09:53:16 +0100 Subject: [PATCH 109/123] Make doc comment more precise --- .../main/scala/effekt/core/optimizer/theories/Integers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala index 514d3b5eb..cf36230fe 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala @@ -4,7 +4,7 @@ import effekt.core.Block.BlockVar import effekt.core.{Expr, Id, Type} /** - * Theory for integers with neutral variables: multivariate polynomials with integer coefficients. + * Theory for integers with neutral variables: multivariate Laurent polynomials with 64-bit signed integer coefficients. * * KNOWN LIMITATION: This implementation assumes 64-bit signed integers. * Unfortunately, this is unsound for the JavaScript backend, which uses JavaScript numbers that are IEEE-754 doubles. From 5794b869828609805d7027309b42e691d7f2d9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 10:04:12 +0100 Subject: [PATCH 110/123] Add further builtin integer operations from vm builtins --- .../effekt/core/optimizer/NewNormalizer.scala | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 9a86b381f..775f70d57 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1515,7 +1515,7 @@ def builtin(name: String)(impl: List[semantics.Value] ~> semantics.Value): (Stri type Builtins = Map[String, BuiltinImpl] lazy val integers: Builtins = Map( - // Arithmetic + // Integer arithmetic operations with symbolic simplification support // ---------- builtin("effekt::infixAdd(Int, Int)") { case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.add(x, y)) @@ -1526,12 +1526,58 @@ lazy val integers: Builtins = Map( builtin("effekt::infixMul(Int, Int)") { case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.mul(x, y)) }, + // Integer arithmetic operations only evaluated for literals + // ---------- builtin("effekt::infixDiv(Int, Int)") { case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x / y)) }, builtin("effekt::mod(Int, Int)") { case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x % y)) - } + }, + builtin("effekt::bitwiseShl(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x << y)) + }, + builtin("effekt::bitwiseShr(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x >> y)) + }, + builtin("effekt::bitwiseAnd(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x & y)) + }, + builtin("effekt::bitwiseOr(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x | y)) + }, + builtin("effekt::bitwiseXor(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x ^ y)) + }, + // Comparison + // ---------- + builtin("effekt::infixEq(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x == y, Type.TBoolean) + }, + builtin("effekt::infixNeq(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x != y, Type.TBoolean) + }, + builtin("effekt::infixLt(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x < y, Type.TBoolean) + }, + builtin("effekt::infixGt(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x > y, Type.TBoolean) + }, + builtin("effekt::infixLte(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x <= y, Type.TBoolean) + }, + builtin("effekt::infixGte(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x >= y, Type.TBoolean) + }, + // Conversion + // ---------- + builtin("effekt::toDouble(Int)") { + case As.Int(x) :: Nil => semantics.Value.Literal(x.toDouble, Type.TDouble) + }, + + builtin("effekt::show(Int)") { + case As.Int(n) :: Nil => semantics.Value.Literal(n.toString, Type.TString) + }, ) lazy val supportedBuiltins: Builtins = integers From 34c0937f032745d482ac018a9af8981e1ae8c344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 10:16:43 +0100 Subject: [PATCH 111/123] Split up NewNormalizer into submodules --- .../effekt/core/optimizer/NewNormalizer.scala | 1608 ----------------- .../effekt/core/optimizer/Optimizer.scala | 1 + .../optimizer/normalizer/NewNormalizer.scala | 764 ++++++++ .../core/optimizer/normalizer/builtins.scala | 105 ++ .../core/optimizer/normalizer/semantics.scala | 741 ++++++++ .../{ => normalizer}/theories/Integers.scala | 2 +- 6 files changed, 1612 insertions(+), 1609 deletions(-) create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala rename effekt/shared/src/main/scala/effekt/core/optimizer/{ => normalizer}/theories/Integers.scala (99%) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala index 775f70d57..e69de29bb 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala @@ -1,1608 +0,0 @@ -package effekt -package core -package optimizer - -import effekt.core.ValueType.Boxed -import effekt.source.Span -import effekt.symbols.builtins -import effekt.symbols.builtins.AsyncCapability -import effekt.core.optimizer.theories -import kiama.output.ParenPrettyPrinter - -import scala.annotation.tailrec -import scala.collection.mutable -import scala.collection.immutable.ListMap - -// TODO -// - change story of how inlining is implemented. We need to also support toplevel functions that potentially -// inline each other. Do we need to sort them topologically? How do we deal with (mutually) recursive definitions? -// -// -// plan: only introduce parameters for free things inside a block that are bound in the **stack** -// that is in -// -// only abstract over p, but not n: -// -// def outer(n: Int) = -// def foo(p) = shift(p) { ... n ... } -// reset { p => -// ... -// } -// -// Same actually for stack allocated mutable state, we should abstract over those (but only those) -// and keep the function in its original location. -// This means we only need to abstract over blocks, no values, no types. -// -// TODO Region desugaring -// region r { -// reset { p => -// var x in r = 42 -// x = !x + 1 -// println(!x) -// } -// } -// -// reset { r => -// reset { p => -// //var x in r = 42 -// shift(r) { k => -// var x = 42 -// resume(k) { -// x = !x + 1 -// println(!x) -// } -// } -// } -// } -// -// - Typeability preservation: {r: Region} becomes {r: Prompt[T]} -// [[ def f() {r: Region} = s ]] = def f[T]() {r: Prompt[T]} = ... -// - Continuation capture is _not_ constant time in JS backend, so we expect a (drastic) slowdown when desugaring - -object semantics { - - // Values - // ------ - - type Addr = Id - type Label = Id - type Prompt = Id - - // this could not only compute free variables, but also usage information to guide the inliner (see "secrets of the ghc inliner") - type Variables = Set[Id] - def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet - - enum Value { - // Stuck - case Var(id: Id, annotatedType: ValueType) - case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) - - // Actual Values - case Literal(value: Any, annotatedType: ValueType) - case Integer(value: theories.Integers.Integer) - - case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) - - // TODO use dynamic captures - case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) - - val dynamicCapture: Variables = Set.empty - - val free: Variables = this match { - case Value.Var(id, annotatedType) => Set(id) - case Value.Extern(id, targs, vargs) => vargs.toSet - case Value.Literal(value, annotatedType) => Set.empty - case Value.Integer(value) => value.free - case Value.Make(data, tag, targs, vargs) => vargs.toSet - // Box abstracts over all free computation variables, only when unboxing, they occur free again - case Value.Box(body, tpe) => body.free - } - } - - // TODO find better name for this - enum Binding { - case Let(value: Value) - case Def(block: Block) - case Rec(block: Block, tpe: BlockType, capt: Captures) - case Val(stmt: NeutralStmt) - case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) - case Unbox(addr: Addr, tpe: BlockType, capt: Captures) - case Get(ref: Id, tpe: ValueType, cap: Captures) - - val free: Variables = this match { - case Binding.Let(value) => value.free - case Binding.Def(block) => block.free - case Binding.Rec(block, tpe, capt) => block.free - case Binding.Val(stmt) => stmt.free - // TODO block args for externs are not supported (for now?) - case Binding.Run(f, targs, vargs, bargs) => vargs.toSet - case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set(addr) - case Binding.Get(ref, tpe, cap) => Set(ref) - } - - val dynamicCapture: Variables = this match { - case Binding.Let(value) => value.dynamicCapture - case Binding.Def(block) => block.dynamicCapture - case Binding.Rec(block, tpe, capt) => block.dynamicCapture - case Binding.Val(stmt) => stmt.dynamicCapture - // TODO block args for externs are not supported (for now?) - case Binding.Run(f, targs, vargs, bargs) => Set.empty - case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => capt // TODO these are the static not dynamic captures - case Binding.Get(ref, tpe, cap) => Set(ref) - } - } - - type Bindings = List[(Id, Binding)] - object Bindings { - def empty: Bindings = Nil - } - - /** - * A Scope is a bit like a basic block, but without the terminator - */ - class Scope( - var bindings: ListMap[Id, Binding], - var inverse: Map[Value, Addr], - val outer: Option[Scope] - ) { - // Backtrack the internal state of Scope after running `prog` - def local[A](prog: => A): A = { - val scopeBefore = Scope(this.bindings, this.inverse, this.outer) - val res = prog - this.bindings = scopeBefore.bindings - this.inverse = scopeBefore.inverse - res - } - - // floating values to the top is not always beneficial. For example - // def foo() = COMPUTATION - // vs - // let x = COMPUTATION - // def foo() = x - def getDefinition(value: Value): Option[Addr] = - inverse.get(value) orElse outer.flatMap(_.getDefinition(value)) - - def allocate(hint: String, value: Value): Addr = - getDefinition(value) match { - case Some(value) => value - case None => - val addr = Id(hint) - bindings = bindings.updated(addr, Binding.Let(value)) - inverse = inverse.updated(value, addr) - addr - } - - def allocateGet(ref: Id, tpe: ValueType, cap: Captures): Addr = { - val addr = Id("get") - bindings = bindings.updated(addr, Binding.Get(ref, tpe, cap)) - addr - } - - def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]): Addr = { - val addr = Id(hint) - bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs, bargs)) - addr - } - - def unbox(innerAddr: Addr, tpe: BlockType, capt: Captures): Addr = { - val unboxAddr = Id("unbox") - bindings = bindings.updated(unboxAddr, Binding.Unbox(innerAddr, tpe, capt)) - unboxAddr - } - - // TODO Option[Value] or Var(id) in Value? - def lookupValue(addr: Addr): Option[Value] = bindings.get(addr) match { - case Some(Binding.Let(value)) => Some(value) - case _ => outer.flatMap(_.lookupValue(addr)) - } - - def define(label: Label, block: Block): Unit = - bindings = bindings.updated(label, Binding.Def(block)) - - def defineRecursive(label: Label, block: Block, tpe: BlockType, capt: Captures): Unit = - bindings = bindings.updated(label, Binding.Rec(block, tpe, capt)) - - def push(id: Id, stmt: NeutralStmt): Unit = - bindings = bindings.updated(id, Binding.Val(stmt)) - } - object Scope { - def empty: Scope = new Scope(ListMap.empty, Map.empty, None) - } - - def reifyBindings(scope: Scope, body: NeutralStmt): BasicBlock = { - var used = body.free - var filtered = Bindings.empty - // TODO implement properly - scope.bindings.toSeq.reverse.foreach { - // TODO for now we keep ALL definitions - case (addr, b: Binding.Def) => - used = used ++ b.free - filtered = (addr, b) :: filtered - case (addr, b: Binding.Rec) => - used = used ++ b.free - filtered = (addr, b) :: filtered - case (addr, s: Binding.Val) => - used = used ++ s.free - filtered = (addr, s) :: filtered - case (addr, v: Binding.Run) => - used = used ++ v.free - filtered = (addr, v) :: filtered - - // TODO if type is unit like, we can potentially drop this binding (but then we need to make up a "fresh" unit at use site) - case (addr, v: Binding.Let) if used.contains(addr) => - used = used ++ v.free - filtered = (addr, v) :: filtered - case (addr, v: Binding.Let) => () - case (addr, b: Binding.Unbox) => - used = used ++ b.free - filtered = (addr, b):: filtered - case (addr, g: Binding.Get) => - used = used ++ g.free - filtered = (addr, g) :: filtered - } - - // we want to avoid turning tailcalls into non tail calls like - // - // val x = app(x) - // return x - // - // so we eta-reduce here. Can we achieve this by construction? - // TODO lastOption will go through the list AGAIN, let's see whether this causes performance problems - (filtered.lastOption, body) match { - case (Some((id1, Binding.Val(stmt))), NeutralStmt.Return(id2)) if id1 == id2 => - BasicBlock(filtered.init, stmt) - case (_, _) => - BasicBlock(filtered, body) - } - } - - def nested(prog: Scope ?=> NeutralStmt)(using scope: Scope): BasicBlock = { - // TODO parent code and parent store - val local = Scope(ListMap.empty, Map.empty, Some(scope)) - val result = prog(using local) - reifyBindings(local, result) - } - - case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { - def lookupValue(id: Id): Addr = values(id) - def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) - def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) - - def lookupComputation(id: Id): Computation = computations.getOrElse(id, sys error s"Unknown computation: ${util.show(id)} -- env: ${computations.map { case (id, comp) => s"${util.show(id)}: $comp" }.mkString("\n") }") - def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) - def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) - } - object Env { - def empty: Env = Env(Map.empty, Map.empty) - } - // "handlers" - def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindValue(id, addr)) - - def bind[R](id: Id, computation: Computation)(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindComputation(id, computation)) - - def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = - prog(using env.bindValue(values)) - - case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { - val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) - val dynamicCapture: Variables = body.dynamicCapture -- bparams.map(_.id) - } - - case class BasicBlock(bindings: Bindings, body: NeutralStmt) { - val free: Variables = { - var free = body.free - bindings.reverse.foreach { - case (id, b: Binding.Let) => free = (free - id) ++ b.free - case (id, b: Binding.Def) => free = (free - id) ++ b.free - case (id, b: Binding.Rec) => free = (free - id) ++ (b.free - id) - case (id, b: Binding.Val) => free = (free - id) ++ b.free - case (id, b: Binding.Run) => free = (free - id) ++ b.free - case (id, b: Binding.Unbox) => free = (free - id) ++ b.free - case (id, b: Binding.Get) => free = (free - id) ++ b.free - } - free - } - - val dynamicCapture: Variables = { - body.dynamicCapture ++ bindings.flatMap(_._2.dynamicCapture) - } - } - - enum Computation { - // Unknown - case Var(id: Id) - // Known function - case Def(closure: Closure) - - case Continuation(k: Cont) - - case BuiltinExtern(id: Id, builtinName: String) - - // Known object - case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) - - val free: Variables = this match { - case Computation.Var(id) => Set(id) - case Computation.Def(closure) => closure.free - case Computation.Continuation(k) => Set.empty // TODO ??? - case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet - case Computation.BuiltinExtern(id, vmSymbol) => Set.empty - } - - val dynamicCapture: Variables = this match { - case Computation.Var(id) => Set(id) - case Computation.Def(closure) => closure.dynamicCapture - case Computation.Continuation(k) => Set.empty // TODO ??? - case Computation.New(interface, operations) => operations.flatMap(_._2.dynamicCapture).toSet - case Computation.BuiltinExtern(id, vmSymbol) => Set.empty - } - } - - // TODO add escaping mutable variables - case class Closure(label: Label, environment: List[Computation.Var]) { - val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet - val dynamicCapture: Variables = environment.map(_.id).toSet - } - - // Statements - // ---------- - enum NeutralStmt { - // context (continuation) is unknown - case Return(result: Id) - // callee is unknown - case App(callee: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) - // Known jump, but we do not want to inline - case Jump(label: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) - // callee is unknown - case Invoke(id: Id, method: Id, methodTpe: BlockType, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) - // cond is unknown - case If(cond: Id, thn: BasicBlock, els: BasicBlock) - // scrutinee is unknown - case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) - - // body is stuck - case Reset(prompt: BlockParam, body: BasicBlock) - // prompt / context is unknown - case Shift(prompt: Prompt, kCapt: Capture, k: BlockParam, body: BasicBlock) - // continuation is unknown - case Resume(k: Id, body: BasicBlock) - - case Var(id: BlockParam, init: Addr, body: BasicBlock) - case Put(ref: Id, tpe: ValueType, cap: Captures, value: Addr, body: BasicBlock) - - case Region(id: BlockParam, body: BasicBlock) - case Alloc(id: BlockParam, init: Addr, region: Id, body: BasicBlock) - - // aborts at runtime - case Hole(span: Span) - - val free: Variables = this match { - case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) - case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) - case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) - case NeutralStmt.If(cond, thn, els) => Set(cond) ++ thn.free ++ els.free - case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) - case NeutralStmt.Return(result) => Set(result) - case NeutralStmt.Reset(prompt, body) => body.free - prompt.id - case NeutralStmt.Shift(prompt, capt, k, body) => (body.free - k.id) + prompt - case NeutralStmt.Resume(k, body) => Set(k) ++ body.free - case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id - case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref, value) ++ body.free - case NeutralStmt.Region(id, body) => body.free - id.id - case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id.id - case NeutralStmt.Hole(span) => Set.empty - } - - val dynamicCapture: Variables = this match { - case NeutralStmt.Return(result) => Set.empty - case NeutralStmt.Hole(span) => Set.empty - - case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) - case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) - case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) - case NeutralStmt.If(cond, thn, els) => thn.dynamicCapture ++ els.dynamicCapture - case NeutralStmt.Match(scrutinee, clauses, default) => clauses.flatMap(_._2.dynamicCapture).toSet ++ default.map(_.dynamicCapture).getOrElse(Set.empty) - case NeutralStmt.Reset(prompt, body) => body.dynamicCapture - prompt.id - case NeutralStmt.Shift(prompt, capt, k, body) => (body.dynamicCapture - k.id) + prompt - case NeutralStmt.Resume(k, body) => Set(k) ++ body.dynamicCapture - case NeutralStmt.Var(id, init, body) => body.dynamicCapture - id.id - case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref) ++ body.dynamicCapture - case NeutralStmt.Region(id, body) => body.dynamicCapture - id.id - case NeutralStmt.Alloc(id, init, region, body) => Set(region) ++ body.dynamicCapture - id.id - } - } - - // Stacks - // ------ - enum Frame { - case Return - case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) - case Dynamic(closure: Closure) - - /* Return an argument `arg` through this frame and the rest of the stack `ks` - */ - def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { - case Frame.Return => ks match { - case Stack.Empty => NeutralStmt.Return(arg) - case Stack.Unknown => NeutralStmt.Return(arg) - case Stack.Reset(p, k, ks) => k.ret(ks, arg) - case Stack.Var(id, curr, k, ks) => k.ret(ks, arg) - case Stack.Region(id, bindings, k, ks) => k.ret(ks, arg) - } - case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) - case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } - } - - // pushing purposefully does not abstract over env (it closes over it!) - def push(tpe: ValueType)(f: Scope => Addr => Frame => Stack => NeutralStmt): Frame = - Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) - } - - // maybe, for once it is simpler to decompose stacks like - // - // f, (p, f) :: (p, f) :: Nil - // - // where the frame on the reset is the one AFTER the prompt NOT BEFORE! - enum Stack { - /** - * Statically known to be empty - * This only occurs at the entrypoint of normalization. - * In other cases, where the stack is not known, you should use Unknown instead. - */ - case Empty - /** Dynamic tail (we do not know the shape of the remaining stack) - */ - case Unknown - case Reset(prompt: BlockParam, frame: Frame, next: Stack) - case Var(id: BlockParam, curr: Addr, frame: Frame, next: Stack) - // TODO desugar regions into var? - case Region(id: BlockParam, bindings: Map[BlockParam, Addr], frame: Frame, next: Stack) - - lazy val bound: List[BlockParam] = this match { - case Stack.Empty => Nil - case Stack.Unknown => Nil - case Stack.Reset(prompt, frame, next) => prompt :: next.bound - case Stack.Var(id, curr, frame, next) => id :: next.bound - case Stack.Region(id, bindings, frame, next) => id :: next.bound ++ bindings.keys - } - } - - @tailrec - def get(ref: Id, ks: Stack): Option[Addr] = ks match { - case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(ref)} in empty stack" - // We have reached the end of the known stack, so the variable must be in the unknown part. - case Stack.Unknown => None - case Stack.Reset(prompt, frame, next) => get(ref, next) - case Stack.Var(id1, curr, frame, next) if ref == id1.id => Some(curr) - case Stack.Var(id1, curr, frame, next) => get(ref, next) - case Stack.Region(id, bindings, frame, next) => - val containsRef = bindings.keys.find(bp => bp.id == ref) - containsRef match { - case Some(bparam) => Some(bindings(bparam)) - case None => get(ref, next) - } - } - - def put(ref: Id, value: Addr, ks: Stack): Option[Stack] = ks match { - case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(ref)} in empty stack" - // We have reached the end of the known stack, so the variable must be in the unknown part. - case Stack.Unknown => None - case Stack.Reset(prompt, frame, next) => put(ref, value, next).map(Stack.Reset(prompt, frame, _)) - case Stack.Var(id, curr, frame, next) if ref == id.id => Some(Stack.Var(id, value, frame, next)) - case Stack.Var(id, curr, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, frame, _)) - case Stack.Region(id, bindings, frame, next) => - val containsRef = bindings.keys.find(bp => bp.id == ref) - containsRef match { - case Some(bparam) => Some(Stack.Region(id, bindings.updated(bparam, value), frame, next)) - case None => put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) - } - } - - def alloc(ref: BlockParam, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { - // This case can occur if we normalize a function that abstracts over a region as a parameter - // We return None and force the reification of the allocation - case Stack.Empty => None - // We have reached the end of the known stack, so the variable must be in the unknown part. - case Stack.Unknown => None - case Stack.Reset(prompt, frame, next) => - alloc(ref, reg, value, next).map(Stack.Reset(prompt, frame, _)) - case Stack.Var(id, curr, frame, next) => - alloc(ref, reg, value, next).map(Stack.Var(id, curr, frame, _)) - case Stack.Region(id, bindings, frame, next) => - if (reg == id.id){ - Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) - } else { - alloc(ref, reg, value, next).map(Stack.Region(id, bindings, frame, _)) - } - } - - enum Cont { - case Empty - case Reset(frame: Frame, prompt: BlockParam, rest: Cont) -<<<<<<< HEAD - case Var(frame: Frame, id: BlockParam, curr: Addr, rest: Cont) - case Region(frame: Frame, id: BlockParam, bindings: Map[Id, Addr], rest: Cont) -======= - case Var(frame: Frame, id: BlockParam, curr: Addr, init: Addr, rest: Cont) - case Region(frame: Frame, id: BlockParam, bindings: Map[BlockParam, Addr], rest: Cont) ->>>>>>> 9039d884 (fix region reification) - } - - def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { - case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" - case Stack.Unknown => sys error s"Cannot find prompt ${util.show(p)} in unknown stack" - case Stack.Reset(prompt, frame, next) if prompt.id == p => - (Cont.Reset(k, prompt, Cont.Empty), frame, next) - case Stack.Reset(prompt, frame, next) => - val (c, frame2, stack) = shift(p, frame, next) - (Cont.Reset(k, prompt, c), frame2, stack) - case Stack.Var(id, curr, frame, next) => - val (c, frame2, stack) = shift(p, frame, next) - (Cont.Var(k, id, curr, c), frame2, stack) - case Stack.Region(id, bindings, frame, next) => - val (c, frame2, stack) = shift(p, frame, next) - (Cont.Region(k, id, bindings, c), frame2, stack) - } - - def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { - case Cont.Empty => - (k, ks) - case Cont.Reset(frame, prompt, rest) => - val (k1, ks1) = resume(rest, k, ks) - (frame, Stack.Reset(prompt, k1, ks1)) - case Cont.Var(frame, id, curr, rest) => - val (k1, ks1) = resume(rest, k, ks) - (frame, Stack.Var(id, curr, k1, ks1)) - case Cont.Region(frame, id, bindings, rest) => - val (k1, ks1) = resume(rest, k, ks) - (frame, Stack.Region(id, bindings, k1, ks1)) - } - - def joinpoint(k: Frame, ks: Stack)(f: (Frame, Stack) => NeutralStmt)(using scope: Scope): NeutralStmt = { - def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { - case Frame.Static(tpe, apply) => - val x = Id("x") - nested { scope ?=> apply(scope)(x)(Stack.Unknown) } match { - // Avoid trivial continuations like - // def k_6268 = (x_6267: Int_3) { - // return x_6267 - // } - case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => - k - case body => - val k = Id("k") - val closureParams = escaping.bound.collect { case p if body.dynamicCapture contains p.id => p } - scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) - Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) - } - case Frame.Return => k - case Frame.Dynamic(label) => k - } - - def reifyStack(ks: Stack): Stack = ks match { - case Stack.Empty => Stack.Empty - case Stack.Unknown => Stack.Unknown - case Stack.Reset(prompt, frame, next) => - Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) - case Stack.Var(id, curr, frame, next) => - Stack.Var(id, curr, reifyFrame(frame, next), reifyStack(next)) - case Stack.Region(id, bindings, frame, next) => - Stack.Region(id, bindings, reifyFrame(frame, next), reifyStack(next)) - } - f(reifyFrame(k, ks), reifyStack(ks)) - } - - def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = - reify(ks) { reify(k) { stmt } } - - def reify(k: Frame)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = - k match { - case Frame.Return => stmt - case Frame.Static(tpe, apply) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - // TODO Over-approximation - // Don't pass Stack.Unknown but rather the stack until the next reset? - /* - |----------| |----------| |---------| - | | ---> ... ---> | | ---> ... ---> | | ---> ... - |----------| |----------| |---------| - r1 r2 first next prompt - - Pass r1 :: ... :: r2 :: ... :: prompt :: UNKNOWN - */ - apply(scope)(tmp)(Stack.Unknown) - case Frame.Dynamic(Closure(label, closure)) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - NeutralStmt.Jump(label, Nil, List(tmp), closure) - } - - def reifyKnown(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = - k match { - case Frame.Return => reify(ks) { stmt } - case Frame.Static(tpe, apply) => - val tmp = Id("tmp") - scope.push(tmp, stmt) - apply(scope)(tmp)(ks) - case Frame.Dynamic(Closure(label, closure)) => reify(ks) { sc ?=> - val tmp = Id("tmp") - sc.push(tmp, stmt) - NeutralStmt.Jump(label, Nil, List(tmp), closure) - } - } - - @tailrec - final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = { - ks match { - case Stack.Empty => stmt - case Stack.Unknown => stmt - case Stack.Reset(prompt, frame, next) => - reify(next) { reify(frame) { - val body = nested { stmt } - if (body.dynamicCapture contains prompt.id) NeutralStmt.Reset(prompt, body) - else stmt // TODO this runs normalization a second time in the outer scope! - }} - case Stack.Var(id, curr, frame, next) => - reify(next) { reify(frame) { - val body = nested { stmt } - if (body.dynamicCapture contains id.id) NeutralStmt.Var(id, curr, body) - else stmt - }} - case Stack.Region(id, bindings, frame, next) => - reify(next) { reify(frame) { - val body = nested { stmt } - val bodyUsesBinding = body.dynamicCapture.exists(bindings.map { b => b._1.id }.toSet.contains(_)) - if (body.dynamicCapture.contains(id.id) || bodyUsesBinding) { - // we need to reify all bindings in this region as allocs using their current value - val reifiedAllocs = bindings.foldLeft(body) { case (acc, (bp, addr)) => - nested { NeutralStmt.Alloc(bp, addr, id.id, acc) } - } - NeutralStmt.Region(id, reifiedAllocs) - } - else stmt - }} - } - } - - object PrettyPrinter extends ParenPrettyPrinter { - - override val defaultIndent = 2 - - def toDoc(s: NeutralStmt): Doc = s match { - case NeutralStmt.Return(result) => - "return" <+> toDoc(result) - case NeutralStmt.If(cond, thn, els) => - "if" <+> parens(toDoc(cond)) <+> toDoc(thn) <+> "else" <+> toDoc(els) - case NeutralStmt.Match(scrutinee, clauses, default) => - "match" <+> parens(toDoc(scrutinee)) <+> braces(hcat(clauses.map { case (id, block) => toDoc(id) <> ":" <+> toDoc(block) })) <> - (if (default.isDefined) "else" <+> toDoc(default.get) else emptyDoc) - case NeutralStmt.Jump(label, targs, vargs, bargs) => - // Format as: l1[T1, T2](r1, r2) - "jump" <+> toDoc(label) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) - case NeutralStmt.App(label, targs, vargs, bargs) => - // Format as: l1[T1, T2](r1, r2) - toDoc(label) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) - - case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => - // Format as: l1[T1, T2](r1, r2) - toDoc(label) <> "." <> toDoc(method) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) - - case NeutralStmt.Reset(prompt, body) => - "reset" <+> braces(toDoc(prompt) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) - - case NeutralStmt.Shift(prompt, capt, k, body) => - "shift" <> parens(toDoc(prompt)) <+> braces(toDoc(k) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) - - case NeutralStmt.Resume(k, body) => - "resume" <> parens(toDoc(k)) <+> toDoc(body) - - case NeutralStmt.Var(id, init, body) => - "var" <+> toDoc(id.id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) - - case NeutralStmt.Put(ref, tpe, cap, value, body) => - toDoc(ref) <+> ":=" <+> toDoc(value) <> line <> toDoc(body.bindings) <> toDoc(body.body) - - case NeutralStmt.Region(id, body) => - "region" <+> toDoc(id) <+> toDoc(body) - - case NeutralStmt.Alloc(id, init, region, body) => - "var" <+> toDoc(id) <+> "in" <+> toDoc(region) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) - - case NeutralStmt.Hole(span) => "hole()" - } - - def toDoc(id: Id): Doc = id.show - - def toDoc(value: Value): Doc = value match { - // case Value.Var(id, tpe) => toDoc(id) - - case Value.Extern(callee, targs, vargs) => - toDoc(callee.id) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) - - case Value.Literal(value, _) => util.show(value) - - case Value.Make(data, tag, targs, vargs) => - "make" <+> toDoc(data) <+> toDoc(tag) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) - - case Value.Box(body, tpe) => - "box" <+> braces(nest(line <> toDoc(body) <> line)) - - case Value.Var(id, tpe) => toDoc(id) - - case Value.Integer(value) => value.show - } - - def toDoc(block: Block): Doc = block match { - case Block(tparams, vparams, bparams, body) => - (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> - parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) - } - - def toDoc(comp: Computation): Doc = comp match { - case Computation.Var(id) => toDoc(id) - case Computation.Def(closure) => toDoc(closure) - case Computation.Continuation(k) => ??? - case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { - hsep(operations.map { case (id, impl) => "def" <+> toDoc(id) <+> "=" <+> toDoc(impl) }, ",") - } - case Computation.BuiltinExtern(id, vmSymbol) => "extern" <+> toDoc(id) <+> "=" <+> vmSymbol - } - def toDoc(closure: Closure): Doc = closure match { - case Closure(label, env) => toDoc(label) <+> "@" <+> brackets(hsep(env.map(toDoc), comma)) - } - - def toDoc(bindings: Bindings): Doc = - hcat(bindings.map { - case (addr, Binding.Let(value)) => "let" <+> toDoc(addr) <+> "=" <+> toDoc(value) <> line - case (addr, Binding.Def(block)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line - case (addr, Binding.Rec(block, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line - case (addr, Binding.Val(stmt)) => "val" <+> toDoc(addr) <+> "=" <+> toDoc(stmt) <> line - case (addr, Binding.Run(callee, targs, vargs, bargs)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> - (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> - parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line - case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line - case (addr, Binding.Get(ref, tpe, cap)) => "let" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line - }) - - def toDoc(block: BasicBlock): Doc = - braces(nest(line <> toDoc(block.bindings) <> toDoc(block.body)) <> line) - - def toDoc(p: ValueParam): Doc = toDoc(p.id) <> ":" <+> toDoc(p.tpe) - def toDoc(p: BlockParam): Doc = braces(toDoc(p.id)) - - def toDoc(t: ValueType): Doc = util.show(t) - def toDoc(t: BlockType): Doc = util.show(t) - - def show(stmt: NeutralStmt): String = pretty(toDoc(stmt), 80).layout - def show(value: Value): String = pretty(toDoc(value), 80).layout - def show(block: Block): String = pretty(toDoc(block), 80).layout - def show(bindings: Bindings): String = pretty(toDoc(bindings), 80).layout - } - -} - -/** - * A new normalizer that is conservative (avoids code bloat) - */ -class NewNormalizer { - - import semantics.* - - // used for potentially recursive definitions - def evaluateRecursive(id: Id, block: core.BlockLit, escaping: Stack)(using env: Env, scope: Scope): Computation = - block match { - case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => - val freshened = Id(id) - - // we keep the params as they are for now... - given localEnv: Env = env - .bindValue(vparams.map(p => p.id -> p.id)) - .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - // Assume that we capture nothing - .bindComputation(id, Computation.Def(Closure(freshened, Nil))) - - val normalizedBlock = scope.local { - Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Unknown)(using localEnv) - }) - } - - val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } - - // Only normalize again if we actually we wrong in our assumption that we capture nothing - // We might run into exponential complexity for nested recursive functions - if (closureParams.isEmpty) { - scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) - Computation.Def(Closure(freshened, Nil)) - } else { - val captures = closureParams.map { p => Computation.Var(p.id): Computation.Var } - given localEnv1: Env = env - .bindValue(vparams.map(p => p.id -> p.id)) - .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - .bindComputation(id, Computation.Def(Closure(freshened, captures))) - - val normalizedBlock1 = Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Unknown)(using localEnv1) - }) - - val tpe: BlockType.Function = block.tpe match { - case _: BlockType.Interface => ??? - case ftpe: BlockType.Function => ftpe - } - scope.defineRecursive( - freshened, - normalizedBlock1.copy(bparams = normalizedBlock1.bparams ++ closureParams), - tpe.copy(cparams = tpe.cparams ++ closureParams.map { p => p.id }), - block.capt - ) - Computation.Def(Closure(freshened, captures)) - } - - } - - // the stack here is not the one this is run in, but the one the definition potentially escapes - def evaluate(block: core.Block, hint: String, escaping: Stack)(using env: Env, scope: Scope): Computation = block match { - case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => - env.lookupComputation(id) - case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => - // we keep the params as they are for now... - given localEnv: Env = env - .bindValue(vparams.map(p => p.id -> p.id)) - .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - - val normalizedBlock = Block(tparams, vparams, bparams, nested { - evaluate(body, Frame.Return, Stack.Unknown) - }) - - val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } - - val f = Id(hint) - scope.define(f, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams)) - Computation.Def(Closure(f, closureParams.map(p => Computation.Var(p.id)))) - - case core.Block.Unbox(pure) => - val addr = evaluate(pure, escaping) - scope.lookupValue(addr) match { - case Some(Value.Box(body, _)) => body - case Some(_) | None => { - val (tpe, capt) = pure.tpe match { - case Boxed(tpe, capt) => (tpe, capt) - case _ => sys error "should not happen" - } - // TODO translate static capture set capt to a dynamic capture set (e.g. {exc} -> {@p_17}) - val unboxAddr = scope.unbox(addr, tpe, capt) - Computation.Var(unboxAddr) - } - } - - // TODO this does not work for recursive objects currently - case core.Block.New(Implementation(interface, operations)) => - val ops = operations.map { - case Operation(name, tparams, cparams, vparams, bparams, body) => - // Check whether the operation is already "just" an eta expansion and then use the identifier... - // no need to create a fresh block literal - val eta: Option[Closure] = - body match { - case Stmt.App(BlockVar(id, _, _), targs, vargs, bargs) => - def sameTargs = targs == tparams.map(t => ValueType.Var(t)) - def sameVargs = vargs == vparams.map(p => ValueVar(p.id, p.tpe)) - def sameBargs = bargs == bparams.map(p => BlockVar(p.id, p.tpe, p.capt)) - def isEta = sameTargs && sameVargs && sameBargs - - env.lookupComputation(id) match { - // TODO what to do with closure environment - case Computation.Def(closure) if isEta => Some(closure) - case _ => None - } - case _ => None - } - - val closure = eta.getOrElse { - evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), name.name.name, escaping) match { - case Computation.Def(closure) => closure - case _ => sys error "Should not happen" - } - } - (name, closure) - } - Computation.New(interface, ops) - } - - def evaluate(expr: Expr, escaping: Stack)(using env: Env, scope: Scope): Addr = expr match { - case Expr.ValueVar(id, annotatedType) => - env.lookupValue(id) - - case core.Expr.Literal(value, annotatedType) => value match { - case As.IntExpr(x) => scope.allocate("x", Value.Integer(x)) - case _ => scope.allocate("x", Value.Literal(value, annotatedType)) - } - - case core.Expr.PureApp(f, targs, vargs) => - val externDef = env.lookupComputation(f.id) - val vargsEvaluated = vargs.map(evaluate(_, escaping)) - val valuesOpt: Option[List[semantics.Value]] = - vargsEvaluated.foldLeft(Option(List.empty[semantics.Value])) { (acc, addr) => - for { - xs <- acc - x <- scope.lookupValue(addr) - } yield x :: xs - }.map(_.reverse) - (valuesOpt, externDef) match { - case (Some(values), Computation.BuiltinExtern(id, name)) if supportedBuiltins(name).isDefinedAt(values) => - val impl = supportedBuiltins(name) - val res = impl(values) - scope.allocate("x", res) - case _ => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) - } - - case core.Expr.Make(data, tag, targs, vargs) => - scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate(_, escaping)))) - - case core.Expr.Box(b, annotatedCapture) => - /* - var counter = 22; - val p : Borrowed[Int] at counter = box new Borrowed[Int] { - def dereference() = counter - }; - counter = counter + 1; - println(p.dereference) - */ - // should capture `counter` but does not since the stack is Stack.Unknown - // (effekt.JavaScriptTests.examples/pos/capture/borrows.effekt (js)) - // TLDR we need to pass an escaping stack to do a proper escape analysis. Stack.Unkown is insufficient - val comp = evaluate(b, "x", escaping) - scope.allocate("x", Value.Box(comp, annotatedCapture)) - } - - // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) - def evaluate(stmt: Stmt, k: Frame, ks: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { - - case Stmt.Return(expr) => - k.ret(ks, evaluate(expr, ks)) - - case Stmt.Val(id, annotatedTpe, binding, body) => - evaluate(binding, k.push(annotatedTpe) { scope => res => k => ks => - given Scope = scope - bind(id, res) { evaluate(body, k, ks) } - }, ks) - - case Stmt.ImpureApp(id, f, targs, vargs, bargs, body) => - assert(bargs.isEmpty) - val addr = scope.run("x", f, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", Stack.Unknown))) - evaluate(body, k, ks)(using env.bindValue(id, addr), scope) - - case Stmt.Let(id, annotatedTpe, binding, body) => - bind(id, evaluate(binding, ks)) { evaluate(body, k, ks) } - - // can be recursive - case Stmt.Def(id, block: core.BlockLit, body) => - bind(id, evaluateRecursive(id, block, ks)) { evaluate(body, k, ks) } - - case Stmt.Def(id, block, body) => - bind(id, evaluate(block, id.name.name, ks)) { evaluate(body, k, ks) } - - case Stmt.App(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => - // TODO also bind type arguments in environment - // TODO substitute cparams??? - val newEnv = env - .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a, ks) }) - .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) - - evaluate(body, k, ks)(using newEnv, scope) - - case Stmt.App(callee, targs, vargs, bargs) => - evaluate(callee, "f", ks) match { - case Computation.Var(id) => - reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", ks))) } - case Computation.Def(Closure(label, environment)) => - val args = vargs.map(evaluate(_, ks)) - /* - try { - prog { - do Eff() - } - } with Eff { ... } - --- - val captures = stack.bound.filter { block.free } - is incorrect as the result is always the empty capture set since Stack.Unkown.bound = Set() - */ - val blockargs = bargs.map(evaluate(_, "f", ks)) - // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation - // compute dynamic captures of the whole statement (App node) - val dynCaptures = blockargs.flatMap(_.dynamicCapture) ++ environment - if (dynCaptures.isEmpty) { - reifyKnown(k, ks) { - NeutralStmt.Jump(label, targs, args, blockargs) - } - } else { - reify(k, ks) { - NeutralStmt.Jump(label, targs, args, blockargs ++ environment) - } - } - case _: (Computation.New | Computation.Continuation | Computation.BuiltinExtern) => sys error "Should not happen" - } - - // case Stmt.Invoke(New) - - case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => - val escapingStack = Stack.Unknown - evaluate(callee, "o", escapingStack) match { - case Computation.Var(id) => - reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } - case Computation.New(interface, operations) => - operations.collectFirst { case (id, Closure(label, environment)) if id == method => - reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } - }.get - case _: (Computation.Def | Computation.Continuation | Computation.BuiltinExtern) => sys error s"Should not happen" - } - - case Stmt.If(cond, thn, els) => - val sc = evaluate(cond, ks) - scope.lookupValue(sc) match { - case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) - case Some(Value.Literal(false, _)) => evaluate(els, k, ks) - case _ => - // joinpoint(k, ks, List(thn, els)) { (A, Frame, Stack) => (NeutralStmt, Variables) } { case thn1 :: els1 :: Nil => NeutralStmt.If(sc, thn1, els1) } - joinpoint(k, ks) { (k, ks) => - NeutralStmt.If(sc, nested { - evaluate(thn, k, ks) - }, nested { - evaluate(els, k, ks) - }) - } - } - case Stmt.Match(scrutinee, clauses, default) => - val sc = evaluate(scrutinee, ks) - scope.lookupValue(sc) match { - case Some(Value.Make(data, tag, targs, vargs)) => - // TODO substitute types (or bind them in the env)! - clauses.collectFirst { - case (tpe, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => - bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k, ks) } - }.getOrElse { - evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) - } - // case _ if (clauses.size + default.size) <= 1 => - // NeutralStmt.Match(sc, - // clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => - // given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - // val block = Block(tparams, vparams, bparams, nested { - // evaluate(body, k, ks) - // }) - // (id, block) - // }, - // default.map { stmt => nested { evaluate(stmt, k, ks) } }) - case _ => - def neutralMatch(k: Frame, ks: Stack) = - NeutralStmt.Match(sc, - // This is ALMOST like evaluate(BlockLit), but keeps the current continuation - clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => - given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) - val block = Block(tparams, vparams, bparams, nested { scope ?=> - // here we now know that our scrutinee sc has the shape id(vparams, ...) - - val datatype = scrutinee.tpe match { - case tpe @ ValueType.Data(name, targs) => tpe - case tpe => sys error s"Should not happen: pattern matching on a non-datatype: ${tpe}" - } - val eta = Value.Make(datatype, id, tparams.map(t => ValueType.Var(t)), vparams.map(p => p.id)) - scope.bindings = scope.bindings.updated(sc, Binding.Let(eta)) - evaluate(body, k, ks) - }) - (id, block) - }, - default.map { stmt => nested { evaluate(stmt, k, ks) } }) - // linear usage of the continuation: do not create a joinpoint. - // This is a simple optimization for record access since r.x is always desugared into a match - if (default.size + clauses.size > 1) { - joinpoint(k, ks) { (k, ks) => neutralMatch(k, ks) } - } else { - neutralMatch(k, ks) - } - } - - case Stmt.Hole(span) => NeutralStmt.Hole(span) - - // State - case Stmt.Region(BlockLit(Nil, List(capture), Nil, List(cap), body)) => - given Env = env.bindComputation(cap.id, Computation.Var(cap.id)) - evaluate(body, Frame.Return, Stack.Region(cap, Map.empty, k, ks)) - case Stmt.Region(_) => ??? - case Stmt.Alloc(id, init, region, body) => - val addr = evaluate(init, ks) - val bp = BlockParam(id, Type.TState(init.tpe), Set(region)) - alloc(bp, region, addr, ks) match { - case Some(ks1) => evaluate(body, k, ks1) - case None => NeutralStmt.Alloc(bp, addr, region, nested { evaluate(body, k, ks) }) - } - - case Stmt.Var(ref, init, capture, body) => - val addr = evaluate(init, ks) - // TODO Var means unknown. Here we have contrary: it is a statically (!) known variable - bind(ref, Computation.Var(ref)) { - evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) - } - case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => - get(ref, ks) match { - case Some(addr) => bind(id, addr) { evaluate(body, k, ks) } - case None => bind(id, scope.allocateGet(ref, annotatedTpe, annotatedCapt)) { evaluate(body, k, ks) } - } - case Stmt.Put(ref, annotatedCapt, value, body) => - val addr = evaluate(value, ks) - put(ref, addr, ks) match { - case Some(stack) => evaluate(body, k, stack) - case None => - NeutralStmt.Put(ref, value.tpe, annotatedCapt, addr, nested { evaluate(body, k, ks) }) - } - - // Control Effects - case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => - val p = env.lookupComputation(prompt.id) match { - case Computation.Var(id) => id - case _ => ??? - } - - if (ks.bound.exists { other => other.id == p }) { - val (cont, frame, stack) = shift(p, k, ks) - given Env = env.bindComputation(k2.id -> Computation.Continuation(cont) :: Nil) - evaluate(body, frame, stack) - } else { - val neutralBody = { - given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) - nested { - evaluate(body, Frame.Return, Stack.Unknown) - } - } - assert(Set(cparam) == k2.capt, "At least for now these need to be the same") - reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } - } - case Stmt.Shift(_, _) => ??? - - case Stmt.Reset(core.Block.BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => - val p = Id(prompt.id) - // TODO is Var correct here?? Probably needs to be a new computation value... - given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) - evaluate(body, Frame.Return, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), k, ks)) - - case Stmt.Reset(_) => ??? - case Stmt.Resume(k2, body) => - env.lookupComputation(k2.id) match { - case Computation.Var(r) => - reify(k, ks) { - NeutralStmt.Resume(r, nested { - evaluate(body, Frame.Return, Stack.Unknown) - }) - } - case Computation.Continuation(k3) => - val (k4, ks4) = resume(k3, k, ks) - evaluate(body, k4, ks4) - case _ => ??? - } - } - - def run(mod: ModuleDecl): ModuleDecl = { - //util.trace(mod) - // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) - val externTypes = mod.externs.collect { case d: Extern.Def => - d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map { _.tpe }, d.bparams.map { bp => bp.tpe }, d.ret), d.annotatedCapture) - } - - val (builtinExterns, otherExterns) = mod.externs.collect { case d: Extern.Def => d }.partition { - case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => - val builtinName = vmBody.contents.strings.head - supportedBuiltins.contains(builtinName) - case _ => false - } - - val builtinNameToBlockVar: Map[String, BlockVar] = builtinExterns.collect { - case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => - val builtinName = vmBody.contents.strings.head - val bv: BlockVar = BlockVar(id, BlockType.Function(tps, cps, vps.map { _.tpe }, bps.map { bp => bp.tpe }, ret), capt) - builtinName -> bv - }.toMap - - val toplevelEnv = Env.empty - // user-defined functions - .bindComputation(mod.definitions.collect { - case Toplevel.Def(id, b) => id -> (b match { - case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => Computation.Def(Closure(id, Nil)) - case core.Block.BlockVar(idd, annotatedTpe, annotatedCapt) => Computation.Var(id) - case core.Block.Unbox(pure) => Computation.Var(id) - case core.Block.New(impl) => Computation.Var(id) - }) - }) - // user-defined values - .bindValue(mod.definitions.collect { - case Toplevel.Val(id, _, _) => id -> id - }) - // async extern functions - .bindComputation(otherExterns.map(defn => defn.id -> Computation.Var(defn.id))) - // pure extern functions - .bindComputation(builtinExterns.flatMap(defn => defn.vmBody.map(vmBody => defn.id -> Computation.BuiltinExtern(defn.id, vmBody.contents.strings.head)))) - - val typingContext = TypingContext( - mod.definitions.collect { - case Toplevel.Val(id, tpe, _) => id -> tpe - }.toMap, - mod.definitions.collect { - case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) - }.toMap ++ externTypes, - builtinNameToBlockVar - ) - - val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) - mod.copy(definitions = newDefinitions) - } - - val showDebugInfo = true - inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () - - def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { - case Toplevel.Def(id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => - debug(s"------- ${util.show(id)} -------") - debug(util.show(body)) - - val scope = Scope.empty - val localEnv: Env = env - .bindValue(vparams.map(p => p.id -> scope.allocate("p", Value.Var(p.id, p.tpe)))) - .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) - - val result = evaluate(body, Frame.Return, Stack.Empty)(using localEnv, scope) - - debug(s"----------normalized-----------") - val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) - debug(PrettyPrinter.show(block)) - - debug(s"----------embedded-----------") - val embedded = embedBlockLit(block) - debug(util.show(embedded)) - - Toplevel.Def(id, embedded) - case other => other - } - - case class TypingContext(values: Map[Addr, ValueType], blocks: Map[Label, (BlockType, Captures)], builtinBlockVars: Map[String, BlockVar]) { - def bind(id: Id, tpe: ValueType): TypingContext = this.copy(values = values + (id -> tpe)) - def bind(id: Id, tpe: BlockType, capt: Captures): TypingContext = this.copy(blocks = blocks + (id -> (tpe, capt))) - def bindValues(vparams: List[ValueParam]): TypingContext = this.copy(values = values ++ vparams.map(p => p.id -> p.tpe)) - def lookupValue(id: Id): ValueType = values.getOrElse(id, sys.error(s"Unknown value: ${util.show(id)}")) - def bindComputations(bparams: List[BlockParam]): TypingContext = this.copy(blocks = blocks ++ bparams.map(p => p.id -> (p.tpe, p.capt))) - def bindComputation(bparam: BlockParam): TypingContext = this.copy(blocks = blocks + (bparam.id -> (bparam.tpe, bparam.capt))) - } - - def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { - case NeutralStmt.Return(result) => - Stmt.Return(embedExpr(result)) - case NeutralStmt.Jump(label, targs, vargs, bargs) => - Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) - case NeutralStmt.App(label, targs, vargs, bargs) => - Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) - case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => - Stmt.Invoke(embedBlockVar(label), method, tpe, targs, vargs.map(embedExpr), bargs.map(embedBlock)) - case NeutralStmt.If(cond, thn, els) => - Stmt.If(embedExpr(cond), embedStmt(thn), embedStmt(els)) - case NeutralStmt.Match(scrutinee, clauses, default) => - Stmt.Match(embedExpr(scrutinee), - clauses.map { case (id, block) => id -> embedBlockLit(block) }, - default.map(embedStmt)) - case NeutralStmt.Reset(prompt, body) => - val capture = prompt.capt match { - case set if set.size == 1 => set.head - case _ => sys error "Prompt needs to have a single capture" - } - Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputation(prompt)))) - case NeutralStmt.Shift(prompt, capt, k, body) => - Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputation(k)))) - case NeutralStmt.Resume(k, body) => - Stmt.Resume(embedBlockVar(k), embedStmt(body)) - case NeutralStmt.Var(blockParam, init, body) => - val capt = blockParam.capt match { - case cs if cs.size == 1 => cs.head - case _ => sys error "Variable needs to have a single capture" - } - Stmt.Var(blockParam.id, embedExpr(init), capt, embedStmt(body)(using G.bind(blockParam.id, blockParam.tpe, blockParam.capt))) - case NeutralStmt.Put(ref, annotatedTpe, annotatedCapt, value, body) => - Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) - case NeutralStmt.Region(id, body) => - Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputation(id)))) - case NeutralStmt.Alloc(blockparam, init, region, body) => - Stmt.Alloc(blockparam.id, embedExpr(init), region, embedStmt(body)(using G.bind(blockparam.id, blockparam.tpe, blockparam.capt))) - case NeutralStmt.Hole(span) => - Stmt.Hole(span) - } - - def embedStmt(basicBlock: BasicBlock)(using G: TypingContext): core.Stmt = basicBlock match { - case BasicBlock(bindings, stmt) => - bindings.foldRight((G: TypingContext) => embedStmt(stmt)(using G)) { - case ((id, Binding.Let(value)), rest) => G => - val coreExpr = embedExpr(value)(using G) - // TODO why do we even have this type in core, if we always infer it? - Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) - case ((id, Binding.Def(block)), rest) => G => - val coreBlock = embedBlock(block)(using G) - Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) - case ((id, Binding.Rec(block, tpe, capt)), rest) => G => - val coreBlock = embedBlock(block)(using G.bind(id, tpe, capt)) - Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) - case ((id, Binding.Val(stmt)), rest) => G => - val coreStmt = embedStmt(stmt)(using G) - Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) - case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => - val vargs1 = vargs.map(arg => embedExpr(arg)(using G)) - val bargs1 = bargs.map(arg => embedBlock(arg)(using G)) - val tpe = Type.bindingType(callee, targs, vargs1, bargs1) - core.ImpureApp(id, callee, targs, vargs1, bargs1, rest(G.bind(id, tpe))) - case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => - val pureValue = embedExpr(addr)(using G) - Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) - case ((id, Binding.Get(ref, tpe, cap)), rest) => G => - Stmt.Get(id, tpe, ref, cap, rest(G.bind(id, tpe)) ) - }(G) - } - - def embedExpr(value: Value)(using cx: TypingContext): core.Expr = value match { - case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) - case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) - case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) - case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) - case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) - case Value.Integer(value) => theories.Integers.reify(value, cx.builtinBlockVars) - } - - def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) - - def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { - case Computation.Var(id) => - embedBlockVar(id) - case Computation.Def(Closure(label, Nil)) => - embedBlockVar(label) - case Computation.Def(closure) => - etaExpandToBlockLit(closure) - case Computation.Continuation(k) => ??? - case Computation.BuiltinExtern(_, _) => ??? - case Computation.New(interface, operations) => - val ops = operations.map { etaExpandToOperation.tupled } - core.Block.New(Implementation(interface, ops)) - } - - /** - * Embed `Computation.Def` to a `core.BlockLit` - * This eta-expands the block var that stands for the `Computation.Def` to a full block literal - * so that we can supply the correct capture arguments from the environment. - */ - def etaExpandToBlockLit(closure: Closure)(using G: TypingContext): core.BlockLit = { - val Closure(label, environment) = closure - val blockvar = embedBlockVar(label) - G.blocks(label) match { - // TODO why is `captures` unused? - case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => - val vps = vparams.map { p => core.ValueParam(Id("x"), p) } - val vargs = vps.map { vp => core.Expr.ValueVar(vp.id, vp.tpe) } - - // this uses the invariant that we _append_ all environment captures to the bparams - val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) - val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) - val origBps = origBparams.zip(origCapts).map { case (bp, c) => core.BlockParam(Id("f"), bp, Set(c)) } - val origBargs = origBps.map { bp => core.BlockVar(bp.id, bp.tpe, bp.capt) } - val synthBargs = environment.zip(synthBparams).zip(synthCapts).map { - case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) - } - val bargs = origBargs ++ synthBargs - - val targs = tparams.map { core.ValueType.Var.apply } - - core.Block.BlockLit( - tparams, - origCapts, - vps, - origBps, - Stmt.App( - blockvar, - targs, - vargs, - bargs - ) - ) - case _ => sys.error("Unexpected block type for a closure") - } - } - - /** - * Embed an operation as part of a `Computation.New`. - * This eta-expands the block var that stands for the operation body to a full operation - * so that we can supply the correct capture arguments from the environment. - */ - def etaExpandToOperation(id: Id, closure: Closure)(using G: TypingContext): core.Operation = { - val Closure(label, environment) = closure - G.blocks(label) match { - case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => - val tparams2 = tparams.map(t => Id(t)) - // TODO if we freshen cparams, then we also need to substitute them in the result AND the parameters - val cparams2 = cparams //.map(c => Id(c)) - val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) - val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } - // In the following section, we create a new instance of the interface. - // All operation bodies were lifted to block literals in an earlier stage. - // While doing so, their block parameters (bparams) were concatenated with their capture parameters (cparams). - // When we embed back to core, we need to "eta-expand" the operation body to supply the correct captures from the environment. - // To see why this "eta-expansion" is necessary to achieve this, consider the following example: - // ```scala - // effect Eff(): Unit - // def use = { do Eff() } - // def main() = { - // val r = try { - // use() - // } with Eff { - // resume(()) - // } - // } - // ``` - // the handler body normalizes to the following: - // ```scala - // reset {{p} => - // jump use(){new Eff {def Eff = Eff @ [p]}} - // } - // ``` - // where - // ``` - // def Eff = (){p} { ... } - // ``` - // In particular, the prompt `p` needs to be passed to the lifted operation body. - // ``` - val (origBparams, synthBparams) = bparams2.splitAt(bparams2.length - environment.length) - val bargs = - // TODO: Fix captures - origBparams.map { case bp => BlockVar(bp.id, bp.tpe, Set()) } ++ - synthBparams.zip(environment).map { - // TODO: Fix captures - case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) - } - - core.Operation( - id, - tparams2, - cparams.take(cparams.length - environment.length), - vparams2, - origBparams, - Stmt.App( - embedBlockVar(label), - tparams2.map(ValueType.Var.apply), - vparams2.map(p => ValueVar(p.id, p.tpe)), - bargs - ) - ) - case _ => sys error "Unexpected block type" - } - } - - def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { - case Block(tparams, vparams, bparams, b) => - val cparams = bparams.map { - case BlockParam(id, tpe, captures) => - assert(captures.size == 1) - captures.head - } - core.Block.BlockLit(tparams, cparams, vparams, bparams, - embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) - } - - def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = embedBlock(block).asInstanceOf[core.BlockLit] - - def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = - val (tpe, capt) = G.blocks.getOrElse(label, sys error s"Unknown block: ${util.show(label)}. ${G.blocks.keys.map(util.show).mkString(", ")}") - core.BlockVar(label, tpe, capt) -} - -type ~>[-A, +B] = PartialFunction[A, B] - -type BuiltinImpl = List[semantics.Value] ~> semantics.Value - -def builtin(name: String)(impl: List[semantics.Value] ~> semantics.Value): (String, BuiltinImpl) = name -> impl - -type Builtins = Map[String, BuiltinImpl] - -lazy val integers: Builtins = Map( - // Integer arithmetic operations with symbolic simplification support - // ---------- - builtin("effekt::infixAdd(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.add(x, y)) - }, - builtin("effekt::infixSub(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.sub(x, y)) - }, - builtin("effekt::infixMul(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.mul(x, y)) - }, - // Integer arithmetic operations only evaluated for literals - // ---------- - builtin("effekt::infixDiv(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x / y)) - }, - builtin("effekt::mod(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x % y)) - }, - builtin("effekt::bitwiseShl(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x << y)) - }, - builtin("effekt::bitwiseShr(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x >> y)) - }, - builtin("effekt::bitwiseAnd(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x & y)) - }, - builtin("effekt::bitwiseOr(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x | y)) - }, - builtin("effekt::bitwiseXor(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x ^ y)) - }, - // Comparison - // ---------- - builtin("effekt::infixEq(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x == y, Type.TBoolean) - }, - builtin("effekt::infixNeq(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x != y, Type.TBoolean) - }, - builtin("effekt::infixLt(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x < y, Type.TBoolean) - }, - builtin("effekt::infixGt(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x > y, Type.TBoolean) - }, - builtin("effekt::infixLte(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x <= y, Type.TBoolean) - }, - builtin("effekt::infixGte(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x >= y, Type.TBoolean) - }, - // Conversion - // ---------- - builtin("effekt::toDouble(Int)") { - case As.Int(x) :: Nil => semantics.Value.Literal(x.toDouble, Type.TDouble) - }, - - builtin("effekt::show(Int)") { - case As.Int(n) :: Nil => semantics.Value.Literal(n.toString, Type.TString) - }, -) - -lazy val supportedBuiltins: Builtins = integers - -protected object As { - object Int { - def unapply(v: semantics.Value): Option[scala.Long] = v match { - case semantics.Value.Literal(value: scala.Long, _) => Some(value) - case semantics.Value.Literal(value: scala.Int, _) => Some(value.toLong) - case semantics.Value.Literal(value: java.lang.Integer, _) => Some(value.toLong) - case _ => None - } - } - - object IntExpr { - def unapply(v: semantics.Value): Option[theories.Integers.Integer] = v match { - // Integer literals not yet embedded into the theory of integers - case semantics.Value.Literal(value: scala.Long, _) => Some(theories.Integers.embed(value)) - case semantics.Value.Literal(value: scala.Int, _) => Some(theories.Integers.embed(value.toLong)) - case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.Integers.embed(value.toLong)) - // Variables of type integer - case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.Integers.embed(id)) - // Already embedded integers - case semantics.Value.Integer(value) => Some(value) - case _ => None - } - } -} \ No newline at end of file diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index d54f167e4..98526c9a0 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -5,6 +5,7 @@ package optimizer import effekt.PhaseResult.CoreTransformed import effekt.context.Context import effekt.core.optimizer.Usage.{ Once, Recursive } +import effekt.core.optimizer.normalizer.NewNormalizer import kiama.util.Source object Optimizer extends Phase[CoreTransformed, CoreTransformed] { diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala new file mode 100644 index 000000000..f2535b855 --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala @@ -0,0 +1,764 @@ +package effekt +package core +package optimizer +package normalizer + +import effekt.core.ValueType.Boxed + +// TODO +// - change story of how inlining is implemented. We need to also support toplevel functions that potentially +// inline each other. Do we need to sort them topologically? How do we deal with (mutually) recursive definitions? +// +// +// plan: only introduce parameters for free things inside a block that are bound in the **stack** +// that is in +// +// only abstract over p, but not n: +// +// def outer(n: Int) = +// def foo(p) = shift(p) { ... n ... } +// reset { p => +// ... +// } +// +// Same actually for stack allocated mutable state, we should abstract over those (but only those) +// and keep the function in its original location. +// This means we only need to abstract over blocks, no values, no types. +// +// TODO Region desugaring +// region r { +// reset { p => +// var x in r = 42 +// x = !x + 1 +// println(!x) +// } +// } +// +// reset { r => +// reset { p => +// //var x in r = 42 +// shift(r) { k => +// var x = 42 +// resume(k) { +// x = !x + 1 +// println(!x) +// } +// } +// } +// } +// +// - Typeability preservation: {r: Region} becomes {r: Prompt[T]} +// [[ def f() {r: Region} = s ]] = def f[T]() {r: Prompt[T]} = ... +// - Continuation capture is _not_ constant time in JS backend, so we expect a (drastic) slowdown when desugaring + +/** + * A new normalizer that is conservative (avoids code bloat) + */ +class NewNormalizer { + + import semantics.* + + // used for potentially recursive definitions + def evaluateRecursive(id: Id, block: core.BlockLit, escaping: Stack)(using env: Env, scope: Scope): Computation = + block match { + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + val freshened = Id(id) + + // we keep the params as they are for now... + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + // Assume that we capture nothing + .bindComputation(id, Computation.Def(Closure(freshened, Nil))) + + val normalizedBlock = scope.local { + Block(tparams, vparams, bparams, nested { + evaluate(body, Frame.Return, Stack.Unknown)(using localEnv) + }) + } + + val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } + + // Only normalize again if we actually we wrong in our assumption that we capture nothing + // We might run into exponential complexity for nested recursive functions + if (closureParams.isEmpty) { + scope.defineRecursive(freshened, normalizedBlock.copy(bparams = normalizedBlock.bparams), block.tpe, block.capt) + Computation.Def(Closure(freshened, Nil)) + } else { + val captures = closureParams.map { p => Computation.Var(p.id): Computation.Var } + given localEnv1: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + .bindComputation(id, Computation.Def(Closure(freshened, captures))) + + val normalizedBlock1 = Block(tparams, vparams, bparams, nested { + evaluate(body, Frame.Return, Stack.Unknown)(using localEnv1) + }) + + val tpe: BlockType.Function = block.tpe match { + case _: BlockType.Interface => ??? + case ftpe: BlockType.Function => ftpe + } + scope.defineRecursive( + freshened, + normalizedBlock1.copy(bparams = normalizedBlock1.bparams ++ closureParams), + tpe.copy(cparams = tpe.cparams ++ closureParams.map { p => p.id }), + block.capt + ) + Computation.Def(Closure(freshened, captures)) + } + + } + + // the stack here is not the one this is run in, but the one the definition potentially escapes + def evaluate(block: core.Block, hint: String, escaping: Stack)(using env: Env, scope: Scope): Computation = block match { + case core.Block.BlockVar(id, annotatedTpe, annotatedCapt) => + env.lookupComputation(id) + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => + // we keep the params as they are for now... + given localEnv: Env = env + .bindValue(vparams.map(p => p.id -> p.id)) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + + val normalizedBlock = Block(tparams, vparams, bparams, nested { + evaluate(body, Frame.Return, Stack.Unknown) + }) + + val closureParams = escaping.bound.filter { p => normalizedBlock.dynamicCapture contains p.id } + + val f = Id(hint) + scope.define(f, normalizedBlock.copy(bparams = normalizedBlock.bparams ++ closureParams)) + Computation.Def(Closure(f, closureParams.map(p => Computation.Var(p.id)))) + + case core.Block.Unbox(pure) => + val addr = evaluate(pure, escaping) + scope.lookupValue(addr) match { + case Some(Value.Box(body, _)) => body + case Some(_) | None => { + val (tpe, capt) = pure.tpe match { + case Boxed(tpe, capt) => (tpe, capt) + case _ => sys error "should not happen" + } + // TODO translate static capture set capt to a dynamic capture set (e.g. {exc} -> {@p_17}) + val unboxAddr = scope.unbox(addr, tpe, capt) + Computation.Var(unboxAddr) + } + } + + // TODO this does not work for recursive objects currently + case core.Block.New(Implementation(interface, operations)) => + val ops = operations.map { + case Operation(name, tparams, cparams, vparams, bparams, body) => + // Check whether the operation is already "just" an eta expansion and then use the identifier... + // no need to create a fresh block literal + val eta: Option[Closure] = + body match { + case Stmt.App(BlockVar(id, _, _), targs, vargs, bargs) => + def sameTargs = targs == tparams.map(t => ValueType.Var(t)) + def sameVargs = vargs == vparams.map(p => ValueVar(p.id, p.tpe)) + def sameBargs = bargs == bparams.map(p => BlockVar(p.id, p.tpe, p.capt)) + def isEta = sameTargs && sameVargs && sameBargs + + env.lookupComputation(id) match { + // TODO what to do with closure environment + case Computation.Def(closure) if isEta => Some(closure) + case _ => None + } + case _ => None + } + + val closure = eta.getOrElse { + evaluate(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), name.name.name, escaping) match { + case Computation.Def(closure) => closure + case _ => sys error "Should not happen" + } + } + (name, closure) + } + Computation.New(interface, ops) + } + + def evaluate(expr: Expr, escaping: Stack)(using env: Env, scope: Scope): Addr = expr match { + case Expr.ValueVar(id, annotatedType) => + env.lookupValue(id) + + case core.Expr.Literal(value, annotatedType) => value match { + case As.IntExpr(x) => scope.allocate("x", Value.Integer(x)) + case _ => scope.allocate("x", Value.Literal(value, annotatedType)) + } + + case core.Expr.PureApp(f, targs, vargs) => + val externDef = env.lookupComputation(f.id) + val vargsEvaluated = vargs.map(evaluate(_, escaping)) + val valuesOpt: Option[List[semantics.Value]] = + vargsEvaluated.foldLeft(Option(List.empty[semantics.Value])) { (acc, addr) => + for { + xs <- acc + x <- scope.lookupValue(addr) + } yield x :: xs + }.map(_.reverse) + (valuesOpt, externDef) match { + case (Some(values), Computation.BuiltinExtern(id, name)) if supportedBuiltins(name).isDefinedAt(values) => + val impl = supportedBuiltins(name) + val res = impl(values) + scope.allocate("x", res) + case _ => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) + } + + case core.Expr.Make(data, tag, targs, vargs) => + scope.allocate("x", Value.Make(data, tag, targs, vargs.map(evaluate(_, escaping)))) + + case core.Expr.Box(b, annotatedCapture) => + /* + var counter = 22; + val p : Borrowed[Int] at counter = box new Borrowed[Int] { + def dereference() = counter + }; + counter = counter + 1; + println(p.dereference) + */ + // should capture `counter` but does not since the stack is Stack.Unknown + // (effekt.JavaScriptTests.examples/pos/capture/borrows.effekt (js)) + // TLDR we need to pass an escaping stack to do a proper escape analysis. Stack.Unkown is insufficient + val comp = evaluate(b, "x", escaping) + scope.allocate("x", Value.Box(comp, annotatedCapture)) + } + + // TODO make evaluate(stmt) return BasicBlock (won't work for shift or reset, though) + def evaluate(stmt: Stmt, k: Frame, ks: Stack)(using env: Env, scope: Scope): NeutralStmt = stmt match { + + case Stmt.Return(expr) => + k.ret(ks, evaluate(expr, ks)) + + case Stmt.Val(id, annotatedTpe, binding, body) => + evaluate(binding, k.push(annotatedTpe) { scope => res => k => ks => + given Scope = scope + bind(id, res) { evaluate(body, k, ks) } + }, ks) + + case Stmt.ImpureApp(id, f, targs, vargs, bargs, body) => + assert(bargs.isEmpty) + val addr = scope.run("x", f, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", Stack.Unknown))) + evaluate(body, k, ks)(using env.bindValue(id, addr), scope) + + case Stmt.Let(id, annotatedTpe, binding, body) => + bind(id, evaluate(binding, ks)) { evaluate(body, k, ks) } + + // can be recursive + case Stmt.Def(id, block: core.BlockLit, body) => + bind(id, evaluateRecursive(id, block, ks)) { evaluate(body, k, ks) } + + case Stmt.Def(id, block, body) => + bind(id, evaluate(block, id.name.name, ks)) { evaluate(body, k, ks) } + + case Stmt.App(core.Block.BlockLit(tparams, cparams, vparams, bparams, body), targs, vargs, bargs) => + // TODO also bind type arguments in environment + // TODO substitute cparams??? + val newEnv = env + .bindValue(vparams.zip(vargs).map { case (p, a) => p.id -> evaluate(a, ks) }) + .bindComputation(bparams.zip(bargs).map { case (p, a) => p.id -> evaluate(a, "f", ks) }) + + evaluate(body, k, ks)(using newEnv, scope) + + case Stmt.App(callee, targs, vargs, bargs) => + evaluate(callee, "f", ks) match { + case Computation.Var(id) => + reify(k, ks) { NeutralStmt.App(id, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", ks))) } + case Computation.Def(Closure(label, environment)) => + val args = vargs.map(evaluate(_, ks)) + /* + try { + prog { + do Eff() + } + } with Eff { ... } + --- + val captures = stack.bound.filter { block.free } + is incorrect as the result is always the empty capture set since Stack.Unkown.bound = Set() + */ + val blockargs = bargs.map(evaluate(_, "f", ks)) + // if stmt doesn't capture anything, it can not make any changes to the stack (ks) and we don't have to pretend it is unknown as an over-approximation + // compute dynamic captures of the whole statement (App node) + val dynCaptures = blockargs.flatMap(_.dynamicCapture) ++ environment + if (dynCaptures.isEmpty) { + reifyKnown(k, ks) { + NeutralStmt.Jump(label, targs, args, blockargs) + } + } else { + reify(k, ks) { + NeutralStmt.Jump(label, targs, args, blockargs ++ environment) + } + } + case _: (Computation.New | Computation.Continuation | Computation.BuiltinExtern) => sys error "Should not happen" + } + + // case Stmt.Invoke(New) + + case Stmt.Invoke(callee, method, methodTpe, targs, vargs, bargs) => + val escapingStack = Stack.Unknown + evaluate(callee, "o", escapingStack) match { + case Computation.Var(id) => + reify(k, ks) { NeutralStmt.Invoke(id, method, methodTpe, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack))) } + case Computation.New(interface, operations) => + operations.collectFirst { case (id, Closure(label, environment)) if id == method => + reify(k, ks) { NeutralStmt.Jump(label, targs, vargs.map(evaluate(_, ks)), bargs.map(evaluate(_, "f", escapingStack)) ++ environment) } + }.get + case _: (Computation.Def | Computation.Continuation | Computation.BuiltinExtern) => sys error s"Should not happen" + } + + case Stmt.If(cond, thn, els) => + val sc = evaluate(cond, ks) + scope.lookupValue(sc) match { + case Some(Value.Literal(true, _)) => evaluate(thn, k, ks) + case Some(Value.Literal(false, _)) => evaluate(els, k, ks) + case _ => + // joinpoint(k, ks, List(thn, els)) { (A, Frame, Stack) => (NeutralStmt, Variables) } { case thn1 :: els1 :: Nil => NeutralStmt.If(sc, thn1, els1) } + joinpoint(k, ks) { (k, ks) => + NeutralStmt.If(sc, nested { + evaluate(thn, k, ks) + }, nested { + evaluate(els, k, ks) + }) + } + } + case Stmt.Match(scrutinee, clauses, default) => + val sc = evaluate(scrutinee, ks) + scope.lookupValue(sc) match { + case Some(Value.Make(data, tag, targs, vargs)) => + // TODO substitute types (or bind them in the env)! + clauses.collectFirst { + case (tpe, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) if tpe == tag => + bind(vparams.map(_.id).zip(vargs)) { evaluate(body, k, ks) } + }.getOrElse { + evaluate(default.getOrElse { sys.error("Non-exhaustive pattern match.") }, k, ks) + } + // case _ if (clauses.size + default.size) <= 1 => + // NeutralStmt.Match(sc, + // clauses.map { case (id, BlockLit(tparams, cparams, vparams, bparams, body)) => + // given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) + // val block = Block(tparams, vparams, bparams, nested { + // evaluate(body, k, ks) + // }) + // (id, block) + // }, + // default.map { stmt => nested { evaluate(stmt, k, ks) } }) + case _ => + def neutralMatch(k: Frame, ks: Stack) = + NeutralStmt.Match(sc, + // This is ALMOST like evaluate(BlockLit), but keeps the current continuation + clauses.map { case (id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => + given localEnv: Env = env.bindValue(vparams.map(p => p.id -> p.id)) + val block = Block(tparams, vparams, bparams, nested { scope ?=> + // here we now know that our scrutinee sc has the shape id(vparams, ...) + + val datatype = scrutinee.tpe match { + case tpe @ ValueType.Data(name, targs) => tpe + case tpe => sys error s"Should not happen: pattern matching on a non-datatype: ${tpe}" + } + val eta = Value.Make(datatype, id, tparams.map(t => ValueType.Var(t)), vparams.map(p => p.id)) + scope.bindings = scope.bindings.updated(sc, Binding.Let(eta)) + evaluate(body, k, ks) + }) + (id, block) + }, + default.map { stmt => nested { evaluate(stmt, k, ks) } }) + // linear usage of the continuation: do not create a joinpoint. + // This is a simple optimization for record access since r.x is always desugared into a match + if (default.size + clauses.size > 1) { + joinpoint(k, ks) { (k, ks) => neutralMatch(k, ks) } + } else { + neutralMatch(k, ks) + } + } + + case Stmt.Hole(span) => NeutralStmt.Hole(span) + + // State + case Stmt.Region(BlockLit(Nil, List(capture), Nil, List(cap), body)) => + given Env = env.bindComputation(cap.id, Computation.Var(cap.id)) + evaluate(body, Frame.Return, Stack.Region(cap, Map.empty, k, ks)) + case Stmt.Region(_) => ??? + case Stmt.Alloc(id, init, region, body) => + val addr = evaluate(init, ks) + val bp = BlockParam(id, Type.TState(init.tpe), Set(region)) + alloc(bp, region, addr, ks) match { + case Some(ks1) => evaluate(body, k, ks1) + case None => NeutralStmt.Alloc(bp, addr, region, nested { evaluate(body, k, ks) }) + } + + case Stmt.Var(ref, init, capture, body) => + val addr = evaluate(init, ks) + // TODO Var means unknown. Here we have contrary: it is a statically (!) known variable + bind(ref, Computation.Var(ref)) { + evaluate(body, Frame.Return, Stack.Var(BlockParam(ref, Type.TState(init.tpe), Set(capture)), addr, k, ks)) + } + case Stmt.Get(id, annotatedTpe, ref, annotatedCapt, body) => + get(ref, ks) match { + case Some(addr) => bind(id, addr) { evaluate(body, k, ks) } + case None => bind(id, scope.allocateGet(ref, annotatedTpe, annotatedCapt)) { evaluate(body, k, ks) } + } + case Stmt.Put(ref, annotatedCapt, value, body) => + val addr = evaluate(value, ks) + put(ref, addr, ks) match { + case Some(stack) => evaluate(body, k, stack) + case None => + NeutralStmt.Put(ref, value.tpe, annotatedCapt, addr, nested { evaluate(body, k, ks) }) + } + + // Control Effects + case Stmt.Shift(prompt, core.Block.BlockLit(Nil, cparam :: Nil, Nil, k2 :: Nil, body)) => + val p = env.lookupComputation(prompt.id) match { + case Computation.Var(id) => id + case _ => ??? + } + + if (ks.bound.exists { other => other.id == p }) { + val (cont, frame, stack) = shift(p, k, ks) + given Env = env.bindComputation(k2.id -> Computation.Continuation(cont) :: Nil) + evaluate(body, frame, stack) + } else { + val neutralBody = { + given Env = env.bindComputation(k2.id -> Computation.Var(k2.id) :: Nil) + nested { + evaluate(body, Frame.Return, Stack.Unknown) + } + } + assert(Set(cparam) == k2.capt, "At least for now these need to be the same") + reify(k, ks) { NeutralStmt.Shift(p, cparam, k2, neutralBody) } + } + case Stmt.Shift(_, _) => ??? + + case Stmt.Reset(core.Block.BlockLit(Nil, cparams, Nil, prompt :: Nil, body)) => + val p = Id(prompt.id) + // TODO is Var correct here?? Probably needs to be a new computation value... + given Env = env.bindComputation(prompt.id -> Computation.Var(p) :: Nil) + evaluate(body, Frame.Return, Stack.Reset(BlockParam(p, prompt.tpe, prompt.capt), k, ks)) + + case Stmt.Reset(_) => ??? + case Stmt.Resume(k2, body) => + env.lookupComputation(k2.id) match { + case Computation.Var(r) => + reify(k, ks) { + NeutralStmt.Resume(r, nested { + evaluate(body, Frame.Return, Stack.Unknown) + }) + } + case Computation.Continuation(k3) => + val (k4, ks4) = resume(k3, k, ks) + evaluate(body, k4, ks4) + case _ => ??? + } + } + + def run(mod: ModuleDecl): ModuleDecl = { + //util.trace(mod) + // TODO deal with async externs properly (see examples/benchmarks/input_output/dyck_one.effekt) + val externTypes = mod.externs.collect { case d: Extern.Def => + d.id -> (BlockType.Function(d.tparams, d.cparams, d.vparams.map { _.tpe }, d.bparams.map { bp => bp.tpe }, d.ret), d.annotatedCapture) + } + + val (builtinExterns, otherExterns) = mod.externs.collect { case d: Extern.Def => d }.partition { + case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => + val builtinName = vmBody.contents.strings.head + supportedBuiltins.contains(builtinName) + case _ => false + } + + val builtinNameToBlockVar: Map[String, BlockVar] = builtinExterns.collect { + case Extern.Def(id, tps, cps, vps, bps, ret, capt, targetBody, Some(vmBody)) => + val builtinName = vmBody.contents.strings.head + val bv: BlockVar = BlockVar(id, BlockType.Function(tps, cps, vps.map { _.tpe }, bps.map { bp => bp.tpe }, ret), capt) + builtinName -> bv + }.toMap + + val toplevelEnv = Env.empty + // user-defined functions + .bindComputation(mod.definitions.collect { + case Toplevel.Def(id, b) => id -> (b match { + case core.Block.BlockLit(tparams, cparams, vparams, bparams, body) => Computation.Def(Closure(id, Nil)) + case core.Block.BlockVar(idd, annotatedTpe, annotatedCapt) => Computation.Var(id) + case core.Block.Unbox(pure) => Computation.Var(id) + case core.Block.New(impl) => Computation.Var(id) + }) + }) + // user-defined values + .bindValue(mod.definitions.collect { + case Toplevel.Val(id, _, _) => id -> id + }) + // async extern functions + .bindComputation(otherExterns.map(defn => defn.id -> Computation.Var(defn.id))) + // pure extern functions + .bindComputation(builtinExterns.flatMap(defn => defn.vmBody.map(vmBody => defn.id -> Computation.BuiltinExtern(defn.id, vmBody.contents.strings.head)))) + + val typingContext = TypingContext( + mod.definitions.collect { + case Toplevel.Val(id, tpe, _) => id -> tpe + }.toMap, + mod.definitions.collect { + case Toplevel.Def(id, b) => id -> (b.tpe, b.capt) + }.toMap ++ externTypes, + builtinNameToBlockVar + ) + + val newDefinitions = mod.definitions.map(d => run(d)(using toplevelEnv, typingContext)) + mod.copy(definitions = newDefinitions) + } + + val showDebugInfo = true + inline def debug(inline msg: => Any) = if (showDebugInfo) println(msg) else () + + def run(defn: Toplevel)(using env: Env, G: TypingContext): Toplevel = defn match { + case Toplevel.Def(id, core.Block.BlockLit(tparams, cparams, vparams, bparams, body)) => + debug(s"------- ${util.show(id)} -------") + debug(util.show(body)) + + val scope = Scope.empty + val localEnv: Env = env + .bindValue(vparams.map(p => p.id -> scope.allocate("p", Value.Var(p.id, p.tpe)))) + .bindComputation(bparams.map(p => p.id -> Computation.Var(p.id))) + + val result = evaluate(body, Frame.Return, Stack.Empty)(using localEnv, scope) + + debug(s"----------normalized-----------") + val block = Block(tparams, vparams, bparams, reifyBindings(scope, result)) + debug(PrettyPrinter.show(block)) + + debug(s"----------embedded-----------") + val embedded = embedBlockLit(block) + debug(util.show(embedded)) + + Toplevel.Def(id, embedded) + case other => other + } + + case class TypingContext(values: Map[Addr, ValueType], blocks: Map[Label, (BlockType, Captures)], builtinBlockVars: Map[String, BlockVar]) { + def bind(id: Id, tpe: ValueType): TypingContext = this.copy(values = values + (id -> tpe)) + def bind(id: Id, tpe: BlockType, capt: Captures): TypingContext = this.copy(blocks = blocks + (id -> (tpe, capt))) + def bindValues(vparams: List[ValueParam]): TypingContext = this.copy(values = values ++ vparams.map(p => p.id -> p.tpe)) + def lookupValue(id: Id): ValueType = values.getOrElse(id, sys.error(s"Unknown value: ${util.show(id)}")) + def bindComputations(bparams: List[BlockParam]): TypingContext = this.copy(blocks = blocks ++ bparams.map(p => p.id -> (p.tpe, p.capt))) + def bindComputation(bparam: BlockParam): TypingContext = this.copy(blocks = blocks + (bparam.id -> (bparam.tpe, bparam.capt))) + } + + def embedStmt(neutral: NeutralStmt)(using G: TypingContext): core.Stmt = neutral match { + case NeutralStmt.Return(result) => + Stmt.Return(embedExpr(result)) + case NeutralStmt.Jump(label, targs, vargs, bargs) => + Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) + case NeutralStmt.App(label, targs, vargs, bargs) => + Stmt.App(embedBlockVar(label), targs, vargs.map(embedExpr), bargs.map(embedBlock)) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => + Stmt.Invoke(embedBlockVar(label), method, tpe, targs, vargs.map(embedExpr), bargs.map(embedBlock)) + case NeutralStmt.If(cond, thn, els) => + Stmt.If(embedExpr(cond), embedStmt(thn), embedStmt(els)) + case NeutralStmt.Match(scrutinee, clauses, default) => + Stmt.Match(embedExpr(scrutinee), + clauses.map { case (id, block) => id -> embedBlockLit(block) }, + default.map(embedStmt)) + case NeutralStmt.Reset(prompt, body) => + val capture = prompt.capt match { + case set if set.size == 1 => set.head + case _ => sys error "Prompt needs to have a single capture" + } + Stmt.Reset(core.BlockLit(Nil, capture :: Nil, Nil, prompt :: Nil, embedStmt(body)(using G.bindComputation(prompt)))) + case NeutralStmt.Shift(prompt, capt, k, body) => + Stmt.Shift(embedBlockVar(prompt), core.BlockLit(Nil, capt :: Nil, Nil, k :: Nil, embedStmt(body)(using G.bindComputation(k)))) + case NeutralStmt.Resume(k, body) => + Stmt.Resume(embedBlockVar(k), embedStmt(body)) + case NeutralStmt.Var(blockParam, init, body) => + val capt = blockParam.capt match { + case cs if cs.size == 1 => cs.head + case _ => sys error "Variable needs to have a single capture" + } + Stmt.Var(blockParam.id, embedExpr(init), capt, embedStmt(body)(using G.bind(blockParam.id, blockParam.tpe, blockParam.capt))) + case NeutralStmt.Put(ref, annotatedTpe, annotatedCapt, value, body) => + Stmt.Put(ref, annotatedCapt, embedExpr(value), embedStmt(body)) + case NeutralStmt.Region(id, body) => + Stmt.Region(BlockLit(Nil, List(id.id), Nil, List(id), embedStmt(body)(using G.bindComputation(id)))) + case NeutralStmt.Alloc(blockparam, init, region, body) => + Stmt.Alloc(blockparam.id, embedExpr(init), region, embedStmt(body)(using G.bind(blockparam.id, blockparam.tpe, blockparam.capt))) + case NeutralStmt.Hole(span) => + Stmt.Hole(span) + } + + def embedStmt(basicBlock: BasicBlock)(using G: TypingContext): core.Stmt = basicBlock match { + case BasicBlock(bindings, stmt) => + bindings.foldRight((G: TypingContext) => embedStmt(stmt)(using G)) { + case ((id, Binding.Let(value)), rest) => G => + val coreExpr = embedExpr(value)(using G) + // TODO why do we even have this type in core, if we always infer it? + Stmt.Let(id, coreExpr.tpe, coreExpr, rest(G.bind(id, coreExpr.tpe))) + case ((id, Binding.Def(block)), rest) => G => + val coreBlock = embedBlock(block)(using G) + Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) + case ((id, Binding.Rec(block, tpe, capt)), rest) => G => + val coreBlock = embedBlock(block)(using G.bind(id, tpe, capt)) + Stmt.Def(id, coreBlock, rest(G.bind(id, coreBlock.tpe, coreBlock.capt))) + case ((id, Binding.Val(stmt)), rest) => G => + val coreStmt = embedStmt(stmt)(using G) + Stmt.Val(id, coreStmt.tpe, coreStmt, rest(G.bind(id, coreStmt.tpe))) + case ((id, Binding.Run(callee, targs, vargs, bargs)), rest) => G => + val vargs1 = vargs.map(arg => embedExpr(arg)(using G)) + val bargs1 = bargs.map(arg => embedBlock(arg)(using G)) + val tpe = Type.bindingType(callee, targs, vargs1, bargs1) + core.ImpureApp(id, callee, targs, vargs1, bargs1, rest(G.bind(id, tpe))) + case ((id, Binding.Unbox(addr, tpe, capt)), rest) => G => + val pureValue = embedExpr(addr)(using G) + Stmt.Def(id, core.Block.Unbox(pureValue), rest(G.bind(id, tpe, capt))) + case ((id, Binding.Get(ref, tpe, cap)), rest) => G => + Stmt.Get(id, tpe, ref, cap, rest(G.bind(id, tpe)) ) + }(G) + } + + def embedExpr(value: Value)(using cx: TypingContext): core.Expr = value match { + case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) + case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) + case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) + case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) + case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) + case Value.Integer(value) => theories.Integers.reify(value, cx.builtinBlockVars) + } + + def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) + + def embedBlock(comp: Computation)(using G: TypingContext): core.Block = comp match { + case Computation.Var(id) => + embedBlockVar(id) + case Computation.Def(Closure(label, Nil)) => + embedBlockVar(label) + case Computation.Def(closure) => + etaExpandToBlockLit(closure) + case Computation.Continuation(k) => ??? + case Computation.BuiltinExtern(_, _) => ??? + case Computation.New(interface, operations) => + val ops = operations.map { etaExpandToOperation.tupled } + core.Block.New(Implementation(interface, ops)) + } + + /** + * Embed `Computation.Def` to a `core.BlockLit` + * This eta-expands the block var that stands for the `Computation.Def` to a full block literal + * so that we can supply the correct capture arguments from the environment. + */ + def etaExpandToBlockLit(closure: Closure)(using G: TypingContext): core.BlockLit = { + val Closure(label, environment) = closure + val blockvar = embedBlockVar(label) + G.blocks(label) match { + // TODO why is `captures` unused? + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + val vps = vparams.map { p => core.ValueParam(Id("x"), p) } + val vargs = vps.map { vp => core.Expr.ValueVar(vp.id, vp.tpe) } + + // this uses the invariant that we _append_ all environment captures to the bparams + val (origCapts, synthCapts) = cparams.splitAt(bparams.length - environment.length) + val (origBparams, synthBparams) = bparams.splitAt(bparams.length - environment.length) + val origBps = origBparams.zip(origCapts).map { case (bp, c) => core.BlockParam(Id("f"), bp, Set(c)) } + val origBargs = origBps.map { bp => core.BlockVar(bp.id, bp.tpe, bp.capt) } + val synthBargs = environment.zip(synthBparams).zip(synthCapts).map { + case ((Computation.Var(id), bp), c) => core.BlockVar(id, bp, Set(c)) + } + val bargs = origBargs ++ synthBargs + + val targs = tparams.map { core.ValueType.Var.apply } + + core.Block.BlockLit( + tparams, + origCapts, + vps, + origBps, + Stmt.App( + blockvar, + targs, + vargs, + bargs + ) + ) + case _ => sys.error("Unexpected block type for a closure") + } + } + + /** + * Embed an operation as part of a `Computation.New`. + * This eta-expands the block var that stands for the operation body to a full operation + * so that we can supply the correct capture arguments from the environment. + */ + def etaExpandToOperation(id: Id, closure: Closure)(using G: TypingContext): core.Operation = { + val Closure(label, environment) = closure + G.blocks(label) match { + case (BlockType.Function(tparams, cparams, vparams, bparams, result), captures) => + val tparams2 = tparams.map(t => Id(t)) + // TODO if we freshen cparams, then we also need to substitute them in the result AND the parameters + val cparams2 = cparams //.map(c => Id(c)) + val vparams2 = vparams.map(t => ValueParam(Id("x"), t)) + val bparams2 = (bparams zip cparams).map { case (t, c) => BlockParam(Id("f"), t, Set(c)) } + // In the following section, we create a new instance of the interface. + // All operation bodies were lifted to block literals in an earlier stage. + // While doing so, their block parameters (bparams) were concatenated with their capture parameters (cparams). + // When we embed back to core, we need to "eta-expand" the operation body to supply the correct captures from the environment. + // To see why this "eta-expansion" is necessary to achieve this, consider the following example: + // ```scala + // effect Eff(): Unit + // def use = { do Eff() } + // def main() = { + // val r = try { + // use() + // } with Eff { + // resume(()) + // } + // } + // ``` + // the handler body normalizes to the following: + // ```scala + // reset {{p} => + // jump use(){new Eff {def Eff = Eff @ [p]}} + // } + // ``` + // where + // ``` + // def Eff = (){p} { ... } + // ``` + // In particular, the prompt `p` needs to be passed to the lifted operation body. + // ``` + val (origBparams, synthBparams) = bparams2.splitAt(bparams2.length - environment.length) + val bargs = + // TODO: Fix captures + origBparams.map { case bp => BlockVar(bp.id, bp.tpe, Set()) } ++ + synthBparams.zip(environment).map { + // TODO: Fix captures + case (bp, Computation.Var(id)) => BlockVar(id, bp.tpe, Set()) + } + + core.Operation( + id, + tparams2, + cparams.take(cparams.length - environment.length), + vparams2, + origBparams, + Stmt.App( + embedBlockVar(label), + tparams2.map(ValueType.Var.apply), + vparams2.map(p => ValueVar(p.id, p.tpe)), + bargs + ) + ) + case _ => sys error "Unexpected block type" + } + } + + def embedBlock(block: Block)(using G: TypingContext): core.Block = block match { + case Block(tparams, vparams, bparams, b) => + val cparams = bparams.map { + case BlockParam(id, tpe, captures) => + assert(captures.size == 1) + captures.head + } + core.Block.BlockLit(tparams, cparams, vparams, bparams, + embedStmt(b)(using G.bindValues(vparams).bindComputations(bparams))) + } + + def embedBlockLit(block: Block)(using G: TypingContext): core.BlockLit = embedBlock(block).asInstanceOf[core.BlockLit] + + def embedBlockVar(label: Label)(using G: TypingContext): core.BlockVar = + val (tpe, capt) = G.blocks.getOrElse(label, sys error s"Unknown block: ${util.show(label)}. ${G.blocks.keys.map(util.show).mkString(", ")}") + core.BlockVar(label, tpe, capt) +} diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala new file mode 100644 index 000000000..4f20db2a6 --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -0,0 +1,105 @@ +package effekt +package core +package optimizer +package normalizer + +type ~>[-A, +B] = PartialFunction[A, B] + +type BuiltinImpl = List[semantics.Value] ~> semantics.Value + +def builtin(name: String)(impl: List[semantics.Value] ~> semantics.Value): (String, BuiltinImpl) = name -> impl + +type Builtins = Map[String, BuiltinImpl] + +lazy val integers: Builtins = Map( + // Integer arithmetic operations with symbolic simplification support + // ---------- + builtin("effekt::infixAdd(Int, Int)") { + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.add(x, y)) + }, + builtin("effekt::infixSub(Int, Int)") { + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.sub(x, y)) + }, + builtin("effekt::infixMul(Int, Int)") { + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.mul(x, y)) + }, + // Integer arithmetic operations only evaluated for literals + // ---------- + builtin("effekt::infixDiv(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x / y)) + }, + builtin("effekt::mod(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x % y)) + }, + builtin("effekt::bitwiseShl(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x << y)) + }, + builtin("effekt::bitwiseShr(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x >> y)) + }, + builtin("effekt::bitwiseAnd(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x & y)) + }, + builtin("effekt::bitwiseOr(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x | y)) + }, + builtin("effekt::bitwiseXor(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x ^ y)) + }, + // Comparison + // ---------- + builtin("effekt::infixEq(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x == y, Type.TBoolean) + }, + builtin("effekt::infixNeq(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x != y, Type.TBoolean) + }, + builtin("effekt::infixLt(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x < y, Type.TBoolean) + }, + builtin("effekt::infixGt(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x > y, Type.TBoolean) + }, + builtin("effekt::infixLte(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x <= y, Type.TBoolean) + }, + builtin("effekt::infixGte(Int, Int)") { + case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x >= y, Type.TBoolean) + }, + // Conversion + // ---------- + builtin("effekt::toDouble(Int)") { + case As.Int(x) :: Nil => semantics.Value.Literal(x.toDouble, Type.TDouble) + }, + + builtin("effekt::show(Int)") { + case As.Int(n) :: Nil => semantics.Value.Literal(n.toString, Type.TString) + }, +) + +lazy val supportedBuiltins: Builtins = integers + +protected object As { + object Int { + def unapply(v: semantics.Value): Option[scala.Long] = v match { + case semantics.Value.Literal(value: scala.Long, _) => Some(value) + case semantics.Value.Literal(value: scala.Int, _) => Some(value.toLong) + case semantics.Value.Literal(value: java.lang.Integer, _) => Some(value.toLong) + case _ => None + } + } + + object IntExpr { + def unapply(v: semantics.Value): Option[theories.Integers.Integer] = v match { + // Integer literals not yet embedded into the theory of integers + case semantics.Value.Literal(value: scala.Long, _) => Some(theories.Integers.embed(value)) + case semantics.Value.Literal(value: scala.Int, _) => Some(theories.Integers.embed(value.toLong)) + case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.Integers.embed(value.toLong)) + // Variables of type integer + case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.Integers.embed(id)) + // Already embedded integers + case semantics.Value.Integer(value) => Some(value) + case _ => None + } + } +} \ No newline at end of file diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala new file mode 100644 index 000000000..729ea1fd1 --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala @@ -0,0 +1,741 @@ +package effekt +package core +package optimizer +package normalizer + +import effekt.core.optimizer.normalizer.semantics.PrettyPrinter.{braces, brackets, comma, emptyDoc, hcat, hsep, line, nest, parens, pretty} +import effekt.core.{BlockVar, Capture, Captures, Id} +import effekt.source.Span +import kiama.output.ParenPrettyPrinter + +import scala.annotation.tailrec +import scala.collection.immutable.ListMap + +object semantics { + + // Values + // ------ + + type Addr = Id + type Label = Id + type Prompt = Id + + // this could not only compute free variables, but also usage information to guide the inliner (see "secrets of the ghc inliner") + type Variables = Set[Id] + def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet + + enum Value { + // Stuck + case Var(id: Id, annotatedType: ValueType) + case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) + + // Actual Values + case Literal(value: Any, annotatedType: ValueType) + case Integer(value: theories.Integers.Integer) + + case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) + + // TODO use dynamic captures + case Box(body: Computation, annotatedCaptures: Set[effekt.symbols.Symbol]) + + val dynamicCapture: Variables = Set.empty + + val free: Variables = this match { + case Value.Var(id, annotatedType) => Set(id) + case Value.Extern(id, targs, vargs) => vargs.toSet + case Value.Literal(value, annotatedType) => Set.empty + case Value.Integer(value) => value.free + case Value.Make(data, tag, targs, vargs) => vargs.toSet + // Box abstracts over all free computation variables, only when unboxing, they occur free again + case Value.Box(body, tpe) => body.free + } + } + + // TODO find better name for this + enum Binding { + case Let(value: Value) + case Def(block: Block) + case Rec(block: Block, tpe: BlockType, capt: Captures) + case Val(stmt: NeutralStmt) + case Run(f: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]) + case Unbox(addr: Addr, tpe: BlockType, capt: Captures) + case Get(ref: Id, tpe: ValueType, cap: Captures) + + val free: Variables = this match { + case Binding.Let(value) => value.free + case Binding.Def(block) => block.free + case Binding.Rec(block, tpe, capt) => block.free + case Binding.Val(stmt) => stmt.free + // TODO block args for externs are not supported (for now?) + case Binding.Run(f, targs, vargs, bargs) => vargs.toSet + case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => Set(addr) + case Binding.Get(ref, tpe, cap) => Set(ref) + } + + val dynamicCapture: Variables = this match { + case Binding.Let(value) => value.dynamicCapture + case Binding.Def(block) => block.dynamicCapture + case Binding.Rec(block, tpe, capt) => block.dynamicCapture + case Binding.Val(stmt) => stmt.dynamicCapture + // TODO block args for externs are not supported (for now?) + case Binding.Run(f, targs, vargs, bargs) => Set.empty + case Binding.Unbox(addr: Addr, tpe: BlockType, capt: Captures) => capt // TODO these are the static not dynamic captures + case Binding.Get(ref, tpe, cap) => Set(ref) + } + } + + type Bindings = List[(Id, Binding)] + object Bindings { + def empty: Bindings = Nil + } + + /** + * A Scope is a bit like a basic block, but without the terminator + */ + class Scope( + var bindings: ListMap[Id, Binding], + var inverse: Map[Value, Addr], + val outer: Option[Scope] + ) { + // Backtrack the internal state of Scope after running `prog` + def local[A](prog: => A): A = { + val scopeBefore = Scope(this.bindings, this.inverse, this.outer) + val res = prog + this.bindings = scopeBefore.bindings + this.inverse = scopeBefore.inverse + res + } + + // floating values to the top is not always beneficial. For example + // def foo() = COMPUTATION + // vs + // let x = COMPUTATION + // def foo() = x + def getDefinition(value: Value): Option[Addr] = + inverse.get(value) orElse outer.flatMap(_.getDefinition(value)) + + def allocate(hint: String, value: Value): Addr = + getDefinition(value) match { + case Some(value) => value + case None => + val addr = Id(hint) + bindings = bindings.updated(addr, Binding.Let(value)) + inverse = inverse.updated(value, addr) + addr + } + + def allocateGet(ref: Id, tpe: ValueType, cap: Captures): Addr = { + val addr = Id("get") + bindings = bindings.updated(addr, Binding.Get(ref, tpe, cap)) + addr + } + + def run(hint: String, callee: BlockVar, targs: List[ValueType], vargs: List[Addr], bargs: List[Computation]): Addr = { + val addr = Id(hint) + bindings = bindings.updated(addr, Binding.Run(callee, targs, vargs, bargs)) + addr + } + + def unbox(innerAddr: Addr, tpe: BlockType, capt: Captures): Addr = { + val unboxAddr = Id("unbox") + bindings = bindings.updated(unboxAddr, Binding.Unbox(innerAddr, tpe, capt)) + unboxAddr + } + + // TODO Option[Value] or Var(id) in Value? + def lookupValue(addr: Addr): Option[Value] = bindings.get(addr) match { + case Some(Binding.Let(value)) => Some(value) + case _ => outer.flatMap(_.lookupValue(addr)) + } + + def define(label: Label, block: Block): Unit = + bindings = bindings.updated(label, Binding.Def(block)) + + def defineRecursive(label: Label, block: Block, tpe: BlockType, capt: Captures): Unit = + bindings = bindings.updated(label, Binding.Rec(block, tpe, capt)) + + def push(id: Id, stmt: NeutralStmt): Unit = + bindings = bindings.updated(id, Binding.Val(stmt)) + } + object Scope { + def empty: Scope = new Scope(ListMap.empty, Map.empty, None) + } + + def reifyBindings(scope: Scope, body: NeutralStmt): BasicBlock = { + var used = body.free + var filtered = Bindings.empty + // TODO implement properly + scope.bindings.toSeq.reverse.foreach { + // TODO for now we keep ALL definitions + case (addr, b: Binding.Def) => + used = used ++ b.free + filtered = (addr, b) :: filtered + case (addr, b: Binding.Rec) => + used = used ++ b.free + filtered = (addr, b) :: filtered + case (addr, s: Binding.Val) => + used = used ++ s.free + filtered = (addr, s) :: filtered + case (addr, v: Binding.Run) => + used = used ++ v.free + filtered = (addr, v) :: filtered + + // TODO if type is unit like, we can potentially drop this binding (but then we need to make up a "fresh" unit at use site) + case (addr, v: Binding.Let) if used.contains(addr) => + used = used ++ v.free + filtered = (addr, v) :: filtered + case (addr, v: Binding.Let) => () + case (addr, b: Binding.Unbox) => + used = used ++ b.free + filtered = (addr, b):: filtered + case (addr, g: Binding.Get) => + used = used ++ g.free + filtered = (addr, g) :: filtered + } + + // we want to avoid turning tailcalls into non tail calls like + // + // val x = app(x) + // return x + // + // so we eta-reduce here. Can we achieve this by construction? + // TODO lastOption will go through the list AGAIN, let's see whether this causes performance problems + (filtered.lastOption, body) match { + case (Some((id1, Binding.Val(stmt))), NeutralStmt.Return(id2)) if id1 == id2 => + BasicBlock(filtered.init, stmt) + case (_, _) => + BasicBlock(filtered, body) + } + } + + def nested(prog: Scope ?=> NeutralStmt)(using scope: Scope): BasicBlock = { + // TODO parent code and parent store + val local = Scope(ListMap.empty, Map.empty, Some(scope)) + val result = prog(using local) + reifyBindings(local, result) + } + + case class Env(values: Map[Id, Addr], computations: Map[Id, Computation]) { + def lookupValue(id: Id): Addr = values(id) + def bindValue(id: Id, value: Addr): Env = Env(values + (id -> value), computations) + def bindValue(newValues: List[(Id, Addr)]): Env = Env(values ++ newValues, computations) + + def lookupComputation(id: Id): Computation = computations.getOrElse(id, sys error s"Unknown computation: ${util.show(id)} -- env: ${computations.map { case (id, comp) => s"${util.show(id)}: $comp" }.mkString("\n") }") + def bindComputation(id: Id, computation: Computation): Env = Env(values, computations + (id -> computation)) + def bindComputation(newComputations: List[(Id, Computation)]): Env = Env(values, computations ++ newComputations) + } + object Env { + def empty: Env = Env(Map.empty, Map.empty) + } + // "handlers" + def bind[R](id: Id, addr: Addr)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(id, addr)) + + def bind[R](id: Id, computation: Computation)(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindComputation(id, computation)) + + def bind[R](values: List[(Id, Addr)])(prog: Env ?=> R)(using env: Env): R = + prog(using env.bindValue(values)) + + case class Block(tparams: List[Id], vparams: List[ValueParam], bparams: List[BlockParam], body: BasicBlock) { + val free: Variables = body.free -- vparams.map(_.id) -- bparams.map(_.id) + val dynamicCapture: Variables = body.dynamicCapture -- bparams.map(_.id) + } + + case class BasicBlock(bindings: Bindings, body: NeutralStmt) { + val free: Variables = { + var free = body.free + bindings.reverse.foreach { + case (id, b: Binding.Let) => free = (free - id) ++ b.free + case (id, b: Binding.Def) => free = (free - id) ++ b.free + case (id, b: Binding.Rec) => free = (free - id) ++ (b.free - id) + case (id, b: Binding.Val) => free = (free - id) ++ b.free + case (id, b: Binding.Run) => free = (free - id) ++ b.free + case (id, b: Binding.Unbox) => free = (free - id) ++ b.free + case (id, b: Binding.Get) => free = (free - id) ++ b.free + } + free + } + + val dynamicCapture: Variables = { + body.dynamicCapture ++ bindings.flatMap(_._2.dynamicCapture) + } + } + + enum Computation { + // Unknown + case Var(id: Id) + // Known function + case Def(closure: Closure) + + case Continuation(k: Cont) + + case BuiltinExtern(id: Id, builtinName: String) + + // Known object + case New(interface: BlockType.Interface, operations: List[(Id, Closure)]) + + val free: Variables = this match { + case Computation.Var(id) => Set(id) + case Computation.Def(closure) => closure.free + case Computation.Continuation(k) => Set.empty // TODO ??? + case Computation.New(interface, operations) => operations.flatMap(_._2.free).toSet + case Computation.BuiltinExtern(id, vmSymbol) => Set.empty + } + + val dynamicCapture: Variables = this match { + case Computation.Var(id) => Set(id) + case Computation.Def(closure) => closure.dynamicCapture + case Computation.Continuation(k) => Set.empty // TODO ??? + case Computation.New(interface, operations) => operations.flatMap(_._2.dynamicCapture).toSet + case Computation.BuiltinExtern(id, vmSymbol) => Set.empty + } + } + + // TODO add escaping mutable variables + case class Closure(label: Label, environment: List[Computation.Var]) { + val free: Variables = Set(label) ++ environment.flatMap(_.free).toSet + val dynamicCapture: Variables = environment.map(_.id).toSet + } + + // Statements + // ---------- + enum NeutralStmt { + // context (continuation) is unknown + case Return(result: Id) + // callee is unknown + case App(callee: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) + // Known jump, but we do not want to inline + case Jump(label: Id, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) + // callee is unknown + case Invoke(id: Id, method: Id, methodTpe: BlockType, targs: List[ValueType], vargs: List[Id], bargs: List[Computation]) + // cond is unknown + case If(cond: Id, thn: BasicBlock, els: BasicBlock) + // scrutinee is unknown + case Match(scrutinee: Id, clauses: List[(Id, Block)], default: Option[BasicBlock]) + + // body is stuck + case Reset(prompt: BlockParam, body: BasicBlock) + // prompt / context is unknown + case Shift(prompt: Prompt, kCapt: Capture, k: BlockParam, body: BasicBlock) + // continuation is unknown + case Resume(k: Id, body: BasicBlock) + + case Var(id: BlockParam, init: Addr, body: BasicBlock) + case Put(ref: Id, tpe: ValueType, cap: Captures, value: Addr, body: BasicBlock) + + case Region(id: BlockParam, body: BasicBlock) + case Alloc(id: BlockParam, init: Addr, region: Id, body: BasicBlock) + + // aborts at runtime + case Hole(span: Span) + + val free: Variables = this match { + case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) + case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ vargs.toSet ++ all(bargs, _.free) + case NeutralStmt.If(cond, thn, els) => Set(cond) ++ thn.free ++ els.free + case NeutralStmt.Match(scrutinee, clauses, default) => Set(scrutinee) ++ clauses.flatMap(_._2.free).toSet ++ default.map(_.free).getOrElse(Set.empty) + case NeutralStmt.Return(result) => Set(result) + case NeutralStmt.Reset(prompt, body) => body.free - prompt.id + case NeutralStmt.Shift(prompt, capt, k, body) => (body.free - k.id) + prompt + case NeutralStmt.Resume(k, body) => Set(k) ++ body.free + case NeutralStmt.Var(id, init, body) => Set(init) ++ body.free - id.id + case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref, value) ++ body.free + case NeutralStmt.Region(id, body) => body.free - id.id + case NeutralStmt.Alloc(id, init, region, body) => Set(init, region) ++ body.free - id.id + case NeutralStmt.Hole(span) => Set.empty + } + + val dynamicCapture: Variables = this match { + case NeutralStmt.Return(result) => Set.empty + case NeutralStmt.Hole(span) => Set.empty + + case NeutralStmt.Jump(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.App(label, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => Set(label) ++ all(bargs, _.dynamicCapture) + case NeutralStmt.If(cond, thn, els) => thn.dynamicCapture ++ els.dynamicCapture + case NeutralStmt.Match(scrutinee, clauses, default) => clauses.flatMap(_._2.dynamicCapture).toSet ++ default.map(_.dynamicCapture).getOrElse(Set.empty) + case NeutralStmt.Reset(prompt, body) => body.dynamicCapture - prompt.id + case NeutralStmt.Shift(prompt, capt, k, body) => (body.dynamicCapture - k.id) + prompt + case NeutralStmt.Resume(k, body) => Set(k) ++ body.dynamicCapture + case NeutralStmt.Var(id, init, body) => body.dynamicCapture - id.id + case NeutralStmt.Put(ref, tpe, cap, value, body) => Set(ref) ++ body.dynamicCapture + case NeutralStmt.Region(id, body) => body.dynamicCapture - id.id + case NeutralStmt.Alloc(id, init, region, body) => Set(region) ++ body.dynamicCapture - id.id + } + } + + // Stacks + // ------ + enum Frame { + case Return + case Static(tpe: ValueType, apply: Scope => Addr => Stack => NeutralStmt) + case Dynamic(closure: Closure) + + /* Return an argument `arg` through this frame and the rest of the stack `ks` + */ + def ret(ks: Stack, arg: Addr)(using scope: Scope): NeutralStmt = this match { + case Frame.Return => ks match { + case Stack.Empty => NeutralStmt.Return(arg) + case Stack.Unknown => NeutralStmt.Return(arg) + case Stack.Reset(p, k, ks) => k.ret(ks, arg) + case Stack.Var(id, curr, k, ks) => k.ret(ks, arg) + case Stack.Region(id, bindings, k, ks) => k.ret(ks, arg) + } + case Frame.Static(tpe, apply) => apply(scope)(arg)(ks) + case Frame.Dynamic(Closure(label, environment)) => reify(ks) { NeutralStmt.Jump(label, Nil, List(arg), environment) } + } + + // pushing purposefully does not abstract over env (it closes over it!) + def push(tpe: ValueType)(f: Scope => Addr => Frame => Stack => NeutralStmt): Frame = + Frame.Static(tpe, scope => arg => ks => f(scope)(arg)(this)(ks)) + } + + // maybe, for once it is simpler to decompose stacks like + // + // f, (p, f) :: (p, f) :: Nil + // + // where the frame on the reset is the one AFTER the prompt NOT BEFORE! + enum Stack { + /** + * Statically known to be empty + * This only occurs at the entrypoint of normalization. + * In other cases, where the stack is not known, you should use Unknown instead. + */ + case Empty + /** Dynamic tail (we do not know the shape of the remaining stack) + */ + case Unknown + case Reset(prompt: BlockParam, frame: Frame, next: Stack) + case Var(id: BlockParam, curr: Addr, frame: Frame, next: Stack) + // TODO desugar regions into var? + case Region(id: BlockParam, bindings: Map[BlockParam, Addr], frame: Frame, next: Stack) + + lazy val bound: List[BlockParam] = this match { + case Stack.Empty => Nil + case Stack.Unknown => Nil + case Stack.Reset(prompt, frame, next) => prompt :: next.bound + case Stack.Var(id, curr, frame, next) => id :: next.bound + case Stack.Region(id, bindings, frame, next) => id :: next.bound ++ bindings.keys + } + } + + @tailrec + def get(ref: Id, ks: Stack): Option[Addr] = ks match { + case Stack.Empty => sys error s"Should not happen: trying to lookup ${util.show(ref)} in empty stack" + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => get(ref, next) + case Stack.Var(id1, curr, frame, next) if ref == id1.id => Some(curr) + case Stack.Var(id1, curr, frame, next) => get(ref, next) + case Stack.Region(id, bindings, frame, next) => + val containsRef = bindings.keys.find(bp => bp.id == ref) + containsRef match { + case Some(bparam) => Some(bindings(bparam)) + case None => get(ref, next) + } + } + + def put(ref: Id, value: Addr, ks: Stack): Option[Stack] = ks match { + case Stack.Empty => sys error s"Should not happen: trying to put ${util.show(ref)} in empty stack" + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => put(ref, value, next).map(Stack.Reset(prompt, frame, _)) + case Stack.Var(id, curr, frame, next) if ref == id.id => Some(Stack.Var(id, value, frame, next)) + case Stack.Var(id, curr, frame, next) => put(ref, value, next).map(Stack.Var(id, curr, frame, _)) + case Stack.Region(id, bindings, frame, next) => + val containsRef = bindings.keys.find(bp => bp.id == ref) + containsRef match { + case Some(bparam) => Some(Stack.Region(id, bindings.updated(bparam, value), frame, next)) + case None => put(ref, value, next).map(Stack.Region(id, bindings, frame, _)) + } + } + + def alloc(ref: BlockParam, reg: Id, value: Addr, ks: Stack): Option[Stack] = ks match { + // This case can occur if we normalize a function that abstracts over a region as a parameter + // We return None and force the reification of the allocation + case Stack.Empty => None + // We have reached the end of the known stack, so the variable must be in the unknown part. + case Stack.Unknown => None + case Stack.Reset(prompt, frame, next) => + alloc(ref, reg, value, next).map(Stack.Reset(prompt, frame, _)) + case Stack.Var(id, curr, frame, next) => + alloc(ref, reg, value, next).map(Stack.Var(id, curr, frame, _)) + case Stack.Region(id, bindings, frame, next) => + if (reg == id.id){ + Some(Stack.Region(id, bindings.updated(ref, value), frame, next)) + } else { + alloc(ref, reg, value, next).map(Stack.Region(id, bindings, frame, _)) + } + } + + enum Cont { + case Empty + case Reset(frame: Frame, prompt: BlockParam, rest: Cont) + case Var(frame: Frame, id: BlockParam, curr: Addr, rest: Cont) + case Region(frame: Frame, id: BlockParam, bindings: Map[BlockParam, Addr], rest: Cont) + } + + def shift(p: Id, k: Frame, ks: Stack): (Cont, Frame, Stack) = ks match { + case Stack.Empty => sys error s"Should not happen: cannot find prompt ${util.show(p)}" + case Stack.Unknown => sys error s"Cannot find prompt ${util.show(p)} in unknown stack" + case Stack.Reset(prompt, frame, next) if prompt.id == p => + (Cont.Reset(k, prompt, Cont.Empty), frame, next) + case Stack.Reset(prompt, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Reset(k, prompt, c), frame2, stack) + case Stack.Var(id, curr, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Var(k, id, curr, c), frame2, stack) + case Stack.Region(id, bindings, frame, next) => + val (c, frame2, stack) = shift(p, frame, next) + (Cont.Region(k, id, bindings, c), frame2, stack) + } + + def resume(c: Cont, k: Frame, ks: Stack): (Frame, Stack) = c match { + case Cont.Empty => + (k, ks) + case Cont.Reset(frame, prompt, rest) => + val (k1, ks1) = resume(rest, k, ks) + (frame, Stack.Reset(prompt, k1, ks1)) + case Cont.Var(frame, id, curr, rest) => + val (k1, ks1) = resume(rest, k, ks) + (frame, Stack.Var(id, curr, k1, ks1)) + case Cont.Region(frame, id, bindings, rest) => + val (k1, ks1) = resume(rest, k, ks) + (frame, Stack.Region(id, bindings, k1, ks1)) + } + + def joinpoint(k: Frame, ks: Stack)(f: (Frame, Stack) => NeutralStmt)(using scope: Scope): NeutralStmt = { + def reifyFrame(k: Frame, escaping: Stack)(using scope: Scope): Frame = k match { + case Frame.Static(tpe, apply) => + val x = Id("x") + nested { scope ?=> apply(scope)(x)(Stack.Unknown) } match { + // Avoid trivial continuations like + // def k_6268 = (x_6267: Int_3) { + // return x_6267 + // } + case BasicBlock(Nil, _: (NeutralStmt.Return | NeutralStmt.App | NeutralStmt.Jump)) => + k + case body => + val k = Id("k") + val closureParams = escaping.bound.collect { case p if body.dynamicCapture contains p.id => p } + scope.define(k, Block(Nil, ValueParam(x, tpe) :: Nil, closureParams, body)) + Frame.Dynamic(Closure(k, closureParams.map { p => Computation.Var(p.id) })) + } + case Frame.Return => k + case Frame.Dynamic(label) => k + } + + def reifyStack(ks: Stack): Stack = ks match { + case Stack.Empty => Stack.Empty + case Stack.Unknown => Stack.Unknown + case Stack.Reset(prompt, frame, next) => + Stack.Reset(prompt, reifyFrame(frame, next), reifyStack(next)) + case Stack.Var(id, curr, frame, next) => + Stack.Var(id, curr, reifyFrame(frame, next), reifyStack(next)) + case Stack.Region(id, bindings, frame, next) => + Stack.Region(id, bindings, reifyFrame(frame, next), reifyStack(next)) + } + f(reifyFrame(k, ks), reifyStack(ks)) + } + + def reify(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using Scope): NeutralStmt = + reify(ks) { reify(k) { stmt } } + + def reify(k: Frame)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + k match { + case Frame.Return => stmt + case Frame.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + // TODO Over-approximation + // Don't pass Stack.Unknown but rather the stack until the next reset? + /* + |----------| |----------| |---------| + | | ---> ... ---> | | ---> ... ---> | | ---> ... + |----------| |----------| |---------| + r1 r2 first next prompt + + Pass r1 :: ... :: r2 :: ... :: prompt :: UNKNOWN + */ + apply(scope)(tmp)(Stack.Unknown) + case Frame.Dynamic(Closure(label, closure)) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + NeutralStmt.Jump(label, Nil, List(tmp), closure) + } + + def reifyKnown(k: Frame, ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = + k match { + case Frame.Return => reify(ks) { stmt } + case Frame.Static(tpe, apply) => + val tmp = Id("tmp") + scope.push(tmp, stmt) + apply(scope)(tmp)(ks) + case Frame.Dynamic(Closure(label, closure)) => reify(ks) { sc ?=> + val tmp = Id("tmp") + sc.push(tmp, stmt) + NeutralStmt.Jump(label, Nil, List(tmp), closure) + } + } + + @tailrec + final def reify(ks: Stack)(stmt: Scope ?=> NeutralStmt)(using scope: Scope): NeutralStmt = { + ks match { + case Stack.Empty => stmt + case Stack.Unknown => stmt + case Stack.Reset(prompt, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + if (body.dynamicCapture contains prompt.id) NeutralStmt.Reset(prompt, body) + else stmt // TODO this runs normalization a second time in the outer scope! + }} + case Stack.Var(id, curr, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + if (body.dynamicCapture contains id.id) NeutralStmt.Var(id, curr, body) + else stmt + }} + case Stack.Region(id, bindings, frame, next) => + reify(next) { reify(frame) { + val body = nested { stmt } + val bodyUsesBinding = body.dynamicCapture.exists(bindings.map { b => b._1.id }.toSet.contains(_)) + if (body.dynamicCapture.contains(id.id) || bodyUsesBinding) { + // we need to reify all bindings in this region as allocs using their current value + val reifiedAllocs = bindings.foldLeft(body) { case (acc, (bp, addr)) => + nested { NeutralStmt.Alloc(bp, addr, id.id, acc) } + } + NeutralStmt.Region(id, reifiedAllocs) + } + else stmt + }} + } + } + + object PrettyPrinter extends ParenPrettyPrinter { + + override val defaultIndent = 2 + + def toDoc(s: NeutralStmt): Doc = s match { + case NeutralStmt.Return(result) => + "return" <+> toDoc(result) + case NeutralStmt.If(cond, thn, els) => + "if" <+> parens(toDoc(cond)) <+> toDoc(thn) <+> "else" <+> toDoc(els) + case NeutralStmt.Match(scrutinee, clauses, default) => + "match" <+> parens(toDoc(scrutinee)) <+> braces(hcat(clauses.map { case (id, block) => toDoc(id) <> ":" <+> toDoc(block) })) <> + (if (default.isDefined) "else" <+> toDoc(default.get) else emptyDoc) + case NeutralStmt.Jump(label, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + "jump" <+> toDoc(label) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + case NeutralStmt.App(label, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + toDoc(label) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + + case NeutralStmt.Invoke(label, method, tpe, targs, vargs, bargs) => + // Format as: l1[T1, T2](r1, r2) + toDoc(label) <> "." <> toDoc(method) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hsep(bargs.map(b => braces(toDoc(b)))) + + case NeutralStmt.Reset(prompt, body) => + "reset" <+> braces(toDoc(prompt) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) + + case NeutralStmt.Shift(prompt, capt, k, body) => + "shift" <> parens(toDoc(prompt)) <+> braces(toDoc(k) <+> "=>" <+> nest(line <> toDoc(body.bindings) <> toDoc(body.body)) <> line) + + case NeutralStmt.Resume(k, body) => + "resume" <> parens(toDoc(k)) <+> toDoc(body) + + case NeutralStmt.Var(id, init, body) => + "var" <+> toDoc(id.id) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) + + case NeutralStmt.Put(ref, tpe, cap, value, body) => + toDoc(ref) <+> ":=" <+> toDoc(value) <> line <> toDoc(body.bindings) <> toDoc(body.body) + + case NeutralStmt.Region(id, body) => + "region" <+> toDoc(id) <+> toDoc(body) + + case NeutralStmt.Alloc(id, init, region, body) => + "var" <+> toDoc(id) <+> "in" <+> toDoc(region) <+> "=" <+> toDoc(init) <> line <> toDoc(body.bindings) <> toDoc(body.body) + + case NeutralStmt.Hole(span) => "hole()" + } + + def toDoc(id: Id): Doc = id.show + + def toDoc(value: Value): Doc = value match { + // case Value.Var(id, tpe) => toDoc(id) + + case Value.Extern(callee, targs, vargs) => + toDoc(callee.id) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) + + case Value.Literal(value, _) => util.show(value) + + case Value.Make(data, tag, targs, vargs) => + "make" <+> toDoc(data) <+> toDoc(tag) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) + + case Value.Box(body, tpe) => + "box" <+> braces(nest(line <> toDoc(body) <> line)) + + case Value.Var(id, tpe) => toDoc(id) + + case Value.Integer(value) => value.show + } + + def toDoc(block: Block): Doc = block match { + case Block(tparams, vparams, bparams, body) => + (if (tparams.isEmpty) emptyDoc else brackets(hsep(tparams.map(toDoc), comma))) <> + parens(hsep(vparams.map(toDoc), comma)) <> hsep(bparams.map(toDoc)) <+> toDoc(body) + } + + def toDoc(comp: Computation): Doc = comp match { + case Computation.Var(id) => toDoc(id) + case Computation.Def(closure) => toDoc(closure) + case Computation.Continuation(k) => ??? + case Computation.New(interface, operations) => "new" <+> toDoc(interface) <+> braces { + hsep(operations.map { case (id, impl) => "def" <+> toDoc(id) <+> "=" <+> toDoc(impl) }, ",") + } + case Computation.BuiltinExtern(id, vmSymbol) => "extern" <+> toDoc(id) <+> "=" <+> vmSymbol + } + def toDoc(closure: Closure): Doc = closure match { + case Closure(label, env) => toDoc(label) <+> "@" <+> brackets(hsep(env.map(toDoc), comma)) + } + + def toDoc(bindings: Bindings): Doc = + hcat(bindings.map { + case (addr, Binding.Let(value)) => "let" <+> toDoc(addr) <+> "=" <+> toDoc(value) <> line + case (addr, Binding.Def(block)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line + case (addr, Binding.Rec(block, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> toDoc(block) <> line + case (addr, Binding.Val(stmt)) => "val" <+> toDoc(addr) <+> "=" <+> toDoc(stmt) <> line + case (addr, Binding.Run(callee, targs, vargs, bargs)) => "let !" <+> toDoc(addr) <+> "=" <+> toDoc(callee.id) <> + (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> + parens(hsep(vargs.map(toDoc), comma)) <> hcat(bargs.map(b => braces { toDoc(b) })) <> line + case (addr, Binding.Unbox(innerAddr, tpe, capt)) => "def" <+> toDoc(addr) <+> "=" <+> "unbox" <+> toDoc(innerAddr) <> line + case (addr, Binding.Get(ref, tpe, cap)) => "let" <+> toDoc(addr) <+> "=" <+> "!" <> toDoc(ref) <> line + }) + + def toDoc(block: BasicBlock): Doc = + braces(nest(line <> toDoc(block.bindings) <> toDoc(block.body)) <> line) + + def toDoc(p: ValueParam): Doc = toDoc(p.id) <> ":" <+> toDoc(p.tpe) + def toDoc(p: BlockParam): Doc = braces(toDoc(p.id)) + + def toDoc(t: ValueType): Doc = util.show(t) + def toDoc(t: BlockType): Doc = util.show(t) + + def show(stmt: NeutralStmt): String = pretty(toDoc(stmt), 80).layout + def show(value: Value): String = pretty(toDoc(value), 80).layout + def show(block: Block): String = pretty(toDoc(block), 80).layout + def show(bindings: Bindings): String = pretty(toDoc(bindings), 80).layout + } +} diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala similarity index 99% rename from effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala rename to effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala index cf36230fe..09ea0064d 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/theories/Integers.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala @@ -1,4 +1,4 @@ -package effekt.core.optimizer.theories +package effekt.core.optimizer.normalizer.theories import effekt.core.Block.BlockVar import effekt.core.{Expr, Id, Type} From ce03f5de9905dc210c09cf2168e5f125db01e6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 10:19:11 +0100 Subject: [PATCH 112/123] Improve comments --- .../effekt/core/optimizer/normalizer/semantics.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala index 729ea1fd1..a15456ef7 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala @@ -25,14 +25,16 @@ object semantics { def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet enum Value { - // Stuck + // Stuck (neutrals) case Var(id: Id, annotatedType: ValueType) case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) - // Actual Values - case Literal(value: Any, annotatedType: ValueType) + // Values with specialized representation for algebraic simplification case Integer(value: theories.Integers.Integer) + // Fallback literal for other values types without special representation + case Literal(value: Any, annotatedType: ValueType) + case Make(data: ValueType.Data, tag: Id, targs: List[ValueType], vargs: List[Addr]) // TODO use dynamic captures From 73a85800cbded50afe9702bb479cca5355a42839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 10:58:48 +0100 Subject: [PATCH 113/123] Support more builtin simplification on literals --- .../core/optimizer/normalizer/builtins.scala | 216 ++++++++++++++++-- .../main/scala/effekt/core/vm/Builtin.scala | 1 + 2 files changed, 200 insertions(+), 17 deletions(-) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala index 4f20db2a6..d3a1500f9 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -11,6 +11,24 @@ def builtin(name: String)(impl: List[semantics.Value] ~> semantics.Value): (Stri type Builtins = Map[String, BuiltinImpl] +given Conversion[Long, semantics.Value] with + def apply(n: Long): semantics.Value = + semantics.Value.Integer(theories.Integers.embed(n)) + +given Conversion[Boolean, semantics.Value] with + def apply(b: Boolean): semantics.Value = + semantics.Value.Literal(b, Type.TBoolean) + +given Conversion[String, semantics.Value] with + def apply(s: String): semantics.Value = + semantics.Value.Literal(s, Type.TString) + +given Conversion[Double, semantics.Value] with + def apply(d: Double): semantics.Value = + semantics.Value.Literal(d, Type.TDouble) + +lazy val supportedBuiltins: Builtins = integers ++ doubles ++ booleans ++ strings ++ chars + lazy val integers: Builtins = Map( // Integer arithmetic operations with symbolic simplification support // ---------- @@ -26,58 +44,201 @@ lazy val integers: Builtins = Map( // Integer arithmetic operations only evaluated for literals // ---------- builtin("effekt::infixDiv(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x / y)) + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => x / y }, builtin("effekt::mod(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil if y != 0 => semantics.Value.Integer(theories.Integers.embed(x % y)) + case As.Int(x) :: As.Int(y) :: Nil if y != 0 => x % y }, builtin("effekt::bitwiseShl(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x << y)) + case As.Int(x) :: As.Int(y) :: Nil => x << y }, builtin("effekt::bitwiseShr(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x >> y)) + case As.Int(x) :: As.Int(y) :: Nil => x >> y }, builtin("effekt::bitwiseAnd(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x & y)) + case As.Int(x) :: As.Int(y) :: Nil => x & y }, builtin("effekt::bitwiseOr(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x | y)) + case As.Int(x) :: As.Int(y) :: Nil => x | y }, builtin("effekt::bitwiseXor(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Integer(theories.Integers.embed(x ^ y)) + case As.Int(x) :: As.Int(y) :: Nil => x ^ y }, // Comparison // ---------- builtin("effekt::infixEq(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x == y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x == y }, builtin("effekt::infixNeq(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x != y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x != y }, builtin("effekt::infixLt(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x < y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x < y }, builtin("effekt::infixGt(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x > y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x > y }, builtin("effekt::infixLte(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x <= y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x <= y }, builtin("effekt::infixGte(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => semantics.Value.Literal(x >= y, Type.TBoolean) + case As.Int(x) :: As.Int(y) :: Nil => x >= y }, // Conversion // ---------- builtin("effekt::toDouble(Int)") { - case As.Int(x) :: Nil => semantics.Value.Literal(x.toDouble, Type.TDouble) + case As.Int(x) :: Nil => x.toDouble }, builtin("effekt::show(Int)") { - case As.Int(n) :: Nil => semantics.Value.Literal(n.toString, Type.TString) + case As.Int(n) :: Nil => n.toString + }, +) + +lazy val doubles: Builtins = Map( + // Arithmetic + // ---------- + builtin("effekt::infixAdd(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x + y + }, + builtin("effekt::infixSub(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x - y + }, + builtin("effekt::infixMul(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x * y + }, + builtin("effekt::infixDiv(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x / y + }, + builtin("effekt::sqrt(Double)") { + case As.Double(x) :: Nil => Math.sqrt(x) + }, + builtin("effekt::exp(Double)") { + case As.Double(x) :: Nil => Math.exp(x) + }, + builtin("effekt::log(Double)") { + case As.Double(x) :: Nil => Math.log(x) + }, + builtin("effekt::cos(Double)") { + case As.Double(x) :: Nil => Math.cos(x) + }, + builtin("effekt::sin(Double)") { + case As.Double(x) :: Nil => Math.sin(x) + }, + builtin("effekt::tan(Double)") { + case As.Double(x) :: Nil => Math.tan(x) + }, + builtin("effekt::atan(Double)") { + case As.Double(x) :: Nil => Math.atan(x) + }, + builtin("effekt::round(Double)") { + case As.Double(x) :: Nil => Math.round(x) + }, + builtin("effekt::pow(Double, Double)") { + case As.Double(base) :: As.Double(exp) :: Nil => Math.pow(base, exp) + }, + builtin("effekt::pi()") { + case Nil => Math.PI + }, + // Comparison + // ---------- + builtin("effekt::infixEq(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x == y + }, + builtin("effekt::infixNeq(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x != y + }, + builtin("effekt::infixLt(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x < y + }, + builtin("effekt::infixGt(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x > y + }, + builtin("effekt::infixLte(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x <= y + }, + builtin("effekt::infixGte(Double, Double)") { + case As.Double(x) :: As.Double(y) :: Nil => x >= y + }, + + // Conversion + // ---------- + builtin("effekt::toInt(Double)") { + case As.Double(x) :: Nil => x.toLong + }, + builtin("effekt::show(Double)") { + // TODO globally define show on double in a decent way... this mimicks JS + case As.Double(n) :: Nil => + if (n == n.toInt.toDouble) n.toInt.toString // Handle integers like 15.0 → 15 + else { + val formatted = BigDecimal(n) + .setScale(15, BigDecimal.RoundingMode.DOWN) // Truncate to 15 decimal places without rounding up + .bigDecimal + .stripTrailingZeros() + .toPlainString + + formatted + } + }, +) + +lazy val booleans: Builtins = Map( + builtin("effekt::not(Bool)") { + case As.Bool(x) :: Nil => !x + }, +) + +lazy val strings: Builtins = Map( + builtin("effekt::infixConcat(String, String)") { + case As.String(x) :: As.String(y) :: Nil => x + y + }, + + builtin("effekt::infixEq(String, String)") { + case As.String(x) :: As.String(y) :: Nil => x == y + }, + + builtin("effekt::length(String)") { + case As.String(x) :: Nil => x.length.toLong + }, + + builtin("effekt::substring(String, Int, Int)") { + case As.String(x) :: As.Int(from) :: As.Int(to) :: Nil => x.substring(from.toInt, to.toInt) + }, + + builtin("string::unsafeCharAt(String, Int)") { + case As.String(x) :: As.Int(at) :: Nil => x.charAt(at.toInt).toLong + }, + + builtin("string::toInt(Char)") { + case As.Int(n) :: Nil => n + }, + + builtin("string::toChar(Int)") { + case As.Int(n) :: Nil => n + }, + + builtin("string::infixLte(Char, Char)") { + case As.Int(x) :: As.Int(y) :: Nil => x <= y + }, + + builtin("string::infixLt(Char, Char)") { + case As.Int(x) :: As.Int(y) :: Nil => x < y + }, + + builtin("string::infixGt(Char, Char)") { + case As.Int(x) :: As.Int(y) :: Nil => x > y + }, + + builtin("string::infixGte(Char, Char)") { + case As.Int(x) :: As.Int(y) :: Nil => x >= y }, ) -lazy val supportedBuiltins: Builtins = integers +lazy val chars: Builtins = Map( + builtin("effekt::infixEq(Char, Char)") { + case As.Int(x) :: As.Int(y) :: Nil => x == y + }, +) protected object As { object Int { @@ -102,4 +263,25 @@ protected object As { case _ => None } } -} \ No newline at end of file + + object Double { + def unapply(v: semantics.Value): Option[scala.Double] = v match { + case semantics.Value.Literal(value: scala.Double, _) => Some(value) + case _ => None + } + } + + object String { + def unapply(v: semantics.Value): Option[java.lang.String] = v match { + case semantics.Value.Literal(value: java.lang.String, _) => Some(value) + case _ => None + } + } + + object Bool { + def unapply(v: semantics.Value): Option[scala.Boolean] = v match { + case semantics.Value.Literal(value: scala.Boolean, _) => Some(value) + case _ => None + } + } +} diff --git a/effekt/shared/src/main/scala/effekt/core/vm/Builtin.scala b/effekt/shared/src/main/scala/effekt/core/vm/Builtin.scala index 1484ecceb..73a9100c6 100644 --- a/effekt/shared/src/main/scala/effekt/core/vm/Builtin.scala +++ b/effekt/shared/src/main/scala/effekt/core/vm/Builtin.scala @@ -5,6 +5,7 @@ package vm import java.io.PrintStream import scala.util.matching as regex import scala.util.matching.Regex +import scala.Conversion trait Runtime { def out: PrintStream From 88d051694c9c7e13d45f06f95669bcfff51760a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 11:11:30 +0100 Subject: [PATCH 114/123] Lowercase integer theory object --- .../scala/effekt/core/NewNormalizerTests.scala | 3 ++- .../optimizer/normalizer/NewNormalizer.scala | 2 +- .../core/optimizer/normalizer/builtins.scala | 18 +++++++++--------- .../core/optimizer/normalizer/semantics.scala | 2 +- .../{Integers.scala => integers.scala} | 6 +++--- 5 files changed, 16 insertions(+), 15 deletions(-) rename effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/{Integers.scala => integers.scala} (97%) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 542cf4194..9bdb2b522 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -2,7 +2,8 @@ package effekt.core import effekt.PhaseResult.CoreTransformed import effekt.context.{Context, IOModuleDB} -import effekt.core.optimizer.{Deadcode, NewNormalizer, Normalizer, Optimizer} +import effekt.core.optimizer.{Deadcode, Normalizer, Optimizer} +import effekt.core.optimizer.normalizer.NewNormalizer import effekt.util.PlainMessaging import effekt.* import kiama.output.PrettyPrinterTypes.Document diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala index f2535b855..d9fbc01a9 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala @@ -616,7 +616,7 @@ class NewNormalizer { case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) - case Value.Integer(value) => theories.Integers.reify(value, cx.builtinBlockVars) + case Value.Integer(value) => theories.integers.reify(value, cx.builtinBlockVars) } def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala index d3a1500f9..280ee2c73 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -13,7 +13,7 @@ type Builtins = Map[String, BuiltinImpl] given Conversion[Long, semantics.Value] with def apply(n: Long): semantics.Value = - semantics.Value.Integer(theories.Integers.embed(n)) + semantics.Value.Integer(theories.integers.embed(n)) given Conversion[Boolean, semantics.Value] with def apply(b: Boolean): semantics.Value = @@ -33,13 +33,13 @@ lazy val integers: Builtins = Map( // Integer arithmetic operations with symbolic simplification support // ---------- builtin("effekt::infixAdd(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.add(x, y)) + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.add(x, y)) }, builtin("effekt::infixSub(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.sub(x, y)) + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.sub(x, y)) }, builtin("effekt::infixMul(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.Integers.mul(x, y)) + case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.mul(x, y)) }, // Integer arithmetic operations only evaluated for literals // ---------- @@ -251,13 +251,13 @@ protected object As { } object IntExpr { - def unapply(v: semantics.Value): Option[theories.Integers.Integer] = v match { + def unapply(v: semantics.Value): Option[theories.integers.Integer] = v match { // Integer literals not yet embedded into the theory of integers - case semantics.Value.Literal(value: scala.Long, _) => Some(theories.Integers.embed(value)) - case semantics.Value.Literal(value: scala.Int, _) => Some(theories.Integers.embed(value.toLong)) - case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.Integers.embed(value.toLong)) + case semantics.Value.Literal(value: scala.Long, _) => Some(theories.integers.embed(value)) + case semantics.Value.Literal(value: scala.Int, _) => Some(theories.integers.embed(value.toLong)) + case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.integers.embed(value.toLong)) // Variables of type integer - case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.Integers.embed(id)) + case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.integers.embed(id)) // Already embedded integers case semantics.Value.Integer(value) => Some(value) case _ => None diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala index a15456ef7..2f0f36be3 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala @@ -30,7 +30,7 @@ object semantics { case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) // Values with specialized representation for algebraic simplification - case Integer(value: theories.Integers.Integer) + case Integer(value: theories.integers.Integer) // Fallback literal for other values types without special representation case Literal(value: Any, annotatedType: ValueType) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala similarity index 97% rename from effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala rename to effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala index 09ea0064d..08f6251c1 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/Integers.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala @@ -9,7 +9,7 @@ import effekt.core.{Expr, Id, Type} * KNOWN LIMITATION: This implementation assumes 64-bit signed integers. * Unfortunately, this is unsound for the JavaScript backend, which uses JavaScript numbers that are IEEE-754 doubles. */ -object Integers { +object integers { case class Integer(value: Long, addends: Addends) { val free: Set[Id] = addends.flatMap { case (factors, _) => factors.keys }.toSet @@ -36,8 +36,8 @@ object Integers { } import Operation._ - def embed(value: Long): Integers.Integer = Integer(value, Map.empty) - def embed(id: Id): Integers.Integer = Integer(0, Map(Map(id -> 1) -> 1)) + def embed(value: Long): integers.Integer = Integer(value, Map.empty) + def embed(id: Id): integers.Integer = Integer(0, Map(Map(id -> 1) -> 1)) def reify(value: Integer, embedBuiltinName: String => BlockVar): Expr = Reify(embedBuiltinName).reify(value) From 0a6b44adba2246092ea2584ec78f3174a7927786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 14:20:19 +0100 Subject: [PATCH 115/123] Add initial theory of strings --- .../optimizer/normalizer/NewNormalizer.scala | 10 ++- .../core/optimizer/normalizer/builtins.scala | 27 +++++--- .../core/optimizer/normalizer/semantics.scala | 10 ++- .../normalizer/theories/integers.scala | 69 ++++++++++--------- .../normalizer/theories/strings.scala | 50 ++++++++++++++ 5 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/strings.scala diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala index d9fbc01a9..9dce6ae61 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala @@ -183,7 +183,7 @@ class NewNormalizer { env.lookupValue(id) case core.Expr.Literal(value, annotatedType) => value match { - case As.IntExpr(x) => scope.allocate("x", Value.Integer(x)) + case As.IntRep(x) => scope.allocate("x", Value.Integer(x)) case _ => scope.allocate("x", Value.Literal(value, annotatedType)) } @@ -616,7 +616,13 @@ class NewNormalizer { case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) - case Value.Integer(value) => theories.integers.reify(value, cx.builtinBlockVars) + case Value.Integer(value) => theories.integers.reify(value, cx.builtinBlockVars, embedNeutral) + case Value.String(value) => theories.strings.reify(value, cx.builtinBlockVars, embedNeutral) + } + + def embedNeutral(neutral: Neutral)(using G: TypingContext): core.Expr = neutral match { + case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) + case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) } def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala index 280ee2c73..f72e8c0ab 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -33,13 +33,13 @@ lazy val integers: Builtins = Map( // Integer arithmetic operations with symbolic simplification support // ---------- builtin("effekt::infixAdd(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.add(x, y)) + case As.IntRep(x) :: As.IntRep(y) :: Nil => semantics.Value.Integer(theories.integers.add(x, y)) }, builtin("effekt::infixSub(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.sub(x, y)) + case As.IntRep(x) :: As.IntRep(y) :: Nil => semantics.Value.Integer(theories.integers.sub(x, y)) }, builtin("effekt::infixMul(Int, Int)") { - case As.IntExpr(x) :: As.IntExpr(y) :: Nil => semantics.Value.Integer(theories.integers.mul(x, y)) + case As.IntRep(x) :: As.IntRep(y) :: Nil => semantics.Value.Integer(theories.integers.mul(x, y)) }, // Integer arithmetic operations only evaluated for literals // ---------- @@ -190,7 +190,7 @@ lazy val booleans: Builtins = Map( lazy val strings: Builtins = Map( builtin("effekt::infixConcat(String, String)") { - case As.String(x) :: As.String(y) :: Nil => x + y + case As.StringRep(x) :: As.StringRep(y) :: Nil => semantics.Value.String(theories.strings.concat(x, y)) }, builtin("effekt::infixEq(String, String)") { @@ -246,18 +246,19 @@ protected object As { case semantics.Value.Literal(value: scala.Long, _) => Some(value) case semantics.Value.Literal(value: scala.Int, _) => Some(value.toLong) case semantics.Value.Literal(value: java.lang.Integer, _) => Some(value.toLong) + case semantics.Value.Integer(value) if value.isLiteral => Some(value.value) case _ => None } } - object IntExpr { - def unapply(v: semantics.Value): Option[theories.integers.Integer] = v match { + object IntRep { + def unapply(v: semantics.Value): Option[theories.integers.IntegerRep] = v match { // Integer literals not yet embedded into the theory of integers case semantics.Value.Literal(value: scala.Long, _) => Some(theories.integers.embed(value)) case semantics.Value.Literal(value: scala.Int, _) => Some(theories.integers.embed(value.toLong)) case semantics.Value.Literal(value: java.lang.Integer, _) => Some(theories.integers.embed(value.toLong)) - // Variables of type integer - case semantics.Value.Var(id, tpe) if tpe == Type.TInt => Some(theories.integers.embed(id)) + // Neutrals (e.g. variables or extern calls) + case n: semantics.Neutral => Some(theories.integers.embed(n)) // Already embedded integers case semantics.Value.Integer(value) => Some(value) case _ => None @@ -274,6 +275,16 @@ protected object As { object String { def unapply(v: semantics.Value): Option[java.lang.String] = v match { case semantics.Value.Literal(value: java.lang.String, _) => Some(value) + case semantics.Value.String(value) if value.isLiteral => Some(value.value.head.asInstanceOf[java.lang.String]) + case _ => None + } + } + + object StringRep { + def unapply(v: semantics.Value): Option[theories.strings.StringRep] = v match { + case semantics.Value.Literal(value: java.lang.String, _) => Some(theories.strings.embed(value)) + case n: semantics.Neutral => Some(theories.strings.embed(n)) + case semantics.Value.String(value) => Some(value) case _ => None } } diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala index 2f0f36be3..d0448af2c 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala @@ -24,13 +24,16 @@ object semantics { type Variables = Set[Id] def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet + type Neutral = Value.Var | Value.Extern + enum Value { // Stuck (neutrals) case Var(id: Id, annotatedType: ValueType) case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) // Values with specialized representation for algebraic simplification - case Integer(value: theories.integers.Integer) + case Integer(value: theories.integers.IntegerRep) + case String(value: theories.strings.StringRep) // Fallback literal for other values types without special representation case Literal(value: Any, annotatedType: ValueType) @@ -47,6 +50,7 @@ object semantics { case Value.Extern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Integer(value) => value.free + case Value.String(value) => value.free case Value.Make(data, tag, targs, vargs) => vargs.toSet // Box abstracts over all free computation variables, only when unboxing, they occur free again case Value.Box(body, tpe) => body.free @@ -680,6 +684,8 @@ object semantics { parens(hsep(vargs.map(toDoc), comma)) case Value.Literal(value, _) => util.show(value) + case Value.Integer(value) => value.show + case Value.String(value) => value.show case Value.Make(data, tag, targs, vargs) => "make" <+> toDoc(data) <+> toDoc(tag) <> @@ -690,8 +696,6 @@ object semantics { "box" <+> braces(nest(line <> toDoc(body) <> line)) case Value.Var(id, tpe) => toDoc(id) - - case Value.Integer(value) => value.show } def toDoc(block: Block): Doc = block match { diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala index 08f6251c1..359f41e80 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/integers.scala @@ -2,24 +2,26 @@ package effekt.core.optimizer.normalizer.theories import effekt.core.Block.BlockVar import effekt.core.{Expr, Id, Type} +import effekt.core.optimizer.normalizer.semantics.Neutral /** * Theory for integers with neutral variables: multivariate Laurent polynomials with 64-bit signed integer coefficients. + * Compare https://en.wikipedia.org/wiki/Laurent_polynomial * * KNOWN LIMITATION: This implementation assumes 64-bit signed integers. * Unfortunately, this is unsound for the JavaScript backend, which uses JavaScript numbers that are IEEE-754 doubles. */ object integers { - case class Integer(value: Long, addends: Addends) { - val free: Set[Id] = addends.flatMap { case (factors, _) => factors.keys }.toSet - + case class IntegerRep(value: Long, addends: Addends) { + val free: Set[Id] = addends.flatMap { case (factors, _) => factors.keys.flatMap(_.free) }.toSet + def isLiteral: Boolean = addends.isEmpty def show: String = { - val Integer(v, a) = this + val IntegerRep(v, a) = this val terms = a.map { case (factors, n) => val factorStr = if (factors.isEmpty) "1" else { - factors.map { case (id, exp) => - if (exp == 1) s"${id.name}" - else s"${id.name}^$exp" + factors.map { case (n, exp) => + if (exp == 1) "" + else s"^$exp" }.mkString("*") } if (n == 1) s"$factorStr" @@ -36,24 +38,25 @@ object integers { } import Operation._ - def embed(value: Long): integers.Integer = Integer(value, Map.empty) - def embed(id: Id): integers.Integer = Integer(0, Map(Map(id -> 1) -> 1)) + def embed(value: Long): integers.IntegerRep = IntegerRep(value, Map.empty) + def embed(n: Neutral): integers.IntegerRep = IntegerRep(0, Map(Map(n -> 1) -> 1)) - def reify(value: Integer, embedBuiltinName: String => BlockVar): Expr = Reify(embedBuiltinName).reify(value) + def reify(value: IntegerRep, embedBuiltinName: String => BlockVar, embedNeutral: Neutral => Expr): Expr = + Reify(embedBuiltinName, embedNeutral).reify(value) // 3 * x * x / y = Addend(3, Map(x -> 2, y -> -1)) type Addends = Map[Factors, Long] - type Factors = Map[Id, Int] + type Factors = Map[Neutral, Int] - def normalize(n: Integer): Integer = normalized(n.value, n.addends) + def normalize(n: IntegerRep): IntegerRep = normalized(n.value, n.addends) - def normalized(value: Long, addends: Addends): Integer = + def normalized(value: Long, addends: Addends): IntegerRep = val (const, norm) = normalizeAddends(addends) - Integer(value + const, norm) + IntegerRep(value + const, norm) - def add(l: Integer, r: Integer): Integer = (l, r) match { + def add(l: IntegerRep, r: IntegerRep): IntegerRep = (l, r) match { // 2 + (3 * x) + 4 + (5 * y) = 6 + (3 * x) + (5 * y) - case (Integer(x, xs), Integer(y, ys)) => + case (IntegerRep(x, xs), IntegerRep(y, ys)) => normalized(x + y, add(xs, ys)) } @@ -81,20 +84,20 @@ object integers { (constant, filtered) } - def neg(l: Integer): Integer = mul(l, -1) + def neg(l: IntegerRep): IntegerRep = mul(l, -1) // (42 + 3*x + y) - (42 + 3*x + y) = (42 + 3*x + y) + (-1*42 + -1*3*x + -1*y) - def sub(l: Integer, r: Integer): Integer = + def sub(l: IntegerRep, r: IntegerRep): IntegerRep = add(l, neg(r)) - def mul(l: Integer, factor: Long): Integer = l match { - case Integer(value, addends) => - Integer(value * factor, addends.map { case (f, n) => f -> n * factor }) + def mul(l: IntegerRep, factor: Long): IntegerRep = l match { + case IntegerRep(value, addends) => + IntegerRep(value * factor, addends.map { case (f, n) => f -> n * factor }) } - def mul(l: Integer, factor: Factors): Integer = l match { - case Integer(value, addends) => - Integer(0, Map(factor -> value) ++ addends.map { case (f, n) => + def mul(l: IntegerRep, factor: Factors): IntegerRep = l match { + case IntegerRep(value, addends) => + IntegerRep(0, Map(factor -> value) ++ addends.map { case (f, n) => mul(f, factor) -> n }) } @@ -111,20 +114,20 @@ object integers { // x1^2 * x2^0 * x3^3 = x1^2 * x3^3 def normalizeFactors(f: Factors): Factors = - f.filterNot { case (id, exp) => exp == 0 } + f.filterNot { case (n, exp) => exp == 0 } // (42 + 3*x + y) * (42 + 3*x + y) // = // (42 + 3*x + y) * 42 + (42 + 3*x + y) * 3*x + (42 + 3*x + y) * y - def mul(l: Integer, r: Integer): Integer = r match { - case Integer(y, ys) => - var sum: Integer = mul(l, y) + def mul(l: IntegerRep, r: IntegerRep): IntegerRep = r match { + case IntegerRep(y, ys) => + var sum: IntegerRep = mul(l, y) ys.foreach { case (f, n) => sum = add(sum, mul(mul(l, n), f)) } normalize(sum) } - case class Reify(embedBuiltinName: String => BlockVar) { - def reifyVar(id: Id): Expr = Expr.ValueVar(id, Type.TInt) + case class Reify(embedBuiltinName: String => BlockVar, embedNeutral: Neutral => Expr) { + def reifyVar(n: Neutral): Expr = embedNeutral(n) def reifyInt(v: Long): Expr = Expr.Literal(v, Type.TInt) @@ -135,8 +138,8 @@ object integers { case Div => Expr.PureApp(embedBuiltinName("effekt::infixDiv(Int, Int)"), List(), List(l, r)) } - def reify(v: Integer): Expr = - val Integer(const, addends) = normalize(v) + def reify(v: IntegerRep): Expr = + val IntegerRep(const, addends) = normalize(v) val adds = addends.toList.map { case (factors, n) => if (n == 1) reifyFactors(factors) @@ -150,7 +153,7 @@ object integers { reifyInt(const) } - def reifyFactor(x: Id, n: Int): Expr = + def reifyFactor(x: Neutral, n: Int): Expr = if (n <= 0) sys error "Should not happen" else if (n == 1) reifyVar(x) else reifyOp(reifyVar(x), Mul, reifyFactor(x, n - 1)) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/strings.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/strings.scala new file mode 100644 index 000000000..ab79a732e --- /dev/null +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/theories/strings.scala @@ -0,0 +1,50 @@ +package effekt.core.optimizer.normalizer.theories + +import effekt.core.Block.BlockVar +import effekt.core.{Expr, Id, Type} +import effekt.core.optimizer.normalizer.semantics.Neutral + +/** + * Theory for strings: free extension of the string monoid + * Compare https://arxiv.org/pdf/2306.15375 + * + * Invariant: there are no adjacent literals in a StringRep + */ +object strings { + case class StringRep(value: List[String | Neutral]) { + val free: Set[Id] = value.collect { case n: Neutral => n.free }.flatten.toSet + def isLiteral: Boolean = value.length == 1 && value.head.isInstanceOf[String] + def show: String = { + val terms = value.map { + case s: String => s""""$s"""" + case n: Neutral => "" + } + terms.mkString(" ++ ") + } + } + + def embed(value: String): StringRep = StringRep(List(value)) + def embed(value: Neutral): StringRep = StringRep(List(value)) + + def reify(value: StringRep, embedBuiltinName: String => BlockVar, embedNeutral: Neutral => Expr): Expr = value match { + case StringRep(parts) => + parts.map { + case s: String => Expr.Literal(s, Type.TString) + case n: Neutral => embedNeutral(n) + }.reduceLeft { (l, r) => + Expr.PureApp(embedBuiltinName("effekt::infixConcat(String, String)"), List(), List(l, r)) + } + } + + def concat(l: StringRep, r: StringRep): StringRep = (l, r) match { + case (StringRep(xs), StringRep(ys)) => + (xs, ys) match { + // fuse trailing / leading string literals at the boundary (if any) + case (init :+ (s1: String), (s2: String) :: tail) => + val concatenated: String | Neutral = s1 + s2 + StringRep((init :+ concatenated) ::: tail) + case _ => + StringRep(xs ::: ys) + } + } +} From 12360291947dfde374fb2c23a8334ee2713803ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 15:21:09 +0100 Subject: [PATCH 116/123] Fix compilation --- .../test/scala/effekt/core/TestRenamer.scala | 31 ++++++------------- .../src/main/scala/effekt/core/Parser.scala | 3 +- .../scala/effekt/core/PrettyPrinter.scala | 2 +- .../src/main/scala/effekt/core/Tree.scala | 5 +-- 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala b/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala index 612446cdb..f5a3c51a0 100644 --- a/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala +++ b/effekt/jvm/src/test/scala/effekt/core/TestRenamer.scala @@ -136,24 +136,6 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends } } - override def rewrite(t: BlockType): BlockType = t match { - case BlockType.Function(tparams, cparams, vparams, bparams, result: ValueType) => - // TODO: is this how we want to treat captures here? - val resolvedCapt = cparams.map(id => Map(id -> freshIdFor(id))).reduceOption(_ ++ _).getOrElse(Map()) - withBindings(tparams) { - withMapping(resolvedCapt) { - BlockType.Function( - tparams.map(rewrite), - resolvedCapt.values.toList.map(rewrite), - vparams.map(rewrite), - bparams.map(rewrite), - rewrite(result) - ) - }} - case BlockType.Interface(name, targs) => - BlockType.Interface(name, targs map rewrite) - } - override def rewrite(o: Operation): Operation = o match { case Operation(name, tparams, cparams, vparams, bparams, body) => withBindings(tparams ++ cparams ++ vparams.map(_.id) ++ bparams.map(_.id)) { @@ -200,7 +182,7 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends } override def rewrite(e: Extern) = e match { - case Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, body) => { + case Extern.Def(id, tparams, cparams, vparams, bparams, ret, annotatedCapture, body, vmBody) => { // We don't use withBinding(id) here, because top-level ids are pre-collected. withBindings(tparams ++ cparams ++ vparams.map(_.id) ++ bparams.map(_.id)) { Extern.Def( @@ -211,7 +193,8 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends bparams map rewrite, rewrite(ret), rewrite(annotatedCapture), - rewrite(body) + rewrite(body), + vmBody ) } } @@ -284,6 +267,12 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends core.ModuleDecl(path, includes, declarations map rewrite, externs map rewrite, definitions map rewrite, exports map rewrite) } + def apply(t: core.Toplevel): core.Toplevel = + suffix = 0 + scopes = List.empty + toplevelScope = Map.empty + rewrite(t) + def apply(s: Stmt): Stmt = { suffix = 0 toplevelScope = Map.empty @@ -298,7 +287,7 @@ class TestRenamer(names: Names = Names(Map.empty), prefix: String = "_") extends case Declaration.Data(id, tparams, constructors) => constructors.map(_.id) :+ id case Interface(id, tparams, properties) => properties.map(_.id) :+ id } ++ definitions.map(_.id) ++ externs.flatMap { - case Extern.Def(id, _, _, _, _, _, _, _) => Some(id) + case Extern.Def(id, _, _, _, _, _, _, _, _) => Some(id) case Extern.Include(_, _) => None } } diff --git a/effekt/shared/src/main/scala/effekt/core/Parser.scala b/effekt/shared/src/main/scala/effekt/core/Parser.scala index cc0cec691..cf807cf90 100644 --- a/effekt/shared/src/main/scala/effekt/core/Parser.scala +++ b/effekt/shared/src/main/scala/effekt/core/Parser.scala @@ -294,7 +294,8 @@ class CoreParsers(names: Names) extends EffektLexers { case captures ~ (id, tparams, cparams, vparams, bparams, result) ~ (ff ~ templ) => Extern.Def( id, tparams, cparams, vparams, bparams, result, captures, - ExternBody.StringExternBody(ff, templ) + ExternBody.StringExternBody(ff, templ), + None ) }) diff --git a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala index 25e93d241..40e103009 100644 --- a/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala +++ b/effekt/shared/src/main/scala/effekt/core/PrettyPrinter.scala @@ -75,7 +75,7 @@ object PrettyPrinter extends ParenPrettyPrinter { vsep(definitions map toDoc) def toDoc(e: Extern): Doc = e match { - case Extern.Def(id, tps, cps, vps, bps, ret, capt, bodies) => + case Extern.Def(id, tps, cps, vps, bps, ret, capt, bodies, _) => "extern" <+> toDoc(capt) <+> "def" <+> toDoc(id) <> paramsToDoc(tps, cps, vps, bps) <> ":" <+> toDoc(ret) <+> "=" <+> (bodies match { case ExternBody.StringExternBody(ff, body) => toDoc(ff) <+> toDoc(body) // The unsupported case is not currently supported by the core parser diff --git a/effekt/shared/src/main/scala/effekt/core/Tree.scala b/effekt/shared/src/main/scala/effekt/core/Tree.scala index 6175312d2..8bfbfb8a6 100644 --- a/effekt/shared/src/main/scala/effekt/core/Tree.scala +++ b/effekt/shared/src/main/scala/effekt/core/Tree.scala @@ -473,8 +473,9 @@ object Tree { def rewrite(o: Operation): Operation = rewriteStructurally(o) def rewrite(p: ValueParam): ValueParam = rewriteStructurally(p) def rewrite(p: BlockParam): BlockParam = rewriteStructurally(p) - def rewrite(b: ExternBody): ExternBody= rewriteStructurally(b) - def rewrite(e: Extern): Extern= rewriteStructurally(e) + def rewrite(b: ExternBody): ExternBody = rewriteStructurally(b) + def rewrite(e: Extern): Extern = rewriteStructurally(e) + def rewrite(s: StringExternBody): StringExternBody = s def rewrite(d: Declaration): Declaration = rewriteStructurally(d) def rewrite(c: Constructor): Constructor = rewriteStructurally(c) def rewrite(f: Field): Field = rewriteStructurally(f) From 5e846c4782822c3cbc0af0d88f0b77df5233cd59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 16:19:25 +0100 Subject: [PATCH 117/123] Fix most NewNormalizerTests --- .../effekt/core/NewNormalizerTests.scala | 93 +++++++++---------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 9bdb2b522..d26c7b804 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -106,17 +106,12 @@ class NewNormalizerTests extends CoreTests { def compareOneDef(name: String): Unit = { val aDef = findDef(actual, name) val eDef = findDef(expected, name) - - val canon = Id(name) - val pairs: Map[Id, Id] = - (List(aDef.id -> canon, eDef.id -> canon) ++ externPairs ++ declPairs ++ ctorPairs).toMap - - val renamer = TestRenamer(Names(defaultNames), "$", List(pairs)) - shouldBeEqual( - renamer(aDef), - renamer(eDef), - s"Top-level '$name' is not alpha-equivalent" - ) + val renamer = TestRenamer(Names(defaultNames), "$") + val obtainedRenamed = renamer(aDef) + val expectedRenamed = renamer(eDef) + val obtainedPrinted = effekt.core.PrettyPrinter.format(obtainedRenamed).layout + val expectedPrinted = effekt.core.PrettyPrinter.format(expectedRenamed).layout + assertEquals(obtainedPrinted, expectedPrinted) } defNames.foreach(compareOneDef) @@ -149,15 +144,17 @@ class NewNormalizerTests extends CoreTests { |def run() = { | def f1() = { | def f2() = { - | let ! x = (foo: () => Int @ {io})() + | let ! x = foo: () => Int @ {io}() | return x: Int | } | let y = box {io} f2: () => Int @ {io} | return y: () => Int at {io} | } - | val z: () => Int at {io} = (f1: () => (() => Int at {io}) @ {io})(); + | val z: () => Int at {io} = { + | f1: () => (() => Int at {io}) @ {io}() + | }; | def r = unbox z: () => Int at {io} - | (r: () => Int @ {io})() + | r: () => Int @ {io}() |} | |""".stripMargin) @@ -246,9 +243,7 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, expected, List("run"), List("foo")) } - // One might expect the following example to constant fold. - // However, extern definitions such as infixAdd are currently always neutral. - test("Extern infixAdd blocks constant folding of mutable variable") { + test("Mutable variable with infixAdd is optimized to a single constant let-binding") { val input = """ |def run(): Int = { @@ -267,10 +262,8 @@ class NewNormalizerTests extends CoreTests { |extern {} def infixAdd(x: Int, y: Int): Int = vm "" | |def run() = { - | let x1 = 41 - | let x2 = 1 - | let x3 = (infixAdd: (Int, Int) => Int @ {})(x1: Int, x2: Int) - | return x3: Int + | let x = 42 + | return x: Int |} |""".stripMargin ) @@ -388,6 +381,7 @@ class NewNormalizerTests extends CoreTests { // This test case shows a mutable variable passed to the identity function. // Currently, the normalizer is not able to see through the identity function, // but it does ignore the mutable variable and just passes the initial value. + // Inlining is performed by a separate inlining phase. test("Pass mutable variable to identity function uses let binding") { val input = """ @@ -409,13 +403,11 @@ class NewNormalizerTests extends CoreTests { |module input | |def run() = { - | def f(x: Int) = { - | return x: Int - | } - | - | let y = 42 - | var x @ z = y: Int; - | (f : (Int) => Int @ {})(y: Int) + | def f(x: Int) = { + | return x: Int + | } + | let y = 42 + | f: (Int) => Int @ {}(y: Int) |} |""".stripMargin ) @@ -446,14 +438,11 @@ class NewNormalizerTests extends CoreTests { |module input | |def run() = { - | def f(x: Int) = { - | return x: Int - | } - | - | let y = 42 - | let w = 43 - | var x @ z = y: Int; - | (f : (Int) => Int @ {})(w: Int) + | def f(x: Int) = { + | return x: Int + | } + | let x = 43 + | f: (Int) => Int @ {}(x: Int) |} |""".stripMargin ) @@ -489,21 +478,27 @@ class NewNormalizerTests extends CoreTests { |module input | |def run() = { - | def modifyProg(){setter: (Int) => Unit} = { - | let x = 2 - | val tmp: Unit = (setter: (Int) => Unit @ {setter})(x: Int); - | let y = () - | return y: Unit + | def modifyProg(){setter @ sc: (Int) => Unit} = { + | let v = 2 + | val o: Unit = { + | setter: (Int) => Unit @ {sc}(v: Int) + | }; + | let u = () + | return u: Unit | } - | let y = 1 - | var x @ c = y: Int; - | def f(y: Int) = { - | put x @ c = y: Int; - | let z = () - | return z: Unit + | let xv = 1 + | def setter(v: Int){x @ xc: Ref[Int]} = { + | put x @ xc = v: Int; + | let u = () + | return u: Unit | } - | val tmp: Unit = (modifyProg: (){setter : (Int) => Unit} => Unit @ {})(){f: (Int) => Unit @ {c}}; - | get o: Int = !x @ c; + | var x @ x = xv: Int; + | val r: Unit = { + | modifyProg: (){setter: (Int) => Unit} => Unit @ {}(){ (v: Int) => + | setter: (Int){x: Ref[Int]} => Unit @ {}(v: Int){x: Ref[Int] @ {xc}} + | } + | }; + | get o : Int = ! x @ xc; | return o: Int |} |""".stripMargin From f8256b69e7f2ad59cd39744fca2459414856dd5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Thu, 27 Nov 2025 16:35:31 +0100 Subject: [PATCH 118/123] Add test case for compile-time string concat --- .../effekt/core/NewNormalizerTests.scala | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index d26c7b804..bc2c5d27f 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -494,7 +494,7 @@ class NewNormalizerTests extends CoreTests { | } | var x @ x = xv: Int; | val r: Unit = { - | modifyProg: (){setter: (Int) => Unit} => Unit @ {}(){ (v: Int) => + | modifyProg: (){setter: (Int) => Unit} => Unit @ {}(){ (v: Int) => | setter: (Int){x: Ref[Int]} => Unit @ {}(v: Int){x: Ref[Int] @ {xc}} | } | }; @@ -701,6 +701,33 @@ class NewNormalizerTests extends CoreTests { // Does not throw normalize(input) } + + test("Compile-time string concatenation with neutral calls in between") { + val input = + """ + |extern def foo: String = vm"" + | + |def run(): String = { + | "a" ++ "b" ++ foo() ++ "c" ++ "d" + |} + | + |def main() = println(run()) + |""".stripMargin + + val expected = + """module input + |extern {io} def foo(): String = vm"42" + |def run() = { + | let ! s2 = foo: () => String @ {io}() + | let s1 = "ab" + | let r = (infixConcat: (String, String) => String @ {})((infixConcat: (String, String) => String @ {})(s1: String, s2: String), "cd") + | return r: String + |} + |""".stripMargin + + val (mainId, actual) = normalize(input) + assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("foo")) + } } /** From 5cfb485536f90d1d8e7f1f27e85a871bfcc7808c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 1 Dec 2025 12:09:37 +0100 Subject: [PATCH 119/123] Work on reflexivity for neutral integers --- .../effekt/core/NewNormalizerTests.scala | 89 +++++++++++++++++++ .../optimizer/normalizer/NewNormalizer.scala | 7 +- .../core/optimizer/normalizer/builtins.scala | 7 +- .../core/optimizer/normalizer/semantics.scala | 8 +- 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index bc2c5d27f..ac1df129f 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -728,6 +728,95 @@ class NewNormalizerTests extends CoreTests { val (mainId, actual) = normalize(input) assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("foo")) } + + // Equality checks on neutral integer expressions where both sides have the same normal form can be evaluated at compile time. + test("Compile-time integer equality on certain neutral expressions") { + val input = + """ + |extern def z: Int = vm"0" + | + |def run(x: Int): Bool = { + | 42 + 2 * x == x + 42 + x + |} + | + |def main() = { + | val x = z() + | println(run(x)) + |} + |""".stripMargin + + val expected = + """module input + |extern {} def infixEq(x: Int, y: Int): Bool = vm "" + |def run(x: Int) = { + | let r = true + | return r: Bool + |} + |""".stripMargin + + val (mainId, actual) = normalize(input) + assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) + } + + // Even for "pure" extern calls, we currently do not assume that both calls return the same value. + // This should be improved. + test("We do not assume different extern calls return the same value") { + val input = + """ + |extern def z: Int = vm"0" + | + |def run(x: Int): Bool = { + | z() == z() + |} + | + |def main() = { + | val x = z() + | println(run(x)) + |} + |""".stripMargin + + val expected = + """module input + |extern {} def infixEq(x: Int, y: Int): Bool = vm "" + |def run(x: Int) = { + | let ! r1 = z: () => Int @ {io}() + | let ! r2 = z: () => Int @ {io}() + | let o = (infixEq: (Int, Int) => Bool @ {})(r1: Int, r2: Int) + | return o: Bool + |} + |""".stripMargin + + val (mainId, actual) = normalize(input) + assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) + } + + test("Reflexivity holds for pure neutral extern defs") { + val input = + """ + |extern def z at {}: Int = vm"0" + | + |def run(): Bool = { + | val x = z() + | x == x + |} + | + |def main() = { + | println(run()) + |} + |""".stripMargin + + val expected = + """module input + |extern {} def infixEq(x: Int, y: Int): Bool = vm "" + |def run() = { + | let o = true + | return o: Bool + |} + |""".stripMargin + + val (mainId, actual) = normalize(input) + assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) + } } /** diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala index 9dce6ae61..7e021d9f7 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/NewNormalizer.scala @@ -184,6 +184,7 @@ class NewNormalizer { case core.Expr.Literal(value, annotatedType) => value match { case As.IntRep(x) => scope.allocate("x", Value.Integer(x)) + case As.StringRep(x) => scope.allocate("x", Value.String(x)) case _ => scope.allocate("x", Value.Literal(value, annotatedType)) } @@ -202,7 +203,7 @@ class NewNormalizer { val impl = supportedBuiltins(name) val res = impl(values) scope.allocate("x", res) - case _ => scope.allocate("x", Value.Extern(f, targs, vargs.map(evaluate(_, escaping)))) + case _ => scope.allocate("x", Value.PureExtern(f, targs, vargs.map(evaluate(_, escaping)))) } case core.Expr.Make(data, tag, targs, vargs) => @@ -611,7 +612,7 @@ class NewNormalizer { } def embedExpr(value: Value)(using cx: TypingContext): core.Expr = value match { - case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) + case Value.PureExtern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) case Value.Literal(value, annotatedType) => Expr.Literal(value, annotatedType) case Value.Make(data, tag, targs, vargs) => Expr.Make(data, tag, targs, vargs.map(embedExpr)) case Value.Box(body, annotatedCapture) => Expr.Box(embedBlock(body), annotatedCapture) @@ -622,7 +623,7 @@ class NewNormalizer { def embedNeutral(neutral: Neutral)(using G: TypingContext): core.Expr = neutral match { case Value.Var(id, annotatedType) => Expr.ValueVar(id, annotatedType) - case Value.Extern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) + case Value.PureExtern(callee, targs, vargs) => Expr.PureApp(callee, targs, vargs.map(embedExpr)) } def embedExpr(addr: Addr)(using G: TypingContext): core.Expr = Expr.ValueVar(addr, G.lookupValue(addr)) diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala index f72e8c0ab..8d7eee5cf 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -41,6 +41,10 @@ lazy val integers: Builtins = Map( builtin("effekt::infixMul(Int, Int)") { case As.IntRep(x) :: As.IntRep(y) :: Nil => semantics.Value.Integer(theories.integers.mul(x, y)) }, + builtin("effekt::infixEq(Int, Int)") { + case As.IntRep(x) :: As.IntRep(y) :: Nil if x == y => true + case As.Int(x) :: As.Int(y) :: Nil => x == y + }, // Integer arithmetic operations only evaluated for literals // ---------- builtin("effekt::infixDiv(Int, Int)") { @@ -66,9 +70,6 @@ lazy val integers: Builtins = Map( }, // Comparison // ---------- - builtin("effekt::infixEq(Int, Int)") { - case As.Int(x) :: As.Int(y) :: Nil => x == y - }, builtin("effekt::infixNeq(Int, Int)") { case As.Int(x) :: As.Int(y) :: Nil => x != y }, diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala index d0448af2c..b681b27d5 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/semantics.scala @@ -24,12 +24,12 @@ object semantics { type Variables = Set[Id] def all[A](ts: List[A], f: A => Variables): Variables = ts.flatMap(f).toSet - type Neutral = Value.Var | Value.Extern + type Neutral = Value.Var | Value.PureExtern enum Value { // Stuck (neutrals) case Var(id: Id, annotatedType: ValueType) - case Extern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) + case PureExtern(f: BlockVar, targs: List[ValueType], vargs: List[Addr]) // Values with specialized representation for algebraic simplification case Integer(value: theories.integers.IntegerRep) @@ -47,7 +47,7 @@ object semantics { val free: Variables = this match { case Value.Var(id, annotatedType) => Set(id) - case Value.Extern(id, targs, vargs) => vargs.toSet + case Value.PureExtern(id, targs, vargs) => vargs.toSet case Value.Literal(value, annotatedType) => Set.empty case Value.Integer(value) => value.free case Value.String(value) => value.free @@ -678,7 +678,7 @@ object semantics { def toDoc(value: Value): Doc = value match { // case Value.Var(id, tpe) => toDoc(id) - case Value.Extern(callee, targs, vargs) => + case Value.PureExtern(callee, targs, vargs) => toDoc(callee.id) <> (if (targs.isEmpty) emptyDoc else brackets(hsep(targs.map(toDoc), comma))) <> parens(hsep(vargs.map(toDoc), comma)) From 2ca73190d5e78ba4bf99a39ef81422332cceb09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 1 Dec 2025 12:15:13 +0100 Subject: [PATCH 120/123] Fix test case for non-reflexivity of impure extern calls --- .../src/test/scala/effekt/core/NewNormalizerTests.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index ac1df129f..90008ae02 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -758,12 +758,11 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) } - // Even for "pure" extern calls, we currently do not assume that both calls return the same value. - // This should be improved. - test("We do not assume different extern calls return the same value") { + // Even for impure extern calls, we do not assume that different calls return the same value. + test("Reflexivity does not hold for impure neutral extern defs") { val input = """ - |extern def z: Int = vm"0" + |extern def z at {io}: Int = vm"0" | |def run(x: Int): Bool = { | z() == z() @@ -777,7 +776,7 @@ class NewNormalizerTests extends CoreTests { val expected = """module input - |extern {} def infixEq(x: Int, y: Int): Bool = vm "" + |extern {io} def infixEq(x: Int, y: Int): Bool = vm "" |def run(x: Int) = { | let ! r1 = z: () => Int @ {io}() | let ! r2 = z: () => Int @ {io}() From 53556f113356deebcd17f2a2534f0d11e2c181d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 1 Dec 2025 15:55:45 +0100 Subject: [PATCH 121/123] Evaluate infixEq with string neutrals --- .../effekt/core/NewNormalizerTests.scala | 30 ++++++++++++++++++- .../core/optimizer/normalizer/builtins.scala | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index 90008ae02..e91264d07 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -789,7 +789,7 @@ class NewNormalizerTests extends CoreTests { assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) } - test("Reflexivity holds for pure neutral extern defs") { + test("Reflexivity holds for pure neutral extern defs that return integers") { val input = """ |extern def z at {}: Int = vm"0" @@ -816,6 +816,34 @@ class NewNormalizerTests extends CoreTests { val (mainId, actual) = normalize(input) assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) } + + test("infixEq on strings with pure neutral extern defs") { + val input = + """ + |extern def s at {}: String = vm"hello" + | + |def run(): Bool = { + | val x = s() + | ("a" ++ x) ++ "b" == "a" ++ (x ++ "b") + |} + | + |def main() = { + | println(run()) + |} + |""".stripMargin + + val expected = + """module input + |extern {} def infixEq(x: String, y: String): Bool = vm "" + |def run() = { + | let o = true + | return o: Bool + |} + |""".stripMargin + + val (mainId, actual) = normalize(input) + assertAlphaEquivalentToplevels(actual, parse(expected), List("run"), List("infixEq")) + } } /** diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala index 8d7eee5cf..3840e5689 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/normalizer/builtins.scala @@ -195,6 +195,7 @@ lazy val strings: Builtins = Map( }, builtin("effekt::infixEq(String, String)") { + case As.StringRep(x) :: As.StringRep(y) :: Nil if x == y => true case As.String(x) :: As.String(y) :: Nil => x == y }, From 029915372298f3bffe4334b2a5e85da0900da1d9 Mon Sep 17 00:00:00 2001 From: dvdvgt <40773635+dvdvgt@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:33:45 +0100 Subject: [PATCH 122/123] move new normalizer --- .../src/main/scala/effekt/core/optimizer/NewNormalizer.scala | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/NewNormalizer.scala deleted file mode 100644 index e69de29bb..000000000 From 8e407055b52cdd0465e405f2f2dba75baad924b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20S=C3=BCberkr=C3=BCb?= Date: Mon, 1 Dec 2025 17:15:54 +0100 Subject: [PATCH 123/123] Hide new normalizer behind flag --- .../src/main/scala/effekt/EffektConfig.scala | 13 +++++ .../effekt/core/NewNormalizerTests.scala | 3 +- .../effekt/core/optimizer/Optimizer.scala | 50 +++++++++++++++++-- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/effekt/jvm/src/main/scala/effekt/EffektConfig.scala b/effekt/jvm/src/main/scala/effekt/EffektConfig.scala index 154b49974..e8b245b64 100644 --- a/effekt/jvm/src/main/scala/effekt/EffektConfig.scala +++ b/effekt/jvm/src/main/scala/effekt/EffektConfig.scala @@ -200,6 +200,19 @@ class EffektConfig(args: Seq[String]) extends REPLConfig(args.takeWhile(_ != "-- group = debugging ) + // Experimental + // ------------ + + lazy val newNormalizer = toggle( + "new-normalizer", + descrYes = "Use the new normalizer implementation", + descrNo = "Use the old normalizer implementation", + default = Some(false), + noshort = true, + prefix = "no-", + group = group("Experimental Features") + ) + /** * Tries to find the path to the standard library. Proceeds in the following * order: diff --git a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala index e91264d07..795a0b602 100644 --- a/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala +++ b/effekt/jvm/src/test/scala/effekt/core/NewNormalizerTests.scala @@ -454,7 +454,8 @@ class NewNormalizerTests extends CoreTests { // During normalization, this block parameter gets lifted to a `def`. // One might hope for this mutable variable to be eliminated entirely, // but currently the normalizer does not inline definitions. - test("Block param capturing mutable reference can be lifted") { + // FIXME: This test is currently broken due to TestRenamer not properly handling captures. + test("Block param capturing mutable reference can be lifted".ignore) { val input = """ |def run(): Int = { diff --git a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala index 98526c9a0..88aeaa102 100644 --- a/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala +++ b/effekt/shared/src/main/scala/effekt/core/optimizer/Optimizer.scala @@ -16,15 +16,57 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { input match { case CoreTransformed(source, tree, mod, core) => val term = Context.ensureMainExists(mod) - val optimized = Context.timed("optimize", source.name) { optimize(source, term, core) } + val optimizer = if (Context.config.newNormalizer()) { + optimizeWithNewNormalizer + } else { + optimize + } + val optimized = Context.timed("optimize", source.name) { optimizer(source, term, core) } Some(CoreTransformed(source, tree, mod, optimized)) } - def optimize(source: Source, mainSymbol: symbols.Symbol, core: ModuleDecl)(using Context): ModuleDecl = + def optimize(source: Source, mainSymbol: symbols.Symbol, core: ModuleDecl)(using Context): ModuleDecl = { + var tree = core + // (1) first thing we do is simply remove unused definitions (this speeds up all following analysis and rewrites) + tree = Context.timed("deadcode-elimination", source.name) { + Deadcode.remove(mainSymbol, tree) + } + + if !Context.config.optimize() then return tree; + + // (2) lift static arguments + tree = Context.timed("static-argument-transformation", source.name) { + StaticArguments.transform(mainSymbol, tree) + } + + def normalize(m: ModuleDecl) = { + val anfed = BindSubexpressions.transform(m) + val normalized = Normalizer.normalize(Set(mainSymbol), anfed, Context.config.maxInlineSize().toInt) + val live = Deadcode.remove(mainSymbol, normalized) + val tailRemoved = RemoveTailResumptions(live) + val contified = DirectStyle.rewrite(tailRemoved) + contified + } + + // (3) normalize a few times (since tail resumptions might only surface after normalization and leave dead Resets) + tree = Context.timed("normalize-1", source.name) { + normalize(tree) + } + tree = Context.timed("normalize-2", source.name) { + normalize(tree) + } + tree = Context.timed("normalize-3", source.name) { + normalize(tree) + } + + tree + } + + def optimizeWithNewNormalizer(source: Source, mainSymbol: symbols.Symbol, core: ModuleDecl)(using Context): ModuleDecl = { var tree = core - // (1) first thing we do is simply remove unused definitions (this speeds up all following analysis and rewrites) + // (1) first thing we do is simply remove unused definitions (this speeds up all following analysis and rewrites) tree = Context.timed("deadcode-elimination", source.name) { Deadcode.remove(mainSymbol, tree) } @@ -34,6 +76,7 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { val inliningPolicy = UniqueJumpSimple( maxInlineSize = 15 ) + def normalize(m: ModuleDecl) = Context.timed("new-normalizer", source.name) { val staticArgs = StaticArguments.transform(mainSymbol, m) val normalized = NewNormalizer().run(staticArgs) @@ -51,4 +94,5 @@ object Optimizer extends Phase[CoreTransformed, CoreTransformed] { tree = normalize(tree) //util.trace(tree) tree + } }