From cc7cf52d7b4a63ca085be82dfccde54f2a7ad291 Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Sat, 25 Feb 2023 12:40:59 -0500 Subject: [PATCH 1/8] First-pass fix peeking into next block heavily optimizable most likely --- UndertaleModLib/Compiler/AssemblyWriter.cs | 7 +--- UndertaleModLib/Decompiler/Decompiler.cs | 44 ++++++++++++++++------ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/UndertaleModLib/Compiler/AssemblyWriter.cs b/UndertaleModLib/Compiler/AssemblyWriter.cs index 0c51303e3..4c126ae40 100644 --- a/UndertaleModLib/Compiler/AssemblyWriter.cs +++ b/UndertaleModLib/Compiler/AssemblyWriter.cs @@ -1016,12 +1016,7 @@ private static void AssembleStatement(CodeWriter cw, Parser.Statement s, int rem else { // Returns nothing, basically the same as exit - // TODO: I'm pretty sure the "remaining" part is actually just because the decompiler keeps adding - // returns and it's necessary to preserve 1:1 compilation - // But this workaround causes issue https://github.com/krzys-h/UndertaleModTool/issues/900 - // So it would be fixed by cutting the "remaining" check here and removing the extra from decompilation. - if (!(CompileContext.GMS2_3 && remaining == 1)) - AssembleExit(cw); + AssembleExit(cw); } break; case Parser.Statement.StatementKind.Exit: diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index 716048296..9dfad2dab 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2065,17 +2065,39 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary blockAddresses = new(blocks.Keys); + blockAddresses.Sort(); + if (blockAddresses.Count > blockAddresses.IndexOf(block.Address ?? 0) + 1) + { + uint nextBlockAddress = blockAddresses[blockAddresses.IndexOf(block.Address ?? 0) + 1]; + nextBlock = blocks[nextBlockAddress]; + } + } + + if (!(DecompileContext.GMS2_3 + && instr.Kind == UndertaleInstruction.Opcode.Exit + && nextBlock is not null + && nextBlock.Instructions.Count > 0 + && nextBlock.Instructions[0].Kind == UndertaleInstruction.Opcode.Push + && nextBlock.Instructions[0].Value.GetType() != typeof(int))) + { + ReturnStatement stmt = new ReturnStatement(instr.Kind == UndertaleInstruction.Opcode.Ret ? stack.Pop() : null); + /* + This shouldn't be necessary: all unused things on the stack get converted to tempvars at the end anyway, and this fixes decompilation of repeat() + See #85 + + foreach (var expr in stack.Reverse()) + if (!(expr is ExpressionTempVar)) + statements.Add(expr); + stack.Clear(); + */ + statements.Add(stmt); + } end = true; returned = true; break; From 729a6bc4d20ae1c707602d4550278b33256e676d Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Mon, 27 Feb 2023 07:57:30 -0500 Subject: [PATCH 2/8] [m] rm one IndexOf --- UndertaleModLib/Decompiler/Decompiler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index 9dfad2dab..2f58e8464 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2072,15 +2072,15 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary blockAddresses = new(blocks.Keys); blockAddresses.Sort(); - if (blockAddresses.Count > blockAddresses.IndexOf(block.Address ?? 0) + 1) + int nextBlockIndex = blockAddresses.IndexOf(block.Address ?? 0) + 1; + if (blockAddresses.Count > nextBlockIndex) { - uint nextBlockAddress = blockAddresses[blockAddresses.IndexOf(block.Address ?? 0) + 1]; + uint nextBlockAddress = blockAddresses[nextBlockIndex]; nextBlock = blocks[nextBlockAddress]; } } - if (!(DecompileContext.GMS2_3 - && instr.Kind == UndertaleInstruction.Opcode.Exit + if (!(instr.Kind == UndertaleInstruction.Opcode.Exit && nextBlock is not null && nextBlock.Instructions.Count > 0 && nextBlock.Instructions[0].Kind == UndertaleInstruction.Opcode.Push From 2339f5c7139d33a2cb70c0abf3b78098012fe2e3 Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Mon, 27 Feb 2023 08:09:44 -0500 Subject: [PATCH 3/8] Move opcode check earlier to optimize. It might be more readable to condense all of these checks into however many nested ifs resulting in a boolean. "dontProduceStatement" or somesuch --- UndertaleModLib/Decompiler/Decompiler.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index 2f58e8464..ef2997179 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2068,7 +2068,7 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary blockAddresses = new(blocks.Keys); blockAddresses.Sort(); @@ -2080,8 +2080,7 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary 0 && nextBlock.Instructions[0].Kind == UndertaleInstruction.Opcode.Push && nextBlock.Instructions[0].Value.GetType() != typeof(int))) From 606b3eaf459987c321782b0ac8e4696f3a58b3cf Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:04:18 -0500 Subject: [PATCH 4/8] use uint[] per VladiStep Co-authored-by: VladiStep --- UndertaleModLib/Decompiler/Decompiler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index ef2997179..92c42af02 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2070,9 +2070,9 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary blockAddresses = new(blocks.Keys); - blockAddresses.Sort(); - int nextBlockIndex = blockAddresses.IndexOf(block.Address ?? 0) + 1; + uint[] blockAddresses = blocks.Keys.ToArray(); + Array.Sort(blockAddresses); + int nextBlockIndex = Array.IndexOf(blockAddresses, block.Address ?? 0) + 1; if (blockAddresses.Count > nextBlockIndex) { uint nextBlockAddress = blockAddresses[nextBlockIndex]; From ccc032c37afc350b020529ef1041073005ca2fb8 Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:12:14 -0500 Subject: [PATCH 5/8] [m] list has Count, array has Count() --- UndertaleModLib/Decompiler/Decompiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index 92c42af02..78b0f5e60 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2073,7 +2073,7 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary nextBlockIndex) + if (blockAddresses.Count() > nextBlockIndex) { uint nextBlockAddress = blockAddresses[nextBlockIndex]; nextBlock = blocks[nextBlockAddress]; From 94e4cd0f3ce37d87837596df3c8c360dce97fd2b Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:21:34 -0500 Subject: [PATCH 6/8] [m] or Length, even better, thanks Vladi --- UndertaleModLib/Decompiler/Decompiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index 78b0f5e60..ebef143c4 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2073,7 +2073,7 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary nextBlockIndex) + if (blockAddresses.Length > nextBlockIndex) { uint nextBlockAddress = blockAddresses[nextBlockIndex]; nextBlock = blocks[nextBlockAddress]; From 7a3b158449d108f9270ed734ce72df043d24c3ac Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Wed, 1 Mar 2023 19:52:01 -0500 Subject: [PATCH 7/8] [m] it's a function in "expression mode", clarify --- UndertaleModLib/Compiler/Parser.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/UndertaleModLib/Compiler/Parser.cs b/UndertaleModLib/Compiler/Parser.cs index fe5f6a672..723b24c55 100644 --- a/UndertaleModLib/Compiler/Parser.cs +++ b/UndertaleModLib/Compiler/Parser.cs @@ -660,12 +660,12 @@ private static Statement ParseFunction(CompileContext context) { Statement result = new Statement(Statement.StatementKind.FunctionDef, EnsureTokenKind(TokenKind.KeywordFunction).Token); Statement args = new Statement(); - bool expression = true; + bool expressionMode = true; Statement destination = null; if (GetNextTokenKind() == TokenKind.ProcFunction) { - expression = false; + expressionMode = false; Statement s = remainingStageOne.Dequeue(); destination = new Statement(Statement.StatementKind.ExprFuncName, s.Token) { ID = s.ID }; } @@ -691,7 +691,7 @@ private static Statement ParseFunction(CompileContext context) if (EnsureTokenKind(TokenKind.CloseParen) == null) return null; result.Children.Add(ParseStatement(context)); - if (expression) + if (expressionMode) return result; else // Whatever you call non-anonymous definitions { From 60ea473c6787886e1930537f27e6879fb2b563ab Mon Sep 17 00:00:00 2001 From: Jacky720 <32578221+Jacky720@users.noreply.github.com> Date: Wed, 1 Mar 2023 19:52:53 -0500 Subject: [PATCH 8/8] Fix offsets, arg counts, new functions --- UndertaleModLib/Compiler/AssemblyWriter.cs | 69 ++++++++++++++++++++-- UndertaleModLib/Decompiler/Decompiler.cs | 6 +- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/UndertaleModLib/Compiler/AssemblyWriter.cs b/UndertaleModLib/Compiler/AssemblyWriter.cs index 4c126ae40..08b8acb29 100644 --- a/UndertaleModLib/Compiler/AssemblyWriter.cs +++ b/UndertaleModLib/Compiler/AssemblyWriter.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using UndertaleModLib.Decompiler; using UndertaleModLib.Models; using static UndertaleModLib.Models.UndertaleInstruction; @@ -246,8 +247,46 @@ bool hasLocal(string name) Decompiler.Decompiler.BuildSubFunctionCache(compileContext.Data); foreach (var patch in funcPatches) { + if (patch.isNewFunc) + { + UndertaleString childName = new("gml_Script_" + patch.Name); + int childNameIndex = compileContext.Data.Strings.Count; + compileContext.Data.Strings.Add(childName); + + UndertaleCode childEntry = new() + { + Name = childName, + Length = compileContext.OriginalCode.Length, // todo: get a more certainly up-to-date length + ParentEntry = compileContext.OriginalCode, + Offset = patch.Offset, + ArgumentsCount = (ushort)patch.ArgCount, + LocalsCount = compileContext.OriginalCode.LocalsCount // todo: use just the locals for the individual script + }; + compileContext.OriginalCode.ChildEntries.Add(childEntry); + int childEntryIndex = compileContext.Data.Code.IndexOf(compileContext.OriginalCode) + compileContext.OriginalCode.ChildEntries.Count; + compileContext.Data.Code.Insert(childEntryIndex, childEntry); + + UndertaleScript childScript = new() + { + Name = childName, + Code = childEntry + }; + compileContext.Data.Scripts.Add(childScript); + + UndertaleFunction childFunction = new() + { + Name = childName, + NameStringID = childNameIndex, + Autogenerated = true + }; + compileContext.Data.Functions.Add(childFunction); + + compileContext.Data.KnownSubFunctions.Add(patch.Name, childFunction); + + continue; + } + UndertaleFunction def; - // Exceptions do occur, but for the moment only in 2.3. if (patch.ArgCount >= 0) { patch.Target.ArgumentsCount = (ushort)patch.ArgCount; @@ -274,7 +313,7 @@ bool hasLocal(string name) else { def = compileContext.Data.Functions.ByName(patch.Name); - // This is locked by the 2.3 block above, but this code is only reachable using a 2.3 function definition. + // This code is only reachable using a 2.3 function definition. ("push.i gml_Script_scr_stuff") def ??= compileContext.Data.KnownSubFunctions.GetValueOrDefault(patch.Name); if (compileContext.ensureFunctionsDefined) def ??= compileContext.Data.Functions.EnsureDefined(patch.Name, compileContext.Data.Strings, true); @@ -342,6 +381,8 @@ public class FunctionPatch public UndertaleInstruction Target; public string Name; public int ArgCount; + public uint Offset; + public bool isNewFunc = false; } public class StringPatch @@ -1232,6 +1273,27 @@ private static void AssembleExpression(CodeWriter cw, Parser.Statement e, Parser Patch startPatch = Patch.StartHere(cw); Patch endPatch = Patch.Start(); endPatch.Add(cw.Emit(Opcode.B)); + // we're accessing a subfunction here, so build the cache if needed + Decompiler.Decompiler.BuildSubFunctionCache(cw.compileContext.Data); + if (cw.compileContext.Data.KnownSubFunctions.ContainsKey(funcDefName.Text)) + { + string subFunctionName = cw.compileContext.Data.KnownSubFunctions[funcDefName.Text].Name.Content; + UndertaleCode childEntry = cw.compileContext.OriginalCode.ChildEntries.ByName(subFunctionName); + childEntry.Offset = cw.offset * 4; + childEntry.ArgumentsCount = (ushort)e.Children[0].Children.Count; + childEntry.LocalsCount = cw.compileContext.OriginalCode.LocalsCount; // todo: use just the locals for the individual script + } + else // we're making a new function baby + { + cw.funcPatches.Add(new FunctionPatch() + { + Name = funcDefName.Text, + Offset = cw.offset * 4, + ArgCount = (ushort)e.Children[0].Children.Count, + isNewFunc = true + }); + } + cw.loopContexts.Push(new LoopContext(endPatch, startPatch)); AssembleStatement(cw, e.Children[1]); // body AssembleExit(cw); @@ -1244,7 +1306,6 @@ private static void AssembleExpression(CodeWriter cw, Parser.Statement e, Parser Name = funcDefName.Text, ArgCount = -1 }); - //cw.compileContext.Data.Code.ByName("gml_GlobalScript_" + funcDefName.Text).ArgumentsCount = (ushort)e.Children[0].Children.Count; // Figure this out in non-convoluted and working way cw.Emit(Opcode.Conv, DataType.Int32, DataType.Variable); cw.Emit(Opcode.PushI, DataType.Int16).Value = (short)-1; cw.Emit(Opcode.Conv, DataType.Int32, DataType.Variable); @@ -1256,7 +1317,7 @@ private static void AssembleExpression(CodeWriter cw, Parser.Statement e, Parser }); cw.typeStack.Push(DataType.Variable); cw.Emit(Opcode.Dup, DataType.Variable).Extra = 0; - cw.Emit(Opcode.PushI, DataType.Int16).Value = (short)-1; + cw.Emit(Opcode.PushI, DataType.Int16).Value = (short)-1; // todo: -6 sometimes? } break; case Parser.Statement.StatementKind.ExprBinaryOp: diff --git a/UndertaleModLib/Decompiler/Decompiler.cs b/UndertaleModLib/Decompiler/Decompiler.cs index ebef143c4..9de1d01d5 100644 --- a/UndertaleModLib/Decompiler/Decompiler.cs +++ b/UndertaleModLib/Decompiler/Decompiler.cs @@ -2405,7 +2405,7 @@ internal static void DecompileFromBlock(DecompileContext context, Dictionary blocks2 = PrepareDecompileFlow(anonymousCodeObject.ParentEntry, new List() { 0 }); DecompileFromBlock(childContext, blocks2, blocks2[0]); // This hack handles decompilation of code entries getting shorter, but not longer or out of place. + // Probably is no longer needed since we now update Length mostly-correctly Block lastBlock; if (!blocks2.TryGetValue(anonymousCodeObject.Length / 4, out lastBlock)) lastBlock = blocks2[blocks2.Keys.Max()]; @@ -2424,7 +2425,8 @@ static string FindActualNameForAnonymousCodeObject(DecompileContext context, Und foreach (Statement stmt2 in statements) { if (stmt2 is AssignmentStatement assign && - assign.Value is FunctionDefinition funcDef) + assign.Value is FunctionDefinition funcDef && + funcDef.FunctionBodyCodeEntry == anonymousCodeObject) { if (funcDef.FunctionBodyEntryBlock.Address == anonymousCodeObject.Offset / 4) return assign.Destination.Var.Name.Content;