Fix #333, export kkpView symbol maps (#337)

Checker/main.fs

@@ -15,9 +15,9 @@ type CliArguments =
     interface IArgParserTemplate with
         member s.Usage =
             match s with
-            | Update_Golden _ -> "Update the golden tests"
-            | Skip_GLSL_Compile _ -> "Skip the GLSL compilation of shaders"
-            | Skip_Performance_Tests _ -> "Skip the tests of performance"
+            | Update_Golden -> "Update the golden tests"
+            | Skip_GLSL_Compile -> "Skip the GLSL compilation of shaders"
+            | Skip_Performance_Tests -> "Skip the tests of performance"
 
 let cliArgs = ArgumentParser.Create<CliArguments>().ParseCommandLine()
 //let cliArgs = ArgumentParser.Create<CliArguments>().ParseCommandLine([|"--update-golden"|])

README.md

@@ -138,6 +138,8 @@ OPTIONS:
     --no-remove-unused    Do not remove unused code
     --move-declarations   Move declarations to group them
     --preprocess          Evaluate some of the file preprocessor directives
+    --export-kkp-symbol-maps
+                          Export kkpView symbol maps
     --help                display this list of options.
 ```
 
@@ -278,6 +280,12 @@ the output. If two functions have a different number of arguments, they may have
 the same name in the output. This reduces the number of identifiers used by the
 shader and make it more compression friendly.
 
+### kkpView symbol maps
+
+Shader Minifier can export symbol files that map the minified code back to names
+from the original source code. This lets you visualize shader size using
+https://github.com/ConspiracyHu/rekkrunchy-with-analytics
+
 ### Shader Minifier performance
 
 On my machine, it takes around 1s to minify a shader. A lot of that time comes

src/formatter.fs

@@ -4,6 +4,11 @@ open System
 open System.IO
 open Options.Globals
 
+let minify shader =
+    if options.exportKkpSymbolMaps
+    then Printer.printAndWriteSymbols shader
+    else Printer.print shader.code
+
 let private formatPrefix = function
     | Ast.ExportPrefix.Variable -> "var"
     | Ast.ExportPrefix.HlslFunction -> "F"
@@ -25,7 +30,7 @@ let private printCVariables out (shaders: Ast.Shader[]) exportedNames =
     fprintfn out ""
     for shader in shaders do
         let name = (Path.GetFileName shader.filename).Replace(".", "_")
-        fprintfn out "const char *%s =%s \"%s\";" name Environment.NewLine (Printer.print shader.code)
+        fprintfn out "const char *%s =%s \"%s\";" name Environment.NewLine (minify shader)
         fprintfn out ""
 
     fprintfn out "#endif // %s" macroName
@@ -47,20 +52,20 @@ let private printCArray out (shaders: Ast.Shader[]) exportedNames =
     fprintfn out ""
     for shader in shaders do
         fprintfn out "// %s" shader.filename
-        fprintfn out "\"%s\"," (Printer.print shader.code)
+        fprintfn out "\"%s\"," (minify shader)
         fprintfn out ""
 
     fprintfn out "#endif"
 
 let private printNoHeader out (shaders: Ast.Shader[]) =
-    let str = [for shader in shaders -> Printer.print shader.code] |> String.concat "\n"
+    let str = [for shader in shaders -> minify shader] |> String.concat "\n"
     fprintf out "%s" str
 
 let private printIndented out (shaders: Ast.Shader[]) =
     [for shader in shaders do
         if shaders.Length > 1 then
             yield "// " + shader.filename
-        yield Printer.print shader.code]
+        yield minify shader]
     |> String.concat "\n\n"
     |> fprintf out "%s"
 
@@ -73,7 +78,7 @@ let private printJSHeader out (shaders: Ast.Shader[]) exportedNames =
     fprintfn out ""
     for shader in shaders do
         let name = (Path.GetFileName shader.filename).Replace(".", "_")
-        fprintfn out "var %s = `%s`" name (Printer.print shader.code)
+        fprintfn out "var %s = `%s`" name (minify shader)
         fprintfn out ""
 
 let private printNasmHeader out (shaders: Ast.Shader[]) exportedNames =
@@ -85,7 +90,7 @@ let private printNasmHeader out (shaders: Ast.Shader[]) exportedNames =
     fprintfn out ""
     for shader in shaders do
         let name = (Path.GetFileName shader.filename).Replace(".", "_")
-        fprintfn out "_%s:%s\tdb '%s', 0" name Environment.NewLine (Printer.print shader.code)
+        fprintfn out "_%s:%s\tdb '%s', 0" name Environment.NewLine (minify shader)
         fprintfn out ""
 
 let private printRustHeader out (shaders: Ast.Shader[]) exportedNames =
@@ -97,7 +102,7 @@ let private printRustHeader out (shaders: Ast.Shader[]) exportedNames =
     for shader in shaders do
         fprintfn out ""
         let name = (Path.GetFileName shader.filename).Replace(".", "_")    
-        fprintfn out "pub const %s: &'static [u8] = b\"\\%s %s\\0\";" (name.ToUpper()) Environment.NewLine (Printer.print shader.code)
+        fprintfn out "pub const %s: &'static [u8] = b\"\\%s %s\\0\";" (name.ToUpper()) Environment.NewLine (minify shader)
 
 let print out shaders exportedNames = function
     | Options.IndentedText -> printIndented out shaders

src/options.fs

@@ -39,6 +39,7 @@ type CliArguments =
     | [<CustomCommandLine("--no-remove-unused")>] NoRemoveUnused
     | [<CustomCommandLine("--move-declarations")>] MoveDeclarations
     | [<CustomCommandLine("--preprocess")>] Preprocess
+    | [<CustomCommandLine("--export-kkp-symbol-maps")>] ExportKkpSymbolMaps
     | [<MainCommand>] Filenames of filename:string list
 
     interface IArgParserTemplate with
@@ -49,8 +50,8 @@ type CliArguments =
             | Hlsl -> "Use HLSL (default is GLSL)"
             | FormatArg _ -> "Choose to format the output (use 'text' if you want just the shader)"
             | FieldNames _ -> "Choose the field names for vectors: 'rgba', 'xyzw', or 'stpq'"
-            | PreserveExternals _ -> "Do not rename external values (e.g. uniform)"
-            | PreserveAllGlobals _ -> "Do not rename functions and global variables"
+            | PreserveExternals -> "Do not rename external values (e.g. uniform)"
+            | PreserveAllGlobals -> "Do not rename functions and global variables"
             | NoInlining -> "Do not automatically inline variables and functions"
             | AggroInlining -> "Aggressively inline constants. This can reduce output size due to better constant folding. It can also increase output size due to repeated inlined constants, but this increased redundancy can be beneficial to compression, leading to a smaller final compressed size anyway. Does nothing if inlining is disabled."
             | NoRenaming -> "Do not rename anything"
@@ -60,6 +61,7 @@ type CliArguments =
             | NoRemoveUnused -> "Do not remove unused code"
             | MoveDeclarations -> "Move declarations to group them"
             | Preprocess -> "Evaluate some of the file preprocessor directives"
+            | ExportKkpSymbolMaps -> "Export kkpView symbol maps"
             | Filenames _ -> "List of files to minify"
 
 type Options() =
@@ -79,6 +81,7 @@ type Options() =
     member val noRemoveUnused = false with get, set
     member val moveDeclarations = false with get, set
     member val preprocess = false with get, set
+    member val exportKkpSymbolMaps = false with get, set
     member val filenames = [||]: string[] with get, set
 
 module Globals =
@@ -122,6 +125,7 @@ let private initPrivate argv needFiles =
         options.noRemoveUnused <- args.Contains(NoRemoveUnused)
         options.moveDeclarations <- args.Contains(MoveDeclarations)
         options.preprocess <- args.Contains(Preprocess)
+        options.exportKkpSymbolMaps <- args.Contains(ExportKkpSymbolMaps)
         options.noRenamingList <- noRenamingList
         options.filenames <- filenames
         true

src/printer.fs

@@ -1,10 +1,45 @@
 module Printer
 
 open System
+open System.Linq
+open System.Collections.Generic
 open Ast
 open Options.Globals
 open System.Text.RegularExpressions
 
+let private kkpSymFormat shaderSymbol minifiedSize (symbolPool: string array) (symbolIndexes: int16 array) =
+    // https://github.com/ConspiracyHu/kkpView-public/blob/main/README.md
+    let bytes = List<byte>(capacity = 2 * symbolIndexes.Length)
+    let ascii (s: string) = bytes.AddRange(System.Text.Encoding.ASCII.GetBytes s)
+    let asciiz s = ascii s; bytes.Add(byte 0)
+    let fourByteInteger n = bytes.AddRange(List.map byte [ n; n >>> 8; n >>> 16; n >>> 24 ])
+    let twoByteInteger n = bytes.AddRange(List.map byte [ n; n >>> 8 ])
+    ascii "PHXP"                        // 4 bytes: FOURCC: 'PHXP'
+    asciiz shaderSymbol                 // ASCIIZ string: name of the shader described by this sym file.
+    fourByteInteger minifiedSize        // 4 bytes: minified data size (Ds) of the shader.
+    fourByteInteger symbolPool.Length   // 4 bytes: symbol count (Sc).
+    for symbolName in symbolPool do     // For each symbol (Sc),
+        asciiz symbolName               //     ASCIIZ string: name of the symbol.
+    for symbolIndex in symbolIndexes do // For each byte in the minified shader (Ds),
+        twoByteInteger symbolIndex      //     2 bytes: symbol index in the symbol pool.
+    bytes.ToArray()
+
+type SymbolMap() =
+    let symbolRefs = List<string>() // one per byte in the minified shader
+    member _.AddMapping (str: string) (symbolName: string) =
+        if str.ToCharArray() |> Array.exists (fun c -> int(c) >= 256) then failwith "cannot process a non-byte char"
+        for i in 1..str.Length do
+            symbolRefs.Add(symbolName)
+    member _.SymFileBytes (shaderFilename: string) (minifiedShader: string) =
+        if minifiedShader.Length <> symbolRefs.Count then failwith "minified byte size doesn't match symbols"
+        let shaderSymbol = "shader::" + shaderFilename // "::" is the separator for tree structure
+        let mutable i = 0 // indexMap maps each distinct symbol name to its index in the pool
+        let indexMap = symbolRefs.Distinct().ToDictionary(id, (fun _ -> i <- i + 1; int16(i - 1)))
+        let symbolIndexes = symbolRefs.Select(fun symbolRef -> indexMap[symbolRef]).ToArray()
+        let symbolPool = indexMap.OrderBy(fun kv -> kv.Value).Select(fun kv -> kv.Key).ToArray()
+        let bytes = kkpSymFormat shaderSymbol minifiedShader.Length symbolPool symbolIndexes
+        bytes
+
 type PrinterImpl(outputFormat) =
 
     let out a = sprintf a
@@ -270,7 +305,7 @@ type PrinterImpl(outputFormat) =
         | TLDecl decl -> out "%s%s;" (nl 0) (declToS 0 decl)
         | TypeDecl t -> out "%s;" (typeSpecToS t)
 
-    member _.Print tl =
+    let print tl = 
         let mutable wasMacro = true
         // handle the required \n before a macro
         ignoreFirstNewLine <- true
@@ -280,12 +315,32 @@ type PrinterImpl(outputFormat) =
             wasMacro <- isMacro
             if needEndLine then out "%s%s" (backslashN()) (topLevelToS x)
             else topLevelToS x
+        tl |> List.map f
 
-        tl |> List.map f |> String.concat ""
     member _.ExprToS = exprToS
     member _.TypeToS = typeToS
+    member _.Print tl = print tl |> String.concat ""
+    member _.PrintAndWriteSymbols shader =
+        let tlStrings = print shader.code
+        let minifiedShader = tlStrings |> String.concat ""
+        let symbolMap = SymbolMap()
+        for (tl, tlString) in List.zip shader.code tlStrings do
+            let symbolName =
+                match tl with
+                | Function (fct, _) -> fct.fName.OldName
+                | TLDecl (_, declElts) -> declElts |> List.map (fun declElt -> declElt.name.OldName) |> String.concat ","
+                | TypeDecl (TypeBlock (_, Some name, _)) -> name.OldName
+                | TypeDecl _ -> "*type decl*" // unnamed TypeBlock (a top-level TypeDecl cannot be a TypeName)
+                | Precision _ -> "*precision*"
+                | TLVerbatim s when s.StartsWith("#define") -> "#define"
+                | TLVerbatim _ -> "*verbatim*" // HLSL attribute, //[ skipped //]
+            symbolMap.AddMapping tlString symbolName
+        let bytes = symbolMap.SymFileBytes shader.filename minifiedShader
+        System.IO.File.WriteAllBytes(shader.filename + ".sym", bytes)
+        minifiedShader
 
 let print tl = (new PrinterImpl(options.outputFormat)).Print(tl)
+let printAndWriteSymbols shader = (new PrinterImpl(options.outputFormat)).PrintAndWriteSymbols shader
 let printText tl = (new PrinterImpl(Options.Text)).Print(tl)
 let exprToS x = (new PrinterImpl(Options.Text)).ExprToS 0 x
 let typeToS ty = (new PrinterImpl(Options.Text)).TypeToS ty

tests/commands.txt

@@ -61,6 +61,10 @@
 
 --no-remove-unused --move-declarations --format c-array -o tests/unit/inout.expected tests/unit/inout.frag tests/unit/inout2.frag
 
+# kkp symbols tests
+
+--no-remove-unused --no-renaming --export-kkp-symbol-maps --format indented -o tests/unit/symbols.frag.expected tests/unit/symbols.frag
+
 # Tests with full renaming
 
 --no-remove-unused --format c-array --no-inlining -o tests/unit/many_variables.expected tests/unit/many_variables.frag

tests/unit/symbols.frag

@@ -0,0 +1,21 @@
+#   extension GL_EXT_gpu_shader4 : enable
+#  define TEST
+# ifndef A
+# endif
+
+vec3 color; // TLDecl
+
+struct s { // TypeDecl
+  vec4 n, f;
+  bool ok;
+};
+
+float sdf(vec3 p) { // Function
+  color = vec3(1.);
+  return length(p) - 1.;
+}
+
+void main() {
+    int ijk = 32;
+    ijk += ijk;
+}

tests/unit/symbols.frag.expected

@@ -0,0 +1,19 @@
+#extension GL_EXT_gpu_shader4:enable
+
+#define TEST
+
+#ifndef A
+
+#endif
+
+vec3 color;struct s{vec4 n,f;bool ok;};
+float sdf(vec3 p)
+{
+  color=vec3(1);
+  return length(p)-1.;
+}
+void main()
+{
+  int ijk=32;
+  ijk+=ijk;
+}