diff --git a/Checker/main.fs b/Checker/main.fs index 541d30b6..c6528648 100644 --- a/Checker/main.fs +++ b/Checker/main.fs @@ -15,9 +15,9 @@ type CliArguments = interface IArgParserTemplate with member s.Usage = match s with - | Update_Golden _ -> "Update the golden tests" - | Skip_GLSL_Compile _ -> "Skip the GLSL compilation of shaders" - | Skip_Performance_Tests _ -> "Skip the tests of performance" + | Update_Golden -> "Update the golden tests" + | Skip_GLSL_Compile -> "Skip the GLSL compilation of shaders" + | Skip_Performance_Tests -> "Skip the tests of performance" let cliArgs = ArgumentParser.Create().ParseCommandLine() //let cliArgs = ArgumentParser.Create().ParseCommandLine([|"--update-golden"|]) diff --git a/README.md b/README.md index ec3d72d1..935bb49c 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ OPTIONS: --no-remove-unused Do not remove unused code --move-declarations Move declarations to group them --preprocess Evaluate some of the file preprocessor directives + --export-kkp-symbol-maps + Export kkpView symbol maps --help display this list of options. ``` @@ -278,6 +280,12 @@ the output. If two functions have a different number of arguments, they may have the same name in the output. This reduces the number of identifiers used by the shader and make it more compression friendly. +### kkpView symbol maps + +Shader Minifier can export symbol files that map the minified code back to names +from the original source code. This lets you visualize shader size using +https://github.com/ConspiracyHu/rekkrunchy-with-analytics + ### Shader Minifier performance On my machine, it takes around 1s to minify a shader. A lot of that time comes diff --git a/src/formatter.fs b/src/formatter.fs index 0ab2c071..bbfb9cf5 100644 --- a/src/formatter.fs +++ b/src/formatter.fs @@ -4,6 +4,11 @@ open System open System.IO open Options.Globals +let minify shader = + if options.exportKkpSymbolMaps + then Printer.printAndWriteSymbols shader + else Printer.print shader.code + let private formatPrefix = function | Ast.ExportPrefix.Variable -> "var" | Ast.ExportPrefix.HlslFunction -> "F" @@ -25,7 +30,7 @@ let private printCVariables out (shaders: Ast.Shader[]) exportedNames = fprintfn out "" for shader in shaders do let name = (Path.GetFileName shader.filename).Replace(".", "_") - fprintfn out "const char *%s =%s \"%s\";" name Environment.NewLine (Printer.print shader.code) + fprintfn out "const char *%s =%s \"%s\";" name Environment.NewLine (minify shader) fprintfn out "" fprintfn out "#endif // %s" macroName @@ -47,20 +52,20 @@ let private printCArray out (shaders: Ast.Shader[]) exportedNames = fprintfn out "" for shader in shaders do fprintfn out "// %s" shader.filename - fprintfn out "\"%s\"," (Printer.print shader.code) + fprintfn out "\"%s\"," (minify shader) fprintfn out "" fprintfn out "#endif" let private printNoHeader out (shaders: Ast.Shader[]) = - let str = [for shader in shaders -> Printer.print shader.code] |> String.concat "\n" + let str = [for shader in shaders -> minify shader] |> String.concat "\n" fprintf out "%s" str let private printIndented out (shaders: Ast.Shader[]) = [for shader in shaders do if shaders.Length > 1 then yield "// " + shader.filename - yield Printer.print shader.code] + yield minify shader] |> String.concat "\n\n" |> fprintf out "%s" @@ -73,7 +78,7 @@ let private printJSHeader out (shaders: Ast.Shader[]) exportedNames = fprintfn out "" for shader in shaders do let name = (Path.GetFileName shader.filename).Replace(".", "_") - fprintfn out "var %s = `%s`" name (Printer.print shader.code) + fprintfn out "var %s = `%s`" name (minify shader) fprintfn out "" let private printNasmHeader out (shaders: Ast.Shader[]) exportedNames = @@ -85,7 +90,7 @@ let private printNasmHeader out (shaders: Ast.Shader[]) exportedNames = fprintfn out "" for shader in shaders do let name = (Path.GetFileName shader.filename).Replace(".", "_") - fprintfn out "_%s:%s\tdb '%s', 0" name Environment.NewLine (Printer.print shader.code) + fprintfn out "_%s:%s\tdb '%s', 0" name Environment.NewLine (minify shader) fprintfn out "" let private printRustHeader out (shaders: Ast.Shader[]) exportedNames = @@ -97,7 +102,7 @@ let private printRustHeader out (shaders: Ast.Shader[]) exportedNames = for shader in shaders do fprintfn out "" let name = (Path.GetFileName shader.filename).Replace(".", "_") - fprintfn out "pub const %s: &'static [u8] = b\"\\%s %s\\0\";" (name.ToUpper()) Environment.NewLine (Printer.print shader.code) + fprintfn out "pub const %s: &'static [u8] = b\"\\%s %s\\0\";" (name.ToUpper()) Environment.NewLine (minify shader) let print out shaders exportedNames = function | Options.IndentedText -> printIndented out shaders diff --git a/src/options.fs b/src/options.fs index 51e8b8b1..65502215 100644 --- a/src/options.fs +++ b/src/options.fs @@ -39,6 +39,7 @@ type CliArguments = | [] NoRemoveUnused | [] MoveDeclarations | [] Preprocess + | [] ExportKkpSymbolMaps | [] Filenames of filename:string list interface IArgParserTemplate with @@ -49,8 +50,8 @@ type CliArguments = | Hlsl -> "Use HLSL (default is GLSL)" | FormatArg _ -> "Choose to format the output (use 'text' if you want just the shader)" | FieldNames _ -> "Choose the field names for vectors: 'rgba', 'xyzw', or 'stpq'" - | PreserveExternals _ -> "Do not rename external values (e.g. uniform)" - | PreserveAllGlobals _ -> "Do not rename functions and global variables" + | PreserveExternals -> "Do not rename external values (e.g. uniform)" + | PreserveAllGlobals -> "Do not rename functions and global variables" | NoInlining -> "Do not automatically inline variables and functions" | AggroInlining -> "Aggressively inline constants. This can reduce output size due to better constant folding. It can also increase output size due to repeated inlined constants, but this increased redundancy can be beneficial to compression, leading to a smaller final compressed size anyway. Does nothing if inlining is disabled." | NoRenaming -> "Do not rename anything" @@ -60,6 +61,7 @@ type CliArguments = | NoRemoveUnused -> "Do not remove unused code" | MoveDeclarations -> "Move declarations to group them" | Preprocess -> "Evaluate some of the file preprocessor directives" + | ExportKkpSymbolMaps -> "Export kkpView symbol maps" | Filenames _ -> "List of files to minify" type Options() = @@ -79,6 +81,7 @@ type Options() = member val noRemoveUnused = false with get, set member val moveDeclarations = false with get, set member val preprocess = false with get, set + member val exportKkpSymbolMaps = false with get, set member val filenames = [||]: string[] with get, set module Globals = @@ -122,6 +125,7 @@ let private initPrivate argv needFiles = options.noRemoveUnused <- args.Contains(NoRemoveUnused) options.moveDeclarations <- args.Contains(MoveDeclarations) options.preprocess <- args.Contains(Preprocess) + options.exportKkpSymbolMaps <- args.Contains(ExportKkpSymbolMaps) options.noRenamingList <- noRenamingList options.filenames <- filenames true diff --git a/src/printer.fs b/src/printer.fs index 0f77072e..a19c7c37 100644 --- a/src/printer.fs +++ b/src/printer.fs @@ -1,10 +1,45 @@ module Printer open System +open System.Linq +open System.Collections.Generic open Ast open Options.Globals open System.Text.RegularExpressions +let private kkpSymFormat shaderSymbol minifiedSize (symbolPool: string array) (symbolIndexes: int16 array) = + // https://github.com/ConspiracyHu/kkpView-public/blob/main/README.md + let bytes = List(capacity = 2 * symbolIndexes.Length) + let ascii (s: string) = bytes.AddRange(System.Text.Encoding.ASCII.GetBytes s) + let asciiz s = ascii s; bytes.Add(byte 0) + let fourByteInteger n = bytes.AddRange(List.map byte [ n; n >>> 8; n >>> 16; n >>> 24 ]) + let twoByteInteger n = bytes.AddRange(List.map byte [ n; n >>> 8 ]) + ascii "PHXP" // 4 bytes: FOURCC: 'PHXP' + asciiz shaderSymbol // ASCIIZ string: name of the shader described by this sym file. + fourByteInteger minifiedSize // 4 bytes: minified data size (Ds) of the shader. + fourByteInteger symbolPool.Length // 4 bytes: symbol count (Sc). + for symbolName in symbolPool do // For each symbol (Sc), + asciiz symbolName // ASCIIZ string: name of the symbol. + for symbolIndex in symbolIndexes do // For each byte in the minified shader (Ds), + twoByteInteger symbolIndex // 2 bytes: symbol index in the symbol pool. + bytes.ToArray() + +type SymbolMap() = + let symbolRefs = List() // one per byte in the minified shader + member _.AddMapping (str: string) (symbolName: string) = + if str.ToCharArray() |> Array.exists (fun c -> int(c) >= 256) then failwith "cannot process a non-byte char" + for i in 1..str.Length do + symbolRefs.Add(symbolName) + member _.SymFileBytes (shaderFilename: string) (minifiedShader: string) = + if minifiedShader.Length <> symbolRefs.Count then failwith "minified byte size doesn't match symbols" + let shaderSymbol = "shader::" + shaderFilename // "::" is the separator for tree structure + let mutable i = 0 // indexMap maps each distinct symbol name to its index in the pool + let indexMap = symbolRefs.Distinct().ToDictionary(id, (fun _ -> i <- i + 1; int16(i - 1))) + let symbolIndexes = symbolRefs.Select(fun symbolRef -> indexMap[symbolRef]).ToArray() + let symbolPool = indexMap.OrderBy(fun kv -> kv.Value).Select(fun kv -> kv.Key).ToArray() + let bytes = kkpSymFormat shaderSymbol minifiedShader.Length symbolPool symbolIndexes + bytes + type PrinterImpl(outputFormat) = let out a = sprintf a @@ -270,7 +305,7 @@ type PrinterImpl(outputFormat) = | TLDecl decl -> out "%s%s;" (nl 0) (declToS 0 decl) | TypeDecl t -> out "%s;" (typeSpecToS t) - member _.Print tl = + let print tl = let mutable wasMacro = true // handle the required \n before a macro ignoreFirstNewLine <- true @@ -280,12 +315,32 @@ type PrinterImpl(outputFormat) = wasMacro <- isMacro if needEndLine then out "%s%s" (backslashN()) (topLevelToS x) else topLevelToS x + tl |> List.map f - tl |> List.map f |> String.concat "" member _.ExprToS = exprToS member _.TypeToS = typeToS + member _.Print tl = print tl |> String.concat "" + member _.PrintAndWriteSymbols shader = + let tlStrings = print shader.code + let minifiedShader = tlStrings |> String.concat "" + let symbolMap = SymbolMap() + for (tl, tlString) in List.zip shader.code tlStrings do + let symbolName = + match tl with + | Function (fct, _) -> fct.fName.OldName + | TLDecl (_, declElts) -> declElts |> List.map (fun declElt -> declElt.name.OldName) |> String.concat "," + | TypeDecl (TypeBlock (_, Some name, _)) -> name.OldName + | TypeDecl _ -> "*type decl*" // unnamed TypeBlock (a top-level TypeDecl cannot be a TypeName) + | Precision _ -> "*precision*" + | TLVerbatim s when s.StartsWith("#define") -> "#define" + | TLVerbatim _ -> "*verbatim*" // HLSL attribute, //[ skipped //] + symbolMap.AddMapping tlString symbolName + let bytes = symbolMap.SymFileBytes shader.filename minifiedShader + System.IO.File.WriteAllBytes(shader.filename + ".sym", bytes) + minifiedShader let print tl = (new PrinterImpl(options.outputFormat)).Print(tl) +let printAndWriteSymbols shader = (new PrinterImpl(options.outputFormat)).PrintAndWriteSymbols shader let printText tl = (new PrinterImpl(Options.Text)).Print(tl) let exprToS x = (new PrinterImpl(Options.Text)).ExprToS 0 x let typeToS ty = (new PrinterImpl(Options.Text)).TypeToS ty diff --git a/tests/commands.txt b/tests/commands.txt index 35492ca1..d579078e 100644 --- a/tests/commands.txt +++ b/tests/commands.txt @@ -61,6 +61,10 @@ --no-remove-unused --move-declarations --format c-array -o tests/unit/inout.expected tests/unit/inout.frag tests/unit/inout2.frag +# kkp symbols tests + +--no-remove-unused --no-renaming --export-kkp-symbol-maps --format indented -o tests/unit/symbols.frag.expected tests/unit/symbols.frag + # Tests with full renaming --no-remove-unused --format c-array --no-inlining -o tests/unit/many_variables.expected tests/unit/many_variables.frag diff --git a/tests/unit/symbols.frag b/tests/unit/symbols.frag new file mode 100644 index 00000000..a8da180f --- /dev/null +++ b/tests/unit/symbols.frag @@ -0,0 +1,21 @@ +# extension GL_EXT_gpu_shader4 : enable +# define TEST +# ifndef A +# endif + +vec3 color; // TLDecl + +struct s { // TypeDecl + vec4 n, f; + bool ok; +}; + +float sdf(vec3 p) { // Function + color = vec3(1.); + return length(p) - 1.; +} + +void main() { + int ijk = 32; + ijk += ijk; +} \ No newline at end of file diff --git a/tests/unit/symbols.frag.expected b/tests/unit/symbols.frag.expected new file mode 100644 index 00000000..a52088a2 --- /dev/null +++ b/tests/unit/symbols.frag.expected @@ -0,0 +1,19 @@ +#extension GL_EXT_gpu_shader4:enable + +#define TEST + +#ifndef A + +#endif + +vec3 color;struct s{vec4 n,f;bool ok;}; +float sdf(vec3 p) +{ + color=vec3(1); + return length(p)-1.; +} +void main() +{ + int ijk=32; + ijk+=ijk; +} \ No newline at end of file diff --git a/tests/unit/symbols.frag.sym b/tests/unit/symbols.frag.sym new file mode 100644 index 00000000..ac4a1793 Binary files /dev/null and b/tests/unit/symbols.frag.sym differ