From 671f4e31b0679ad50005f02badd002102ee3b198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Thu, 7 Nov 2024 20:26:45 +0100 Subject: [PATCH 1/7] pattern dedup --- code/C#/DBDefsDumper/Pattern.cs | 54 +++++++++++---- code/C#/DBDefsDumper/PatternBuilder.cs | 93 +++++--------------------- code/C#/DBDefsDumper/Program.cs | 54 +-------------- 3 files changed, 60 insertions(+), 141 deletions(-) diff --git a/code/C#/DBDefsDumper/Pattern.cs b/code/C#/DBDefsDumper/Pattern.cs index f51c1da3297..ed7013ec3c4 100644 --- a/code/C#/DBDefsDumper/Pattern.cs +++ b/code/C#/DBDefsDumper/Pattern.cs @@ -1,35 +1,61 @@ using System.Collections.Generic; +using System; namespace DBDefsDumper { + class VersionRange + { + public List mmps; + public int minBuild; + public int maxBuild; + + public VersionRange(List mmps, int minBuild = 0, int maxBuild = 0x7fffffff) + { + this.mmps = mmps; + this.minBuild = minBuild; + this.maxBuild = maxBuild; + } + + public bool allows(string mmpb) + { + string[] parts = mmpb.Split('.'); + if (parts.Length != 4) + { + throw new Exception("Bad major.minor.patch.build: " + mmpb); + } + var build = Int32.Parse(parts[3]); + return this.mmps.Contains(parts[0] + "." + parts[1] + "." + parts[2]) && + this.minBuild <= build && + build <= this.maxBuild; + } + }; + class Pattern { public string name; - public List compatiblePatches; + public List compatible; public int cur_pos; public string cur_pattern; - public int minBuild; - public int maxBuild; public Dictionary offsets = new Dictionary(); - public Pattern(string name, List compatiblePatches, int minBuild, int maxBuild) + public Pattern(string name, List compatible) { this.name = name; - this.compatiblePatches = compatiblePatches; + this.compatible = compatible; this.cur_pos = 0; this.cur_pattern = ""; - this.minBuild = minBuild; - this.maxBuild = maxBuild; } - public Pattern(string name, List compatiblePatches) + public bool allows(string mmpb) { - this.name = name; - this.compatiblePatches = compatiblePatches; - this.cur_pos = 0; - this.cur_pattern = ""; - this.minBuild = 0; - this.maxBuild = 0; + foreach (var range in this.compatible) + { + if (range.allows(mmpb)) + { + return true; + } + } + return false; } // Utilities diff --git a/code/C#/DBDefsDumper/PatternBuilder.cs b/code/C#/DBDefsDumper/PatternBuilder.cs index 2e8f8a81099..fba39a7ecb3 100644 --- a/code/C#/DBDefsDumper/PatternBuilder.cs +++ b/code/C#/DBDefsDumper/PatternBuilder.cs @@ -45,7 +45,7 @@ public PatternBuilder() //); patterns.Add( - new Pattern("11.0.0", new() { "11.0.0", "11.0.2", "1.15.4", "4.4.1", "11.0.5", "11.0.7", "1.15.5" }, 54210, 0) + new Pattern("11.0.0", new() { new VersionRange (new() { "11.0.0", "11.0.2", "1.15.4", "4.4.1", "11.0.5", "11.0.7", "1.15.5" }, 54210 ) }) .Pointer(Name.DB_NAME) .FileDataID(Name.FDID) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -77,7 +77,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("10.1.0", new List { "10.1.0", "10.1.5", "1.14.4", "1.15.0", "3.4.2", "10.1.7", "10.2.0", "3.4.3", "10.2.5", "4.4.0", "1.15.1", "10.2.7", "10.2.6", "1.15.2", "1.15.3" }, 48480, 0) + new Pattern("10.1.0", new() { new VersionRange (new() { "10.1.0", "10.1.5", "1.14.4", "1.15.0", "3.4.2", "10.1.7", "10.2.0", "3.4.3", "10.2.5", "4.4.0", "1.15.1", "10.2.7", "10.2.6", "1.15.2", "1.15.3" }, 48480 ) }) .Pointer(Name.DB_NAME) .FileDataID(Name.FDID) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -107,7 +107,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("8.0.1", new List { "8.3.0", "8.2.5", "8.2.0", "8.1.5", "8.1.0", "8.0.1", "1.13.2", "1.13.3", "1.13.4", "9.0.1", "1.13.5", "8.3.7", "9.0.2", "1.13.6", "1.13.7", "9.0.5", "2.5.1", "9.1.0", "2.5.2", "1.14.0", "9.1.5", "1.14.1", "9.2.0", "1.14.2", "2.5.3", "2.5.4", "9.2.5", "1.14.3", "3.4.0", "10.0.0", "10.0.2", "9.2.7", "3.4.1", "10.0.5", "10.0.7"}, 26232, 0) + new Pattern("8.0.1", new() { new VersionRange (new() { "8.3.0", "8.2.5", "8.2.0", "8.1.5", "8.1.0", "8.0.1", "1.13.2", "1.13.3", "1.13.4", "9.0.1", "1.13.5", "8.3.7", "9.0.2", "1.13.6", "1.13.7", "9.0.5", "2.5.1", "9.1.0", "2.5.2", "1.14.0", "9.1.5", "1.14.1", "9.2.0", "1.14.2", "2.5.3", "2.5.4", "9.2.5", "1.14.3", "3.4.0", "10.0.0", "10.0.2", "9.2.7", "3.4.1", "10.0.5", "10.0.7" }, 26232 ) }) .Pointer(Name.DB_NAME) .FileDataID(Name.FDID) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -139,7 +139,10 @@ public PatternBuilder() ); patterns.Add( - new Pattern("8.0.1-prefdid", new List { "8.0.1" }, 25901, 26231) + new Pattern("8.0.1-prefdid,1.13.0-retail,7.3.5-release", + new() { new VersionRange (new() { "8.0.1" }, 25901, 26231 ), + new VersionRange (new() { "1.13.0" }), + new VersionRange (new() { "7.3.5" }, 25875, 26972 ) }) .Pointer(Name.DB_NAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -170,69 +173,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("1.13.0-retail", new List { "1.13.0" }, 28211, 28211) - .Pointer(Name.DB_NAME) - .FieldReference(Name.NUM_FIELD_IN_FILE) - .RecordSize(Name.RECORD_SIZE) - .FieldReference(Name.NUM_FIELD) - .OptionalFieldReference(Name.ID_COLUMN) - .Boolean(Name.SPARSE_TABLE) - .Pointer(Name.FIELD_OFFSETS) - .Pointer(Name.FIELD_SIZES) - .Pointer(Name.FIELD_TYPES) - .Pointer(Name.FIELD_FLAGS) - .Pointer(Name.FIELD_SIZES_IN_FILE) - .Pointer(Name.FIELD_TYPES_IN_FILE) - .Pointer(Name.FIELD_FLAGS_IN_FILE) - .Uint8(Name.FLAGS_58_21) - .Hash(Name.TABLE_HASH) - .Hash(Name.SIBLING_TABLE_HASH) - .Hash(Name.LAYOUT_HASH) - .Uint8(Name.FLAGS_68_421) - .FieldReference(Name.FIELD_NUM_IDX_INT) - .FieldReference(Name.FIELD_NUM_IDX_STRING) - .OptionalPointer(Name.FIELD_IDX_INT) - .OptionalPointer(Name.FIELD_IDX_STRING) - .Boolean(Name.HAS_RELATION) - .OptionalFieldReference(Name.FIELD_RELATION) - .OptionalFieldReference(Name.FIELD_RELATION_IN_FILE) - .OptionalPointer(Name.SORT_FUNC) - .Boolean(Name.UNKC0) - ); - - patterns.Add( - new Pattern("7.3.5-release", new List { "7.3.5" }, 25875, 26972) - .Pointer(Name.DB_NAME) - .FieldReference(Name.NUM_FIELD_IN_FILE) - .RecordSize(Name.RECORD_SIZE) - .FieldReference(Name.NUM_FIELD) - .OptionalFieldReference(Name.ID_COLUMN) - .Boolean(Name.SPARSE_TABLE) - .Pointer(Name.FIELD_OFFSETS) - .Pointer(Name.FIELD_SIZES) - .Pointer(Name.FIELD_TYPES) - .Pointer(Name.FIELD_FLAGS) - .Pointer(Name.FIELD_SIZES_IN_FILE) - .Pointer(Name.FIELD_TYPES_IN_FILE) - .Pointer(Name.FIELD_FLAGS_IN_FILE) - .Uint8(Name.FLAGS_58_21) - .Hash(Name.TABLE_HASH) - .Hash(Name.SIBLING_TABLE_HASH) - .Hash(Name.LAYOUT_HASH) - .Uint8(Name.FLAGS_68_421) - .FieldReference(Name.FIELD_NUM_IDX_INT) - .FieldReference(Name.FIELD_NUM_IDX_STRING) - .OptionalPointer(Name.FIELD_IDX_INT) - .OptionalPointer(Name.FIELD_IDX_STRING) - .Boolean(Name.HAS_RELATION) - .OptionalFieldReference(Name.FIELD_RELATION) - .OptionalFieldReference(Name.FIELD_RELATION_IN_FILE) - .OptionalPointer(Name.SORT_FUNC) - .Boolean(Name.UNKC0) - ); - - patterns.Add( - new Pattern("7.3.5-ptr", new List { "7.3.5" }, 25600, 25864) + new Pattern("7.3.5-ptr", new() { new VersionRange (new() { "7.3.5" }, 25600, 25864 ) }) .Pointer(Name.DB_NAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -262,7 +203,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("7.3.{0/2}", new List { "7.3.0", "7.3.2" }, 24473, 25549) // note: also matches release-7.3.5 even though different struct + new Pattern("7.3.{0/2}", new() { new VersionRange (new() { "7.3.0", "7.3.2" } ) }) // note: also matches release-7.3.5 even though different struct .Pointer(Name.DB_NAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -292,7 +233,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("7.{0,2.5}-release", new List { "7.2.0", "7.2.5"}, 23514, 24973) // note: also matches release-7.3.5 even though different struct + new Pattern("7.{0,2.5}-release", new() { new VersionRange (new() { "7.2.0", "7.2.5"}, 23514, 24973 ) }) // note: also matches release-7.3.5 even though different struct .Pointer(Name.DB_NAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -322,7 +263,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("7.{0,1}.{0,5}-release", new List { "7.0.3", "7.1.0", "7.1.5", "7.2.0" }, 20740, 23476) //todo: find minbuild + new Pattern("7.{0,1}.{0,5}-release", new() { new VersionRange (new() { "7.0.3", "7.1.0", "7.1.5", "7.2.0" }, 20740, 23476 ) }) //todo: find minbuild .Pointer(Name.DB_NAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -341,7 +282,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.x-db2-experimental", new List { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742) // note: subset of internal-6.0.1-dbc, so conflicts with that + new Pattern("6.x-db2-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that .Pointer(Name.DB_FILENAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -355,7 +296,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.x-dbc-experimental", new List { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742) // note: subset of internal-6.0.1-dbc, so conflicts with that + new Pattern("6.x-dbc-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that .Pointer(Name.DB_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -370,7 +311,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.0.1-db2-internal", new List { "6.0.1" }, 18179, 18179) // note: conflicts with internal-6.0.1-dbc + new Pattern("6.0.1-db2-internal", new() { new VersionRange (new() { "6.0.1" }, 18179, 18179 ) }) // note: conflicts with internal-6.0.1-dbc .Pointer(Name.DB_FILENAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -395,7 +336,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.0.1-dbc-internal", new List { "6.0.1" }, 18179, 18179) // note: conflicts with internal-6.0.1-db2 + new Pattern("6.0.1-dbc-internal", new() { new VersionRange (new() { "6.0.1" }, 18179, 18179 ) }) // note: conflicts with internal-6.0.1-db2 .Pointer(Name.DB_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -414,7 +355,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("5.0.1-dbc-internal", new List { "5.0.1" }) // note: subset of internal-6.0.1-dbc, so conflicts with that + new Pattern("5.0.1-dbc-internal", new() { new VersionRange (new() { "5.0.1" } ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that .Pointer(Name.DB_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) @@ -427,7 +368,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("5.0.1-db2-internal", new List { "5.0.1" }) // note: subset of internal-6.0.1-db2, so conflicts with that and internal-5.0.1-dbc + new Pattern("5.0.1-db2-internal", new() { new VersionRange (new() { "5.0.1" } ) }) // note: subset of internal-6.0.1-db2, so conflicts with that and internal-5.0.1-dbc .Pointer(Name.DB_FILENAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) diff --git a/code/C#/DBDefsDumper/Program.cs b/code/C#/DBDefsDumper/Program.cs index 3673fa830b0..17a94e3c21b 100644 --- a/code/C#/DBDefsDumper/Program.cs +++ b/code/C#/DBDefsDumper/Program.cs @@ -261,58 +261,10 @@ static void Main(string[] args) { if (patternOverride == "") { - // Skip versions of the pattern that aren't for this expansion - if (build.StartsWith("1")) + if (!pattern.allows(build)) { - if (!pattern.compatiblePatches.Contains(build.Substring(0, 6))) - { - Console.WriteLine("Skipping " + pattern.name + " as it does not list " + build + " as compatible!"); - continue; - } - - if (!pattern.compatiblePatches.Contains(build.Substring(0, 6))) - { - Console.WriteLine("Skipping " + pattern.name + " as it does not list " + build + " as compatible!"); - continue; - } - - if (pattern.minBuild != 0 && pattern.minBuild > int.Parse(build.Substring(7))) - { - Console.WriteLine("Skipping " + pattern.name + " as minimum build " + pattern.minBuild + " exceeds build of " + build.Substring(6)); - continue; - } - - if (pattern.maxBuild != 0 && int.Parse(build.Substring(7)) > pattern.maxBuild) - { - Console.WriteLine("Skipping " + pattern.name + " as maximum build " + pattern.maxBuild + " exceeds build of " + build.Substring(6)); - continue; - } - } - else - { - if (!pattern.compatiblePatches.Contains(build.Substring(0, 5))) - { - Console.WriteLine("Skipping " + pattern.name + " as it does not list " + build + " as compatible!"); - continue; - } - - if (!pattern.compatiblePatches.Contains(build.Substring(0, 5))) - { - Console.WriteLine("Skipping " + pattern.name + " as it does not list " + build + " as compatible!"); - continue; - } - - if (pattern.minBuild != 0 && pattern.minBuild > int.Parse(build.Substring(6))) - { - Console.WriteLine("Skipping " + pattern.name + " as minimum build " + pattern.minBuild + " exceeds build of " + build.Substring(6)); - continue; - } - - if (pattern.maxBuild != 0 && int.Parse(build.Substring(6)) > pattern.maxBuild) - { - Console.WriteLine("Skipping " + pattern.name + " as maximum build " + pattern.maxBuild + " exceeds build of " + build.Substring(6)); - continue; - } + Console.WriteLine("Skipping " + pattern.name + " as it does not list " + build + " as compatible!"); + continue; } } else From f07ca7412ca38ec6872f68d6dcf015479e7b4e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Thu, 7 Nov 2024 20:26:52 +0100 Subject: [PATCH 2/7] mention build requirement --- code/C#/DBDefsDumper/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/C#/DBDefsDumper/Program.cs b/code/C#/DBDefsDumper/Program.cs index 17a94e3c21b..7a99bb5cf8a 100644 --- a/code/C#/DBDefsDumper/Program.cs +++ b/code/C#/DBDefsDumper/Program.cs @@ -16,7 +16,7 @@ static void Main(string[] args) { if (args.Length < 2) { - throw new ArgumentException("Not enough arguments! Required: file, outdir, (build in x.x.x format), (pattern name to always use)"); + throw new ArgumentException("Not enough arguments! Required: file, outdir, (build in x.x.x.build format), (pattern name to always use)"); } if (!File.Exists(args[0])) From 93cb6b7c9341c510fcd03d657b480480672575d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Thu, 7 Nov 2024 20:27:11 +0100 Subject: [PATCH 3/7] also filter DBFilesClient\ in other pattern type --- code/C#/DBDefsDumper/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/C#/DBDefsDumper/Program.cs b/code/C#/DBDefsDumper/Program.cs index 7a99bb5cf8a..6153d51cc63 100644 --- a/code/C#/DBDefsDumper/Program.cs +++ b/code/C#/DBDefsDumper/Program.cs @@ -423,6 +423,10 @@ static void Main(string[] args) { bin.BaseStream.Position = (long)translate((ulong)meta.dbFilenameOffs); var name = bin.ReadCString(); + if (name.Contains("DBFilesClient")) + { + name = name.Substring(name.IndexOf("\\") + 1); + } if (badNames.Contains(Path.GetFileNameWithoutExtension(name))) { From 66ed1159837fb48667dcb37ec4bd93ad6a8cd250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Thu, 7 Nov 2024 20:27:29 +0100 Subject: [PATCH 4/7] filter pbeWithMD5AndDES-CBC --- code/C#/DBDefsDumper/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/C#/DBDefsDumper/Program.cs b/code/C#/DBDefsDumper/Program.cs index 6153d51cc63..32e6b85b57c 100644 --- a/code/C#/DBDefsDumper/Program.cs +++ b/code/C#/DBDefsDumper/Program.cs @@ -252,7 +252,7 @@ static void Main(string[] args) var metas = new Dictionary(); // Manual list of names to ignore as these are known bad pattern matches - var badNames = new List() { "rippleDetail", "RAIDrippleDetail" }; + var badNames = new List() { "rippleDetail", "RAIDrippleDetail", "pbeWithMD5AndDES-CBC" }; var patternBuilder = new PatternBuilder(); Pattern usedPattern = null; From 611692af81923755c82b8120c845c1424b650bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Mon, 11 Nov 2024 21:36:33 +0100 Subject: [PATCH 5/7] dumper: 7.* pattern stuff --- code/C#/DBDefsDumper/DBDefsDumper.csproj | 2 +- code/C#/DBDefsDumper/Pattern.cs | 1 + code/C#/DBDefsDumper/PatternBuilder.cs | 70 ++++++++++++++++++++++-- code/C#/DBDefsDumper/Program.cs | 22 ++++---- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/code/C#/DBDefsDumper/DBDefsDumper.csproj b/code/C#/DBDefsDumper/DBDefsDumper.csproj index bcac44563d6..ced679d5502 100644 --- a/code/C#/DBDefsDumper/DBDefsDumper.csproj +++ b/code/C#/DBDefsDumper/DBDefsDumper.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 diff --git a/code/C#/DBDefsDumper/Pattern.cs b/code/C#/DBDefsDumper/Pattern.cs index ed7013ec3c4..9d4ad39bcab 100644 --- a/code/C#/DBDefsDumper/Pattern.cs +++ b/code/C#/DBDefsDumper/Pattern.cs @@ -197,5 +197,6 @@ class Name public const string UNK_BOOL_601dbc_x3b = "unkown bool x3b 6.0.1"; public const string UNK_BOOL_11DB2_x1C = "unknown bool x1C 11.0.0"; public const string UNK_BOOL_11DB2_x1D = "unknown bool x1D 11.0.0"; + public const string UNK_EXTRA_POINTER_IN_720 = "UNK_EXTRA_POINTER_IN_720"; } } diff --git a/code/C#/DBDefsDumper/PatternBuilder.cs b/code/C#/DBDefsDumper/PatternBuilder.cs index fba39a7ecb3..d86d6017685 100644 --- a/code/C#/DBDefsDumper/PatternBuilder.cs +++ b/code/C#/DBDefsDumper/PatternBuilder.cs @@ -210,10 +210,10 @@ public PatternBuilder() .FieldReference(Name.NUM_FIELD) .OptionalFieldReference(Name.ID_COLUMN) .Boolean(Name.SPARSE_TABLE) - .Pointer(Name.FIELD_FLAGS) .Pointer(Name.FIELD_OFFSETS) .Pointer(Name.FIELD_SIZES) .Pointer(Name.FIELD_TYPES) + .Pointer(Name.FIELD_FLAGS) .Pointer(Name.FIELD_SIZES_IN_FILE) .Pointer(Name.FIELD_TYPES_IN_FILE) .Pointer(Name.FIELD_FLAGS_IN_FILE) @@ -233,14 +233,76 @@ public PatternBuilder() ); patterns.Add( - new Pattern("7.{0,2.5}-release", new() { new VersionRange (new() { "7.2.0", "7.2.5"}, 23514, 24973 ) }) // note: also matches release-7.3.5 even though different struct + new Pattern("7.2.0-ptr-a", new() { new VersionRange (new() { "7.2.0" } ) }) .Pointer(Name.DB_NAME) + .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE) .FieldReference(Name.NUM_FIELD) .OptionalFieldReference(Name.ID_COLUMN) .Boolean(Name.SPARSE_TABLE) - .Pointer(Name.FIELD_OFFSETS) + .Pointer(Name.FIELD_OFFSETS) // might be dynamic in 7.2.0 builds!? + .Pointer(Name.FIELD_SIZES) + .Pointer(Name.FIELD_TYPES) + .Pointer(Name.FIELD_FLAGS) + .Pointer(Name.FIELD_SIZES_IN_FILE) + .Pointer(Name.FIELD_TYPES_IN_FILE) + .Pointer(Name.FIELD_FLAGS_IN_FILE) + .Uint8(Name.FLAGS_58_21) + .Hash(Name.TABLE_HASH) + .Hash(Name.SIBLING_TABLE_HASH) + .Hash(Name.LAYOUT_HASH) + .Uint8(Name.FLAGS_68_421) + .FieldReference(Name.FIELD_NUM_IDX_INT) + .FieldReference(Name.FIELD_NUM_IDX_STRING) + .OptionalPointer(Name.FIELD_IDX_INT) + .OptionalPointer(Name.FIELD_IDX_STRING) + .OptionalFieldReference(Name.FIELD_RELATION) + .OptionalFieldReference(Name.FIELD_RELATION_IN_FILE) + .OptionalPointer(Name.SORT_FUNC) + .Boolean(Name.UNKC0) + ); + + patterns.Add( + new Pattern("7.2.0-ptr-b", new() { new VersionRange (new() { "7.2.0" } ) }) + .Pointer(Name.DB_NAME) + .FieldReference(Name.NUM_FIELD_IN_FILE) + .RecordSize(Name.RECORD_SIZE) + .FieldReference(Name.NUM_FIELD) + .OptionalFieldReference(Name.ID_COLUMN) + .Boolean(Name.SPARSE_TABLE) + .OptionalPointer(Name.UNK_EXTRA_POINTER_IN_720) + .Pointer(Name.FIELD_OFFSETS) // might be dynamic in 7.2.0 builds!? + .Pointer(Name.FIELD_SIZES) + .Pointer(Name.FIELD_TYPES) + .Pointer(Name.FIELD_FLAGS) + .Pointer(Name.FIELD_SIZES_IN_FILE) + .Pointer(Name.FIELD_TYPES_IN_FILE) + .Pointer(Name.FIELD_FLAGS_IN_FILE) + .Uint8(Name.FLAGS_58_21) + .Hash(Name.TABLE_HASH) + .Hash(Name.SIBLING_TABLE_HASH) + .Hash(Name.LAYOUT_HASH) + .Uint8(Name.FLAGS_68_421) + .FieldReference(Name.FIELD_NUM_IDX_INT) + .FieldReference(Name.FIELD_NUM_IDX_STRING) + .OptionalPointer(Name.FIELD_IDX_INT) + .OptionalPointer(Name.FIELD_IDX_STRING) + .OptionalFieldReference(Name.FIELD_RELATION) + .OptionalFieldReference(Name.FIELD_RELATION_IN_FILE) + .OptionalPointer(Name.SORT_FUNC) + .Boolean(Name.UNKC0) + ); + + patterns.Add( + new Pattern("7.{0,2.5}-release", new() { new VersionRange (new() { "7.2.0", "7.2.5"} ) }) // note: also matches release-7.3.5 even though different struct + .Pointer(Name.DB_NAME) + .FieldReference(Name.NUM_FIELD_IN_FILE) + .RecordSize(Name.RECORD_SIZE) + .FieldReference(Name.NUM_FIELD) + .OptionalFieldReference(Name.ID_COLUMN) + .Boolean(Name.SPARSE_TABLE) + .Pointer(Name.FIELD_OFFSETS) // might be dynamic in 7.2.0 builds!? .Pointer(Name.FIELD_SIZES) .Pointer(Name.FIELD_TYPES) .Pointer(Name.FIELD_FLAGS) @@ -263,7 +325,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("7.{0,1}.{0,5}-release", new() { new VersionRange (new() { "7.0.3", "7.1.0", "7.1.5", "7.2.0" }, 20740, 23476 ) }) //todo: find minbuild + new Pattern("7.{0,1}.{0,5}-release", new() { new VersionRange (new() { "7.0.1", "7.0.3", "7.1.0", "7.1.5", "7.2.0" }, 20740, 23476 ) }) //todo: find minbuild .Pointer(Name.DB_NAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) diff --git a/code/C#/DBDefsDumper/Program.cs b/code/C#/DBDefsDumper/Program.cs index 32e6b85b57c..5c49930e734 100644 --- a/code/C#/DBDefsDumper/Program.cs +++ b/code/C#/DBDefsDumper/Program.cs @@ -296,7 +296,7 @@ static void Main(string[] args) { bin.BaseStream.Position = matchPos + pattern.offsets[Name.FDID]; var fdid = bin.ReadUInt32(); - if (fdid < 53183 || fdid > (4058697 * 4) /*4x max at March 23 2021*/) + if (fdid < 53183 || fdid > (6363366 * 4) /*4x max at 2024-11-07*/) { Console.WriteLine("Invalid filedataid " + fdid + ", skipping match.."); continue; @@ -459,9 +459,9 @@ static void Main(string[] args) // Process DBMetas foreach (var meta in metas) { - if ((long)translate((ulong)meta.Value.field_offsets_offs) > bin.BaseStream.Length) + if ((long)translate((ulong)meta.Value.field_sizes_offs) > bin.BaseStream.Length) { - Console.WriteLine("Skipping reading of " + meta.Key + " because field offset (" + (long)translate((ulong)meta.Value.field_offsets_offs) + ") is outside of file range (" + bin.BaseStream.Length + ")!"); + Console.WriteLine("Skipping reading of " + meta.Key + " because field sizes offset (" + (long)translate((ulong)meta.Value.field_sizes_offs) + ") is outside of file range (" + bin.BaseStream.Length + ")!"); continue; } @@ -494,7 +494,8 @@ static void Main(string[] args) fieldCount = meta.Value.num_fields; } - var field_offsets = ReadFieldArray(bin, fieldCount, (long)translate((ulong)meta.Value.field_offsets_offs)); // TODO: Field offsets is currently unused, use to verify sizes + // \todo For some reason in 7.2.0, these are dynamically initialized, so unavailable. Luckily we don't use them anyway. + /// var field_offsets = ReadFieldArray(bin, fieldCount, (long)translate((ulong)meta.Value.field_offsets_offs)); // TODO: Field offsets is currently unused, use to verify sizes var field_sizes = ReadFieldArray(bin, fieldCount, (long)translate((ulong)meta.Value.field_sizes_offs)); var field_types = ReadFieldArray(bin, fieldCount, (long)translate((ulong)meta.Value.field_types_offs)); @@ -1013,18 +1014,19 @@ public static (string, int) TypeToT(int type, FieldFlags flag) case 0 | FieldFlags.f_unsigned | FieldFlags.f_maybe_compressed | FieldFlags.f_maybe_fk: return ("uint", 32); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } case 1: switch (flag) { case 0 | 0 | 0 | 0: + case 0 | 0 | FieldFlags.f_maybe_compressed | FieldFlags.f_maybe_fk: return ("int", 64); case 0 | FieldFlags.f_unsigned | 0 | 0: case 0 | FieldFlags.f_unsigned | 0 | FieldFlags.f_maybe_fk: return ("uint", 64); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } case 2: switch (flag) @@ -1034,7 +1036,7 @@ public static (string, int) TypeToT(int type, FieldFlags flag) case FieldFlags.f_localized | 0 | 0 | 0: return ("locstring", 0); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } case 3: switch (flag) @@ -1043,7 +1045,7 @@ public static (string, int) TypeToT(int type, FieldFlags flag) case 0 | 0 | FieldFlags.f_maybe_compressed | 0: return ("float", 0); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } case 4: switch (flag) @@ -1056,7 +1058,7 @@ public static (string, int) TypeToT(int type, FieldFlags flag) case 0 | FieldFlags.f_unsigned | FieldFlags.f_maybe_compressed | FieldFlags.f_maybe_fk: return ("uint", 8); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } case 5: switch (flag) @@ -1072,7 +1074,7 @@ public static (string, int) TypeToT(int type, FieldFlags flag) case 0 | FieldFlags.f_unsigned | FieldFlags.f_maybe_compressed | FieldFlags.f_maybe_fk: return ("uint", 16); default: - throw new Exception("Unknown flag combination!"); + throw new Exception("Unknown flag combination! type=" + type + ", flag=" + flag); } default: throw new Exception("Ran into unknown field type: " + type); From de9cd09d0887652fdf5084a3f9c7d2011da00341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Mon, 11 Nov 2024 23:34:44 +0100 Subject: [PATCH 6/7] update net7 -> net8 --- code/C#/DBDefsMerge/DBDefsMerge.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/C#/DBDefsMerge/DBDefsMerge.csproj b/code/C#/DBDefsMerge/DBDefsMerge.csproj index 1a56c7b0e05..00994218586 100644 --- a/code/C#/DBDefsMerge/DBDefsMerge.csproj +++ b/code/C#/DBDefsMerge/DBDefsMerge.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 From 68b86cc4cdbea3c2be33f188ac0bb8deb2c76bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20L=C3=B6rwald?= Date: Mon, 18 Nov 2024 22:51:32 +0100 Subject: [PATCH 7/7] allow 6.2.0a --- code/C#/DBDefsDumper/PatternBuilder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/C#/DBDefsDumper/PatternBuilder.cs b/code/C#/DBDefsDumper/PatternBuilder.cs index d86d6017685..0ddc13382a9 100644 --- a/code/C#/DBDefsDumper/PatternBuilder.cs +++ b/code/C#/DBDefsDumper/PatternBuilder.cs @@ -344,7 +344,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.x-db2-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that + new Pattern("6.x-db2-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.0a", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that .Pointer(Name.DB_FILENAME) .Pointer(Name.DB_CACHE_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) @@ -358,7 +358,7 @@ public PatternBuilder() ); patterns.Add( - new Pattern("6.x-dbc-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that + new Pattern("6.x-dbc-experimental", new() { new VersionRange (new() { "6.0.1", "6.0.2", "6.0.3", "6.1.0", "6.1.2", "6.2.0", "6.2.0a", "6.2.1", "6.2.2", "6.2.3", "6.2.4" }, 18125, 21742 ) }) // note: subset of internal-6.0.1-dbc, so conflicts with that .Pointer(Name.DB_FILENAME) .FieldReference(Name.NUM_FIELD_IN_FILE) .RecordSize(Name.RECORD_SIZE)