From feb8db0a24204f54218b74255b13706a6158617c Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Fri, 29 May 2020 16:41:38 +0800 Subject: [PATCH 1/6] Add BloomFilter to levelDB --- src/LevelDBStore/IO/Data/LevelDB/Native.cs | 3 +++ src/LevelDBStore/LevelDBStore/config.json | 5 +++-- src/LevelDBStore/Plugins/Storage/LevelDBStore.cs | 6 +++++- src/LevelDBStore/Plugins/Storage/Store.cs | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/LevelDBStore/IO/Data/LevelDB/Native.cs b/src/LevelDBStore/IO/Data/LevelDB/Native.cs index f9991ff81..8959ae390 100644 --- a/src/LevelDBStore/IO/Data/LevelDB/Native.cs +++ b/src/LevelDBStore/IO/Data/LevelDB/Native.cs @@ -155,6 +155,9 @@ public static class Native [DllImport("libleveldb", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)] public static extern void leveldb_options_set_filter_policy(IntPtr /*Options*/ options, IntPtr /*FilterPolicy*/ policy); + + [DllImport("libleveldb", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)] + public static extern IntPtr leveldb_filterpolicy_create_bloom(int bits_per_key); #endregion #region ReadOptions diff --git a/src/LevelDBStore/LevelDBStore/config.json b/src/LevelDBStore/LevelDBStore/config.json index e6683b8e4..9126e212a 100644 --- a/src/LevelDBStore/LevelDBStore/config.json +++ b/src/LevelDBStore/LevelDBStore/config.json @@ -1,5 +1,6 @@ -{ +{ "PluginConfiguration": { - "Path": "Data_LevelDB_{0}" + "Path": "Data_LevelDB_{0}", + "BloomFilterBitsPerKey": "20" } } diff --git a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs index 265c4286b..9e6c9ccaf 100644 --- a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs +++ b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Neo.Persistence; namespace Neo.Plugins.Storage @@ -5,10 +6,13 @@ namespace Neo.Plugins.Storage public class LevelDBStore : Plugin, IStoragePlugin { private string path; + private int bloomFilterBitsPerKey; protected override void Configure() { - path = string.Format(GetConfiguration().GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); + IConfigurationSection config = GetConfiguration(); + path = string.Format(config.GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); + bloomFilterBitsPerKey = int.Parse(config.GetSection("BloomFilterBitsPerKey").Value ?? "20"); } public IStore GetStore() diff --git a/src/LevelDBStore/Plugins/Storage/Store.cs b/src/LevelDBStore/Plugins/Storage/Store.cs index fef486d27..aac9c31db 100644 --- a/src/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/LevelDBStore/Plugins/Storage/Store.cs @@ -14,7 +14,7 @@ internal class Store : IStore public Store(string path) { - this.db = DB.Open(path, new Options { CreateIfMissing = true }); + this.db = DB.Open(path, new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(20) }); byte[] value = db.Get(ReadOptions.Default, Helper.CreateKey(SYS_Version)); if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0")) return; From c7c00f49b35207c726dfb419c9dd73f542aa49d5 Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Fri, 29 May 2020 17:31:42 +0800 Subject: [PATCH 2/6] Code optimization --- src/LevelDBStore/Plugins/Storage/LevelDBStore.cs | 4 ++-- src/LevelDBStore/Plugins/Storage/Store.cs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs index 9e6c9ccaf..3aab8aa85 100644 --- a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs +++ b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs @@ -12,12 +12,12 @@ protected override void Configure() { IConfigurationSection config = GetConfiguration(); path = string.Format(config.GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); - bloomFilterBitsPerKey = int.Parse(config.GetSection("BloomFilterBitsPerKey").Value ?? "20"); + int.TryParse(config.GetSection("BloomFilterBitsPerKey")?.Value, out bloomFilterBitsPerKey); } public IStore GetStore() { - return new Store(path); + return new Store(path, bloomFilterBitsPerKey); } } } diff --git a/src/LevelDBStore/Plugins/Storage/Store.cs b/src/LevelDBStore/Plugins/Storage/Store.cs index aac9c31db..90c3fd9c7 100644 --- a/src/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/LevelDBStore/Plugins/Storage/Store.cs @@ -12,9 +12,12 @@ internal class Store : IStore private const byte SYS_Version = 0xf0; private readonly DB db; - public Store(string path) + public Store(string path, int bloomFilterBitsPerKey) { - this.db = DB.Open(path, new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(20) }); + if (bloomFilterBitsPerKey > 0) + this.db = DB.Open(path, new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(bloomFilterBitsPerKey) }); + else + this.db = DB.Open(path, new Options { CreateIfMissing = true }); byte[] value = db.Get(ReadOptions.Default, Helper.CreateKey(SYS_Version)); if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0")) return; From 782b88ee1312f77b3cf5a7b05b4e005cfce7fd4e Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Mon, 1 Jun 2020 15:48:17 +0800 Subject: [PATCH 3/6] Code optimization --- src/LevelDBStore/LevelDBStore/config.json | 2 +- src/LevelDBStore/Plugins/Storage/LevelDBStore.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LevelDBStore/LevelDBStore/config.json b/src/LevelDBStore/LevelDBStore/config.json index 9126e212a..b3ede47ba 100644 --- a/src/LevelDBStore/LevelDBStore/config.json +++ b/src/LevelDBStore/LevelDBStore/config.json @@ -1,6 +1,6 @@ { "PluginConfiguration": { "Path": "Data_LevelDB_{0}", - "BloomFilterBitsPerKey": "20" + "BloomFilterBitsPerKey": 20 } } diff --git a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs index 3aab8aa85..576a26814 100644 --- a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs +++ b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs @@ -12,7 +12,7 @@ protected override void Configure() { IConfigurationSection config = GetConfiguration(); path = string.Format(config.GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); - int.TryParse(config.GetSection("BloomFilterBitsPerKey")?.Value, out bloomFilterBitsPerKey); + bloomFilterBitsPerKey = config.GetSection("BloomFilterBitsPerKey")?.Get() ?? 0; } public IStore GetStore() From 7bccc9f4f18d9c51c62f2e981f0bb9b3ee6eacb2 Mon Sep 17 00:00:00 2001 From: Qiao Jin <43407364+Qiao-Jin@users.noreply.github.com> Date: Tue, 14 Jul 2020 17:38:10 +0800 Subject: [PATCH 4/6] Optimize bloomfilter bits per key value --- src/LevelDBStore/LevelDBStore/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LevelDBStore/LevelDBStore/config.json b/src/LevelDBStore/LevelDBStore/config.json index b3ede47ba..6020cdc17 100644 --- a/src/LevelDBStore/LevelDBStore/config.json +++ b/src/LevelDBStore/LevelDBStore/config.json @@ -1,6 +1,6 @@ { "PluginConfiguration": { "Path": "Data_LevelDB_{0}", - "BloomFilterBitsPerKey": 20 + "BloomFilterBitsPerKey": 15 } } From 3f49b2b6b64c9b873ef23d2a30a655c4d7b04fa2 Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Wed, 15 Jul 2020 10:29:52 +0800 Subject: [PATCH 5/6] Hard-code bloomfilter bitsPeyKey to 15 --- src/LevelDBStore/LevelDBStore/config.json | 3 +-- src/LevelDBStore/Plugins/Storage/LevelDBStore.cs | 7 ++----- src/LevelDBStore/Plugins/Storage/Store.cs | 7 ++----- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/LevelDBStore/LevelDBStore/config.json b/src/LevelDBStore/LevelDBStore/config.json index 6020cdc17..f31d07fd6 100644 --- a/src/LevelDBStore/LevelDBStore/config.json +++ b/src/LevelDBStore/LevelDBStore/config.json @@ -1,6 +1,5 @@ { "PluginConfiguration": { - "Path": "Data_LevelDB_{0}", - "BloomFilterBitsPerKey": 15 + "Path": "Data_LevelDB_{0}" } } diff --git a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs index efe56abe9..fa4ab0922 100644 --- a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs +++ b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs @@ -6,20 +6,17 @@ namespace Neo.Plugins.Storage public class LevelDBStore : Plugin, IStorageProvider { private string path; - private int bloomFilterBitsPerKey; public override string Description => "Uses LevelDB to store the blockchain data"; protected override void Configure() { - IConfigurationSection config = GetConfiguration(); - path = string.Format(config.GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); - bloomFilterBitsPerKey = config.GetSection("BloomFilterBitsPerKey")?.Get() ?? 0; + path = string.Format(GetConfiguration().GetSection("Path").Value ?? "Data_LevelDB_{0}", ProtocolSettings.Default.Magic.ToString("X8")); } public IStore GetStore() { - return new Store(path, bloomFilterBitsPerKey); + return new Store(path); } } } diff --git a/src/LevelDBStore/Plugins/Storage/Store.cs b/src/LevelDBStore/Plugins/Storage/Store.cs index ed6164b07..6d9aba937 100644 --- a/src/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/LevelDBStore/Plugins/Storage/Store.cs @@ -14,12 +14,9 @@ internal class Store : IStore private const byte SYS_Version = 0xf0; private readonly DB db; - public Store(string path, int bloomFilterBitsPerKey) + public Store(string path) { - if (bloomFilterBitsPerKey > 0) - this.db = DB.Open(path, new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(bloomFilterBitsPerKey) }); - else - this.db = DB.Open(path, new Options { CreateIfMissing = true }); + this.db = DB.Open(path, new Options { CreateIfMissing = true, FilterPolicy = Native.leveldb_filterpolicy_create_bloom(15) }); byte[] value = db.Get(ReadOptions.Default, LHelper.CreateKey(SYS_Version)); if (value != null && Version.TryParse(Encoding.ASCII.GetString(value), out Version version) && version >= Version.Parse("3.0.0")) return; From aca9ac9c54feaea83249952346b83ffd2a587d31 Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Wed, 15 Jul 2020 10:32:41 +0800 Subject: [PATCH 6/6] Code optimization --- src/LevelDBStore/Plugins/Storage/LevelDBStore.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs index fa4ab0922..09a526bcc 100644 --- a/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs +++ b/src/LevelDBStore/Plugins/Storage/LevelDBStore.cs @@ -1,4 +1,3 @@ -using Microsoft.Extensions.Configuration; using Neo.Persistence; namespace Neo.Plugins.Storage