From 1e73aa8e0d4af4191719c2fbfccf46195c74c296 Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Tue, 7 Nov 2017 16:51:13 -0300 Subject: [PATCH 01/46] market_history default buckets https://github.com/bitshares/bitshares-core/issues/465 --- libraries/plugins/market_history/market_history_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/market_history/market_history_plugin.cpp b/libraries/plugins/market_history/market_history_plugin.cpp index 6ec38968bc..94633caf89 100644 --- a/libraries/plugins/market_history/market_history_plugin.cpp +++ b/libraries/plugins/market_history/market_history_plugin.cpp @@ -255,7 +255,7 @@ void market_history_plugin::plugin_set_program_options( ) { cli.add_options() - ("bucket-size", boost::program_options::value()->default_value("[15,60,300,3600,86400]"), + ("bucket-size", boost::program_options::value()->default_value("[60,300,900,1800,3600,14400,86400]"), "Track market history by grouping orders into buckets of equal size measured in seconds specified as a JSON array of numbers") ("history-per-size", boost::program_options::value()->default_value(1000), "How far back in time to track history for each bucket size, measured in the number of buckets (default: 1000)") From af2bce6a3b93f0f213d402466719a5fafc8c1b9d Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Tue, 7 Nov 2017 16:55:27 -0300 Subject: [PATCH 02/46] mistake sorry, i made a commit directly to master instead of sending as pull request to develop. this commit reverts back. --- libraries/plugins/market_history/market_history_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/market_history/market_history_plugin.cpp b/libraries/plugins/market_history/market_history_plugin.cpp index 94633caf89..6ec38968bc 100644 --- a/libraries/plugins/market_history/market_history_plugin.cpp +++ b/libraries/plugins/market_history/market_history_plugin.cpp @@ -255,7 +255,7 @@ void market_history_plugin::plugin_set_program_options( ) { cli.add_options() - ("bucket-size", boost::program_options::value()->default_value("[60,300,900,1800,3600,14400,86400]"), + ("bucket-size", boost::program_options::value()->default_value("[15,60,300,3600,86400]"), "Track market history by grouping orders into buckets of equal size measured in seconds specified as a JSON array of numbers") ("history-per-size", boost::program_options::value()->default_value(1000), "How far back in time to track history for each bucket size, measured in the number of buckets (default: 1000)") From d6fda25b25e8838d075e5992368e010b79742232 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 9 May 2018 08:47:37 -0400 Subject: [PATCH 03/46] Add: Bug Report & Feature Request --- .github/ISSUE_TEMPLATE/bug_report.md | 40 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 34 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..46f4671897 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,40 @@ +--- +name: Bug Report +about: Create a detailed report about a deficiency in the BitShares Core implementation. + +--- + +**Bug Description** +A clear and concise description of what the bug is. + +**Steps To Reproduce** +Steps to reproduce the behavior: +1. Execute API call '...' +2. Using JSON payload '...' +3. Received response '...' +4. See error + +**Expected Behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain process flow and behavior. + +**witness_node --version (please complete the following information):** + - Version: [e.g. 2.0.180425] + - SHA: [e.g. bf263b1db47a91ea9748118df1abb5a22cafc40b] + - SSL: [e.g. OpenSSL 1.0.2g 1 Mar 2016] + - Boost: [e.g. 1.62] + - Websocket++: [e.g. 0.7.0] + - Host OS: [e.g. Ubuntu 17.10] + +**Additional Context** +Add any other context about the problem here. + +## CORE TEAM TASK LIST +- [ ] Evaluate / Prioritize Bug Request +- [ ] Refine User Stories / Requirements +- [ ] Define Test Cases +- [ ] Design / Develop Solution +- [ ] Perform QA/Testing +- [ ] Update Documentation diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..df5d009fb1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,34 @@ +--- +name: Feature Request +about: Suggest an idea for the BitShares Core Team to evaluate and prioritize for development. + +--- + +**User Story** +Please tell us about your feature request using the User Story format: +As a `` I want `` so that ``. + +At minimum, please define the ``, `` and `` for your feature request. The `` may be the system software, a component thereof, the end user, etc.; please be specific describing the context. The `` details the solution your feature will provide; please describe the process flow for the functionality. The `` details the benefits the feature will deliver; consider referencing alternative implementations for context. + +**Impacts** +Describe which portion(s) of BitShares Core may be impacted by your request. Please tick at least one box. +- [ ] API (the application programming interface) +- [ ] Build (the build process or something prior to compiled code) +- [ ] CLI (the command line wallet) +- [ ] Consensus (the validation or commitment of data) +- [ ] Deployment (the deployment process after building such as Docker, Travis, etc.) +- [ ] P2P (the peer-to-peer network for transaction/block propagation) +- [ ] Performance (system or user efficiency, performance, etc.) +- [ ] Security (system or user security) +- [ ] Other (please add below) + +**Additional Context** +Add any other context about your request here. + +## CORE TEAM TASK LIST +- [ ] Evaluate / Prioritize Feature Request +- [ ] Refine User Stories / Requirements +- [ ] Define Test Cases +- [ ] Design / Develop Solution +- [ ] Perform QA/Testing +- [ ] Update Documentation From 16de19c9ee8d5b445b954ef83c6f68c60f04c16f Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 9 May 2018 09:54:47 -0400 Subject: [PATCH 04/46] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 46f4671897..25e2c59024 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,6 +4,17 @@ about: Create a detailed report about a deficiency in the BitShares Core impleme --- +**Instructions** +Pleaae include a detailed Title above. Next, please complete the following sections below: +* Bug Description +* Steps To Reproduce +* Expected Behavior +* Screenshots (optional) +* witness_node --version (optional) +* Aditional Context (optional) + +Finally, press the 'submit issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. + **Bug Description** A clear and concise description of what the bug is. @@ -17,10 +28,10 @@ Steps to reproduce the behavior: **Expected Behavior** A clear and concise description of what you expected to happen. -**Screenshots** +**Screenshots (optional)** If applicable, add screenshots to help explain process flow and behavior. -**witness_node --version (please complete the following information):** +**witness_node --version (optional):** - Version: [e.g. 2.0.180425] - SHA: [e.g. bf263b1db47a91ea9748118df1abb5a22cafc40b] - SSL: [e.g. OpenSSL 1.0.2g 1 Mar 2016] @@ -28,7 +39,7 @@ If applicable, add screenshots to help explain process flow and behavior. - Websocket++: [e.g. 0.7.0] - Host OS: [e.g. Ubuntu 17.10] -**Additional Context** +**Additional Context (optional)** Add any other context about the problem here. ## CORE TEAM TASK LIST From 5f1f6d984c458267bb2f973646e623d26e281462 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 9 May 2018 09:58:09 -0400 Subject: [PATCH 05/46] Update feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index df5d009fb1..962fcffb23 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,6 +4,14 @@ about: Suggest an idea for the BitShares Core Team to evaluate and prioritize fo --- +**Instructions** +Pleaae include a detailed Title above. Next, please complete the following sections below: +* User Story +* Impacts +* Aditional Context (optional) + +Finally, press the 'submit issue' button. The Core Team will evaluate and prioritize your Feature Request for future development. + **User Story** Please tell us about your feature request using the User Story format: As a `` I want `` so that ``. @@ -22,7 +30,7 @@ Describe which portion(s) of BitShares Core may be impacted by your request. Ple - [ ] Security (system or user security) - [ ] Other (please add below) -**Additional Context** +**Additional Context (optional)** Add any other context about your request here. ## CORE TEAM TASK LIST From 54f72685ad6a6c3fd7ea7878de359bf4e65d2b2a Mon Sep 17 00:00:00 2001 From: Alfredo Date: Mon, 14 May 2018 10:08:17 -0300 Subject: [PATCH 06/46] update project number --- Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index fad32a549a..c0af095a2e 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "Bitshares-Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "2.0.180202" +PROJECT_NUMBER = "2.0.180425" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From d23896db3b101196fb072b126d64871c1ad5c964 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 23 May 2018 16:51:28 -0400 Subject: [PATCH 07/46] Fixup: ISSUE_TEMPLATE --- .github/ISSUE_TEMPLATE/bug_report.md | 16 +++++++--------- .github/ISSUE_TEMPLATE/feature_request.md | 14 ++++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 25e2c59024..040adcb44b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -5,25 +5,25 @@ about: Create a detailed report about a deficiency in the BitShares Core impleme --- **Instructions** -Pleaae include a detailed Title above. Next, please complete the following sections below: +Please include a detailed Title above. Next, please complete the following sections below: * Bug Description * Steps To Reproduce * Expected Behavior * Screenshots (optional) * witness_node --version (optional) -* Aditional Context (optional) +* Additional Context (optional) -Finally, press the 'submit issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. +Finally, press the 'Submit new issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. **Bug Description** A clear and concise description of what the bug is. **Steps To Reproduce** -Steps to reproduce the behavior: +Steps to reproduce the behavior (example outlined below): 1. Execute API call '...' 2. Using JSON payload '...' 3. Received response '...' -4. See error +4. See error in screenshot **Expected Behavior** A clear and concise description of what you expected to happen. @@ -31,19 +31,17 @@ A clear and concise description of what you expected to happen. **Screenshots (optional)** If applicable, add screenshots to help explain process flow and behavior. -**witness_node --version (optional):** +**CLI output for `witness_node --version` (optional):** - Version: [e.g. 2.0.180425] - - SHA: [e.g. bf263b1db47a91ea9748118df1abb5a22cafc40b] - SSL: [e.g. OpenSSL 1.0.2g 1 Mar 2016] - Boost: [e.g. 1.62] - - Websocket++: [e.g. 0.7.0] - Host OS: [e.g. Ubuntu 17.10] **Additional Context (optional)** Add any other context about the problem here. ## CORE TEAM TASK LIST -- [ ] Evaluate / Prioritize Bug Request +- [ ] Evaluate / Prioritize Bug Report - [ ] Refine User Stories / Requirements - [ ] Define Test Cases - [ ] Design / Develop Solution diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 962fcffb23..de6e5b661c 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -5,12 +5,12 @@ about: Suggest an idea for the BitShares Core Team to evaluate and prioritize fo --- **Instructions** -Pleaae include a detailed Title above. Next, please complete the following sections below: +Please include a detailed Title above. Next, please complete the following sections below: * User Story * Impacts -* Aditional Context (optional) +* Additional Context (optional) -Finally, press the 'submit issue' button. The Core Team will evaluate and prioritize your Feature Request for future development. +Finally, press the 'Submit new issue' button. The Core Team will evaluate and prioritize your Feature Request for future development. **User Story** Please tell us about your feature request using the User Story format: @@ -23,11 +23,13 @@ Describe which portion(s) of BitShares Core may be impacted by your request. Ple - [ ] API (the application programming interface) - [ ] Build (the build process or something prior to compiled code) - [ ] CLI (the command line wallet) -- [ ] Consensus (the validation or commitment of data) - [ ] Deployment (the deployment process after building such as Docker, Travis, etc.) +- [ ] DEX (the Decentralized EXchange, market engine, etc.) - [ ] P2P (the peer-to-peer network for transaction/block propagation) -- [ ] Performance (system or user efficiency, performance, etc.) -- [ ] Security (system or user security) +- [ ] Performance (system or user efficiency, etc.) +- [ ] Protocol (the blockchain logic, consensus, validation, etc.) +- [ ] Security (the security of system or user data, etc.) +- [ ] UX (the User Experience) - [ ] Other (please add below) **Additional Context (optional)** From 0ac87af0dd9152fa990fa6b3041c37d49a262075 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 6 Jun 2018 10:48:58 -0400 Subject: [PATCH 08/46] Add: Build Error --- .github/ISSUE_TEMPLATE/bug_report.md | 31 +++++++++++++++----- .github/ISSUE_TEMPLATE/build_error.md | 42 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/build_error.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 040adcb44b..3b76358a4e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,10 +7,11 @@ about: Create a detailed report about a deficiency in the BitShares Core impleme **Instructions** Please include a detailed Title above. Next, please complete the following sections below: * Bug Description +* Impacts * Steps To Reproduce * Expected Behavior * Screenshots (optional) -* witness_node --version (optional) +* Host Environment (optional) * Additional Context (optional) Finally, press the 'Submit new issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. @@ -18,6 +19,20 @@ Finally, press the 'Submit new issue' button. The Core Team will evaluate and pr **Bug Description** A clear and concise description of what the bug is. +**Impacts** +Describe which portion(s) of BitShares Core may be impacted by this bug. Please tick at least one box. +- [ ] API (the application programming interface) +- [ ] Build (the build process or something prior to compiled code) +- [ ] CLI (the command line wallet) +- [ ] Deployment (the deployment process after building such as Docker, Travis, etc.) +- [ ] DEX (the Decentralized EXchange, market engine, etc.) +- [ ] P2P (the peer-to-peer network for transaction/block propagation) +- [ ] Performance (system or user efficiency, etc.) +- [ ] Protocol (the blockchain logic, consensus, validation, etc.) +- [ ] Security (the security of system or user data, etc.) +- [ ] UX (the User Experience) +- [ ] Other (please add below) + **Steps To Reproduce** Steps to reproduce the behavior (example outlined below): 1. Execute API call '...' @@ -31,12 +46,14 @@ A clear and concise description of what you expected to happen. **Screenshots (optional)** If applicable, add screenshots to help explain process flow and behavior. -**CLI output for `witness_node --version` (optional):** - - Version: [e.g. 2.0.180425] - - SSL: [e.g. OpenSSL 1.0.2g 1 Mar 2016] - - Boost: [e.g. 1.62] - - Host OS: [e.g. Ubuntu 17.10] - +**Host Environment** +Please provide details about the host environment. Much of this information can be found running: `witness_node --version`. + - Host OS: [e.g. Ubuntu 18.04 LTS] + - Host Physical RAM [e.g. 4GB] + - BitShares Version: [e.g. 2.0.180425] + - OpenSSL Version: [e.g. 1.1.0g] + - Boost Version: [e.g. 1.65.1] + **Additional Context (optional)** Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/build_error.md b/.github/ISSUE_TEMPLATE/build_error.md new file mode 100644 index 0000000000..2769078685 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/build_error.md @@ -0,0 +1,42 @@ +--- +name: Build Error +about: Create a detailed report about an error during in the BitShares Core build process. + +--- + +**Instructions** +Please include a detailed Title above. Next, please complete the following sections below: +* Build Error +* Build Environment +* Steps To Reproduce +* Screenshots (optional) + +Finally, press the 'Submit new issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. + +**Build Error Description** +A clear and concise description of what the build error is. + +**Build Environment** +Please provide details about the build environment, including the relevant required libraries. Much of this information can be found in the `CMakeFiles/CMakeOutput.log`. + - Host OS: [e.g. Ubuntu 18.04 LTS] + - Host Physical RAM [e.g. 4GB] + - Source Branch/Tag: [e.g. master or 2.0.180425] + - OpenSSL Version: [e.g. 1.1.0g] + - Boost Version: [e.g. 1.65.1] + - C++ Compiler: [e.g. gcc version 4.8.5] + +**Steps To Reproduce** +Steps to reproduce the behavior (example outlined below): +1. Using installation guide from this URL... +2. This is my complete build script... +3. It fails at this step... +4. Here is the error output... +5. See the error in the screenshot below... + +**Screenshots (optional)** +If applicable, add screenshots to demonstrate the build process and error output. + +## CORE TEAM TASK LIST +- [ ] Evaluate `Build Error` +- [ ] Provide build guidance +- [ ] Create `Bug Report` From e591d639cd46aca9b03d3d40bacb6faa0fd82f60 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 6 Jun 2018 13:08:54 -0400 Subject: [PATCH 09/46] Add: Build Error Template --- .github/ISSUE_TEMPLATE/build_error.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/build_error.md b/.github/ISSUE_TEMPLATE/build_error.md index 2769078685..511766a46d 100644 --- a/.github/ISSUE_TEMPLATE/build_error.md +++ b/.github/ISSUE_TEMPLATE/build_error.md @@ -9,7 +9,7 @@ Please include a detailed Title above. Next, please complete the following secti * Build Error * Build Environment * Steps To Reproduce -* Screenshots (optional) +* Console Logs (optional) Finally, press the 'Submit new issue' button. The Core Team will evaluate and prioritize your Bug Report for future development. @@ -33,8 +33,8 @@ Steps to reproduce the behavior (example outlined below): 4. Here is the error output... 5. See the error in the screenshot below... -**Screenshots (optional)** -If applicable, add screenshots to demonstrate the build process and error output. +**Console Logs (optional)** +Please provide the full console log, including all commands entered and their output. This will allow detailed troubleshooting. ## CORE TEAM TASK LIST - [ ] Evaluate `Build Error` From 6911192102733d669250fd045d990ec847bd9603 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 6 Jun 2018 13:58:19 -0400 Subject: [PATCH 10/46] Add: Build Error Template --- .github/ISSUE_TEMPLATE/build_error.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/build_error.md b/.github/ISSUE_TEMPLATE/build_error.md index 511766a46d..403f5e8af8 100644 --- a/.github/ISSUE_TEMPLATE/build_error.md +++ b/.github/ISSUE_TEMPLATE/build_error.md @@ -29,9 +29,8 @@ Please provide details about the build environment, including the relevant requi Steps to reproduce the behavior (example outlined below): 1. Using installation guide from this URL... 2. This is my complete build script... -3. It fails at this step... -4. Here is the error output... -5. See the error in the screenshot below... +3. It fails at this step with the following output... +4. See the error in the console log below... **Console Logs (optional)** Please provide the full console log, including all commands entered and their output. This will allow detailed troubleshooting. From 1d1db0b1a85669d5abaae7f8d52c6a2a685a1fe1 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Wed, 6 Jun 2018 15:21:47 -0400 Subject: [PATCH 11/46] Add: Build Error Template --- .github/ISSUE_TEMPLATE/build_error.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/build_error.md b/.github/ISSUE_TEMPLATE/build_error.md index 403f5e8af8..20be1bef17 100644 --- a/.github/ISSUE_TEMPLATE/build_error.md +++ b/.github/ISSUE_TEMPLATE/build_error.md @@ -1,6 +1,6 @@ --- name: Build Error -about: Create a detailed report about an error during in the BitShares Core build process. +about: Create a detailed report about an error encountered during the BitShares Core build process. --- @@ -17,7 +17,7 @@ Finally, press the 'Submit new issue' button. The Core Team will evaluate and pr A clear and concise description of what the build error is. **Build Environment** -Please provide details about the build environment, including the relevant required libraries. Much of this information can be found in the `CMakeFiles/CMakeOutput.log`. +Details about the build environment, including the relevant required libraries. Much of this information can be found in the `CMakeFiles/CMakeOutput.log`. - Host OS: [e.g. Ubuntu 18.04 LTS] - Host Physical RAM [e.g. 4GB] - Source Branch/Tag: [e.g. master or 2.0.180425] From 1297e1b9d3afd01071d9fcaebf5ee3e8c4a8ae0d Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Tue, 12 Jun 2018 09:59:58 -0700 Subject: [PATCH 12/46] update doxyfile and readme --- Doxyfile | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doxyfile b/Doxyfile index c0af095a2e..f5bc590bd3 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,13 +38,13 @@ PROJECT_NAME = "Bitshares-Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "2.0.180425" +PROJECT_NUMBER = "2.0.180612" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = "Your share in the Decentralized Exchange" +PROJECT_BRIEF = "The underlying software running the BitShares Blockchain" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 diff --git a/README.md b/README.md index d7e7a700af..c271550aeb 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ To build after all dependencies are installed: **NOTE:** BitShares requires an [OpenSSL](https://www.openssl.org/) version in the 1.0.x series. OpenSSL 1.1.0 and newer are NOT supported. If your system OpenSSL version is newer, then you will need to manually provide an older version of OpenSSL and specify it to CMake using `-DOPENSSL_INCLUDE_DIR`, `-DOPENSSL_SSL_LIBRARY`, and `-DOPENSSL_CRYPTO_LIBRARY`. -**NOTE:** BitShares requires a [Boost](http://www.boost.org/) version in the range [1.57, 1.63]. Versions earlier than +**NOTE:** BitShares requires a [Boost](http://www.boost.org/) version in the range [1.57, 1.65]. Versions earlier than 1.57 or newer than 1.63 are NOT supported. If your system Boost version is newer, then you will need to manually build an older version of Boost and specify it to CMake using `DBOOST_ROOT`. From c9a3c525415dcfe379e79e86956974bc17075d78 Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Tue, 12 Jun 2018 11:28:23 -0700 Subject: [PATCH 13/46] update max boost in sentence --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c271550aeb..a1e630a972 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ To build after all dependencies are installed: **NOTE:** BitShares requires an [OpenSSL](https://www.openssl.org/) version in the 1.0.x series. OpenSSL 1.1.0 and newer are NOT supported. If your system OpenSSL version is newer, then you will need to manually provide an older version of OpenSSL and specify it to CMake using `-DOPENSSL_INCLUDE_DIR`, `-DOPENSSL_SSL_LIBRARY`, and `-DOPENSSL_CRYPTO_LIBRARY`. **NOTE:** BitShares requires a [Boost](http://www.boost.org/) version in the range [1.57, 1.65]. Versions earlier than -1.57 or newer than 1.63 are NOT supported. If your system Boost version is newer, then you will need to manually build +1.57 or newer than 1.65 are NOT supported. If your system Boost version is newer, then you will need to manually build an older version of Boost and specify it to CMake using `DBOOST_ROOT`. After building, the witness node can be launched with: From bce877770f70952deb1e703fa29eab86efc07cef Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 15 Jun 2018 16:08:28 -0300 Subject: [PATCH 14/46] sync project description in doxyfile for https://github.com/bitshares/bitshares-core/pull/1062 --- Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index f5bc590bd3..469a3705cf 100644 --- a/Doxyfile +++ b/Doxyfile @@ -44,7 +44,7 @@ PROJECT_NUMBER = "2.0.180612" # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = "The underlying software running the BitShares Blockchain" +PROJECT_BRIEF = "BitShares blockchain implementation and command-line interface software" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 From 859262acde1bcc0b3f982a10246f4e6a6448768f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 2 Jul 2018 16:29:47 -0300 Subject: [PATCH 15/46] Update node requirements for #1107 --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ff894659ba..6641c2e4f4 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,12 @@ the blockchain. After syncing, you can exit the node using Ctrl+C and setup the rpc-endpoint = 127.0.0.1:8090 -**NOTE:** By default the witness node will start in reduced memory ram mode by using some of the commands detailed in [Memory reduction for nodes](https://github.com/bitshares/bitshares-core/wiki/Memory-reduction-for-nodes). -In order to run a full node with all the account history you need to remove `partial-operations` and `max-ops-per-account` from your config file. Please note that currently(2017-12-23) a full node need 54GB of RAM to operate and required memory is growing fast. +**IMPORTANT:** By default the witness node will start in reduced memory ram mode by using some of the commands detailed in [Memory reduction for nodes](https://github.com/bitshares/bitshares-core/wiki/Memory-reduction-for-nodes). +In order to run a full node with all the account history you need to remove `partial-operations` and `max-ops-per-account` from your config file. Please note that currently(2018-07-02) a full node need more than 100GB of RAM to operate and required memory is growing fast. Consider the following table before running a node: + +| Default | Full | Minimal | ElasticSearch +| --- | --- | --- | --- +| 20G RAM | 120G RAM | 4G RAM | 200G SSD HD, 16G RAM After starting the witness node again, in a separate terminal you can run: From 76c808a887e95724e9bd4a6b7afa568a109fa791 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 2 Jul 2018 16:46:38 -0300 Subject: [PATCH 16/46] fix syntax error --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6641c2e4f4..cb20c2d76d 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ the blockchain. After syncing, you can exit the node using Ctrl+C and setup the rpc-endpoint = 127.0.0.1:8090 **IMPORTANT:** By default the witness node will start in reduced memory ram mode by using some of the commands detailed in [Memory reduction for nodes](https://github.com/bitshares/bitshares-core/wiki/Memory-reduction-for-nodes). -In order to run a full node with all the account history you need to remove `partial-operations` and `max-ops-per-account` from your config file. Please note that currently(2018-07-02) a full node need more than 100GB of RAM to operate and required memory is growing fast. Consider the following table before running a node: +In order to run a full node with all the account history you need to remove `partial-operations` and `max-ops-per-account` from your config file. Please note that currently(2018-07-02) a full node will need more than 100GB of RAM to operate and required memory is growing fast. Consider the following table before running a node: | Default | Full | Minimal | ElasticSearch | --- | --- | --- | --- From cabb3c9738ff1c451f0aa17a68863e96595099bb Mon Sep 17 00:00:00 2001 From: ryanRfox Date: Tue, 19 Jun 2018 15:29:47 -0400 Subject: [PATCH 17/46] Add: Require 64-bit OS --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cb20c2d76d..b80ce3ae17 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Getting Started Build instructions and additional documentation are available in the [wiki](https://github.com/bitshares/bitshares-core/wiki). -We recommend building on Ubuntu 16.04 LTS, and the build dependencies may be installed with: +We recommend building on Ubuntu 16.04 LTS (64-bit), and the build dependencies may be installed with: sudo apt-get update sudo apt-get install autoconf cmake make automake libtool git libboost-all-dev libssl-dev g++ libcurl4-openssl-dev @@ -55,6 +55,8 @@ To build after all dependencies are installed: 1.57 or newer than 1.63 are NOT supported. If your system Boost version is newer, then you will need to manually build an older version of Boost and specify it to CMake using `DBOOST_ROOT`. +**NOTE:** BitShares requires a 64-bit operating system to build, and will not build on a 32-bit OS. + After building, the witness node can be launched with: ./programs/witness_node/witness_node From 716387b9b4851b358c6a75f0706c87ed142a92f3 Mon Sep 17 00:00:00 2001 From: Anzhy Cherrnyavski Date: Mon, 25 Feb 2019 09:28:59 +0300 Subject: [PATCH 18/46] Fixed saving keys in wallet --- libraries/wallet/wallet.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 096d075f65..ae3214b951 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -265,9 +265,10 @@ class wallet_api_impl private: void claim_registered_account(const account_object& account) { + std::vector import_keys; auto it = _wallet.pending_account_registrations.find( account.name ); FC_ASSERT( it != _wallet.pending_account_registrations.end() ); - for (const std::string& wif_key : it->second) + for (const std::string& wif_key : it->second) { if( !import_key( account.name, wif_key ) ) { // somebody else beat our pending registration, there is @@ -280,8 +281,22 @@ class wallet_api_impl // possibility of migrating to a fork where the // name is available, the user can always // manually re-register) + } else { + import_keys.push_back( wif_key ); } + } _wallet.pending_account_registrations.erase( it ); + + for( const auto& k : import_keys ) { + fc::optional optional_private_key = wif_to_key( k ); + if (!optional_private_key) + FC_THROW("Invalid private key"); + string shorthash = detail::address_to_shorthash(optional_private_key->get_public_key()); + copy_wallet_file( "before-import-key-" + shorthash ); + + save_wallet_file(); + copy_wallet_file( "after-import-key-" + shorthash ); + } } // after a witness registration succeeds, this saves the private key in the wallet permanently From c77c0699488f5923258d9e0d92256239e652ab53 Mon Sep 17 00:00:00 2001 From: Anzhy Cherrnyavski Date: Wed, 27 Feb 2019 10:03:16 +0300 Subject: [PATCH 19/46] Added test. --- libraries/fc | 2 +- libraries/wallet/wallet.cpp | 13 +++--------- tests/cli/main.cpp | 40 +++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/libraries/fc b/libraries/fc index 1a411b813a..8ebd99b786 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 1a411b813a61f12587234d1bb5f32ad1a95883a9 +Subproject commit 8ebd99b786623bc8d55e89d42df82644a71a6885 diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index ae3214b951..8d579e1100 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -265,7 +265,7 @@ class wallet_api_impl private: void claim_registered_account(const account_object& account) { - std::vector import_keys; + bool import_keys = false; auto it = _wallet.pending_account_registrations.find( account.name ); FC_ASSERT( it != _wallet.pending_account_registrations.end() ); for (const std::string& wif_key : it->second) { @@ -282,20 +282,13 @@ class wallet_api_impl // name is available, the user can always // manually re-register) } else { - import_keys.push_back( wif_key ); + import_keys = true; } } _wallet.pending_account_registrations.erase( it ); - for( const auto& k : import_keys ) { - fc::optional optional_private_key = wif_to_key( k ); - if (!optional_private_key) - FC_THROW("Invalid private key"); - string shorthash = detail::address_to_shorthash(optional_private_key->get_public_key()); - copy_wallet_file( "before-import-key-" + shorthash ); - + if( import_keys ) { save_wallet_file(); - copy_wallet_file( "after-import-key-" + shorthash ); } } diff --git a/tests/cli/main.cpp b/tests/cli/main.cpp index 102300f835..f4e19ba784 100644 --- a/tests/cli/main.cpp +++ b/tests/cli/main.cpp @@ -37,6 +37,8 @@ #include #include +#include + #ifdef _WIN32 #ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 @@ -580,3 +582,41 @@ BOOST_FIXTURE_TEST_CASE( account_history_pagination, cli_fixture ) throw; } } + +graphene::wallet::plain_keys decrypt_keys( const std::string& password, const vector& cipher_keys ) +{ + auto pw = fc::sha512::hash( password.c_str(), password.size() ); + vector decrypted = fc::aes_decrypt( pw, cipher_keys ); + return fc::raw::unpack( decrypted ); +} + +BOOST_AUTO_TEST_CASE( saving_keys_wallet_test ) { + cli_fixture cli; + + cli.con.wallet_api_ptr->import_balance( "nathan", cli.nathan_keys, true ); + cli.con.wallet_api_ptr->upgrade_account( "nathan", true ); + std::string brain_key( "FICTIVE WEARY MINIBUS LENS HAWKIE MAIDISH MINTY GLYPH GYTE KNOT COCKSHY LENTIGO PROPS BIFORM KHUTBAH BRAZIL" ); + cli.con.wallet_api_ptr->create_account_with_brain_key( brain_key, "account1", "nathan", "nathan", true ); + + BOOST_CHECK_NO_THROW( cli.con.wallet_api_ptr->transfer( "nathan", "account1", "9000", "1.3.0", "", true ) ); + + std::string path( cli.app_dir.path().generic_string() + "/wallet.json" ); + graphene::wallet::wallet_data wallet = fc::json::from_file( path ).as( 2 * GRAPHENE_MAX_NESTED_OBJECTS ); + BOOST_CHECK( wallet.extra_keys.size() == 1 ); // nathan + BOOST_CHECK( wallet.pending_account_registrations.size() == 1 ); // account1 + BOOST_CHECK( wallet.pending_account_registrations["account1"].size() == 2 ); // account1 active key + account1 memo key + + graphene::wallet::plain_keys pk = decrypt_keys( "supersecret", wallet.cipher_keys ); + BOOST_CHECK( pk.keys.size() == 1 ); // nathan key + + BOOST_CHECK( generate_block( cli.app1 ) ); + fc::usleep( fc::seconds(1) ); + + wallet = fc::json::from_file( path ).as( 2 * GRAPHENE_MAX_NESTED_OBJECTS ); + BOOST_CHECK( wallet.extra_keys.size() == 2 ); // nathan + account1 + BOOST_CHECK( wallet.pending_account_registrations.empty() ); + BOOST_CHECK_NO_THROW( cli.con.wallet_api_ptr->transfer( "account1", "nathan", "1000", "1.3.0", "", true ) ); + + pk = decrypt_keys( "supersecret", wallet.cipher_keys ); + BOOST_CHECK( pk.keys.size() == 3 ); // nathan key + account1 active key + account1 memo key +} From e6ecf100ae623f4180b3d9503fed512b2de3861a Mon Sep 17 00:00:00 2001 From: Anzhy Cherrnyavski Date: Thu, 28 Feb 2019 09:46:30 +0300 Subject: [PATCH 20/46] fc commit changed --- libraries/fc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fc b/libraries/fc index 8ebd99b786..1a411b813a 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 8ebd99b786623bc8d55e89d42df82644a71a6885 +Subproject commit 1a411b813a61f12587234d1bb5f32ad1a95883a9 From 05ceb7c2b0792168e88b2b7781aa39536ad98fc3 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 5 Mar 2019 06:13:28 -0500 Subject: [PATCH 21/46] Prevent Boost/WinSock.h multiple inclusion --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bef600f32..339381b9ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,7 @@ if( WIN32 ) set(CRYPTO_LIB) if( MSVC ) + add_definitions(-DWIN32_LEAN_AND_MEAN) #looks like this flag can have different default on some machines. SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SAFESEH:NO") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO") From 37f9aaafff98aa5eb9dd24da64d140529d1cdaad Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 5 Mar 2019 06:14:55 -0500 Subject: [PATCH 22/46] Fix uin64_t / safe comparision --- libraries/chain/db_maint.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index 51e5596de4..a0ddba36f8 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -311,7 +311,7 @@ void database::update_active_committee_members() if( stake_target > 0 ) { while( (committee_member_count < _committee_count_histogram_buffer.size() - 1) - && (stake_tally <= stake_target) ) + && (stake_tally <= stake_target.value) ) { stake_tally += _committee_count_histogram_buffer[++committee_member_count]; } From bb2a757889c543e7a91948bb60dc16130cba999d Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 13 Feb 2019 10:06:06 -0300 Subject: [PATCH 23/46] update full node specs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e5147f0112..43c9daa823 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ In order to run a full node with all the account history you need to remove `par | Default | Full | Minimal | ElasticSearch | --- | --- | --- | --- -| 100G HDD, 16G RAM | 200G HDD, 160G RAM | 80G HDD, 4G RAM | 500G SSD, 32G RAM +| 100G HDD, 16G RAM | 640 SSD, 64G RAM | 80G HDD, 4G RAM | 500G SSD, 32G RAM After starting the witness node again, in a separate terminal you can run: From 00f9c903b69591960aa3dad1993cd5cb9449bd77 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 13 Feb 2019 16:14:11 -0300 Subject: [PATCH 24/46] add note to clarify new full node requirements --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 43c9daa823..3387ccf444 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,9 @@ In order to run a full node with all the account history you need to remove `par | Default | Full | Minimal | ElasticSearch | --- | --- | --- | --- -| 100G HDD, 16G RAM | 640 SSD, 64G RAM | 80G HDD, 4G RAM | 500G SSD, 32G RAM +| 100G HDD, 16G RAM | 640G SSD, 64G RAM * | 80G HDD, 4G RAM | 500G SSD, 32G RAM + +\* For this setup to work allocate all SSD space left(excluding OS and software) as Swap. After starting the witness node again, in a separate terminal you can run: From 93050ba44714af7c02a81257974025d41a5ef6e4 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 14 Feb 2019 15:58:46 -0300 Subject: [PATCH 25/46] update full nodes setup notes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3387ccf444..a5c719aaa0 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ In order to run a full node with all the account history you need to remove `par | --- | --- | --- | --- | 100G HDD, 16G RAM | 640G SSD, 64G RAM * | 80G HDD, 4G RAM | 500G SSD, 32G RAM -\* For this setup to work allocate all SSD space left(excluding OS and software) as Swap. +\* For this setup, allocate at least 500GB of SSD as swap. After starting the witness node again, in a separate terminal you can run: From 38746ff441ae8461cc1343fa5ae3122c6f307427 Mon Sep 17 00:00:00 2001 From: John Jones Date: Wed, 27 Feb 2019 09:56:34 -0500 Subject: [PATCH 26/46] reject oversized transactions --- libraries/chain/db_block.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index edffd102e0..e340d2dea1 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -223,6 +223,8 @@ bool database::_push_block(const signed_block& new_block) */ processed_transaction database::push_transaction( const precomputable_transaction& trx, uint32_t skip ) { try { + // see https://github.com/bitshares/bitshares-core/issues/1573 + FC_ASSERT( fc::raw::pack_size( trx ) < (1024 * 1024), "Transaction exceeds maximum transaction size." ); processed_transaction result; detail::with_skip_flags( *this, skip, [&]() { From b9a2284c5d569c2bb1a712d792e82acac0f079b6 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 27 Feb 2019 11:45:45 -0300 Subject: [PATCH 27/46] initialize percent_change in market_ticker constructor --- libraries/app/database_api.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index c97db8273b..b454e9b15f 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -387,6 +387,7 @@ market_ticker::market_ticker(const market_ticker_object& mto, time = now; base = asset_base.symbol; quote = asset_quote.symbol; + percent_change = "0"; fc::uint128 bv; fc::uint128 qv; price latest_price = asset( mto.latest_base, mto.base ) / asset( mto.latest_quote, mto.quote ); From 6b8f15768e7899b721a8ff64d7e1ecbe824d5531 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 27 Feb 2019 20:31:38 -0300 Subject: [PATCH 28/46] add a second contructor for empty ticker --- libraries/app/database_api.cpp | 16 +++++++++++++++- .../app/include/graphene/app/database_api.hpp | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index b454e9b15f..aab507bc71 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -420,6 +420,20 @@ market_ticker::market_ticker(const market_ticker_object& mto, if(!orders.bids.empty()) highest_bid = orders.bids[0].price; } +market_ticker::market_ticker(const fc::time_point_sec& now, + const asset_object& asset_base, + const asset_object& asset_quote) +{ + time = now; + base = base; + quote = quote; + latest = "0"; + lowest_ask = "0"; + highest_bid = "0"; + percent_change = "0"; + base_volume = "0"; + quote_volume = "0"; +} ////////////////////////////////////////////////////////////////////// // // @@ -1434,7 +1448,7 @@ market_ticker database_api_impl::get_ticker( const string& base, const string& q return market_ticker(*itr, now, *assets[0], *assets[1], orders); } // if no ticker is found for this market we return an empty ticker - market_ticker empty_result; + market_ticker empty_result(now, *assets[0], *assets[1]); return empty_result; } diff --git a/libraries/app/include/graphene/app/database_api.hpp b/libraries/app/include/graphene/app/database_api.hpp index b2ce640460..fe97083956 100644 --- a/libraries/app/include/graphene/app/database_api.hpp +++ b/libraries/app/include/graphene/app/database_api.hpp @@ -97,6 +97,9 @@ struct market_ticker const asset_object& asset_base, const asset_object& asset_quote, const order_book& orders); + market_ticker(const fc::time_point_sec& now, + const asset_object& asset_base, + const asset_object& asset_quote); }; struct market_volume From 1f1155ac78ff7db41993b130b2b30c482eb504a3 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 27 Feb 2019 21:46:20 -0300 Subject: [PATCH 29/46] fix asset symbols in empty ticker contructor --- libraries/app/database_api.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index aab507bc71..794195cdc0 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -425,8 +425,8 @@ market_ticker::market_ticker(const fc::time_point_sec& now, const asset_object& asset_quote) { time = now; - base = base; - quote = quote; + base = asset_base.symbol; + quote = asset_quote.symbol; latest = "0"; lowest_ask = "0"; highest_bid = "0"; From 5c4b6fc4361c7b850129b21d59ddd14f1f969d6c Mon Sep 17 00:00:00 2001 From: Alfredo Date: Thu, 28 Feb 2019 12:14:42 -0300 Subject: [PATCH 30/46] initialize lowest_ask and highest_bid --- libraries/app/database_api.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 794195cdc0..2eafd284de 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -388,6 +388,9 @@ market_ticker::market_ticker(const market_ticker_object& mto, base = asset_base.symbol; quote = asset_quote.symbol; percent_change = "0"; + lowest_ask = "0"; + highest_bid = "0"; + fc::uint128 bv; fc::uint128 qv; price latest_price = asset( mto.latest_base, mto.base ) / asset( mto.latest_quote, mto.quote ); From 51814596358eaa73cb737f228596693849f47109 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 4 Mar 2019 16:45:07 -0300 Subject: [PATCH 31/46] bump database to last release date --- libraries/chain/include/graphene/chain/config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index e97bacc032..fdcd3312f8 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -121,7 +121,7 @@ #define GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT 4 #define GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT 3 -#define GRAPHENE_CURRENT_DB_VERSION "BTS2.181221" +#define GRAPHENE_CURRENT_DB_VERSION "20190219" #define GRAPHENE_IRREVERSIBLE_THRESHOLD (70 * GRAPHENE_1_PERCENT) From 511076c04849b6952fb821655defcc72367de1f2 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 1 Mar 2019 18:15:29 +0100 Subject: [PATCH 32/46] Removed initial_chain_id from example genesis file to avoid confusion --- libraries/egenesis/genesis-dev.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/egenesis/genesis-dev.json b/libraries/egenesis/genesis-dev.json index f400055244..7d6f03a46c 100644 --- a/libraries/egenesis/genesis-dev.json +++ b/libraries/egenesis/genesis-dev.json @@ -370,11 +370,10 @@ } ], "initial_worker_candidates": [], - "initial_chain_id": "aa34045518f1469a28fa4578240d5f039afa9959c0b95ce3b39674efa691fb21", "immutable_parameters": { "min_committee_member_count": 11, "min_witness_count": 11, "num_special_accounts": 0, "num_special_assets": 0 } -} \ No newline at end of file +} From d80e1d9c09866c51d5f4bfc541083127346d8888 Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 9 Mar 2019 09:52:24 -0500 Subject: [PATCH 33/46] Add plugin section headers to generated config.ini --- libraries/app/config_util.cpp | 8 ++++++++ libraries/app/include/graphene/app/application.hpp | 11 +++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index 6bfe946bd7..fcef822821 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -265,6 +265,13 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa if( !od->description().empty() ) out_cfg << "# " << od->description() << "\n"; + + if( od->long_name().find("plugin-cfg-header-") == 0 ) + { + out_cfg << "\n"; + continue; + } + boost::any store; if( !od->semantic()->apply_default(store) ) out_cfg << "# " << od->long_name() << " = \n"; @@ -284,6 +291,7 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa } out_cfg << "\n" + << "# =============== logging options ===============\n" << "# Logging configuration is loaded from logging.ini by default.\n" << "# If logging.ini exists, logging configuration added in this file will be ignored.\n"; out_cfg.close(); diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index 66a73f3999..c62c510fe4 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -62,13 +62,20 @@ namespace graphene { namespace app { auto plug = std::make_shared(); plug->plugin_set_app(this); - boost::program_options::options_description plugin_cli_options(plug->plugin_name() + " plugin. " + plug->plugin_description() + "\nOptions"), plugin_cfg_options; - //boost::program_options::options_description plugin_cli_options("Options for plugin " + plug->plugin_name()), plugin_cfg_options; + string cli_plugin_desc = plug->plugin_name() + " plugin. " + plug->plugin_description() + "\nOptions"; + boost::program_options::options_description plugin_cli_options( cli_plugin_desc ), plugin_cfg_options; plug->plugin_set_program_options(plugin_cli_options, plugin_cfg_options); + if( !plugin_cli_options.options().empty() ) _cli_options.add(plugin_cli_options); + if( !plugin_cfg_options.options().empty() ) + { + std::string header_name = "plugin-cfg-header-" + plug->plugin_name(); + std::string header_desc = "=============== " + plug->plugin_name() + " plugin options ==============="; + _cfg_options.add_options()(header_name.c_str(), header_desc.c_str()); _cfg_options.add(plugin_cfg_options); + } add_available_plugin( plug ); From d4a5b40bf60c2b1cc85d6ab80b8af3475caa47ae Mon Sep 17 00:00:00 2001 From: abitmore Date: Sat, 9 Mar 2019 12:17:26 -0500 Subject: [PATCH 34/46] Highlight plugin section headers in config.ini --- libraries/app/config_util.cpp | 17 ++++++++++++----- .../app/include/graphene/app/application.hpp | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/libraries/app/config_util.cpp b/libraries/app/config_util.cpp index fcef822821..f06291b788 100644 --- a/libraries/app/config_util.cpp +++ b/libraries/app/config_util.cpp @@ -258,20 +258,24 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa }; deduplicator dedup(modify_option_defaults); std::ofstream out_cfg(config_ini_path.preferred_string()); + std::string plugin_header_surrounding( 78, '=' ); for( const boost::shared_ptr opt : cfg_options.options() ) { const boost::shared_ptr od = dedup.next(opt); if( !od ) continue; - if( !od->description().empty() ) - out_cfg << "# " << od->description() << "\n"; - - if( od->long_name().find("plugin-cfg-header-") == 0 ) + if( od->long_name().find("plugin-cfg-header-") == 0 ) // it's a plugin header { + out_cfg << "\n"; + out_cfg << "# " << plugin_header_surrounding << "\n"; + out_cfg << "# " << od->description() << "\n"; + out_cfg << "# " << plugin_header_surrounding << "\n"; out_cfg << "\n"; continue; } + if( !od->description().empty() ) + out_cfg << "# " << od->description() << "\n"; boost::any store; if( !od->semantic()->apply_default(store) ) out_cfg << "# " << od->long_name() << " = \n"; @@ -291,7 +295,10 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa } out_cfg << "\n" - << "# =============== logging options ===============\n" + << "# " << plugin_header_surrounding << "\n" + << "# logging options\n" + << "# " << plugin_header_surrounding << "\n" + << "#\n" << "# Logging configuration is loaded from logging.ini by default.\n" << "# If logging.ini exists, logging configuration added in this file will be ignored.\n"; out_cfg.close(); diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index c62c510fe4..598974f7ad 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -72,7 +72,7 @@ namespace graphene { namespace app { if( !plugin_cfg_options.options().empty() ) { std::string header_name = "plugin-cfg-header-" + plug->plugin_name(); - std::string header_desc = "=============== " + plug->plugin_name() + " plugin options ==============="; + std::string header_desc = plug->plugin_name() + " plugin options"; _cfg_options.add_options()(header_name.c_str(), header_desc.c_str()); _cfg_options.add(plugin_cfg_options); } From 81933c550b4dca47e1c44c0f92c41785f6cad35d Mon Sep 17 00:00:00 2001 From: dimfred Date: Sun, 17 Mar 2019 23:57:36 +0100 Subject: [PATCH 35/46] added new signals to database, calling signals from maintenance --- libraries/chain/db_maint.cpp | 5 +++++ libraries/chain/include/graphene/chain/database.hpp | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index a0ddba36f8..a5066483b4 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -44,6 +44,7 @@ #include #include #include +#include namespace graphene { namespace chain { @@ -1081,6 +1082,8 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g distribute_fba_balances(*this); create_buyback_orders(*this); + on_maintenance_begin( next_block.id() ); + struct vote_tally_helper { database& d; const global_property_object& props; @@ -1110,6 +1113,8 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g + (stake_account.cashback_vb.valid() ? (*stake_account.cashback_vb)(d).balance.amount.value: 0) + stats.core_in_balance.value; + d.on_voting_stake_calculated( stake_account, opinion_account, voting_stake ); + for( vote_id_type id : opinion_account.options.votes ) { uint32_t offset = id.instance(); diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 7fa190cb66..6ab090c96a 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -204,6 +204,16 @@ namespace graphene { namespace chain { */ fc::signal&, const vector&, const flat_set&)> removed_objects; + /** + * Emitted after the calculation of the voting_stake (occurs in the maintenance interval) + */ + fc::signal on_voting_stake_calculated; + + /** + * Emitted after the beginning of the maintenance interval + */ + fc::signal on_maintenance_begin; + //////////////////// db_witness_schedule.cpp //////////////////// /** From fe13e7a8c313d1ca3b1153feb4e0f605f9075b65 Mon Sep 17 00:00:00 2001 From: dimfred Date: Sat, 16 Mar 2019 17:31:16 +0100 Subject: [PATCH 36/46] added voting_statistics_object --- libraries/chain/CMakeLists.txt | 1 + .../include/graphene/chain/protocol/types.hpp | 6 + .../chain/voting_statistics_object.hpp | 103 ++++++++++++++++++ libraries/chain/voting_statistics_object.cpp | 30 +++++ 4 files changed, 140 insertions(+) create mode 100644 libraries/chain/include/graphene/chain/voting_statistics_object.hpp create mode 100644 libraries/chain/voting_statistics_object.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 4e25cd6b39..924b082ac0 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -88,6 +88,7 @@ add_library( graphene_chain market_object.cpp proposal_object.cpp vesting_balance_object.cpp + voting_statistics_object.cpp block_database.cpp diff --git a/libraries/chain/include/graphene/chain/protocol/types.hpp b/libraries/chain/include/graphene/chain/protocol/types.hpp index 4e96abf9f9..a5d25dab16 100644 --- a/libraries/chain/include/graphene/chain/protocol/types.hpp +++ b/libraries/chain/include/graphene/chain/protocol/types.hpp @@ -138,6 +138,7 @@ namespace graphene { namespace chain { vesting_balance_object_type, worker_object_type, balance_object_type, + voting_statistics_object_type, OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types }; @@ -180,6 +181,7 @@ namespace graphene { namespace chain { class worker_object; class balance_object; class blinded_balance_object; + class voting_statistics_object; typedef object_id< protocol_ids, account_object_type, account_object> account_id_type; typedef object_id< protocol_ids, asset_object_type, asset_object> asset_id_type; @@ -195,6 +197,7 @@ namespace graphene { namespace chain { typedef object_id< protocol_ids, vesting_balance_object_type, vesting_balance_object> vesting_balance_id_type; typedef object_id< protocol_ids, worker_object_type, worker_object> worker_id_type; typedef object_id< protocol_ids, balance_object_type, balance_object> balance_id_type; + typedef object_id< protocol_ids, voting_statistics_object_type, voting_statistics_object> voting_statistics_id_type; // implementation types class global_property_object; @@ -351,6 +354,8 @@ FC_REFLECT_ENUM( graphene::chain::object_type, (vesting_balance_object_type) (worker_object_type) (balance_object_type) + (voting_statistics_object_type) + (OBJECT_TYPE_COUNT) ) FC_REFLECT_ENUM( graphene::chain::impl_object_type, @@ -404,6 +409,7 @@ FC_REFLECT_TYPENAME( graphene::chain::special_authority_id_type ) FC_REFLECT_TYPENAME( graphene::chain::buyback_id_type ) FC_REFLECT_TYPENAME( graphene::chain::fba_accumulator_id_type ) FC_REFLECT_TYPENAME( graphene::chain::collateral_bid_id_type ) +FC_REFLECT_TYPENAME( graphene::chain::voting_statistics_id_type ) FC_REFLECT( graphene::chain::void_t, ) diff --git a/libraries/chain/include/graphene/chain/voting_statistics_object.hpp b/libraries/chain/include/graphene/chain/voting_statistics_object.hpp new file mode 100644 index 0000000000..94b3795cf2 --- /dev/null +++ b/libraries/chain/include/graphene/chain/voting_statistics_object.hpp @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 Blockchain Projects B.V. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include + +#include + +#include +#include + +#include + + +namespace graphene { namespace chain { + + /** + * @brief tracks the history of the voting stake for an account + * @ingroup object + * @ingroup implementation + * + * The calculation of the voting stake, performed in the maintenance interval, results + * in the creation or, if present, in the update of a voting_statistics_object. + * + * @note By default these objects are not tracked, the voting_stat_plugin must + * be loaded for these objects to be maintained. + */ + class voting_statistics_object : public abstract_object + { + public: + static const uint8_t space_id = protocol_ids; + static const uint8_t type_id = voting_statistics_object_type; + + voting_statistics_object(){} + + /* the block_id where the maintenance interval was performed */ + static block_id_type block_id; + + /* the owner of the stake */ + account_id_type account; + /* the stake which was generated by this account */ + uint64_t stake = 0; + /* the proxy for this account */ + account_id_type proxy = GRAPHENE_PROXY_TO_SELF_ACCOUNT; + /* the accounts which this account was a proxy for with the proxied stakes */ + flat_map< account_id_type, uint64_t > proxy_for; + /* the vote_id's this account was voting for */ + flat_set votes; + + /* returns the total voting stake this account can vote with */ + const uint64_t get_total_voting_stake() const + { + uint64_t init = has_proxy() ? 0 : stake; + return boost::accumulate( proxy_for | boost::adaptors::map_values, init ); + } + + inline bool has_proxy() const + { + return GRAPHENE_PROXY_TO_SELF_ACCOUNT != proxy; + } + }; + + + struct by_owner{}; + + typedef multi_index_container< + voting_statistics_object, + indexed_by< + ordered_unique< tag, member< object, object_id_type, &object::id > >, + ordered_unique< tag, + member< voting_statistics_object, account_id_type, &voting_statistics_object::account > + > + > + > voting_statistics_multi_index_type; + + typedef generic_index voting_statistics_index; + +}} // graphene::chain + +FC_REFLECT_DERIVED( graphene::chain::voting_statistics_object, (graphene::chain::object), + (account)(stake)(proxy)(proxy_for)(votes) ) diff --git a/libraries/chain/voting_statistics_object.cpp b/libraries/chain/voting_statistics_object.cpp new file mode 100644 index 0000000000..c1df979586 --- /dev/null +++ b/libraries/chain/voting_statistics_object.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 Blockchain Projects B.V. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include + +namespace graphene { namespace chain { + + block_id_type voting_statistics_object::block_id; + +}} // graphene::chain From 3dd13850f5fba3149c02959b84aa3a28ebd71d0c Mon Sep 17 00:00:00 2001 From: dimfred Date: Tue, 12 Mar 2019 16:02:17 +0100 Subject: [PATCH 37/46] added voting_stat_plugin --- libraries/plugins/CMakeLists.txt | 1 + libraries/plugins/voting_stat/CMakeLists.txt | 20 ++ .../voting_stat/voting_stat_plugin.hpp | 69 ++++++ .../voting_stat/voting_stat_plugin.cpp | 216 ++++++++++++++++++ 4 files changed, 306 insertions(+) create mode 100644 libraries/plugins/voting_stat/CMakeLists.txt create mode 100644 libraries/plugins/voting_stat/include/graphene/voting_stat/voting_stat_plugin.hpp create mode 100644 libraries/plugins/voting_stat/voting_stat_plugin.cpp diff --git a/libraries/plugins/CMakeLists.txt b/libraries/plugins/CMakeLists.txt index caacb8bd53..a6f4d3bd49 100644 --- a/libraries/plugins/CMakeLists.txt +++ b/libraries/plugins/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory( delayed_node ) add_subdirectory( debug_witness ) add_subdirectory( snapshot ) add_subdirectory( es_objects ) +add_subdirectory( voting_stat ) diff --git a/libraries/plugins/voting_stat/CMakeLists.txt b/libraries/plugins/voting_stat/CMakeLists.txt new file mode 100644 index 0000000000..b3dc852938 --- /dev/null +++ b/libraries/plugins/voting_stat/CMakeLists.txt @@ -0,0 +1,20 @@ +file( GLOB HEADER "include/graphene/account_history/*.hpp" ) + +add_library( graphene_voting_stat voting_stat_plugin.cpp ) + +target_link_libraries( graphene_voting_stat graphene_chain graphene_app ) +target_include_directories( graphene_voting_stat + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +if(MSVC) + set_source_files_properties( voting_stat_plugin.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) +endif(MSVC) + +install( TARGETS + graphene_voting_stat + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib ) + +INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/voting_stat" ) diff --git a/libraries/plugins/voting_stat/include/graphene/voting_stat/voting_stat_plugin.hpp b/libraries/plugins/voting_stat/include/graphene/voting_stat/voting_stat_plugin.hpp new file mode 100644 index 0000000000..f4fae9df8b --- /dev/null +++ b/libraries/plugins/voting_stat/include/graphene/voting_stat/voting_stat_plugin.hpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019 Blockchain Projects BV. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include + +namespace graphene { namespace voting_stat { + using namespace chain; +// +// Plugins should #define their SPACE_ID's so plugins with +// conflicting SPACE_ID assignments can be compiled into the +// same binary (by simply re-assigning some of the conflicting #defined +// SPACE_ID's in a build script). +// +// Assignment of SPACE_ID's cannot be done at run-time because +// various template automagic depends on them being known at compile +// time. +// +#ifndef VOTING_STAT_SPACE_ID +#define VOTING_STAT_SPACE_ID 4 +#endif + +namespace detail +{ + class voting_stat_plugin_impl; +} + +class voting_stat_plugin : public graphene::app::plugin +{ + public: + voting_stat_plugin(); + virtual ~voting_stat_plugin(); + + std::string plugin_name()const override; + virtual void plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg) override; + virtual void plugin_initialize(const boost::program_options::variables_map& options) override; + virtual void plugin_startup() override; + + friend class detail::voting_stat_plugin_impl; + std::unique_ptr my; +}; + +}} // graphene::voting_stat + + diff --git a/libraries/plugins/voting_stat/voting_stat_plugin.cpp b/libraries/plugins/voting_stat/voting_stat_plugin.cpp new file mode 100644 index 0000000000..36787a8feb --- /dev/null +++ b/libraries/plugins/voting_stat/voting_stat_plugin.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2019 Blockchain Projects BV. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include + +#include + +#include + +namespace graphene { namespace voting_stat { + +namespace detail { + +class voting_stat_plugin_impl +{ +public: + voting_stat_plugin_impl(voting_stat_plugin& _plugin) + : _self( _plugin ){} + + virtual ~voting_stat_plugin_impl(){} + + graphene::chain::database& database() + { + return _self.database(); + } + + /** + * @brief callback for database::on_stake_calculated + * updates the given stake anc proxy account + */ + void on_stake_calculated( + const account_object& stake_account, + const account_object& proxy_account, + uint64_t stake ); + + /** + * @brief callback for database::on_maintenance_begin + * updates the block_id to the one where the maintenance interval occurs + */ + void on_maintenance_begin( const block_id_type& block_id ); + +private: + voting_stat_plugin& _self; + +}; + +void voting_stat_plugin_impl::on_maintenance_begin( const block_id_type& block_id ) +{ + graphene::chain::voting_statistics_object::block_id = block_id; +} + +void voting_stat_plugin_impl::on_stake_calculated( + const account_object& stake_account, + const account_object& proxy_account, + uint64_t stake ) +{ + auto& db = database(); + + account_id_type stake_id = stake_account.id; + account_id_type proxy_id = proxy_account.id; + account_id_type new_proxy_id = stake_id == proxy_id ? + GRAPHENE_PROXY_TO_SELF_ACCOUNT : proxy_id; + account_id_type old_proxy_id; + bool has_stake_changed; + + const auto& idx = db.get_index_type().indices().get(); + auto stake_stat_it = idx.find( stake_id ); + if( stake_stat_it == idx.end() ) + { + has_stake_changed = true; + old_proxy_id = GRAPHENE_PROXY_TO_SELF_ACCOUNT; + db.create( + [&stake_account, new_proxy_id, stake] (voting_statistics_object& o){ + o.account = stake_account.id; + o.stake = stake; + o.proxy = new_proxy_id; + o.votes = stake_account.options.votes; + } + ); + stake_stat_it = idx.find( stake_id ); + } + else + { + has_stake_changed = stake_stat_it->stake != stake; + old_proxy_id = stake_stat_it->proxy; + db.modify( *stake_stat_it, + [&stake_account, new_proxy_id, stake] (voting_statistics_object& o) { + o.stake = stake; + o.proxy = new_proxy_id; + o.votes = stake_account.options.votes; + } + ); + } + + if( old_proxy_id != new_proxy_id ) + { + if( old_proxy_id != GRAPHENE_PROXY_TO_SELF_ACCOUNT ) + { + /* if the proxy account has changed delete the proxied stake from the old proxy account */ + auto old_proxy_stat_it = idx.find( old_proxy_id ); + db.modify( *old_proxy_stat_it, + [stake_id] (voting_statistics_object& o) { + o.proxy_for.erase( stake_id ); + } + ); + } + + if( new_proxy_id != GRAPHENE_PROXY_TO_SELF_ACCOUNT ) + { + auto proxy_stat_it = idx.find( new_proxy_id ); + if( proxy_stat_it == idx.end() ) + { + /* if the new proxy account doesn't exist, create and add the proxied stake */ + db.create( + [stake_id, new_proxy_id, stake] (voting_statistics_object& o) { + o.account = new_proxy_id; + o.proxy_for.emplace( stake_id, stake ); + } + ); + } + else + { + /* insert the stake into the new proxy account */ + auto proxy_stat_it = idx.find( proxy_account.id ); + db.modify( *proxy_stat_it, + [stake_id, stake] (voting_statistics_object& o) { + auto insertion_return = o.proxy_for.emplace( stake_id, stake ); + if( !insertion_return.second ) + insertion_return.first->second = stake; + } + ); + } + } + } + else if( stake_stat_it->has_proxy() && has_stake_changed ) + { + /* when the proxied stake has changed update the proxy account with the new stake */ + auto proxy_stat_it = idx.find( proxy_account.id ); + db.modify( *proxy_stat_it, + [stake_id, stake] (voting_statistics_object& o) { + auto insertion_return = o.proxy_for.emplace( stake_id, stake ); + if( !insertion_return.second ) + insertion_return.first->second = stake; + } + ); + } +} + +} // namespace::detail + + + +voting_stat_plugin::voting_stat_plugin() : + my( new detail::voting_stat_plugin_impl(*this) ) +{ +} + +voting_stat_plugin::~voting_stat_plugin() +{ +} + +std::string voting_stat_plugin::plugin_name()const +{ + return "voting_stat"; +} + +void voting_stat_plugin::plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg + ) +{ +} + +void voting_stat_plugin::plugin_initialize(const boost::program_options::variables_map& options) +{ + auto& db = database(); + db.add_index< primary_index< voting_statistics_index > >(); + + db.on_voting_stake_calculated.connect( + [&](const account_object& stake_account, const account_object& proxy_account, + const uint64_t stake) { + my->on_stake_calculated( stake_account, proxy_account, stake ); + } + ); + + db.on_maintenance_begin.connect( + [&](const block_id_type& block_id){ my->on_maintenance_begin( block_id ); + }); + +} + +void voting_stat_plugin::plugin_startup() +{ +} + +} } From 1ba8840f39de2e8b9b4ab336c9f60bcbc503544f Mon Sep 17 00:00:00 2001 From: dimfred Date: Mon, 18 Mar 2019 00:00:34 +0100 Subject: [PATCH 38/46] added test --- tests/CMakeLists.txt | 4 + tests/voting_stat/main.cpp | 250 +++++++++++++++++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 tests/voting_stat/main.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b863c6347e..218f3d11b6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,4 +40,8 @@ file(GLOB ES_SOURCES "elasticsearch/*.cpp") add_executable( es_test ${COMMON_SOURCES} ${ES_SOURCES} ) target_link_libraries( es_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +file( GLOB VOTING_STAT_SOURCES "voting_stat/*.cpp" ) +add_executable( voting_stat_test ${COMMON_SOURCES} ${VOTING_STAT_SOURCES} ) +target_link_libraries( voting_stat_test graphene_chain graphene_app graphene_voting_stat graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) + add_subdirectory( generate_empty_blocks ) diff --git a/tests/voting_stat/main.cpp b/tests/voting_stat/main.cpp new file mode 100644 index 0000000000..d9e83de7eb --- /dev/null +++ b/tests/voting_stat/main.cpp @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2019 Blockchain Projects BV. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include + +#include + +#include + +#include + +#include + +#include + +#include "../common/database_fixture.hpp" + +#define BOOST_TEST_MODULE Voting Statistics Plugin Test +#include + +using namespace fc; +using namespace graphene::app; +using namespace graphene::chain; +using namespace graphene::chain::test; +using namespace graphene::voting_stat; + +struct voting_stat_fixture : public database_fixture +{ + vote_id_type default_vote_id; + + voting_stat_fixture() + { + app.register_plugin( true ); + app.initialize_plugins( boost::program_options::variables_map() ); // TODO + app.startup_plugins(); + }; + + void make_next_maintenance_interval() + { + generate_blocks( db.get_dynamic_global_properties().next_maintenance_time ); + generate_block(); + } + + void set_account_options( account_id_type id, optional proxy = optional() ) + { + witness_id_type wit_id = *db.get_global_properties().active_witnesses.begin(); + default_vote_id = wit_id(db).vote_id; + + account_options opt; + opt.votes = { default_vote_id }; + opt.num_witness = opt.votes.size(); + if( proxy ) + opt.voting_account = *proxy; + + account_update_operation op; + op.account = id; + op.new_options = opt; + + signed_transaction trx; + trx.operations.push_back( op ); + set_expiration( db, trx ); + PUSH_TX( db, trx, ~0 ); + } + + const voting_statistics_object& get_voting_statistics_object( account_id_type id ) + { + const auto& idx = db.get_index_type().indices().get(); + return *idx.find( id ); + } +}; + +BOOST_FIXTURE_TEST_SUITE( voting_stat_tests, voting_stat_fixture ) + + +BOOST_AUTO_TEST_CASE( block_id_changes_with_each_interval ) +{ try { + + make_next_maintenance_interval(); + auto first_block_id = voting_statistics_object::block_id; + + make_next_maintenance_interval(); + auto second_block_id = voting_statistics_object::block_id; + BOOST_CHECK( first_block_id != second_block_id ); + + make_next_maintenance_interval(); + auto third_block_id = voting_statistics_object::block_id; + BOOST_CHECK( second_block_id != third_block_id ); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE( voting_statistics_without_proxy ) +{ try { + + ACTOR( alice ); + transfer( committee_account, alice_id, asset(1) ); + set_account_options( alice_id ); + + const auto& alice_acc = alice_id(db); + BOOST_CHECK( *alice_acc.options.votes.begin() == default_vote_id ); + BOOST_CHECK( alice_acc.options.voting_account == GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + + + make_next_maintenance_interval(); + const auto& alice_stat1 = get_voting_statistics_object( alice_id ); + + BOOST_CHECK( alice_stat1.proxy == GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + BOOST_CHECK( !alice_stat1.has_proxy() ); + BOOST_CHECK( alice_stat1.proxy_for.empty() ); + BOOST_CHECK( alice_stat1.stake == 1 ); + BOOST_CHECK( *alice_stat1.votes.begin() == default_vote_id ); + BOOST_CHECK( alice_stat1.get_total_voting_stake() == 1 ); + + + /* increase stake */ + transfer( committee_account, alice_id, asset(10) ); + + make_next_maintenance_interval(); + const auto& alice_stat2 = get_voting_statistics_object( alice_id ); + + BOOST_CHECK( alice_stat2.proxy == GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + BOOST_CHECK( !alice_stat2.has_proxy() ); + BOOST_CHECK( alice_stat2.proxy_for.empty() ); + BOOST_CHECK( alice_stat2.stake == 11 ); + BOOST_CHECK( *alice_stat1.votes.begin() == default_vote_id ); + BOOST_CHECK( alice_stat1.get_total_voting_stake() == 11 ); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE( voting_statistics_with_proxy ) +{ try { + + ACTORS( (alice)(bob)(charlie) ); + transfer( committee_account, alice_id, asset(1), asset() ); + transfer( committee_account, bob_id, asset(2), asset() ); + transfer( committee_account, charlie_id, asset(3), asset() ); + + /* proxy chain: alice => bob => charlie */ + set_account_options( alice_id, bob_id ); + set_account_options( bob_id, charlie_id ); + set_account_options( charlie_id ); + + make_next_maintenance_interval(); + const auto& alice_stat1 = get_voting_statistics_object( alice_id ); + const auto& bob_stat1 = get_voting_statistics_object( bob_id ); + const auto& charlie_stat1 = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat1.has_proxy() ); + BOOST_CHECK( alice_stat1.proxy == bob_id ); + BOOST_CHECK( alice_stat1.get_total_voting_stake() == 0 ); + + BOOST_CHECK( bob_stat1.has_proxy() ); + BOOST_CHECK( bob_stat1.proxy == charlie_id ); + auto alice_proxied = *bob_stat1.proxy_for.begin(); + BOOST_CHECK( alice_proxied.first == alice_id && alice_proxied.second == 1 ); + BOOST_CHECK( bob_stat1.get_total_voting_stake() == 1 ); + + BOOST_CHECK( !charlie_stat1.has_proxy() ); + auto bob_proxied = *charlie_stat1.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat1.get_total_voting_stake() == (2 + 3) ); + + + /* proxy: alice => alice; bob => charlie; */ + set_account_options( alice_id, GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + + make_next_maintenance_interval(); + const auto& alice_stat2 = get_voting_statistics_object( alice_id ); + const auto& bob_stat2 = get_voting_statistics_object( bob_id ); + const auto& charlie_stat2 = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat2.has_proxy() ); + BOOST_CHECK( alice_stat2.proxy_for.empty() ); + BOOST_CHECK( alice_stat2.get_total_voting_stake() == 1 ); + + BOOST_CHECK( bob_stat2.has_proxy() ); + BOOST_CHECK( bob_stat2.proxy_for.empty() ); + BOOST_CHECK( bob_stat2.get_total_voting_stake() == 0 ); + + BOOST_CHECK( !charlie_stat2.has_proxy() ); + bob_proxied = *charlie_stat2.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat2.get_total_voting_stake() == (2 + 3) ); + + + /* proxy: alice => alice; bob => charlie; charlie => alice; stake increase */ + set_account_options( charlie_id, alice_id ); + transfer( committee_account, alice_id, asset(10) ); + transfer( committee_account, bob_id, asset(20) ); + transfer( committee_account, charlie_id, asset(30) ); + + make_next_maintenance_interval(); + const auto& alice_stat3 = get_voting_statistics_object( alice_id ); + const auto& bob_stat3 = get_voting_statistics_object( bob_id ); + const auto& charlie_stat3 = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat3.has_proxy() ); + auto charlie_proxied = *alice_stat3.proxy_for.begin(); + BOOST_CHECK( charlie_proxied.first == charlie_id && charlie_proxied.second == 33 ); + BOOST_CHECK( alice_stat3.get_total_voting_stake() == (11 + 33) ); + + BOOST_CHECK( bob_stat3.has_proxy() ); + BOOST_CHECK( bob_stat3.proxy_for.empty() ); + BOOST_CHECK( bob_stat3.get_total_voting_stake() == 0 ); + + BOOST_CHECK( charlie_stat3.has_proxy() ); + bob_proxied = *charlie_stat2.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 22 ); + BOOST_CHECK( charlie_stat3.get_total_voting_stake() == 22 ); + + + /* only stake increase */ + transfer( committee_account, alice_id, asset(100) ); + transfer( committee_account, bob_id, asset(200) ); + transfer( committee_account, charlie_id, asset(300) ); + + make_next_maintenance_interval(); + const auto& alice_stat4 = get_voting_statistics_object( alice_id ); + const auto& bob_stat4 = get_voting_statistics_object( bob_id ); + const auto& charlie_stat4 = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat4.stake == 111 ); + BOOST_CHECK( alice_stat4.get_total_voting_stake() == (111 + 333) ); + BOOST_CHECK( bob_stat4.stake == 222 ); + BOOST_CHECK( bob_stat4.get_total_voting_stake() == 0 ); + BOOST_CHECK( charlie_stat4.stake == 333 ); + BOOST_CHECK( charlie_stat4.get_total_voting_stake() == 222 ); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() From 61d8f6b1c8bcf0e5ac3f13e0136225b303ef5c3f Mon Sep 17 00:00:00 2001 From: dimfred Date: Mon, 25 Mar 2019 17:22:43 +0100 Subject: [PATCH 39/46] added voting statistics to es_objects --- libraries/plugins/es_objects/es_objects.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 21ae19094e..ab7f54eaec 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -60,6 +61,7 @@ class es_objects_plugin_impl bool _es_objects_balances = true; bool _es_objects_limit_orders = true; bool _es_objects_asset_bitasset = true; + bool _es_objects_voting_statistics = true; std::string _es_objects_index_prefix = "objects-"; uint32_t _es_objects_start_es_after_block = 0; CURL *curl; // curl handler @@ -148,6 +150,15 @@ bool es_objects_plugin_impl::index_database( const vector& ids, else prepareTemplate(*ba, "bitasset"); } + } else if(value.is() && _es_objects_voting_statistics) { + auto obj = db.find_object(value); + auto vs = static_cast(obj); + if (vs != nullptr) { + if (action == "delete") + remove_from_database(vs->id, "voting_statistics"); + else + prepareTemplate(*vs, "voting-statistics"); + } } } @@ -253,6 +264,7 @@ void es_objects_plugin::plugin_set_program_options( ("es-objects-balances", boost::program_options::value(), "Store balances objects(true)") ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects(true)") ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data(true)") + ("es-objects-voting-statistics", boost::program_options::value(), "Store voting statistcs(true)") ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") ("es-objects-keep-only-current", boost::program_options::value(), "Keep only current state of the objects(true)") ("es-objects-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") @@ -312,6 +324,9 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable if (options.count("es-objects-asset-bitasset")) { my->_es_objects_asset_bitasset = options["es-objects-asset-bitasset"].as(); } + if (options.count("es-objects-voting-statistics")) { + my->_es_objects_voting_statistics = options["es-objects-voting-statistics"].as(); + } if (options.count("es-objects-index-prefix")) { my->_es_objects_index_prefix = options["es-objects-index-prefix"].as(); } From f67075d6a3e97c3bda6e5a0613015dd07bf26668 Mon Sep 17 00:00:00 2001 From: dimfred Date: Mon, 25 Mar 2019 17:22:58 +0100 Subject: [PATCH 40/46] added test to check for voting statistics in elasticsearch --- tests/voting_stat/main.cpp | 67 ++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/tests/voting_stat/main.cpp b/tests/voting_stat/main.cpp index d9e83de7eb..f34403a2d7 100644 --- a/tests/voting_stat/main.cpp +++ b/tests/voting_stat/main.cpp @@ -26,6 +26,8 @@ #include #include +#include +#include #include @@ -43,16 +45,55 @@ using namespace graphene::app; using namespace graphene::chain; using namespace graphene::chain::test; using namespace graphene::voting_stat; +using namespace graphene::es_objects; +using namespace graphene::utilities; +namespace bpo = boost::program_options; struct voting_stat_fixture : public database_fixture { vote_id_type default_vote_id; - + CURL *_curl; + ES _es; + voting_stat_fixture() { - app.register_plugin( true ); - app.initialize_plugins( boost::program_options::variables_map() ); // TODO - app.startup_plugins(); + try + { + _curl = curl_easy_init(); + _es.curl = _curl; + _es.elasticsearch_url = "http://localhost:9200/"; + _es.index_prefix = "objects-"; + + app.register_plugin( true ); + app.register_plugin( true ); + + auto es_object = app.get_plugin("es_objects"); + bpo::options_description cli; + bpo::options_description cfi; + es_object->plugin_set_program_options( cli, cfi ); + + bpo::variables_map var_map; + + int plugin_argc = 17; + const char* const plugin_argv[]{ "voting_stat", + "--es-objects-bulk-replay", "1", + "--es-objects-proposals", "false", + "--es-objects-accounts", "false", + "--es-objects-assets", "false", + "--es-objects-balances", "false", + "--es-objects-limit-orders", "false", + "--es-objects-asset-bitasset", "false", + "--es-objects-keep-only-current", "false" + }; + + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfi ), var_map ); + app.initialize_plugins( var_map ); + } + catch(fc::exception &e) + { + edump((e.to_detail_string() )); + } + }; void make_next_maintenance_interval() @@ -91,7 +132,6 @@ struct voting_stat_fixture : public database_fixture BOOST_FIXTURE_TEST_SUITE( voting_stat_tests, voting_stat_fixture ) - BOOST_AUTO_TEST_CASE( block_id_changes_with_each_interval ) { try { @@ -148,7 +188,12 @@ BOOST_AUTO_TEST_CASE( voting_statistics_without_proxy ) BOOST_AUTO_TEST_CASE( voting_statistics_with_proxy ) { try { - + /* clear es-db */ + auto objects_deleted = graphene::utilities::deleteAll(_es); + if( !objects_deleted ) + BOOST_FAIL( "elastic search DB could not be deleted" ); + + ACTORS( (alice)(bob)(charlie) ); transfer( committee_account, alice_id, asset(1), asset() ); transfer( committee_account, bob_id, asset(2), asset() ); @@ -244,6 +289,16 @@ BOOST_AUTO_TEST_CASE( voting_statistics_with_proxy ) BOOST_CHECK( bob_stat4.get_total_voting_stake() == 0 ); BOOST_CHECK( charlie_stat4.stake == 333 ); BOOST_CHECK( charlie_stat4.get_total_voting_stake() == 222 ); + + + /* wait for es */ + fc::usleep(fc::milliseconds(1000)); + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + _es.endpoint = _es.index_prefix + "*/data/_count"; + _es.query = query; + auto res = graphene::utilities::simpleQuery(_es); + variant j = fc::json::from_string(res); + BOOST_CHECK( j["count"].as_int64() == 12 ); } FC_LOG_AND_RETHROW() } From 468050dec1280cd199f546b8a4a989e0948a8f93 Mon Sep 17 00:00:00 2001 From: dimfred Date: Mon, 1 Jul 2019 12:52:52 +0200 Subject: [PATCH 41/46] added plugin to witness node --- programs/witness_node/CMakeLists.txt | 2 +- programs/witness_node/main.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/witness_node/CMakeLists.txt b/programs/witness_node/CMakeLists.txt index 4815879a40..5dc9a9bcb3 100644 --- a/programs/witness_node/CMakeLists.txt +++ b/programs/witness_node/CMakeLists.txt @@ -12,7 +12,7 @@ endif() # We have to link against graphene_debug_witness because deficiency in our API infrastructure doesn't allow plugins to be fully abstracted #246 target_link_libraries( witness_node -PRIVATE graphene_app graphene_delayed_node graphene_account_history graphene_elasticsearch graphene_market_history graphene_grouped_orders graphene_witness graphene_chain graphene_debug_witness graphene_egenesis_full graphene_snapshot graphene_es_objects fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +PRIVATE graphene_app graphene_delayed_node graphene_account_history graphene_elasticsearch graphene_market_history graphene_grouped_orders graphene_witness graphene_chain graphene_debug_witness graphene_egenesis_full graphene_snapshot graphene_es_objects graphene_voting_stat fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) install( TARGETS witness_node diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index cc10ecca01..44a47341ff 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -82,7 +83,8 @@ int main(int argc, char** argv) { auto snapshot_plug = node->register_plugin(); auto es_objects_plug = node->register_plugin(); auto grouped_orders_plug = node->register_plugin(); - + auto voting_stat_plug = node->register_plugin(); + try { bpo::options_description cli, cfg; From dd5233ff25704304bd46802806b5ce815d82dfa2 Mon Sep 17 00:00:00 2001 From: dimfred Date: Tue, 30 Jul 2019 12:53:52 +0200 Subject: [PATCH 42/46] es objects switch case optimization want to add static constexpr method to abstract object for better case checks --- libraries/chain/db_maint.cpp | 4 +- .../chain/include/graphene/chain/database.hpp | 9 +- libraries/plugins/es_objects/es_objects.cpp | 182 ++++++++++++------ 3 files changed, 135 insertions(+), 60 deletions(-) diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index c910694ba6..f590aa96df 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -1151,7 +1151,7 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g distribute_fba_balances(*this); create_buyback_orders(*this); - on_maintenance_begin( next_block.id() ); + on_maintenance_begin( next_block.block_num() ); struct vote_tally_helper { database& d; @@ -1332,6 +1332,8 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g // process_budget needs to run at the bottom because // it needs to know the next_maintenance_time process_budget(); + + on_maintenance_end(); } } } diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index f48095d572..6a7d96b0b2 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -213,7 +213,12 @@ namespace graphene { namespace chain { /** * Emitted after the beginning of the maintenance interval */ - fc::signal on_maintenance_begin; + fc::signal on_maintenance_begin; + + /** + * Emitted after the end of the maintenance interval + */ + fc::signal on_maintenance_end; //////////////////// db_witness_schedule.cpp //////////////////// @@ -331,7 +336,7 @@ namespace graphene { namespace chain { * to newly created VBID and return it. * * Otherwise, credit amount to ovbid. - * + * * @return ID of newly created VBO, but only if VBO was created. */ optional< vesting_balance_id_type > deposit_lazy_vesting( diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index fb39359015..36acb1c7a1 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include @@ -63,6 +64,9 @@ class es_objects_plugin_impl bool _es_objects_limit_orders = true; bool _es_objects_asset_bitasset = true; bool _es_objects_voting_statistics = true; + bool _es_objects_voteable_statistics = true; + bool _es_objects_statistics_delete_allowed = true; + std::string _es_objects_index_prefix = "objects-"; uint32_t _es_objects_start_es_after_block = 0; CURL *curl; // curl handler @@ -144,70 +148,126 @@ bool es_objects_plugin_impl::index_database(const vector& ids, s limit_documents = _es_objects_bulk_replay; - for (auto const &value: ids) { - if (value.is() && _es_objects_proposals) { - auto obj = db.find_object(value); - auto p = static_cast(obj); - if (p != nullptr) { - if (action == "delete") - remove_from_database(p->id, "proposal"); - else - prepareTemplate(*p, "proposal"); + for (auto const &value: ids) + { + switch( value.space_type() ) + { + case( proposal_object::space_id << 8 | proposal_object::type_id ): + { + if( _es_objects_proposals ) { + auto obj = db.find_object(value); + auto p = static_cast(obj); + if (p != nullptr) { + if (action == "delete") + remove_from_database(p->id, "proposal"); + else + prepareTemplate(*p, "proposal"); + } + } + break; + } + case( account_object::space_id << 8 | account_object::type_id ): + { + if( _es_objects_accounts ) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if (a != nullptr) { + if (action == "delete") + remove_from_database(a->id, "account"); + else + prepareTemplate(*a, "account"); + } + } + break; } - } else if (value.is() && _es_objects_accounts) { - auto obj = db.find_object(value); - auto a = static_cast(obj); - if (a != nullptr) { - if (action == "delete") - remove_from_database(a->id, "account"); - else - prepareTemplate(*a, "account"); + case( asset_object::space_id << 8 | asset_object::type_id ): + { + if( _es_objects_assets ) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if (a != nullptr) { + if (action == "delete") + remove_from_database(a->id, "asset"); + else + prepareTemplate(*a, "asset"); + } + } + break; } - } else if (value.is() && _es_objects_assets) { - auto obj = db.find_object(value); - auto a = static_cast(obj); - if (a != nullptr) { - if (action == "delete") - remove_from_database(a->id, "asset"); - else - prepareTemplate(*a, "asset"); + case( account_balance_object::space_id << 8 | account_balance_object::type_id ): + { + if( _es_objects_balances ) { + auto obj = db.find_object(value); + auto b = static_cast(obj); + if (b != nullptr) { + if (action == "delete") + remove_from_database(b->id, "balance"); + else + prepareTemplate(*b, "balance"); + } + } + break; } - } else if (value.is() && _es_objects_balances) { - auto obj = db.find_object(value); - auto b = static_cast(obj); - if (b != nullptr) { - if (action == "delete") - remove_from_database(b->id, "balance"); - else - prepareTemplate(*b, "balance"); + case( limit_order_object::space_id << 8 | limit_order_object::type_id ): + { + if( _es_objects_limit_orders ) { + auto obj = db.find_object(value); + auto l = static_cast(obj); + if (l != nullptr) { + if (action == "delete") + remove_from_database(l->id, "limitorder"); + else + prepareTemplate(*l, "limitorder"); + } + } + break; } - } else if (value.is() && _es_objects_limit_orders) { - auto obj = db.find_object(value); - auto l = static_cast(obj); - if (l != nullptr) { - if (action == "delete") - remove_from_database(l->id, "limitorder"); - else - prepareTemplate(*l, "limitorder"); + case( asset_bitasset_data_object::space_id << 8 | asset_bitasset_data_object::type_id ): + { + if( _es_objects_asset_bitasset ) { + auto obj = db.find_object(value); + auto ba = static_cast(obj); + if (ba != nullptr) { + if (action == "delete") + remove_from_database(ba->id, "bitasset"); + else + prepareTemplate(*ba, "bitasset"); + } + } + break; } - } else if (value.is() && _es_objects_asset_bitasset) { - auto obj = db.find_object(value); - auto ba = static_cast(obj); - if (ba != nullptr) { - if (action == "delete") - remove_from_database(ba->id, "bitasset"); - else - prepareTemplate(*ba, "bitasset"); + case( voting_statistics_object::space_id << 8 | voting_statistics_object::type_id ): + { + if( _es_objects_voting_statistics ) { + auto obj = db.find_object(value); + auto vs = static_cast(obj); + + if (vs != nullptr) { + if (action == "delete" && _es_objects_statistics_delete_allowed ) + remove_from_database(vs->id, "voting-statistics"); + else + prepareTemplate(*vs, "voting-statistics"); + } + } + break; } - } else if(value.is() && _es_objects_voting_statistics) { - auto obj = db.find_object(value); - auto vs = static_cast(obj); - if (vs != nullptr) { - if (action == "delete") - remove_from_database(vs->id, "voting_statistics"); - else - prepareTemplate(*vs, "voting-statistics"); + case( voteable_statistics_object::space_id << 8 | voteable_statistics_object::type_id ): + { + if( _es_objects_voteable_statistics ) { + auto obj = db.find_object(value); + auto vs = static_cast(obj); + + if (vs != nullptr) { + if (action == "delete" && _es_objects_statistics_delete_allowed ) + remove_from_database(vs->id, "voteable-statistics"); + else + prepareTemplate(*vs, "voteable-statistics"); + } + } + break; } + default: + break; } } @@ -318,6 +378,8 @@ void es_objects_plugin::plugin_set_program_options( ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects(true)") ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data(true)") ("es-objects-voting-statistics", boost::program_options::value(), "Store voting statistcs(true)") + ("es-objects-voteable-statistics", boost::program_options::value(), "Store voteable statistcs(true)") + ("es-objects-statistics-delete-allowed", boost::program_options::value(), "Allows the deletion of statistics objects from es(true)") ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") ("es-objects-keep-only-current", boost::program_options::value(), "Keep only current state of the objects(true)") ("es-objects-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") @@ -387,6 +449,12 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable if (options.count("es-objects-voting-statistics")) { my->_es_objects_voting_statistics = options["es-objects-voting-statistics"].as(); } + if (options.count("es-objects-voteable-statistics")) { + my->_es_objects_voteable_statistics = options["es-objects-voteable-statistics"].as(); + } + if (options.count("es-objects-statistics-delete-allowed")) { + my->_es_objects_statistics_delete_allowed = options["es-objects-statistics-delete-allowed"].as(); + } if (options.count("es-objects-index-prefix")) { my->_es_objects_index_prefix = options["es-objects-index-prefix"].as(); } From ca014624786ad477df929e934625c494075f69f3 Mon Sep 17 00:00:00 2001 From: dimfred Date: Tue, 30 Jul 2019 16:47:54 +0200 Subject: [PATCH 43/46] voteable statistics object --- .../include/graphene/chain/protocol/types.hpp | 5 + .../chain/voteable_statistics_object.hpp | 95 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 libraries/chain/include/graphene/chain/voteable_statistics_object.hpp diff --git a/libraries/chain/include/graphene/chain/protocol/types.hpp b/libraries/chain/include/graphene/chain/protocol/types.hpp index 28485a69d1..77d43bc6ab 100644 --- a/libraries/chain/include/graphene/chain/protocol/types.hpp +++ b/libraries/chain/include/graphene/chain/protocol/types.hpp @@ -156,6 +156,7 @@ namespace graphene { namespace chain { balance_object_type, htlc_object_type, voting_statistics_object_type, + voteable_statistics_object_type, OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types }; @@ -200,6 +201,7 @@ namespace graphene { namespace chain { class blinded_balance_object; class voting_statistics_object; class htlc_object; + class voteable_statistics_object; typedef object_id< protocol_ids, account_object_type, account_object> account_id_type; typedef object_id< protocol_ids, asset_object_type, asset_object> asset_id_type; @@ -217,6 +219,7 @@ namespace graphene { namespace chain { typedef object_id< protocol_ids, balance_object_type, balance_object> balance_id_type; typedef object_id< protocol_ids, voting_statistics_object_type, voting_statistics_object> voting_statistics_id_type; typedef object_id< protocol_ids, htlc_object_type, htlc_object> htlc_id_type; + typedef object_id< protocol_ids, voteable_statistics_object_type, voteable_statistics_object> voteable_statistics_id_type; // implementation types class global_property_object; @@ -392,6 +395,7 @@ FC_REFLECT_ENUM( graphene::chain::object_type, (balance_object_type) (htlc_object_type) (voting_statistics_object_type) + (voteable_statistics_object_type) (OBJECT_TYPE_COUNT) ) FC_REFLECT_ENUM( graphene::chain::impl_object_type, @@ -447,6 +451,7 @@ FC_REFLECT_TYPENAME( graphene::chain::fba_accumulator_id_type ) FC_REFLECT_TYPENAME( graphene::chain::collateral_bid_id_type ) FC_REFLECT_TYPENAME( graphene::chain::voting_statistics_id_type ) FC_REFLECT_TYPENAME( graphene::chain::htlc_id_type ) +FC_REFLECT_TYPENAME( graphene::chain::voteable_statistics_id_type ) FC_REFLECT( graphene::chain::void_t, ) diff --git a/libraries/chain/include/graphene/chain/voteable_statistics_object.hpp b/libraries/chain/include/graphene/chain/voteable_statistics_object.hpp new file mode 100644 index 0000000000..978f8b7be9 --- /dev/null +++ b/libraries/chain/include/graphene/chain/voteable_statistics_object.hpp @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 Blockchain Projects B.V. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include + +#include + +#include +#include + +#include + + +namespace graphene { namespace chain { + + /** + * @brief tracks the history of the voting stake for an account + * @ingroup object + * @ingroup implementation + * + * The calculation of the voting stake, performed in the maintenance interval, results + * in the creation or, if present, in the update of a voting_statistics_object. + * + * @note By default these objects are not tracked, the voting_stat_plugin must + * be loaded for these objects to be maintained. + */ + class voteable_statistics_object : public abstract_object + { + public: + static const uint8_t space_id = protocol_ids; + static const uint8_t type_id = voteable_statistics_object_type; + + voteable_statistics_object(){} + + /* the block_num where the maintenance interval was performed */ + uint32_t block_number; + /* vote_id of the voteable_object */ + vote_id_type vote_id; + /* the accounts which this voteable is voted by with the number of votes */ + flat_map< account_id_type, uint64_t > voted_by; + + /* returns the total votes of this object */ + uint64_t get_votes() const + { + return boost::accumulate( voted_by | boost::adaptors::map_values, 0 ); + } + }; + + + //struct by_block_number{}; // declared in voting_statistics_object just let it like this? + typedef multi_index_container< voteable_statistics_object, + indexed_by< + ordered_unique< tag, + member< object, object_id_type, &object::id > + >, + ordered_unique< tag, + composite_key< voteable_statistics_object, + member< voteable_statistics_object, uint32_t, &voteable_statistics_object::block_number >, + member< voteable_statistics_object, vote_id_type, &voteable_statistics_object::vote_id > + > + > + > + > voteable_statistics_multi_index_type; + + typedef generic_index< voteable_statistics_object, + voteable_statistics_multi_index_type + > voteable_statistics_index; + +}} // graphene::chain + +FC_REFLECT_DERIVED( graphene::chain::voteable_statistics_object, (graphene::chain::object), + (block_number)(vote_id)(voted_by) ) From cd7f8d5912b01e060a0538ba4a243cff86222511 Mon Sep 17 00:00:00 2001 From: dimfred Date: Tue, 30 Jul 2019 16:48:20 +0200 Subject: [PATCH 44/46] voting statisitics object --- libraries/chain/CMakeLists.txt | 1 - .../chain/voting_statistics_object.hpp | 42 ++++++++++--------- libraries/chain/voting_statistics_object.cpp | 30 ------------- 3 files changed, 22 insertions(+), 51 deletions(-) delete mode 100644 libraries/chain/voting_statistics_object.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index ab48478b3f..411f92661f 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -89,7 +89,6 @@ add_library( graphene_chain market_object.cpp proposal_object.cpp vesting_balance_object.cpp - voting_statistics_object.cpp block_database.cpp diff --git a/libraries/chain/include/graphene/chain/voting_statistics_object.hpp b/libraries/chain/include/graphene/chain/voting_statistics_object.hpp index 94b3795cf2..40e06b810f 100644 --- a/libraries/chain/include/graphene/chain/voting_statistics_object.hpp +++ b/libraries/chain/include/graphene/chain/voting_statistics_object.hpp @@ -32,7 +32,7 @@ #include #include - +#include namespace graphene { namespace chain { @@ -41,23 +41,22 @@ namespace graphene { namespace chain { * @ingroup object * @ingroup implementation * - * The calculation of the voting stake, performed in the maintenance interval, results - * in the creation or, if present, in the update of a voting_statistics_object. + * The calculation of the voting stake, performed in the maintenance interval, results in the creation or, + * if present, in the update of a voting_statistics_object. * - * @note By default these objects are not tracked, the voting_stat_plugin must - * be loaded for these objects to be maintained. + * @note By default these objects are not tracked, the voting_stat_plugin must be loaded for these objects to + * be maintained. */ class voting_statistics_object : public abstract_object { public: static const uint8_t space_id = protocol_ids; static const uint8_t type_id = voting_statistics_object_type; - - voting_statistics_object(){} - /* the block_id where the maintenance interval was performed */ - static block_id_type block_id; + voting_statistics_object(){} + /* the block number where the maintenance interval was performed */ + uint32_t block_number; /* the owner of the stake */ account_id_type account; /* the stake which was generated by this account */ @@ -70,34 +69,37 @@ namespace graphene { namespace chain { flat_set votes; /* returns the total voting stake this account can vote with */ - const uint64_t get_total_voting_stake() const + uint64_t get_total_voting_stake() const { uint64_t init = has_proxy() ? 0 : stake; return boost::accumulate( proxy_for | boost::adaptors::map_values, init ); } - inline bool has_proxy() const + inline bool has_proxy() const { return GRAPHENE_PROXY_TO_SELF_ACCOUNT != proxy; } }; - struct by_owner{}; - - typedef multi_index_container< - voting_statistics_object, + struct by_block_number{}; + typedef multi_index_container< voting_statistics_object, indexed_by< - ordered_unique< tag, member< object, object_id_type, &object::id > >, - ordered_unique< tag, - member< voting_statistics_object, account_id_type, &voting_statistics_object::account > + ordered_unique< tag, + member< object, object_id_type, &object::id > + >, + ordered_unique< tag, + composite_key< voting_statistics_object, + member< voting_statistics_object, uint32_t, &voting_statistics_object::block_number >, + member< voting_statistics_object, account_id_type, &voting_statistics_object::account > + > > > > voting_statistics_multi_index_type; - typedef generic_index voting_statistics_index; + typedef generic_index< voting_statistics_object, voting_statistics_multi_index_type > voting_statistics_index; }} // graphene::chain FC_REFLECT_DERIVED( graphene::chain::voting_statistics_object, (graphene::chain::object), - (account)(stake)(proxy)(proxy_for)(votes) ) + (block_number)(account)(stake)(proxy)(proxy_for)(votes) ) diff --git a/libraries/chain/voting_statistics_object.cpp b/libraries/chain/voting_statistics_object.cpp deleted file mode 100644 index c1df979586..0000000000 --- a/libraries/chain/voting_statistics_object.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 Blockchain Projects B.V. - * - * The MIT License - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include - -namespace graphene { namespace chain { - - block_id_type voting_statistics_object::block_id; - -}} // graphene::chain From 592845e27488976e9d17f25eb7a40cfc8f3f9a67 Mon Sep 17 00:00:00 2001 From: dimfred Date: Tue, 30 Jul 2019 16:48:42 +0200 Subject: [PATCH 45/46] voting stat plugin --- .../voting_stat/voting_stat_plugin.cpp | 357 +++++++++++++----- 1 file changed, 255 insertions(+), 102 deletions(-) diff --git a/libraries/plugins/voting_stat/voting_stat_plugin.cpp b/libraries/plugins/voting_stat/voting_stat_plugin.cpp index 36787a8feb..64295398b3 100644 --- a/libraries/plugins/voting_stat/voting_stat_plugin.cpp +++ b/libraries/plugins/voting_stat/voting_stat_plugin.cpp @@ -24,9 +24,18 @@ #include #include +#include +#include +#include +#include #include +#include +#include + +#include + namespace graphene { namespace voting_stat { namespace detail { @@ -34,133 +43,230 @@ namespace detail { class voting_stat_plugin_impl { public: - voting_stat_plugin_impl(voting_stat_plugin& _plugin) - : _self( _plugin ){} - + voting_stat_plugin_impl(voting_stat_plugin& _plugin) : _self( _plugin ){} virtual ~voting_stat_plugin_impl(){} - graphene::chain::database& database() - { - return _self.database(); - } + boost::signals2::connection _on_voting_stake_calc_conn; + std::unique_ptr _on_voting_stake_calc_block; + + uint16_t _maint_counter = 0; + + /** + * plugin parameters + */ + bool _keep_objects_in_db = true; + uint16_t _track_every_x_maint = 12; + bool _track_worker_votes = true; + bool _track_witness_votes = true; + bool _track_committee_votes = true; /** * @brief callback for database::on_stake_calculated - * updates the given stake anc proxy account + * + * This function is triggered when the calculation of a stake for a given account is done inside the maintenance + * interval. It creates/updates the voting_statistics_object for a stake account. Optionally if a proxy is set + * in the stake account, also the voting_statistics_object for the proxy account is created/updated. */ - void on_stake_calculated( - const account_object& stake_account, - const account_object& proxy_account, - uint64_t stake ); + void on_stake_calculated(const account_object& stake_account, const account_object& proxy_account, uint64_t stake); /** * @brief callback for database::on_maintenance_begin - * updates the block_id to the one where the maintenance interval occurs + * + * Updates the block number to the one where the maintenance interval occurs and unblocks the + * database::on_stake_calculated signal, so that statistics objects can be created. */ - void on_maintenance_begin( const block_id_type& block_id ); + void on_maintenance_begin(uint32_t block_num); + + /** + * @brief callback for database::on_maintenance_end + * + * Disconnects the on_stake_calculated callback and deletes all statistics objects if + * _voting_stat_delete_objects_after_interval=true + */ + void on_maintenance_end(); + + void create_voteable_statistics_objects(); + void delete_all_statistics_objects(); + + graphene::chain::database& database() + { + return _self.database(); + } private: voting_stat_plugin& _self; - + + uint32_t _maint_block; + bool _create_voteable = false; }; -void voting_stat_plugin_impl::on_maintenance_begin( const block_id_type& block_id ) +void voting_stat_plugin_impl::on_maintenance_begin(uint32_t block_num) { - graphene::chain::voting_statistics_object::block_id = block_id; + if( !_keep_objects_in_db ) + delete_all_statistics_objects(); + + if( _maint_counter == _track_every_x_maint ) + { + _on_voting_stake_calc_block->unblock(); + _maint_counter = 0; + _maint_block = block_num; + _create_voteable = true; + } + ++_maint_counter; +} + +void voting_stat_plugin_impl::on_maintenance_end() +{ + _on_voting_stake_calc_block->block(); + if( _create_voteable ) { + _create_voteable = false; + create_voteable_statistics_objects(); + } +} + +void voting_stat_plugin_impl::delete_all_statistics_objects() +{ + auto& db = database(); + + // TODO find better way to delete + const auto& voting_idx = db.get_index_type().indices().get(); + for( const auto& voting_obj : voting_idx ) { + db.remove( voting_obj ); + } + + const auto& voteable_idx = db.get_index_type().indices().get(); + for( const auto& voteable_obj : voteable_idx ) { + db.remove( voteable_obj ); + } +} + +void voting_stat_plugin_impl::create_voteable_statistics_objects() +{ + auto& db = database(); + + // TODO secondary index for workers where current_time < worker_end_time + // will reduce the iteration time + if( _track_worker_votes ) + { + const auto& worker_idx = db.get_index_type().indices().get(); + + auto now = db.head_block_time(); + for( const auto& worker : worker_idx ) + { + if( now > worker.work_end_date ) + continue; + + db.create( [this, &worker]( voteable_statistics_object& o ){ + o.block_number = _maint_block; + o.vote_id = worker.vote_for; + }); + } + } + + if( _track_witness_votes ) + { + const auto& witness_idx = db.get_index_type().indices().get(); + for( const auto& witness : witness_idx ) + { + db.create( [this, &witness]( voteable_statistics_object& o ){ + o.block_number = _maint_block; + o.vote_id = witness.vote_id; + }); + } + } + + if( _track_committee_votes ) + { + const auto& committee_idx = db.get_index_type().indices().get(); + for( const auto& committee_member : committee_idx ) + { + db.create( [this, &committee_member](voteable_statistics_object& o){ + o.block_number = _maint_block; + o.vote_id = committee_member.vote_id; + }); + } + } + + + const auto& voting_stat_idx = db.get_index_type().indices().get(); + auto voting_stat_range = voting_stat_idx.equal_range( _maint_block ); + + const auto& voteable_idx = db.get_index_type().indices().get(); + + for( auto voting_stat_it = voting_stat_range.first; voting_stat_it != voting_stat_range.second; ++voting_stat_it ) + { + uint64_t total_stake = voting_stat_it->get_total_voting_stake(); + if( !total_stake ) + continue; // don't bother inserting a 0 stake + + const flat_set& votes = voting_stat_it->votes; + for( const auto& vote_id : votes ) + { + auto voteable_obj_range = voteable_idx.equal_range( boost::make_tuple( _maint_block, vote_id ) ); + if( voteable_obj_range.first == voteable_obj_range.second ) + continue; // when the obj isn't found it isn't needed hence skip it + + db.modify( *voteable_obj_range.first, + [voting_stat_it, total_stake]( voteable_statistics_object& o ){ + o.voted_by.emplace( voting_stat_it->account, total_stake ); + } + ); + } + } } -void voting_stat_plugin_impl::on_stake_calculated( +void voting_stat_plugin_impl::on_stake_calculated( const account_object& stake_account, - const account_object& proxy_account, + const account_object& proxy_account, uint64_t stake ) { auto& db = database(); account_id_type stake_id = stake_account.id; account_id_type proxy_id = proxy_account.id; - account_id_type new_proxy_id = stake_id == proxy_id ? - GRAPHENE_PROXY_TO_SELF_ACCOUNT : proxy_id; - account_id_type old_proxy_id; - bool has_stake_changed; - - const auto& idx = db.get_index_type().indices().get(); - auto stake_stat_it = idx.find( stake_id ); - if( stake_stat_it == idx.end() ) + proxy_id = ( stake_id == proxy_id ? GRAPHENE_PROXY_TO_SELF_ACCOUNT : proxy_id ); + + const auto& voting_stat_idx = db.get_index_type().indices().get(); + + auto stake_stat_range = voting_stat_idx.equal_range( boost::make_tuple( _maint_block, stake_id ) ); + if( stake_stat_range.first == stake_stat_range.second ) { - has_stake_changed = true; - old_proxy_id = GRAPHENE_PROXY_TO_SELF_ACCOUNT; - db.create( - [&stake_account, new_proxy_id, stake] (voting_statistics_object& o){ - o.account = stake_account.id; - o.stake = stake; - o.proxy = new_proxy_id; - o.votes = stake_account.options.votes; - } - ); - stake_stat_it = idx.find( stake_id ); + db.create( [this, &stake_account, &proxy_id, stake]( voting_statistics_object& o ){ + o.block_number = _maint_block; + o.account = stake_account.id; + o.stake = stake; + o.proxy = proxy_id; + o.votes = stake_account.options.votes; + }); } else { - has_stake_changed = stake_stat_it->stake != stake; - old_proxy_id = stake_stat_it->proxy; - db.modify( *stake_stat_it, - [&stake_account, new_proxy_id, stake] (voting_statistics_object& o) { + db.modify( *stake_stat_range.first, + [this, &stake_account, &proxy_id, stake]( voting_statistics_object& o ){ o.stake = stake; - o.proxy = new_proxy_id; + o.proxy = proxy_id; o.votes = stake_account.options.votes; } - ); + ); } - - if( old_proxy_id != new_proxy_id ) + + if( proxy_id == GRAPHENE_PROXY_TO_SELF_ACCOUNT ) + return; + + auto proxy_stat_range = voting_stat_idx.equal_range( boost::make_tuple( _maint_block, proxy_id ) ); + if( proxy_stat_range.first == proxy_stat_range.second ) { - if( old_proxy_id != GRAPHENE_PROXY_TO_SELF_ACCOUNT ) - { - /* if the proxy account has changed delete the proxied stake from the old proxy account */ - auto old_proxy_stat_it = idx.find( old_proxy_id ); - db.modify( *old_proxy_stat_it, - [stake_id] (voting_statistics_object& o) { - o.proxy_for.erase( stake_id ); - } - ); - } - - if( new_proxy_id != GRAPHENE_PROXY_TO_SELF_ACCOUNT ) - { - auto proxy_stat_it = idx.find( new_proxy_id ); - if( proxy_stat_it == idx.end() ) - { - /* if the new proxy account doesn't exist, create and add the proxied stake */ - db.create( - [stake_id, new_proxy_id, stake] (voting_statistics_object& o) { - o.account = new_proxy_id; - o.proxy_for.emplace( stake_id, stake ); - } - ); - } - else - { - /* insert the stake into the new proxy account */ - auto proxy_stat_it = idx.find( proxy_account.id ); - db.modify( *proxy_stat_it, - [stake_id, stake] (voting_statistics_object& o) { - auto insertion_return = o.proxy_for.emplace( stake_id, stake ); - if( !insertion_return.second ) - insertion_return.first->second = stake; - } - ); - } - } + db.create( [this, &stake_id, &proxy_id, stake]( voting_statistics_object& o ){ + o.block_number = _maint_block; + o.account = proxy_id; + o.proxy_for.emplace( stake_id, stake ); + }); } - else if( stake_stat_it->has_proxy() && has_stake_changed ) + else { - /* when the proxied stake has changed update the proxy account with the new stake */ - auto proxy_stat_it = idx.find( proxy_account.id ); - db.modify( *proxy_stat_it, - [stake_id, stake] (voting_statistics_object& o) { - auto insertion_return = o.proxy_for.emplace( stake_id, stake ); - if( !insertion_return.second ) - insertion_return.first->second = stake; + db.modify( *proxy_stat_range.first, + [&stake_id, stake]( voting_statistics_object& o ){ + o.proxy_for.emplace( stake_id, stake ); } ); } @@ -186,27 +292,74 @@ std::string voting_stat_plugin::plugin_name()const void voting_stat_plugin::plugin_set_program_options( boost::program_options::options_description& cli, - boost::program_options::options_description& cfg - ) + boost::program_options::options_description& cfg ) { + cli.add_options() + ( + "voting-stat-track-every-x-maint", boost::program_options::value(), + "Every x maintenance interval statistic objects will be created (12=2per day)" + ) + ( + "voting-stat-keep-objects-in-db", boost::program_options::value(), + "Every created object will be deleted after the maintenance interval (true)" + ) + ( + "voting-stat-track-worker-votes", boost::program_options::value(), + "Worker votes will be tracked (true)" + ) + ( + "voting-stat-track-witness-votes", boost::program_options::value(), + "Witness votes will be tracked (true)" + ) + ( + "voting-stat-track-committee-votes", boost::program_options::value(), + "Committee votes will be tracked (true)" + ); + cfg.add(cli); } void voting_stat_plugin::plugin_initialize(const boost::program_options::variables_map& options) { auto& db = database(); - db.add_index< primary_index< voting_statistics_index > >(); + db.add_index< primary_index >(); + db.add_index< primary_index >(); - db.on_voting_stake_calculated.connect( - [&](const account_object& stake_account, const account_object& proxy_account, - const uint64_t stake) { - my->on_stake_calculated( stake_account, proxy_account, stake ); - } + if( options.count("voting-stat-track-every-x-maint") ){ + my->_track_every_x_maint = options["voting-stat-track-every-x-maint"].as(); + if( my->_track_every_x_maint == 0 ) + my->_track_every_x_maint = 1; + my->_maint_counter = my->_track_every_x_maint; + } + if( options.count("voting-stat-keep-objects-in-db") ){ + my->_keep_objects_in_db = options["voting-stat-keep-objects-in-db"].as(); + } + if( options.count("voting-stat-track-worker-votes") ){ + my->_track_worker_votes = options["voting-stat-track-worker-votes"].as(); + } + if( options.count("voting-stat-track-witness-votes") ){ + my->_track_witness_votes = options["voting-stat-track-witness-votes"].as(); + } + if( options.count("voting-stat-track-committee-votes") ){ + my->_track_committee_votes = options["voting-stat-track-committee-votes"].as(); + } + + my->_on_voting_stake_calc_conn = db.on_voting_stake_calculated.connect( + [&]( const account_object& stake_account, const account_object& proxy_account, const uint64_t stake ){ + my->on_stake_calculated( stake_account, proxy_account, stake ); + } + ); + + my->_on_voting_stake_calc_block = std::unique_ptr( + new boost::signals2::shared_connection_block(my->_on_voting_stake_calc_conn) ); - - db.on_maintenance_begin.connect( - [&](const block_id_type& block_id){ my->on_maintenance_begin( block_id ); + + db.on_maintenance_begin.connect( [this](uint32_t block_num){ + my->on_maintenance_begin( block_num ); }); + db.on_maintenance_end.connect( [this](){ + my->on_maintenance_end(); + }); } void voting_stat_plugin::plugin_startup() From e741346c44e0e7d98e924ce6359d90af75a29b4c Mon Sep 17 00:00:00 2001 From: dimfred Date: Thu, 1 Aug 2019 15:57:43 +0200 Subject: [PATCH 46/46] tests --- tests/voting_stat/main.cpp | 704 ++++++++++++++++++++++++++++++------- 1 file changed, 577 insertions(+), 127 deletions(-) diff --git a/tests/voting_stat/main.cpp b/tests/voting_stat/main.cpp index f34403a2d7..28682d83a4 100644 --- a/tests/voting_stat/main.cpp +++ b/tests/voting_stat/main.cpp @@ -22,6 +22,8 @@ * THE SOFTWARE. */ #include +#include +#include #include @@ -34,6 +36,7 @@ #include #include +#include #include "../common/database_fixture.hpp" @@ -54,46 +57,24 @@ struct voting_stat_fixture : public database_fixture vote_id_type default_vote_id; CURL *_curl; ES _es; - + voting_stat_fixture() { - try + try { _curl = curl_easy_init(); _es.curl = _curl; _es.elasticsearch_url = "http://localhost:9200/"; - _es.index_prefix = "objects-"; - + _es.index_prefix = "objects-"; + app.register_plugin( true ); app.register_plugin( true ); - - auto es_object = app.get_plugin("es_objects"); - bpo::options_description cli; - bpo::options_description cfi; - es_object->plugin_set_program_options( cli, cfi ); - - bpo::variables_map var_map; - - int plugin_argc = 17; - const char* const plugin_argv[]{ "voting_stat", - "--es-objects-bulk-replay", "1", - "--es-objects-proposals", "false", - "--es-objects-accounts", "false", - "--es-objects-assets", "false", - "--es-objects-balances", "false", - "--es-objects-limit-orders", "false", - "--es-objects-asset-bitasset", "false", - "--es-objects-keep-only-current", "false" - }; - - bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfi ), var_map ); - app.initialize_plugins( var_map ); - } + } catch(fc::exception &e) { edump((e.to_detail_string() )); } - + }; void make_next_maintenance_interval() @@ -106,7 +87,7 @@ struct voting_stat_fixture : public database_fixture { witness_id_type wit_id = *db.get_global_properties().active_witnesses.begin(); default_vote_id = wit_id(db).vote_id; - + account_options opt; opt.votes = { default_vote_id }; opt.num_witness = opt.votes.size(); @@ -125,41 +106,47 @@ struct voting_stat_fixture : public database_fixture const voting_statistics_object& get_voting_statistics_object( account_id_type id ) { - const auto& idx = db.get_index_type().indices().get(); - return *idx.find( id ); + const auto& idx = db.get_index_type().indices().get(); + auto last_block = idx.rbegin()->block_number; + auto range = idx.equal_range( boost::make_tuple( last_block, id ) ); + if( range.first == range.second ) + BOOST_FAIL( "object could not be found, which should never happen" ); + return *range.first; } }; BOOST_FIXTURE_TEST_SUITE( voting_stat_tests, voting_stat_fixture ) -BOOST_AUTO_TEST_CASE( block_id_changes_with_each_interval ) +BOOST_AUTO_TEST_CASE( test_voting_statistics_object_tracking_without_proxy ) { try { - make_next_maintenance_interval(); - auto first_block_id = voting_statistics_object::block_id; + bpo::options_description cli; + bpo::options_description cfg; - make_next_maintenance_interval(); - auto second_block_id = voting_statistics_object::block_id; - BOOST_CHECK( first_block_id != second_block_id ); + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli, cfg ); - make_next_maintenance_interval(); - auto third_block_id = voting_statistics_object::block_id; - BOOST_CHECK( second_block_id != third_block_id ); + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "true" + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); -} FC_LOG_AND_RETHROW() } + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); -BOOST_AUTO_TEST_CASE( voting_statistics_without_proxy ) -{ try { ACTOR( alice ); - transfer( committee_account, alice_id, asset(1) ); set_account_options( alice_id ); - + + const auto& alice_acc = alice_id(db); BOOST_CHECK( *alice_acc.options.votes.begin() == default_vote_id ); BOOST_CHECK( alice_acc.options.voting_account == GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + transfer( committee_account, alice_id, asset(1) ); make_next_maintenance_interval(); const auto& alice_stat1 = get_voting_statistics_object( alice_id ); @@ -171,128 +158,316 @@ BOOST_AUTO_TEST_CASE( voting_statistics_without_proxy ) BOOST_CHECK( alice_stat1.get_total_voting_stake() == 1 ); - /* increase stake */ - transfer( committee_account, alice_id, asset(10) ); - + transfer( committee_account, alice_id, asset(1) ); make_next_maintenance_interval(); const auto& alice_stat2 = get_voting_statistics_object( alice_id ); BOOST_CHECK( alice_stat2.proxy == GRAPHENE_PROXY_TO_SELF_ACCOUNT ); BOOST_CHECK( !alice_stat2.has_proxy() ); BOOST_CHECK( alice_stat2.proxy_for.empty() ); - BOOST_CHECK( alice_stat2.stake == 11 ); - BOOST_CHECK( *alice_stat1.votes.begin() == default_vote_id ); - BOOST_CHECK( alice_stat1.get_total_voting_stake() == 11 ); + BOOST_CHECK( alice_stat2.stake == 2 ); + BOOST_CHECK( *alice_stat2.votes.begin() == default_vote_id ); + BOOST_CHECK( alice_stat2.get_total_voting_stake() == 2 ); } FC_LOG_AND_RETHROW() } -BOOST_AUTO_TEST_CASE( voting_statistics_with_proxy ) +BOOST_AUTO_TEST_CASE( test_voting_statistics_with_proxy_delete_after_interval ) { try { - /* clear es-db */ + + bpo::options_description cli_vs; + bpo::options_description cli_es; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli_vs, cfg ); + + auto es_objects = app.get_plugin("es_objects"); + es_objects->plugin_set_program_options( cli_es, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "false", + "--voting-stat-track-witness-votes", "false", + "--voting-stat-track-committee-votes", "false", + "--voting-stat-track-worker-votes", "false", + + "--es-objects-bulk-replay", "1", + "--es-objects-proposals", "false", + "--es-objects-accounts", "false", + "--es-objects-assets", "false", + "--es-objects-balances", "false", + "--es-objects-limit-orders", "false", + "--es-objects-asset-bitasset", "false", + "--es-objects-keep-only-current", "true", + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + auto objects_deleted = graphene::utilities::deleteAll(_es); if( !objects_deleted ) BOOST_FAIL( "elastic search DB could not be deleted" ); ACTORS( (alice)(bob)(charlie) ); - transfer( committee_account, alice_id, asset(1), asset() ); - transfer( committee_account, bob_id, asset(2), asset() ); - transfer( committee_account, charlie_id, asset(3), asset() ); - - /* proxy chain: alice => bob => charlie */ + transfer( committee_account, alice_id, asset(1) ); + transfer( committee_account, bob_id, asset(2) ); + transfer( committee_account, charlie_id, asset(3) ); + + // proxy chain: alice => bob => charlie set_account_options( alice_id, bob_id ); set_account_options( bob_id, charlie_id ); set_account_options( charlie_id ); - - make_next_maintenance_interval(); - const auto& alice_stat1 = get_voting_statistics_object( alice_id ); - const auto& bob_stat1 = get_voting_statistics_object( bob_id ); - const auto& charlie_stat1 = get_voting_statistics_object( charlie_id ); - - BOOST_CHECK( alice_stat1.has_proxy() ); - BOOST_CHECK( alice_stat1.proxy == bob_id ); - BOOST_CHECK( alice_stat1.get_total_voting_stake() == 0 ); - - BOOST_CHECK( bob_stat1.has_proxy() ); - BOOST_CHECK( bob_stat1.proxy == charlie_id ); - auto alice_proxied = *bob_stat1.proxy_for.begin(); - BOOST_CHECK( alice_proxied.first == alice_id && alice_proxied.second == 1 ); - BOOST_CHECK( bob_stat1.get_total_voting_stake() == 1 ); - - BOOST_CHECK( !charlie_stat1.has_proxy() ); - auto bob_proxied = *charlie_stat1.proxy_for.begin(); - BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); - BOOST_CHECK( charlie_stat1.get_total_voting_stake() == (2 + 3) ); - - - /* proxy: alice => alice; bob => charlie; */ - set_account_options( alice_id, GRAPHENE_PROXY_TO_SELF_ACCOUNT ); make_next_maintenance_interval(); - const auto& alice_stat2 = get_voting_statistics_object( alice_id ); - const auto& bob_stat2 = get_voting_statistics_object( bob_id ); - const auto& charlie_stat2 = get_voting_statistics_object( charlie_id ); - - BOOST_CHECK( !alice_stat2.has_proxy() ); - BOOST_CHECK( alice_stat2.proxy_for.empty() ); - BOOST_CHECK( alice_stat2.get_total_voting_stake() == 1 ); - - BOOST_CHECK( bob_stat2.has_proxy() ); - BOOST_CHECK( bob_stat2.proxy_for.empty() ); - BOOST_CHECK( bob_stat2.get_total_voting_stake() == 0 ); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat.has_proxy() ); + BOOST_CHECK( alice_stat.proxy == bob_id ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == 0 ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy == charlie_id ); + auto alice_proxied = *bob_stat.proxy_for.begin(); + BOOST_CHECK( alice_proxied.first == alice_id && alice_proxied.second == 1 ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 1 ); + + BOOST_CHECK( !charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == (2 + 3) ); + } - BOOST_CHECK( !charlie_stat2.has_proxy() ); - bob_proxied = *charlie_stat2.proxy_for.begin(); - BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); - BOOST_CHECK( charlie_stat2.get_total_voting_stake() == (2 + 3) ); + // proxy: alice => alice; bob => charlie; + set_account_options( alice_id, GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + make_next_maintenance_interval(); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat.has_proxy() ); + BOOST_CHECK( alice_stat.proxy_for.empty() ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == 1 ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy_for.empty() ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + + BOOST_CHECK( !charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == (2 + 3) ); + } - /* proxy: alice => alice; bob => charlie; charlie => alice; stake increase */ + // proxy: alice => alice; bob => charlie; charlie => alice; stake increase set_account_options( charlie_id, alice_id ); transfer( committee_account, alice_id, asset(10) ); transfer( committee_account, bob_id, asset(20) ); transfer( committee_account, charlie_id, asset(30) ); make_next_maintenance_interval(); - const auto& alice_stat3 = get_voting_statistics_object( alice_id ); - const auto& bob_stat3 = get_voting_statistics_object( bob_id ); - const auto& charlie_stat3 = get_voting_statistics_object( charlie_id ); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat.has_proxy() ); + auto charlie_proxied = *alice_stat.proxy_for.begin(); + BOOST_CHECK( charlie_proxied.first == charlie_id && charlie_proxied.second == 33 ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == (11 + 33) ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy_for.empty() ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + BOOST_CHECK( bob_stat.stake == 22 ); + + BOOST_CHECK( charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 22 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == 22 ); + } + + // only stake increase + transfer( committee_account, alice_id, asset(100) ); + transfer( committee_account, bob_id, asset(200) ); + transfer( committee_account, charlie_id, asset(300) ); - BOOST_CHECK( !alice_stat3.has_proxy() ); - auto charlie_proxied = *alice_stat3.proxy_for.begin(); - BOOST_CHECK( charlie_proxied.first == charlie_id && charlie_proxied.second == 33 ); - BOOST_CHECK( alice_stat3.get_total_voting_stake() == (11 + 33) ); + make_next_maintenance_interval(); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat.stake == 111 ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == (111 + 333) ); + BOOST_CHECK( bob_stat.stake == 222 ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + BOOST_CHECK( charlie_stat.stake == 333 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == 222 ); + } - BOOST_CHECK( bob_stat3.has_proxy() ); - BOOST_CHECK( bob_stat3.proxy_for.empty() ); - BOOST_CHECK( bob_stat3.get_total_voting_stake() == 0 ); + fc::usleep(fc::milliseconds(2000)); + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + _es.endpoint = _es.index_prefix + "*/data/_count"; + _es.query = query; + auto res = graphene::utilities::simpleQuery(_es); + variant j = fc::json::from_string(res); + BOOST_CHECK( j["count"].as_int64() == 12 ); - BOOST_CHECK( charlie_stat3.has_proxy() ); - bob_proxied = *charlie_stat2.proxy_for.begin(); - BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 22 ); - BOOST_CHECK( charlie_stat3.get_total_voting_stake() == 22 ); +} FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE( test_voting_statistics_with_proxy_keep_after_interval ) +{ try { - /* only stake increase */ + // this test is the same as above just that objects are not deleted with each interval + + bpo::options_description cli_vs; + bpo::options_description cli_es; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli_vs, cfg ); + + auto es_objects = app.get_plugin("es_objects"); + es_objects->plugin_set_program_options( cli_es, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "true", + "--voting-stat-track-witness-votes", "false", + "--voting-stat-track-committee-votes", "false", + "--voting-stat-track-worker-votes", "false", + + "--es-objects-bulk-replay", "1", + "--es-objects-proposals", "false", + "--es-objects-accounts", "false", + "--es-objects-assets", "false", + "--es-objects-balances", "false", + "--es-objects-limit-orders", "false", + "--es-objects-asset-bitasset", "false", + "--es-objects-keep-only-current", "true", + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + + auto objects_deleted = graphene::utilities::deleteAll(_es); + if( !objects_deleted ) + BOOST_FAIL( "elastic search DB could not be deleted" ); + + + ACTORS( (alice)(bob)(charlie) ); + transfer( committee_account, alice_id, asset(1) ); + transfer( committee_account, bob_id, asset(2) ); + transfer( committee_account, charlie_id, asset(3) ); + + // proxy chain: alice => bob => charlie + set_account_options( alice_id, bob_id ); + set_account_options( bob_id, charlie_id ); + set_account_options( charlie_id ); + + make_next_maintenance_interval(); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat.has_proxy() ); + BOOST_CHECK( alice_stat.proxy == bob_id ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == 0 ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy == charlie_id ); + auto alice_proxied = *bob_stat.proxy_for.begin(); + BOOST_CHECK( alice_proxied.first == alice_id && alice_proxied.second == 1 ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 1 ); + + BOOST_CHECK( !charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == (2 + 3) ); + } + + // proxy: alice => alice; bob => charlie; + set_account_options( alice_id, GRAPHENE_PROXY_TO_SELF_ACCOUNT ); + + make_next_maintenance_interval(); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat.has_proxy() ); + BOOST_CHECK( alice_stat.proxy_for.empty() ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == 1 ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy_for.empty() ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + + BOOST_CHECK( !charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 2 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == (2 + 3) ); + } + + // proxy: alice => alice; bob => charlie; charlie => alice; stake increase + set_account_options( charlie_id, alice_id ); + transfer( committee_account, alice_id, asset(10) ); + transfer( committee_account, bob_id, asset(20) ); + transfer( committee_account, charlie_id, asset(30) ); + + make_next_maintenance_interval(); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( !alice_stat.has_proxy() ); + auto charlie_proxied = *alice_stat.proxy_for.begin(); + BOOST_CHECK( charlie_proxied.first == charlie_id && charlie_proxied.second == 33 ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == (11 + 33) ); + + BOOST_CHECK( bob_stat.has_proxy() ); + BOOST_CHECK( bob_stat.proxy_for.empty() ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + BOOST_CHECK( bob_stat.stake == 22 ); + + BOOST_CHECK( charlie_stat.has_proxy() ); + auto bob_proxied = *charlie_stat.proxy_for.begin(); + BOOST_CHECK( bob_proxied.first == bob_id && bob_proxied.second == 22 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == 22 ); + } + + // only stake increase transfer( committee_account, alice_id, asset(100) ); transfer( committee_account, bob_id, asset(200) ); transfer( committee_account, charlie_id, asset(300) ); make_next_maintenance_interval(); - const auto& alice_stat4 = get_voting_statistics_object( alice_id ); - const auto& bob_stat4 = get_voting_statistics_object( bob_id ); - const auto& charlie_stat4 = get_voting_statistics_object( charlie_id ); - - BOOST_CHECK( alice_stat4.stake == 111 ); - BOOST_CHECK( alice_stat4.get_total_voting_stake() == (111 + 333) ); - BOOST_CHECK( bob_stat4.stake == 222 ); - BOOST_CHECK( bob_stat4.get_total_voting_stake() == 0 ); - BOOST_CHECK( charlie_stat4.stake == 333 ); - BOOST_CHECK( charlie_stat4.get_total_voting_stake() == 222 ); - - - /* wait for es */ - fc::usleep(fc::milliseconds(1000)); + { + const auto& alice_stat = get_voting_statistics_object( alice_id ); + const auto& bob_stat = get_voting_statistics_object( bob_id ); + const auto& charlie_stat = get_voting_statistics_object( charlie_id ); + + BOOST_CHECK( alice_stat.stake == 111 ); + BOOST_CHECK( alice_stat.get_total_voting_stake() == (111 + 333) ); + BOOST_CHECK( bob_stat.stake == 222 ); + BOOST_CHECK( bob_stat.get_total_voting_stake() == 0 ); + BOOST_CHECK( charlie_stat.stake == 333 ); + BOOST_CHECK( charlie_stat.get_total_voting_stake() == 222 ); + } + + fc::usleep(fc::milliseconds(2000)); string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; _es.endpoint = _es.index_prefix + "*/data/_count"; _es.query = query; @@ -302,4 +477,279 @@ BOOST_AUTO_TEST_CASE( voting_statistics_with_proxy ) } FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE( test_voteable_objects_tracking_with_es ) +{ try { + + bpo::options_description cli_vs; + bpo::options_description cli_es; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli_vs, cfg ); + + auto es_objects = app.get_plugin("es_objects"); + es_objects->plugin_set_program_options( cli_es, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "false", + "--voting-stat-track-worker-votes", "true", + "--voting-stat-track-witness-votes", "true", + "--voting-stat-track-committee-votes", "true", + + "--es-objects-voting-statistics", "true", + "--es-objects-voteable-statistics", "true", + "--es-objects-statistics-delete-allowed", "false", + + "--es-objects-bulk-replay", "1", + "--es-objects-proposals", "false", + "--es-objects-accounts", "false", + "--es-objects-assets", "false", + "--es-objects-balances", "false", + "--es-objects-limit-orders", "false", + "--es-objects-asset-bitasset", "false", + "--es-objects-keep-only-current", "true", + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + + auto objects_deleted = graphene::utilities::deleteAll(_es); + if( !objects_deleted ) + BOOST_FAIL( "elastic search DB could not be deleted" ); + + + ACTOR( alice ); + uint64_t alice_stake = 100; + upgrade_to_lifetime_member( alice_id ); + transfer( committee_account, alice_id, asset(alice_stake) ); + set_account_options( alice_id ); + + create_worker( alice_id ); + create_worker( alice_id ); + + + const auto& voteable_idx = db.get_index_type().indices().get(); + const auto& witness_idx = db.get_index_type().indices().get(); + const auto& committee_idx = db.get_index_type().indices().get(); + const auto& worker_idx = db.get_index_type().indices().get(); + + uint64_t num_witnesses = witness_idx.size(); + uint64_t num_committee_members = committee_idx.size(); + uint64_t num_workers = worker_idx.size(); + + uint64_t expected_voteables = num_witnesses + num_committee_members + num_workers; + + make_next_maintenance_interval(); + BOOST_CHECK( voteable_idx.size() == expected_voteables ); + auto last_block = voteable_idx.rbegin()->block_number; + auto default_voteable_votes = voteable_idx.equal_range( boost::make_tuple( last_block, default_vote_id) ) + .first->get_votes(); + BOOST_CHECK( default_voteable_votes == alice_stake ); + + make_next_maintenance_interval(); + BOOST_CHECK( voteable_idx.size() == expected_voteables ); + + + auto expected_objs_in_es = 2*(expected_voteables + 1); + + fc::usleep(fc::milliseconds(2000)); + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + _es.endpoint = _es.index_prefix + "*/data/_count"; + _es.query = query; + auto res = graphene::utilities::simpleQuery(_es); + variant j = fc::json::from_string(res); + uint64_t obj_count = j["count"].as_uint64(); + BOOST_CHECK( obj_count == expected_objs_in_es ); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE( test_voting_stat_plugin_track_every_x_interval ) +{ try { + + bpo::options_description cli; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "2", + "--voting-stat-keep-objects-in-db", "true" + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + + + ACTOR( alice ); + set_account_options( alice_id ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + const auto& alice_stat1 = get_voting_statistics_object( alice_id ); + BOOST_CHECK( alice_stat1.stake == 1 ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + const auto& alice_stat2 = get_voting_statistics_object( alice_id ); + // since this interval is even it should not be tracked + BOOST_CHECK( alice_stat2.stake == 1 ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + const auto& alice_stat3 = get_voting_statistics_object( alice_id ); + // this should result in the correct object since all odd intervals are tracked + BOOST_CHECK( alice_stat3.stake == 3 ); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE( test_delete_after_interval_and_pushed_to_es ) +{ try { + + bpo::options_description cli_vs; + bpo::options_description cli_es; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli_vs, cfg ); + + auto es_objects = app.get_plugin("es_objects"); + es_objects->plugin_set_program_options( cli_es, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "false", + + "--es-objects-voting-statistics", "true", + "--es-objects-voteable-statistics", "false", + "--es-objects-statistics-delete-allowed", "false", + + "--es-objects-bulk-replay", "1", + "--es-objects-proposals", "false", + "--es-objects-accounts", "false", + "--es-objects-assets", "false", + "--es-objects-balances", "false", + "--es-objects-limit-orders", "false", + "--es-objects-asset-bitasset", "false", + "--es-objects-keep-only-current", "true", + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + + auto objects_deleted = graphene::utilities::deleteAll(_es); + if( !objects_deleted ) + BOOST_FAIL( "elastic search DB could not be deleted" ); + + + const auto& idx = db.get_index_type().indices().get(); + ACTOR( alice ); + set_account_options( alice_id ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + BOOST_CHECK( idx.size() == 1 ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + BOOST_CHECK( idx.size() == 1 ); + + + transfer( committee_account, alice_id, asset(1) ); + make_next_maintenance_interval(); + BOOST_CHECK( idx.size() == 1 ); + + + fc::usleep(fc::milliseconds(2000)); + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + _es.endpoint = _es.index_prefix + "*/data/_count"; + _es.query = query; + auto res = graphene::utilities::simpleQuery(_es); + variant j = fc::json::from_string(res); + int64_t obj_count = j["count"].as_int64(); + BOOST_CHECK( obj_count == 3 ); + +} FC_LOG_AND_RETHROW() } + +// TODO REMOVE +BOOST_AUTO_TEST_CASE( test_indices ) +{ try { + + bpo::options_description cli_vs; + bpo::options_description cli_es; + bpo::options_description cfg; + + auto voting_stat = app.get_plugin("voting_stat"); + voting_stat->plugin_set_program_options( cli_vs, cfg ); + + const char* const plugin_argv[]{ "voting_stat", + "--voting-stat-track-every-x-maint", "1", + "--voting-stat-keep-objects-in-db", "true", + }; + int plugin_argc = sizeof(plugin_argv)/sizeof(char*); + + bpo::variables_map var_map; + bpo::store( bpo::parse_command_line( plugin_argc, plugin_argv, cfg ), var_map ); + app.initialize_plugins( var_map ); + + + ACTORS((alice)(bob)); + + transfer( committee_account, alice_id, asset(1) ); + transfer( committee_account, bob_id, asset(1) ); + make_next_maintenance_interval(); + + transfer( committee_account, alice_id, asset(1) ); + transfer( committee_account, bob_id, asset(1) ); + make_next_maintenance_interval(); + + transfer( committee_account, alice_id, asset(1) ); + transfer( committee_account, bob_id, asset(1) ); + make_next_maintenance_interval(); + + const auto& block_idx = db.get_index_type().indices().get(); + edump(("BLOCK")); + for( const auto& o : block_idx ) + edump((o.block_number)(o.account)); + + auto beg = block_idx.lower_bound( block_idx.begin()->block_number ); + auto end = block_idx.upper_bound( block_idx.begin()->block_number ); + + edump(("BLOCK BOUND")); + for(; beg != end; ++beg ) + edump((beg->block_number)(beg->account)); + + + edump(("RANGEPLAY")); + auto last_block = block_idx.begin()->block_number; + auto block_range = block_idx.equal_range( last_block ); + for( auto o = block_range.first; o != block_range.second; ++o ) { + edump((o->block_number)(o->account)); + } + + for( const auto& o : block_idx ) + db.remove(o); + + auto range = block_idx.equal_range( boost::make_tuple( last_block, alice_id ) ); + edump((range.first==range.second)); + + uint64_t delt = std::distance( range.first, range.second ); + edump((delt)); + + //edump((range.first->block_number)(range.first->account)); + +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END()