merge release to master

.travis.yml

@@ -5,14 +5,12 @@ cache: ccache
 git:
   depth: 1
 
-dist: trusty
+dist: xenial
 
 sudo: true
 
 install:
- - echo "deb http://de.archive.ubuntu.com/ubuntu xenial main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
- - sudo apt-get update
- - sudo apt-get install --allow-unauthenticated g++ libboost-all-dev cmake libreadline-dev libssl-dev autoconf parallel ccache
+ - sudo apt-get install --allow-unauthenticated libboost-thread-dev libboost-iostreams-dev libboost-date-time-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev libboost-chrono-dev libboost-test-dev libboost-context-dev libboost-regex-dev libboost-coroutine-dev cmake parallel
 
 addons:
   sonarcloud:
@@ -27,15 +25,21 @@ env:
     - CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
 
 script:
- - 'echo $((`date +%s` - 120)) >_start_time'
+ - programs/build_helpers/buildstep -s 3500
  - ccache -s
- - '( [ `ccache -s | grep "files in cache" | cut -c 20-` = 0 ] && touch _empty_cache ) || true'
- - sed -i '/tests/d' libraries/fc/CMakeLists.txt
- - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON .
- - 'which build-wrapper-linux-x86-64 && build-wrapper-linux-x86-64 --out-dir bw-output make -j 2 cli_wallet witness_node chain_test cli_test || make -j 2 cli_wallet witness_node chain_test cli_test'
+ - programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
+ - programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
+ - programs/build_helpers/buildstep make.cli_wallet 1600 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_wallet"
+ - programs/build_helpers/buildstep make.witness_node 300 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node"
+ - programs/build_helpers/buildstep make.serializer 45 "programs/build_helpers/make_with_sonar bw-output -j 2 js_operation_serializer"
+ - programs/build_helpers/buildstep make.get_dev_key 10 "programs/build_helpers/make_with_sonar bw-output -j 2 get_dev_key"
+ - programs/build_helpers/buildstep make.chain_test 900 "programs/build_helpers/make_with_sonar bw-output -j 2 chain_test"
+ - programs/build_helpers/buildstep make.cli_test 200 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_test"
+ - programs/build_helpers/buildstep make.perf_test 120 "programs/build_helpers/make_with_sonar bw-output -j 2 performance_test"
  - set -o pipefail
- - '[ -r _empty_cache ] || tests/chain_test 2>&1 | cat'
- - '[ -r _empty_cache ] || tests/cli_test 2>&1 | cat'
- - 'find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o "$d" "${d/CMakeFiles*.dir//}"/*.cpp; done >/dev/null'
- - '( [ -r _empty_cache -o $((`date +%s` - `cat _start_time`)) -gt $((42 * 60)) ] && echo "WARNING! Skipping sonar scanner due to time constraints!" ) || ( which sonar-scanner && sonar-scanner || true )'
- - '[ ! -r _empty_cache ] || ( echo "Please restart with populated cache" && false )'
+ - programs/build_helpers/buildstep run.chain_test 240 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test"
+ - programs/build_helpers/buildstep run.cli_test 30 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test"
+ - programs/build_helpers/buildstep prepare.sonar 20 'find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o "$d" "${d/CMakeFiles*.dir//}"/*.cpp; done >/dev/null'
+ - programs/build_helpers/buildstep run.sonar 400 "which sonar-scanner && sonar-scanner || true"
+ - programs/build_helpers/buildstep end 0
+ - ccache -s

CMakeLists.txt

@@ -48,12 +48,9 @@ LIST(APPEND BOOST_COMPONENTS thread
                              system
                              filesystem
                              program_options
-                             signals
-                             serialization
                              chrono
                              unit_test_framework
-                             context
-                             locale)
+                             context)
 SET( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 
 IF( WIN32 )

Dockerfile

@@ -10,16 +10,25 @@ RUN \
       cmake \
       git \
       libbz2-dev \
-      libreadline-dev \
-      libboost-all-dev \
       libcurl4-openssl-dev \
       libssl-dev \
       libncurses-dev \
+      libboost-thread-dev \
+      libboost-iostreams-dev \
+      libboost-date-time-dev \
+      libboost-system-dev \
+      libboost-filesystem-dev \
+      libboost-program-options-dev \
+      libboost-chrono-dev \
+      libboost-test-dev \
+      libboost-context-dev \
+      libboost-regex-dev \
+      libboost-coroutine-dev \
+      libtool \
       doxygen \
       ca-certificates \
+      fish \
     && \
-    apt-get update -y && \
-    apt-get install -y fish && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -38,9 +47,10 @@ RUN \
     git submodule update --init --recursive && \
     cmake \
         -DCMAKE_BUILD_TYPE=Release \
+	-DGRAPHENE_DISABLE_UNITY_BUILD=ON \
         . && \
-    make witness_node cli_wallet && \
-    install -s programs/witness_node/witness_node programs/cli_wallet/cli_wallet /usr/local/bin && \
+    make witness_node cli_wallet get_dev_key && \
+    install -s programs/witness_node/witness_node programs/genesis_util/get_dev_key programs/cli_wallet/cli_wallet /usr/local/bin && \
     #
     # Obtain version
     mkdir /etc/bitshares && \

README.md

@@ -80,7 +80,9 @@ In order to run a full node with all the account history you need to remove `par
 
 | Default | Full | Minimal  | ElasticSearch 
 | --- | --- | --- | ---
-| 100G HDD, 16G RAM | 200G HDD, 160G RAM | 80G HDD, 4G RAM | 500G SSD, 32G RAM
+| 100G HDD, 16G RAM | 640G SSD, 64G RAM * | 80G HDD, 4G RAM | 500G SSD, 32G RAM
+
+\* For this setup, allocate at least 500GB of SSD as swap.
 
 After starting the witness node again, in a separate terminal you can run:
 

libraries/README.md

@@ -0,0 +1,20 @@
+# BitShares Libraries
+
+The libraries are the core of the project and defines everything where applications can build on top.
+
+A **graphene** blockchain software will use the `app` library to define what the application will do, what services it will offer. The blockchain is defined by the `chain` library and include all the objects, types, operations, protocols that builds current consensus blockchain. The lowest level in memory database of Bitshares is developed at the `db` library. The `fc` is a helper module broadly used in the libraries code, `egenesis` will help with the genesis file, `plugins` will be loaded optionally to the application. Wallet software like the cli_wallet will benefit from the `wallet` library.
+
+Code in libraries is the most important part of **bitshares-core** project and it is maintained by the Bitshares Core Team and contributors.
+# Available Libraries
+
+Folder | Name | Description | Status
+---|---|---|---
+[app](app) | Application | Bundles component libraries (chain, network, plugins) into a useful application. Also provides API access. | Active 
+[chain](chain) | Blockchain | Defines all objects, operations and types. This include the consensus protocol, defines the whole blockchain behaviour. | Active 
+[db](db) | Database | Defines the internal database graphene uses. | Active 
+[egenesis](egenesis) | Genesis | Hardcodes the `genesis.json` file into the `witness_node` executable.| Active
+[fc](fc) | Fast-compiling C++ library | https://github.com/bitshares/bitshares-fc | Active 
+[net](net) | Network | The graphene p2p layer. | Active 
+[plugins](plugins) | Plugins | Collection of singleton designed modules used for extending the bitshares-core.  | Active 
+[utilities](utilities) | Utilities | Common utility calls used in applications or other libraries. | Active 
+[wallet](wallet) | Wallet | Wallet definition for the `cli_wallet` software. | Active

libraries/app/api.cpp

@@ -37,7 +37,6 @@
 #include <graphene/chain/worker_object.hpp>
 
 #include <fc/crypto/hex.hpp>
-#include <fc/smart_ref_impl.hpp>
 #include <fc/thread/future.hpp>
 
 namespace graphene { namespace app {
@@ -101,7 +100,7 @@ namespace graphene { namespace app {
        }
        else if( api_name == "asset_api" )
        {
-          _asset_api = std::make_shared< asset_api >( std::ref( *_app.chain_database() ) );
+          _asset_api = std::make_shared< asset_api >( _app );
        }
        else if( api_name == "orders_api" )
        {
@@ -159,18 +158,18 @@ namespace graphene { namespace app {
        }
     }
 
-    void network_broadcast_api::broadcast_transaction(const signed_transaction& trx)
+    void network_broadcast_api::broadcast_transaction(const precomputable_transaction& trx)
     {
-       trx.validate();
+       _app.chain_database()->precompute_parallel( trx ).wait();
        _app.chain_database()->push_transaction(trx);
        if( _app.p2p_node() != nullptr )
           _app.p2p_node()->broadcast_transaction(trx);
     }
 
-    fc::variant network_broadcast_api::broadcast_transaction_synchronous(const signed_transaction& trx)
+    fc::variant network_broadcast_api::broadcast_transaction_synchronous(const precomputable_transaction& trx)
     {
        fc::promise<fc::variant>::ptr prom( new fc::promise<fc::variant>() );
-       broadcast_transaction_with_callback( [=]( const fc::variant& v ){
+       broadcast_transaction_with_callback( [prom]( const fc::variant& v ){
         prom->set_value(v);
        }, trx );
 
@@ -179,14 +178,15 @@ namespace graphene { namespace app {
 
     void network_broadcast_api::broadcast_block( const signed_block& b )
     {
+       _app.chain_database()->precompute_parallel( b ).wait();
        _app.chain_database()->push_block(b);
        if( _app.p2p_node() != nullptr )
           _app.p2p_node()->broadcast( net::block_message( b ));
     }
 
-    void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const signed_transaction& trx)
+    void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const precomputable_transaction& trx)
     {
-       trx.validate();
+       _app.chain_database()->precompute_parallel( trx ).wait();
        _callbacks[trx.id()] = cb;
        _app.chain_database()->push_transaction(trx);
        if( _app.p2p_node() != nullptr )
@@ -283,10 +283,12 @@ namespace graphene { namespace app {
        return *_debug_api;
     }
 
-    vector<order_history_object> history_api::get_fill_order_history( asset_id_type a, asset_id_type b, uint32_t limit  )const
+    vector<order_history_object> history_api::get_fill_order_history( std::string asset_a, std::string asset_b, uint32_t limit  )const
     {
        FC_ASSERT(_app.chain_database());
        const auto& db = *_app.chain_database();
+       asset_id_type a = database_api.get_asset_id_from_string( asset_a );
+       asset_id_type b = database_api.get_asset_id_from_string( asset_b );
        if( a > b ) std::swap(a,b);
        const auto& history_idx = db.get_index_type<graphene::market_history::history_index>().indices().get<by_key>();
        history_key hkey;
@@ -344,7 +346,7 @@ namespace graphene { namespace app {
     }
 
     vector<operation_history_object> history_api::get_account_history_operations( const std::string account_id_or_name,
-                                                                       int operation_id,
+                                                                       int operation_type,
                                                                        operation_history_id_type start,
                                                                        operation_history_id_type stop,
                                                                        unsigned limit) const
@@ -367,17 +369,17 @@ namespace graphene { namespace app {
        {
           if( node->operation_id.instance.value <= start.instance.value ) {
 
-             if(node->operation_id(db).op.which() == operation_id)
+             if(node->operation_id(db).op.which() == operation_type)
                result.push_back( node->operation_id(db) );
-             }
+          }
           if( node->next == account_transaction_history_id_type() )
              node = nullptr;
           else node = &node->next(db);
        }
        if( stop.instance.value == 0 && result.size() < limit ) {
-          const account_transaction_history_object head = account_transaction_history_id_type()(db);
-          if( head.account == account && head.operation_id(db).op.which() == operation_id )
-             result.push_back(head.operation_id(db));
+          auto head = db.find(account_transaction_history_id_type());
+          if (head != nullptr && head->account == account && head->operation_id(db).op.which() == operation_type)
+            result.push_back(head->operation_id(db));
        }
        return result;
     }
@@ -442,11 +444,13 @@ namespace graphene { namespace app {
         return result;
     }
 
-    vector<bucket_object> history_api::get_market_history( asset_id_type a, asset_id_type b,
+    vector<bucket_object> history_api::get_market_history( std::string asset_a, std::string asset_b,
                                                            uint32_t bucket_seconds, fc::time_point_sec start, fc::time_point_sec end )const
     { try {
        FC_ASSERT(_app.chain_database());
        const auto& db = *_app.chain_database();
+       asset_id_type a = database_api.get_asset_id_from_string( asset_a );
+       asset_id_type b = database_api.get_asset_id_from_string( asset_b );
        vector<bucket_object> result;
        result.reserve(200);
 
@@ -466,7 +470,7 @@ namespace graphene { namespace app {
           ++itr;
        }
        return result;
-    } FC_CAPTURE_AND_RETHROW( (a)(b)(bucket_seconds)(start)(end) ) }
+    } FC_CAPTURE_AND_RETHROW( (asset_a)(asset_b)(bucket_seconds)(start)(end) ) }
 
     crypto_api::crypto_api(){};
 
@@ -525,12 +529,17 @@ namespace graphene { namespace app {
     }
 
     // asset_api
-    asset_api::asset_api(graphene::chain::database& db) : _db(db) { }
+    asset_api::asset_api(graphene::app::application& app) :
+         _db( *app.chain_database()), 
+         database_api( std::ref(*app.chain_database()), &(app.get_options()) 
+         ) { }
     asset_api::~asset_api() { }
 
-    vector<account_asset_balance> asset_api::get_asset_holders( asset_id_type asset_id, uint32_t start, uint32_t limit ) const {
+    vector<account_asset_balance> asset_api::get_asset_holders( std::string asset, uint32_t start, uint32_t limit ) const {
       FC_ASSERT(limit <= 100);
 
+      asset_id_type asset_id = database_api.get_asset_id_from_string( asset );
+
       const auto& bal_idx = _db.get_index_type< account_balance_index >().indices().get< by_asset_balance >();
       auto range = bal_idx.equal_range( boost::make_tuple( asset_id ) );
 
@@ -561,9 +570,10 @@ namespace graphene { namespace app {
       return result;
     }
     // get number of asset holders.
-    int asset_api::get_asset_holders_count( asset_id_type asset_id ) const {
+    int asset_api::get_asset_holders_count( std::string asset ) const {
 
       const auto& bal_idx = _db.get_index_type< account_balance_index >().indices().get< by_asset_balance >();
+      asset_id_type asset_id = database_api.get_asset_id_from_string( asset );      
       auto range = bal_idx.equal_range( boost::make_tuple( asset_id ) );
 
       int count = boost::distance(range) - 1;
@@ -606,8 +616,8 @@ namespace graphene { namespace app {
       return plugin->tracked_groups();
    }
 
-   vector< limit_order_group > orders_api::get_grouped_limit_orders( asset_id_type base_asset_id,
-                                                               asset_id_type quote_asset_id,
+   vector< limit_order_group > orders_api::get_grouped_limit_orders( std::string base_asset,
+                                                               std::string quote_asset,
                                                                uint16_t group,
                                                                optional<price> start,
                                                                uint32_t limit )const
@@ -618,6 +628,9 @@ namespace graphene { namespace app {
       const auto& limit_groups = plugin->limit_order_groups();
       vector< limit_order_group > result;
 
+      asset_id_type base_asset_id = database_api.get_asset_id_from_string( base_asset );
+      asset_id_type quote_asset_id = database_api.get_asset_id_from_string( quote_asset );
+
       price max_price = price::max( base_asset_id, quote_asset_id );
       price min_price = price::min( base_asset_id, quote_asset_id );
       if( start.valid() && !start->is_null() )