Merge pull request #8050 from EOSIO/global-greylist-2.0

Add greylist limit - v2.0.x

libraries/chain/controller.cpp

@@ -1653,11 +1653,10 @@ struct controller_impl {
       // Update resource limits:
       resource_limits.process_account_limit_updates();
       const auto& chain_config = self.get_global_properties().configuration;
-      uint32_t max_virtual_mult = 1000;
       uint64_t CPU_TARGET = EOS_PERCENT(chain_config.max_block_cpu_usage, chain_config.target_block_cpu_usage_pct);
       resource_limits.set_block_parameters(
-         { CPU_TARGET, chain_config.max_block_cpu_usage, config::block_cpu_usage_average_window_ms / config::block_interval_ms, max_virtual_mult, {99, 100}, {1000, 999}},
-         {EOS_PERCENT(chain_config.max_block_net_usage, chain_config.target_block_net_usage_pct), chain_config.max_block_net_usage, config::block_size_average_window_ms / config::block_interval_ms, max_virtual_mult, {99, 100}, {1000, 999}}
+         { CPU_TARGET, chain_config.max_block_cpu_usage, config::block_cpu_usage_average_window_ms / config::block_interval_ms, config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}},
+         {EOS_PERCENT(chain_config.max_block_net_usage, chain_config.target_block_net_usage_pct), chain_config.max_block_net_usage, config::block_size_average_window_ms / config::block_interval_ms, config::maximum_elastic_resource_multiplier, {99, 100}, {1000, 999}}
       );
       resource_limits.process_block_usage(pbhs.block_num);
 
@@ -3131,6 +3130,20 @@ void controller::set_subjective_cpu_leeway(fc::microseconds leeway) {
    my->subjective_cpu_leeway = leeway;
 }
 
+void controller::set_greylist_limit( uint32_t limit ) {
+   EOS_ASSERT( 0 < limit && limit <= chain::config::maximum_elastic_resource_multiplier,
+               misc_exception,
+               "Invalid limit (${limit}) passed into set_greylist_limit. "
+               "Must be between 1 and ${max}.",
+               ("limit", limit)("max", chain::config::maximum_elastic_resource_multiplier)
+   );
+   my->conf.greylist_limit = limit;
+}
+
+uint32_t controller::get_greylist_limit()const {
+   return my->conf.greylist_limit;
+}
+
 void controller::add_resource_greylist(const account_name &name) {
    my->conf.resource_greylist.insert(name);
 }

libraries/chain/include/eosio/chain/config.hpp

@@ -48,6 +48,7 @@ static const uint32_t account_cpu_usage_average_window_ms  = 24*60*60*1000l;
 static const uint32_t account_net_usage_average_window_ms  = 24*60*60*1000l;
 static const uint32_t block_cpu_usage_average_window_ms    = 60*1000l;
 static const uint32_t block_size_average_window_ms         = 60*1000l;
+static const uint32_t maximum_elastic_resource_multiplier  = 1000;
 
 //const static uint64_t   default_max_storage_size       = 10 * 1024;
 //const static uint32_t   default_max_trx_runtime        = 10*1000;

libraries/chain/include/eosio/chain/controller.hpp

@@ -94,6 +94,7 @@ namespace eosio { namespace chain {
 
             flat_set<account_name>   resource_greylist;
             flat_set<account_name>   trusted_producers;
+            uint32_t                 greylist_limit         = chain::config::maximum_elastic_resource_multiplier;
          };
 
          enum class block_status {
@@ -282,6 +283,8 @@ namespace eosio { namespace chain {
          validation_mode get_validation_mode()const;
 
          void set_subjective_cpu_leeway(fc::microseconds leeway);
+         void set_greylist_limit( uint32_t limit );
+         uint32_t get_greylist_limit()const;
 
          void add_to_ram_correction( account_name account, uint64_t ram_bytes );
          bool all_subjective_mitigations_disabled()const;
@@ -350,21 +353,3 @@ namespace eosio { namespace chain {
    };
 
 } }  /// eosio::chain
-
-FC_REFLECT( eosio::chain::controller::config,
-            (actor_whitelist)
-            (actor_blacklist)
-            (contract_whitelist)
-            (contract_blacklist)
-            (blocks_dir)
-            (state_dir)
-            (state_size)
-            (reversible_cache_size)
-            (read_only)
-            (force_all_checks)
-            (disable_replay_opts)
-            (contracts_console)
-            (wasm_runtime)
-            (resource_greylist)
-            (trusted_producers)
-          )

libraries/chain/include/eosio/chain/resource_limits.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <eosio/chain/exceptions.hpp>
 #include <eosio/chain/types.hpp>
+#include <eosio/chain/config.hpp>
 #include <eosio/chain/snapshot.hpp>
 #include <chainbase/chainbase.hpp>
 #include <set>
@@ -87,11 +88,11 @@ namespace eosio { namespace chain { namespace resource_limits {
          uint64_t get_block_cpu_limit() const;
          uint64_t get_block_net_limit() const;
 
-         int64_t get_account_cpu_limit( const account_name& name, bool elastic = true) const;
-         int64_t get_account_net_limit( const account_name& name, bool elastic = true) const;
+         std::pair<int64_t, bool> get_account_cpu_limit( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const;
+         std::pair<int64_t, bool> get_account_net_limit( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const;
 
-         account_resource_limit get_account_cpu_limit_ex( const account_name& name, bool elastic = true) const;
-         account_resource_limit get_account_net_limit_ex( const account_name& name, bool elastic = true) const;
+         std::pair<account_resource_limit, bool> get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const;
+         std::pair<account_resource_limit, bool> get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const;
 
          int64_t get_account_ram_usage( const account_name& name ) const;
 

libraries/chain/include/eosio/chain/resource_limits_private.hpp

@@ -233,7 +233,7 @@ namespace eosio { namespace chain { namespace resource_limits {
        * real maximum block is less, this virtual number is only used for rate limiting users.
        *
        * It's lowest possible value is max_block_size * blocksize_average_window_ms / block_interval
-       * It's highest possible value is 1000 times its lowest possible value
+       * It's highest possible value is config::maximum_elastic_resource_multiplier (1000) times its lowest possible value
        *
        * This means that the most an account can consume during idle periods is 1000x the bandwidth
        * it is gauranteed under congestion.

libraries/chain/resource_limits.cpp

@@ -365,12 +365,12 @@ uint64_t resource_limits_manager::get_block_net_limit() const {
    return config.net_limit_parameters.max - state.pending_net_usage;
 }
 
-int64_t resource_limits_manager::get_account_cpu_limit( const account_name& name, bool elastic ) const {
-   auto arl = get_account_cpu_limit_ex(name, elastic);
-   return arl.available;
+std::pair<int64_t, bool> resource_limits_manager::get_account_cpu_limit( const account_name& name, uint32_t greylist_limit ) const {
+   auto [arl, greylisted] = get_account_cpu_limit_ex(name, greylist_limit);
+   return {arl.available, greylisted};
 }
 
-account_resource_limit resource_limits_manager::get_account_cpu_limit_ex( const account_name& name, bool elastic) const {
+std::pair<account_resource_limit, bool> resource_limits_manager::get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit ) const {
 
    const auto& state = _db.get<resource_limits_state_object>();
    const auto& usage = _db.get<resource_usage_object, by_owner>(name);
@@ -380,14 +380,23 @@ account_resource_limit resource_limits_manager::get_account_cpu_limit_ex( const
    get_account_limits( name, x, y, cpu_weight );
 
    if( cpu_weight < 0 || state.total_cpu_weight == 0 ) {
-      return { -1, -1, -1 };
+      return {{ -1, -1, -1 }, false};
    }
 
    account_resource_limit arl;
 
    uint128_t window_size = config.account_cpu_usage_average_window;
+   uint64_t  greylisted_virtual_cpu_limit = config.cpu_limit_parameters.max * greylist_limit;
+
+   bool greylisted = false;
+   uint128_t virtual_cpu_capacity_in_window = window_size;
+   if( greylisted_virtual_cpu_limit < state.virtual_cpu_limit ) {
+      virtual_cpu_capacity_in_window *= greylisted_virtual_cpu_limit;
+      greylisted = true;
+   } else {
+      virtual_cpu_capacity_in_window *= state.virtual_cpu_limit;
+   }
 
-   uint128_t virtual_cpu_capacity_in_window = (uint128_t)(elastic ? state.virtual_cpu_limit : config.cpu_limit_parameters.max) * window_size;
    uint128_t user_weight     = (uint128_t)cpu_weight;
    uint128_t all_user_weight = (uint128_t)state.total_cpu_weight;
 
@@ -401,15 +410,15 @@ account_resource_limit resource_limits_manager::get_account_cpu_limit_ex( const
 
    arl.used = impl::downgrade_cast<int64_t>(cpu_used_in_window);
    arl.max = impl::downgrade_cast<int64_t>(max_user_use_in_window);
-   return arl;
+   return {arl, greylisted};
 }
 
-int64_t resource_limits_manager::get_account_net_limit( const account_name& name, bool elastic) const {
-   auto arl = get_account_net_limit_ex(name, elastic);
-   return arl.available;
+std::pair<int64_t, bool> resource_limits_manager::get_account_net_limit( const account_name& name, uint32_t greylist_limit ) const {
+   auto [arl, greylisted] = get_account_net_limit_ex(name, greylist_limit);
+   return {arl.available, greylisted};
 }
 
-account_resource_limit resource_limits_manager::get_account_net_limit_ex( const account_name& name, bool elastic) const {
+std::pair<account_resource_limit, bool> resource_limits_manager::get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit ) const {
    const auto& config = _db.get<resource_limits_config_object>();
    const auto& state  = _db.get<resource_limits_state_object>();
    const auto& usage  = _db.get<resource_usage_object, by_owner>(name);
@@ -418,18 +427,26 @@ account_resource_limit resource_limits_manager::get_account_net_limit_ex( const
    get_account_limits( name, x, net_weight, y );
 
    if( net_weight < 0 || state.total_net_weight == 0) {
-      return { -1, -1, -1 };
+      return {{ -1, -1, -1 }, false};
    }
 
    account_resource_limit arl;
 
    uint128_t window_size = config.account_net_usage_average_window;
+   uint64_t  greylisted_virtual_net_limit = config.net_limit_parameters.max * greylist_limit;
+
+   bool greylisted = false;
+   uint128_t virtual_network_capacity_in_window = window_size;
+   if( greylisted_virtual_net_limit < state.virtual_net_limit ) {
+      virtual_network_capacity_in_window *= greylisted_virtual_net_limit;
+      greylisted = true;
+   } else {
+      virtual_network_capacity_in_window *= state.virtual_net_limit;
+   }
 
-   uint128_t virtual_network_capacity_in_window = (uint128_t)(elastic ? state.virtual_net_limit : config.net_limit_parameters.max) * window_size;
    uint128_t user_weight     = (uint128_t)net_weight;
    uint128_t all_user_weight = (uint128_t)state.total_net_weight;
 
-
    auto max_user_use_in_window = (virtual_network_capacity_in_window * user_weight) / all_user_weight;
    auto net_used_in_window  = impl::integer_divide_ceil((uint128_t)usage.net_usage.value_ex * window_size, (uint128_t)config::rate_limiting_precision);
 
@@ -440,7 +457,7 @@ account_resource_limit resource_limits_manager::get_account_net_limit_ex( const
 
    arl.used = impl::downgrade_cast<int64_t>(net_used_in_window);
    arl.max = impl::downgrade_cast<int64_t>(max_user_use_in_window);
-   return arl;
+   return {arl, greylisted};
 }
 
 } } } /// eosio::chain::resource_limits

libraries/chain/transaction_context.cpp

@@ -481,17 +481,26 @@ namespace eosio { namespace chain {
       int64_t account_cpu_limit = large_number_no_overflow;
       bool greylisted_net = false;
       bool greylisted_cpu = false;
+
+      uint32_t specified_greylist_limit = control.get_greylist_limit();
       for( const auto& a : bill_to_accounts ) {
-         bool elastic = force_elastic_limits || !(control.is_producing_block() && control.is_resource_greylisted(a));
-         auto net_limit = rl.get_account_net_limit(a, elastic);
+         uint32_t greylist_limit = config::maximum_elastic_resource_multiplier;
+         if( !force_elastic_limits && control.is_producing_block() ) {
+            if( control.is_resource_greylisted(a) ) {
+               greylist_limit = 1;
+            } else {
+               greylist_limit = specified_greylist_limit;
+            }
+         }
+         auto [net_limit, net_was_greylisted] = rl.get_account_net_limit(a, greylist_limit);
          if( net_limit >= 0 ) {
             account_net_limit = std::min( account_net_limit, net_limit );
-            if (!elastic) greylisted_net = true;
+            greylisted_net |= net_was_greylisted;
          }
-         auto cpu_limit = rl.get_account_cpu_limit(a, elastic);
+         auto [cpu_limit, cpu_was_greylisted] = rl.get_account_cpu_limit(a, greylist_limit);
          if( cpu_limit >= 0 ) {
             account_cpu_limit = std::min( account_cpu_limit, cpu_limit );
-            if (!elastic) greylisted_cpu = true;
+            greylisted_cpu |= cpu_was_greylisted;
          }
       }
 

plugins/chain_plugin/chain_plugin.cpp

@@ -1615,7 +1615,7 @@ string convert_to_string(const chain::key256_t& source, const string& key_type,
          return std::string(val);
       }
       EOS_ASSERT( false, chain_type_exception, "Incompatible key_type and encode_type for key256_t next_key" );
-     
+
    } FC_RETHROW_EXCEPTIONS(warn, "Could not convert ${desc} source '${source}' to string.", ("desc", desc)("source",source) )
 }
 
@@ -2328,9 +2328,9 @@ read_only::get_account_results read_only::get_account( const get_account_params&
    result.last_code_update = accnt_metadata_obj.last_code_update;
    result.created          = accnt_obj.creation_date;
 
-   bool grelisted = db.is_resource_greylisted(result.account_name);
-   result.net_limit = rm.get_account_net_limit_ex( result.account_name, !grelisted);
-   result.cpu_limit = rm.get_account_cpu_limit_ex( result.account_name, !grelisted);
+   uint32_t greylist_limit = db.is_resource_greylisted(result.account_name) ? 1 : config::maximum_elastic_resource_multiplier;
+   result.net_limit = rm.get_account_net_limit_ex( result.account_name, greylist_limit).first;
+   result.cpu_limit = rm.get_account_cpu_limit_ex( result.account_name, greylist_limit).first;
    result.ram_usage = rm.get_account_ram_usage( result.account_name );
 
    const auto& permissions = d.get_index<permission_index,by_owner>();

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -14,13 +14,14 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    APPBASE_PLUGIN_REQUIRES((chain_plugin)(http_client_plugin))
 
    struct runtime_options {
-      fc::optional<int32_t> max_transaction_time;
-      fc::optional<int32_t> max_irreversible_block_age;
-      fc::optional<int32_t> produce_time_offset_us;
-      fc::optional<int32_t> last_block_time_offset_us;
-      fc::optional<int32_t> max_scheduled_transaction_time_per_block_ms;
-      fc::optional<int32_t> subjective_cpu_leeway_us;
-      fc::optional<double>  incoming_defer_ratio;
+      fc::optional<int32_t>   max_transaction_time;
+      fc::optional<int32_t>   max_irreversible_block_age;
+      fc::optional<int32_t>   produce_time_offset_us;
+      fc::optional<int32_t>   last_block_time_offset_us;
+      fc::optional<int32_t>   max_scheduled_transaction_time_per_block_ms;
+      fc::optional<int32_t>   subjective_cpu_leeway_us;
+      fc::optional<double>    incoming_defer_ratio;
+      fc::optional<uint32_t>  greylist_limit;
    };
 
    struct whitelist_blacklist {
@@ -115,7 +116,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
 
 } //eosio
 
-FC_REFLECT(eosio::producer_plugin::runtime_options, (max_transaction_time)(max_irreversible_block_age)(produce_time_offset_us)(last_block_time_offset_us)(max_scheduled_transaction_time_per_block_ms)(subjective_cpu_leeway_us)(incoming_defer_ratio));
+FC_REFLECT(eosio::producer_plugin::runtime_options, (max_transaction_time)(max_irreversible_block_age)(produce_time_offset_us)(last_block_time_offset_us)(max_scheduled_transaction_time_per_block_ms)(subjective_cpu_leeway_us)(incoming_defer_ratio)(greylist_limit));
 FC_REFLECT(eosio::producer_plugin::greylist_params, (accounts));
 FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_blacklist)(contract_whitelist)(contract_blacklist)(action_blacklist)(key_blacklist) )
 FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash))

plugins/producer_plugin/producer_plugin.cpp

@@ -646,6 +646,8 @@ void producer_plugin::set_program_options(
           "Limits the maximum time (in milliseconds) that is allowed for sending blocks to a keosd provider for signing")
          ("greylist-account", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "account that can not access to extended CPU/NET virtual resources")
+         ("greylist-limit", boost::program_options::value<uint32_t>()->default_value(1000),
+          "Limit (between 1 and 1000) on the multiple that CPU/NET virtual resources can extend during low usage (only enforced subjectively; use 1000 to not enforce any limit)")
          ("produce-time-offset-us", boost::program_options::value<int32_t>()->default_value(0),
           "offset of non last block producing time in microseconds. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later")
          ("last-block-time-offset-us", boost::program_options::value<int32_t>()->default_value(0),
@@ -867,6 +869,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       add_greylist_accounts(param);
    }
 
+   {
+      uint32_t greylist_limit = options.at("greylist-limit").as<uint32_t>();
+      chain.set_greylist_limit( greylist_limit );
+   }
+
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -955,6 +962,7 @@ bool producer_plugin::paused() const {
 }
 
 void producer_plugin::update_runtime_options(const runtime_options& options) {
+   chain::controller& chain = my->chain_plug->chain();
    bool check_speculating = false;
 
    if (options.max_transaction_time) {
@@ -983,15 +991,17 @@ void producer_plugin::update_runtime_options(const runtime_options& options) {
    }
 
    if (check_speculating && my->_pending_block_mode == pending_block_mode::speculating) {
-      chain::controller& chain = my->chain_plug->chain();
       my->_unapplied_transactions.add_aborted( chain.abort_block() );
       my->schedule_production_loop();
    }
 
    if (options.subjective_cpu_leeway_us) {
-      chain::controller& chain = my->chain_plug->chain();
       chain.set_subjective_cpu_leeway(fc::microseconds(*options.subjective_cpu_leeway_us));
    }
+
+   if (options.greylist_limit) {
+      chain.set_greylist_limit(*options.greylist_limit);
+   }
 }
 
 producer_plugin::runtime_options producer_plugin::get_runtime_options() const {