Merge remote-tracking branch 'origin/develop' into bg-task-init

.github/ISSUE_TEMPLATE/bug-report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Please tell us if you encountered an issue with Golem
 title: ''
-labels: brass, bug
+labels: P3, brass, bug
 assignees: ZmijaWA
 
 ---
@@ -21,6 +21,12 @@ assignees: ZmijaWA
 
 **Mainnet/Testnet**:
 
+**Priority label is set to the lowest by default. To setup higher priority please change the label**
+_P0 label is set for Severity-Critical/Effort-easy
+P1 label is set  for Severity-Critical/Effort-hard
+P2 label is set for Severity-Low/ Effort-easy
+P3 label is set for Severity-Low/Effort-hard_
+
 **Description of the issue**:
 
 _A clear and concise description of what went wrong, in which component, when and where._

.github/ISSUE_TEMPLATE/new-feature.md

@@ -2,7 +2,7 @@
 name: New feature
 about: 'Define a new feature / change for Golem '
 title: ''
-labels: ''
+labels: P3
 assignees: ''
 
 ---
@@ -99,6 +99,13 @@ _Add test scenarios for the new feature_
 * [ ] Concent integration tests pass
 * [ ] Concent acceptance tests pass
 
+
+**Please choose the priority label for QA. It is set to the lowest by default. To setup higher priority please change the label**
+_P0 label is set for Severity-Critical/Effort-easy
+P1 label is set  for Severity-Critical/Effort-hard
+P2 label is set for Severity-Low/ Effort-easy
+P3 label is set for Severity-Low/Effort-hard_
+
 ### QA team
 * [ ] Base scenario passes
 * [ ] Additional test 1 passes... 

golem/appconfig.py

@@ -8,7 +8,6 @@
 from golem.config.active import ENABLE_TALKBACK
 from golem.clientconfigdescriptor import ClientConfigDescriptor
 from golem.core.simpleconfig import SimpleConfig, ConfigEntry
-from golem.core.variables import KEY_DIFFICULTY
 
 from golem.ranking.helper.trust_const import \
     REQUESTING_TRUST, \
@@ -151,7 +150,6 @@ def load_config(cls, datadir, cfg_file_name=CONFIG_FILENAME):
             seed_port=START_PORT,
             seeds="",
             opt_peer_num=OPTIMAL_PEER_NUM,
-            key_difficulty=KEY_DIFFICULTY,
             # flags
             in_shutdown=0,
             accept_tasks=ACCEPT_TASKS,

golem/client.py

@@ -753,10 +753,6 @@ def get_dir_manager(self):
     def get_key_id(self):
         return self.keys_auth.key_id
 
-    @rpc_utils.expose('crypto.difficulty')
-    def get_difficulty(self):
-        return self.keys_auth.get_difficulty()
-
     @rpc_utils.expose('net.ident.key')
     def get_node_key(self):
         key = self.node.key

golem/clientconfigdescriptor.py

@@ -1,8 +1,6 @@
 import logging
 import typing
 
-from golem.core.variables import KEY_DIFFICULTY
-
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +19,6 @@ def __init__(self):
         self.send_pings = 0
         self.pings_interval = 0.0
         self.use_ipv6 = 0
-        self.key_difficulty = 0
         self.use_upnp = 0
         self.enable_talkback = 0
         self.enable_monitor = 0
@@ -108,7 +105,6 @@ class ConfigApprover(object):
     to_int_opt = {
         'seed_port', 'num_cores', 'opt_peer_num', 'p2p_session_timeout',
         'task_session_timeout', 'pings_interval', 'max_results_sending_delay',
-        'key_difficulty',
     }
     to_big_int_opt = {
         'min_price', 'max_price',
@@ -117,7 +113,6 @@ class ConfigApprover(object):
         'getting_peers_interval', 'getting_tasks_interval', 'computing_trust',
         'requesting_trust'
     }
-    max_opt = {'key_difficulty': KEY_DIFFICULTY}
 
     def __init__(self, config_desc):
         """ Create config approver class that keeps old config descriptor
@@ -129,7 +124,6 @@ def __init__(self, config_desc):
             (self.to_int_opt, self._to_int),
             (self.to_big_int_opt, self._to_int),
             (self.to_float_opt, self._to_float),
-            (self.max_opt, self._max_value)
         ]
         self.config_desc = config_desc
 
@@ -188,16 +182,3 @@ def _to_float(val, name):
         except ValueError:
             logger.warning("{} value '{}' is not a number".format(name, val))
         return val
-
-    @classmethod
-    def _max_value(cls, val, name):
-        """Try to set a maximum numeric value of val or the default value.
-        :param val: value that should be changed to float
-        :param str name: name of a config description option for logs
-        :return: max(val, min_value) or unchanged value if it's not possible
-        """
-        try:
-            return max(val, cls.max_opt[name])
-        except (KeyError, ValueError):
-            logger.warning('Cannot apply a minimum value to %r', name)
-        return val

golem/core/keysauth.py

@@ -53,9 +53,8 @@ class WrongPassword(Exception):
 class KeysAuth:
     """
     Elliptical curves cryptographic authorization manager. Generates
-    private and public keys based on ECC (curve secp256k1) with specified
-    difficulty. Private key is stored in file. When this file not exist, is
-    broken or contain key below requested difficulty new key is generated.
+    private and public keys based on ECC (curve secp256k1). Private key is
+    stored in file. When this file not exist or is broken new key is generated.
     """
     KEYS_SUBDIR = 'keys'
 
@@ -64,29 +63,28 @@ class KeysAuth:
     key_id: str = ""
     ecc: ECCx = None
 
-    def __init__(self, datadir: str, private_key_name: str, password: str,
-                 difficulty: int = 0) -> None:
+    def __init__(
+            self,
+            datadir: str,
+            private_key_name: str,
+            password: str,
+    ) -> None:
         """
         Create new ECC keys authorization manager, load or create keys.
 
         :param datadir where to store files
         :param private_key_name: name of the file containing private key
         :param password: user password to protect private key
-        :param difficulty:
-            desired key difficulty level. It's a number of leading zeros in
-            binary representation of public key. Value in range <0, 255>.
-            0 accepts all keys, 255 is nearly impossible.
         """
 
         prv, pub = KeysAuth._load_or_generate_keys(
-            datadir, private_key_name, password, difficulty)
+            datadir, private_key_name, password)
 
         self._private_key = prv
         self.ecc = ECCx(prv)
         self.public_key = pub
         self.key_id = encode_hex(pub)[2:]
         self.eth_addr = pubkey_to_address(pub)
-        self.difficulty = KeysAuth.get_difficulty(self.key_id)
 
     @staticmethod
     def key_exists(datadir: str, private_key_name: str) -> bool:
@@ -95,23 +93,25 @@ def key_exists(datadir: str, private_key_name: str) -> bool:
         return os.path.isfile(priv_key_path)
 
     @staticmethod
-    def _load_or_generate_keys(datadir: str, filename: str, password: str,
-                               difficulty: int) -> Tuple[bytes, bytes]:
+    def _load_or_generate_keys(
+            datadir: str,
+            filename: str,
+            password: str,
+    ) -> Tuple[bytes, bytes]:
         keys_dir = KeysAuth._get_or_create_keys_dir(datadir)
         priv_key_path = os.path.join(keys_dir, filename)
 
         loaded_keys = KeysAuth._load_and_check_keys(
             priv_key_path,
             password,
-            difficulty,
         )
 
         if loaded_keys:
             logger.debug('Existing keys loaded')
             priv_key, pub_key = loaded_keys
         else:
             logger.debug('No keys found, generating new one')
-            priv_key, pub_key = KeysAuth._generate_keys(difficulty)
+            priv_key, pub_key = KeysAuth._generate_keys()
             logger.debug('Generation completed, saving keys')
             KeysAuth._save_private_key(priv_key, priv_key_path, password)
             logger.debug('Keys stored succesfully')
@@ -126,9 +126,10 @@ def _get_or_create_keys_dir(datadir: str) -> str:
         return keys_dir
 
     @staticmethod
-    def _load_and_check_keys(priv_key_path: str,
-                             password: str,
-                             difficulty: int) -> Optional[Tuple[bytes, bytes]]:
+    def _load_and_check_keys(
+            priv_key_path: str,
+            password: str,
+    ) -> Optional[Tuple[bytes, bytes]]:
         try:
             with open(priv_key_path, 'r') as f:
                 keystore = f.read()
@@ -143,29 +144,13 @@ def _load_and_check_keys(priv_key_path: str,
 
         pub_key = privtopub(priv_key)
 
-        if not KeysAuth.is_pubkey_difficult(pub_key, difficulty):
-            raise Exception("Loaded key is not difficult enough")
-
         return priv_key, pub_key
 
     @staticmethod
-    def _generate_keys(difficulty: int) -> Tuple[bytes, bytes]:
-        from twisted.internet import reactor
-        reactor_started = reactor.running
+    def _generate_keys() -> Tuple[bytes, bytes]:
         logger.info("Generating new key pair")
-        started = time.time()
-        while True:
-            priv_key = mk_privkey(str(get_random_float()))
-            pub_key = privtopub(priv_key)
-            if KeysAuth.is_pubkey_difficult(pub_key, difficulty):
-                break
-
-            # lets be responsive to reactor stop (eg. ^C hit by user)
-            if reactor_started and not reactor.running:
-                logger.warning("reactor stopped, aborting key generation ..")
-                raise Exception("aborting key generation")
-
-        logger.info("Keys generated in %.2fs", time.time() - started)
+        priv_key = mk_privkey(str(get_random_float()))
+        pub_key = privtopub(priv_key)
         return priv_key, pub_key
 
     @staticmethod
@@ -178,29 +163,6 @@ def _save_private_key(key, key_path, password: str):
         with open(key_path, 'w') as f:
             f.write(json.dumps(keystore))
 
-    @staticmethod
-    def _count_max_hash(difficulty: int) -> int:
-        return 2 << (256 - difficulty - 1)
-
-    @staticmethod
-    def is_pubkey_difficult(pub_key: Union[bytes, str],
-                            difficulty: int) -> bool:
-        if isinstance(pub_key, str):
-            pub_key = decode_hex(pub_key)
-        return sha2(pub_key) < KeysAuth._count_max_hash(difficulty)
-
-    def is_difficult(self, difficulty: int) -> bool:
-        return self.is_pubkey_difficult(self.public_key, difficulty)
-
-    @staticmethod
-    def get_difficulty(key_id: str) -> int:
-        """
-        Calculate given key difficulty.
-        This is more expensive to calculate than is_difficult, so use
-        the latter if possible.
-        """
-        return int(math.floor(256 - math.log2(sha2(decode_hex(key_id)))))
-
     def sign(self, data: bytes) -> bytes:
         """
         Sign given data with ECDSA;

golem/core/variables.py

@@ -63,7 +63,6 @@
 
 # Number of task headers transmitted per message
 TASK_HEADERS_LIMIT = 20
-KEY_DIFFICULTY = 14
 
 # Maximum acceptable difference between node time and monitor time (seconds)
 MAX_TIME_DIFF = 10

golem/database/database.py

@@ -55,7 +55,7 @@ def execute_sql(self, sql, params=None, require_commit=True):
 
 
 class Database:
-    SCHEMA_VERSION = 30
+    SCHEMA_VERSION = 32
 
     def __init__(self,  # noqa pylint: disable=too-many-arguments
                  db: peewee.Database,

golem/database/schemas/031_migrate_payment_to_task_payment.py

@@ -0,0 +1,74 @@
+# pylint: disable=no-member,unused-argument
+import json
+import logging
+
+SCHEMA_VERSION = 31
+
+logger = logging.getLogger('golem.database')
+
+
+STATUS_MAPPING = {
+    1: 'awaiting',
+    2: 'sent',
+    3: 'confirmed',
+    4: 'overdue',
+}
+
+
+def migrate_payment(database, db_row):
+    details = json.loads(db_row['details'])
+    status = STATUS_MAPPING[db_row['status']]
+    cursor = database.execute_sql(
+        "INSERT INTO walletoperation"
+        " (tx_hash, direction, operation_type, status, sender_address,"
+        "  recipient_address, amount, currency, gas_cost,"
+        "  created_date, modified_date)"
+        " VALUES (?, 'outgoing', 'task_payment', ?, '', ?, ?, 'GNT', ?,"
+        "        ?, datetime('now'))",
+        (
+            f"0x{details['tx']}",
+            status,
+            f'0x{db_row["payee"]}',
+            db_row['value'],
+            details['fee'],
+            db_row['created_date'],
+        ),
+    )
+    wallet_operation_id = cursor.lastrowid
+    cursor.execute(
+        "INSERT INTO taskpayment"
+        " (wallet_operation_id, node, task, subtask,"
+        "  expected_amount, created_date, modified_date)"
+        " VALUES (?, ?, '', ?, ?, ?, datetime('now'))",
+        (
+            wallet_operation_id,
+            details['node_info']['key'],
+            db_row['subtask'],
+            db_row['value'],
+            db_row['created_date'],
+        ),
+    )
+
+
+def migrate(migrator, database, fake=False, **kwargs):
+    cursor = database.execute_sql(
+        'SELECT details, status, payee, value, subtask, created_date'
+        ' FROM payment'
+    )
+    for db_row in cursor.fetchall():
+        dict_row = {
+            'details': db_row[0],
+            'status': db_row[1],
+            'payee': db_row[2],
+            'value': db_row[3],
+            'subtask': db_row[4],
+            'created_date': db_row[5],
+        }
+        try:
+            migrate_payment(database, dict_row)
+        except Exception:  # pylint: disable=broad-except
+            logger.error("Migration problem. db_row=%s", db_row, exc_info=True)
+
+
+def rollback(migrator, database, fake=False, **kwargs):
+    pass