thread locks and daemon

src/cryptoadvance/specter/__main__.py

@@ -50,10 +50,17 @@ def server(daemon, stop, restart, force, port, host, cert, key, tor, hwibridge):
             with open(pid_file) as f:
                 pid = int(f.read())
             os.kill(pid, signal.SIGTERM)
+            time.sleep(0.3)
+            try:
+                os.remove(pid_file)
+            except Exception as e:
+                pass
         elif daemon:
             if not force:
                 print(f"PID file \"{pid_file}\" already exists. Use --force to overwrite")
                 return
+            else:
+                os.remove(pid_file)
         if stop:
             return
     else:
@@ -129,13 +136,14 @@ def run(debug=False):
 
     # check if we should run a daemon or not
     if daemon or restart:
-        from daemonize import Daemonize
         print("Starting server in background...")
         print("* Hopefully running on %s://%s:%d/" % (protocol, host, port))
         if tor is not None:
             print("* For onion address check the file %s" % toraddr_file)
-        # Note: we can't run flask as a deamon in debug mode,
-        #       so use debug=False by default
+        # macOS + python3.7 is buggy
+        if sys.platform=="darwin" and (sys.version_info.major==3 and sys.version_info.minor < 8):
+            print("* WARNING: --daemon mode might not work properly in python 3.7 and lower on MacOS. Upgrade to python 3.8+")
+        from daemonize import Daemonize
         d = Daemonize(app="specter", pid=pid_file, action=run)
         d.start()
     else:

src/cryptoadvance/specter/device.py

@@ -1,9 +1,8 @@
 import json
 from .key import Key
-
+from .helpers import fslock
 
 class Device:
-
     def __init__(self, name, alias, device_type, keys, fullpath, manager):
         self.name = name
         self.alias = alias
@@ -32,11 +31,12 @@ def json(self):
         }
 
     def _update_keys(self):
-        with open(self.fullpath, "r") as f:
-            content = json.loads(f.read())
-        content['keys'] = [key.json for key in self.keys]
-        with open(self.fullpath, "w") as f:
-            f.write(json.dumps(content,indent=4))
+        with fslock:
+            with open(self.fullpath, "r") as f:
+                content = json.loads(f.read())
+            content['keys'] = [key.json for key in self.keys]
+            with open(self.fullpath, "w") as f:
+                f.write(json.dumps(content,indent=4))
         self.manager.update()
 
     def remove_key(self, key):
@@ -58,8 +58,9 @@ def wallets(self, wallet_manager):
 
     def set_type(self, device_type):
         self.device_type = device_type
-        with open(self.fullpath, "w") as f:
-            f.write(json.dumps(self.json,indent=4))
+        with fslock:
+            with open(self.fullpath, "w") as f:
+                f.write(json.dumps(self.json,indent=4))
         self.manager.update()
 
     def __eq__(self, other):

src/cryptoadvance/specter/device_manager.py

@@ -6,7 +6,7 @@
 from .devices.keepkey import Keepkey
 from .devices.specter import Specter
 from .devices.cobo import Cobo
-from .helpers import alias, load_jsons
+from .helpers import alias, load_jsons, fslock
 
 
 logger = logging.getLogger(__name__)
@@ -72,8 +72,9 @@ def add_device(self, name, device_type, keys):
                 non_dup_keys.append(key)
         keys = non_dup_keys
         device = get_device_class(device_type)(name, device_alias, device_type, keys, fullpath, self)
-        with open(fullpath, "w") as file:
-            file.write(json.dumps(device.json, indent=4))
+        with fslock:
+            with open(fullpath, "w") as file:
+                file.write(json.dumps(device.json, indent=4))
 
         self.update() # reload files
         return device

src/cryptoadvance/specter/helpers.py

@@ -3,6 +3,24 @@
 from .descriptor import AddChecksum
 from hwilib.serializations import PSBT
 from .bcur import bcur_decode
+import threading
+
+# use this for all fs operations
+fslock = threading.Lock()
+
+def locked(customlock=fslock):
+    """
+    @locked(lock) decorator.
+    Make sure you are not calling 
+    @locked function from another @locked function
+    with the same lock argument.
+    """
+    def wrapper(fn):
+        def wrapper_fn(*args, **kwargs):
+            with customlock:
+                return fn(*args, **kwargs)
+        return wrapper_fn
+    return wrapper
 
 try:
     collectionsAbc = collections.abc
@@ -30,8 +48,9 @@ def load_jsons(folder, key=None):
     files.sort(key=lambda x: os.path.getmtime(os.path.join(folder, x)))
     dd = OrderedDict()
     for fname in files:
-        with open(os.path.join(folder, fname)) as f:
-            d = json.loads(f.read())
+        with fslock:
+            with open(os.path.join(folder, fname)) as f:
+                d = json.loads(f.read())
         if key is None:
             dd[fname[:-5]] = d
         else:
@@ -193,31 +212,32 @@ def get_users_json(specter):
             'is_admin': True
         }
     ]
-
 
     # if users.json file exists - load from it
     if os.path.isfile(os.path.join(specter.data_folder, "users.json")):
-        with open(os.path.join(specter.data_folder, "users.json"), "r") as f:
-            users = json.loads(f.read())
+        with fslock:
+            with open(os.path.join(specter.data_folder, "users.json"), "r") as f:
+                users = json.loads(f.read())
     # otherwise - create one and assign unique id
     else:
         save_users_json(specter, users)
     return users
 
 def save_users_json(specter, users):
-    with open(os.path.join(specter.data_folder, 'users.json'), "w") as f:
-        f.write(json.dumps(users, indent=4))
+    with fslock:
+        with open(os.path.join(specter.data_folder, 'users.json'), "w") as f:
+            f.write(json.dumps(users, indent=4))
 
 def hwi_get_config(specter):
     config = {
         'whitelisted_domains': 'http://127.0.0.1:25441/'
     }
-
     # if hwi_bridge_config.json file exists - load from it
     if os.path.isfile(os.path.join(specter.data_folder, "hwi_bridge_config.json")):
-        with open(os.path.join(specter.data_folder, "hwi_bridge_config.json"), "r") as f:
-            file_config = json.loads(f.read())
-            deep_update(config, file_config)
+        with fslock:
+            with open(os.path.join(specter.data_folder, "hwi_bridge_config.json"), "r") as f:
+                file_config = json.loads(f.read())
+                deep_update(config, file_config)
     # otherwise - create one and assign unique id
     else:
         save_hwi_bridge_config(specter, config)
@@ -232,8 +252,9 @@ def save_hwi_bridge_config(specter, config):
                 url += "/"
             whitelisted_domains += url.strip() + '\n'
         config['whitelisted_domains'] = whitelisted_domains
-    with open(os.path.join(specter.data_folder, 'hwi_bridge_config.json'), "w") as f:
-        f.write(json.dumps(config, indent=4))
+    with fslock:
+        with open(os.path.join(specter.data_folder, 'hwi_bridge_config.json'), "w") as f:
+            f.write(json.dumps(config, indent=4))
 
 def der_to_bytes(derivation):
     items = derivation.split("/")

src/cryptoadvance/specter/hwi_rpc.py

@@ -1,10 +1,13 @@
 from hwilib.serializations import PSBT
 import hwilib.commands as hwi_commands
 from hwilib import bech32
-from .helpers import convert_xpub_prefix
+from .helpers import convert_xpub_prefix, locked
 from .specter_hwi import SpecterClient, enumerate as specter_enumerate
 from .json_rpc import JSONRPC
+import threading
 
+# use this lock for all hwi operations
+hwilock = threading.Lock()
 
 class HWIBridge(JSONRPC):
     """
@@ -27,6 +30,7 @@ def __init__(self):
         # devices once per session and save them.
         self.enumerate()
 
+    @locked(hwilock)
     def enumerate(self):
         """
         Returns a list of all connected devices (dicts).
@@ -56,6 +60,7 @@ def detect_device(self, device_type=None, path=None, fingerprint=None, rescan_de
         if len(res) > 0:
             return res[0]
 
+    @locked(hwilock)
     def prompt_pin(self, device_type=None, path=None, passphrase='', chain=''):
         if device_type == "keepkey" or device_type == "trezor":
             # The device will randomize its pin entry matrix on the device
@@ -69,6 +74,7 @@ def prompt_pin(self, device_type=None, path=None, passphrase='', chain=''):
         else:
             raise Exception("Invalid HWI device type %s, prompt_pin is only supported for Trezor and Keepkey devices" % device_type)
 
+    @locked(hwilock)
     def send_pin(self, pin='', device_type=None, path=None, passphrase='', chain=''):
         if device_type == "keepkey" or device_type == "trezor":
             if pin == '':
@@ -78,11 +84,13 @@ def send_pin(self, pin='', device_type=None, path=None, passphrase='', chain='')
         else:
             raise Exception("Invalid HWI device type %s, send_pin is only supported for Trezor and Keepkey devices" % device_type)
 
+    @locked(hwilock)
     def extract_xpubs(self, device_type=None, path=None, fingerprint=None, passphrase='', chain=''):
         client = self._get_client(device_type=device_type, fingerprint=fingerprint, path=path, passphrase=passphrase, chain=chain)
         xpubs = self._extract_xpubs_from_client(client)
         return xpubs
 
+    @locked(hwilock)
     def display_address(self, descriptor='', device_type=None, path=None, fingerprint=None, passphrase='', chain=''):
         if descriptor == '':
             raise Exception("Descriptor must not be empty")
@@ -102,6 +110,7 @@ def display_address(self, descriptor='', device_type=None, path=None, fingerprin
                 client.close()
             raise e
 
+    @locked(hwilock)
     def sign_tx(self, psbt='', device_type=None, path=None, fingerprint=None, passphrase='', chain=''):
         if psbt == '':
             raise Exception("PSBT must not be empty")

src/cryptoadvance/specter/specter.py

@@ -6,7 +6,7 @@
 from .wallet_manager import WalletManager
 from .user import User
 from flask_login import current_user
-
+import threading
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +30,8 @@ def get_cli(conf):
 class Specter:
     ''' A central Object mostly holding app-settings '''
     CONFIG_FILE_NAME = "config.json"
+    # use this lock for all fs operations
+    lock = threading.Lock()
     def __init__(self, data_folder="./data", config={}):
         if data_folder.startswith("~"):
             data_folder = os.path.expanduser(data_folder)
@@ -74,15 +76,13 @@ def __init__(self, data_folder="./data", config={}):
         self.check()
 
     def check(self, user=current_user):
-        if self.is_checking:
-            return
-        self.is_checking = True
         # if config.json file exists - load from it
         if os.path.isfile(os.path.join(self.data_folder, "config.json")):
-            with open(os.path.join(self.data_folder, "config.json"), "r") as f:
-                self.file_config = json.loads(f.read())
-                deep_update(self.config, self.file_config)
-        # otherwise - create one and assign unique id
+            with self.lock:
+                with open(os.path.join(self.data_folder, "config.json"), "r") as f:
+                    self.file_config = json.loads(f.read())
+                    deep_update(self.config, self.file_config)
+            # otherwise - create one and assign unique id
         else:
             if self.config["uid"] == "":
                 self.config["uid"] = random.randint(0,256**8).to_bytes(8,'big').hex()
@@ -132,7 +132,6 @@ def check(self, user=current_user):
                     self.cli, 
                     chain=chain
                 )
-        self.is_checking = False
 
     def clear_user_session(self):
         self.device_manager = None
@@ -185,8 +184,9 @@ def test_rpc(self, **kwargs):
         return r
 
     def _save(self):
-        with open(os.path.join(self.data_folder, self.CONFIG_FILE_NAME), "w") as f:
-            f.write(json.dumps(self.config, indent=4))
+        with self.lock:
+            with open(os.path.join(self.data_folder, self.CONFIG_FILE_NAME), "w") as f:
+                f.write(json.dumps(self.config, indent=4))
 
     def update_rpc(self, **kwargs):
         need_update = False

src/cryptoadvance/specter/wallet.py

@@ -3,17 +3,16 @@
 from .descriptor import AddChecksum
 from .device import Device
 from .key import Key
-from .helpers import decode_base58, der_to_bytes, get_xpub_fingerprint, sort_descriptor
+from .helpers import decode_base58, der_to_bytes, get_xpub_fingerprint, sort_descriptor, fslock
 from hwilib.serializations import PSBT, CTransaction
 from io import BytesIO
 from .specter_error import SpecterError
-
+import threading
 
 # a gap of 20 addresses is what many wallets do
 WALLET_CHUNK = 20
 
 class Wallet():
-
     def __init__(
         self,
         name,
@@ -192,8 +191,9 @@ def json(self):
         }
 
     def save_to_file(self):
-        with open(self.fullpath, "w+") as f:
-            f.write(json.dumps(self.json, indent=4))
+        with fslock:
+            with open(self.fullpath, "w+") as f:
+                f.write(json.dumps(self.json, indent=4))
         self.manager.update()
 
     @property