Merge pull request #2 from arcolife/master

update master with latest

README.md

@@ -4,6 +4,9 @@ Refer to:
 https://arxiv.org/abs/1805.01457
 https://eprint.iacr.org/2016/917.pdf
 
+## Run
+
+Refer to `trueconsensus/README.md` for further instructions
 
 ### Parameterized by following..
 

docs/TODO.md

@@ -0,0 +1,20 @@
+## Generic Infra
+
+- [ ] Separate Dapp architecture: `dapps/bank.py` is currently tightly coupled with node.py. 
+- [ ] no gRPC communiction between nodes yet
+- [ ] add network_latency check 
+
+## Fastchain
+
+- [ ] nodes call quits on any error
+- [ ] xyz.pem not found error on running client.py. Batch size not playing well in proto_message.message check
+- [ ] complete subprotocol.py
+- [ ] complete bft_committee.py
+- [ ] return blocks of transactions
+
+## Snailchain / Committee election
+
+- [ ] complete minerva/vrf.py function
+- [ ] complete snailchain/fpow.py dummy
+- [ ] integrate level db wrappers
+- [ ] integrate py-evm functionalities

requirements.txt

@@ -1,6 +1,7 @@
 bitcoin==1.1.42
 docopt==0.6.2
 ecdsa==0.13
+protobuf==3.5.2.post1
 pycrypto==2.6.1
 pykwalify==1.6.1
 PySocks==1.6.8

trueconsensus/fastchain/pbft-py/README.md → trueconsensus/README.md

@@ -4,50 +4,51 @@ This is a `Practical Byzantine Fault Tolerance` implementation of Miguel Castro
 
 ### Setup
 
-#### Install Google's Protobuf
-
-Download and install [google protobuf](https://github.com/google/protobuf/tree/master/python/google)
-
-```
-# brew install protobuf
-OR 
-# pip install protobuf
-```
-
 #### Configure paths and tunables
 
-Fill up the config files `pbft_logistics.cfg` and `pbft_tunables.yaml` or use defaults.
-
+Fill up the config files `conf/pbft_logistics.cfg` and `conf/pbft_tunables.yaml` or use defaults.
 
 #### Install dependencies
 
 __Recommended__: Use a python3 virtual environment
 
 ```
 virtualenv -p python3 venv
-virtualenv -p python3 venv
 source venv/bin/activate
 pip install -r requirements.txt
 ```
 
+##### Install Google's Protobuf
+
+Download and install [google protobuf](https://github.com/google/protobuf/tree/master/python/google)
+
+```
+# brew install protobuf
+OR 
+# pip install protobuf
+```
+
 #### Generate required content as a precusory
 
 Then proceed as follows:
 
 ```
-# generate requests_pb2.py from requests.proto file
-./proto.sh
+# change to folder trueconsensus
+cd trueconsensus/
+
+# generate proto/requests_pb2.py from proto/requests.proto file
+./utils/generate_proto.sh
 
 # generate asymm keypairs
-python make_keys.py
+python -m utils.make_keys
 
 # generate requests
-./generate_requests_dat.py 
+python -m utils.generate_requests_dat
 ```
 
 ### Run
 
-Server: `./server.py`
+Server: `./engine.py`
 
 Client: `./client.py`
 

trueconsensus/fastchain/pbft-py/client.py → trueconsensus/client.py

@@ -11,8 +11,8 @@
 import socks
 # import time
 
-import request_pb2
-from config import client_address, \
+from proto import request_pb2
+from fastchain.config import client_address, \
     client_id, \
     RL, \
     client_logger, \

trueconsensus/conf/pbft_tunables.yaml

@@ -0,0 +1,28 @@
+# pbft core configurations
+# name: pbft base configuration
+testbed_config:
+  total: 5 # excluding base client
+  client_id: 5 # total-1 => 0-indexed
+  server_id_init: 4 # => 0 indexed
+  threading_enabled: true
+  requests:
+    max: 256
+    batch_size: 32
+general:
+  max_fail: 1
+  base_port: 49500
+  tor_socksport_range: 9050,9150
+# refer to  README.md#parameterized-by-following section for these configurables
+slowchain:
+  csize: 0
+bft_committee:
+  lambda: 1
+  timeout: 300 # seconds
+  tbft: 1
+  csize: 0
+  th: 10 # adjusted to ⌈csize/3⌉ later
+  actual_delta: 0
+  delta: 1 # ms to be adjusted from network ping test
+  # https://github.com/matthieu-lapeyre/network-benchmark/blob/master/network_test.py
+  chain: []
+  alpha: 0 # initial adversary hash power

trueconsensus/fastchain/pbft-py/bank.py → trueconsensus/dapps/bank.py

@@ -1,5 +1,5 @@
-import proto_message as message
-from config import config_general
+from proto import proto_message as message
+from fastchain.config import config_general
 
 
 class bank:

trueconsensus/engine.py

@@ -14,19 +14,164 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import sys
+# import yaml
+import signal
+from datetime import datetime
+import socket
+# from random import random
+from argparse import RawTextHelpFormatter, \
+                ArgumentParser
+
+from threading import Timer, Thread
+
+from fastchain import node
+from fastchain.config import config_yaml, \
+    threading_enabled, \
+    _logger, \
+    N, \
+    RL
+
 from snailchain import SnailChain
-from fastchain.bft import NodeBFT, \
+from fastchain.bft_committee import NodeBFT, \
     ViewChangeInit, \
     LedgerLog, \
-    BFTcommittee, \
-    SubProtoDailyBFT, \
+    BFTcommittee
+
+from fastchain.subprotocol import SubProtoDailyBFT, \
     Mempools
 
+parser = ArgumentParser(formatter_class=RawTextHelpFormatter,
+                        description="""PBFT standalone server demo""")
+
+
+def suicide():
+    # import pdb; pdb.set_trace()
+    # sys.exit()
+    # quit()
+    os.kill(os.getpid(), signal.SIGINT)
+
+
+def signal_handler(event, frame):
+    sys.stdout.write("handling signal: %s\n" % event)
+    sys.stdout.flush()
+    _logger.error("Kill signal (%s) detected. Stopping pbft.." % event)
+    countdown = 3  # seconds
+    if event == signal.SIGINT:
+        print("Committing deliberate suicide in %s seconds" % countdown)
+        t = Timer(countdown, suicide)
+        t.start()
+        sys.exit(130)  # Ctrl-C for bash
+
+
+def init_server(id):
+    global RL
+    try:
+        ip, port = RL[id]
+    except IndexError as E:
+        quit("%s Ran out of replica list. No more server config to try" % E)
+    s = socket.socket()
+    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    # host = socket.gethostname()
+    host = "0.0.0.0"
+    s.bind((host, port))  # on EC2 we cannot directly bind on public addr
+    s.listen(50)
+    s.setblocking(0)
+    _logger.debug("Server [%s] -- listening on port %s" % (id, port))
+    _logger.debug("IP: %s" % ip)
+    return s
+
+
+class ThreadedExecution(object):
 
-class DailyOffChainConsensus(object):
     def __init__(self):
-        self.chain = []
-        self._lambda = None
+        pass
+
+    def run(self, ID):
+        sys.stdout.write("run started\n")
+        sys.stdout.flush()
+        socket_obj = init_server(ID)
+        n = node.Node(ID, 0, N)
+        # n.init_keys(N)
+        n.init_replica_map(socket_obj)
+        n.server_loop()
+        sys.stdout.write("run exited\n")
+        sys.stdout.flush()
+
+    def launch(self):
+        threads = []
+        for i in range(N):
+            thread = Thread(target=self.run, args=[i])
+            thread.start()
+            threads.append(thread)
 
-    def preproess(self):
+        for thread in threads:
+            thread.join()
+
+        sys.stdout.write("join completed\n")
+        sys.stdout.flush()
+
+
+class NonThreadedExecution(object):
+    '''
+    Finds sockets that aren't busy and attempts to establish and launch testbed
+    '''
+    def __init__(self):
         pass
+
+    def init_server_socket(self, _id=None):
+        """
+        triggers setup using testbed_config. Increments given server id
+        if that (ip, socket) from Replica List RL is already in use.
+        """
+        global N
+        c = _id
+        while c < N:
+            s = None
+            try:
+                s = init_server(c)
+            except OSError as E:
+                _logger.error("%s -- Server ID: [%s]" % (E, c))
+                c -= 1
+            if s:
+                return s, c
+
+    def launch(self):
+        socket_obj, _id = self.init_server_socket(
+            _id=config_yaml["testbed_config"]["server_id_init"] - 1
+        )
+        n = node.Node(_id, 0, N)
+        # n.init_keys(N)
+        n.init_replica_map(socket_obj)
+        n.server_loop()
+
+
+# def pbft_usage():
+#     parser.add_argument("-n", "--nodes", dest="node_count", action='store',
+#                         help="# of PBFT nodes to be launched")
+#     parser.add_argument("-id", "--node-id", dest="node_id",
+#                         action='store_true',
+#                         help="")
+#     parser.add_argument("-ts", "--tune-settings", dest="tune",
+#                         action='store_true',
+#                         help="")
+
+
+def main():
+    print("Start time: ", datetime.now())
+    print("Threading enabled: ", threading_enabled)
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    # import pdb; pdb.set_trace()
+    if threading_enabled:
+        ThreadedExecution().launch()
+    else:
+        NonThreadedExecution().launch()
+
+
+if __name__ == "__main__":
+    # import pdb; pdb.set_trace()
+    main()

trueconsensus/fastchain/__init__.py

@@ -2,6 +2,5 @@
 Package import structure
 '''
 
-from fastchain.bft import *
+from fastchain.bft_committee import *
 from fastchain.node import *
-