CCF base is now directly downloaded and isntalled from CCF release pa…

…ge. Currently 0.11.7 is installed.

With this change, CCF is no longer a submodule within PDO. Every PDO client must also install the CCF base
to be able to use CCF client modules for submitting transactions.

Signed-off-by: prakashngit <prakash.narayana.moorthy@intel.com>

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "interpreters/wasm-micro-runtime"]
 	path = interpreters/wasm-micro-runtime
 	url = https://github.com/bytecodealliance/wasm-micro-runtime.git
-[submodule "ccf_transaction_processor/CCF"]
-	path = ccf_transaction_processor/CCF
-	url = https://github.com/microsoft/CCF.git

build/Makefile

@@ -37,6 +37,12 @@ ifndef PDO_INTERPRETER
 $(error Incomplete configuration, PDO_INTERPRETER is not defined)
 endif
 
+ifeq ($(PDO_LEDGER_TYPE),ccf)
+ifndef CCF_BASE
+$(error Incomplete configuration, CCF_BASE is not defined)
+endif
+endif
+
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 DSTDIR ?= $(PDO_INSTALL_ROOT)
 SRCDIR ?= $(abspath $(SCRIPTDIR)/..)

ccf_transaction_processor/Makefile

@@ -25,10 +25,14 @@ ifndef SGX_MODE
 $(error Incomplete configuration, SGX_MODE is not defined)
 endif
 
+ifndef CCF_BASE
+$(error Incomplete configuration, CCF_BASE is not defined)
+endif
+
+
 NINJA ?= ninja $(NINJA_OPTIONS)
 
 SCRIPTDIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-CCFDIR := $(SCRIPTDIR)/CCF
 CCFDSTDIR ?= $(PDO_INSTALL_ROOT)/opt/pdo/ccf
 SRCDIR ?= $(abspath $(SCRIPTDIR)/..)
 KEYDIR := $(CCFDSTDIR)/keys/
@@ -48,29 +52,17 @@ all: environment keys config install
 
 build: build-ccf build-pdo-tp
 
-build-ccf :
-	mkdir -p $(CCFDIR)/build
-	cd $(CCFDIR)/build && cmake -GNinja \
-		-DCOMPILE_TARGETS=virtual \
-		-DCMAKE_INSTALL_PREFIX=$(CCFDSTDIR) \
-		-DBUILD_END_TO_END_TESTS=OFF \
-		-DBUILD_SMALLBANK=OFF \
-		-DBUILD_TESTS=OFF \
-		-DBUILD_UNIT_TESTS=OFF \
-		..
-	cd $(CCFDIR)/build && $(NINJA) && $(NINJA) install
-
-build-pdo-tp : install-ccf $(BLDDIR)
+build-pdo-tp : $(BLDDIR)
 	cd $(BLDDIR) && cmake .. -GNinja \
-		-DCCF_DIR=$(CCFDSTDIR) \
+		-DCCF_DIR=$(CCF_BASE) \
 		-DCOMPILE_TARGETS=virtual \
 		-DCMAKE_INSTALL_PREFIX=$(CCFDSTDIR)
 	cd $(BLDDIR) && $(NINJA)
 
 clean : clean-build clean-install
 
 clean-build:
-	rm -rf $(BLDDIR) $(CCFDIR)/build
+	rm -rf $(BLDDIR)
 
 clean-install :
 	rm -rf $(CCFDSTDIR)
@@ -84,8 +76,8 @@ $(ETCDIR)/cchost.toml : etc/cchost.toml
 			--output-directory $(dir $@) \
 			single --file $(notdir $@)
 
-$(ETCDIR)/gov.lua : $(CCFDIR)/src/runtime_config/gov.lua
-	cp $(CCFDIR)/src/runtime_config/gov.lua $(ETCDIR)/gov.lua
+$(ETCDIR)/gov.lua : etc/gov.lua
+	cp etc/gov.lua $(ETCDIR)/gov.lua
 
 environment : $(CCFDSTDIR) $(PYTHON_DIR)
 
@@ -106,13 +98,10 @@ $(PYTHON_DIR) :
 	virtualenv -p python3.7 --no-download $(CCFDSTDIR)
 	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade pip
 	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade setuptools
-	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade -r $(CCFDIR)/tests/requirements.txt
+	. $(abspath $(CCFDSTDIR)/bin/activate) && pip install --upgrade -r $(CCF_BASE)/bin/requirements.txt
 
 install : install-ccf install-pdo-tp
 
-install-ccf : build-ccf
-	cd $(CCFDIR)/build && $(NINJA) install
-
 PDO_BASH_SCRIPTS=start_ccf_network.sh start_cchost.sh stop_cchost.sh
 PDO_PYTHON_SCRIPTS=configure_ccf_network.py generate_ledger_authority.py fetch_ledger_authority.py
 
@@ -123,8 +112,8 @@ install-pdo-tp : build-pdo-tp
 	cp $(addprefix scripts/,$(PDO_PYTHON_SCRIPTS)) $(BINDIR)
 
 keys :
-	cd $(KEYDIR) && $(CCFDIR)/tests/keygenerator.sh --name memberccf --gen-enc-key
-	cd $(KEYDIR) && $(CCFDIR)/tests/keygenerator.sh --name userccf --gen-enc-key
+	cd $(KEYDIR) && $(CCF_BASE)/bin/keygenerator.sh --name memberccf --gen-enc-key
+	cd $(KEYDIR) && $(CCF_BASE)/bin/keygenerator.sh --name userccf --gen-enc-key
 
 .PHONY : all build build-ccf build-pdo-tp
 .PHONY : clean clean-build clean-install

ccf_transaction_processor/README.md

@@ -14,9 +14,12 @@ guaranteed.
 The instructions below  can be used to build and deploy CCF based PDO
 TP. Make sure the following env variables are defined:
 
+1) CCF_BASE : directory where CCF base is installed using the tarball from CCF release page.
+We recommend setting `export CCF_BASE=/opt/intel/ccf`.
+
 1) PDO_SOURCE_ROOT : points to PDO local git repo directory.
 
-2) PDO_INSTALL_ROOT : CCF will be installed at PDO_INSTALL_ROOT/opt/pdo/ccf/.
+2) PDO_INSTALL_ROOT : PTO TP will be installed at PDO_INSTALL_ROOT/opt/pdo/ccf/.
 
 3) HOSTNAME : CCF's first node will be deployed at HOSTNAME:6600. One can simply set HOSTNAME to be the `ip-address` of the VM that can be used to ping the VM from other machines. For local testing, set HOSTNAME to `127.0.0.1`.
 
@@ -30,17 +33,23 @@ used to sign the pdo contract enclaves.
 IMPORTANT: When installing CCF and PDO on the same VM for local testing, please install PDO first and
 then CCF. See [PDO docs](../docs) for detailed instructions on installing PDO.
 
-## Get CCF Source Code
+## Install CCF Base
 
-CCF tag 0.11.7 is included as a submodule within PDO. Download the
-submodule via the following commands:
+CCF Base with tag 0.11.7 is to be directly installed using the tarball from CCF release page.
+The following commands will install CCF base @ CCF_BASE folder
 
 ```bash
-cd $PDO_SOURCE_ROOT
-git submodule update --init
+wget https://github.com/microsoft/CCF/releases/download/ccf-0.11.7/ccf.tar.gz -P /tmp
+tar -xvf /tmp/ccf.tar.gz -C /tmp
+mv /tmp/ccf-0.11.7 $CCF_BASE
 ```
 
-## Install CCF Dependencies
+We note that CCF Base needs to be installed in PDO clients/eservice nodes when CCF is used as PDO ledger.
+The CCF base contains CCFClient modules that will be ued by PDO clients/eservice when submitting
+transactions to the CCF ledger. The rest of the steps below are only needed on the node
+where CCF based pdo-tp is getting built.
+
+## Install CCF Dependencies for building PDO TP
 
 CCF/PDO combo has been tested under a scenario where CCF is deployed in
 a standalone VM, and where PDO cients/services are deployed either locally
@@ -49,7 +58,7 @@ The dependencies needed to deploy CCF in an Ubuntu 18.04 VM with virtual enclave
 installed by running the following command:
 
 ```bash
-cd $PDO_SOURCE_ROOT/ccf_transaction_processor/CCF/getting_started/setup_vm/
+cd $CCF_BASE/getting_started/setup_vm/
 ./run.sh ccf-dev.yml
 ```
 
@@ -104,15 +113,21 @@ export NINJA_OPTION=-j2
 See the CCF documentation for information about configuring CCF. The
 `cchost` configuration file used by the PDO control scripts can be found
 at `${PDO_HOME}/ccf/etc/cchost.toml`. The CCF governance script can be
-found at `${PDO_HOME}/ccf/etc/gov.lua`.
-
+found at `${PDO_HOME}/ccf/etc/gov.lua`. We note that this governance script is
+the template governance script found as part of the CCF repo.
 
 ## Start/Stop CCF Network
 
 You can start a new CCF network with the PDO transaction processor using
-the script at `${PDO_HOME}/ccf/bin/start_ccf_network.sh`. That script
-will start the first node in the CCF network, open the network, add the
-user account that will be used for other PDO transactions, and generate
+the following commands:
+
+```bash
+source $PDO_HOME/ccf/bin/activate
+${PDO_HOME}/ccf/bin/start_ccf_network.sh
+```
+
+The above script will start the first node in the CCF network, open the network,
+add the user account that will be used for other PDO transactions, and generate
 the ledger authority key. The ledger authority key will be stored in the
 file `${PDO_HOME}/ccf/keys/ledger_authority_pub.pem`. This key can be
 used to verify claims about the state of the ledger.

ccf_transaction_processor/etc/gov.lua

@@ -0,0 +1,100 @@
+-- Copyright (c) Microsoft Corporation. All rights reserved.
+-- Licensed under the Apache 2.0 License.
+
+-- This file defines the default initial contents (ie, Lua scripts) of the governance scripts table.
+return {
+  pass = [[
+  tables, calls, votes = ...
+
+  -- interface definitions
+  PASSED = 1
+  PENDING = 0
+  REJECTED = -1
+  STATE_ACTIVE = "ACTIVE"
+
+  -- count member votes
+  member_votes = 0
+
+  for member, vote in pairs(votes) do
+    if vote then
+      member_votes = member_votes + 1
+    end
+  end
+
+  -- count active members
+  members_active = 0
+
+  tables["ccf.members"]:foreach(function(member, details)
+    if details["status"] == STATE_ACTIVE then
+      members_active = members_active + 1
+    end
+  end)
+
+  -- check for raw_puts to sensitive tables
+  SENSITIVE_TABLES = {"ccf.whitelists", "ccf.governance.scripts"}
+  for _, call in pairs(calls) do
+    if call.func == "raw_puts" then
+      for _, sensitive_table in pairs(SENSITIVE_TABLES) do
+        if call.args[sensitive_table] then
+          -- require unanimity
+          if member_votes == members_active then
+            return PASSED
+          else
+            return PENDING
+          end
+        end
+      end
+    end
+  end
+
+  -- a majority of members can pass votes
+  if member_votes > math.floor(members_active / 2) then
+    return PASSED
+  end
+
+  return PENDING]],
+
+  environment_proposal = [[
+  __Puts = {}
+  function __Puts:new(o)
+    o = o or {}
+    setmetatable(o, self)
+    self.__index = self
+    return o
+  end
+
+  function __Puts:put(t, key, value)
+    self[t] = self[t] or {}
+    table.insert(self[t], {k = key, v = value})
+    return self
+  end
+  -- create a frontend for __Puts that hides function entries
+  Puts = setmetatable({}, {__index = __Puts})
+
+  __Calls = {}
+  function __Calls:new(o)
+    o = o or {}
+    setmetatable(o, self)
+    self.__index = self
+    return o
+  end
+
+  function __Calls:call(_func, _args)
+    table.insert(self, {func=_func, args=_args})
+    return self
+  end
+  Calls =  setmetatable({}, {__index = __Calls})
+  ]],
+
+  -- scripts that can be proposed to be called
+
+  raw_puts = [[
+  tables, puts = ...
+  for table_name, entries in pairs(puts) do
+    t = tables[table_name]
+    for _,entry in pairs(entries) do
+      t:put(entry.k, entry.v)
+    end
+  end
+  return true]],
+}

ccf_transaction_processor/scripts/configure_ccf_network.py

@@ -30,7 +30,8 @@
 
 ## -----------------------------------------------------------------
 ContractHome = os.environ.get("PDO_HOME") or os.path.realpath("/opt/pdo")
-CCF_Bin = os.path.join(ContractHome, "ccf", "bin")
+CCF_BASE = os.environ.get("CCF_BASE")
+CCF_Bin = os.path.join(CCF_BASE, "bin")
 CCF_Etc = os.path.join(ContractHome, "ccf", "etc")
 CCF_Keys = os.path.join(ContractHome, "ccf", "keys")
 

ccf_transaction_processor/scripts/fetch_ledger_authority.py

@@ -26,7 +26,8 @@
 
 ## -----------------------------------------------------------------
 ContractHome = os.environ.get("PDO_HOME") or os.path.realpath("/opt/pdo")
-CCF_Bin = os.path.join(ContractHome, "ccf", "bin")
+CCF_BASE = os.environ.get("CCF_BASE")
+CCF_Bin = os.path.join(CCF_BASE, "bin")
 CCF_Etc = os.path.join(ContractHome, "ccf", "etc")
 CCF_Keys = os.path.join(ContractHome, "ccf", "keys")
 

ccf_transaction_processor/scripts/generate_ledger_authority.py

@@ -26,7 +26,8 @@
 
 ## -----------------------------------------------------------------
 ContractHome = os.environ.get("PDO_HOME") or os.path.realpath("/opt/pdo")
-CCF_Bin = os.path.join(ContractHome, "ccf", "bin")
+CCF_BASE = os.environ.get("CCF_BASE")
+CCF_Bin = os.path.join(CCF_BASE, "bin")
 CCF_Etc = os.path.join(ContractHome, "ccf", "etc")
 CCF_Keys = os.path.join(ContractHome, "ccf", "keys")
 

ccf_transaction_processor/scripts/ping_test.py

@@ -27,12 +27,12 @@
 
 ## -----------------------------------------------------------------
 ContractHome = os.environ.get("PDO_HOME") or os.path.realpath("/opt/pdo")
-CCF_Bin = os.path.join(ContractHome, "ccf", "bin")
+CCF_BASE = os.environ.get("CCF_BASE")
+CCF_Bin = os.path.join(CCF_BASE, "bin")
 CCF_Etc = os.path.join(ContractHome, "ccf", "etc")
 CCF_Keys = os.environ.get("PDO_LEDGER_KEY_ROOT") or os.path.join(ContractHome, "ccf", "keys")
 
 sys.path.insert(1, CCF_Bin)
-sys.path.insert(1, "../CCF/tests")
 
 from infra.clients import CCFClient
 
@@ -99,8 +99,8 @@ def Main() :
             version="2.0",
             connection_timeout=3,
             request_timeout=3)
-    except :
-        LOG.error('failed to connect to CCF service')
+    except Exception as e :
+        LOG.error('failed to connect to CCF service: {}'.format(str(e)))
         sys.exit(-1)
 
     ping_test(user_client, options)

ccf_transaction_processor/scripts/start_cchost.sh

@@ -23,9 +23,9 @@ fi
 F_SERVICEHOME="$( cd -P "$( dirname ${BASH_SOURCE[0]} )/.." && pwd )"
 source ${F_SERVICEHOME}/bin/lib/pdo_common.sh
 
-CCHOST=${F_SERVICEHOME}/bin/cchost
+CCHOST=${CCF_BASE}/bin/cchost
 if [ $SGX_MODE == "SIM" ]; then
-   CCHOST=${F_SERVICEHOME}/bin/cchost.virtual
+   CCHOST=${CCF_BASE}/bin/cchost.virtual
 fi
 
 EFILE="${F_SERVICEHOME}/logs/error.log"

ccf_transaction_processor/transaction_processor/pdo_tp.cpp

@@ -292,8 +292,6 @@ namespace ccfapp
                 }
 
                 //verify enclave signature
-                // To Do: This signature verification does not work. The following function returns "true" until this is fixed
-                // see git issues for status
                 if (!verify_enclave_signature_add_enclave(enclave_info_temp.signature, this->enclave_pubk_verifier[enclave_r.value().verifying_key], \
                     contract_info.contract_creator_verifying_key_PEM, in.contract_id, enclave_info_temp.provisioning_key_state_secret_pairs, \
                     enclave_info_temp.encrypted_state_encryption_key)){

docs/host_install.md

@@ -221,6 +221,8 @@ the SIMULATE mode for PDO enclaves, and virtual mode for CCF enclaves.
 See [HERE](../ccf_transaction_processor/Readme.md) to learn more about the ccf based transaction processor.
 The default ledger choice for PDO is Sawtooth. CCF based ledger can be enabled by setting the
 environment variable PDO_LEDGER_TYPE=ccf (assuming that a CCF ledger has already been set up).
+When using CCF based ledger, every pdo client must install CCF base using tarball from CCF release page.
+This enables use of CCF client modules from the CCF base package. Please see instructions in (../ccf_transaction_processor/Readme.md) for installation details.
 
 ## Build and Install PDO
 

python/MANIFEST

@@ -48,6 +48,4 @@
 ./pdo/submitter/sawtooth/sawtooth_protos/transaction_pb2.py
 ./pdo/submitter/ccf/__init__.py
 ./pdo/submitter/ccf/ccf_submitter.py
-./pdo/submitter/ccf/helpers/__init__.py
-./pdo/submitter/ccf/helpers/clients.py
 ./setup.py

python/pdo/submitter/ccf/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__all__ = [ 'ccf_submitter', 'helpers']
+__all__ = [ 'ccf_submitter']