FAB-1547 initial create/join chain support

https://jira.hyperledger.org/browse/FAB-1547

With this change the fabric has basic support for create / join
chain begun with the implementation of CSCC (configuraton system
chaincode) in https://jira.hyperledger.org/browse/FAB-1022.

Some todos remain for follow up CRs
  . docker based commands to channel-setup.md
  . CONFIGURATION_TRANSACTION validation - https://jira.hyperledger.org/browse/FAB-1639
  . further MSP integration (still uses default MSP)
  . absorption of deliver client management into gossip later - https://jira.hyperledger.org/browse/FAB-1580
  . adding specific configuration items to channel create - https://jira.hyperledger.org/browse/FAB-1642

Steps to test chain create / join below. All commands assume shell
in "fabric/" directory.

Vagrant window 1 - start orderer
  cd orderer
  ORDERER_GENERAL_LOGLEVEL=debug ./orderer

Vagrant window 2 - ask orderer to create a chain
  cd peer
  peer channel create -c myc1

  #on successful creation, a genesis block myc1.block is saved
  #in the same directory

Vagrant window 3 - start the peer without **TEST_CHAINID**
                   (basically in a "chainless" mode)

  #to start with a clean env do "rm -rf /var/hyperledger/*"
  cd peer
  peer node start --peer-defaultchain=false

  #in "--peer-defaultchain=false" mode the peer has to join
  #chains to create leader and do transactions. It does not
  #have a default chain or ledger (the **TEST_CHAINID** chain)

Vagrant window 4 - ask peer to join a chain
  cd peer
  peer channel join -b myc1.block

At this point we can issue transactions

Vagrant window 2 - deploy a chaincode to myc1
  cd peer
  peer chaincode deploy -C myc1 -n mycc -p github.com/hyperledger/fabric/examples/chaincode/go/chaincode_example02 -c '{"Args":["init","a","100","b","200"]}'

  #note the use of "-C myc1"
  #wait for 10 secs or so

Vagrant window 2 - query chaincode
  cd peer
  peer chaincode query -C myc1 -n mycc -c '{"Args":["query","a"]}'

Change-Id: I7d1d04e8a207eb57597a1e6eb8b986e1080e7811
Signed-off-by: Srinivasan Muralidharan <muralisr@us.ibm.com>

bddtests/chaincode.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric/common/util"
+	"github.com/hyperledger/fabric/protos/common"
 	pb "github.com/hyperledger/fabric/protos/peer"
 	putils "github.com/hyperledger/fabric/protos/utils"
 )
@@ -52,5 +53,5 @@ func createProposalForChaincode(ccChaincodeDeploymentSpec *pb.ChaincodeDeploymen
 	uuid := createProposalID()
 
 	// make proposal
-	return putils.CreateChaincodeProposal(uuid, util.GetTestChainID(), lcChaincodeInvocationSpec, creator)
+	return putils.CreateChaincodeProposal(uuid, common.HeaderType_ENDORSER_TRANSACTION, util.GetTestChainID(), lcChaincodeInvocationSpec, creator)
 }

common/configtx/test/helper.go

@@ -63,3 +63,8 @@ func init() {
 func MakeGenesisBlock(chainID string) (*cb.Block, error) {
 	return genesisFactory.Block(chainID)
 }
+
+// GetOrderererTemplate returns the test orderer template
+func GetOrdererTemplate() configtx.Template {
+	return template
+}

core/chaincode/configer.go

@@ -111,9 +111,9 @@ func joinChain(blockBytes []byte) ([]byte, error) {
 		return nil, fmt.Errorf("Failed to get the chain ID from the configuration block, %s", err)
 	}
 
-	// Initialize all system chainodes on this chain
-	// TODO: Fix this code to initialize instead of deploy chaincodes
-	DeploySysCCs(chainID)
+	if err = peer.CreateDeliveryService(chainID); err != nil {
+		return nil, err
+	}
 
 	return []byte("200"), nil
 }

core/chaincode/configer_test.go

@@ -100,6 +100,7 @@ func TestConfigerInvokeJoinChainWrongParams(t *testing.T) {
 func TestConfigerInvokeJoinChainCorrectParams(t *testing.T) {
 	//t.Skip("Test CI build")
 	viper.Set("peer.fileSystemPath", "/var/hyperledger/test/")
+	peer.MockInitialize()
 	ledgermgmt.InitializeTestEnv()
 	defer ledgermgmt.CleanupTestEnv()
 	defer os.RemoveAll("/var/hyperledger/test/")

core/chaincode/exectransaction_test.go

@@ -153,7 +153,7 @@ func endTxSimulationCIS(chainID string, txid string, txsim ledger.TxSimulator, p
 		return err
 	}
 	// get a proposal - we need it to get a transaction
-	prop, err := putils.CreateProposalFromCIS(txid, chainID, cis, ss)
+	prop, err := putils.CreateProposalFromCIS(txid, common.HeaderType_ENDORSER_TRANSACTION, chainID, cis, ss)
 	if err != nil {
 		return err
 	}

core/committer/txvalidator/validator.go

@@ -90,9 +90,14 @@ func (v *txValidator) Validate(block *common.Block) {
 				// NOT check the validity of endorsements, though. That's a
 				// job for VSCC below
 				logger.Debug("Validating transaction peer.ValidateTransaction()")
-				if payload, _, err := peer.ValidateTransaction(env); err != nil {
+				var payload *common.Payload
+				var err error
+				if payload, _, err = peer.ValidateTransaction(env); err != nil {
 					logger.Errorf("Invalid transaction with index %d, error %s", tIdx, err)
-				} else {
+					continue
+				}
+
+				if common.HeaderType(payload.Header.ChainHeader.Type) == common.HeaderType_ENDORSER_TRANSACTION {
 					// Check duplicate transactions
 					txID := payload.Header.ChainHeader.TxID
 					if _, err := v.ledger.GetTransactionByID(txID); err == nil {
@@ -107,15 +112,17 @@ func (v *txValidator) Validate(block *common.Block) {
 						logger.Errorf("VSCCValidateTx for transaction txId = %s returned error %s", txID, err)
 						continue
 					}
+				} else if common.HeaderType(payload.Header.ChainHeader.Type) == common.HeaderType_CONFIGURATION_TRANSACTION {
+					logger.Warningf("Validation for common.HeaderType_CONFIGURATION_TRANSACTION %d pending JIRA-1639", tIdx)
+				}
 
-					if _, err := proto.Marshal(env); err != nil {
-						logger.Warningf("Cannot marshal transaction due to %s", err)
-						continue
-					}
-					// Succeeded to pass down here, transaction is valid,
-					// just unset the filter bit array flag.
-					txsfltr.Unset(uint(tIdx))
+				if _, err := proto.Marshal(env); err != nil {
+					logger.Warningf("Cannot marshal transaction due to %s", err)
+					continue
 				}
+				// Succeeded to pass down here, transaction is valid,
+				// just unset the filter bit array flag.
+				txsfltr.Unset(uint(tIdx))
 			} else {
 				logger.Warning("Nil tx from block")
 			}

core/endorser/endorser_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/hyperledger/fabric/core/peer"
 	"github.com/hyperledger/fabric/core/peer/msp"
 	"github.com/hyperledger/fabric/msp"
+	"github.com/hyperledger/fabric/protos/common"
 	pb "github.com/hyperledger/fabric/protos/peer"
 	pbutils "github.com/hyperledger/fabric/protos/utils"
 	"github.com/spf13/viper"
@@ -109,7 +110,7 @@ func closeListenerAndSleep(l net.Listener) {
 //Currently supported only for Invokes (Queries still go through devops client)
 func getInvokeProposal(cis *pb.ChaincodeInvocationSpec, chainID string, creator []byte) (*pb.Proposal, error) {
 	uuid := util.GenerateUUID()
-	return pbutils.CreateChaincodeProposal(uuid, chainID, cis, creator)
+	return pbutils.CreateChaincodeProposal(uuid, common.HeaderType_ENDORSER_TRANSACTION, chainID, cis, creator)
 }
 
 func getDeployProposal(cds *pb.ChaincodeDeploymentSpec, chainID string, creator []byte) (*pb.Proposal, error) {

core/ledger/kvledger/txmgmt/validator/statebasedval/state_based_validator.go

@@ -39,10 +39,53 @@ func NewValidator(db statedb.VersionedDB) *Validator {
 	return &Validator{db}
 }
 
+//validate endorser transaction
+func (v *Validator) validateEndorserTX(envBytes []byte, doMVCCValidation bool, updates *statedb.UpdateBatch) (*rwset.TxReadWriteSet, error) {
+	// extract actions from the envelope message
+	respPayload, err := putils.GetActionFromEnvelope(envBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	//preparation for extracting RWSet from transaction
+	txRWSet := &rwset.TxReadWriteSet{}
+
+	// Get the Result from the Action
+	// and then Unmarshal it into a TxReadWriteSet using custom unmarshalling
+	if err = txRWSet.Unmarshal(respPayload.Results); err != nil {
+		return nil, err
+	}
+
+	// trace the first 2000 characters of RWSet only, in case it is huge
+	if logger.IsEnabledFor(logging.DEBUG) {
+		txRWSetString := txRWSet.String()
+		if len(txRWSetString) < 2000 {
+			logger.Debugf("validating txRWSet:[%s]", txRWSetString)
+		} else {
+			logger.Debugf("validating txRWSet:[%s...]", txRWSetString[0:2000])
+		}
+	}
+
+	//mvccvalidation, may invalidate transaction
+	if doMVCCValidation {
+		if valid, err := v.validateTx(txRWSet, updates); err != nil {
+			return nil, err
+		} else if !valid {
+			txRWSet = nil
+		}
+	}
+
+	return txRWSet, err
+}
+
+// TODO validate configuration transaction
+func (v *Validator) validateConfigTX(env *common.Envelope) (bool, error) {
+	return true, nil
+}
+
 // ValidateAndPrepareBatch implements method in Validator interface
 func (v *Validator) ValidateAndPrepareBatch(block *common.Block, doMVCCValidation bool) (*statedb.UpdateBatch, error) {
 	logger.Debugf("New block arrived for validation:%#v, doMVCCValidation=%t", block, doMVCCValidation)
-	var valid bool
 	updates := statedb.NewUpdateBatch()
 	logger.Debugf("Validating a block with [%d] transactions", len(block.Data.Data))
 	txsFilter := util.NewFilterBitArrayFromBytes(block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER])
@@ -53,41 +96,39 @@ func (v *Validator) ValidateAndPrepareBatch(block *common.Block, doMVCCValidatio
 			continue
 		}
 
-		// extract actions from the envelope message
-		respPayload, err := putils.GetActionFromEnvelope(envBytes)
+		env, err := putils.GetEnvelopeFromBlock(envBytes)
 		if err != nil {
 			return nil, err
 		}
 
-		//preparation for extracting RWSet from transaction
-		txRWSet := &rwset.TxReadWriteSet{}
-
-		// Get the Result from the Action
-		// and then Unmarshal it into a TxReadWriteSet using custom unmarshalling
-		if err = txRWSet.Unmarshal(respPayload.Results); err != nil {
+		payload, err := putils.GetPayload(env)
+		if err != nil {
 			return nil, err
 		}
 
-		// trace the first 2000 characters of RWSet only, in case it is huge
-		if logger.IsEnabledFor(logging.DEBUG) {
-			txRWSetString := txRWSet.String()
-			if len(txRWSetString) < 2000 {
-				logger.Debugf("validating txRWSet:[%s]", txRWSetString)
-			} else {
-				logger.Debugf("validating txRWSet:[%s...]", txRWSetString[0:2000])
+		valid := false
+		if common.HeaderType(payload.Header.ChainHeader.Type) == common.HeaderType_ENDORSER_TRANSACTION {
+			txRWSet, err := v.validateEndorserTX(envBytes, doMVCCValidation, updates)
+			if err != nil {
+				return nil, err
 			}
+			//txRWSet != nil => t is valid
+			if txRWSet != nil {
+				committingTxHeight := version.NewHeight(block.Header.Number, uint64(txIndex+1))
+				addWriteSetToBatch(txRWSet, committingTxHeight, updates)
+				valid = true
+			}
+		} else if common.HeaderType(payload.Header.ChainHeader.Type) == common.HeaderType_CONFIGURATION_TRANSACTION {
+			valid, err = v.validateConfigTX(env)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			logger.Errorf("Skipping transaction %d that's not an endorsement or configuration %d", txIndex, payload.Header.ChainHeader.Type)
+			valid = false
 		}
 
-		if !doMVCCValidation {
-			valid = true
-		} else if valid, err = v.validateTx(txRWSet, updates); err != nil {
-			return nil, err
-		}
-
-		if valid {
-			committingTxHeight := version.NewHeight(block.Header.Number, uint64(txIndex+1))
-			addWriteSetToBatch(txRWSet, committingTxHeight, updates)
-		} else {
+		if !valid {
 			// Unset bit in byte array corresponded to the invalid transaction
 			txsFilter.Set(uint(txIndex))
 		}

core/peer/fullflow_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/hyperledger/fabric/common/util"
 	"github.com/hyperledger/fabric/core/peer/msp"
 	"github.com/hyperledger/fabric/msp"
+	"github.com/hyperledger/fabric/protos/common"
 	"github.com/hyperledger/fabric/protos/peer"
 	"github.com/hyperledger/fabric/protos/utils"
 )
@@ -39,7 +40,7 @@ func getProposal() (*peer.Proposal, error) {
 
 	uuid := util.GenerateUUID()
 
-	return utils.CreateProposalFromCIS(uuid, util.GetTestChainID(), cis, signerSerialized)
+	return utils.CreateProposalFromCIS(uuid, common.HeaderType_ENDORSER_TRANSACTION, util.GetTestChainID(), cis, signerSerialized)
 }
 
 func TestGoodPath(t *testing.T) {

core/peer/msgvalidation.go

@@ -89,6 +89,12 @@ func ValidateProposalMessage(signedProp *pb.SignedProposal) (*pb.Proposal, *comm
 
 	// continue the validation in a way that depends on the type specified in the header
 	switch common.HeaderType(hdr.ChainHeader.Type) {
+	case common.HeaderType_CONFIGURATION_TRANSACTION:
+		//which the types are different the validation is the same
+		//viz, validate a proposal to a chaincode. If we need other
+		//special validation for confguration, we would have to implement
+		//special validation
+		fallthrough
 	case common.HeaderType_ENDORSER_TRANSACTION:
 		// validation of the proposal message knowing it's of type CHAINCODE
 		chaincodeHdrExt, err := validateChaincodeProposalMessage(prop, hdr)
@@ -98,6 +104,7 @@ func ValidateProposalMessage(signedProp *pb.SignedProposal) (*pb.Proposal, *comm
 
 		return prop, hdr, chaincodeHdrExt, err
 	default:
+		//NOTE : we proably need a case
 		return nil, nil, nil, fmt.Errorf("Unsupported proposal type %d", common.HeaderType(hdr.ChainHeader.Type))
 	}
 }
@@ -315,6 +322,11 @@ func ValidateTransaction(e *common.Envelope) (*common.Payload, []*pb.Transaction
 
 	putilsLogger.Infof("Header is %s", payload.Header)
 
+	if common.HeaderType(payload.Header.ChainHeader.Type) == common.HeaderType_CONFIGURATION_TRANSACTION {
+		putilsLogger.Warningf("Skipping common.HeaderType_CONFIGURATION_TRANSACTION validation pending JIRA-1639\n")
+		return payload, nil, nil
+	}
+
 	// validate the header
 	err = validateCommonHeader(payload.Header)
 	if err != nil {

core/peer/msp/config.go

@@ -43,13 +43,16 @@ func LoadFakeSetupWithLocalMspAndTestChainMsp(dir string) error {
 }
 
 // GetMSPManagerFromBlock returns a new MSP manager from a ConfigurationEnvelope
-func GetMSPManagerFromBlock(b *common.Block) (msp.MSPManager, error) {
+// Note that chainID should really be obtained from the block. Passing it for
+// two reasons (1) efficiency (2) getting chainID from block using protos/utils
+// will cause import cycles
+func GetMSPManagerFromBlock(cid string, b *common.Block) (msp.MSPManager, error) {
 	mgrConfig, err := msputils.GetMSPManagerConfigFromBlock(b)
 	if err != nil {
 		return nil, err
 	}
 
-	mgr := msp.NewMSPManager()
+	mgr := GetManagerForChain(cid)
 	err = mgr.Setup(mgrConfig)
 	if err != nil {
 		return nil, err

core/peer/msp/peermsp_test.go

@@ -53,7 +53,7 @@ func TestGetMSPManagerFromBlock(t *testing.T) {
 		t.Fatalf("getTestBlockFromMspConfig failed, err %s", err)
 	}
 
-	mgr, err := GetMSPManagerFromBlock(block)
+	mgr, err := GetMSPManagerFromBlock("testchainid", block)
 	if err != nil {
 		t.Fatalf("GetMSPManagerFromBlock failed, err %s", err)
 	} else if mgr == nil {

core/peer/peer.go

@@ -59,13 +59,17 @@ func MockInitialize() {
 	ledgermgmt.InitializeTestEnv()
 	chains.list = nil
 	chains.list = make(map[string]*chain)
+	deliveryServiceProvider = func(string) error { return nil }
 }
 
+var deliveryServiceProvider func(string) error
+
 // Initialize sets up any chains that the peer has from the persistence. This
 // function should be called at the start up when the ledger and gossip
 // ready
-func Initialize() {
-	//Till JoinChain works, we continue to use default chain
+func Initialize(dsProvider func(string) error) {
+	deliveryServiceProvider = dsProvider
+
 	var cb *common.Block
 	var ledger ledger.ValidatedLedger
 	ledgermgmt.Initialize()
@@ -76,12 +80,12 @@ func Initialize() {
 	for _, cid := range ledgerIds {
 		peerLogger.Infof("Loading chain %s", cid)
 		if ledger, err = ledgermgmt.OpenLedger(cid); err != nil {
-			peerLogger.Warning("Failed to load ledger %s", cid)
+			peerLogger.Warningf("Failed to load ledger %s(%s)", cid, err)
 			peerLogger.Debug("Error while loading ledger %s with message %s. We continue to the next ledger rather than abort.", cid, err)
 			continue
 		}
 		if cb, err = getCurrConfigBlockFromLedger(ledger); err != nil {
-			peerLogger.Warning("Failed to find configuration block on ledger %s", cid)
+			peerLogger.Warningf("Failed to find configuration block on ledger %s(%s)", cid, err)
 			peerLogger.Debug("Error while looking for config block on ledger %s with message %s. We continue to the next ledger rather than abort.", cid, err)
 			continue
 		}
@@ -90,7 +94,20 @@ func Initialize() {
 			peerLogger.Warning("Failed to load chain %s", cid)
 			peerLogger.Debug("Error reloading chain %s with message %s. We continue to the next chain rather than abort.", cid, err)
 		}
+
+		// now create the delivery service for this chain
+		if err = deliveryServiceProvider(cid); err != nil {
+			peerLogger.Errorf("Error creating delivery service for %s(err - %s)", cid, err)
+		}
+	}
+}
+
+//CreateDeliveryService creates the delivery service for the chainID
+func CreateDeliveryService(chainID string) error {
+	if deliveryServiceProvider == nil {
+		return fmt.Errorf("delivery service provider not available")
 	}
+	return deliveryServiceProvider(chainID)
 }
 
 func getCurrConfigBlockFromLedger(ledger ledger.ValidatedLedger) (*common.Block, error) {
@@ -129,7 +146,7 @@ func getCurrConfigBlockFromLedger(ledger ledger.ValidatedLedger) (*common.Block,
 func createChain(cid string, ledger ledger.ValidatedLedger, cb *common.Block) error {
 	c := committer.NewLedgerCommitter(ledger)
 
-	mgr, err := mspmgmt.GetMSPManagerFromBlock(cb)
+	mgr, err := mspmgmt.GetMSPManagerFromBlock(cid, cb)
 	if err != nil {
 		return err
 	}

core/peer/peer_test.go

@@ -33,7 +33,7 @@ import (
 func TestInitialize(t *testing.T) {
 	viper.Set("peer.fileSystemPath", "/var/hyperledger/test/")
 
-	Initialize()
+	Initialize(nil)
 }
 
 func TestCreateChainFromBlock(t *testing.T) {
@@ -84,7 +84,7 @@ func TestCreateChainFromBlock(t *testing.T) {
 	}
 
 	// Chaos monkey test
-	Initialize()
+	Initialize(nil)
 
 	SetCurrConfigBlock(block, testChainID)
 }