Merge "FAB-13011 add kafka consensus type to byfn sample"

first-network/base/docker-compose-base.yaml

@@ -22,6 +22,8 @@ services:
  - ORDERER_GENERAL_TLS_PRIVATEKEY=/var/hyperledger/orderer/tls/server.key
  - ORDERER_GENERAL_TLS_CERTIFICATE=/var/hyperledger/orderer/tls/server.crt
  - ORDERER_GENERAL_TLS_ROOTCAS=[/var/hyperledger/orderer/tls/ca.crt]
+ - ORDERER_KAFKA_TOPIC_REPLICATIONFACTOR=1
+ - ORDERER_KAFKA_VERBOSE=true
  working_dir: /opt/gopath/src/github.com/hyperledger/fabric
  command: orderer
  volumes:

first-network/byfn.sh

@@ -35,7 +35,7 @@ export VERBOSE=false
 # Print the usage message
 function printHelp() {
  echo "Usage: "
- echo " byfn.sh <mode> [-c <channel name>] [-t <timeout>] [-d <delay>] [-f <docker-compose-file>] [-s <dbtype>] [-l <language>] [-i <imagetag>] [-v]"
+ echo " byfn.sh <mode> [-c <channel name>] [-t <timeout>] [-d <delay>] [-f <docker-compose-file>] [-s <dbtype>] [-l <language>] [-o <consensus-type>] [-i <imagetag>] [-v]"
  echo " <mode> - one of 'up', 'down', 'restart', 'generate' or 'upgrade'"
  echo " - 'up' - bring up the network with docker-compose up"
  echo " - 'down' - clear the network with docker-compose down"
@@ -48,6 +48,7 @@ function printHelp() {
  echo " -f <docker-compose-file> - specify which docker-compose file use (defaults to docker-compose-cli.yaml)"
  echo " -s <dbtype> - the database backend to use: goleveldb (default) or couchdb"
  echo " -l <language> - the chaincode language: golang (default) or node"
+ echo " -o <consensus-type> - the consensus-type of the ordering service: solo (default) or kafka"
  echo " -i <imagetag> - the tag to be used to launch the network (defaults to \"latest\")"
  echo " -v - verbose mode"
  echo " byfn.sh -h (print this message)"
@@ -156,14 +157,29 @@ function networkUp() {
  generateChannelArtifacts
  fi
  if [ "${IF_COUCHDB}" == "couchdb" ]; then
- IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1
+ if [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH up -d 2>&1
+ else
+ IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1
+ fi
  else
- IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1
+ if [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA up -d 2>&1
+ else
+ IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1
+ fi
  fi
  if [ $? -ne 0 ]; then
  echo "ERROR !!!! Unable to start network"
  exit 1
  fi
+
+ if [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ sleep 1
+ echo "Sleeping 10s to allow kafka cluster to complete booting"
+ sleep 9
+ fi
+
  # now run the end to end script
  docker exec cli scripts/script.sh $CHANNEL_NAME $CLI_DELAY $LANGUAGE $CLI_TIMEOUT $VERBOSE
  if [ $? -ne 0 ]; then
@@ -190,9 +206,17 @@ function upgradeNetwork() {
 
  export IMAGE_TAG=$IMAGETAG
  if [ "${IF_COUCHDB}" == "couchdb" ]; then
- COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH"
+ if [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH"
+ else
+ COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH"
+ fi
  else
- COMPOSE_FILES="-f $COMPOSE_FILE"
+ if [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA"
+ else
+ COMPOSE_FILES="-f $COMPOSE_FILE"
+ fi
  fi
 
  # removing the cli container
@@ -238,7 +262,8 @@ function upgradeNetwork() {
 # Tear down running network
 function networkDown() {
  # stop org3 containers also in addition to org1 and org2, in case we were running sample to add org3
- docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans
+ # stop kafka and zookeeper containers in case we're running with kafka consensus-type
+ docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans
 
  # Don't remove the generated artifacts -- note, the ledgers are always removed
  if [ "$MODE" != "restart" ]; then
@@ -382,8 +407,17 @@ function generateChannelArtifacts() {
  echo "##########################################################"
  # Note: For some unknown reason (at least for now) the block file can't be
  # named orderer.genesis.block or the orderer will fail to launch!
+ echo "CONSENSUS_TYPE="$CONSENSUS_TYPE
  set -x
- configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
+ if [ "$CONSENSUS_TYPE" == "solo" ]; then
+ configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
+ elif [ "$CONSENSUS_TYPE" == "kafka" ]; then
+ configtxgen -profile SampleDevModeKafka -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
+ else
+ set +x
+ echo "unrecognized CONSESUS_TYPE='$CONSENSUS_TYPE'. exiting"
+ exit 1
+ fi
  res=$?
  set +x
  if [ $res -ne 0 ]; then
@@ -448,11 +482,15 @@ COMPOSE_FILE=docker-compose-cli.yaml
 COMPOSE_FILE_COUCH=docker-compose-couch.yaml
 # org3 docker compose file
 COMPOSE_FILE_ORG3=docker-compose-org3.yaml
+# kafka and zookeeper compose file
+COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml
 #
 # use golang as the default language for chaincode
 LANGUAGE=golang
 # default image tag
 IMAGETAG="latest"
+# default consensus type
+CONSENSUS_TYPE="solo"
 # Parse commandline args
 if [ "$1" = "-m" ]; then # supports old usage, muscle memory is powerful!
  shift
@@ -475,7 +513,7 @@ else
  exit 1
 fi
 
-while getopts "h?c:t:d:f:s:l:i:v" opt; do
+while getopts "h?c:t:d:f:s:l:i:o:v" opt; do
  case "$opt" in
  h | \?)
  printHelp
@@ -502,6 +540,9 @@ while getopts "h?c:t:d:f:s:l:i:v" opt; do
  i)
  IMAGETAG=$(go env GOARCH)"-"$OPTARG
  ;;
+ o)
+ CONSENSUS_TYPE=$OPTARG
+ ;;
  v)
  VERBOSE=true
  ;;

first-network/configtx.yaml

@@ -321,3 +321,28 @@ Profiles:
  - *Org2
  Capabilities:
  <<: *ApplicationCapabilities
+
+ SampleDevModeKafka:
+ <<: *ChannelDefaults
+ Capabilities:
+ <<: *ChannelCapabilities
+ Orderer:
+ <<: *OrdererDefaults
+ OrdererType: kafka
+ Kafka:
+ Brokers:
+ - kafka.example.com:9092
+
+ Organizations:
+ - *OrdererOrg
+ Capabilities:
+ <<: *OrdererCapabilities
+ Application:
+ <<: *ApplicationDefaults
+ Organizations:
+ - <<: *OrdererOrg
+ Consortiums:
+ SampleConsortium:
+ Organizations:
+ - *Org1
+ - *Org2

first-network/docker-compose-kafka.yaml

@@ -0,0 +1,43 @@
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+# NOTE: This is not the way a Kafka cluster would normally be deployed in production, as it is not secure
+# and is not fault tolerant. This example is a toy deployment that is only meant to exercise the Kafka code path
+# of the ordering service.
+
+version: '2'
+
+networks:
+ byfn:
+
+services:
+ zookeeper.example.com:
+ container_name: zookeeper.example.com
+ image: hyperledger/fabric-zookeeper:$IMAGE_TAG
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 32181
+ ZOOKEEPER_TICK_TIME: 2000
+ networks:
+ - byfn
+
+ kafka.example.com:
+ container_name: kafka.example.com
+ image: hyperledger/fabric-kafka:$IMAGE_TAG
+ depends_on:
+ - zookeeper.example.com
+ environment:
+ - KAFKA_BROKER_ID=1
+ - KAFKA_ZOOKEEPER_CONNECT=zookeeper.example.com:2181
+ - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka.example.com:9092
+ - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+ - KAFKA_MESSAGE_MAX_BYTES=1048576 # 1 * 1024 * 1024 B
+ - KAFKA_REPLICA_FETCH_MAX_BYTES=1048576 # 1 * 1024 * 1024 B
+ - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false
+ - KAFKA_LOG_RETENTION_MS=-1
+ - KAFKA_MIN_INSYNC_REPLICAS=1
+ - KAFKA_DEFAULT_REPLICATION_FACTOR=1
+ networks:
+ - byfn

first-network/eyfn.sh

@@ -136,7 +136,7 @@ function networkUp () {
 
 # Tear down running network
 function networkDown () {
- docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans
+ docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans
  # Don't remove containers, images, etc if restarting
  if [ "$MODE" != "restart" ]; then
  #Cleanup the chaincode containers
@@ -245,6 +245,8 @@ COMPOSE_FILE_COUCH=docker-compose-couch.yaml
 COMPOSE_FILE_ORG3=docker-compose-org3.yaml
 #
 COMPOSE_FILE_COUCH_ORG3=docker-compose-couch-org3.yaml
+# kafka and zookeeper compose file
+COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml
 # use golang as the default language for chaincode
 LANGUAGE=golang
 # default image tag