Skip to content

Commit

Permalink
FAB-13011 add kafka consensus type to byfn sample
Browse files Browse the repository at this point in the history
Augment the fabric-samples first-network sample to include a flag that allows a choice of  consensus-type:
solo (default) or kafka.

- Add note on production deployment of kafka
- Remove port mapping for kafka and zookeeper containars

Change-Id: I3204ba44e5974961eb69649b6f5744f52097ce80
Signed-off-by: Yoav Tock <tock@il.ibm.com>
  • Loading branch information
tock-ibm committed Dec 5, 2018
1 parent 4bde7d4 commit 7ad9f19
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 9 deletions.
2 changes: 2 additions & 0 deletions first-network/base/docker-compose-base.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ services:
- ORDERER_GENERAL_TLS_PRIVATEKEY=/var/hyperledger/orderer/tls/server.key
- ORDERER_GENERAL_TLS_CERTIFICATE=/var/hyperledger/orderer/tls/server.crt
- ORDERER_GENERAL_TLS_ROOTCAS=[/var/hyperledger/orderer/tls/ca.crt]
- ORDERER_KAFKA_TOPIC_REPLICATIONFACTOR=1
- ORDERER_KAFKA_VERBOSE=true
working_dir: /opt/gopath/src/github.com/hyperledger/fabric
command: orderer
volumes:
Expand Down
57 changes: 49 additions & 8 deletions first-network/byfn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export VERBOSE=false
# Print the usage message
function printHelp() {
echo "Usage: "
echo " byfn.sh <mode> [-c <channel name>] [-t <timeout>] [-d <delay>] [-f <docker-compose-file>] [-s <dbtype>] [-l <language>] [-i <imagetag>] [-v]"
echo " byfn.sh <mode> [-c <channel name>] [-t <timeout>] [-d <delay>] [-f <docker-compose-file>] [-s <dbtype>] [-l <language>] [-o <consensus-type>] [-i <imagetag>] [-v]"
echo " <mode> - one of 'up', 'down', 'restart', 'generate' or 'upgrade'"
echo " - 'up' - bring up the network with docker-compose up"
echo " - 'down' - clear the network with docker-compose down"
Expand All @@ -48,6 +48,7 @@ function printHelp() {
echo " -f <docker-compose-file> - specify which docker-compose file use (defaults to docker-compose-cli.yaml)"
echo " -s <dbtype> - the database backend to use: goleveldb (default) or couchdb"
echo " -l <language> - the chaincode language: golang (default) or node"
echo " -o <consensus-type> - the consensus-type of the ordering service: solo (default) or kafka"
echo " -i <imagetag> - the tag to be used to launch the network (defaults to \"latest\")"
echo " -v - verbose mode"
echo " byfn.sh -h (print this message)"
Expand Down Expand Up @@ -156,14 +157,29 @@ function networkUp() {
generateChannelArtifacts
fi
if [ "${IF_COUCHDB}" == "couchdb" ]; then
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1
if [ "$CONSENSUS_TYPE" == "kafka" ]; then
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH up -d 2>&1
else
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1
fi
else
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1
if [ "$CONSENSUS_TYPE" == "kafka" ]; then
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA up -d 2>&1
else
IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1
fi
fi
if [ $? -ne 0 ]; then
echo "ERROR !!!! Unable to start network"
exit 1
fi

if [ "$CONSENSUS_TYPE" == "kafka" ]; then
sleep 1
echo "Sleeping 10s to allow kafka cluster to complete booting"
sleep 9
fi

# now run the end to end script
docker exec cli scripts/script.sh $CHANNEL_NAME $CLI_DELAY $LANGUAGE $CLI_TIMEOUT $VERBOSE
if [ $? -ne 0 ]; then
Expand All @@ -190,9 +206,17 @@ function upgradeNetwork() {

export IMAGE_TAG=$IMAGETAG
if [ "${IF_COUCHDB}" == "couchdb" ]; then
COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH"
if [ "$CONSENSUS_TYPE" == "kafka" ]; then
COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH"
else
COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH"
fi
else
COMPOSE_FILES="-f $COMPOSE_FILE"
if [ "$CONSENSUS_TYPE" == "kafka" ]; then
COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA"
else
COMPOSE_FILES="-f $COMPOSE_FILE"
fi
fi

# removing the cli container
Expand Down Expand Up @@ -238,7 +262,8 @@ function upgradeNetwork() {
# Tear down running network
function networkDown() {
# stop org3 containers also in addition to org1 and org2, in case we were running sample to add org3
docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans
# stop kafka and zookeeper containers in case we're running with kafka consensus-type
docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans

# Don't remove the generated artifacts -- note, the ledgers are always removed
if [ "$MODE" != "restart" ]; then
Expand Down Expand Up @@ -382,8 +407,17 @@ function generateChannelArtifacts() {
echo "##########################################################"
# Note: For some unknown reason (at least for now) the block file can't be
# named orderer.genesis.block or the orderer will fail to launch!
echo "CONSENSUS_TYPE="$CONSENSUS_TYPE
set -x
configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
if [ "$CONSENSUS_TYPE" == "solo" ]; then
configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
elif [ "$CONSENSUS_TYPE" == "kafka" ]; then
configtxgen -profile SampleDevModeKafka -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block
else
set +x
echo "unrecognized CONSESUS_TYPE='$CONSENSUS_TYPE'. exiting"
exit 1
fi
res=$?
set +x
if [ $res -ne 0 ]; then
Expand Down Expand Up @@ -448,11 +482,15 @@ COMPOSE_FILE=docker-compose-cli.yaml
COMPOSE_FILE_COUCH=docker-compose-couch.yaml
# org3 docker compose file
COMPOSE_FILE_ORG3=docker-compose-org3.yaml
# kafka and zookeeper compose file
COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml
#
# use golang as the default language for chaincode
LANGUAGE=golang
# default image tag
IMAGETAG="latest"
# default consensus type
CONSENSUS_TYPE="solo"
# Parse commandline args
if [ "$1" = "-m" ]; then # supports old usage, muscle memory is powerful!
shift
Expand All @@ -475,7 +513,7 @@ else
exit 1
fi

while getopts "h?c:t:d:f:s:l:i:v" opt; do
while getopts "h?c:t:d:f:s:l:i:o:v" opt; do
case "$opt" in
h | \?)
printHelp
Expand All @@ -502,6 +540,9 @@ while getopts "h?c:t:d:f:s:l:i:v" opt; do
i)
IMAGETAG=$(go env GOARCH)"-"$OPTARG
;;
o)
CONSENSUS_TYPE=$OPTARG
;;
v)
VERBOSE=true
;;
Expand Down
25 changes: 25 additions & 0 deletions first-network/configtx.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -319,3 +319,28 @@ Profiles:
- *Org2
Capabilities:
<<: *ApplicationCapabilities

SampleDevModeKafka:
<<: *ChannelDefaults
Capabilities:
<<: *ChannelCapabilities
Orderer:
<<: *OrdererDefaults
OrdererType: kafka
Kafka:
Brokers:
- kafka.example.com:9092

Organizations:
- *OrdererOrg
Capabilities:
<<: *OrdererCapabilities
Application:
<<: *ApplicationDefaults
Organizations:
- <<: *OrdererOrg
Consortiums:
SampleConsortium:
Organizations:
- *Org1
- *Org2
43 changes: 43 additions & 0 deletions first-network/docker-compose-kafka.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#


# NOTE: This is not the way a Kafka cluster would normally be deployed in production, as it is not secure
# and is not fault tolerant. This example is a toy deployment that is only meant to exercise the Kafka code path
# of the ordering service.

version: '2'

networks:
byfn:

services:
zookeeper.example.com:
container_name: zookeeper.example.com
image: hyperledger/fabric-zookeeper:$IMAGE_TAG
environment:
ZOOKEEPER_CLIENT_PORT: 32181
ZOOKEEPER_TICK_TIME: 2000
networks:
- byfn

kafka.example.com:
container_name: kafka.example.com
image: hyperledger/fabric-kafka:$IMAGE_TAG
depends_on:
- zookeeper.example.com
environment:
- KAFKA_BROKER_ID=1
- KAFKA_ZOOKEEPER_CONNECT=zookeeper.example.com:2181
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka.example.com:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- KAFKA_MESSAGE_MAX_BYTES=1048576 # 1 * 1024 * 1024 B
- KAFKA_REPLICA_FETCH_MAX_BYTES=1048576 # 1 * 1024 * 1024 B
- KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false
- KAFKA_LOG_RETENTION_MS=-1
- KAFKA_MIN_INSYNC_REPLICAS=1
- KAFKA_DEFAULT_REPLICATION_FACTOR=1
networks:
- byfn
4 changes: 3 additions & 1 deletion first-network/eyfn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ function networkUp () {

# Tear down running network
function networkDown () {
docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans
docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans
# Don't remove containers, images, etc if restarting
if [ "$MODE" != "restart" ]; then
#Cleanup the chaincode containers
Expand Down Expand Up @@ -245,6 +245,8 @@ COMPOSE_FILE_COUCH=docker-compose-couch.yaml
COMPOSE_FILE_ORG3=docker-compose-org3.yaml
#
COMPOSE_FILE_COUCH_ORG3=docker-compose-couch-org3.yaml
# kafka and zookeeper compose file
COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml
# use golang as the default language for chaincode
LANGUAGE=golang
# default image tag
Expand Down

0 comments on commit 7ad9f19

Please sign in to comment.