diff --git a/first-network/base/docker-compose-base.yaml b/first-network/base/docker-compose-base.yaml index f4a19b618c..756ac97a1b 100644 --- a/first-network/base/docker-compose-base.yaml +++ b/first-network/base/docker-compose-base.yaml @@ -22,6 +22,8 @@ services: - ORDERER_GENERAL_TLS_PRIVATEKEY=/var/hyperledger/orderer/tls/server.key - ORDERER_GENERAL_TLS_CERTIFICATE=/var/hyperledger/orderer/tls/server.crt - ORDERER_GENERAL_TLS_ROOTCAS=[/var/hyperledger/orderer/tls/ca.crt] + - ORDERER_KAFKA_TOPIC_REPLICATIONFACTOR=1 + - ORDERER_KAFKA_VERBOSE=true working_dir: /opt/gopath/src/github.com/hyperledger/fabric command: orderer volumes: diff --git a/first-network/byfn.sh b/first-network/byfn.sh index 5573afe505..73d3b74085 100755 --- a/first-network/byfn.sh +++ b/first-network/byfn.sh @@ -35,7 +35,7 @@ export VERBOSE=false # Print the usage message function printHelp() { echo "Usage: " - echo " byfn.sh [-c ] [-t ] [-d ] [-f ] [-s ] [-l ] [-i ] [-v]" + echo " byfn.sh [-c ] [-t ] [-d ] [-f ] [-s ] [-l ] [-o ] [-i ] [-v]" echo " - one of 'up', 'down', 'restart', 'generate' or 'upgrade'" echo " - 'up' - bring up the network with docker-compose up" echo " - 'down' - clear the network with docker-compose down" @@ -48,6 +48,7 @@ function printHelp() { echo " -f - specify which docker-compose file use (defaults to docker-compose-cli.yaml)" echo " -s - the database backend to use: goleveldb (default) or couchdb" echo " -l - the chaincode language: golang (default) or node" + echo " -o - the consensus-type of the ordering service: solo (default) or kafka" echo " -i - the tag to be used to launch the network (defaults to \"latest\")" echo " -v - verbose mode" echo " byfn.sh -h (print this message)" @@ -156,14 +157,29 @@ function networkUp() { generateChannelArtifacts fi if [ "${IF_COUCHDB}" == "couchdb" ]; then - IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1 + if [ "$CONSENSUS_TYPE" == "kafka" ]; then + IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH up -d 2>&1 + else + IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH up -d 2>&1 + fi else - IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1 + if [ "$CONSENSUS_TYPE" == "kafka" ]; then + IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA up -d 2>&1 + else + IMAGE_TAG=$IMAGETAG docker-compose -f $COMPOSE_FILE up -d 2>&1 + fi fi if [ $? -ne 0 ]; then echo "ERROR !!!! Unable to start network" exit 1 fi + + if [ "$CONSENSUS_TYPE" == "kafka" ]; then + sleep 1 + echo "Sleeping 10s to allow kafka cluster to complete booting" + sleep 9 + fi + # now run the end to end script docker exec cli scripts/script.sh $CHANNEL_NAME $CLI_DELAY $LANGUAGE $CLI_TIMEOUT $VERBOSE if [ $? -ne 0 ]; then @@ -190,9 +206,17 @@ function upgradeNetwork() { export IMAGE_TAG=$IMAGETAG if [ "${IF_COUCHDB}" == "couchdb" ]; then - COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH" + if [ "$CONSENSUS_TYPE" == "kafka" ]; then + COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_COUCH" + else + COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH" + fi else - COMPOSE_FILES="-f $COMPOSE_FILE" + if [ "$CONSENSUS_TYPE" == "kafka" ]; then + COMPOSE_FILES="-f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA" + else + COMPOSE_FILES="-f $COMPOSE_FILE" + fi fi # removing the cli container @@ -238,7 +262,8 @@ function upgradeNetwork() { # Tear down running network function networkDown() { # stop org3 containers also in addition to org1 and org2, in case we were running sample to add org3 - docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans + # stop kafka and zookeeper containers in case we're running with kafka consensus-type + docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_COUCH -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 down --volumes --remove-orphans # Don't remove the generated artifacts -- note, the ledgers are always removed if [ "$MODE" != "restart" ]; then @@ -382,8 +407,17 @@ function generateChannelArtifacts() { echo "##########################################################" # Note: For some unknown reason (at least for now) the block file can't be # named orderer.genesis.block or the orderer will fail to launch! + echo "CONSENSUS_TYPE="$CONSENSUS_TYPE set -x - configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block + if [ "$CONSENSUS_TYPE" == "solo" ]; then + configtxgen -profile TwoOrgsOrdererGenesis -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block + elif [ "$CONSENSUS_TYPE" == "kafka" ]; then + configtxgen -profile SampleDevModeKafka -channelID byfn-sys-channel -outputBlock ./channel-artifacts/genesis.block + else + set +x + echo "unrecognized CONSESUS_TYPE='$CONSENSUS_TYPE'. exiting" + exit 1 + fi res=$? set +x if [ $res -ne 0 ]; then @@ -448,11 +482,15 @@ COMPOSE_FILE=docker-compose-cli.yaml COMPOSE_FILE_COUCH=docker-compose-couch.yaml # org3 docker compose file COMPOSE_FILE_ORG3=docker-compose-org3.yaml +# kafka and zookeeper compose file +COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml # # use golang as the default language for chaincode LANGUAGE=golang # default image tag IMAGETAG="latest" +# default consensus type +CONSENSUS_TYPE="solo" # Parse commandline args if [ "$1" = "-m" ]; then # supports old usage, muscle memory is powerful! shift @@ -475,7 +513,7 @@ else exit 1 fi -while getopts "h?c:t:d:f:s:l:i:v" opt; do +while getopts "h?c:t:d:f:s:l:i:o:v" opt; do case "$opt" in h | \?) printHelp @@ -502,6 +540,9 @@ while getopts "h?c:t:d:f:s:l:i:v" opt; do i) IMAGETAG=$(go env GOARCH)"-"$OPTARG ;; + o) + CONSENSUS_TYPE=$OPTARG + ;; v) VERBOSE=true ;; diff --git a/first-network/configtx.yaml b/first-network/configtx.yaml index b5ac5aa07c..3d29ebfce0 100644 --- a/first-network/configtx.yaml +++ b/first-network/configtx.yaml @@ -321,3 +321,28 @@ Profiles: - *Org2 Capabilities: <<: *ApplicationCapabilities + + SampleDevModeKafka: + <<: *ChannelDefaults + Capabilities: + <<: *ChannelCapabilities + Orderer: + <<: *OrdererDefaults + OrdererType: kafka + Kafka: + Brokers: + - kafka.example.com:9092 + + Organizations: + - *OrdererOrg + Capabilities: + <<: *OrdererCapabilities + Application: + <<: *ApplicationDefaults + Organizations: + - <<: *OrdererOrg + Consortiums: + SampleConsortium: + Organizations: + - *Org1 + - *Org2 \ No newline at end of file diff --git a/first-network/docker-compose-kafka.yaml b/first-network/docker-compose-kafka.yaml new file mode 100644 index 0000000000..06deac8515 --- /dev/null +++ b/first-network/docker-compose-kafka.yaml @@ -0,0 +1,43 @@ +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# NOTE: This is not the way a Kafka cluster would normally be deployed in production, as it is not secure +# and is not fault tolerant. This example is a toy deployment that is only meant to exercise the Kafka code path +# of the ordering service. + +version: '2' + +networks: + byfn: + +services: + zookeeper.example.com: + container_name: zookeeper.example.com + image: hyperledger/fabric-zookeeper:$IMAGE_TAG + environment: + ZOOKEEPER_CLIENT_PORT: 32181 + ZOOKEEPER_TICK_TIME: 2000 + networks: + - byfn + + kafka.example.com: + container_name: kafka.example.com + image: hyperledger/fabric-kafka:$IMAGE_TAG + depends_on: + - zookeeper.example.com + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper.example.com:2181 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka.example.com:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_MESSAGE_MAX_BYTES=1048576 # 1 * 1024 * 1024 B + - KAFKA_REPLICA_FETCH_MAX_BYTES=1048576 # 1 * 1024 * 1024 B + - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false + - KAFKA_LOG_RETENTION_MS=-1 + - KAFKA_MIN_INSYNC_REPLICAS=1 + - KAFKA_DEFAULT_REPLICATION_FACTOR=1 + networks: + - byfn diff --git a/first-network/eyfn.sh b/first-network/eyfn.sh index cee408b829..c915189b36 100755 --- a/first-network/eyfn.sh +++ b/first-network/eyfn.sh @@ -136,7 +136,7 @@ function networkUp () { # Tear down running network function networkDown () { - docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans + docker-compose -f $COMPOSE_FILE -f $COMPOSE_FILE_KAFKA -f $COMPOSE_FILE_ORG3 -f $COMPOSE_FILE_COUCH down --volumes --remove-orphans # Don't remove containers, images, etc if restarting if [ "$MODE" != "restart" ]; then #Cleanup the chaincode containers @@ -245,6 +245,8 @@ COMPOSE_FILE_COUCH=docker-compose-couch.yaml COMPOSE_FILE_ORG3=docker-compose-org3.yaml # COMPOSE_FILE_COUCH_ORG3=docker-compose-couch-org3.yaml +# kafka and zookeeper compose file +COMPOSE_FILE_KAFKA=docker-compose-kafka.yaml # use golang as the default language for chaincode LANGUAGE=golang # default image tag