[FAB-3387] Clean up Docker Compose configs

Some cruft has been accumulated over time on the orderer-related Docker
Compose configurations under the "bddtests" folder. This changeset:

1. Removes environment variables that are either deprecated (e.g.
`ORDERER_RAMLEDGER`) or not in use by the BDD tests (e.g. all `CONFIGTX`
variables since `bootstrap.feature` does not use the `configtxgen` tool,
or the `provisional` bootstrapper).

2. Removes configurations that are either no longer relevant (e.g.
`bdd-docker/docker-compose-4-consensus-batch-nosnapshotbuffer.yml`) or
no longer in use.

3. Moves all Docker Compose files to the `bddtests` folder and renames
them to filenames that are hopefully a bit more expressive. Thanks to
Jeff Garratt for guiding me through his.

4. Modified the Kafka environment so that it uses the bare-minimum
sensisble defaults, and added comments for the critical settings.

UPDATE: Rebased, added more comments, expanded base orderer
configuration to a ZK ensemble consisting of 3 ZK servers, and a Kafka
cluster consisting of 4 brokers, modified the `SystemUpWaitTime`
accordingly.

Change-Id: Iac0f55a0bfc3a7a67b70d3de0918cf727058ae16
Signed-off-by: Kostas Christidis <kostas@christidis.io>

Makefile

@@ -151,14 +151,7 @@ test-cmd:
 docker: $(patsubst %,build/image/%/$(DUMMY), $(IMAGES))
 native: peer orderer configtxgen cryptogen
 
-BEHAVE_ENVIRONMENTS = kafka orderer-1-kafka-1 orderer-1-kafka-3
-BEHAVE_ENVIRONMENT_TARGETS = $(patsubst %,bddtests/environments/%, $(BEHAVE_ENVIRONMENTS))
-.PHONY: behave-environments $(BEHAVE_ENVIRONMENT_TARGETS)
-behave-environments: $(BEHAVE_ENVIRONMENT_TARGETS)
-$(BEHAVE_ENVIRONMENT_TARGETS):
-	@docker-compose --file $@/docker-compose.yml build
-
-behave-deps: docker peer build/bin/block-listener behave-environments configtxgen cryptogen
+behave-deps: docker peer build/bin/block-listener configtxgen cryptogen
 behave: behave-deps
 	@echo "Running behave tests"
 	@cd bddtests; behave $(BEHAVE_OPTS)

bddtests/docker-compose-next-4.yml → bddtests/dc-base.yml

@@ -9,21 +9,19 @@ services:
 
   orderer0:
     extends:
-      file: docker-compose-orderer-solo.yml
-      service: orderer0
+      file: dc-orderer-base.yml
+      service: orderer
     environment:
       - ORDERER_GENERAL_LOCALMSPDIR=${ORDERER0_ORDERER_GENERAL_LOCALMSPDIR}
       - ORDERER_GENERAL_LOCALMSPID=${ORDERER0_ORDERER_GENERAL_LOCALMSPID}
-      - ORDERER_GENERAL_TLS_ENABLED=true
       - ORDERER_GENERAL_TLS_PRIVATEKEY=${ORDERER0_ORDERER_GENERAL_TLS_PRIVATEKEY}
       - ORDERER_GENERAL_TLS_CERTIFICATE=${ORDERER0_ORDERER_GENERAL_TLS_CERTIFICATE}
       - ORDERER_GENERAL_TLS_ROOTCAS=${ORDERER0_ORDERER_GENERAL_TLS_ROOTCAS}
 
-
   peer0:
     extends:
-      file: docker-compose-next.yml
-      service: vpNext
+      file: dc-peer-base.yml
+      service: peer
     environment:
       - CORE_PEER_ID=vp0
       - CORE_PEER_GOSSIP_BOOTSTRAP=peer1:7051
@@ -37,14 +35,13 @@ services:
       - CORE_PEER_TLS_SERVERHOSTOVERRIDE=${PEER0_CORE_PEER_TLS_SERVERHOSTOVERRIDE}
     depends_on:
       - orderer0
-    # ports:
-    #   - 7050:6060
-
+    #ports:
+       #- 7050:6060
 
   peer1:
     extends:
-      file: docker-compose-next.yml
-      service: vpNext
+      file: dc-peer-base.yml
+      service: peer
     environment:
       - CORE_PEER_ID=vp1
       - CORE_PEER_GOSSIP_BOOTSTRAP=peer0:7051
@@ -61,8 +58,8 @@ services:
 
   peer2:
     extends:
-      file: docker-compose-next.yml
-      service: vpNext
+      file: dc-peer-base.yml
+      service: peer
     environment:
       - CORE_PEER_ID=vp2
       - CORE_PEER_GOSSIP_BOOTSTRAP=peer3:7051
@@ -79,8 +76,8 @@ services:
 
   peer3:
     extends:
-      file: docker-compose-next.yml
-      service: vpNext
+      file: dc-peer-base.yml
+      service: peer
     environment:
       - CORE_PEER_ID=vp3
       - CORE_PEER_GOSSIP_BOOTSTRAP=peer2:7051

bddtests/dc-orderer-base.yml

@@ -0,0 +1,25 @@
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+version: '2'
+
+services:
+
+  orderer:
+    image: hyperledger/fabric-orderer
+    environment:
+      - ORDERER_GENERAL_GENESISMETHOD=${ORDERER_GENERAL_GENESISMETHOD}
+      - ORDERER_GENERAL_GENESISFILE=${ORDERER_GENERAL_GENESISFILE}
+      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
+      - ORDERER_GENERAL_LOGLEVEL=debug
+      - ORDERER_KAFKA_VERBOSE=true
+      - ORDERER_GENERAL_TLS_ENABLED=true
+    volumes:
+      - ./volumes/orderer:/var/hyperledger/bddtests/volumes/orderer
+      - /etc/hyperledger/msp:/etc/hyperledger/msp
+    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
+    command: orderer
+    ports:
+      - '7050'

bddtests/dc-orderer-kafka-base.yml

@@ -0,0 +1,52 @@
+# Copyright IBM Corp. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+version: '2'
+
+services:
+
+  zookeeper:
+    image: hyperledger/fabric-zookeeper
+    restart: always
+    ports:
+      - '2181'
+      - '2888'
+      - '3888'
+
+  kafka:
+    image: hyperledger/fabric-kafka
+    restart: always
+    environment:
+      # ========================================================================
+      #     Reference: https://kafka.apache.org/documentation/#configuration
+      # ========================================================================
+      #
+      # socket.request.max.bytes
+      # The maximum number of bytes in a socket request. ATTN: If you set this
+      # env var, you should make sure that the value assigned to
+      # `brokerConfig.Producer.MaxMessageBytes` in `newBrokerConfig()` in
+      # `fabric/orderer/kafka/util.go` matches it.
+      #- KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 # 100 * 1024 * 1024 B
+      #
+      # message.max.bytes
+      # The maximum size of envelope that the broker can receive.
+      - KAFKA_MESSAGE_MAX_BYTES=103809024 # 99 * 1024 * 1024 B
+      #
+      # replica.fetch.max.bytes
+      # The number of bytes of messages to attempt to fetch for each channel.
+      # This is not an absolute maximum, if the fetched envelope is larger than
+      # this value, the envelope will still be returned to ensure that progress
+      # can be made. The maximum message size accepted by the broker is defined
+      # via message.max.bytes above.
+      - KAFKA_REPLICA_FETCH_MAX_BYTES=103809024 # 99 * 1024 * 1024 B
+      #
+      # unclean.leader.election.enable
+      # Data consistency is key in a blockchain environment. We cannot have a
+      # leader chosen outside of the in-sync replica set, or we run the risk of
+      # overwriting the offsets that the previous leader produced, and --as a
+      # result-- rewriting the blockchain that the orderers produce.
+      - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false
+    ports:
+      - '9092'