Skip to content

Commit

Permalink
FAB-16695 Enable split admin, cluster ports in IT
Browse files Browse the repository at this point in the history 
There was a bug where if the prometheus metrics provider was used, and
either the peer admin service port, or the orderer cluster port were set
to be different from the primary listen port, then metrics were double
registered, resulting in a peer crash.  This CR enables splitting these
ports in the integration tests, to confirm the bug has been eliminated,
and to detect other such regressions in the future.

Signed-off-by: Jason Yellick <jyellick@us.ibm.com>
Change-Id: I52d3721a51e80e1aefd15d59ca14bba9669bdd91
  • Loading branch information
Jason Yellick committed Sep 30, 2019
1 parent 23de844 commit 3345371
Show file tree
Hide file tree
Showing 7 changed files with 27 additions and 25 deletions.
5 changes: 0 additions & 5 deletions integration/e2e/cft_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,17 +446,12 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {

for i, runner := range []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} {
// Switch between the general port and the cluster listener port
port := network.OrdererPort(network.Orderers[i], nwo.ListenPort)
runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_LISTENPORT=%d", port))
runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_LISTENPORT=%d", network.ReservePort()))
runner.Command.Env = append(runner.Command.Env, "ORDERER_GENERAL_CLUSTER_TLSHANDSHAKETIMESHIFT=90s")
runner.Command.Env = append(runner.Command.Env, "ORDERER_GENERAL_CLUSTER_LISTENADDRESS=127.0.0.1")
tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(network.Orderers[i]), "server.crt")
tlsKeyPath := filepath.Join(network.OrdererLocalTLSDir(network.Orderers[i]), "server.key")
runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERCERTIFICATE=%s", tlsCertPath))
runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERPRIVATEKEY=%s", tlsKeyPath))
runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_ROOTCAS=%s", ordererTLSCACertPath))
runner.Command.Env = append(runner.Command.Env, "ORDERER_METRICS_PROVIDER=disabled")
}

o1Proc = ifrit.Invoke(o1Runner)
Expand Down
20 changes: 10 additions & 10 deletions integration/e2e/etcdraft_reconfig_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: secondOrdererCertificate,
ClientTlsCert: secondOrdererCertificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(orderer2, nwo.ListenPort)),
Port: uint32(network.OrdererPort(orderer2, nwo.ClusterPort)),
})

By("Obtaining the last config block from the orderer")
Expand Down Expand Up @@ -242,7 +242,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
return ret
}()
submitterOrderer := network.Orderers[submitter]
port := network.OrdererPort(targetOrderer, nwo.ListenPort)
port := network.OrdererPort(targetOrderer, nwo.ClusterPort)

fmt.Fprintf(GinkgoWriter, "Rotating certificate of orderer node %d\n", target+1)
swap(submitterOrderer, rotation.oldCert, etcdraft.Consenter{
Expand All @@ -262,7 +262,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ChannelID: network.SystemChannel.Name,
Block: "newest",
OutputFile: "/dev/null",
Orderer: network.OrdererAddress(targetOrderer, nwo.ListenPort),
Orderer: network.OrdererAddress(targetOrderer, nwo.ClusterPort),
}
Eventually(func() string {
sess, err := network.OrdererAdminSession(targetOrderer, peer, c)
Expand Down Expand Up @@ -394,7 +394,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {

for i, rotation := range certificateRotations {
o := network.Orderers[i]
port := network.OrdererPort(o, nwo.ListenPort)
port := network.OrdererPort(o, nwo.ClusterPort)

By(fmt.Sprintf("Adding the future certificate of orderer node %d", i))
for _, channelName := range []string{"systemchannel", "testchannel"} {
Expand Down Expand Up @@ -468,7 +468,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: orderer4Certificate,
ClientTlsCert: orderer4Certificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(o4, nwo.ListenPort)),
Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)),
})
}

Expand Down Expand Up @@ -535,7 +535,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: orderer4Certificate,
ClientTlsCert: orderer4Certificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(o4, nwo.ListenPort)),
Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)),
})

By("Waiting for orderer4 and to replicate testchannel2")
Expand Down Expand Up @@ -619,7 +619,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: ordererCertificate,
ClientTlsCert: ordererCertificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(o2, nwo.ListenPort)),
Port: uint32(network.OrdererPort(o2, nwo.ClusterPort)),
})

By("Waiting for orderer2 to join the channel")
Expand All @@ -635,7 +635,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: ordererCertificate,
ClientTlsCert: ordererCertificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(o3, nwo.ListenPort)),
Port: uint32(network.OrdererPort(o3, nwo.ClusterPort)),
})

By("Waiting for orderer3 to join the channel")
Expand Down Expand Up @@ -816,7 +816,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: certificatesOfOrderers[0].oldCert,
ClientTlsCert: certificatesOfOrderers[0].oldCert,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(orderers[0], nwo.ListenPort)),
Port: uint32(network.OrdererPort(orderers[0], nwo.ClusterPort)),
})

By("Ensuring the re-added orderer joins the Raft cluster")
Expand Down Expand Up @@ -915,7 +915,7 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
ServerTlsCert: ordererCertificate,
ClientTlsCert: ordererCertificate,
Host: "127.0.0.1",
Port: uint32(network.OrdererPort(orderers[i], nwo.ListenPort)),
Port: uint32(network.OrdererPort(orderers[i], nwo.ClusterPort)),
})
blockSeq++

Expand Down
2 changes: 1 addition & 1 deletion integration/e2e/migration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1043,7 +1043,7 @@ func prepareRaftMetadata(network *nwo.Network) []byte {
fullTlsPath := network.OrdererLocalTLSDir(o)
certBytes, err := ioutil.ReadFile(filepath.Join(fullTlsPath, "server.crt"))
Expect(err).NotTo(HaveOccurred())
port := network.OrdererPort(o, nwo.ListenPort)
port := network.OrdererPort(o, nwo.ClusterPort)

consenter := &protosraft.Consenter{
ClientTlsCert: certBytes,
Expand Down
2 changes: 1 addition & 1 deletion integration/nwo/configtx_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ Profiles:{{ range .Profiles }}
SnapshotIntervalSize: 1 KB
Consenters:{{ range .Orderers }}{{ with $w.Orderer . }}
- Host: 127.0.0.1
Port: {{ $w.OrdererPort . "Listen" }}
Port: {{ $w.OrdererPort . "Cluster" }}
ClientTLSCert: {{ $w.OrdererLocalCryptoDir . "tls" }}/server.crt
ServerTLSCert: {{ $w.OrdererLocalCryptoDir . "tls" }}/server.crt
{{- end }}{{- end }}
Expand Down
1 change: 1 addition & 0 deletions integration/nwo/core_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ peer:
enabled: false
listenAddress: 127.0.0.1:{{ .PeerPort Peer "ProfilePort" }}
adminService:
listenAddress: 127.0.0.1:{{ .PeerPort Peer "AdminServicePort" }}
handlers:
authFilters:
- name: DefaultAuth
Expand Down
18 changes: 10 additions & 8 deletions integration/nwo/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1277,23 +1277,25 @@ type PortName string
type Ports map[PortName]uint16

const (
ChaincodePort PortName = "Chaincode"
EventsPort PortName = "Events"
HostPort PortName = "HostPort"
ListenPort PortName = "Listen"
ProfilePort PortName = "Profile"
OperationsPort PortName = "Operations"
ChaincodePort PortName = "Chaincode"
EventsPort PortName = "Events"
HostPort PortName = "HostPort"
ListenPort PortName = "Listen"
ProfilePort PortName = "Profile"
OperationsPort PortName = "Operations"
AdminServicePort PortName = "AdminService"
ClusterPort PortName = "Cluster"
)

// PeerPortNames returns the list of ports that need to be reserved for a Peer.
func PeerPortNames() []PortName {
return []PortName{ListenPort, ChaincodePort, EventsPort, ProfilePort, OperationsPort}
return []PortName{ListenPort, ChaincodePort, EventsPort, ProfilePort, OperationsPort, AdminServicePort}
}

// OrdererPortNames returns the list of ports that need to be reserved for an
// Orderer.
func OrdererPortNames() []PortName {
return []PortName{ListenPort, ProfilePort, OperationsPort}
return []PortName{ListenPort, ProfilePort, OperationsPort, ClusterPort}
}

// BrokerPortNames returns the list of ports that need to be reserved for a
Expand Down
4 changes: 4 additions & 0 deletions integration/nwo/orderer_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,15 @@ General:
Cluster:
ClientCertificate: {{ $w.OrdererLocalTLSDir Orderer }}/server.crt
ClientPrivateKey: {{ $w.OrdererLocalTLSDir Orderer }}/server.key
ServerCertificate: {{ $w.OrdererLocalTLSDir Orderer }}/server.crt
ServerPrivateKey: {{ $w.OrdererLocalTLSDir Orderer }}/server.key
DialTimeout: 5s
RPCTimeout: 7s
ReplicationBufferSize: 20971520
ReplicationPullTimeout: 5s
ReplicationRetryTimeout: 5s
ListenAddress: 127.0.0.1
ListenPort: {{ .OrdererPort Orderer "Cluster" }}
Keepalive:
ServerMinInterval: 60s
ServerInterval: 7200s
Expand Down

0 comments on commit 3345371

Please sign in to comment.