From ad22d8e5aa2075b976dba1c5de029001b542cdea Mon Sep 17 00:00:00 2001 From: Jason Yellick Date: Thu, 9 Feb 2017 15:10:04 -0500 Subject: [PATCH] [FAB-2156] Move orderer viper utils to common https://jira.hyperledger.org/browse/FAB-2156 Before moving the orderer genesis tooling to common, it requires some viper extensions which resided in the orderer, this CR moves them to common. Change-Id: I5c68cad8903819d490a0e908a7ac07a2f6d86fdc Signed-off-by: Jason Yellick --- common/viperutil/config_test.go | 312 ++++++++++++++++++ .../viperutil}/config_util.go | 9 +- orderer/localconfig/config.go | 6 +- orderer/localconfig/config_test.go | 289 +--------------- 4 files changed, 325 insertions(+), 291 deletions(-) create mode 100644 common/viperutil/config_test.go rename {orderer/localconfig => common/viperutil}/config_util.go (96%) diff --git a/common/viperutil/config_test.go b/common/viperutil/config_test.go new file mode 100644 index 00000000000..93dcf9cea65 --- /dev/null +++ b/common/viperutil/config_test.go @@ -0,0 +1,312 @@ +/* +Copyright IBM Corp. 2016 All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "reflect" + "strings" + "testing" + + "github.com/hyperledger/fabric/orderer/mocks/util" + "github.com/spf13/viper" +) + +const Prefix = "VIPERUTIL" + +type testSlice struct { + Inner struct { + Slice []string + } +} + +func TestEnvSlice(t *testing.T) { + envVar := "VIPERUTIL_INNER_SLICE" + envVal := "[a, b, c]" + os.Setenv(envVar, envVal) + defer os.Unsetenv(envVar) + config := viper.New() + config.SetEnvPrefix(Prefix) + config.AutomaticEnv() + replacer := strings.NewReplacer(".", "_") + config.SetEnvKeyReplacer(replacer) + config.SetConfigType("yaml") + + data := "---\nInner:\n Slice: [d,e,f]" + + err := config.ReadConfig(bytes.NewReader([]byte(data))) + + if err != nil { + t.Fatalf("Error reading %s plugin config: %s", Prefix, err) + } + + var uconf testSlice + + err = EnhancedExactUnmarshal(config, &uconf) + if err != nil { + t.Fatalf("Failed to unmarshal with: %s", err) + } + + expected := []string{"a", "b", "c"} + if !reflect.DeepEqual(uconf.Inner.Slice, expected) { + t.Fatalf("Did not get back the right slice, expeced: %v got %v", expected, uconf.Inner.Slice) + } +} + +type testByteSize struct { + Inner struct { + ByteSize uint32 + } +} + +func TestByteSize(t *testing.T) { + config := viper.New() + config.SetConfigType("yaml") + + testCases := []struct { + data string + expected uint32 + }{ + {"", 0}, + {"42", 42}, + {"42k", 42 * 1024}, + {"42kb", 42 * 1024}, + {"42K", 42 * 1024}, + {"42KB", 42 * 1024}, + {"42 K", 42 * 1024}, + {"42 KB", 42 * 1024}, + {"42m", 42 * 1024 * 1024}, + {"42mb", 42 * 1024 * 1024}, + {"42M", 42 * 1024 * 1024}, + {"42MB", 42 * 1024 * 1024}, + {"42 M", 42 * 1024 * 1024}, + {"42 MB", 42 * 1024 * 1024}, + {"3g", 3 * 1024 * 1024 * 1024}, + {"3gb", 3 * 1024 * 1024 * 1024}, + {"3G", 3 * 1024 * 1024 * 1024}, + {"3GB", 3 * 1024 * 1024 * 1024}, + {"3 G", 3 * 1024 * 1024 * 1024}, + {"3 GB", 3 * 1024 * 1024 * 1024}, + } + + for _, tc := range testCases { + t.Run(tc.data, func(t *testing.T) { + data := fmt.Sprintf("---\nInner:\n ByteSize: %s", tc.data) + err := config.ReadConfig(bytes.NewReader([]byte(data))) + if err != nil { + t.Fatalf("Error reading config: %s", err) + } + var uconf testByteSize + err = EnhancedExactUnmarshal(config, &uconf) + if err != nil { + t.Fatalf("Failed to unmarshal with: %s", err) + } + if uconf.Inner.ByteSize != tc.expected { + t.Fatalf("Did not get back the right byte size, expeced: %v got %v", tc.expected, uconf.Inner.ByteSize) + } + }) + } +} + +func TestByteSizeOverflow(t *testing.T) { + config := viper.New() + config.SetConfigType("yaml") + + data := "---\nInner:\n ByteSize: 4GB" + err := config.ReadConfig(bytes.NewReader([]byte(data))) + if err != nil { + t.Fatalf("Error reading config: %s", err) + } + var uconf testByteSize + err = EnhancedExactUnmarshal(config, &uconf) + if err == nil { + t.Fatalf("Should have failed to unmarshal") + } +} + +type stringFromFileConfig struct { + Inner struct { + Single string + Multiple []string + } +} + +func TestStringNotFromFile(t *testing.T) { + + expectedValue := "expected_value" + yaml := fmt.Sprintf("---\nInner:\n Single: %s\n", expectedValue) + + config := viper.New() + config.SetConfigType("yaml") + + if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { + t.Fatalf("Error reading config: %s", err) + } + + var uconf stringFromFileConfig + if err := EnhancedExactUnmarshal(config, &uconf); err != nil { + t.Fatalf("Failed to unmarshall: %s", err) + } + + if uconf.Inner.Single != expectedValue { + t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single) + } + +} + +func TestStringFromFile(t *testing.T) { + + expectedValue := "this is the text in the file" + + // create temp file + file, err := ioutil.TempFile(os.TempDir(), "test") + if err != nil { + t.Fatalf("Unable to create temp file.") + } + defer os.Remove(file.Name()) + + // write temp file + if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil { + t.Fatalf("Unable to write to temp file.") + } + + yaml := fmt.Sprintf("---\nInner:\n Single:\n File: %s", file.Name()) + + config := viper.New() + config.SetConfigType("yaml") + + if err = config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { + t.Fatalf("Error reading config: %s", err) + } + var uconf stringFromFileConfig + if err = EnhancedExactUnmarshal(config, &uconf); err != nil { + t.Fatalf("Failed to unmarshall: %s", err) + } + + if uconf.Inner.Single != expectedValue { + t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single) + } +} + +func TestPEMBlocksFromFile(t *testing.T) { + + // create temp file + file, err := ioutil.TempFile(os.TempDir(), "test") + if err != nil { + t.Fatalf("Unable to create temp file.") + } + defer os.Remove(file.Name()) + + numberOfCertificates := 3 + var pems []byte + for i := 0; i < numberOfCertificates; i++ { + publicKeyCert, _, err := util.GenerateMockPublicPrivateKeyPairPEM(true) + if err != nil { + t.Fatalf("Enable to generate a signer certificate: %v", err) + } + pems = append(pems, publicKeyCert...) + } + + // write temp file + if err := ioutil.WriteFile(file.Name(), pems, 0666); err != nil { + t.Fatalf("Unable to write to temp file: %v", err) + } + + yaml := fmt.Sprintf("---\nInner:\n Multiple:\n File: %s", file.Name()) + + config := viper.New() + config.SetConfigType("yaml") + + if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { + t.Fatalf("Error reading config: %v", err) + } + var uconf stringFromFileConfig + if err := EnhancedExactUnmarshal(config, &uconf); err != nil { + t.Fatalf("Failed to unmarshall: %v", err) + } + + if len(uconf.Inner.Multiple) != 3 { + t.Fatalf(`Expected: "%v", Actual: "%v"`, numberOfCertificates, len(uconf.Inner.Multiple)) + } +} + +func TestStringFromFileNotSpecified(t *testing.T) { + + yaml := fmt.Sprintf("---\nInner:\n Single:\n File:\n") + + config := viper.New() + config.SetConfigType("yaml") + + if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { + t.Fatalf("Error reading config: %s", err) + } + var uconf stringFromFileConfig + if err := EnhancedExactUnmarshal(config, &uconf); err == nil { + t.Fatalf("Should of failed to unmarshall.") + } + +} + +func TestStringFromFileEnv(t *testing.T) { + + expectedValue := "this is the text in the file" + + // create temp file + file, err := ioutil.TempFile(os.TempDir(), "test") + if err != nil { + t.Fatalf("Unable to create temp file.") + } + defer os.Remove(file.Name()) + + // write temp file + if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil { + t.Fatalf("Unable to write to temp file.") + } + + envVar := "VIPERUTIL_INNER_SINGLE_FILE" + envVal := file.Name() + os.Setenv(envVar, envVal) + defer os.Unsetenv(envVar) + config := viper.New() + config.SetEnvPrefix(Prefix) + config.AutomaticEnv() + replacer := strings.NewReplacer(".", "_") + config.SetEnvKeyReplacer(replacer) + config.SetConfigType("yaml") + + data := "---\nInner:\n Single:\n File: wrong_file" + + if err = config.ReadConfig(bytes.NewReader([]byte(data))); err != nil { + t.Fatalf("Error reading %s plugin config: %s", Prefix, err) + } + + var uconf stringFromFileConfig + + err = EnhancedExactUnmarshal(config, &uconf) + if err != nil { + t.Fatalf("Failed to unmarshal with: %s", err) + } + + t.Log(uconf.Inner.Single) + + if !reflect.DeepEqual(uconf.Inner.Single, expectedValue) { + t.Fatalf(`Expected: "%v", Actual: "%v"`, expectedValue, uconf.Inner.Single) + } +} diff --git a/orderer/localconfig/config_util.go b/common/viperutil/config_util.go similarity index 96% rename from orderer/localconfig/config_util.go rename to common/viperutil/config_util.go index 008e3c662d5..4be9fb8b7bc 100644 --- a/orderer/localconfig/config_util.go +++ b/common/viperutil/config_util.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package config +package viperutil import ( "fmt" @@ -30,9 +30,12 @@ import ( "encoding/pem" "github.com/mitchellh/mapstructure" + "github.com/op/go-logging" "github.com/spf13/viper" ) +var logger = logging.MustGetLogger("viperutil") + func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}) for key := range nodeKeys { @@ -228,10 +231,10 @@ func pemBlocksFromFileDecodeHook() mapstructure.DecodeHookFunc { } } -// ExactWithDateUnmarshal is intended to unmarshal a config file into a structure +// EnhancedExactUnmarshal is intended to unmarshal a config file into a structure // producing error when extraneous variables are introduced and supporting // the time.Duration type -func ExactWithDateUnmarshal(v *viper.Viper, output interface{}) error { +func EnhancedExactUnmarshal(v *viper.Viper, output interface{}) error { baseKeys := v.AllSettings() // AllKeys doesn't actually return all keys, it only returns the base ones leafKeys := getKeysRecursively("", v, baseKeys) diff --git a/orderer/localconfig/config.go b/orderer/localconfig/config.go index ca50d375052..8d79c1caf15 100644 --- a/orderer/localconfig/config.go +++ b/orderer/localconfig/config.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "github.com/hyperledger/fabric/common/viperutil" + "github.com/Shopify/sarama" "github.com/op/go-logging" "github.com/spf13/viper" @@ -358,7 +360,7 @@ func LoadGenesis() *GenesisTopLevel { var uconf GenesisTopLevel - err = ExactWithDateUnmarshal(config, &uconf) + err = viperutil.EnhancedExactUnmarshal(config, &uconf) if err != nil { panic(fmt.Errorf("Error unmarshaling into structure: %s", err)) } @@ -407,7 +409,7 @@ func Load() *TopLevel { var uconf TopLevel - err = ExactWithDateUnmarshal(config, &uconf) + err = viperutil.EnhancedExactUnmarshal(config, &uconf) if err != nil { panic(fmt.Errorf("Error unmarshaling into structure: %s", err)) } diff --git a/orderer/localconfig/config_test.go b/orderer/localconfig/config_test.go index bdc83bc6fbd..7e7e7db391e 100644 --- a/orderer/localconfig/config_test.go +++ b/orderer/localconfig/config_test.go @@ -17,16 +17,13 @@ limitations under the License. package config import ( - "bytes" "fmt" - "io/ioutil" "os" - "reflect" - "strings" "testing" "time" - "github.com/hyperledger/fabric/orderer/mocks/util" + "github.com/hyperledger/fabric/common/viperutil" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) @@ -51,117 +48,7 @@ func TestBadConfig(t *testing.T) { var uconf struct{} - err = ExactWithDateUnmarshal(config, &uconf) - if err == nil { - t.Fatalf("Should have failed to unmarshal") - } -} - -type testSlice struct { - Inner struct { - Slice []string - } -} - -func TestEnvSlice(t *testing.T) { - envVar := "ORDERER_INNER_SLICE" - envVal := "[a, b, c]" - os.Setenv(envVar, envVal) - defer os.Unsetenv(envVar) - config := viper.New() - config.SetEnvPrefix(Prefix) - config.AutomaticEnv() - replacer := strings.NewReplacer(".", "_") - config.SetEnvKeyReplacer(replacer) - config.SetConfigType("yaml") - - data := "---\nInner:\n Slice: [d,e,f]" - - err := config.ReadConfig(bytes.NewReader([]byte(data))) - - if err != nil { - t.Fatalf("Error reading %s plugin config: %s", Prefix, err) - } - - var uconf testSlice - - err = ExactWithDateUnmarshal(config, &uconf) - if err != nil { - t.Fatalf("Failed to unmarshal with: %s", err) - } - - expected := []string{"a", "b", "c"} - if !reflect.DeepEqual(uconf.Inner.Slice, expected) { - t.Fatalf("Did not get back the right slice, expeced: %v got %v", expected, uconf.Inner.Slice) - } -} - -type testByteSize struct { - Inner struct { - ByteSize uint32 - } -} - -func TestByteSize(t *testing.T) { - config := viper.New() - config.SetConfigType("yaml") - - testCases := []struct { - data string - expected uint32 - }{ - {"", 0}, - {"42", 42}, - {"42k", 42 * 1024}, - {"42kb", 42 * 1024}, - {"42K", 42 * 1024}, - {"42KB", 42 * 1024}, - {"42 K", 42 * 1024}, - {"42 KB", 42 * 1024}, - {"42m", 42 * 1024 * 1024}, - {"42mb", 42 * 1024 * 1024}, - {"42M", 42 * 1024 * 1024}, - {"42MB", 42 * 1024 * 1024}, - {"42 M", 42 * 1024 * 1024}, - {"42 MB", 42 * 1024 * 1024}, - {"3g", 3 * 1024 * 1024 * 1024}, - {"3gb", 3 * 1024 * 1024 * 1024}, - {"3G", 3 * 1024 * 1024 * 1024}, - {"3GB", 3 * 1024 * 1024 * 1024}, - {"3 G", 3 * 1024 * 1024 * 1024}, - {"3 GB", 3 * 1024 * 1024 * 1024}, - } - - for _, tc := range testCases { - t.Run(tc.data, func(t *testing.T) { - data := fmt.Sprintf("---\nInner:\n ByteSize: %s", tc.data) - err := config.ReadConfig(bytes.NewReader([]byte(data))) - if err != nil { - t.Fatalf("Error reading config: %s", err) - } - var uconf testByteSize - err = ExactWithDateUnmarshal(config, &uconf) - if err != nil { - t.Fatalf("Failed to unmarshal with: %s", err) - } - if uconf.Inner.ByteSize != tc.expected { - t.Fatalf("Did not get back the right byte size, expeced: %v got %v", tc.expected, uconf.Inner.ByteSize) - } - }) - } -} - -func TestByteSizeOverflow(t *testing.T) { - config := viper.New() - config.SetConfigType("yaml") - - data := "---\nInner:\n ByteSize: 4GB" - err := config.ReadConfig(bytes.NewReader([]byte(data))) - if err != nil { - t.Fatalf("Error reading config: %s", err) - } - var uconf testByteSize - err = ExactWithDateUnmarshal(config, &uconf) + err = viperutil.EnhancedExactUnmarshal(config, &uconf) if err == nil { t.Fatalf("Should have failed to unmarshal") } @@ -217,173 +104,3 @@ func TestKafkaTLSConfig(t *testing.T) { }) } } - -type stringFromFileConfig struct { - Inner struct { - Single string - Multiple []string - } -} - -func TestStringNotFromFile(t *testing.T) { - - expectedValue := "expected_value" - yaml := fmt.Sprintf("---\nInner:\n Single: %s\n", expectedValue) - - config := viper.New() - config.SetConfigType("yaml") - - if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { - t.Fatalf("Error reading config: %s", err) - } - - var uconf stringFromFileConfig - if err := ExactWithDateUnmarshal(config, &uconf); err != nil { - t.Fatalf("Failed to unmarshall: %s", err) - } - - if uconf.Inner.Single != expectedValue { - t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single) - } - -} - -func TestStringFromFile(t *testing.T) { - - expectedValue := "this is the text in the file" - - // create temp file - file, err := ioutil.TempFile(os.TempDir(), "test") - if err != nil { - t.Fatalf("Unable to create temp file.") - } - defer os.Remove(file.Name()) - - // write temp file - if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil { - t.Fatalf("Unable to write to temp file.") - } - - yaml := fmt.Sprintf("---\nInner:\n Single:\n File: %s", file.Name()) - - config := viper.New() - config.SetConfigType("yaml") - - if err = config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { - t.Fatalf("Error reading config: %s", err) - } - var uconf stringFromFileConfig - if err = ExactWithDateUnmarshal(config, &uconf); err != nil { - t.Fatalf("Failed to unmarshall: %s", err) - } - - if uconf.Inner.Single != expectedValue { - t.Fatalf(`Expected: "%s", Actual: "%s"`, expectedValue, uconf.Inner.Single) - } -} - -func TestPEMBlocksFromFile(t *testing.T) { - - // create temp file - file, err := ioutil.TempFile(os.TempDir(), "test") - if err != nil { - t.Fatalf("Unable to create temp file.") - } - defer os.Remove(file.Name()) - - numberOfCertificates := 3 - var pems []byte - for i := 0; i < numberOfCertificates; i++ { - publicKeyCert, _, err := util.GenerateMockPublicPrivateKeyPairPEM(true) - if err != nil { - t.Fatalf("Enable to generate a signer certificate: %v", err) - } - pems = append(pems, publicKeyCert...) - } - - // write temp file - if err := ioutil.WriteFile(file.Name(), pems, 0666); err != nil { - t.Fatalf("Unable to write to temp file: %v", err) - } - - yaml := fmt.Sprintf("---\nInner:\n Multiple:\n File: %s", file.Name()) - - config := viper.New() - config.SetConfigType("yaml") - - if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { - t.Fatalf("Error reading config: %v", err) - } - var uconf stringFromFileConfig - if err := ExactWithDateUnmarshal(config, &uconf); err != nil { - t.Fatalf("Failed to unmarshall: %v", err) - } - - if len(uconf.Inner.Multiple) != 3 { - t.Fatalf(`Expected: "%v", Actual: "%v"`, numberOfCertificates, len(uconf.Inner.Multiple)) - } -} - -func TestStringFromFileNotSpecified(t *testing.T) { - - yaml := fmt.Sprintf("---\nInner:\n Single:\n File:\n") - - config := viper.New() - config.SetConfigType("yaml") - - if err := config.ReadConfig(bytes.NewReader([]byte(yaml))); err != nil { - t.Fatalf("Error reading config: %s", err) - } - var uconf stringFromFileConfig - if err := ExactWithDateUnmarshal(config, &uconf); err == nil { - t.Fatalf("Should of failed to unmarshall.") - } - -} - -func TestStringFromFileEnv(t *testing.T) { - - expectedValue := "this is the text in the file" - - // create temp file - file, err := ioutil.TempFile(os.TempDir(), "test") - if err != nil { - t.Fatalf("Unable to create temp file.") - } - defer os.Remove(file.Name()) - - // write temp file - if err = ioutil.WriteFile(file.Name(), []byte(expectedValue), 0777); err != nil { - t.Fatalf("Unable to write to temp file.") - } - - envVar := "ORDERER_INNER_SINGLE_FILE" - envVal := file.Name() - os.Setenv(envVar, envVal) - defer os.Unsetenv(envVar) - config := viper.New() - config.SetEnvPrefix(Prefix) - config.AutomaticEnv() - replacer := strings.NewReplacer(".", "_") - config.SetEnvKeyReplacer(replacer) - config.SetConfigType("yaml") - - data := "---\nInner:\n Single:\n File: wrong_file" - - if err = config.ReadConfig(bytes.NewReader([]byte(data))); err != nil { - t.Fatalf("Error reading %s plugin config: %s", Prefix, err) - } - - var uconf stringFromFileConfig - - err = ExactWithDateUnmarshal(config, &uconf) - if err != nil { - t.Fatalf("Failed to unmarshal with: %s", err) - } - - t.Log(uconf.Inner.Single) - - if !reflect.DeepEqual(uconf.Inner.Single, expectedValue) { - t.Fatalf(`Expected: "%v", Actual: "%v"`, expectedValue, uconf.Inner.Single) - } -}