[FAB-4102] Proto translator statically opaque comp

The proto translation framework introduced in FAB-4100 needs to be
applied to messages with opaque fields.  This is one of the key goals of
proto translation, which prevents the protos as defined from being human
readable.

This CR adds a statically opaque proto msg handler to the proto
translation framework.

Change-Id: Ifebc46f3bd3ec2087a9f2d579de30da5a3cb784d
Signed-off-by: Jason Yellick <jyellick@us.ibm.com>

common/tools/protolator/api.go

@@ -0,0 +1,63 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protolator
+
+import (
+	"github.com/golang/protobuf/proto"
+)
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// This set of interfaces and methods is designed to allow protos to have Go methods attached
+// to them, so that they may be automatically marshaled to human readable JSON (where the
+// opaque byte fields are represented as their expanded proto contents) and back once again
+// to standard proto messages.
+//
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+// StaticallyOpaqueFieldProto should be implemented by protos which have bytes fields which
+// are the marshaled value of a fixed type
+type StaticallyOpaqueFieldProto interface {
+	// StaticallyOpaqueFields returns the field names which contain opaque data
+	StaticallyOpaqueFields() []string
+
+	// StaticallyOpaqueFieldProto returns a newly allocated proto message of the correct
+	// type for the field name.
+	StaticallyOpaqueFieldProto(name string) (proto.Message, error)
+}
+
+// StaticallyOpaqueMapFieldProto should be implemented by protos which have maps to bytes fields
+// which are the marshaled value of a fixed type
+type StaticallyOpaqueMapFieldProto interface {
+	// StaticallyOpaqueFields returns the field names which contain opaque data
+	StaticallyOpaqueMapFields() []string
+
+	// StaticallyOpaqueFieldProto returns a newly allocated proto message of the correct
+	// type for the field name.
+	StaticallyOpaqueMapFieldProto(name string, key string) (proto.Message, error)
+}
+
+// StaticallyOpaqueSliceFieldProto should be implemented by protos which have maps to bytes fields
+// which are the marshaled value of a fixed type
+type StaticallyOpaqueSliceFieldProto interface {
+	// StaticallyOpaqueFields returns the field names which contain opaque data
+	StaticallyOpaqueSliceFields() []string
+
+	// StaticallyOpaqueFieldProto returns a newly allocated proto message of the correct
+	// type for the field name.
+	StaticallyOpaqueSliceFieldProto(name string, index int) (proto.Message, error)
+}

common/tools/protolator/json.go

@@ -57,6 +57,7 @@ type protoField interface {
 var (
 	protoMsgType           = reflect.TypeOf((*proto.Message)(nil)).Elem()
 	mapStringInterfaceType = reflect.TypeOf(map[string]interface{}{})
+	bytesType              = reflect.TypeOf([]byte{})
 )
 
 type baseField struct {
@@ -201,6 +202,15 @@ func (sf *sliceField) PopulateTo() (interface{}, error) {
 	return result, nil
 }
 
+func stringInSlice(target string, slice []string) bool {
+	for _, name := range slice {
+		if name == target {
+			return true
+		}
+	}
+	return false
+}
+
 // protoToJSON is a simple shortcut wrapper around the proto JSON marshaler
 func protoToJSON(msg proto.Message) ([]byte, error) {
 	var b bytes.Buffer
@@ -241,6 +251,9 @@ func jsonToMap(marshaled []byte) (map[string]interface{}, error) {
 // Factories listed lower, may depend on factories listed higher being
 // evaluated first.
 var fieldFactories = []protoFieldFactory{
+	staticallyOpaqueSliceFieldFactory{},
+	staticallyOpaqueMapFieldFactory{},
+	staticallyOpaqueFieldFactory{},
 	nestedSliceFieldFactory{},
 	nestedMapFieldFactory{},
 	nestedFieldFactory{},

common/tools/protolator/statically_opaque.go

@@ -0,0 +1,152 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protolator
+
+import (
+	"reflect"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func opaqueFrom(opaqueType func() (proto.Message, error), value interface{}, destType reflect.Type) (reflect.Value, error) {
+	tree := value.(map[string]interface{}) // Safe, already checked
+	nMsg, err := opaqueType()
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	if err := recursivelyPopulateMessageFromTree(tree, nMsg); err != nil {
+		return reflect.Value{}, err
+	}
+	mMsg, err := proto.Marshal(nMsg)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+	return reflect.ValueOf(mMsg), nil
+}
+
+func opaqueTo(opaqueType func() (proto.Message, error), value reflect.Value) (interface{}, error) {
+	nMsg, err := opaqueType()
+	if err != nil {
+		return nil, err
+	}
+	mMsg := value.Interface().([]byte) // Safe, already checked
+	if err = proto.Unmarshal(mMsg, nMsg); err != nil {
+		return nil, err
+	}
+	return recursivelyCreateTreeFromMessage(nMsg)
+}
+
+type staticallyOpaqueFieldFactory struct{}
+
+func (soff staticallyOpaqueFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	opaqueProto, ok := msg.(StaticallyOpaqueFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, opaqueProto.StaticallyOpaqueFields())
+}
+
+func (soff staticallyOpaqueFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	opaqueProto := msg.(StaticallyOpaqueFieldProto) // Type checked in Handles
+
+	return &plainField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: bytesType,
+			value: fieldValue,
+		},
+		populateFrom: func(v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return opaqueFrom(func() (proto.Message, error) { return opaqueProto.StaticallyOpaqueFieldProto(fieldName) }, v, dT)
+		},
+		populateTo: func(v reflect.Value) (interface{}, error) {
+			return opaqueTo(func() (proto.Message, error) { return opaqueProto.StaticallyOpaqueFieldProto(fieldName) }, v)
+		},
+	}, nil
+}
+
+type staticallyOpaqueMapFieldFactory struct{}
+
+func (soff staticallyOpaqueMapFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	opaqueProto, ok := msg.(StaticallyOpaqueMapFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, opaqueProto.StaticallyOpaqueMapFields())
+}
+
+func (soff staticallyOpaqueMapFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	opaqueProto := msg.(StaticallyOpaqueMapFieldProto) // Type checked in Handles
+
+	return &mapField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: fieldType,
+			value: fieldValue,
+		},
+		populateFrom: func(key string, v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return opaqueFrom(func() (proto.Message, error) {
+				return opaqueProto.StaticallyOpaqueMapFieldProto(fieldName, key)
+			}, v, dT)
+		},
+		populateTo: func(key string, v reflect.Value) (interface{}, error) {
+			return opaqueTo(func() (proto.Message, error) {
+				return opaqueProto.StaticallyOpaqueMapFieldProto(fieldName, key)
+			}, v)
+		},
+	}, nil
+}
+
+type staticallyOpaqueSliceFieldFactory struct{}
+
+func (soff staticallyOpaqueSliceFieldFactory) Handles(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) bool {
+	opaqueProto, ok := msg.(StaticallyOpaqueSliceFieldProto)
+	if !ok {
+		return false
+	}
+
+	return stringInSlice(fieldName, opaqueProto.StaticallyOpaqueSliceFields())
+}
+
+func (soff staticallyOpaqueSliceFieldFactory) NewProtoField(msg proto.Message, fieldName string, fieldType reflect.Type, fieldValue reflect.Value) (protoField, error) {
+	opaqueProto := msg.(StaticallyOpaqueSliceFieldProto) // Type checked in Handles
+
+	return &sliceField{
+		baseField: baseField{
+			msg:   msg,
+			name:  fieldName,
+			fType: mapStringInterfaceType,
+			vType: fieldType,
+			value: fieldValue,
+		},
+		populateFrom: func(index int, v interface{}, dT reflect.Type) (reflect.Value, error) {
+			return opaqueFrom(func() (proto.Message, error) {
+				return opaqueProto.StaticallyOpaqueSliceFieldProto(fieldName, index)
+			}, v, dT)
+		},
+		populateTo: func(index int, v reflect.Value) (interface{}, error) {
+			return opaqueTo(func() (proto.Message, error) {
+				return opaqueProto.StaticallyOpaqueSliceFieldProto(fieldName, index)
+			}, v)
+		},
+	}, nil
+}

common/tools/protolator/statically_opaque_test.go

@@ -0,0 +1,135 @@
+/*
+Copyright IBM Corp. 2017 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+                 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protolator
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/hyperledger/fabric/common/tools/protolator/testprotos"
+	"github.com/hyperledger/fabric/protos/utils"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+)
+
+func extractSimpleMsgPlainField(source []byte) string {
+	result := &testprotos.SimpleMsg{}
+	err := proto.Unmarshal(source, result)
+	if err != nil {
+		panic(err)
+	}
+	return result.PlainField
+}
+
+func TestPlainStaticallyOpaqueMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff}
+
+	pfValue := "foo"
+	startMsg := &testprotos.StaticallyOpaqueMsg{
+		PlainOpaqueField: utils.MarshalOrPanic(&testprotos.SimpleMsg{
+			PlainField: pfValue,
+		}),
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.StaticallyOpaqueMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.PlainOpaqueField), extractSimpleMsgPlainField(newMsg.PlainOpaqueField))
+
+	fieldFactories = []protoFieldFactory{tppff, staticallyOpaqueFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.PlainOpaqueField), extractSimpleMsgPlainField(newMsg.PlainOpaqueField))
+}
+
+func TestMapStaticallyOpaqueMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff}
+
+	pfValue := "foo"
+	mapKey := "bar"
+	startMsg := &testprotos.StaticallyOpaqueMsg{
+		MapOpaqueField: map[string][]byte{
+			mapKey: utils.MarshalOrPanic(&testprotos.SimpleMsg{
+				PlainField: pfValue,
+			}),
+		},
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.StaticallyOpaqueMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.MapOpaqueField[mapKey]), extractSimpleMsgPlainField(newMsg.MapOpaqueField[mapKey]))
+
+	fieldFactories = []protoFieldFactory{tppff, staticallyOpaqueMapFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.MapOpaqueField[mapKey]), extractSimpleMsgPlainField(newMsg.MapOpaqueField[mapKey]))
+}
+
+func TestSliceStaticallyOpaqueMsg(t *testing.T) {
+	fromPrefix := "from"
+	toPrefix := "to"
+	tppff := &testProtoPlainFieldFactory{
+		fromPrefix: fromPrefix,
+		toPrefix:   toPrefix,
+	}
+
+	fieldFactories = []protoFieldFactory{tppff}
+
+	pfValue := "foo"
+	startMsg := &testprotos.StaticallyOpaqueMsg{
+		SliceOpaqueField: [][]byte{
+			utils.MarshalOrPanic(&testprotos.SimpleMsg{
+				PlainField: pfValue,
+			}),
+		},
+	}
+
+	var buffer bytes.Buffer
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	newMsg := &testprotos.StaticallyOpaqueMsg{}
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.NotEqual(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.SliceOpaqueField[0]), extractSimpleMsgPlainField(newMsg.SliceOpaqueField[0]))
+
+	fieldFactories = []protoFieldFactory{tppff, staticallyOpaqueSliceFieldFactory{}}
+
+	buffer.Reset()
+	assert.NoError(t, DeepMarshalJSON(&buffer, startMsg))
+	assert.NoError(t, DeepUnmarshalJSON(bytes.NewReader(buffer.Bytes()), newMsg))
+	assert.Equal(t, fromPrefix+toPrefix+extractSimpleMsgPlainField(startMsg.SliceOpaqueField[0]), extractSimpleMsgPlainField(newMsg.SliceOpaqueField[0]))
+}