bzutil - add full parsed json output to get_operation (#469)

* bazel/execution - parsing of operations to json
* add parse_test
* simplify operationResultWrapper, update comment

bazel/execution/client_test.go

@@ -4,15 +4,12 @@ package execution
 //go:generate mockgen -destination=mock_longrunning/opclient_mock.go google.golang.org/genproto/googleapis/longrunning OperationsClient
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/golang/mock/gomock"
-	"github.com/golang/protobuf/ptypes/any"
 	"github.com/golang/protobuf/ptypes/empty"
 	"golang.org/x/net/context"
 	"google.golang.org/genproto/googleapis/longrunning"
-	"google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc"
 
 	"github.com/twitter/scoot/bazel/execution/mock_longrunning"
@@ -109,64 +106,6 @@ func TestClientExecute(t *testing.T) {
 	}
 }
 
-func TestExtractOpFromJsonError(t *testing.T) {
-	opBytes := []byte(`{"name":"testName","metadata":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"},"done":true,"Result":{"Error":{"code":1,"message":"CANCELLED"}}}`)
-
-	metadata := &any.Any{
-		TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
-		Value:   []byte("testVal"),
-	}
-	opErr := &longrunning.Operation_Error{
-		Error: &status.Status{
-			Code:    int32(1),
-			Message: "CANCELLED",
-		},
-	}
-	expOp := &longrunning.Operation{
-		Name:     "testName",
-		Metadata: metadata,
-		Done:     true,
-		Result:   opErr,
-	}
-
-	gotOp, err := ExtractOpFromJson(opBytes)
-	if err != nil {
-		t.Fatalf("Received error extracting operation from json: %s", err)
-	}
-	if !reflect.DeepEqual(gotOp, expOp) {
-		t.Fatalf("Expected gotOp to equal expOp.\ngotOp: %+v\nexpOp: %+v", gotOp, expOp)
-	}
-}
-
-func TestExtractOpFromJsonResponse(t *testing.T) {
-	opBytes := []byte(`{"name":"testName","metadata":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"},"done":true,"Result":{"Response":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"}}}`)
-
-	metadata := &any.Any{
-		TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
-		Value:   []byte("testVal"),
-	}
-	opResp := &longrunning.Operation_Response{
-		Response: &any.Any{
-			TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
-			Value:   []byte("testVal"),
-		},
-	}
-	expOp := &longrunning.Operation{
-		Name:     "testName",
-		Metadata: metadata,
-		Done:     true,
-		Result:   opResp,
-	}
-
-	gotOp, err := ExtractOpFromJson(opBytes)
-	if err != nil {
-		t.Fatalf("Received error extracting operation from json: %s", err)
-	}
-	if !reflect.DeepEqual(gotOp, expOp) {
-		t.Fatalf("Expected gotOp to equal expOp.\ngotOp: %+v\nexpOp: %+v", gotOp, expOp)
-	}
-}
-
 // Fake Execution_ExecuteClient
 // Implements Execution_ExecuteClient interface
 type fakeExecClient struct {

bazel/execution/parse.go

@@ -47,6 +47,7 @@ func ParseExecuteOperation(op *longrunning.Operation) (*remoteexecution.ExecuteO
 	return eom, st, res, nil
 }
 
+// TODO - used by bazel-integration, may be possible to replace with OperationToJson via bzutil and remove this
 // JSON unmarshalling doesn't work for Operations with nested Results, as they're of unexported type isOperation_Result.
 // Thus, we need custom unmarshalling logic.
 func ExtractOpFromJson(opBytes []byte) (*longrunning.Operation, error) {
@@ -152,6 +153,43 @@ func ExtractOpFromJson(opBytes []byte) (*longrunning.Operation, error) {
 	return op, nil
 }
 
+// Converts a longrunning.Operation to a Json-encoded []byte. Because an Operation's Result field is not
+// natively compatible with json.Marshal, this does the minimum necessary extraction of nested data and
+// wrapping with custom types so that it can be used with Marshal.
+func OperationToJson(op *longrunning.Operation) ([]byte, error) {
+	if op == nil {
+		return nil, nil
+	}
+
+	eom, st, res, err := ParseExecuteOperation(op)
+	if err != nil {
+		return nil, err
+	}
+
+	ow := &operationWrapper{
+		Name:     op.GetName(),
+		Metadata: eom,
+		Done:     op.GetDone(),
+		Result: &operationResultWrapper{
+			Error:    st,
+			Response: res,
+		},
+	}
+	return json.Marshal(ow)
+}
+
+type operationWrapper struct {
+	Name     string
+	Metadata *remoteexecution.ExecuteOperationMetadata
+	Done     bool
+	Result   *operationResultWrapper
+}
+
+type operationResultWrapper struct {
+	Error    *status.Status
+	Response *remoteexecution.ExecuteResponse
+}
+
 // Create error for failing to deserialize a field as an expected type
 func deserializeErr(keyName, expectedType string, value interface{}) error {
 	return fmt.Errorf("value for key '%s' was not of type %s. %v: %s", keyName, expectedType, value, reflect.TypeOf(value))

bazel/execution/parse_test.go

@@ -0,0 +1,104 @@
+package execution
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/golang/protobuf/ptypes/any"
+	"google.golang.org/genproto/googleapis/longrunning"
+	"google.golang.org/genproto/googleapis/rpc/status"
+
+	"github.com/twitter/scoot/bazel"
+	"github.com/twitter/scoot/bazel/remoteexecution"
+)
+
+func TestExtractOpFromJsonError(t *testing.T) {
+	opBytes := []byte(`{"name":"testName","metadata":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"},"done":true,"Result":{"Error":{"code":1,"message":"CANCELLED"}}}`)
+
+	metadata := &any.Any{
+		TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
+		Value:   []byte("testVal"),
+	}
+	opErr := &longrunning.Operation_Error{
+		Error: &status.Status{
+			Code:    int32(1),
+			Message: "CANCELLED",
+		},
+	}
+	expOp := &longrunning.Operation{
+		Name:     "testName",
+		Metadata: metadata,
+		Done:     true,
+		Result:   opErr,
+	}
+
+	gotOp, err := ExtractOpFromJson(opBytes)
+	if err != nil {
+		t.Fatalf("Received error extracting operation from json: %s", err)
+	}
+	if !reflect.DeepEqual(gotOp, expOp) {
+		t.Fatalf("Expected gotOp to equal expOp.\ngotOp: %+v\nexpOp: %+v", gotOp, expOp)
+	}
+}
+
+func TestExtractOpFromJsonResponse(t *testing.T) {
+	opBytes := []byte(`{"name":"testName","metadata":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"},"done":true,"Result":{"Response":{"type_url":"type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata","value":"testVal"}}}`)
+
+	metadata := &any.Any{
+		TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
+		Value:   []byte("testVal"),
+	}
+	opResp := &longrunning.Operation_Response{
+		Response: &any.Any{
+			TypeUrl: "type.googleapis.com/build.bazel.remote.execution.v2.ExecuteOperationMetadata",
+			Value:   []byte("testVal"),
+		},
+	}
+	expOp := &longrunning.Operation{
+		Name:     "testName",
+		Metadata: metadata,
+		Done:     true,
+		Result:   opResp,
+	}
+
+	gotOp, err := ExtractOpFromJson(opBytes)
+	if err != nil {
+		t.Fatalf("Received error extracting operation from json: %s", err)
+	}
+	if !reflect.DeepEqual(gotOp, expOp) {
+		t.Fatalf("Expected gotOp to equal expOp.\ngotOp: %+v\nexpOp: %+v", gotOp, expOp)
+	}
+}
+
+func TestOperationToJson(t *testing.T) {
+	eom := &remoteexecution.ExecuteOperationMetadata{
+		Stage:        remoteexecution.ExecuteOperationMetadata_COMPLETED,
+		ActionDigest: &remoteexecution.Digest{Hash: bazel.EmptySha, SizeBytes: bazel.EmptySize},
+	}
+	eomAsPBAny, err := marshalAny(eom)
+	if err != nil {
+		t.Fatalf("Failed to marshal: %s", err)
+	}
+
+	er := &remoteexecution.ExecuteResponse{
+		Result:       &remoteexecution.ActionResult{},
+		CachedResult: false,
+		Status:       &status.Status{},
+	}
+	resAsPBAny, err := marshalAny(er)
+	if err != nil {
+		t.Fatalf("Failed to marshal: %s", err)
+	}
+
+	op := &longrunning.Operation{
+		Name:     "test",
+		Metadata: eomAsPBAny,
+		Done:     true,
+		Result:   &longrunning.Operation_Response{Response: resAsPBAny},
+	}
+
+	_, err = OperationToJson(op)
+	if err != nil {
+		t.Fatalf("Failed to parse to json: %s", err)
+	}
+}

binaries/bzutil/main.go

@@ -109,7 +109,8 @@ func main() {
 	getCommand := flag.NewFlagSet(getOpCmdStr, flag.ExitOnError)
 	getAddr := getCommand.String("grpc_addr", scootapi.DefaultSched_GRPC, "'host:port' of grpc Exec server")
 	getName := getCommand.String("name", "", "Operation name to query")
-	getJson := getCommand.Bool("json", false, "Print operation as JSON to stdout")
+	getJson := getCommand.Bool("json", false, "Print operation as JSON to stdout (some data remains serialized)")
+	getJsonFull := getCommand.Bool("json_full", false, "Print full operation as JSON to stdout (format not guaranteed canonical)")
 	getLogLevel := getCommand.String("log_level", "", "Log everything at this level and above (error|info|debug)")
 
 	// Cancel Operation
@@ -186,7 +187,10 @@ func main() {
 			log.Fatalf("name required for %s", getOpCmdStr)
 		}
 		parseAndSetLevel(*getLogLevel)
-		getOperation(*getAddr, *getName, *getJson)
+		if *getJson && *getJsonFull {
+			log.Fatal("Can only specify one of: json, json_full")
+		}
+		getOperation(*getAddr, *getName, *getJson, *getJsonFull)
 	} else if cancelCommand.Parsed() {
 		if *cancelName == "" {
 			log.Fatalf("name required for %s", cancelOpCmdStr)
@@ -357,20 +361,28 @@ func execute(execAddr, actionDigestStr string, skipCache bool, execJson bool) {
 	}
 }
 
-func getOperation(execAddr, opName string, getJson bool) {
+func getOperation(execAddr, opName string, getJson, getJsonFull bool) {
 	r := dialer.NewConstantResolver(execAddr)
 	operation, err := execution.GetOperation(r, opName)
 	if err != nil {
 		log.Fatalf("Error making GetOperation request: %s", err)
 	}
 	log.Info(execution.ExecuteOperationToStr(operation))
-	// We only use default Marshalling, which leaves most nested fields serialized
 	if getJson {
+		// We only use default Marshalling, which leaves most nested fields serialized
 		b, err := json.Marshal(operation)
 		if err != nil {
 			log.Fatalf("Error converting operation to JSON: %v", err)
 		}
 		fmt.Printf("%s\n", b)
+	} else if getJsonFull {
+		// Gets the "full" deserialized json representation, although to do this custom extraction/parsing is used,
+		// which means that the resulting format could possibly deviate from the canonical data types.
+		b, err := execution.OperationToJson(operation)
+		if err != nil {
+			log.Fatalf("Error parsing operation full JSON: %v", err)
+		}
+		fmt.Printf("%s\n", b)
 	} else {
 		fmt.Printf("%s\n", operation.GetName())
 	}