remove double encoding of payload and signature fields for intoto (#1150

)

Signed-off-by: Bob Callaway <bcallaway@google.com>

Signed-off-by: Bob Callaway <bcallaway@google.com>

pkg/api/entries.go

@@ -237,12 +237,12 @@ func createLogEntry(params entries.CreateLogEntryParams) (models.LogEntry, middl
 		go func() {
 			keys, err := entry.IndexKeys()
 			if err != nil {
-				log.ContextLogger(ctx).Error(err)
+				log.ContextLogger(ctx).Error(fmt.Errorf("error generating index keys: %w", err))
 				return
 			}
 			for _, key := range keys {
 				if err := addToIndex(context.Background(), key, entryID); err != nil {
-					log.ContextLogger(ctx).Error(err)
+					log.ContextLogger(ctx).Error(fmt.Errorf("error inserting key/value pairs into index: %w", err))
 				}
 			}
 		}()

pkg/types/intoto/v0.0.2/entry.go

@@ -84,7 +84,7 @@ func (v V002Entry) IndexKeys() ([]string, error) {
 
 		canonKey, err := keyObj.CanonicalValue()
 		if err != nil {
-			return result, fmt.Errorf("could not canonicize key: %w", err)
+			return result, fmt.Errorf("could not canonicalize key: %w", err)
 		}
 
 		keyHash := sha256.Sum256(canonKey)
@@ -102,7 +102,7 @@ func (v V002Entry) IndexKeys() ([]string, error) {
 	switch *v.IntotoObj.Content.Envelope.PayloadType {
 	case in_toto.PayloadType:
 
-		if v.IntotoObj.Content.Envelope.Payload == nil {
+		if v.IntotoObj.Content.Envelope.Payload == "" {
 			log.Logger.Info("IntotoObj DSSE payload is empty")
 			return result, nil
 		}
@@ -159,7 +159,6 @@ func (v *V002Entry) Unmarshal(pe models.ProposedEntry) error {
 		return errors.New("cannot unmarshal non Intoto v0.0.2 type")
 	}
 
-	var err error
 	if err := types.DecodeEntry(it.Spec, &v.IntotoObj); err != nil {
 		return err
 	}
@@ -173,17 +172,44 @@ func (v *V002Entry) Unmarshal(pe models.ProposedEntry) error {
 		return nil
 	}
 
+	payloadToUse := v.IntotoObj.Content.Envelope.Payload
+	// we need the decoded payload to calculate the correct SHA256 digest
+	decodedPayload, err := base64.StdEncoding.DecodeString(payloadToUse)
+	if err != nil {
+		return fmt.Errorf("envelope payload must be base64 encoded string: %w", err)
+	}
+	// in rekor v0.12 & v1.0.0, rekor-cli would (incorrectly) double encode the payload.
+	// handle that gracefully here
+	doubleDecodedPayload, err := base64.StdEncoding.DecodeString(string(decodedPayload))
+	if err == nil {
+		payloadToUse = string(decodedPayload)
+		v.IntotoObj.Content.Envelope.Payload = payloadToUse
+		decodedPayload = doubleDecodedPayload
+	}
+
 	env := &dsse.Envelope{
-		Payload:     string(v.IntotoObj.Content.Envelope.Payload),
+		Payload:     payloadToUse, // must be base64 encoded string
 		PayloadType: *v.IntotoObj.Content.Envelope.PayloadType,
 	}
 
 	allPubKeyBytes := make([][]byte, 0)
 	for _, sig := range v.IntotoObj.Content.Envelope.Signatures {
+		sigToUse := sig.Sig
+		decodedSig, err := base64.StdEncoding.DecodeString(string(sig.Sig))
+		if err != nil {
+			return fmt.Errorf("envelope signature must be a base64 encoded string: %w", err)
+		}
+		// in rekor v0.12 & v1.0.0, rekor-cli would (incorrectly) double encode the signature.
+		// handle that gracefully here
+		if _, err = base64.StdEncoding.DecodeString(string(decodedSig)); err == nil {
+			sigToUse = string(decodedSig)
+		}
+
 		env.Signatures = append(env.Signatures, dsse.Signature{
 			KeyID: sig.Keyid,
-			Sig:   string(sig.Sig),
+			Sig:   sigToUse, // must be base64 encoded string
 		})
+		sig.Sig = sigToUse // overwrite value in local object to ensure we canonicalize correctly
 
 		allPubKeyBytes = append(allPubKeyBytes, sig.PublicKey)
 	}
@@ -194,11 +220,6 @@ func (v *V002Entry) Unmarshal(pe models.ProposedEntry) error {
 
 	v.env = *env
 
-	decodedPayload, err := base64.StdEncoding.DecodeString(string(v.IntotoObj.Content.Envelope.Payload))
-	if err != nil {
-		return fmt.Errorf("could not decode envelope payload: %w", err)
-	}
-
 	h := sha256.Sum256(decodedPayload)
 	v.IntotoObj.Content.PayloadHash = &models.IntotoV002SchemaContentPayloadHash{
 		Algorithm: swag.String(models.IntotoV002SchemaContentPayloadHashAlgorithmSha256),
@@ -339,8 +360,7 @@ func (v V002Entry) CreateFromArtifactProperties(_ context.Context, props types.A
 		return nil, err
 	}
 
-	b64 := strfmt.Base64([]byte(env.Payload))
-	re.IntotoObj.Content.Envelope.Payload = b64
+	re.IntotoObj.Content.Envelope.Payload = env.Payload
 	re.IntotoObj.Content.Envelope.PayloadType = &env.PayloadType
 
 	for _, sig := range env.Signatures {
@@ -351,13 +371,13 @@ func (v V002Entry) CreateFromArtifactProperties(_ context.Context, props types.A
 
 		canonKey, err := key.CanonicalValue()
 		if err != nil {
-			return nil, fmt.Errorf("could not canonicize key: %w", err)
+			return nil, fmt.Errorf("could not canonicalize key: %w", err)
 		}
 
 		keyBytes := strfmt.Base64(canonKey)
 		re.IntotoObj.Content.Envelope.Signatures = append(re.IntotoObj.Content.Envelope.Signatures, &models.IntotoV002SchemaContentEnvelopeSignaturesItems0{
 			Keyid:     sig.KeyID,
-			Sig:       strfmt.Base64([]byte(sig.Sig)),
+			Sig:       sig.Sig,
 			PublicKey: keyBytes,
 		})
 	}

pkg/types/intoto/v0.0.2/entry_test.go

@@ -108,14 +108,13 @@ func multiSignEnvelope(t *testing.T, k []*ecdsa.PrivateKey, payload []byte) *dss
 func createRekorEnvelope(dsseEnv *dsse.Envelope, pub [][]byte) *models.IntotoV002SchemaContentEnvelope {
 
 	env := &models.IntotoV002SchemaContentEnvelope{}
-	b64 := strfmt.Base64([]byte(dsseEnv.Payload))
-	env.Payload = b64
+	env.Payload = dsseEnv.Payload
 	env.PayloadType = &dsseEnv.PayloadType
 
 	for i, sig := range dsseEnv.Signatures {
 		env.Signatures = append(env.Signatures, &models.IntotoV002SchemaContentEnvelopeSignaturesItems0{
 			Keyid:     sig.KeyID,
-			Sig:       strfmt.Base64([]byte(sig.Sig)),
+			Sig:       sig.Sig,
 			PublicKey: strfmt.Base64(pub[i]),
 		})
 	}
@@ -173,6 +172,12 @@ func TestV002Entry_Unmarshal(t *testing.T) {
 
 	validPayload := "hellothispayloadisvalid"
 
+	doubleEncodedEnvelope := createRekorEnvelope(envelope(t, key, []byte(validPayload)), [][]byte{pub})
+	doubleEncodedPayload := base64.StdEncoding.EncodeToString([]byte(doubleEncodedEnvelope.Payload))
+	doubleEncodedEnvelope.Payload = doubleEncodedPayload
+	doubleEncodedSig := base64.StdEncoding.EncodeToString([]byte(doubleEncodedEnvelope.Signatures[0].Sig))
+	doubleEncodedEnvelope.Signatures[0].Sig = doubleEncodedSig
+
 	tests := []struct {
 		env     *dsse.Envelope
 		name    string
@@ -265,6 +270,20 @@ func TestV002Entry_Unmarshal(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			env:  envelope(t, key, []byte(validPayload)),
+			name: "double base64 encoded envelope (testing backwards compat with v0.12.x and v1.0.0",
+			it: &models.IntotoV002Schema{
+				Content: &models.IntotoV002SchemaContent{
+					Envelope: doubleEncodedEnvelope,
+					Hash: &models.IntotoV002SchemaContentHash{
+						Algorithm: swag.String(models.IntotoV002SchemaContentHashAlgorithmSha256),
+						Value:     swag.String(envelopeHash(t, envelope(t, priv, []byte(validPayload)))),
+					},
+				},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/types/intoto/v0.0.2/intoto_v0_0_2_schema.json

@@ -13,9 +13,9 @@
                     "type": "object",
                     "properties": {
                         "payload": {
-                            "description": "payload of the envelope",
+                            "description": "base64 encoded payload of the envelope",
                             "type": "string",
-                            "format": "byte",
+                            "pattern": "^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=|[A-Za-z0-9+\\/]{4})$",
                             "writeOnly": true
                         },
                         "payloadType": {
@@ -35,9 +35,9 @@
                                         "type": "string"
                                     },
                                     "sig": {
-                                        "description": "signature of the payload",
+                                        "description": "base64 encoded signature of the payload",
                                         "type": "string",
-                                        "format": "byte"
+                                        "pattern": "^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=|[A-Za-z0-9+\\/]{4})$"
                                     },
                                     "publicKey": {
                                         "description": "public key that corresponds to this signature",