fix: emoji and unicode parsing by switching to yaml fork (#54)

Fork is specifically maintained by @braydonk particularly for the interests of yamlfmt. Context: go-yaml/yaml#737 (comment) Before: Any multi-line string with emojis will collapse into a single line string and the emoji will be escaped. After: Multi-line strings with emojis are correctly left unedited.
sethvargo · Aug 8, 2023 · 7f726e8 · 7f726e8
1 parent 5dafc06
commit 7f726e8
Show file tree

Hide file tree

Showing 13 changed files with 110 additions and 11 deletions.
diff --git a/command/command.go b/command/command.go
@@ -9,10 +9,12 @@ import (
 	"strconv"
 	"strings"
 
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 	"github.com/hexops/gotextdiff"
 	"github.com/hexops/gotextdiff/myers"
 	"github.com/hexops/gotextdiff/span"
-	"gopkg.in/yaml.v3"
 
 	"github.com/sethvargo/ratchet/internal/atomic"
 	"github.com/sethvargo/ratchet/internal/version"

diff --git a/command/command_test.go b/command/command_test.go
@@ -1,8 +1,17 @@
 package command
 
 import (
+	"bytes"
+	"fmt"
+	"path/filepath"
 	"reflect"
+	"runtime"
 	"testing"
+
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
+	"github.com/google/go-cmp/cmp"
 )
 
 const (
@@ -319,3 +328,68 @@ func Test_removeNewLineChanges(t *testing.T) {
 		})
 	}
 }
+
+func Test_parseYAMLFile(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name         string
+		yamlFilename string
+		want         string
+	}{
+		{
+			name:         "yamlA_multiple_empty_lines",
+			yamlFilename: "testdata/github.yml",
+			want: `jobs:
+    my_job:
+        runs-on: 'ubuntu-latest'
+        container:
+            image: 'ubuntu:20.04'
+        services:
+            nginx:
+                image: 'nginx:1.21'
+        steps:
+            - uses: 'actions/checkout@v3'
+            - uses: 'docker://ubuntu:20.04'
+              with:
+                uses: '/path/to/user.png'
+                image: '/path/to/image.jpg'
+            - runs: |-
+                echo "Hello 😀"
+    other_job:
+        uses: 'my-org/my-repo/.github/workflows/my-workflow.yml@v0'
+    final_job:
+        uses: './local/path/to/action'
+`,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			node, err := parseYAMLFile(rootPath(tc.yamlFilename))
+			if err != nil {
+				t.Errorf("parseYAMLFile() returned error: %v", err)
+			}
+			var buf bytes.Buffer
+			err = yaml.NewEncoder(&buf).Encode(node)
+			if err != nil {
+				t.Errorf("failed to marshal yaml to string: %v", err)
+			}
+			got := buf.String()
+
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Errorf("removeBindingFromPolicy() returned diff (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func rootPath(filename string) (fn string) {
+	_, fn, _, _ = runtime.Caller(0)
+	repoRoot := filepath.Dir(filepath.Dir(fn))
+	return fmt.Sprintf("%s/%s", repoRoot, filename)
+}
diff --git a/go.mod b/go.mod
@@ -3,13 +3,14 @@ module github.com/sethvargo/ratchet
 go 1.19
 
 require (
+	github.com/braydonk/yaml v0.7.0
+	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.14.0
 	github.com/google/go-github/v51 v51.0.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hexops/gotextdiff v1.0.3
 	golang.org/x/oauth2 v0.7.0
 	golang.org/x/sync v0.1.0
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -1,5 +1,7 @@
 github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310 h1:dGAdTcqheKrQ/TW76sAcmO2IorwXplUw2inPkOzykbw=
 github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
+github.com/braydonk/yaml v0.7.0 h1:ySkqO7r0MGoCNhiRJqE0Xe9yhINMyvOAB3nFjgyJn2k=
+github.com/braydonk/yaml v0.7.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
 github.com/cloudflare/circl v1.3.2 h1:VWp8dY3yH69fdM7lM6A1+NhhVoDu9vqK0jOgmkQHFWk=
@@ -24,6 +26,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
 github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
 github.com/google/go-github/v51 v51.0.0 h1:KCjsbgPV28VoRftdP+K2mQL16jniUsLAJknsOVKwHyU=
@@ -120,5 +123,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
diff --git a/parser/actions.go b/parser/actions.go
@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"strings"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/circleci.go b/parser/circleci.go
@@ -3,7 +3,9 @@ package parser
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/cloudbuild.go b/parser/cloudbuild.go
@@ -3,7 +3,9 @@ package parser
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/drone.go b/parser/drone.go
@@ -3,7 +3,9 @@ package parser
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/gitlabci.go b/parser/gitlabci.go
@@ -3,7 +3,9 @@ package parser
 import (
 	"fmt"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/parser.go b/parser/parser.go
@@ -7,9 +7,12 @@ import (
 	"strings"
 	"sync"
 
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
+
 	"github.com/hashicorp/go-multierror"
 	"golang.org/x/sync/semaphore"
-	"gopkg.in/yaml.v3"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -6,7 +6,9 @@ import (
 	"strings"
 	"testing"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 
 	"github.com/sethvargo/ratchet/resolver"
 )

diff --git a/parser/refs.go b/parser/refs.go
@@ -5,7 +5,9 @@ import (
 	"strings"
 	"sync"
 
-	"gopkg.in/yaml.v3"
+	// Using banydonk/yaml instead of the default yaml pkg because the default
+	// pkg incorrectly escapes unicode. https://github.com/go-yaml/yaml/issues/737
+	"github.com/braydonk/yaml"
 )
 
 type RefsList struct {

diff --git a/testdata/github.yml b/testdata/github.yml
@@ -17,6 +17,9 @@ jobs:
         uses: '/path/to/user.png'
         image: '/path/to/image.jpg'
 
+    - runs: |-
+        echo "Hello 😀"
+
   other_job:
     uses: 'my-org/my-repo/.github/workflows/my-workflow.yml@v0'