Skip to content

Commit

Permalink
Use github.com/yuin/goldmark-meta to parse the meta section
Browse files Browse the repository at this point in the history
Because later parse steps look for the `---` that delineates a meta section, we need to
not only parse but also render back the meta section. We should be able to remove this
when we start delineating sections by node and node kind instead of by re-"parsing" the
document.
  • Loading branch information
iwahbe committed Aug 14, 2024
1 parent 6bcf126 commit 73e27d9
Show file tree
Hide file tree
Showing 12 changed files with 127 additions and 7 deletions.
2 changes: 2 additions & 0 deletions dynamic/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ require (
require (
github.com/teekennedy/goldmark-markdown v0.3.0 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
github.com/yuin/goldmark-meta v1.1.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

require (
Expand Down
2 changes: 2 additions & 0 deletions dynamic/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -873,6 +873,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI=
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ require (
github.com/stretchr/testify v1.9.0
github.com/teekennedy/goldmark-markdown v0.3.0
github.com/yuin/goldmark v1.7.4
github.com/yuin/goldmark-meta v1.1.0
github.com/zclconf/go-cty v1.14.2
golang.org/x/crypto v0.25.0
golang.org/x/mod v0.18.0
Expand Down Expand Up @@ -112,6 +113,7 @@ require (
golang.org/x/tools v0.22.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
mvdan.cc/gofumpt v0.5.0 // indirect
)

Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2038,6 +2038,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
github.com/zclconf/go-cty v1.0.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
Expand Down
2 changes: 2 additions & 0 deletions pf/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ require (
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
github.com/yuin/goldmark-meta v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand All @@ -82,6 +83,7 @@ require (
golang.org/x/tools v0.22.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

require (
Expand Down
2 changes: 2 additions & 0 deletions pf/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2036,6 +2036,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
github.com/zclconf/go-cty v1.0.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
Expand Down
2 changes: 2 additions & 0 deletions pf/tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ require (
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
github.com/yuin/goldmark-meta v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand All @@ -75,6 +76,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
mvdan.cc/gofumpt v0.5.0 // indirect
)

Expand Down
2 changes: 2 additions & 0 deletions pf/tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2074,6 +2074,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
github.com/zclconf/go-cty v1.0.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
Expand Down
2 changes: 2 additions & 0 deletions pkg/tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ require (
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
github.com/yuin/goldmark-meta v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
Expand All @@ -104,6 +105,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
mvdan.cc/gofumpt v0.5.0 // indirect
)

Expand Down
2 changes: 2 additions & 0 deletions pkg/tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2143,6 +2143,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
github.com/zclconf/go-cty v1.0.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
Expand Down
36 changes: 29 additions & 7 deletions pkg/tfgen/parse/extension.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@ import (
"bytes"

"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
markdown "github.com/teekennedy/goldmark-markdown"
"github.com/yuin/goldmark"
meta "github.com/yuin/goldmark-meta"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"

Expand All @@ -37,13 +40,13 @@ var TFRegistryExtension goldmark.Extender = tfRegistryExtension{}
type tfRegistryExtension struct{}

func (s tfRegistryExtension) Extend(md goldmark.Markdown) {
extension.GFM.Extend(md)
section.Extension.Extend(md)
md.Parser().AddOptions(
parser.WithASTTransformers(
util.Prioritized(recognizeHeaderAfterHTML{}, 902),
))

extension.GFM.Extend(md) // GitHub Flavored Markdown
section.Extension.Extend(md) // AST defined sections
meta.New().Extend(md) // Support for YAML metadata blocks
metaExtension{}.Extend(md) // Support rendering out YAML metadata blocks
md.Parser().AddOptions(parser.WithASTTransformers(
util.Prioritized(recognizeHeaderAfterHTML{}, 902),
))
}

// recognizeHeaderAfterHTML allows us to work around a difference in how TF's registry parses
Expand Down Expand Up @@ -86,3 +89,22 @@ func WalkNode[T ast.Node](node ast.Node, f func(T)) {
})
contract.AssertNoErrorf(err, "impossible: ast.Walk never returns an error")
}

func RenderMarkdown() renderer.Renderer {
// [markdown.NewRenderer] does not produce a renderer that can render [ast.String],
// so we augment the [renderer.Renderer] with that type.
r := renderMarkdown{markdown.NewRenderer()}
writeString := func(
writer util.BufWriter, _ []byte, n ast.Node, entering bool,
) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
_, err := writer.Write(n.(*ast.String).Value)
return ast.WalkContinue, err
}
r.Register(ast.KindString, writeString)
return r
}

type renderMarkdown struct{ *markdown.Renderer }
78 changes: 78 additions & 0 deletions pkg/tfgen/parse/meta.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2016-2024, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parse

import (
"github.com/yuin/goldmark"
meta "github.com/yuin/goldmark-meta"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
"gopkg.in/yaml.v2"
)

var MetaKind = ast.NewNodeKind("Meta")

type Meta struct {
ast.BaseBlock
Yaml yaml.MapSlice
}

func (m *Meta) Dump(source []byte, level int) {
ast.DumpHelper(m, source, level, nil, nil)
}

func (m *Meta) Kind() ast.NodeKind { return MetaKind }

var _ goldmark.Extender = metaExtension{}

type metaExtension struct{}

func (m metaExtension) Extend(md goldmark.Markdown) {
md.Parser().AddOptions(parser.WithASTTransformers(
util.Prioritized(m, 1),
))
md.Renderer().AddOptions(renderer.WithNodeRenderers(
util.Prioritized(m, 1),
))
}

func (metaExtension) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
items := meta.GetItems(pc)
if items == nil {
return
}
node.InsertBefore(node, node.FirstChild(), &Meta{Yaml: items})
}

func (m metaExtension) RegisterFuncs(r renderer.NodeRendererFuncRegisterer) {
f := func(b util.BufWriter, _ []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
if entering {
return ast.WalkContinue, nil
}
b.WriteString("---\n")
y, err := yaml.Marshal(n.(*Meta).Yaml)
if err != nil {
return ast.WalkContinue, err
}
b.Write(y)
b.WriteByte('\n')
b.WriteString("---\n")
return ast.WalkContinue, nil
}
r.Register(MetaKind, f)
}

0 comments on commit 73e27d9

Please sign in to comment.