Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(pkger): add jsonnet support for package files #16511

Merged
merged 1 commit into from
Jan 13, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
1. [16260](https://github.com/influxdata/influxdb/pull/16260): Capture User-Agent header as query source for logging purposes
1. [16469](https://github.com/influxdata/influxdb/pull/16469): Add support for configurable max batch size in points write handler
1. [16509](https://github.com/influxdata/influxdb/pull/16509): Add support for applying an influx package via a public facing URL
1. [16511](https://github.com/influxdata/influxdb/pull/16511): Add jsonnet support for influx packages

### Bug Fixes

Expand Down
14 changes: 4 additions & 10 deletions cmd/influx/pkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"reflect"
Expand Down Expand Up @@ -95,7 +94,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command {
cmd.Short = "Apply a pkg to create resources"

cmd.Flags().StringVarP(&b.file, "file", "f", "", "Path to package file")
cmd.MarkFlagFilename("file", "yaml", "yml", "json")
cmd.MarkFlagFilename("file", "yaml", "yml", "json", "jsonnet")
cmd.Flags().BoolVarP(&b.quiet, "quiet", "q", false, "disable output printing")
cmd.Flags().StringVar(&b.applyOpts.force, "force", "", `TTY input, if package will have destructive changes, proceed if set "true"`)
cmd.Flags().StringVarP(&b.applyOpts.url, "url", "u", "", "URL to retrieve a package.")
Expand Down Expand Up @@ -551,14 +550,7 @@ func pkgFromReader(stdin io.Reader) (*pkger.Pkg, error) {
return nil, err
}

var enc pkger.Encoding
switch http.DetectContentType(b[0:512]) {
case "application/json":
enc = pkger.EncodingJSON
default:
enc = pkger.EncodingYAML
}
return pkger.Parse(enc, pkger.FromString(string(b)))
return pkger.Parse(pkger.EncodingSource, pkger.FromString(string(b)))
}

func pkgFromFile(path string) (*pkger.Pkg, error) {
Expand All @@ -568,6 +560,8 @@ func pkgFromFile(path string) (*pkger.Pkg, error) {
enc = pkger.EncodingYAML
case ".json":
enc = pkger.EncodingJSON
case ".jsonnet":
enc = pkger.EncodingJsonnet
default:
return nil, errors.New("file provided must be one of yaml/yml/json extension but got: " + ext)
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ require (
github.com/google/btree v1.0.0
github.com/google/go-cmp v0.3.1
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-jsonnet v0.14.0
github.com/goreleaser/goreleaser v0.97.0
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c // indirect
github.com/hashicorp/raft v1.0.0 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-jsonnet v0.14.0 h1:as/sAfmjOHqY/OMBR4mv9I8ZY0/jNuqN3u44AicwxPs=
github.com/google/go-jsonnet v0.14.0/go.mod h1:zPGC9lj/TbjkBtUACIvYR/ILHrFqKRhxeEA+bLyeMnY=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
Expand Down Expand Up @@ -288,11 +290,14 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
Expand Down Expand Up @@ -539,6 +544,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
4 changes: 4 additions & 0 deletions http/pkger_http_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/influxdata/influxdb"
pctx "github.com/influxdata/influxdb/context"
"github.com/influxdata/influxdb/pkg/httpc"
"github.com/influxdata/influxdb/pkg/jsonnet"
"github.com/influxdata/influxdb/pkger"
"go.uber.org/zap"
"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -232,6 +233,9 @@ func decodeWithEncoding(r *http.Request, v interface{}) (pkger.Encoding, error)
dec interface{ Decode(interface{}) error }
)
switch contentType := r.Header.Get("Content-Type"); contentType {
case "application/x-jsonnet":
encoding = pkger.EncodingJsonnet
dec = jsonnet.NewDecoder(r.Body)
case "text/yml", "application/x-yaml":
encoding = pkger.EncodingYAML
dec = yaml.NewDecoder(r.Body)
Expand Down
32 changes: 32 additions & 0 deletions http/pkger_http_server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,15 @@ func TestPkgerHTTPServer(t *testing.T) {
URL: "https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json",
},
},
{
name: "app jsonnet",
contentType: "application/x-jsonnet",
reqBody: fluxTTP.ReqApplyPkg{
DryRun: true,
OrgID: influxdb.ID(9000).String(),
Pkg: bucketPkg(t, pkger.EncodingJsonnet),
},
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -257,6 +266,29 @@ func bucketPkg(t *testing.T, encoding pkger.Encoding) *pkger.Pkg {

var pkgStr string
switch encoding {
case pkger.EncodingJsonnet:
pkgStr = `
local Bucket(name, desc) = {
kind: 'Bucket',
name: name,
description: desc,
};

{
apiVersion: "0.1.0",
kind: "Package",
meta: {
pkgName: "pkg_name",
pkgVersion: "1",
description: "pack description"
},
spec: {
resources: [
Bucket(name="rucket_1", desc="bucket 1 description"),
]
}
}
`
case pkger.EncodingJSON:
pkgStr = `
{
Expand Down
3 changes: 3 additions & 0 deletions http/swagger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4319,6 +4319,9 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/PkgApply"
application/x-jsonnet:
schema:
$ref: "#/components/schemas/PkgApply"
text/yml:
schema:
$ref: "#/components/schemas/PkgApply"
Expand Down
34 changes: 34 additions & 0 deletions pkg/jsonnet/decode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package jsonnet

import (
"encoding/json"
"io"
"io/ioutil"

"github.com/google/go-jsonnet"
)

// Decoder type can decoce a jsonnet stream into the given output.
type Decoder struct {
r io.Reader
}

// NewDecoder creates a new decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}

// Decode decodes the stream into the provide value.
func (d *Decoder) Decode(v interface{}) error {
b, err := ioutil.ReadAll(d.r)
if err != nil {
return err
}

vm := jsonnet.MakeVM()
jsonStr, err := vm.EvaluateSnippet("memory", string(b))
if err != nil {
return err
}
return json.Unmarshal([]byte(jsonStr), &v)
}
48 changes: 48 additions & 0 deletions pkg/jsonnet/decode_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package jsonnet_test

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"

"github.com/influxdata/influxdb/pkg/jsonnet"
"github.com/stretchr/testify/require"
)

func TestDecoder(t *testing.T) {
type (
person struct {
Name string `json:"name"`
Welcome string `json:"welcome"`
}

persons struct {
Person1 person `json:"person1"`
Person2 person `json:"person2"`
}
)

const entry = `{
person1: {
name: "Alice",
welcome: "Hello " + self.name + "!",
},
person2: self.person1 { name: "Bob" },
}`

var out persons
require.NoError(t, jsonnet.NewDecoder(strings.NewReader(entry)).Decode(&out))

expected := persons{
Person1: person{
Name: "Alice",
Welcome: "Hello Alice!",
},
Person2: person{
Name: "Bob",
Welcome: "Hello Bob!",
},
}
assert.Equal(t, expected, out)
}
18 changes: 17 additions & 1 deletion pkger/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"time"

"github.com/influxdata/influxdb"
"github.com/influxdata/influxdb/pkg/jsonnet"
"gopkg.in/yaml.v3"
)

Expand All @@ -29,6 +30,7 @@ type Encoding int
const (
EncodingUnknown Encoding = iota
EncodingJSON
EncodingJsonnet
EncodingSource // EncodingSource draws the encoding type by inferring it from the source.
EncodingYAML
)
Expand All @@ -38,6 +40,8 @@ func (e Encoding) String() string {
switch e {
case EncodingJSON:
return "json"
case EncodingJsonnet:
return "jsonnet"
case EncodingSource:
return "source"
case EncodingYAML:
Expand All @@ -61,6 +65,8 @@ func Parse(encoding Encoding, readerFn ReaderFn, opts ...ValidateOptFn) (*Pkg, e
switch encoding {
case EncodingJSON:
return parseJSON(r, opts...)
case EncodingJsonnet:
return parseJsonnet(r, opts...)
case EncodingSource:
return parseSource(r, opts...)
case EncodingYAML:
Expand Down Expand Up @@ -122,6 +128,10 @@ func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
return parse(json.NewDecoder(r), opts...)
}

func parseJsonnet(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
return parse(jsonnet.NewDecoder(r), opts...)
}

func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
var b []byte
if byter, ok := r.(interface{ Bytes() []byte }); ok {
Expand All @@ -136,10 +146,16 @@ func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {

contentType := http.DetectContentType(b)
switch {
case strings.Contains(contentType, "jsonnet"):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

strings.Contains(contentType, "json") will always be true, even if the "content-type" is "jsonnet"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ohhhh great catch 🥇, will fix that up 👍

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kelwang push fixed

// highly unlikely to fall in here with supported content type detection as is
return parseJsonnet(bytes.NewReader(b), opts...)
case strings.Contains(contentType, "json"):
return parseJSON(bytes.NewReader(b), opts...)
default:
case strings.Contains(contentType, "yaml"),
strings.Contains(contentType, "yml"):
return parseYAML(bytes.NewReader(b), opts...)
default:
return parseJsonnet(r, opts...)
}
}

Expand Down
46 changes: 46 additions & 0 deletions pkger/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4338,6 +4338,52 @@ spec:
require.True(t, ok)
})
})

t.Run("jsonnet support", func(t *testing.T) {
pkg := validParsedPkg(t, "testdata/bucket_associates_labels.jsonnet", EncodingJsonnet, baseAsserts{
version: "0.1.0",
kind: KindPackage,
description: "pack description",
metaName: "pkg_name",
metaVersion: "1",
})

sum := pkg.Summary()

labels := []SummaryLabel{
{
Name: "label_1",
Properties: struct {
Color string `json:"color"`
Description string `json:"description"`
}{Color: "#eee888", Description: "desc_1"},
},
}
assert.Equal(t, labels, sum.Labels)

bkts := []SummaryBucket{
{
Name: "rucket_1",
Description: "desc_1",
RetentionPeriod: 10000 * time.Second,
LabelAssociations: labels,
},
{
Name: "rucket_2",
Description: "desc_2",
RetentionPeriod: 20000 * time.Second,
LabelAssociations: labels,
},
{
Name: "rucket_3",
Description: "desc_3",
RetentionPeriod: 30000 * time.Second,
LabelAssociations: labels,
},
}
assert.Equal(t, bkts, sum.Buckets)
})

}

func Test_IsParseError(t *testing.T) {
Expand Down
Loading