From da1b541942686dfd734c60daaa27cfe14b1045f6 Mon Sep 17 00:00:00 2001 From: Johnny Steenbergen Date: Sat, 11 Jan 2020 18:25:19 -0800 Subject: [PATCH] feat(pkger): add ability to supply a pkg from a url the following is now possible from the CLI(same in REST API): influx pkg -u https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json --- CHANGELOG.md | 1 + cmd/influx/pkg.go | 12 +++++- http/pkger_http_server.go | 21 ++++++++++ http/pkger_http_server_test.go | 27 ++++++++---- http/swagger.yml | 2 + pkger/models.go | 2 +- pkger/parser.go | 76 ++++++++++++++++++++++++++++++---- pkger/service.go | 21 ++++++---- 8 files changed, 136 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be105613cb..c19a8652472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ 1. [16418](https://github.com/influxdata/influxdb/pull/16418): Add Developer Documentation 1. [16260](https://github.com/influxdata/influxdb/pull/16260): Capture User-Agent header as query source for logging purposes 1. [16469](https://github.com/influxdata/influxdb/pull/16469): Add support for configurable max batch size in points write handler +1. [16509](https://github.com/influxdata/influxdb/pull/16509): Add support for applying an influx package via a public facing URL ### Bug Fixes diff --git a/cmd/influx/pkg.go b/cmd/influx/pkg.go index 381c851728d..476d2de0615 100644 --- a/cmd/influx/pkg.go +++ b/cmd/influx/pkg.go @@ -48,6 +48,7 @@ type cmdPkgBuilder struct { applyOpts struct { force string secrets []string + url string } exportOpts struct { resourceType string @@ -97,6 +98,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command { cmd.MarkFlagFilename("file", "yaml", "yml", "json") cmd.Flags().BoolVarP(&b.quiet, "quiet", "q", false, "disable output printing") cmd.Flags().StringVar(&b.applyOpts.force, "force", "", `TTY input, if package will have destructive changes, proceed if set "true"`) + cmd.Flags().StringVarP(&b.applyOpts.url, "url", "u", "", "URL to retrieve a package.") b.org.register(cmd, false) @@ -132,7 +134,15 @@ func (b *cmdPkgBuilder) pkgApplyRunEFn() func(*cobra.Command, []string) error { return err } - pkg, isTTY, err := b.readPkgStdInOrFile(b.file) + var ( + pkg *pkger.Pkg + isTTY bool + ) + if b.applyOpts.url != "" { + pkg, err = pkger.Parse(pkger.EncodingSource, pkger.FromHTTPRequest(b.applyOpts.url)) + } else { + pkg, isTTY, err = b.readPkgStdInOrFile(b.file) + } if err != nil { return err } diff --git a/http/pkger_http_server.go b/http/pkger_http_server.go index 9b06c9ddb59..ba76c7cbc0a 100644 --- a/http/pkger_http_server.go +++ b/http/pkger_http_server.go @@ -125,6 +125,7 @@ type ( ReqApplyPkg struct { DryRun bool `json:"dryRun" yaml:"dryRun"` OrgID string `json:"orgID" yaml:"orgID"` + URL string `json:"url" yaml:"url"` Pkg *pkger.Pkg `json:"package" yaml:"package"` Secrets map[string]string `json:"secrets"` } @@ -146,6 +147,14 @@ func (s *HandlerPkg) applyPkg(w http.ResponseWriter, r *http.Request) { return } + if reqBody.URL != "" && reqBody.Pkg != nil { + s.HandleHTTPError(r.Context(), &influxdb.Error{ + Code: influxdb.EInvalid, + Msg: "must provide either url or pkg", + }, w) + return + } + orgID, err := influxdb.IDFromString(reqBody.OrgID) if err != nil { s.HandleHTTPError(r.Context(), &influxdb.Error{ @@ -163,6 +172,18 @@ func (s *HandlerPkg) applyPkg(w http.ResponseWriter, r *http.Request) { userID := auth.GetUserID() parsedPkg := reqBody.Pkg + if reqBody.URL != "" { + parsedPkg, err = pkger.Parse(pkger.EncodingSource, pkger.FromHTTPRequest(reqBody.URL)) + if err != nil { + s.HandleHTTPError(r.Context(), &influxdb.Error{ + Code: influxdb.EInvalid, + Msg: "failed to parse package from provided URL", + Err: err, + }, w) + return + } + } + sum, diff, err := s.svc.DryRun(r.Context(), *orgID, userID, parsedPkg) if pkger.IsParseErr(err) { s.encJSONResp(r.Context(), w, http.StatusUnprocessableEntity, RespApplyPkg{ diff --git a/http/pkger_http_server_test.go b/http/pkger_http_server_test.go index 32c1014d68b..d3deddd7227 100644 --- a/http/pkger_http_server_test.go +++ b/http/pkger_http_server_test.go @@ -76,13 +76,32 @@ func TestPkgerHTTPServer(t *testing.T) { tests := []struct { name string contentType string + reqBody fluxTTP.ReqApplyPkg }{ { name: "app json", contentType: "application/json", + reqBody: fluxTTP.ReqApplyPkg{ + DryRun: true, + OrgID: influxdb.ID(9000).String(), + Pkg: bucketPkg(t, pkger.EncodingJSON), + }, }, { name: "defaults json when no content type", + reqBody: fluxTTP.ReqApplyPkg{ + DryRun: true, + OrgID: influxdb.ID(9000).String(), + Pkg: bucketPkg(t, pkger.EncodingJSON), + }, + }, + { + name: "retrieves package from a URL", + reqBody: fluxTTP.ReqApplyPkg{ + DryRun: true, + OrgID: influxdb.ID(9000).String(), + URL: "https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json", + }, }, } @@ -108,11 +127,7 @@ func TestPkgerHTTPServer(t *testing.T) { svr := newMountedHandler(pkgHandler, 1) testttp. - PostJSON(t, "/api/v2/packages/apply", fluxTTP.ReqApplyPkg{ - DryRun: true, - OrgID: influxdb.ID(9000).String(), - Pkg: bucketPkg(t, pkger.EncodingJSON), - }). + PostJSON(t, "/api/v2/packages/apply", tt.reqBody). Headers("Content-Type", tt.contentType). Do(svr). ExpectStatus(http.StatusOK). @@ -257,7 +272,6 @@ func bucketPkg(t *testing.T, encoding pkger.Encoding) *pkger.Pkg { { "kind": "Bucket", "name": "rucket_11", - "retention_period": "1h", "description": "bucket 1 description" } ] @@ -275,7 +289,6 @@ spec: resources: - kind: Bucket name: rucket_11 - retention_period: 1h description: bucket 1 description ` default: diff --git a/http/swagger.yml b/http/swagger.yml index 137baaa0018..2233c34cff7 100644 --- a/http/swagger.yml +++ b/http/swagger.yml @@ -7140,6 +7140,8 @@ components: type: object additionalProperties: type: string + url: + type: string PkgCreate: type: object properties: diff --git a/pkger/models.go b/pkger/models.go index 9f079f2f44f..70d1d1c26d3 100644 --- a/pkger/models.go +++ b/pkger/models.go @@ -1321,7 +1321,7 @@ func (l *label) toInfluxLabel() influxdb.Label { } func toSummaryLabels(labels ...*label) []SummaryLabel { - var iLabels []SummaryLabel + iLabels := make([]SummaryLabel, 0, len(labels)) for _, l := range labels { iLabels = append(iLabels, l.summarize()) } diff --git a/pkger/parser.go b/pkger/parser.go index 9a92920bb7c..e98777001b4 100644 --- a/pkger/parser.go +++ b/pkger/parser.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "sort" "strconv" "strings" @@ -27,8 +28,9 @@ type Encoding int // encoding types const ( EncodingUnknown Encoding = iota - EncodingYAML EncodingJSON + EncodingSource // EncodingSource draws the encoding type by inferring it from the source. + EncodingYAML ) // String provides the string representation of the encoding. @@ -36,6 +38,8 @@ func (e Encoding) String() string { switch e { case EncodingJSON: return "json" + case EncodingSource: + return "source" case EncodingYAML: return "yaml" default: @@ -55,10 +59,12 @@ func Parse(encoding Encoding, readerFn ReaderFn, opts ...ValidateOptFn) (*Pkg, e } switch encoding { - case EncodingYAML: - return parseYAML(r, opts...) case EncodingJSON: return parseJSON(r, opts...) + case EncodingSource: + return parseSource(r, opts...) + case EncodingYAML: + return parseYAML(r, opts...) default: return nil, ErrInvalidEncoding } @@ -93,14 +99,54 @@ func FromString(s string) ReaderFn { } } -func parseYAML(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { - return parse(yaml.NewDecoder(r), opts...) +// FromHTTPRequest parses a pkg from the request body of a HTTP request. This is +// very useful when using packages that are hosted.. +func FromHTTPRequest(addr string) ReaderFn { + return func() (io.Reader, error) { + client := http.Client{Timeout: 5 * time.Minute} + resp, err := client.Get(addr) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var buf bytes.Buffer + if _, err := io.Copy(&buf, resp.Body); err != nil { + return nil, err + } + return &buf, nil + } } func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { return parse(json.NewDecoder(r), opts...) } +func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { + var b []byte + if byter, ok := r.(interface{ Bytes() []byte }); ok { + b = byter.Bytes() + } else { + bb, err := ioutil.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("failed to decode pkg source: %s", err) + } + b = bb + } + + contentType := http.DetectContentType(b) + switch { + case strings.Contains(contentType, "json"): + return parseJSON(bytes.NewReader(b), opts...) + default: + return parseYAML(bytes.NewReader(b), opts...) + } +} + +func parseYAML(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { + return parse(yaml.NewDecoder(r), opts...) +} + type decoder interface { Decode(interface{}) error } @@ -152,7 +198,20 @@ type Pkg struct { // associations the pkg contains. It is very useful for informing users of // the changes that will take place when this pkg would be applied. func (p *Pkg) Summary() Summary { - var sum Summary + // ensure zero values for arrays aren't returned, but instead + // we always returning an initialized slice. + sum := Summary{ + Buckets: []SummaryBucket{}, + Checks: []SummaryCheck{}, + Dashboards: []SummaryDashboard{}, + NotificationEndpoints: []SummaryNotificationEndpoint{}, + NotificationRules: []SummaryNotificationRule{}, + Labels: []SummaryLabel{}, + MissingSecrets: []string{}, + Tasks: []SummaryTask{}, + TelegrafConfigs: []SummaryTelegraf{}, + Variables: []SummaryVariable{}, + } // only add this after dry run has been completed if p.isVerified { @@ -368,8 +427,9 @@ func (p *Pkg) variables() []*variable { // If a resource does not exist yet, a label mapping will not // be returned for it. func (p *Pkg) labelMappings() []SummaryLabelMapping { - var mappings []SummaryLabelMapping - for _, l := range p.mLabels { + labels := p.mLabels + mappings := make([]SummaryLabelMapping, 0, len(labels)) + for _, l := range labels { mappings = append(mappings, l.mappingSummary()...) } diff --git a/pkger/service.go b/pkger/service.go index 1bdb611fbb2..08c5d1fe1f1 100644 --- a/pkger/service.go +++ b/pkger/service.go @@ -754,7 +754,7 @@ func (s *Service) dryRunBuckets(ctx context.Context, orgID influxdb.ID, pkg *Pkg } } - var diffs []DiffBucket + diffs := make([]DiffBucket, 0, len(mExistingBkts)) for _, diff := range mExistingBkts { diffs = append(diffs, diff) } @@ -784,7 +784,7 @@ func (s *Service) dryRunChecks(ctx context.Context, orgID influxdb.ID, pkg *Pkg) } } - var diffs []DiffCheck + diffs := make([]DiffCheck, 0, len(mExistingChecks)) for _, diff := range mExistingChecks { diffs = append(diffs, diff) } @@ -796,8 +796,10 @@ func (s *Service) dryRunChecks(ctx context.Context, orgID influxdb.ID, pkg *Pkg) } func (s *Service) dryRunDashboards(pkg *Pkg) []DiffDashboard { - var diffs []DiffDashboard - for _, d := range pkg.dashboards() { + dashs := pkg.dashboards() + + diffs := make([]DiffDashboard, 0, len(dashs)) + for _, d := range dashs { diffs = append(diffs, newDiffDashboard(d)) } return diffs @@ -862,7 +864,7 @@ func (s *Service) dryRunNotificationEndpoints(ctx context.Context, orgID influxd mExistingToNew[newEndpoint.Name()] = newDiffNotificationEndpoint(newEndpoint, existing) } - var diffs []DiffNotificationEndpoint + diffs := make([]DiffNotificationEndpoint, 0, len(mExistingToNew)) for _, diff := range mExistingToNew { diffs = append(diffs, diff) } @@ -885,7 +887,7 @@ func (s *Service) dryRunNotificationRules(ctx context.Context, orgID influxdb.ID mExisting[e.GetName()] = e } - var diffs []DiffNotificationRule + diffs := make([]DiffNotificationRule, 0, len(mExisting)) for _, r := range pkg.notificationRules() { e, ok := mExisting[r.endpointName] if !ok { @@ -929,8 +931,9 @@ func (s *Service) dryRunTasks(pkg *Pkg) []DiffTask { } func (s *Service) dryRunTelegraf(pkg *Pkg) []DiffTelegraf { - var diffs []DiffTelegraf - for _, t := range pkg.telegrafs() { + telegrafs := pkg.telegrafs() + diffs := make([]DiffTelegraf, 0, len(telegrafs)) + for _, t := range telegrafs { diffs = append(diffs, newDiffTelegraf(t)) } return diffs @@ -1008,7 +1011,7 @@ func (s *Service) dryRunLabelMappings(ctx context.Context, pkg *Pkg) ([]DiffLabe mapperVariables(pkg.variables()), } - var diffs []DiffLabelMapping + diffs := make([]DiffLabelMapping, 0) for _, mapper := range mappers { for i := 0; i < mapper.Len(); i++ { la := mapper.Association(i)