feat(pkger): add ability to supply a pkg from a url

CHANGELOG.md

@@ -37,6 +37,7 @@
 1. [16418](https://github.com/influxdata/influxdb/pull/16418): Add Developer Documentation
 1. [16260](https://github.com/influxdata/influxdb/pull/16260): Capture User-Agent header as query source for logging purposes
 1. [16469](https://github.com/influxdata/influxdb/pull/16469): Add support for configurable max batch size in points write handler
+1. [16509](https://github.com/influxdata/influxdb/pull/16509): Add support for applying an influx package via a public facing URL
 
 ### Bug Fixes
 

cmd/influx/pkg.go

@@ -48,6 +48,7 @@ type cmdPkgBuilder struct {
 	applyOpts struct {
 		force   string
 		secrets []string
+		url     string
 	}
 	exportOpts struct {
 		resourceType string
@@ -97,6 +98,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command {
 	cmd.MarkFlagFilename("file", "yaml", "yml", "json")
 	cmd.Flags().BoolVarP(&b.quiet, "quiet", "q", false, "disable output printing")
 	cmd.Flags().StringVar(&b.applyOpts.force, "force", "", `TTY input, if package will have destructive changes, proceed if set "true"`)
+	cmd.Flags().StringVarP(&b.applyOpts.url, "url", "u", "", "URL to retrieve a package.")
 
 	b.org.register(cmd, false)
 
@@ -132,7 +134,15 @@ func (b *cmdPkgBuilder) pkgApplyRunEFn() func(*cobra.Command, []string) error {
 			return err
 		}
 
-		pkg, isTTY, err := b.readPkgStdInOrFile(b.file)
+		var (
+			pkg   *pkger.Pkg
+			isTTY bool
+		)
+		if b.applyOpts.url != "" {
+			pkg, err = pkger.Parse(pkger.EncodingSource, pkger.FromHTTPRequest(b.applyOpts.url))
+		} else {
+			pkg, isTTY, err = b.readPkgStdInOrFile(b.file)
+		}
 		if err != nil {
 			return err
 		}

http/pkger_http_server.go

@@ -125,6 +125,7 @@ type (
 	ReqApplyPkg struct {
 		DryRun  bool              `json:"dryRun" yaml:"dryRun"`
 		OrgID   string            `json:"orgID" yaml:"orgID"`
+		URL     string            `json:"url" yaml:"url"`
 		Pkg     *pkger.Pkg        `json:"package" yaml:"package"`
 		Secrets map[string]string `json:"secrets"`
 	}
@@ -146,6 +147,14 @@ func (s *HandlerPkg) applyPkg(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if reqBody.URL != "" && reqBody.Pkg != nil {
+		s.HandleHTTPError(r.Context(), &influxdb.Error{
+			Code: influxdb.EInvalid,
+			Msg:  "must provide either url or pkg",
+		}, w)
+		return
+	}
+
 	orgID, err := influxdb.IDFromString(reqBody.OrgID)
 	if err != nil {
 		s.HandleHTTPError(r.Context(), &influxdb.Error{
@@ -163,6 +172,18 @@ func (s *HandlerPkg) applyPkg(w http.ResponseWriter, r *http.Request) {
 	userID := auth.GetUserID()
 
 	parsedPkg := reqBody.Pkg
+	if reqBody.URL != "" {
+		parsedPkg, err = pkger.Parse(pkger.EncodingSource, pkger.FromHTTPRequest(reqBody.URL))
+		if err != nil {
+			s.HandleHTTPError(r.Context(), &influxdb.Error{
+				Code: influxdb.EInvalid,
+				Msg:  "failed to parse package from provided URL",
+				Err:  err,
+			}, w)
+			return
+		}
+	}
+
 	sum, diff, err := s.svc.DryRun(r.Context(), *orgID, userID, parsedPkg)
 	if pkger.IsParseErr(err) {
 		s.encJSONResp(r.Context(), w, http.StatusUnprocessableEntity, RespApplyPkg{

http/pkger_http_server_test.go

@@ -76,13 +76,32 @@ func TestPkgerHTTPServer(t *testing.T) {
 			tests := []struct {
 				name        string
 				contentType string
+				reqBody     fluxTTP.ReqApplyPkg
 			}{
 				{
 					name:        "app json",
 					contentType: "application/json",
+					reqBody: fluxTTP.ReqApplyPkg{
+						DryRun: true,
+						OrgID:  influxdb.ID(9000).String(),
+						Pkg:    bucketPkg(t, pkger.EncodingJSON),
+					},
 				},
 				{
 					name: "defaults json when no content type",
+					reqBody: fluxTTP.ReqApplyPkg{
+						DryRun: true,
+						OrgID:  influxdb.ID(9000).String(),
+						Pkg:    bucketPkg(t, pkger.EncodingJSON),
+					},
+				},
+				{
+					name: "retrieves package from a URL",
+					reqBody: fluxTTP.ReqApplyPkg{
+						DryRun: true,
+						OrgID:  influxdb.ID(9000).String(),
+						URL:    "https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json",
+					},
 				},
 			}
 
@@ -108,11 +127,7 @@ func TestPkgerHTTPServer(t *testing.T) {
 					svr := newMountedHandler(pkgHandler, 1)
 
 					testttp.
-						PostJSON(t, "/api/v2/packages/apply", fluxTTP.ReqApplyPkg{
-							DryRun: true,
-							OrgID:  influxdb.ID(9000).String(),
-							Pkg:    bucketPkg(t, pkger.EncodingJSON),
-						}).
+						PostJSON(t, "/api/v2/packages/apply", tt.reqBody).
 						Headers("Content-Type", tt.contentType).
 						Do(svr).
 						ExpectStatus(http.StatusOK).
@@ -257,7 +272,6 @@ func bucketPkg(t *testing.T, encoding pkger.Encoding) *pkger.Pkg {
       {
         "kind": "Bucket",
         "name": "rucket_11",
-        "retention_period": "1h",
         "description": "bucket 1 description"
       }
     ]
@@ -275,7 +289,6 @@ spec:
   resources:
     - kind: Bucket
       name: rucket_11
-      retention_period: 1h
       description: bucket 1 description
 `
 	default:

http/swagger.yml

@@ -7140,6 +7140,8 @@ components:
           type: object
           additionalProperties:
             type: string
+        url:
+          type: string
     PkgCreate:
       type: object
       properties:

pkger/models.go

@@ -1321,7 +1321,7 @@ func (l *label) toInfluxLabel() influxdb.Label {
 }
 
 func toSummaryLabels(labels ...*label) []SummaryLabel {
-	var iLabels []SummaryLabel
+	iLabels := make([]SummaryLabel, 0, len(labels))
 	for _, l := range labels {
 		iLabels = append(iLabels, l.summarize())
 	}

pkger/parser.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -27,15 +28,18 @@ type Encoding int
 // encoding types
 const (
 	EncodingUnknown Encoding = iota
-	EncodingYAML
 	EncodingJSON
+	EncodingSource // EncodingSource draws the encoding type by inferring it from the source.
+	EncodingYAML
 )
 
 // String provides the string representation of the encoding.
 func (e Encoding) String() string {
 	switch e {
 	case EncodingJSON:
 		return "json"
+	case EncodingSource:
+		return "source"
 	case EncodingYAML:
 		return "yaml"
 	default:
@@ -55,10 +59,12 @@ func Parse(encoding Encoding, readerFn ReaderFn, opts ...ValidateOptFn) (*Pkg, e
 	}
 
 	switch encoding {
-	case EncodingYAML:
-		return parseYAML(r, opts...)
 	case EncodingJSON:
 		return parseJSON(r, opts...)
+	case EncodingSource:
+		return parseSource(r, opts...)
+	case EncodingYAML:
+		return parseYAML(r, opts...)
 	default:
 		return nil, ErrInvalidEncoding
 	}
@@ -93,14 +99,54 @@ func FromString(s string) ReaderFn {
 	}
 }
 
-func parseYAML(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
-	return parse(yaml.NewDecoder(r), opts...)
+// FromHTTPRequest parses a pkg from the request body of a HTTP request. This is
+// very useful when using packages that are hosted..
+func FromHTTPRequest(addr string) ReaderFn {
+	return func() (io.Reader, error) {
+		client := http.Client{Timeout: 5 * time.Minute}
+		resp, err := client.Get(addr)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		var buf bytes.Buffer
+		if _, err := io.Copy(&buf, resp.Body); err != nil {
+			return nil, err
+		}
+		return &buf, nil
+	}
 }
 
 func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
 	return parse(json.NewDecoder(r), opts...)
 }
 
+func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
+	var b []byte
+	if byter, ok := r.(interface{ Bytes() []byte }); ok {
+		b = byter.Bytes()
+	} else {
+		bb, err := ioutil.ReadAll(r)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode pkg source: %s", err)
+		}
+		b = bb
+	}
+
+	contentType := http.DetectContentType(b)
+	switch {
+	case strings.Contains(contentType, "json"):
+		return parseJSON(bytes.NewReader(b), opts...)
+	default:
+		return parseYAML(bytes.NewReader(b), opts...)
+	}
+}
+
+func parseYAML(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
+	return parse(yaml.NewDecoder(r), opts...)
+}
+
 type decoder interface {
 	Decode(interface{}) error
 }
@@ -152,7 +198,20 @@ type Pkg struct {
 // associations the pkg contains. It is very useful for informing users of
 // the changes that will take place when this pkg would be applied.
 func (p *Pkg) Summary() Summary {
-	var sum Summary
+	// ensure zero values for arrays aren't returned, but instead
+	// we always returning an initialized slice.
+	sum := Summary{
+		Buckets:               []SummaryBucket{},
+		Checks:                []SummaryCheck{},
+		Dashboards:            []SummaryDashboard{},
+		NotificationEndpoints: []SummaryNotificationEndpoint{},
+		NotificationRules:     []SummaryNotificationRule{},
+		Labels:                []SummaryLabel{},
+		MissingSecrets:        []string{},
+		Tasks:                 []SummaryTask{},
+		TelegrafConfigs:       []SummaryTelegraf{},
+		Variables:             []SummaryVariable{},
+	}
 
 	// only add this after dry run has been completed
 	if p.isVerified {
@@ -368,8 +427,9 @@ func (p *Pkg) variables() []*variable {
 // If a resource does not exist yet, a label mapping will not
 // be returned for it.
 func (p *Pkg) labelMappings() []SummaryLabelMapping {
-	var mappings []SummaryLabelMapping
-	for _, l := range p.mLabels {
+	labels := p.mLabels
+	mappings := make([]SummaryLabelMapping, 0, len(labels))
+	for _, l := range labels {
 		mappings = append(mappings, l.mappingSummary()...)
 	}
 

pkger/parser_test.go

@@ -121,9 +121,10 @@ spec:
 
 				actual := buckets[0]
 				expectedBucket := SummaryBucket{
-					Name:            "rucket_11",
-					Description:     "bucket 1 description",
-					RetentionPeriod: time.Hour,
+					Name:              "rucket_11",
+					Description:       "bucket 1 description",
+					RetentionPeriod:   time.Hour,
+					LabelAssociations: []SummaryLabel{},
 				}
 				assert.Equal(t, expectedBucket, actual)
 			})

pkger/service.go

@@ -754,7 +754,7 @@ func (s *Service) dryRunBuckets(ctx context.Context, orgID influxdb.ID, pkg *Pkg
 		}
 	}
 
-	var diffs []DiffBucket
+	diffs := make([]DiffBucket, 0, len(mExistingBkts))
 	for _, diff := range mExistingBkts {
 		diffs = append(diffs, diff)
 	}
@@ -784,7 +784,7 @@ func (s *Service) dryRunChecks(ctx context.Context, orgID influxdb.ID, pkg *Pkg)
 		}
 	}
 
-	var diffs []DiffCheck
+	diffs := make([]DiffCheck, 0, len(mExistingChecks))
 	for _, diff := range mExistingChecks {
 		diffs = append(diffs, diff)
 	}
@@ -796,8 +796,10 @@ func (s *Service) dryRunChecks(ctx context.Context, orgID influxdb.ID, pkg *Pkg)
 }
 
 func (s *Service) dryRunDashboards(pkg *Pkg) []DiffDashboard {
-	var diffs []DiffDashboard
-	for _, d := range pkg.dashboards() {
+	dashs := pkg.dashboards()
+
+	diffs := make([]DiffDashboard, 0, len(dashs))
+	for _, d := range dashs {
 		diffs = append(diffs, newDiffDashboard(d))
 	}
 	return diffs
@@ -862,7 +864,7 @@ func (s *Service) dryRunNotificationEndpoints(ctx context.Context, orgID influxd
 		mExistingToNew[newEndpoint.Name()] = newDiffNotificationEndpoint(newEndpoint, existing)
 	}
 
-	var diffs []DiffNotificationEndpoint
+	diffs := make([]DiffNotificationEndpoint, 0, len(mExistingToNew))
 	for _, diff := range mExistingToNew {
 		diffs = append(diffs, diff)
 	}
@@ -885,7 +887,7 @@ func (s *Service) dryRunNotificationRules(ctx context.Context, orgID influxdb.ID
 		mExisting[e.GetName()] = e
 	}
 
-	var diffs []DiffNotificationRule
+	diffs := make([]DiffNotificationRule, 0, len(mExisting))
 	for _, r := range pkg.notificationRules() {
 		e, ok := mExisting[r.endpointName]
 		if !ok {
@@ -929,8 +931,9 @@ func (s *Service) dryRunTasks(pkg *Pkg) []DiffTask {
 }
 
 func (s *Service) dryRunTelegraf(pkg *Pkg) []DiffTelegraf {
-	var diffs []DiffTelegraf
-	for _, t := range pkg.telegrafs() {
+	telegrafs := pkg.telegrafs()
+	diffs := make([]DiffTelegraf, 0, len(telegrafs))
+	for _, t := range telegrafs {
 		diffs = append(diffs, newDiffTelegraf(t))
 	}
 	return diffs
@@ -1008,7 +1011,7 @@ func (s *Service) dryRunLabelMappings(ctx context.Context, pkg *Pkg) ([]DiffLabe
 		mapperVariables(pkg.variables()),
 	}
 
-	var diffs []DiffLabelMapping
+	diffs := make([]DiffLabelMapping, 0)
 	for _, mapper := range mappers {
 		for i := 0; i < mapper.Len(); i++ {
 			la := mapper.Association(i)