Rename config to policy and dataset to data stream

cmd/testrunner.go

@@ -37,7 +37,7 @@ func setupTestCommand() *cobra.Command {
 
 	cmd.PersistentFlags().BoolP(cobraext.FailOnMissingFlagName, "m", false, cobraext.FailOnMissingFlagDescription)
 	cmd.PersistentFlags().BoolP(cobraext.GenerateTestResultFlagName, "g", false, cobraext.GenerateTestResultFlagDescription)
-	cmd.PersistentFlags().StringSliceP(cobraext.DatasetsFlagName, "d", nil, cobraext.DatasetsFlagDescription)
+	cmd.PersistentFlags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription)
 
 	for _, testType := range testrunner.TestTypes() {
 		action := testTypeCommandActionFactory(testType)
@@ -65,9 +65,9 @@ func testTypeCommandActionFactory(testType testrunner.TestType) cobraext.Command
 			return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName)
 		}
 
-		datasets, err := cmd.Flags().GetStringSlice(cobraext.DatasetsFlagName)
+		dataStreams, err := cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName)
 		if err != nil {
-			return cobraext.FlagParsingError(err, cobraext.DatasetsFlagName)
+			return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName)
 		}
 
 		generateTestResult, err := cmd.Flags().GetBool(cobraext.GenerateTestResultFlagName)
@@ -83,14 +83,14 @@ func testTypeCommandActionFactory(testType testrunner.TestType) cobraext.Command
 			return errors.Wrap(err, "locating package root failed")
 		}
 
-		testFolders, err := testrunner.FindTestFolders(packageRootPath, testType, datasets)
+		testFolders, err := testrunner.FindTestFolders(packageRootPath, testType, dataStreams)
 		if err != nil {
 			return errors.Wrap(err, "unable to determine test folder paths")
 		}
 
 		if failOnMissing && len(testFolders) == 0 {
-			if len(datasets) > 0 {
-				return fmt.Errorf("no %s tests found for %s dataset(s)", testType, strings.Join(datasets, ","))
+			if len(dataStreams) > 0 {
+				return fmt.Errorf("no %s tests found for %s data stream(s)", testType, strings.Join(dataStreams, ","))
 			}
 			return fmt.Errorf("no %s tests found", testType)
 		}

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/aymerick/raymond v2.0.2+incompatible
 	github.com/elastic/go-elasticsearch/v7 v7.9.0
-	github.com/elastic/package-spec/code/go v0.0.0-20200918124159-a193a3467b7e
+	github.com/elastic/package-spec/code/go v0.0.0-20200930112347-763ab44be25b
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/google/go-github/v32 v32.1.0

go.sum

@@ -78,8 +78,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M=
 github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
-github.com/elastic/package-spec/code/go v0.0.0-20200918124159-a193a3467b7e h1:Jqnv0q+uGZb/mZYraVT/bqmTN10qxfZas2TvDTPjjMk=
-github.com/elastic/package-spec/code/go v0.0.0-20200918124159-a193a3467b7e/go.mod h1:SuwqJxWP/DP1OiyXaYToOM6J3n8VswrVoswLsWX+NLo=
+github.com/elastic/package-spec/code/go v0.0.0-20200930112347-763ab44be25b h1:yn0xwbErCuRnNB0v1uLZAgmBYIcXW0k9fRx1+/6ri3Y=
+github.com/elastic/package-spec/code/go v0.0.0-20200930112347-763ab44be25b/go.mod h1:Kz0t9hcE0vmayVhw1sYTtIt7fsVwBElVpEhOI3GswBw=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -481,8 +481,8 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2 h1:daAzF/Ytp6YSqJDu1hZJthJIhOrsAa7UbIkziU1t0K4=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0zYfkB7UCa+ayT6uVvbyI=
+golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/cobraext/const.go

@@ -9,8 +9,8 @@ const (
 	DaemonModeFlagName        = "daemon"
 	DaemonModeFlagDescription = "daemon mode"
 
-	DatasetsFlagName        = "datasets"
-	DatasetsFlagDescription = "comma-separated datasets to test"
+	DataStreamsFlagName        = "data-streams"
+	DataStreamsFlagDescription = "comma-separated data streams to test"
 
 	FailOnMissingFlagName        = "fail-on-missing"
 	FailOnMissingFlagDescription = "fail if tests are missing"

internal/kibana/ingestmanager/client_policies.go

@@ -78,8 +78,8 @@ type Vars map[string]Var
 
 // DataStream represents a data stream within a package.
 type DataStream struct {
-	Type    string `json:"type"`
-	Dataset string `json:"dataset"`
+	Type       string `json:"type"`
+	DataStream string `json:"data_stream"`
 }
 
 // Stream encapsulates a data stream and it's variables.

internal/packages/packages.go

@@ -18,18 +18,18 @@ const (
 	// PackageManifestFile is the name of the package's main manifest file.
 	PackageManifestFile = "manifest.yml"
 
-	// DatasetManifestFile is the name of the dataset's manifest file.
-	DatasetManifestFile = "manifest.yml"
+	// DataStreamManifestFile is the name of the dataStream's manifest file.
+	DataStreamManifestFile = "manifest.yml"
 )
 
-// VarValue represents a variable value as defined in a package or dataset
+// VarValue represents a variable value as defined in a package or dataStream
 // manifest file.
 type VarValue struct {
 	scalar string
 	list   []string
 }
 
-// UnmarshalYAML knows how to parse a variable value from a package or dataset
+// UnmarshalYAML knows how to parse a variable value from a package or dataStream
 // manifest file into a VarValue.
 func (vv *VarValue) UnmarshalYAML(value *yaml.Node) error {
 	switch value.Kind {
@@ -68,7 +68,7 @@ type input struct {
 	Vars []variable `json:"vars"`
 }
 
-type configTemplate struct {
+type policyTemplate struct {
 	Inputs []input `json:"inputs"`
 }
 
@@ -78,11 +78,11 @@ type PackageManifest struct {
 	Title           string           `json:"title"`
 	Type            string           `json:"type"`
 	Version         string           `json:"version"`
-	ConfigTemplates []configTemplate `json:"config_templates" yaml:"config_templates"`
+	PolicyTemplates []policyTemplate `json:"policy_templates" yaml:"policy_templates"`
 }
 
-// DatasetManifest represents the structure of a dataset's manifest
-type DatasetManifest struct {
+// DataStreamManifest represents the structure of a dataStream's manifest
+type DataStreamManifest struct {
 	Name          string `json:"name"`
 	Title         string `json:"title"`
 	Type          string `json:"type"`
@@ -124,14 +124,14 @@ func FindPackageRoot() (string, bool, error) {
 	return "", false, nil
 }
 
-// FindDatasetRootForPath finds and returns the path to the root folder of a dataset.
-func FindDatasetRootForPath(workDir string) (string, bool, error) {
+// FindDataStreamRootForPath finds and returns the path to the root folder of a dataStream.
+func FindDataStreamRootForPath(workDir string) (string, bool, error) {
 	dir := workDir
 	for dir != "." {
-		path := filepath.Join(dir, DatasetManifestFile)
+		path := filepath.Join(dir, DataStreamManifestFile)
 		fileInfo, err := os.Stat(path)
 		if err == nil && !fileInfo.IsDir() {
-			ok, err := isDatasetManifest(path)
+			ok, err := isDataStreamManifest(path)
 			if err != nil {
 				return "", false, errors.Wrapf(err, "verifying manifest file failed (path: %s)", path)
 			}
@@ -163,25 +163,25 @@ func ReadPackageManifest(path string) (*PackageManifest, error) {
 	return &m, nil
 }
 
-// ReadDatasetManifest reads and parses the given dataset manifest file.
-func ReadDatasetManifest(path string) (*DatasetManifest, error) {
+// ReadDataStreamManifest reads and parses the given data streammanifest file.
+func ReadDataStreamManifest(path string) (*DataStreamManifest, error) {
 	content, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading file body failed (path: %s)", path)
 	}
 
-	var m DatasetManifest
+	var m DataStreamManifest
 	err = yaml.Unmarshal(content, &m)
 	if err != nil {
-		return nil, errors.Wrapf(err, "unmarshalling dataset manifest failed (path: %s)", path)
+		return nil, errors.Wrapf(err, "unmarshalling data streammanifest failed (path: %s)", path)
 	}
 
 	m.Name = filepath.Base(filepath.Dir(path))
 	return &m, nil
 }
 
-func (ct *configTemplate) FindInputByType(inputType string) *input {
-	for _, input := range ct.Inputs {
+func (pt *policyTemplate) FindInputByType(inputType string) *input {
+	for _, input := range pt.Inputs {
 		if input.Type == inputType {
 			return &input
 		}
@@ -197,8 +197,8 @@ func isPackageManifest(path string) (bool, error) {
 	return m.Type == "integration" && m.Version != "", nil // TODO add support for other package types
 }
 
-func isDatasetManifest(path string) (bool, error) {
-	m, err := ReadDatasetManifest(path)
+func isDataStreamManifest(path string) (bool, error) {
+	m, err := ReadDataStreamManifest(path)
 	if err != nil {
 		return false, errors.Wrapf(err, "reading package manifest failed (path: %s)", path)
 	}

internal/testrunner/runners/pipeline/ingest_pipeline.go

@@ -49,16 +49,16 @@ type pipelineIngestedDocument struct {
 	Doc pipelineDocument `json:"doc"`
 }
 
-func installIngestPipelines(esClient *elasticsearch.Client, datasetPath string) (string, []pipelineResource, error) {
-	datasetManifest, err := packages.ReadDatasetManifest(filepath.Join(datasetPath, packages.DatasetManifestFile))
+func installIngestPipelines(esClient *elasticsearch.Client, dataStreamPath string) (string, []pipelineResource, error) {
+	dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile))
 	if err != nil {
-		return "", nil, errors.Wrap(err, "reading dataset manifest failed")
+		return "", nil, errors.Wrap(err, "reading data stream manifest failed")
 	}
 
 	nonce := time.Now().UnixNano()
 
-	mainPipeline := getWithPipelineNameWithNonce(getPipelineNameOrDefault(datasetManifest), nonce)
-	pipelines, err := loadIngestPipelineFiles(datasetPath, nonce)
+	mainPipeline := getWithPipelineNameWithNonce(getPipelineNameOrDefault(dataStreamManifest), nonce)
+	pipelines, err := loadIngestPipelineFiles(dataStreamPath, nonce)
 	if err != nil {
 		return "", nil, errors.Wrap(err, "loading ingest pipeline files failed")
 	}
@@ -75,15 +75,15 @@ func installIngestPipelines(esClient *elasticsearch.Client, datasetPath string)
 	return mainPipeline, jsonPipelines, nil
 }
 
-func getPipelineNameOrDefault(datasetManifest *packages.DatasetManifest) string {
-	if datasetManifest.Elasticsearch != nil && datasetManifest.Elasticsearch.IngestPipelineName != "" {
-		return datasetManifest.Elasticsearch.IngestPipelineName
+func getPipelineNameOrDefault(dataStreamManifest *packages.DataStreamManifest) string {
+	if dataStreamManifest.Elasticsearch != nil && dataStreamManifest.Elasticsearch.IngestPipelineName != "" {
+		return dataStreamManifest.Elasticsearch.IngestPipelineName
 	}
 	return defaultPipelineName
 }
 
-func loadIngestPipelineFiles(datasetPath string, nonce int64) ([]pipelineResource, error) {
-	elasticsearchPath := filepath.Join(datasetPath, "elasticsearch", "ingest_pipeline")
+func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]pipelineResource, error) {
+	elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline")
 	fis, err := ioutil.ReadDir(elasticsearchPath)
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading ingest pipelines directory failed (path: %s)", elasticsearchPath)

internal/testrunner/runners/pipeline/runner.go

@@ -38,15 +38,15 @@ func (r *runner) run() error {
 		return errors.Wrap(err, "listing test case definitions failed")
 	}
 
-	datasetPath, found, err := packages.FindDatasetRootForPath(r.options.TestFolder.Path)
+	dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path)
 	if err != nil {
-		return errors.Wrap(err, "locating dataset root failed")
+		return errors.Wrap(err, "locating data_stream root failed")
 	}
 	if !found {
-		return errors.New("dataset root not found")
+		return errors.New("dataStream root not found")
 	}
 
-	entryPipeline, pipelineIDs, err := installIngestPipelines(r.options.ESClient, datasetPath)
+	entryPipeline, pipelineIDs, err := installIngestPipelines(r.options.ESClient, dataStreamPath)
 	if err != nil {
 		return errors.Wrap(err, "installing ingest pipelines failed")
 	}

internal/testrunner/runners/system/runner.go

@@ -73,17 +73,17 @@ func (r *runner) run() error {
 		return errors.Wrap(err, "reading package manifest failed")
 	}
 
-	datasetPath, found, err := packages.FindDatasetRootForPath(r.testFolder.Path)
+	dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.testFolder.Path)
 	if err != nil {
-		return errors.Wrap(err, "locating dataset root failed")
+		return errors.Wrap(err, "locating data stream root failed")
 	}
 	if !found {
-		return errors.New("dataset root not found")
+		return errors.New("dataStream root not found")
 	}
 
-	datasetManifest, err := packages.ReadDatasetManifest(filepath.Join(datasetPath, packages.DatasetManifestFile))
+	dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile))
 	if err != nil {
-		return errors.Wrap(err, "reading dataset manifest failed")
+		return errors.Wrap(err, "reading data stream manifest failed")
 	}
 
 	serviceLogsDir, err := install.ServiceLogsDir()
@@ -125,8 +125,8 @@ func (r *runner) run() error {
 	logger.Info("creating test policy...")
 	testTime := time.Now().Format("20060102T15:04:05Z")
 	p := ingestmanager.Policy{
-		Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.testFolder.Package, r.testFolder.Dataset, testTime),
-		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.testFolder.Package, r.testFolder.Dataset),
+		Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),
+		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream),
 		Namespace:   "ep",
 	}
 	policy, err := im.CreatePolicy(p)
@@ -145,10 +145,10 @@ func (r *runner) run() error {
 		return errors.Wrap(err, "unable to load system test configuration")
 	}
 
-	logger.Info("adding package datastream to test policy...")
-	ds := createPackageDatastream(*policy, *pkgManifest, *datasetManifest, *testConfig)
+	logger.Info("adding package data stream to test policy...")
+	ds := createPackageDatastream(*policy, *pkgManifest, *dataStreamManifest, *testConfig)
 	if err := im.AddPackageDataStreamToPolicy(ds); err != nil {
-		return errors.Wrap(err, "could not add dataset Config to policy")
+		return errors.Wrap(err, "could not add data stream config to policy")
 	}
 
 	// Get enrolled agent ID
@@ -168,7 +168,7 @@ func (r *runner) run() error {
 	dataStream := fmt.Sprintf(
 		"%s-%s-%s",
 		ds.Inputs[0].Streams[0].DataStream.Type,
-		ds.Inputs[0].Streams[0].DataStream.Dataset,
+		ds.Inputs[0].Streams[0].DataStream.DataStream,
 		ds.Namespace,
 	)
 
@@ -185,7 +185,7 @@ func (r *runner) run() error {
 	}
 
 	// Assign policy to agent
-	logger.Info("assigning package datastream to agent...")
+	logger.Info("assigning package data streamto agent...")
 	if err := im.AssignPolicyToAgent(agent, *policy); err != nil {
 		return errors.Wrap(err, "could not assign policy to agent")
 	}
@@ -229,10 +229,10 @@ func (r *runner) run() error {
 	}
 
 	if passed {
-		fmt.Printf("System test for %s/%s dataset passed!\n", r.testFolder.Package, r.testFolder.Dataset)
+		fmt.Printf("System test for %s/%s data stream passed!\n", r.testFolder.Package, r.testFolder.DataStream)
 	} else {
-		fmt.Printf("System test for %s/%s dataset failed\n", r.testFolder.Package, r.testFolder.Dataset)
-		return fmt.Errorf("system test for %s/%s dataset failed", r.testFolder.Package, r.testFolder.Dataset)
+		fmt.Printf("System test for %s/%s data stream failed\n", r.testFolder.Package, r.testFolder.DataStream)
+		return fmt.Errorf("system test for %s/%s data stream failed", r.testFolder.Package, r.testFolder.DataStream)
 	}
 	return nil
 }
@@ -273,7 +273,7 @@ func getStackSettingsFromEnv() stackSettings {
 func createPackageDatastream(
 	p ingestmanager.Policy,
 	pkg packages.PackageManifest,
-	ds packages.DatasetManifest,
+	ds packages.DataStreamManifest,
 	c testConfig,
 ) ingestmanager.PackageDataStream {
 	streamInput := ds.Streams[0].Input
@@ -300,18 +300,18 @@ func createPackageDatastream(
 			ID:      fmt.Sprintf("%s-%s.%s", streamInput, pkg.Name, ds.Name),
 			Enabled: true,
 			DataStream: ingestmanager.DataStream{
-				Type:    ds.Type,
-				Dataset: fmt.Sprintf("%s.%s", pkg.Name, ds.Name),
+				Type:       ds.Type,
+				DataStream: fmt.Sprintf("%s.%s", pkg.Name, ds.Name),
 			},
 		},
 	}
 
-	// Add dataset-level vars
+	// Add dataStream-level vars
 	dsVars := ingestmanager.Vars{}
 	for _, dsVar := range ds.Streams[0].Vars {
 		val := dsVar.Default
 
-		cfgVar, exists := c.Dataset.Vars[dsVar.Name]
+		cfgVar, exists := c.DataStream.Vars[dsVar.Name]
 		if exists {
 			// overlay var value from test configuration
 			val = cfgVar
@@ -327,7 +327,7 @@ func createPackageDatastream(
 
 	// Add package-level vars
 	pkgVars := ingestmanager.Vars{}
-	input := pkg.ConfigTemplates[0].FindInputByType(streamInput)
+	input := pkg.PolicyTemplates[0].FindInputByType(streamInput)
 	if input != nil {
 		for _, pkgVar := range input.Vars {
 			val := pkgVar.Default

internal/testrunner/runners/system/test_config.go

@@ -20,10 +20,10 @@ import (
 const configFileName = "config.yml"
 
 type testConfig struct {
-	Vars    map[string]packages.VarValue `yaml:"vars"`
-	Dataset struct {
+	Vars       map[string]packages.VarValue `yaml:"vars"`
+	DataStream struct {
 		Vars map[string]packages.VarValue `yaml:"vars"`
-	} `yaml:"dataset"`
+	} `yaml:"data_stream"`
 }
 
 func newConfig(systemTestFolderPath string, ctxt servicedeployer.ServiceContext) (*testConfig, error) {