discoverybundler: Generate component files from components.go (signal…

…fx#4066)

Also adds `make bundle.d` diff check in tests job.

.github/workflows/build-and-test.yml

@@ -174,6 +174,14 @@ jobs:
           name: unit-test-results
           path: ./unit-test-results
 
+      - name: bundle.d
+        run: |
+          make bundle.d
+          if ! git diff --exit-code; then
+            echo "Discovery bundle.d config has changed. Run 'make bundle.d' and push the changes or ensure correct .tmpl updated."
+            exit 1
+          fi
+
   coverage:
     name: coverage
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved

Makefile

@@ -129,6 +129,8 @@ migratecheckpoint:
 
 .PHONY: bundle.d
 bundle.d:
+	go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler
+	go generate -tags bootstrap.bundle.d ./...
 	go generate -tags bundle.d ./...
 
 .PHONY: add-tag

internal/confmapprovider/discovery/bundle/README.md

@@ -38,13 +38,8 @@ Example `redis.discovery.yaml.tmpl`:
               `{{ configPropertyEnvVar "password" "<username>" }}` environment variable.
 ```
 
-After adding the required generate directive to `bundle_gen.go` and running `make bundle.d`:
-
-```go
-//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl
-```
-
-There is now a corresponding `bundle.d/receiver/redis.discovery.yaml`:
+After adding the required new component filename prefix to the `Components` instance in [`components.go`](./components.go)
+and running `make bundle.d`, there's now a corresponding `bundle.d/receivers/redis.discovery.yaml`:
 
 ```yaml
 #####################################################################################
@@ -69,6 +64,10 @@ redis:
               `SPLUNK_DISCOVERY_RECEIVERS_redis_CONFIG_password="<username>"` environment variable.
 ```
 
+In order for this to be included in the [Windows](./bundledfs_windows.go) and [Linux](./bundledfs_others.go) `BundledFS`
+be sure to include the component filename prefix in the corresponding `Components.Linux` and `Components.Windows`
+functions.
+
 When building the collector afterward, this redis receiver discovery config is now made available to discovery mode, and
 it can be disabled by `--set splunk.discovery.receivers.redis.enabled=false` or
 `SPLUNK_DISCOVERY_RECEIVERS_redis_ENABLED=false`.

internal/confmapprovider/discovery/bundle/bundle_gen.go

@@ -1,4 +1,4 @@
-// Copyright  Splunk, Inc.
+// Copyright Splunk, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,25 +16,22 @@
 
 // These are the discovery config component generating statements.
 // In order to update run go generate -tags bundle.d ./...
-// TODO: add a pre-bundle.d step that generates these generate directives for easier maintenance/management across platforms
-//go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler
-
-//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
-//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
-//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
-//go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
-//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl
-//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/redis.discovery.yaml.tmpl
-
 //go:generate discoverybundler -r -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/extensions/host-observer.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/host-observer.discovery.yaml.tmpl
 //go:generate discoverybundler -r -t bundle.d/extensions/k8s-observer.discovery.yaml.tmpl
 //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/k8s-observer.discovery.yaml.tmpl
 
+//go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl
+//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/redis.discovery.yaml.tmpl
+//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl
+//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl
+//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl
+
 package bundle

internal/confmapprovider/discovery/bundle/bundle_gen.tmpl

@@ -0,0 +1,33 @@
+// Copyright Splunk, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build bundle.d
+
+// These are the discovery config component generating statements.
+// In order to update run go generate -tags bundle.d ./...
+
+{{- range $ext := .Extensions }}
+//go:generate discoverybundler -r -t bundle.d/extensions/{{ $ext }}.discovery.yaml.tmpl
+{{- if index $.Linux $ext }}
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/{{ $ext }}.discovery.yaml.tmpl
+{{- end }}
+{{- end }}
+{{ range $rec := .Receivers }}
+//go:generate discoverybundler -r -t bundle.d/receivers/{{ $rec }}.discovery.yaml.tmpl
+{{- if index $.Linux $rec }}
+//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/{{ $rec }}.discovery.yaml.tmpl
+{{- end }}
+{{- end }}
+
+package bundle

internal/confmapprovider/discovery/bundle/bundle_other.go → internal/confmapprovider/discovery/bundle/bundledfs.tmpl

@@ -1,4 +1,4 @@
-// Copyright  Splunk, Inc.
+// Copyright Splunk, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build !windows
+//go:build {{ tag }}
 
 package bundle
 
@@ -21,11 +21,15 @@ import (
 )
 
 // BundledFS is the in-executable filesystem that contains all bundled discovery config.d components.
-//
-// If you are bootstrapping bundle_gen.go or the `discoverybundler` cmd without any rendered files in bundle.d,
-// comment out the below embed directives before installing to prevent "no matching files found"
-// build errors.
-//
-//go:embed bundle.d/extensions/*.discovery.yaml
-//go:embed bundle.d/receivers/*.discovery.yaml
+{{ range $ext := .Extensions }}
+{{- if included $ext }}
+//go:embed bundle.d/extensions/{{ $ext }}.discovery.yaml
+{{- end }}
+{{- end }}
+
+{{- range $rec := .Receivers }}
+{{- if included $rec }}
+//go:embed bundle.d/receivers/{{ $rec }}.discovery.yaml
+{{- end }}
+{{- end }}
 var BundledFS embed.FS

internal/confmapprovider/discovery/bundle/bundledfs_others.go

@@ -0,0 +1,33 @@
+// Copyright Splunk, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !windows
+
+package bundle
+
+import (
+	"embed"
+)
+
+// BundledFS is the in-executable filesystem that contains all bundled discovery config.d components.
+
+//go:embed bundle.d/extensions/docker-observer.discovery.yaml
+//go:embed bundle.d/extensions/host-observer.discovery.yaml
+//go:embed bundle.d/extensions/k8s-observer.discovery.yaml
+//go:embed bundle.d/receivers/oracledb.discovery.yaml
+//go:embed bundle.d/receivers/redis.discovery.yaml
+//go:embed bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml
+//go:embed bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml
+//go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml
+var BundledFS embed.FS

internal/confmapprovider/discovery/bundle/bundle_windows.go → internal/confmapprovider/discovery/bundle/bundledfs_windows.go

@@ -1,4 +1,4 @@
-// Copyright  Splunk, Inc.
+// Copyright Splunk, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,12 +21,10 @@ import (
 )
 
 // BundledFS is the in-executable filesystem that contains all bundled discovery config.d components.
-//
-// If you are bootstrapping bundle_gen.go or the `discoverybundler` cmd without any rendered files in bundle.d,
-// comment out the below embed directives before installing to prevent "no matching files found"
-// build errors.
-//
-//go:embed bundle.d/extensions/*.discovery.yaml
+
+//go:embed bundle.d/extensions/docker-observer.discovery.yaml
+//go:embed bundle.d/extensions/host-observer.discovery.yaml
+//go:embed bundle.d/extensions/k8s-observer.discovery.yaml
 //go:embed bundle.d/receivers/oracledb.discovery.yaml
 //go:embed bundle.d/receivers/redis.discovery.yaml
 //go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml

internal/confmapprovider/discovery/bundle/cmd/discoverybundler/bootstrap.go

@@ -0,0 +1,21 @@
+// Copyright Splunk, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file will generate bundle_gen.go, bundledfs_others.go, and bundledfs_windows.go for use w/
+// make bundle.d
+//go:build bootstrap.bundle.d
+
+//go:generate discoverybundler --bootstrap
+
+package main

internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go

@@ -51,6 +51,7 @@ type settings struct {
 	dir          string
 	render       bool
 	commented    bool
+	bootstrap    bool
 }
 
 func panicOnError(err error) {
@@ -64,6 +65,7 @@ func loadSettings() *settings {
 	flagSet := flag.NewFlagSet("discoverybundler", flag.ContinueOnError)
 	flagSet.StringVarP(&s.templateFile, "template", "t", "", "the discovery config template (.tmpl) to render")
 	flagSet.BoolVarP(&s.render, "render", "r", false, `whether to render in parent dir (default) or to --dir`)
+	flagSet.BoolVarP(&s.bootstrap, "bootstrap", "b", false, `run the bootstrap`)
 	flagSet.StringVarP(&s.dir, "dir", "d", "", `target directory to render to (sans ".tmpl")`)
 	flagSet.BoolVarP(&s.commented, "commented", "c", false, `whether to comment out all lines`)
 	panicOnError(flagSet.Parse(os.Args[1:]))
@@ -72,6 +74,11 @@ func loadSettings() *settings {
 
 func main() {
 	s := loadSettings()
+	if s.bootstrap {
+		bootstrap()
+		return
+	}
+
 	if s.templateFile == "" {
 		panic("empty templateFile")
 	}
@@ -132,3 +139,51 @@ func commentedTemplate(tmpl []byte) []byte {
 	}
 	return commented
 }
+
+// bootstrap will generate the bundle_gen.go file that invokes this lib, as
+// well as the bundledfs.go files for default discovery mode content.
+func bootstrap() {
+	bundleGenTmpl, err := os.ReadFile(filepath.Join("..", "..", "bundle_gen.tmpl"))
+	panicOnError(err)
+
+	t, err := template.New("bundle_gen").Parse(string(bundleGenTmpl))
+	panicOnError(err)
+
+	out := &bytes.Buffer{}
+	panicOnError(t.Execute(out, bundle.Components))
+
+	filename := filepath.Join("..", "..", "bundle_gen.go")
+	if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { // nolint:gosec // existing project file permissions
+		panicOnError(fmt.Errorf("failed writing to %s: %w", filename, err))
+	}
+
+	genBundledFS()
+}
+
+func genBundledFS() {
+	bundleFSTmpl, err := os.ReadFile(filepath.Join("..", "..", "bundledfs.tmpl"))
+	panicOnError(err)
+
+	for _, tup := range []struct {
+		components map[string]struct{}
+		tag        string
+		suffix     string
+	}{
+		{tag: "windows", suffix: "windows", components: bundle.Components.Windows},
+		{tag: "!windows", suffix: "others", components: bundle.Components.Linux},
+	} {
+		t, err := template.New(fmt.Sprintf("bundledfs_%s", tup.suffix)).Funcs(map[string]any{
+			"tag":      func() string { return tup.tag },
+			"included": func(s string) bool { _, ok := tup.components[s]; return ok },
+		}).Parse(string(bundleFSTmpl))
+		panicOnError(err)
+
+		out := &bytes.Buffer{}
+		panicOnError(t.Execute(out, bundle.Components))
+
+		filename := filepath.Join("..", "..", fmt.Sprintf("bundledfs_%s.go", tup.suffix))
+		if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { // nolint:gosec // existing project file permissions
+			panicOnError(fmt.Errorf("failed writing to %s: %w", filename, err))
+		}
+	}
+}