From 82f3a780a1d4647b628066cbd2a95019cd69b953 Mon Sep 17 00:00:00 2001 From: Ryan Fitzpatrick <10867373+rmfitzpatrick@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:13:57 -0500 Subject: [PATCH] discoverybundler: Generate component files from components.go (#4066) Also adds `make bundle.d` diff check in tests job. --- .github/workflows/build-and-test.yml | 8 ++ Makefile | 2 + .../discovery/bundle/README.md | 13 ++- .../discovery/bundle/bundle_gen.go | 27 +++---- .../discovery/bundle/bundle_gen.tmpl | 33 ++++++++ .../{bundle_other.go => bundledfs.tmpl} | 22 +++--- ..._other_test.go => bundledfs_other_test.go} | 0 .../discovery/bundle/bundledfs_others.go | 33 ++++++++ ...bundle_windows.go => bundledfs_windows.go} | 12 ++- ...dows_test.go => bundledfs_windows_test.go} | 0 .../bundle/cmd/discoverybundler/bootstrap.go | 21 +++++ .../bundle/cmd/discoverybundler/main.go | 55 +++++++++++++ .../discovery/bundle/components.go | 79 +++++++++++++++++++ 13 files changed, 267 insertions(+), 38 deletions(-) create mode 100644 internal/confmapprovider/discovery/bundle/bundle_gen.tmpl rename internal/confmapprovider/discovery/bundle/{bundle_other.go => bundledfs.tmpl} (65%) rename internal/confmapprovider/discovery/bundle/{bundle_other_test.go => bundledfs_other_test.go} (100%) create mode 100644 internal/confmapprovider/discovery/bundle/bundledfs_others.go rename internal/confmapprovider/discovery/bundle/{bundle_windows.go => bundledfs_windows.go} (74%) rename internal/confmapprovider/discovery/bundle/{bundle_windows_test.go => bundledfs_windows_test.go} (100%) create mode 100644 internal/confmapprovider/discovery/bundle/cmd/discoverybundler/bootstrap.go create mode 100644 internal/confmapprovider/discovery/bundle/components.go diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 590fbf9976..04a4c25122 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -174,6 +174,14 @@ jobs: name: unit-test-results path: ./unit-test-results + - name: bundle.d + run: | + make bundle.d + if ! git diff --exit-code; then + echo "Discovery bundle.d config has changed. Run 'make bundle.d' and push the changes or ensure correct .tmpl updated." + exit 1 + fi + coverage: name: coverage # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved diff --git a/Makefile b/Makefile index 28ef628abe..2e44d2c95a 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,8 @@ migratecheckpoint: .PHONY: bundle.d bundle.d: + go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler + go generate -tags bootstrap.bundle.d ./... go generate -tags bundle.d ./... .PHONY: add-tag diff --git a/internal/confmapprovider/discovery/bundle/README.md b/internal/confmapprovider/discovery/bundle/README.md index 67b3ddd434..aebc568f38 100644 --- a/internal/confmapprovider/discovery/bundle/README.md +++ b/internal/confmapprovider/discovery/bundle/README.md @@ -38,13 +38,8 @@ Example `redis.discovery.yaml.tmpl`: `{{ configPropertyEnvVar "password" "" }}` environment variable. ``` -After adding the required generate directive to `bundle_gen.go` and running `make bundle.d`: - -```go -//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl -``` - -There is now a corresponding `bundle.d/receiver/redis.discovery.yaml`: +After adding the required new component filename prefix to the `Components` instance in [`components.go`](./components.go) +and running `make bundle.d`, there's now a corresponding `bundle.d/receivers/redis.discovery.yaml`: ```yaml ##################################################################################### @@ -69,6 +64,10 @@ redis: `SPLUNK_DISCOVERY_RECEIVERS_redis_CONFIG_password=""` environment variable. ``` +In order for this to be included in the [Windows](./bundledfs_windows.go) and [Linux](./bundledfs_others.go) `BundledFS` +be sure to include the component filename prefix in the corresponding `Components.Linux` and `Components.Windows` +functions. + When building the collector afterward, this redis receiver discovery config is now made available to discovery mode, and it can be disabled by `--set splunk.discovery.receivers.redis.enabled=false` or `SPLUNK_DISCOVERY_RECEIVERS_redis_ENABLED=false`. diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.go b/internal/confmapprovider/discovery/bundle/bundle_gen.go index d9fbbdbff9..86870329cb 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_gen.go +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.go @@ -1,4 +1,4 @@ -// Copyright Splunk, Inc. +// Copyright Splunk, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,20 +16,6 @@ // These are the discovery config component generating statements. // In order to update run go generate -tags bundle.d ./... -// TODO: add a pre-bundle.d step that generates these generate directives for easier maintenance/management across platforms -//go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler - -//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl -//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl -//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl -//go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl -//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/redis.discovery.yaml.tmpl - //go:generate discoverybundler -r -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/extensions/host-observer.discovery.yaml.tmpl @@ -37,4 +23,15 @@ //go:generate discoverybundler -r -t bundle.d/extensions/k8s-observer.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/k8s-observer.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/redis.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl + package bundle diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.tmpl b/internal/confmapprovider/discovery/bundle/bundle_gen.tmpl new file mode 100644 index 0000000000..8f5a0beea4 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.tmpl @@ -0,0 +1,33 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build bundle.d + +// These are the discovery config component generating statements. +// In order to update run go generate -tags bundle.d ./... + +{{- range $ext := .Extensions }} +//go:generate discoverybundler -r -t bundle.d/extensions/{{ $ext }}.discovery.yaml.tmpl +{{- if index $.Linux $ext }} +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/{{ $ext }}.discovery.yaml.tmpl +{{- end }} +{{- end }} +{{ range $rec := .Receivers }} +//go:generate discoverybundler -r -t bundle.d/receivers/{{ $rec }}.discovery.yaml.tmpl +{{- if index $.Linux $rec }} +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/{{ $rec }}.discovery.yaml.tmpl +{{- end }} +{{- end }} + +package bundle diff --git a/internal/confmapprovider/discovery/bundle/bundle_other.go b/internal/confmapprovider/discovery/bundle/bundledfs.tmpl similarity index 65% rename from internal/confmapprovider/discovery/bundle/bundle_other.go rename to internal/confmapprovider/discovery/bundle/bundledfs.tmpl index 82196aaf8f..1114a5108f 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_other.go +++ b/internal/confmapprovider/discovery/bundle/bundledfs.tmpl @@ -1,4 +1,4 @@ -// Copyright Splunk, Inc. +// Copyright Splunk, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !windows +//go:build {{ tag }} package bundle @@ -21,11 +21,15 @@ import ( ) // BundledFS is the in-executable filesystem that contains all bundled discovery config.d components. -// -// If you are bootstrapping bundle_gen.go or the `discoverybundler` cmd without any rendered files in bundle.d, -// comment out the below embed directives before installing to prevent "no matching files found" -// build errors. -// -//go:embed bundle.d/extensions/*.discovery.yaml -//go:embed bundle.d/receivers/*.discovery.yaml +{{ range $ext := .Extensions }} +{{- if included $ext }} +//go:embed bundle.d/extensions/{{ $ext }}.discovery.yaml +{{- end }} +{{- end }} + +{{- range $rec := .Receivers }} +{{- if included $rec }} +//go:embed bundle.d/receivers/{{ $rec }}.discovery.yaml +{{- end }} +{{- end }} var BundledFS embed.FS diff --git a/internal/confmapprovider/discovery/bundle/bundle_other_test.go b/internal/confmapprovider/discovery/bundle/bundledfs_other_test.go similarity index 100% rename from internal/confmapprovider/discovery/bundle/bundle_other_test.go rename to internal/confmapprovider/discovery/bundle/bundledfs_other_test.go diff --git a/internal/confmapprovider/discovery/bundle/bundledfs_others.go b/internal/confmapprovider/discovery/bundle/bundledfs_others.go new file mode 100644 index 0000000000..cfbd2de7e5 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundledfs_others.go @@ -0,0 +1,33 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package bundle + +import ( + "embed" +) + +// BundledFS is the in-executable filesystem that contains all bundled discovery config.d components. + +//go:embed bundle.d/extensions/docker-observer.discovery.yaml +//go:embed bundle.d/extensions/host-observer.discovery.yaml +//go:embed bundle.d/extensions/k8s-observer.discovery.yaml +//go:embed bundle.d/receivers/oracledb.discovery.yaml +//go:embed bundle.d/receivers/redis.discovery.yaml +//go:embed bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml +//go:embed bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml +//go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml +var BundledFS embed.FS diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows.go b/internal/confmapprovider/discovery/bundle/bundledfs_windows.go similarity index 74% rename from internal/confmapprovider/discovery/bundle/bundle_windows.go rename to internal/confmapprovider/discovery/bundle/bundledfs_windows.go index 175b19ee13..cdffa4c274 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_windows.go +++ b/internal/confmapprovider/discovery/bundle/bundledfs_windows.go @@ -1,4 +1,4 @@ -// Copyright Splunk, Inc. +// Copyright Splunk, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,12 +21,10 @@ import ( ) // BundledFS is the in-executable filesystem that contains all bundled discovery config.d components. -// -// If you are bootstrapping bundle_gen.go or the `discoverybundler` cmd without any rendered files in bundle.d, -// comment out the below embed directives before installing to prevent "no matching files found" -// build errors. -// -//go:embed bundle.d/extensions/*.discovery.yaml + +//go:embed bundle.d/extensions/docker-observer.discovery.yaml +//go:embed bundle.d/extensions/host-observer.discovery.yaml +//go:embed bundle.d/extensions/k8s-observer.discovery.yaml //go:embed bundle.d/receivers/oracledb.discovery.yaml //go:embed bundle.d/receivers/redis.discovery.yaml //go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows_test.go b/internal/confmapprovider/discovery/bundle/bundledfs_windows_test.go similarity index 100% rename from internal/confmapprovider/discovery/bundle/bundle_windows_test.go rename to internal/confmapprovider/discovery/bundle/bundledfs_windows_test.go diff --git a/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/bootstrap.go b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/bootstrap.go new file mode 100644 index 0000000000..d1a7d4d4d7 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/bootstrap.go @@ -0,0 +1,21 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file will generate bundle_gen.go, bundledfs_others.go, and bundledfs_windows.go for use w/ +// make bundle.d +//go:build bootstrap.bundle.d + +//go:generate discoverybundler --bootstrap + +package main diff --git a/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go index 67ed03a597..06a5034544 100644 --- a/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go +++ b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go @@ -51,6 +51,7 @@ type settings struct { dir string render bool commented bool + bootstrap bool } func panicOnError(err error) { @@ -64,6 +65,7 @@ func loadSettings() *settings { flagSet := flag.NewFlagSet("discoverybundler", flag.ContinueOnError) flagSet.StringVarP(&s.templateFile, "template", "t", "", "the discovery config template (.tmpl) to render") flagSet.BoolVarP(&s.render, "render", "r", false, `whether to render in parent dir (default) or to --dir`) + flagSet.BoolVarP(&s.bootstrap, "bootstrap", "b", false, `run the bootstrap`) flagSet.StringVarP(&s.dir, "dir", "d", "", `target directory to render to (sans ".tmpl")`) flagSet.BoolVarP(&s.commented, "commented", "c", false, `whether to comment out all lines`) panicOnError(flagSet.Parse(os.Args[1:])) @@ -72,6 +74,11 @@ func loadSettings() *settings { func main() { s := loadSettings() + if s.bootstrap { + bootstrap() + return + } + if s.templateFile == "" { panic("empty templateFile") } @@ -132,3 +139,51 @@ func commentedTemplate(tmpl []byte) []byte { } return commented } + +// bootstrap will generate the bundle_gen.go file that invokes this lib, as +// well as the bundledfs.go files for default discovery mode content. +func bootstrap() { + bundleGenTmpl, err := os.ReadFile(filepath.Join("..", "..", "bundle_gen.tmpl")) + panicOnError(err) + + t, err := template.New("bundle_gen").Parse(string(bundleGenTmpl)) + panicOnError(err) + + out := &bytes.Buffer{} + panicOnError(t.Execute(out, bundle.Components)) + + filename := filepath.Join("..", "..", "bundle_gen.go") + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { // nolint:gosec // existing project file permissions + panicOnError(fmt.Errorf("failed writing to %s: %w", filename, err)) + } + + genBundledFS() +} + +func genBundledFS() { + bundleFSTmpl, err := os.ReadFile(filepath.Join("..", "..", "bundledfs.tmpl")) + panicOnError(err) + + for _, tup := range []struct { + components map[string]struct{} + tag string + suffix string + }{ + {tag: "windows", suffix: "windows", components: bundle.Components.Windows}, + {tag: "!windows", suffix: "others", components: bundle.Components.Linux}, + } { + t, err := template.New(fmt.Sprintf("bundledfs_%s", tup.suffix)).Funcs(map[string]any{ + "tag": func() string { return tup.tag }, + "included": func(s string) bool { _, ok := tup.components[s]; return ok }, + }).Parse(string(bundleFSTmpl)) + panicOnError(err) + + out := &bytes.Buffer{} + panicOnError(t.Execute(out, bundle.Components)) + + filename := filepath.Join("..", "..", fmt.Sprintf("bundledfs_%s.go", tup.suffix)) + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { // nolint:gosec // existing project file permissions + panicOnError(fmt.Errorf("failed writing to %s: %w", filename, err)) + } + } +} diff --git a/internal/confmapprovider/discovery/bundle/components.go b/internal/confmapprovider/discovery/bundle/components.go new file mode 100644 index 0000000000..c1a927d252 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/components.go @@ -0,0 +1,79 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bundle + +import ( + "sort" +) + +var ( + // These are extensions that must match corresponding bundle.d/extensions/.discovery.yaml.tmpl files. + // If they are desired for !windows BundledFS inclusion (and a default linux conf.d entry), ensure they are included + // in Components.Linux. If desired in windows BundledFS, ensure they are included in Components.Windows. + extensions = []string{ + "docker-observer", + "host-observer", + "k8s-observer", + } + // These are receivers that must match corresponding bundle.d/receivers/.discovery.yaml.tmpl files + // If they are desired for !windows BundledFS inclusion (and a default linux conf.d entry), ensure they are included + // in Components.Linux. If desired in windows BundledFS, ensure they are included in Components.Windows. + receivers = []string{ + "oracledb", + "redis", + "smartagent-collectd-mysql", + "smartagent-collectd-nginx", + "smartagent-postgresql", + } + + Components = DiscoComponents{ + Extensions: func() []string { + sort.Strings(extensions) + return extensions + }(), + Receivers: func() []string { + sort.Strings(receivers) + return receivers + }(), + Linux: func() map[string]struct{} { + linux := map[string]struct{}{} + for _, extension := range extensions { + linux[extension] = struct{}{} + } + for _, receiver := range receivers { + linux[receiver] = struct{}{} + } + return linux + }(), + Windows: func() map[string]struct{} { + windows := map[string]struct{}{ + "oracledb": {}, + "redis": {}, + "smartagent-postgresql": {}, + } + for _, extension := range extensions { + windows[extension] = struct{}{} + } + return windows + }(), + } +) + +type DiscoComponents struct { + Linux map[string]struct{} + Windows map[string]struct{} + Extensions []string + Receivers []string +}