diff --git a/.chloggen/yamlgen.yaml b/.chloggen/yamlgen.yaml new file mode 100755 index 000000000000..3b3cbeb30338 --- /dev/null +++ b/.chloggen/yamlgen.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: configschema + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: add yaml generation command + +# One or more tracking issues related to the change +issues: [15231] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/cmd/configschema/cfgmetadatagen/cfgmetadatagen/README.md b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/README.md new file mode 100644 index 000000000000..5886f1b84193 --- /dev/null +++ b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/README.md @@ -0,0 +1,79 @@ +# Config Metadata YAML Generator (alpha) + +This CLI application creates a configuration metadata YAML file for each +Collector component where each file describes the field names, types, default +values, and inline documentation for the component's configuration. + +## Operation + +By default, this application creates a new `cfg-metadata` output directory +(overridable via the `-o` flag), subdirectories for each component group +(e.g. `receiver`, `exporter`, etc.), and config metadata YAML files within +those directories for each component. + +### Command line flags + +* `-o ` the name of the default parent directory to be created (defaults to `cfg-metadata`) +* `-s ` the path to the collector source root directory (defaults to `../..`) + +## Example Output + +The following is an example config metadata YAML file (for the File Exporter): + +```yaml +type: '*fileexporter.Config' +doc: | + Config defines configuration for file exporter. +fields: +- name: path + kind: string + default: "" + doc: | + Path of the file to write to. Path is relative to current directory. +- name: rotation + type: '*fileexporter.Rotation' + kind: ptr + doc: | + Rotation defines an option about rotation of telemetry files + fields: + - name: max_megabytes + kind: int + doc: | + MaxMegabytes is the maximum size in megabytes of the file before it gets + rotated. It defaults to 100 megabytes. + - name: max_days + kind: int + doc: | + MaxDays is the maximum number of days to retain old log files based on the + timestamp encoded in their filename. Note that a day is defined as 24 + hours and may not exactly correspond to calendar days due to daylight + savings, leap seconds, etc. The default is not to remove old log files + based on age. + - name: max_backups + kind: int + doc: | + MaxBackups is the maximum number of old log files to retain. The default + is to 100 files. + - name: localtime + kind: bool + default: false + doc: | + LocalTime determines if the time used for formatting the timestamps in + backup files is the computer's local time. The default is to use UTC + time. +- name: format + kind: string + default: json + doc: | + FormatType define the data format of encoded telemetry data + Options: + - json[default]: OTLP json bytes. + - proto: OTLP binary protobuf bytes. +- name: compression + kind: string + default: "" + doc: | + Compression Codec used to export telemetry data + Supported compression algorithms:`zstd` + +``` \ No newline at end of file diff --git a/cmd/configschema/cfgmetadatagen/cfgmetadatagen/cli.go b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/cli.go new file mode 100644 index 000000000000..5f4bf1398fa0 --- /dev/null +++ b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/cli.go @@ -0,0 +1,56 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cfgmetadatagen + +import ( + "fmt" + "reflect" + + "go.opentelemetry.io/collector/component" + "gopkg.in/yaml.v2" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema" +) + +// GenerateFiles is the entry point for cfgmetadatagen. Component factories are +// passed in so it can be used by other distros. +func GenerateFiles(factories component.Factories, sourceDir string, outputDir string) error { + dr := configschema.NewDirResolver(sourceDir, configschema.DefaultModule) + writer := newMetadataFileWriter(outputDir) + configs := configschema.GetAllCfgInfos(factories) + for _, cfg := range configs { + err := writeComponentYAML(writer, cfg, dr) + if err != nil { + fmt.Printf("skipped writing config meta yaml: %v\n", err) + } + } + return nil +} + +func writeComponentYAML(yw metadataWriter, cfg configschema.CfgInfo, dr configschema.DirResolver) error { + fields, err := configschema.ReadFields(reflect.ValueOf(cfg.CfgInstance), dr) + if err != nil { + return fmt.Errorf("error reading fields for component: %w", err) + } + yamlBytes, err := yaml.Marshal(fields) + if err != nil { + return fmt.Errorf("error marshaling to yaml: %w", err) + } + err = yw.write(cfg, yamlBytes) + if err != nil { + return fmt.Errorf("error writing component yaml: %w", err) + } + return nil +} diff --git a/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer.go b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer.go new file mode 100644 index 000000000000..27ada8b69f6c --- /dev/null +++ b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer.go @@ -0,0 +1,59 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cfgmetadatagen + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema" +) + +type metadataWriter interface { + write(cfg configschema.CfgInfo, bytes []byte) error +} + +type metadataFileWriter struct { + baseDir string + dirsCreated map[string]struct{} +} + +func newMetadataFileWriter(dir string) metadataWriter { + return &metadataFileWriter{ + dirsCreated: map[string]struct{}{}, + baseDir: dir, + } +} + +func (w *metadataFileWriter) write(cfg configschema.CfgInfo, yamlBytes []byte) error { + groupDir := filepath.Join(w.baseDir, cfg.Group) + if err := w.prepDir(groupDir); err != nil { + return err + } + filename := filepath.Join(groupDir, fmt.Sprintf("%s.yaml", cfg.Type)) + fmt.Printf("writing file: %s\n", filename) + return os.WriteFile(filename, yamlBytes, 0600) +} + +func (w *metadataFileWriter) prepDir(dir string) error { + if _, ok := w.dirsCreated[dir]; !ok { + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("failed to make dir %q: %w", dir, err) + } + w.dirsCreated[dir] = struct{}{} + } + return nil +} diff --git a/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer_test.go b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer_test.go new file mode 100644 index 000000000000..2ec3eba1cbe2 --- /dev/null +++ b/cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer_test.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// skipping windows to avoid this golang bug: https://github.com/golang/go/issues/51442 +//go:build !windows + +package cfgmetadatagen + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema" +) + +func TestMetadataFileWriter(t *testing.T) { + tempDir := t.TempDir() + w := newMetadataFileWriter(tempDir) + err := w.write(configschema.CfgInfo{Group: "mygroup", Type: "mytype"}, []byte("hello")) + require.NoError(t, err) + file, err := os.Open(filepath.Join(tempDir, "mygroup", "mytype.yaml")) + require.NoError(t, err) + bytes, err := io.ReadAll(file) + require.NoError(t, err) + assert.EqualValues(t, "hello", bytes) +} diff --git a/cmd/configschema/cfgmetadatagen/main.go b/cmd/configschema/cfgmetadatagen/main.go new file mode 100644 index 000000000000..ae98aff52927 --- /dev/null +++ b/cmd/configschema/cfgmetadatagen/main.go @@ -0,0 +1,45 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema/cfgmetadatagen/cfgmetadatagen" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/components" +) + +func main() { + sourceDir, outputDir := getFlags() + c, err := components.Components() + if err != nil { + fmt.Printf("error getting components %v", err) + os.Exit(1) + } + err = cfgmetadatagen.GenerateFiles(c, sourceDir, outputDir) + if err != nil { + fmt.Printf("cfg metadata generator failed: %v\n", err) + } +} + +func getFlags() (string, string) { + sourceDir := flag.String("s", filepath.Join("..", ".."), "") + outputDir := flag.String("o", "cfg-metadata", "output dir") + flag.Parse() + return *sourceDir, *outputDir +} diff --git a/cmd/configschema/docsgen/docsgen/cli.go b/cmd/configschema/docsgen/docsgen/cli.go index 0b38858463fb..4db4482415f5 100644 --- a/cmd/configschema/docsgen/docsgen/cli.go +++ b/cmd/configschema/docsgen/docsgen/cli.go @@ -126,7 +126,7 @@ func writeConfigDoc( mdBytes = append(mdBytes, durationBlock...) } - dir := dr.TypeToProjectPath(v.Type().Elem()) + dir := dr.ReflectValueToProjectPath(v) if dir == "" { log.Printf("writeConfigDoc: skipping, local path not found for component: %s %s", ci.Group, ci.Type) return diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index 485ce6cd260d..2e009ffadeb0 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -13,6 +13,7 @@ require ( go.opentelemetry.io/collector/receiver/otlpreceiver v0.66.1-0.20221202005155-1c54042beb70 golang.org/x/mod v0.6.0 golang.org/x/text v0.4.0 + gopkg.in/yaml.v2 v2.4.0 ) require ( @@ -662,7 +663,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/zorkian/go-datadog-api.v2 v2.30.0 // indirect inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect diff --git a/cmd/configschema/resolver.go b/cmd/configschema/resolver.go index 0b1f64f5472a..a249e4dcde52 100644 --- a/cmd/configschema/resolver.go +++ b/cmd/configschema/resolver.go @@ -34,6 +34,11 @@ const DefaultSrcRoot = "." // DirResolver. const DefaultModule = "github.com/open-telemetry/opentelemetry-collector-contrib" +type DirResolverIntf interface { + TypeToPackagePath(t reflect.Type) (string, error) + ReflectValueToProjectPath(v reflect.Value) string +} + // DirResolver is used to resolve the base directory of a given reflect.Type. type DirResolver struct { SrcRoot string @@ -71,9 +76,10 @@ func (dr DirResolver) TypeToPackagePath(t reflect.Type) (string, error) { return verifiedGoPath, nil } -// TypeToProjectPath accepts a Type and returns its directory in the current project. If +// ReflectValueToProjectPath accepts a reflect.Value and returns its directory in the current project. If // the type doesn't live in the current project, returns "". -func (dr DirResolver) TypeToProjectPath(t reflect.Type) string { +func (dr DirResolver) ReflectValueToProjectPath(v reflect.Value) string { + t := v.Type().Elem() if !strings.HasPrefix(t.PkgPath(), dr.ModuleName) { return "" } diff --git a/cmd/configschema/resolver_test.go b/cmd/configschema/resolver_test.go index 7b728e93c2b4..d338aca055cd 100644 --- a/cmd/configschema/resolver_test.go +++ b/cmd/configschema/resolver_test.go @@ -69,12 +69,12 @@ func TestTypeToPackagePath_Error(t *testing.T) { } func TestTypeToProjectPath(t *testing.T) { - dir := testDR().TypeToProjectPath(reflect.ValueOf(redisreceiver.Config{}).Type()) + dir := testDR().ReflectValueToProjectPath(reflect.ValueOf(&redisreceiver.Config{})) assert.Equal(t, "../../receiver/redisreceiver", dir) } func TestTypetoProjectPath_External(t *testing.T) { - dir := testDR().TypeToProjectPath(reflect.ValueOf(otlpreceiver.Config{}).Type()) + dir := testDR().ReflectValueToProjectPath(reflect.ValueOf(&otlpreceiver.Config{})) assert.Equal(t, "", dir) }