Add command to generate cfgschema yaml files (#15233)

Add a simple CLI to generate config schema yaml files

.chloggen/yamlgen.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: configschema
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add yaml generation command
+
+# One or more tracking issues related to the change
+issues: [15231]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

cmd/configschema/cfgmetadatagen/cfgmetadatagen/README.md

@@ -0,0 +1,79 @@
+# Config Metadata YAML Generator (alpha)
+
+This CLI application creates a configuration metadata YAML file for each
+Collector component where each file describes the field names, types, default
+values, and inline documentation for the component's configuration.
+
+## Operation
+
+By default, this application creates a new `cfg-metadata` output directory
+(overridable via the `-o` flag), subdirectories for each component group
+(e.g. `receiver`, `exporter`, etc.), and config metadata YAML files within
+those directories for each component.
+
+### Command line flags
+
+* `-o <directory>` the name of the default parent directory to be created (defaults to `cfg-metadata`)
+* `-s <directory>` the path to the collector source root directory (defaults to `../..`)
+
+## Example Output
+
+The following is an example config metadata YAML file (for the File Exporter):
+
+```yaml
+type: '*fileexporter.Config'
+doc: |
+  Config defines configuration for file exporter.
+fields:
+- name: path
+  kind: string
+  default: ""
+  doc: |
+    Path of the file to write to. Path is relative to current directory.
+- name: rotation
+  type: '*fileexporter.Rotation'
+  kind: ptr
+  doc: |
+    Rotation defines an option about rotation of telemetry files
+  fields:
+  - name: max_megabytes
+    kind: int
+    doc: |
+      MaxMegabytes is the maximum size in megabytes of the file before it gets
+      rotated. It defaults to 100 megabytes.
+  - name: max_days
+    kind: int
+    doc: |
+      MaxDays is the maximum number of days to retain old log files based on the
+      timestamp encoded in their filename.  Note that a day is defined as 24
+      hours and may not exactly correspond to calendar days due to daylight
+      savings, leap seconds, etc. The default is not to remove old log files
+      based on age.
+  - name: max_backups
+    kind: int
+    doc: |
+      MaxBackups is the maximum number of old log files to retain. The default
+      is to 100 files.
+  - name: localtime
+    kind: bool
+    default: false
+    doc: |
+      LocalTime determines if the time used for formatting the timestamps in
+      backup files is the computer's local time.  The default is to use UTC
+      time.
+- name: format
+  kind: string
+  default: json
+  doc: |
+    FormatType define the data format of encoded telemetry data
+    Options:
+    - json[default]:  OTLP json bytes.
+    - proto:  OTLP binary protobuf bytes.
+- name: compression
+  kind: string
+  default: ""
+  doc: |
+    Compression Codec used to export telemetry data
+    Supported compression algorithms:`zstd`
+
+```

cmd/configschema/cfgmetadatagen/cfgmetadatagen/cli.go

@@ -0,0 +1,56 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cfgmetadatagen
+
+import (
+	"fmt"
+	"reflect"
+
+	"go.opentelemetry.io/collector/component"
+	"gopkg.in/yaml.v2"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema"
+)
+
+// GenerateFiles is the entry point for cfgmetadatagen. Component factories are
+// passed in so it can be used by other distros.
+func GenerateFiles(factories component.Factories, sourceDir string, outputDir string) error {
+	dr := configschema.NewDirResolver(sourceDir, configschema.DefaultModule)
+	writer := newMetadataFileWriter(outputDir)
+	configs := configschema.GetAllCfgInfos(factories)
+	for _, cfg := range configs {
+		err := writeComponentYAML(writer, cfg, dr)
+		if err != nil {
+			fmt.Printf("skipped writing config meta yaml: %v\n", err)
+		}
+	}
+	return nil
+}
+
+func writeComponentYAML(yw metadataWriter, cfg configschema.CfgInfo, dr configschema.DirResolver) error {
+	fields, err := configschema.ReadFields(reflect.ValueOf(cfg.CfgInstance), dr)
+	if err != nil {
+		return fmt.Errorf("error reading fields for component: %w", err)
+	}
+	yamlBytes, err := yaml.Marshal(fields)
+	if err != nil {
+		return fmt.Errorf("error marshaling to yaml: %w", err)
+	}
+	err = yw.write(cfg, yamlBytes)
+	if err != nil {
+		return fmt.Errorf("error writing component yaml: %w", err)
+	}
+	return nil
+}

cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer.go

@@ -0,0 +1,59 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cfgmetadatagen
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema"
+)
+
+type metadataWriter interface {
+	write(cfg configschema.CfgInfo, bytes []byte) error
+}
+
+type metadataFileWriter struct {
+	baseDir     string
+	dirsCreated map[string]struct{}
+}
+
+func newMetadataFileWriter(dir string) metadataWriter {
+	return &metadataFileWriter{
+		dirsCreated: map[string]struct{}{},
+		baseDir:     dir,
+	}
+}
+
+func (w *metadataFileWriter) write(cfg configschema.CfgInfo, yamlBytes []byte) error {
+	groupDir := filepath.Join(w.baseDir, cfg.Group)
+	if err := w.prepDir(groupDir); err != nil {
+		return err
+	}
+	filename := filepath.Join(groupDir, fmt.Sprintf("%s.yaml", cfg.Type))
+	fmt.Printf("writing file: %s\n", filename)
+	return os.WriteFile(filename, yamlBytes, 0600)
+}
+
+func (w *metadataFileWriter) prepDir(dir string) error {
+	if _, ok := w.dirsCreated[dir]; !ok {
+		if err := os.MkdirAll(dir, 0700); err != nil {
+			return fmt.Errorf("failed to make dir %q: %w", dir, err)
+		}
+		w.dirsCreated[dir] = struct{}{}
+	}
+	return nil
+}

cmd/configschema/cfgmetadatagen/cfgmetadatagen/metadata_writer_test.go

@@ -0,0 +1,42 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// skipping windows to avoid this golang bug: https://github.com/golang/go/issues/51442
+//go:build !windows
+
+package cfgmetadatagen
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema"
+)
+
+func TestMetadataFileWriter(t *testing.T) {
+	tempDir := t.TempDir()
+	w := newMetadataFileWriter(tempDir)
+	err := w.write(configschema.CfgInfo{Group: "mygroup", Type: "mytype"}, []byte("hello"))
+	require.NoError(t, err)
+	file, err := os.Open(filepath.Join(tempDir, "mygroup", "mytype.yaml"))
+	require.NoError(t, err)
+	bytes, err := io.ReadAll(file)
+	require.NoError(t, err)
+	assert.EqualValues(t, "hello", bytes)
+}

cmd/configschema/cfgmetadatagen/main.go

@@ -0,0 +1,45 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/cmd/configschema/cfgmetadatagen/cfgmetadatagen"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/components"
+)
+
+func main() {
+	sourceDir, outputDir := getFlags()
+	c, err := components.Components()
+	if err != nil {
+		fmt.Printf("error getting components %v", err)
+		os.Exit(1)
+	}
+	err = cfgmetadatagen.GenerateFiles(c, sourceDir, outputDir)
+	if err != nil {
+		fmt.Printf("cfg metadata generator failed: %v\n", err)
+	}
+}
+
+func getFlags() (string, string) {
+	sourceDir := flag.String("s", filepath.Join("..", ".."), "")
+	outputDir := flag.String("o", "cfg-metadata", "output dir")
+	flag.Parse()
+	return *sourceDir, *outputDir
+}

cmd/configschema/docsgen/docsgen/cli.go

@@ -126,7 +126,7 @@ func writeConfigDoc(
 		mdBytes = append(mdBytes, durationBlock...)
 	}
 
-	dir := dr.TypeToProjectPath(v.Type().Elem())
+	dir := dr.ReflectValueToProjectPath(v)
 	if dir == "" {
 		log.Printf("writeConfigDoc: skipping, local path not found for component: %s %s", ci.Group, ci.Type)
 		return

cmd/configschema/go.mod

@@ -13,6 +13,7 @@ require (
 	go.opentelemetry.io/collector/receiver/otlpreceiver v0.66.1-0.20221202005155-1c54042beb70
 	golang.org/x/mod v0.6.0
 	golang.org/x/text v0.4.0
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
@@ -662,7 +663,6 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gopkg.in/zorkian/go-datadog-api.v2 v2.30.0 // indirect
 	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect

cmd/configschema/resolver.go

@@ -34,6 +34,11 @@ const DefaultSrcRoot = "."
 // DirResolver.
 const DefaultModule = "github.com/open-telemetry/opentelemetry-collector-contrib"
 
+type DirResolverIntf interface {
+	TypeToPackagePath(t reflect.Type) (string, error)
+	ReflectValueToProjectPath(v reflect.Value) string
+}
+
 // DirResolver is used to resolve the base directory of a given reflect.Type.
 type DirResolver struct {
 	SrcRoot    string
@@ -71,9 +76,10 @@ func (dr DirResolver) TypeToPackagePath(t reflect.Type) (string, error) {
 	return verifiedGoPath, nil
 }
 
-// TypeToProjectPath accepts a Type and returns its directory in the current project. If
+// ReflectValueToProjectPath accepts a reflect.Value and returns its directory in the current project. If
 // the type doesn't live in the current project, returns "".
-func (dr DirResolver) TypeToProjectPath(t reflect.Type) string {
+func (dr DirResolver) ReflectValueToProjectPath(v reflect.Value) string {
+	t := v.Type().Elem()
 	if !strings.HasPrefix(t.PkgPath(), dr.ModuleName) {
 		return ""
 	}

cmd/configschema/resolver_test.go

@@ -69,12 +69,12 @@ func TestTypeToPackagePath_Error(t *testing.T) {
 }
 
 func TestTypeToProjectPath(t *testing.T) {
-	dir := testDR().TypeToProjectPath(reflect.ValueOf(redisreceiver.Config{}).Type())
+	dir := testDR().ReflectValueToProjectPath(reflect.ValueOf(&redisreceiver.Config{}))
 	assert.Equal(t, "../../receiver/redisreceiver", dir)
 }
 
 func TestTypetoProjectPath_External(t *testing.T) {
-	dir := testDR().TypeToProjectPath(reflect.ValueOf(otlpreceiver.Config{}).Type())
+	dir := testDR().ReflectValueToProjectPath(reflect.ValueOf(&otlpreceiver.Config{}))
 	assert.Equal(t, "", dir)
 }