Merge pull request #247 from k1LoW/custom-metrics

Collect custom metrics

cmd/root.go

@@ -194,6 +194,10 @@ var rootCmd = &cobra.Command{
 			}
 		}
 
+		if err := r.CollectCustomMetrics(); err != nil {
+			cmd.PrintErrf("Skip collecting custom metrics: %v\n", err)
+		}
+
 		if r.CountMeasured() == 0 {
 			return errors.New("nothing could be measured")
 		}

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/migueleliasweb/go-github-mock v0.0.16
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/samber/lo v1.38.1
 	github.com/shurcooL/githubv4 v0.0.0-20230215024106-420ad0987b9b
 	github.com/spf13/cobra v1.6.1
 	github.com/tenntenn/golden v0.4.0

go.sum

@@ -516,6 +516,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
+github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=

report/custom.go

@@ -0,0 +1,14 @@
+package report
+
+type CustomMetricSet struct {
+	Key     string          `json:"key"`
+	Name    string          `json:"name,omitempty"`
+	Metrics []*CustomMetric `json:"metrics"`
+}
+
+type CustomMetric struct {
+	Key   string  `json:"key"`
+	Name  string  `json:"name,omitempty"`
+	Value float64 `json:"value"`
+	Unit  string  `json:"unit,omitempty"`
+}

report/report.go

@@ -20,6 +20,7 @@ import (
 	"github.com/k1LoW/octocov/pkg/coverage"
 	"github.com/k1LoW/octocov/pkg/ratio"
 	"github.com/olekukonko/tablewriter"
+	"github.com/samber/lo"
 )
 
 const filesHideMin = 30
@@ -33,6 +34,7 @@ type Report struct {
 	CodeToTestRatio   *ratio.Ratio       `json:"code_to_test_ratio,omitempty"`
 	TestExecutionTime *float64           `json:"test_execution_time,omitempty"`
 	Timestamp         time.Time          `json:"timestamp"`
+	CustomMetrics     []*CustomMetricSet `json:"custom_metrics,omitempty"`
 
 	// coverage report paths
 	covPaths []string
@@ -230,6 +232,7 @@ func (r *Report) CountMeasured() int {
 	if r.IsMeasuredTestExecutionTime() {
 		c += 1
 	}
+	c += len(r.CustomMetrics)
 	return c
 }
 
@@ -245,6 +248,10 @@ func (r *Report) IsMeasuredTestExecutionTime() bool {
 	return r.TestExecutionTime != nil
 }
 
+func (r *Report) IsCollectedCustomMetrics() bool {
+	return len(r.CustomMetrics) > 0
+}
+
 func (r *Report) Load(path string) error {
 	f, err := os.Stat(path)
 	if err != nil || f.IsDir() {
@@ -363,6 +370,73 @@ func (r *Report) MeasureTestExecutionTime(ctx context.Context, stepNames []strin
 	return nil
 }
 
+// CollectCustomMetrics collects custom metrics from env.
+func (r *Report) CollectCustomMetrics() error {
+	const envPrefix = "OCTOCOV_CUSTOM_METRICS_"
+	envs := [][]string{}
+	for _, e := range os.Environ() {
+		if !strings.HasPrefix(e, envPrefix) {
+			continue
+		}
+		kv := strings.SplitN(e, "=", 2)
+		if len(kv) != 2 {
+			continue
+		}
+		k := strings.TrimSpace(kv[0])
+		v := strings.TrimSpace(kv[1])
+		envs = append(envs, []string{k, v})
+	}
+	// Sort by key
+	sort.Slice(envs, func(i, j int) bool {
+		return envs[i][0] < envs[j][0]
+	})
+	for _, env := range envs {
+		v := env[1]
+		b, err := os.ReadFile(v)
+		if err != nil {
+			return err
+		}
+		set := &CustomMetricSet{}
+		if err := json.Unmarshal(b, set); err != nil {
+			return err
+		}
+		// Validate
+		if set.Key == "" {
+			return fmt.Errorf("key is required: %s", v)
+		}
+		if set.Name == "" {
+			set.Name = set.Key
+		}
+		for _, m := range set.Metrics {
+			if m.Key == "" {
+				return fmt.Errorf("key of metrics is required: %s", v)
+			}
+			if m.Name == "" {
+				m.Name = m.Key
+			}
+		}
+		if len(set.Metrics) != len(lo.UniqBy(set.Metrics, func(m *CustomMetric) string {
+			return m.Key
+		})) {
+			return fmt.Errorf("key of metrics must be unique: %s", lo.Map(set.Metrics, func(m *CustomMetric, _ int) string {
+				return m.Key
+			}))
+		}
+		r.CustomMetrics = append(r.CustomMetrics, set)
+	}
+
+	// Validate
+	if len(r.CustomMetrics) != len(lo.UniqBy(r.CustomMetrics, func(s *CustomMetricSet) string {
+		return s.Key
+	})) {
+		return fmt.Errorf("key of custom metrics must be unique: %s", lo.Map(r.CustomMetrics, func(s *CustomMetricSet, _ int) string {
+			return s.Key
+		}))
+	}
+
+	return nil
+}
+
 func (r *Report) CoveragePercent() float64 {
 	if r.Coverage == nil || r.Coverage.Total == 0 {
 		return 0.0

report/report_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goccy/go-json"
 	"github.com/google/go-cmp/cmp"
 	"github.com/k1LoW/octocov/gh"
 	"github.com/k1LoW/octocov/pkg/coverage"
@@ -99,6 +100,89 @@ func TestMeasureCoverage(t *testing.T) {
 	}
 }
 
+func TestCollectCustomMetrics(t *testing.T) {
+	tests := []struct {
+		envs    map[string]string
+		want    []*CustomMetricSet
+		wantErr bool
+	}{
+		{
+			map[string]string{
+				"OCTOCOV_CUSTOM_METRICS_BENCHMARK_0": filepath.Join(testdataDir(t), "custom_metrics", "benchmark_0.json"),
+			},
+			[]*CustomMetricSet{
+				{
+					Key:  "benchmark_0",
+					Name: "Benchmark-0",
+					Metrics: []*CustomMetric{
+						{Key: "count", Name: "Count", Value: 1000.0, Unit: ""},
+						{Key: "ns_per_op", Name: "ns/op", Value: 676.0, Unit: "ns/op"},
+					},
+				},
+			},
+			false,
+		},
+		{
+			map[string]string{
+				"OCTOCOV_CUSTOM_METRICS_BENCHMARK_1": filepath.Join(testdataDir(t), "custom_metrics", "benchmark_1.json"),
+				"OCTOCOV_CUSTOM_METRICS_BENCHMARK_0": filepath.Join(testdataDir(t), "custom_metrics", "benchmark_0.json"),
+			},
+			[]*CustomMetricSet{
+				{
+					Key:  "benchmark_0",
+					Name: "Benchmark-0",
+					Metrics: []*CustomMetric{
+						{Key: "count", Name: "Count", Value: 1000.0, Unit: ""},
+						{Key: "ns_per_op", Name: "ns/op", Value: 676.0, Unit: "ns/op"},
+					},
+				},
+				{
+					Key:  "benchmark_1",
+					Name: "Benchmark-1",
+					Metrics: []*CustomMetric{
+						{Key: "count", Name: "Count", Value: 1500.0, Unit: ""},
+						{Key: "ns_per_op", Name: "ns/op", Value: 1340.0, Unit: "ns/op"},
+					},
+				},
+			},
+			false,
+		},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if os.Getenv("UPDATE_GOLDEN") != "" {
+				for _, m := range tt.want {
+					b, err := json.MarshalIndent(m, "", "  ")
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := os.WriteFile(filepath.Join(testdataDir(t), "custom_metrics", fmt.Sprintf("%s.json", m.Key)), b, os.ModePerm); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			for k, v := range tt.envs {
+				t.Setenv(k, v)
+			}
+			r := &Report{}
+			if err := r.CollectCustomMetrics(); err != nil {
+				if !tt.wantErr {
+					t.Error(err)
+				}
+				return
+			}
+			if tt.wantErr {
+				t.Error("want error")
+				return
+			}
+			got := r.CustomMetrics
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Error(diff)
+			}
+		})
+	}
+}
+
 func TestCountMeasured(t *testing.T) {
 	tet := 1000.0
 	tests := []struct {
@@ -109,6 +193,10 @@ func TestCountMeasured(t *testing.T) {
 		{&Report{Coverage: &coverage.Coverage{}}, 1},
 		{&Report{CodeToTestRatio: &ratio.Ratio{}}, 1},
 		{&Report{TestExecutionTime: &tet}, 1},
+		{&Report{CustomMetrics: []*CustomMetricSet{
+			{Key: "m0", Metrics: []*CustomMetric{{}}},
+			{Key: "m1", Metrics: []*CustomMetric{{}}},
+		}}, 2},
 	}
 	for _, tt := range tests {
 		got := tt.r.CountMeasured()

testdata/custom_metrics/benchmark_0.json

@@ -0,0 +1,17 @@
+{
+  "key": "benchmark_0",
+  "name": "Benchmark-0",
+  "metrics": [
+    {
+      "key": "count",
+      "name": "Count",
+      "value": 1000
+    },
+    {
+      "key": "ns_per_op",
+      "name": "ns/op",
+      "value": 676,
+      "unit": "ns/op"
+    }
+  ]
+}

testdata/custom_metrics/benchmark_1.json

@@ -0,0 +1,17 @@
+{
+  "key": "benchmark_1",
+  "name": "Benchmark-1",
+  "metrics": [
+    {
+      "key": "count",
+      "name": "Count",
+      "value": 1500
+    },
+    {
+      "key": "ns_per_op",
+      "name": "ns/op",
+      "value": 1340,
+      "unit": "ns/op"
+    }
+  ]
+}