BREAKING: Remove ClickHouse/Datastore functionality in favor of gRPC …

…exporter

cmd/export.go

@@ -36,7 +36,7 @@ func NewExportCmd(rootConfig *RootConfig, out io.Writer) *ffcli.Command {
 	return &ffcli.Command{
 		Name:       "export",
 		ShortUsage: fmt.Sprintf("%s export <subcommand> [flags] [<args>...]", exeName),
-		ShortHelp:  "Export data from the GitLab API to ClickHouse",
+		ShortHelp:  "Export data from the GitLab API",
 		UsageFunc:  usageFunc,
 		FlagSet:    fs,
 		Subcommands: []*ffcli.Command{

cmd/export_pipeline.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/cluttrdev/gitlab-exporter/pkg/config"
 	"github.com/cluttrdev/gitlab-exporter/pkg/controller"
-	"github.com/cluttrdev/gitlab-exporter/pkg/tasks"
 )
 
 type ExportPipelineConfig struct {
@@ -78,7 +77,7 @@ func (c *ExportPipelineConfig) Exec(ctx context.Context, args []string) error {
 		return fmt.Errorf("error constructing controller: %w", err)
 	}
 
-	opts := tasks.ExportPipelineHierarchyOptions{
+	opts := controller.ExportPipelineHierarchyOptions{
 		ProjectID:  projectID,
 		PipelineID: pipelineID,
 
@@ -88,5 +87,5 @@ func (c *ExportPipelineConfig) Exec(ctx context.Context, args []string) error {
 		ExportJobMetrics:  c.exportSections, // for now, export metrics if we fetch the logs for sections anyway
 	}
 
-	return tasks.ExportPipelineHierarchy(ctx, opts, &ctl.GitLab, ctl.DataStore)
+	return controller.ExportPipelineHierarchy(ctl, ctx, opts)
 }

cmd/fetch_pipeline.go

@@ -21,6 +21,8 @@ type FetchPipelineConfig struct {
 	fetchHierarchy bool
 	fetchSections  bool
 
+	outputTrace bool
+
 	flags *flag.FlagSet
 }
 
@@ -51,6 +53,7 @@ func (c *FetchPipelineConfig) RegisterFlags(fs *flag.FlagSet) {
 
 	fs.BoolVar(&c.fetchHierarchy, "hierarchy", false, "Fetch pipeline hierarchy. (default: false)")
 	fs.BoolVar(&c.fetchSections, "fetch-sections", true, "Fetch job sections. (default: true)")
+	fs.BoolVar(&c.outputTrace, "trace", false, "Output pipeline trace. (default: false)")
 }
 
 func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
@@ -81,7 +84,7 @@ func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
 	}
 
 	var b []byte
-	if c.fetchHierarchy {
+	if c.fetchHierarchy || c.outputTrace {
 		opt := &gitlab.GetPipelineHierarchyOptions{
 			FetchSections: c.fetchSections,
 		}
@@ -92,9 +95,17 @@ func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
 		}
 		ph := phr.PipelineHierarchy
 
-		b, err = json.Marshal(ph)
-		if err != nil {
-			return fmt.Errorf("error marshalling pipeline hierarchy: %w", err)
+		if c.outputTrace {
+			ts := ph.GetAllTraces()
+			b, err = json.Marshal(ts)
+			if err != nil {
+				return fmt.Errorf("error marshalling pipeline traces: %w", err)
+			}
+		} else {
+			b, err = json.Marshal(ph)
+			if err != nil {
+				return fmt.Errorf("error marshalling pipeline hierarchy: %w", err)
+			}
 		}
 	} else {
 		p, err := ctl.GitLab.GetPipeline(ctx, projectID, pipelineID)

cmd/root.go

@@ -57,12 +57,6 @@ func (c *RootConfig) RegisterFlags(fs *flag.FlagSet) {
 	fs.String("gitlab-api-url", defaults.GitLab.Api.URL, fmt.Sprintf("The GitLab API URL (default: '%s').", defaults.GitLab.Api.URL))
 	fs.String("gitlab-api-token", defaults.GitLab.Api.Token, fmt.Sprintf("The GitLab API Token (default: '%s').", defaults.GitLab.Api.Token))
 
-	fs.String("clickhouse-host", defaults.ClickHouse.Host, fmt.Sprintf("The ClickHouse server name (default: '%s').", defaults.ClickHouse.Host))
-	fs.String("clickhouse-port", defaults.ClickHouse.Port, fmt.Sprintf("The ClickHouse port to connect to (default: '%s')", defaults.ClickHouse.Port))
-	fs.String("clickhouse-database", defaults.ClickHouse.Database, fmt.Sprintf("Select the current default ClickHouse database (default: '%s').", defaults.ClickHouse.Database))
-	fs.String("clickhouse-user", defaults.ClickHouse.User, fmt.Sprintf("The ClickHouse username to connect with (default: '%s').", defaults.ClickHouse.User))
-	fs.String("clickhouse-password", defaults.ClickHouse.Password, fmt.Sprintf("The ClickHouse password (default: '%s').", defaults.ClickHouse.Password))
-
 	fs.StringVar(&c.filename, "config", "", "Configuration file to use.")
 }
 
@@ -83,16 +77,6 @@ func loadConfig(filename string, flags *flag.FlagSet, cfg *config.Config) error
 			cfg.GitLab.Api.URL = f.Value.String()
 		case "gitlab-api-token":
 			cfg.GitLab.Api.Token = f.Value.String()
-		case "clickhouse-host":
-			cfg.ClickHouse.Host = f.Value.String()
-		case "clickhouse-port":
-			cfg.ClickHouse.Port = f.Value.String()
-		case "clickhouse-database":
-			cfg.ClickHouse.Database = f.Value.String()
-		case "clickhouse-user":
-			cfg.ClickHouse.User = f.Value.String()
-		case "clickhouse-password":
-			cfg.ClickHouse.Password = f.Value.String()
 		}
 	})
 
@@ -104,12 +88,6 @@ func writeConfig(cfg config.Config, out io.Writer) {
 	fmt.Fprintf(out, "GitLab URL: %s\n", cfg.GitLab.Api.URL)
 	fmt.Fprintf(out, "GitLab Token: %x\n", sha256String(cfg.GitLab.Api.Token))
 	fmt.Fprintln(out, "----")
-	fmt.Fprintf(out, "ClickHouse Host: %s\n", cfg.ClickHouse.Host)
-	fmt.Fprintf(out, "ClickHouse Port: %s\n", cfg.ClickHouse.Port)
-	fmt.Fprintf(out, "ClickHouse Database: %s\n", cfg.ClickHouse.Database)
-	fmt.Fprintf(out, "ClickHouse User: %s\n", cfg.ClickHouse.User)
-	fmt.Fprintf(out, "ClickHouse Password: %x\n", sha256String(cfg.ClickHouse.Password))
-	fmt.Fprintln(out, "----")
 
 	projects := []int64{}
 	for _, p := range cfg.Projects {

configs/gitlab-exporter.yaml

@@ -10,13 +10,6 @@ gitlab:
     rate:
       limit: 0.0
 
-clickhouse:
-  host: "127.0.0.1"
-  port: "9000"
-  database: "default"
-  user: "default"
-  password: ""
-
 # List of gRPC server endpoints to export to
 endpoints: []
   # - address: "127.0.0.1:36275"

go.mod

@@ -3,7 +3,6 @@ module github.com/cluttrdev/gitlab-exporter
 go 1.20
 
 require (
-	github.com/ClickHouse/clickhouse-go/v2 v2.12.0
 	github.com/creasty/defaults v1.7.0
 	github.com/google/go-cmp v0.5.9
 	github.com/peterbourgon/ff/v3 v3.4.0
@@ -18,29 +17,18 @@ require (
 )
 
 require (
-	github.com/ClickHouse/ch-go v0.52.1 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.6.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/uuid v1.3.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
-	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/kr/text v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/paulmach/orb v0.9.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
-	github.com/segmentio/asm v1.2.0 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
-	go.opentelemetry.io/otel v1.16.0 // indirect
-	go.opentelemetry.io/otel/trace v1.16.0 // indirect
+	github.com/stretchr/testify v1.8.3 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect