Skip to content

Commit 9a073f4

Browse files
cluttrdevcluttrdev
committed
BREAKING: Remove ClickHouse/Datastore functionality in favor of gRPC exporter
1 parent cf31800 commit 9a073f4

25 files changed

+299
-2236
lines changed

‎cmd/deduplicate.go

Lines changed: 0 additions & 107 deletions
This file was deleted.

‎cmd/export.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func NewExportCmd(rootConfig *RootConfig, out io.Writer) *ffcli.Command {
3636
return &ffcli.Command{
3737
Name: "export",
3838
ShortUsage: fmt.Sprintf("%s export <subcommand> [flags] [<args>...]", exeName),
39-
ShortHelp: "Export data from the GitLab API to ClickHouse",
39+
ShortHelp: "Export data from the GitLab API",
4040
UsageFunc: usageFunc,
4141
FlagSet: fs,
4242
Subcommands: []*ffcli.Command{

‎cmd/export_pipeline.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010

1111
"github.com/cluttrdev/gitlab-exporter/pkg/config"
1212
"github.com/cluttrdev/gitlab-exporter/pkg/controller"
13-
"github.com/cluttrdev/gitlab-exporter/pkg/tasks"
1413
)
1514

1615
type ExportPipelineConfig struct {
@@ -78,7 +77,7 @@ func (c *ExportPipelineConfig) Exec(ctx context.Context, args []string) error {
7877
return fmt.Errorf("error constructing controller: %w", err)
7978
}
8079

81-
opts := tasks.ExportPipelineHierarchyOptions{
80+
opts := controller.ExportPipelineHierarchyOptions{
8281
ProjectID: projectID,
8382
PipelineID: pipelineID,
8483

@@ -88,5 +87,5 @@ func (c *ExportPipelineConfig) Exec(ctx context.Context, args []string) error {
8887
ExportJobMetrics: c.exportSections, // for now, export metrics if we fetch the logs for sections anyway
8988
}
9089

91-
return tasks.ExportPipelineHierarchy(ctx, opts, &ctl.GitLab, ctl.DataStore)
90+
return controller.ExportPipelineHierarchy(ctl, ctx, opts)
9291
}

‎cmd/fetch_pipeline.go

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ type FetchPipelineConfig struct {
2121
fetchHierarchy bool
2222
fetchSections bool
2323

24+
outputTrace bool
25+
2426
flags *flag.FlagSet
2527
}
2628

@@ -51,6 +53,7 @@ func (c *FetchPipelineConfig) RegisterFlags(fs *flag.FlagSet) {
5153

5254
fs.BoolVar(&c.fetchHierarchy, "hierarchy", false, "Fetch pipeline hierarchy. (default: false)")
5355
fs.BoolVar(&c.fetchSections, "fetch-sections", true, "Fetch job sections. (default: true)")
56+
fs.BoolVar(&c.outputTrace, "trace", false, "Output pipeline trace. (default: false)")
5457
}
5558

5659
func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
@@ -81,7 +84,7 @@ func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
8184
}
8285

8386
var b []byte
84-
if c.fetchHierarchy {
87+
if c.fetchHierarchy || c.outputTrace {
8588
opt := &gitlab.GetPipelineHierarchyOptions{
8689
FetchSections: c.fetchSections,
8790
}
@@ -92,9 +95,17 @@ func (c *FetchPipelineConfig) Exec(ctx context.Context, args []string) error {
9295
}
9396
ph := phr.PipelineHierarchy
9497

95-
b, err = json.Marshal(ph)
96-
if err != nil {
97-
return fmt.Errorf("error marshalling pipeline hierarchy: %w", err)
98+
if c.outputTrace {
99+
ts := ph.GetAllTraces()
100+
b, err = json.Marshal(ts)
101+
if err != nil {
102+
return fmt.Errorf("error marshalling pipeline traces: %w", err)
103+
}
104+
} else {
105+
b, err = json.Marshal(ph)
106+
if err != nil {
107+
return fmt.Errorf("error marshalling pipeline hierarchy: %w", err)
108+
}
98109
}
99110
} else {
100111
p, err := ctl.GitLab.GetPipeline(ctx, projectID, pipelineID)

‎cmd/root.go

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,6 @@ func (c *RootConfig) RegisterFlags(fs *flag.FlagSet) {
5757
fs.String("gitlab-api-url", defaults.GitLab.Api.URL, fmt.Sprintf("The GitLab API URL (default: '%s').", defaults.GitLab.Api.URL))
5858
fs.String("gitlab-api-token", defaults.GitLab.Api.Token, fmt.Sprintf("The GitLab API Token (default: '%s').", defaults.GitLab.Api.Token))
5959

60-
fs.String("clickhouse-host", defaults.ClickHouse.Host, fmt.Sprintf("The ClickHouse server name (default: '%s').", defaults.ClickHouse.Host))
61-
fs.String("clickhouse-port", defaults.ClickHouse.Port, fmt.Sprintf("The ClickHouse port to connect to (default: '%s')", defaults.ClickHouse.Port))
62-
fs.String("clickhouse-database", defaults.ClickHouse.Database, fmt.Sprintf("Select the current default ClickHouse database (default: '%s').", defaults.ClickHouse.Database))
63-
fs.String("clickhouse-user", defaults.ClickHouse.User, fmt.Sprintf("The ClickHouse username to connect with (default: '%s').", defaults.ClickHouse.User))
64-
fs.String("clickhouse-password", defaults.ClickHouse.Password, fmt.Sprintf("The ClickHouse password (default: '%s').", defaults.ClickHouse.Password))
65-
6660
fs.StringVar(&c.filename, "config", "", "Configuration file to use.")
6761
}
6862

@@ -83,16 +77,6 @@ func loadConfig(filename string, flags *flag.FlagSet, cfg *config.Config) error
8377
cfg.GitLab.Api.URL = f.Value.String()
8478
case "gitlab-api-token":
8579
cfg.GitLab.Api.Token = f.Value.String()
86-
case "clickhouse-host":
87-
cfg.ClickHouse.Host = f.Value.String()
88-
case "clickhouse-port":
89-
cfg.ClickHouse.Port = f.Value.String()
90-
case "clickhouse-database":
91-
cfg.ClickHouse.Database = f.Value.String()
92-
case "clickhouse-user":
93-
cfg.ClickHouse.User = f.Value.String()
94-
case "clickhouse-password":
95-
cfg.ClickHouse.Password = f.Value.String()
9680
}
9781
})
9882

@@ -104,12 +88,6 @@ func writeConfig(cfg config.Config, out io.Writer) {
10488
fmt.Fprintf(out, "GitLab URL: %s\n", cfg.GitLab.Api.URL)
10589
fmt.Fprintf(out, "GitLab Token: %x\n", sha256String(cfg.GitLab.Api.Token))
10690
fmt.Fprintln(out, "----")
107-
fmt.Fprintf(out, "ClickHouse Host: %s\n", cfg.ClickHouse.Host)
108-
fmt.Fprintf(out, "ClickHouse Port: %s\n", cfg.ClickHouse.Port)
109-
fmt.Fprintf(out, "ClickHouse Database: %s\n", cfg.ClickHouse.Database)
110-
fmt.Fprintf(out, "ClickHouse User: %s\n", cfg.ClickHouse.User)
111-
fmt.Fprintf(out, "ClickHouse Password: %x\n", sha256String(cfg.ClickHouse.Password))
112-
fmt.Fprintln(out, "----")
11391

11492
projects := []int64{}
11593
for _, p := range cfg.Projects {

‎configs/gitlab-exporter.yaml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,6 @@ gitlab:
1010
rate:
1111
limit: 0.0
1212

13-
clickhouse:
14-
host: "127.0.0.1"
15-
port: "9000"
16-
database: "default"
17-
user: "default"
18-
password: ""
19-
2013
# List of gRPC server endpoints to export to
2114
endpoints: []
2215
# - address: "127.0.0.1:36275"

‎go.mod

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ module github.com/cluttrdev/gitlab-exporter
33
go 1.20
44

55
require (
6-
github.com/ClickHouse/clickhouse-go/v2 v2.12.0
76
github.com/creasty/defaults v1.7.0
87
github.com/google/go-cmp v0.5.9
98
github.com/peterbourgon/ff/v3 v3.4.0
@@ -18,29 +17,18 @@ require (
1817
)
1918

2019
require (
21-
github.com/ClickHouse/ch-go v0.52.1 // indirect
22-
github.com/andybalholm/brotli v1.0.5 // indirect
2320
github.com/beorn7/perks v1.0.1 // indirect
2421
github.com/cespare/xxhash/v2 v2.2.0 // indirect
25-
github.com/go-faster/city v1.0.1 // indirect
26-
github.com/go-faster/errors v0.6.1 // indirect
2722
github.com/golang/protobuf v1.5.3 // indirect
2823
github.com/google/go-querystring v1.1.0 // indirect
29-
github.com/google/uuid v1.3.1 // indirect
3024
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
3125
github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
32-
github.com/klauspost/compress v1.15.15 // indirect
26+
github.com/kr/text v0.1.0 // indirect
3327
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
34-
github.com/paulmach/orb v0.9.0 // indirect
35-
github.com/pierrec/lz4/v4 v4.1.17 // indirect
36-
github.com/pkg/errors v0.9.1 // indirect
3728
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
3829
github.com/prometheus/common v0.44.0 // indirect
3930
github.com/prometheus/procfs v0.11.1 // indirect
40-
github.com/segmentio/asm v1.2.0 // indirect
41-
github.com/shopspring/decimal v1.3.1 // indirect
42-
go.opentelemetry.io/otel v1.16.0 // indirect
43-
go.opentelemetry.io/otel/trace v1.16.0 // indirect
31+
github.com/stretchr/testify v1.8.3 // indirect
4432
golang.org/x/net v0.19.0 // indirect
4533
golang.org/x/oauth2 v0.13.0 // indirect
4634
golang.org/x/sys v0.15.0 // indirect

0 commit comments

Comments
 (0)