Skip to content

Commit 7488884

Browse files
cluttrdevcluttrdev
committed
Replace clickhouse.Client usage with new DataStore interface
1 parent 6b635de commit 7488884

File tree

5 files changed

+54
-45
lines changed

5 files changed

+54
-45
lines changed

‎cmd/export_pipeline.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,5 @@ func (c *ExportPipelineConfig) Exec(ctx context.Context, args []string) error {
8787
ExportTraces: c.exportTraces,
8888
}
8989

90-
return tasks.ExportPipelineHierarchy(ctx, opts, &ctl.GitLab, &ctl.ClickHouse)
90+
return tasks.ExportPipelineHierarchy(ctx, opts, &ctl.GitLab, ctl.DataStore)
9191
}

‎pkg/controller/controller.go‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ import (
1313
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/config"
1414
gitlab "github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/gitlab"
1515
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/worker"
16+
17+
"github.com/cluttrdev/gitlab-clickhouse-exporter/internal/datastore"
1618
)
1719

1820
type Controller struct {
19-
config config.Config
20-
GitLab gitlab.Client
21-
ClickHouse clickhouse.Client
21+
config config.Config
22+
GitLab gitlab.Client
23+
DataStore datastore.DataStore
2224

2325
workers []worker.Worker
2426
}
@@ -40,7 +42,7 @@ func (c *Controller) configure(cfg config.Config) error {
4042
return err
4143
}
4244

43-
if err := c.configureClickHouseClient(cfg.ClickHouse); err != nil {
45+
if err := c.configureClickHouseDataStore(cfg.ClickHouse); err != nil {
4446
return err
4547
}
4648

@@ -60,24 +62,32 @@ func (c *Controller) configureGitLabClient(cfg config.GitLab) error {
6062
})
6163
}
6264

63-
func (c *Controller) configureClickHouseClient(cfg config.ClickHouse) error {
64-
return c.ClickHouse.Configure(clickhouse.ClientConfig{
65+
func (c *Controller) configureClickHouseDataStore(cfg config.ClickHouse) error {
66+
var client clickhouse.Client
67+
68+
conf := clickhouse.ClientConfig{
6569
Host: cfg.Host,
6670
Port: cfg.Port,
6771
Database: cfg.Database,
6872
User: cfg.User,
6973
Password: cfg.Password,
70-
})
74+
}
75+
if err := client.Configure(conf); err != nil {
76+
return err
77+
}
78+
79+
c.DataStore = clickhouse.NewClickHouseDataStore(&client)
80+
return nil
7181
}
7282

7383
func (c *Controller) configureWorkers(cfg config.Config) error {
7484
workers := []worker.Worker{}
7585

7686
for _, prj := range cfg.Projects {
7787
if prj.CatchUp.Enabled {
78-
workers = append(workers, worker.NewCatchUpProjectWorker(prj, &c.GitLab, &c.ClickHouse))
88+
workers = append(workers, worker.NewCatchUpProjectWorker(prj, &c.GitLab, c.DataStore))
7989
}
80-
workers = append(workers, worker.NewExportProjectWorker(prj, &c.GitLab, &c.ClickHouse))
90+
workers = append(workers, worker.NewExportProjectWorker(prj, &c.GitLab, c.DataStore))
8191
}
8292

8393
c.workers = workers
@@ -86,12 +96,8 @@ func (c *Controller) configureWorkers(cfg config.Config) error {
8696
}
8797

8898
func (c *Controller) init(ctx context.Context) error {
89-
if err := c.ClickHouse.CreateDatabase(ctx); err != nil {
90-
return fmt.Errorf("error creating database: %w", err)
91-
}
92-
93-
if err := c.ClickHouse.CreateTables(ctx); err != nil {
94-
return fmt.Errorf("error creating tables: %w", err)
99+
if err := c.DataStore.Initialize(ctx); err != nil {
100+
return err
95101
}
96102

97103
return nil
@@ -102,7 +108,7 @@ func (c *Controller) CheckReadiness(ctx context.Context) error {
102108
return err
103109
}
104110

105-
if err := c.ClickHouse.CheckReadiness(ctx); err != nil {
111+
if err := c.DataStore.CheckReadiness(ctx); err != nil {
106112
return err
107113
}
108114

‎pkg/tasks/export.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ import (
44
"context"
55
"fmt"
66

7-
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/clickhouse"
87
gitlab "github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/gitlab"
98
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/models"
9+
10+
"github.com/cluttrdev/gitlab-clickhouse-exporter/internal/datastore"
1011
)
1112

1213
type ExportPipelineHierarchyOptions struct {
@@ -18,11 +19,11 @@ type ExportPipelineHierarchyOptions struct {
1819
ExportTraces bool
1920
}
2021

21-
func ExportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOptions, gl *gitlab.Client, ch *clickhouse.Client) error {
22-
return <-exportPipelineHierarchy(ctx, opts, gl, ch)
22+
func ExportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOptions, gl *gitlab.Client, ds datastore.DataStore) error {
23+
return <-exportPipelineHierarchy(ctx, opts, gl, ds)
2324
}
2425

25-
func exportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOptions, gl *gitlab.Client, ch *clickhouse.Client) <-chan error {
26+
func exportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOptions, gl *gitlab.Client, ds datastore.DataStore) <-chan error {
2627
out := make(chan error)
2728

2829
go func() {
@@ -39,14 +40,14 @@ func exportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOp
3940
}
4041
ph := phr.PipelineHierarchy
4142

42-
if err := clickhouse.InsertPipelineHierarchy(ctx, ph, ch); err != nil {
43+
if err := ds.InsertPipelineHierarchy(ctx, ph); err != nil {
4344
out <- fmt.Errorf("error inserting pipeline hierarchy: %w", err)
4445
return
4546
}
4647

4748
if opts.ExportTraces {
4849
pts := ph.GetAllTraces()
49-
if err := clickhouse.InsertTraces(ctx, pts, ch); err != nil {
50+
if err := ds.InsertTraces(ctx, pts); err != nil {
5051
out <- fmt.Errorf("error inserting traces: %w", err)
5152
return
5253
}
@@ -66,15 +67,15 @@ func exportPipelineHierarchy(ctx context.Context, opts ExportPipelineHierarchyOp
6667
tcs = append(tcs, ts.TestCases...)
6768
}
6869
}
69-
if err = clickhouse.InsertTestReports(ctx, trs, ch); err != nil {
70+
if err = ds.InsertTestReports(ctx, trs); err != nil {
7071
out <- fmt.Errorf("error inserting testreports: %w", err)
7172
return
7273
}
73-
if err = clickhouse.InsertTestSuites(ctx, tss, ch); err != nil {
74+
if err = ds.InsertTestSuites(ctx, tss); err != nil {
7475
out <- fmt.Errorf("error inserting testsuites: %w", err)
7576
return
7677
}
77-
if err = clickhouse.InsertTestCases(ctx, tcs, ch); err != nil {
78+
if err = ds.InsertTestCases(ctx, tcs); err != nil {
7879
out <- fmt.Errorf("error inserting testcases: %w", err)
7980
return
8081
}

‎pkg/worker/catchup.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@ import (
77
"sync"
88
"time"
99

10-
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/clickhouse"
1110
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/config"
1211
gitlab "github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/gitlab"
1312
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/tasks"
13+
14+
"github.com/cluttrdev/gitlab-clickhouse-exporter/internal/datastore"
1415
)
1516

1617
type catchUpProjectWorker struct {
@@ -22,18 +23,18 @@ type catchUpProjectWorker struct {
2223
// ensure the worker can only be stopped once
2324
stop sync.Once
2425

25-
project config.Project
26-
gitlab *gitlab.Client
27-
clickhouse *clickhouse.Client
26+
project config.Project
27+
gitlab *gitlab.Client
28+
datastore datastore.DataStore
2829
}
2930

30-
func NewCatchUpProjectWorker(cfg config.Project, gl *gitlab.Client, ch *clickhouse.Client) Worker {
31+
func NewCatchUpProjectWorker(cfg config.Project, gl *gitlab.Client, ds datastore.DataStore) Worker {
3132
return &catchUpProjectWorker{
3233
done: make(chan struct{}),
3334

34-
project: cfg,
35-
gitlab: gl,
36-
clickhouse: ch,
35+
project: cfg,
36+
gitlab: gl,
37+
datastore: ds,
3738
}
3839
}
3940

@@ -91,7 +92,7 @@ func (w *catchUpProjectWorker) produce(ctx context.Context, opt gitlab.ListProje
9192
go func() {
9293
defer close(ch)
9394

94-
latestUpdates, err := w.clickhouse.QueryProjectPipelinesLatestUpdate(ctx, w.project.Id)
95+
latestUpdates, err := w.datastore.QueryProjectPipelinesLatestUpdate(ctx, w.project.Id)
9596
if err != nil {
9697
if errors.Is(err, context.Canceled) {
9798
return
@@ -148,7 +149,7 @@ func (w *catchUpProjectWorker) process(ctx context.Context, pipelineChan <-chan
148149
ExportTraces: w.project.Export.Traces.Enabled,
149150
}
150151

151-
if err := tasks.ExportPipelineHierarchy(ctx, opts, w.gitlab, w.clickhouse); err != nil {
152+
if err := tasks.ExportPipelineHierarchy(ctx, opts, w.gitlab, w.datastore); err != nil {
152153
if !errors.Is(err, context.Canceled) {
153154
log.Printf("error exporting pipeline hierarchy: %s\n", err)
154155
}

‎pkg/worker/export.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ import (
66
"sync"
77
"time"
88

9-
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/clickhouse"
109
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/config"
1110
gitlab "github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/gitlab"
1211
"github.com/cluttrdev/gitlab-clickhouse-exporter/pkg/tasks"
12+
13+
"github.com/cluttrdev/gitlab-clickhouse-exporter/internal/datastore"
1314
)
1415

1516
type exportProjectWorker struct {
@@ -21,18 +22,18 @@ type exportProjectWorker struct {
2122
// ensure the worker can only be stopped once
2223
stop sync.Once
2324

24-
project config.Project
25-
gitlab *gitlab.Client
26-
clickhouse *clickhouse.Client
25+
project config.Project
26+
gitlab *gitlab.Client
27+
datastore datastore.DataStore
2728
}
2829

29-
func NewExportProjectWorker(cfg config.Project, gl *gitlab.Client, ch *clickhouse.Client) Worker {
30+
func NewExportProjectWorker(cfg config.Project, gl *gitlab.Client, ds datastore.DataStore) Worker {
3031
return &exportProjectWorker{
3132
done: make(chan struct{}),
3233

33-
project: cfg,
34-
gitlab: gl,
35-
clickhouse: ch,
34+
project: cfg,
35+
gitlab: gl,
36+
datastore: ds,
3637
}
3738
}
3839

@@ -106,7 +107,7 @@ func (w *exportProjectWorker) run(ctx context.Context) {
106107
ExportTraces: w.project.Export.Traces.Enabled,
107108
}
108109

109-
if err := tasks.ExportPipelineHierarchy(ctx, opts, w.gitlab, w.clickhouse); err != nil {
110+
if err := tasks.ExportPipelineHierarchy(ctx, opts, w.gitlab, w.datastore); err != nil {
110111
log.Printf("error exporting pipeline hierarchy: %s\n", err)
111112
} else {
112113
log.Printf("Exported projects/%d/pipelines/%d\n", opts.ProjectID, opts.PipelineID)

0 commit comments

Comments
 (0)