Skip to content

Commit

Permalink
Add exporter type and endpoint config
Browse files Browse the repository at this point in the history
  • Loading branch information
cluttrdev committed Jan 19, 2024
1 parent a7bb87a commit cf31800
Show file tree
Hide file tree
Showing 7 changed files with 230 additions and 2 deletions.
4 changes: 4 additions & 0 deletions configs/gitlab-exporter.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ clickhouse:
user: "default"
password: ""

# List of gRPC server endpoints to export to
endpoints: []
# - address: "127.0.0.1:36275"

# List of projects to export
projects: []
# - # Project ID project (required).
Expand Down
4 changes: 2 additions & 2 deletions grpc/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/cluttrdev/gitlab-exporter/pkg/models"
)

type ClientConfig struct {
type EndpointConfig struct {
Address string
Options []grpc.DialOption
}
Expand All @@ -27,7 +27,7 @@ type Client struct {
client pb.GitLabExporterClient
}

func NewCLient(cfg ClientConfig) (*Client, error) {
func NewCLient(cfg EndpointConfig) (*Client, error) {
conn, err := grpc.Dial(cfg.Address, cfg.Options...)
if err != nil {
return nil, err
Expand Down
5 changes: 5 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
type Config struct {
GitLab GitLab `default:"{}" yaml:"gitlab"`
ClickHouse ClickHouse `default:"{}" yaml:"clickhouse"`
Endpoints []Endpoint `default:"[]" yaml:"endpoints"`
Projects []Project `default:"[]" yaml:"projects"`
Server Server `default:"{}" yaml:"server"`
}
Expand All @@ -32,6 +33,10 @@ type ClickHouse struct {
Password string `default:"" yaml:"password"`
}

type Endpoint struct {
Address string `default:"" yaml:"address"`
}

type Project struct {
ProjectSettings `default:"{}" yaml:",inline"`

Expand Down
4 changes: 4 additions & 0 deletions pkg/config/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) error {
GitLab GitLab `yaml:"gitlab"`
ClickHouse ClickHouse `yaml:"clickhouse"`

Endpoints []Endpoint `yaml:"endpoints"`

Projects []yaml.Node `yaml:"projects"`

Server Server `yaml:"server"`
Expand All @@ -23,6 +25,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) error {
var _cfg _Config
_cfg.GitLab = c.GitLab
_cfg.ClickHouse = c.ClickHouse
_cfg.Endpoints = c.Endpoints
_cfg.Server = c.Server

if err := v.Decode(&_cfg); err != nil {
Expand All @@ -31,6 +34,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) error {

c.GitLab = _cfg.GitLab
c.ClickHouse = _cfg.ClickHouse
c.Endpoints = _cfg.Endpoints
c.Server = _cfg.Server

for _, n := range _cfg.Projects {
Expand Down
28 changes: 28 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ import (
"math/rand"
"time"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"

grpc_client "github.com/cluttrdev/gitlab-exporter/grpc/client"

"github.com/cluttrdev/gitlab-exporter/pkg/clickhouse"
"github.com/cluttrdev/gitlab-exporter/pkg/config"
"github.com/cluttrdev/gitlab-exporter/pkg/gitlab"
Expand All @@ -20,6 +25,7 @@ import (
type Controller struct {
config config.Config
GitLab gitlab.Client
Exporter *Exporter
DataStore datastore.DataStore

workers []worker.Worker
Expand All @@ -46,6 +52,10 @@ func (c *Controller) configure(cfg config.Config) error {
return err
}

if err := c.configureExporter(cfg.Endpoints); err != nil {
return err
}

if err := c.configureWorkers(cfg); err != nil {
return err
}
Expand Down Expand Up @@ -80,6 +90,24 @@ func (c *Controller) configureClickHouseDataStore(cfg config.ClickHouse) error {
return nil
}

func (c *Controller) configureExporter(cfg []config.Endpoint) error {
endpoints := make([]grpc_client.EndpointConfig, 0, len(cfg))
for _, cc := range cfg {
endpoints = append(endpoints, grpc_client.EndpointConfig{
Address: cc.Address,
Options: []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
},
})
}
exp, err := NewExporter(endpoints)
if err != nil {
return err
}
c.Exporter = exp
return nil
}

func (c *Controller) configureWorkers(cfg config.Config) error {
workers := []worker.Worker{}

Expand Down
166 changes: 166 additions & 0 deletions pkg/controller/exporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
package controller

import (
"context"
"errors"

grpc_client "github.com/cluttrdev/gitlab-exporter/grpc/client"
"github.com/cluttrdev/gitlab-exporter/pkg/models"
)

type Exporter struct {
clients []*grpc_client.Client
}

func NewExporter(endpoints []grpc_client.EndpointConfig) (*Exporter, error) {
var clients []*grpc_client.Client
for _, cfg := range endpoints {
c, err := grpc_client.NewCLient(cfg)
if err != nil {
return nil, err
}
clients = append(clients, c)
}

return &Exporter{
clients: clients,
}, nil
}

func (e *Exporter) RecordPipelines(ctx context.Context, data []*models.Pipeline) error {
var errs error
for _, client := range e.clients {
if err := client.RecordPipelines(ctx, data); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordJobs(ctx context.Context, data []*models.Job) error {
var errs error
for _, client := range e.clients {
if err := client.RecordJobs(ctx, data); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordSections(ctx context.Context, data []*models.Section) error {
var errs error
for _, client := range e.clients {
if err := client.RecordSections(ctx, data); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordBridges(ctx context.Context, data []*models.Bridge) error {
var errs error
for _, client := range e.clients {
if err := client.RecordBridges(ctx, data); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordTestReports(ctx context.Context, reports []*models.PipelineTestReport) error {
var errs error
for _, client := range e.clients {
if err := client.RecordTestReports(ctx, reports); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordTestSuites(ctx context.Context, suites []*models.PipelineTestSuite) error {
var errs error
for _, client := range e.clients {
if err := client.RecordTestSuites(ctx, suites); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordTestCases(ctx context.Context, cases []*models.PipelineTestCase) error {
var errs error
for _, client := range e.clients {
if err := client.RecordTestCases(ctx, cases); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordTraces(ctx context.Context, traces []*models.Trace) error {
var errs error
for _, client := range e.clients {
if err := client.RecordTraces(ctx, traces); err != nil {
errs = errors.Join(errs, err)
}
}
return errs
}

func (e *Exporter) RecordPipelineHierarchy(ctx context.Context, ph *models.PipelineHierarchy) error {
data, err := flattenPipelineHierarchy(ph)
if err != nil {
return err
}
var errs error
for _, client := range e.clients {
if err := client.RecordPipelines(ctx, data.Pipelines); err != nil {
errs = errors.Join(errs, err)
continue
}
if err := client.RecordJobs(ctx, data.Jobs); err != nil {
errs = errors.Join(errs, err)
continue
}
if err := client.RecordSections(ctx, data.Sections); err != nil {
errs = errors.Join(errs, err)
continue
}
if err := client.RecordBridges(ctx, data.Bridges); err != nil {
errs = errors.Join(errs, err)
continue
}
}

return errs
}

type pipelineData struct {
Pipelines []*models.Pipeline
Jobs []*models.Job
Sections []*models.Section
Bridges []*models.Bridge
}

func flattenPipelineHierarchy(ph *models.PipelineHierarchy) (pipelineData, error) {
var data pipelineData

data.Pipelines = append(data.Pipelines, ph.Pipeline)
data.Jobs = append(data.Jobs, ph.Jobs...)
data.Sections = append(data.Sections, ph.Sections...)
data.Bridges = append(data.Bridges, ph.Bridges...)

for _, dph := range ph.DownstreamPipelines {
d, err := flattenPipelineHierarchy(dph)
if err != nil {
return data, err
}

data.Pipelines = append(data.Pipelines, d.Pipelines...)
data.Jobs = append(data.Jobs, d.Jobs...)
data.Sections = append(data.Sections, d.Sections...)
data.Bridges = append(data.Bridges, d.Bridges...)
}

return data, nil
}
21 changes: 21 additions & 0 deletions test/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ func defaultConfig() config.Config {
cfg.ClickHouse.User = "default"
cfg.ClickHouse.Password = ""

cfg.Endpoints = []config.Endpoint{}

cfg.Projects = []config.Project{}

cfg.Server.Host = "127.0.0.1"
Expand Down Expand Up @@ -263,3 +265,22 @@ func TestLoad_DataWithCustomServerAddress(t *testing.T) {

checkConfig(t, expected, cfg)
}

func TestLoad_DataWithEndpoints(t *testing.T) {
data := []byte(`
endpoints:
- address: "127.0.0.1:36275"
`)

expected := defaultConfig()
expected.Endpoints = []config.Endpoint{
{Address: "127.0.0.1:36275"},
}

cfg := config.Default()
if err := config.Load(data, &cfg); err != nil {
t.Errorf("Expected no error, got %v", err)
}

checkConfig(t, expected, cfg)
}

0 comments on commit cf31800

Please sign in to comment.