Add support for project namespaces configuration

Author: cluttrdev

cmd/run.go

@@ -110,20 +110,6 @@ func (c *RunConfig) Exec(ctx context.Context, _ []string) error {
 		cfg.Log.Level = "debug"
 	}
 
-	// add projects passed to run command
-	for _, pid := range c.projects {
-		exists := slices.ContainsFunc(cfg.Projects, func(p config.Project) bool {
-			return p.Id == pid
-		})
-
-		if !exists {
-			cfg.Projects = append(cfg.Projects, config.Project{
-				ProjectSettings: config.DefaultProjectSettings(),
-				Id:              pid,
-			})
-		}
-	}
-
 	if cfg.Log.Level == "debug" {
 		writeConfig(c.out, cfg)
 	}
@@ -147,6 +133,26 @@ func (c *RunConfig) Exec(ctx context.Context, _ []string) error {
 		return err
 	}
 
+	// gather projects from config
+	projects, err := resolveProjects(ctx, cfg, gitlabclient)
+	if err != nil {
+		return fmt.Errorf("error resolving projects: %w", err)
+	}
+
+	// add projects passed as arguments
+	for _, pid := range c.projects {
+		exists := slices.ContainsFunc(cfg.Projects, func(p config.Project) bool {
+			return p.Id == pid
+		})
+
+		if !exists {
+			projects = append(cfg.Projects, config.Project{
+				ProjectSettings: config.DefaultProjectSettings(),
+				Id:              pid,
+			})
+		}
+	}
+
 	g := &run.Group{}
 
 	pool := worker.NewWorkerPool(42)
@@ -166,12 +172,12 @@ func (c *RunConfig) Exec(ctx context.Context, _ []string) error {
 		})
 	}
 
-	if len(cfg.Projects) > 0 { // jobs
+	if len(projects) > 0 { // jobs
 		ctx, cancel := context.WithCancel(context.Background())
 
 		g.Add(func() error { // execute
 			var wg sync.WaitGroup
-			for _, p := range cfg.Projects {
+			for _, p := range projects {
 				if c.catchup && p.CatchUp.Enabled {
 					job := jobs.ProjectCatchUpJob{
 						Config:   p,
@@ -288,3 +294,39 @@ func serveHTTP(cfg config.HTTP, reg *prometheus.Registry) (func() error, func(er
 
 	return execute, interrupt
 }
+
+func resolveProjects(ctx context.Context, cfg config.Config, glab *gitlab.Client) ([]config.Project, error) {
+	pm := make(map[int64]config.Project)
+
+	opt := gitlab.ListNamespaceProjectsOptions{}
+	for _, namespace := range cfg.Namespaces {
+		opt.Kind = namespace.Kind
+		opt.Visibility = (*gitlab.VisibilityValue)(&namespace.Visibility)
+		opt.WithShared = namespace.WithShared
+		opt.IncludeSubgroups = namespace.IncludeSubgroups
+
+		ps, err := glab.ListNamespaceProjects(ctx, namespace.Id, opt)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, p := range ps {
+			pm[p.Id] = config.Project{
+				ProjectSettings: namespace.ProjectSettings,
+				Id:              p.Id,
+			}
+		}
+	}
+
+	// overwrite with explicitly configured projects
+	for _, p := range cfg.Projects {
+		pm[p.Id] = p
+	}
+
+	projects := make([]config.Project, 0, len(pm))
+	for _, p := range pm {
+		projects = append(projects, p)
+	}
+
+	return projects, nil
+}

configs/gitlab-exporter.yaml

@@ -61,6 +61,30 @@ projects: []
   #   export: {}
   #   catch_up: {}
 
+# List of namespaces of which to export projects
+namespaces: []
+  # - # The namespace id or url-encoded path
+  #   id: gitlab-exporter
+  #
+  #   # The namespace kind (user or group).
+  #   # Optional, will be determined if not specified.
+  #   kind: group
+  #
+  #   # Limit by visibility (public, internal or private)
+  #   visibility: ""
+  #
+  #   # Include projects shared to this group.
+  #   # (Only applicable for group namespaces)
+  #   with_shared: false
+  #
+  #   # Whether to include projects in subgroups of this namespace.
+  #   # (Only applicable for group namespaces)
+  #   include_subgroups: false
+  #
+  #   # See `project_defaults` for settings that can be overwritten here.
+  #   export: {}
+  #   catch_up: {}
+
 # HTTP server settings
 http:
   # Whether to enable serving http endpoint (metrics, debug info)

internal/config/config.go

@@ -14,6 +14,8 @@ type Config struct {
 	ProjectDefaults ProjectSettings `default:"{}" yaml:"project_defaults"`
 	// List of project to export
 	Projects []Project `default:"[]" yaml:"projects"`
+	// List of namespaces of which to export projects
+	Namespaces []Namespace `default:"[]" yaml:"namespaces"`
 	// HTTP server settings
 	HTTP HTTP `default:"{}" yaml:"http"`
 	// Log configuration settings
@@ -77,6 +79,17 @@ type ProjectCatchUp struct {
 	UpdatedBefore string `default:"" yaml:"updated_before"`
 }
 
+type Namespace struct {
+	ProjectSettings `default:"{}" yaml:",inline"`
+
+	Id   string `yaml:"id"`
+	Kind string `default:"" yaml:"kind"`
+
+	Visibility       string `default:"" yaml:"visibility"`
+	WithShared       bool   `default:"false" yaml:"with_shared"`
+	IncludeSubgroups bool   `default:"false" yaml:"include_subgroups"`
+}
+
 type HTTP struct {
 	Enabled bool   `default:"true" yaml:"enabled"`
 	Host    string `default:"127.0.0.1" yaml:"host"`

internal/config/parser.go

@@ -16,6 +16,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) error {
 		Endpoints       []Endpoint      `yaml:"endpoints"`
 		ProjectDefaults ProjectSettings `yaml:"project_defaults"`
 		Projects        []yaml.Node     `yaml:"projects"`
+		Namespaces      []yaml.Node     `yaml:"namespaces"`
 		HTTP            HTTP            `yaml:"http"`
 		Log             Log             `yaml:"log"`
 	}
@@ -43,12 +44,24 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) error {
 		}
 
 		if err := n.Decode(&p); err != nil {
-			return nil
+			return err
 		}
 
 		c.Projects = append(c.Projects, p)
 	}
 
+	for _, node := range _cfg.Namespaces {
+		n := Namespace{
+			ProjectSettings: c.ProjectDefaults,
+		}
+
+		if err := node.Decode(&n); err != nil {
+			return err
+		}
+
+		c.Namespaces = append(c.Namespaces, n)
+	}
+
 	return nil
 }
 

internal/gitlab/gitlab.go

@@ -1,6 +1,8 @@
 package gitlab
 
 import (
+	"fmt"
+	"strconv"
 	"time"
 
 	durationpb "google.golang.org/protobuf/types/known/durationpb"
@@ -11,6 +13,17 @@ func ptr[T any](v T) *T {
 	return &v
 }
 
+func parseID(id interface{}) (string, error) {
+	switch v := id.(type) {
+	case int:
+		return strconv.Itoa(v), nil
+	case string:
+		return v, nil
+	default:
+		return "", fmt.Errorf("invalid ID type %#v, the ID must be an int or a string", id)
+	}
+}
+
 func convertTime(t *time.Time) *timestamppb.Timestamp {
 	if t == nil {
 		return nil

internal/gitlab/projects.go

@@ -2,15 +2,19 @@ package gitlab
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	gitlab "github.com/xanzy/go-gitlab"
 
 	"github.com/cluttrdev/gitlab-exporter/protobuf/typespb"
 )
 
-type ListProjectOptions = gitlab.ListProjectsOptions
+type ListProjectsOptions = gitlab.ListProjectsOptions
+type ListGroupProjectsOptions = gitlab.ListGroupProjectsOptions
+type VisibilityValue = gitlab.VisibilityValue
 
-func (c *Client) ListProjects(ctx context.Context, opt ListProjectOptions) ([]*typespb.Project, error) {
+func (c *Client) ListProjects(ctx context.Context, opt ListProjectsOptions) ([]*typespb.Project, error) {
 	var projects []*typespb.Project
 
 	for {
@@ -37,14 +41,88 @@ func (c *Client) GetProject(ctx context.Context, id int64) (*typespb.Project, er
 		Statistics: ptr(true),
 	}
 
-	p, _, err := c.client.Projects.GetProject(int(id), &opt)
+	p, _, err := c.client.Projects.GetProject(int(id), &opt, gitlab.WithContext(ctx))
 	if err != nil {
 		return nil, err
 	}
 
 	return convertProject(p), nil
 }
 
+type ListNamespaceProjectsOptions struct {
+	gitlab.ListProjectsOptions
+
+	Kind             string
+	WithShared       bool
+	IncludeSubgroups bool
+}
+
+func (c *Client) ListNamespaceProjects(ctx context.Context, id interface{}, opt ListNamespaceProjectsOptions) ([]*typespb.Project, error) {
+	kind := strings.ToLower(opt.Kind)
+	if !(strings.EqualFold(kind, "user") || strings.EqualFold(kind, "group")) {
+		n, _, err := c.client.Namespaces.GetNamespace(id, gitlab.WithContext(ctx))
+		if err != nil {
+			return nil, fmt.Errorf("error determining namespace kind: %w", err)
+		}
+		kind = n.Kind
+	}
+
+	if kind == "user" {
+		return c.ListUserProjects(ctx, id, opt.ListProjectsOptions)
+	} else if kind == "group" {
+		return c.ListGroupProjects(ctx, id, gitlab.ListGroupProjectsOptions{
+			ListOptions:      opt.ListOptions,
+			WithShared:       &opt.WithShared,
+			IncludeSubGroups: &opt.IncludeSubgroups,
+		})
+	}
+	return nil, fmt.Errorf("invalid namespace kind: %v", kind)
+}
+
+func (c *Client) ListUserProjects(ctx context.Context, uid interface{}, opt ListProjectsOptions) ([]*typespb.Project, error) {
+	var projects []*typespb.Project
+
+	for {
+		ps, resp, err := c.client.Projects.ListUserProjects(uid, &opt, gitlab.WithContext(ctx))
+		if err != nil {
+			return projects, err
+		}
+
+		for _, p := range ps {
+			projects = append(projects, convertProject(p))
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	return projects, nil
+}
+
+func (c *Client) ListGroupProjects(ctx context.Context, gid interface{}, opt ListGroupProjectsOptions) ([]*typespb.Project, error) {
+	var projects []*typespb.Project
+
+	for {
+		ps, resp, err := c.client.Groups.ListGroupProjects(gid, &opt, gitlab.WithContext(ctx))
+		if err != nil {
+			return projects, err
+		}
+
+		for _, p := range ps {
+			projects = append(projects, convertProject(p))
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	return projects, nil
+}
+
 func convertProject(p *gitlab.Project) *typespb.Project {
 	return &typespb.Project{
 		Id:                int64(p.ID),