Add support for exporting project data

grpc/client/client.go

@@ -49,6 +49,18 @@ func (c *Client) MetricsCollector() prometheus.Collector {
 	return c.metrics
 }
 
+func RecordProjects(c *Client, ctx context.Context, data []*typespb.Project) error {
+	req := &servicepb.RecordProjectsRequest{
+		Data: data,
+	}
+	_, err := c.stub.RecordProjects(ctx, req /* opts ...grpc.CallOption */)
+	if err != nil {
+		return fmt.Errorf("error recording projects: %w", err)
+	}
+
+	return nil
+}
+
 func RecordPipelines(c *Client, ctx context.Context, data []*typespb.Pipeline) error {
 	req := &servicepb.RecordPipelinesRequest{
 		Data: data,

internal/exporter/exporter.go

@@ -59,6 +59,10 @@ func export[T any](exporter *Exporter, ctx context.Context, data []*T, record re
 	return errs
 }
 
+func (e *Exporter) ExportProjects(ctx context.Context, data []*typespb.Project) error {
+	return export[typespb.Project](e, ctx, data, grpc_client.RecordProjects)
+}
+
 func (e *Exporter) ExportPipelines(ctx context.Context, data []*typespb.Pipeline) error {
 	return export[typespb.Pipeline](e, ctx, data, grpc_client.RecordPipelines)
 }

internal/gitlab/gitlab.go

@@ -7,6 +7,10 @@ import (
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 )
 
+func ptr[T any](v T) *T {
+	return &v
+}
+
 func convertTime(t *time.Time) *timestamppb.Timestamp {
 	if t == nil {
 		return nil

internal/gitlab/projects.go

@@ -0,0 +1,99 @@
+package gitlab
+
+import (
+	"context"
+
+	gitlab "github.com/xanzy/go-gitlab"
+
+	"github.com/cluttrdev/gitlab-exporter/protobuf/typespb"
+)
+
+type ListProjectOptions = gitlab.ListProjectsOptions
+
+func (c *Client) ListProjects(ctx context.Context, opt ListProjectOptions) ([]*typespb.Project, error) {
+	var projects []*typespb.Project
+
+	for {
+		ps, resp, err := c.client.Projects.ListProjects(&opt, gitlab.WithContext(ctx))
+		if err != nil {
+			return projects, err
+		}
+
+		for _, p := range ps {
+			projects = append(projects, convertProject(p))
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	return projects, nil
+}
+
+func (c *Client) GetProject(ctx context.Context, id int64) (*typespb.Project, error) {
+	opt := gitlab.GetProjectOptions{
+		Statistics: ptr(true),
+	}
+
+	p, _, err := c.client.Projects.GetProject(int(id), &opt)
+	if err != nil {
+		return nil, err
+	}
+
+	return convertProject(p), nil
+}
+
+func convertProject(p *gitlab.Project) *typespb.Project {
+	return &typespb.Project{
+		Id:                int64(p.ID),
+		Name:              p.Name,
+		NameWithNamespace: p.NameWithNamespace,
+		Path:              p.Path,
+		PathWithNamespace: p.PathWithNamespace,
+		Description:       p.Description,
+
+		CreatedAt:      convertTime(p.CreatedAt),
+		LastActivityAt: convertTime(p.LastActivityAt),
+
+		DefaultBranch: p.DefaultBranch,
+		WebUrl:        p.WebURL,
+
+		Namespace:  convertProjectNamespace(p.Namespace),
+		Statistics: convertProjectStatistics(p),
+	}
+}
+
+func convertProjectNamespace(n *gitlab.ProjectNamespace) *typespb.ProjectNamespace {
+	return &typespb.ProjectNamespace{
+		Id:       int64(n.ID),
+		Name:     n.Name,
+		Kind:     n.Kind,
+		Path:     n.Path,
+		FullPath: n.FullPath,
+		ParentId: int64(n.ParentID),
+
+		AvatarUrl: n.AvatarURL,
+		WebUrl:    n.WebURL,
+	}
+}
+
+func convertProjectStatistics(p *gitlab.Project) *typespb.ProjectStatistics {
+	s := &typespb.ProjectStatistics{
+		ForksCount: int64(p.ForksCount),
+		StarsCount: int64(p.StarCount),
+	}
+	if p.Statistics != nil {
+		s.CommitCount = p.Statistics.CommitCount
+		s.StorageSize = p.Statistics.StorageSize
+		s.RepositorySize = p.Statistics.RepositorySize
+		s.WikiSize = p.Statistics.WikiSize
+		s.LfsObjectsSize = p.Statistics.LFSObjectsSize
+		s.JobArtifactsSize = p.Statistics.JobArtifactsSize
+		s.PackagesSize = p.Statistics.PackagesSize
+		s.SnippetsSize = p.Statistics.SnippetsSize
+		s.UploadsSize = p.Statistics.UploadsSize
+	}
+	return s
+}

internal/jobs/export.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cluttrdev/gitlab-exporter/internal/gitlab"
 	"github.com/cluttrdev/gitlab-exporter/internal/tasks"
 	"github.com/cluttrdev/gitlab-exporter/pkg/worker"
+	"github.com/cluttrdev/gitlab-exporter/protobuf/typespb"
 )
 
 type ProjectExportJob struct {
@@ -20,74 +21,97 @@ type ProjectExportJob struct {
 	Exporter *exporter.Exporter
 
 	WorkerPool *worker.Pool
+
+	lastUpdate time.Time
 }
 
 func (j *ProjectExportJob) Run(ctx context.Context) {
 	period := 1 * time.Minute
 
-	opt := gitlab.ListProjectPipelinesOptions{
-		ListOptions: gitlab.ListOptions{
-			PerPage: 100,
-			Page:    1,
-		},
-
-		Scope: &[]string{"finished"}[0],
-	}
+	j.lastUpdate = time.Now().UTC()
 
-	now := time.Now().UTC()
-	opt.UpdatedBefore = &now
+	projectID := j.Config.Id
 
 	ticker := time.NewTicker(period)
 	var iteration int = 0
+	var wg sync.WaitGroup
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
 			iteration++
-			slog.Debug("Exporting updated pipelines", "project", j.Config.Id, "iteration", iteration)
-
-			now := time.Now().UTC()
-			opt.UpdatedAfter = opt.UpdatedBefore
-			opt.UpdatedBefore = &now
-
-			pipelines := j.GitLab.ListProjectPipelines(ctx, j.Config.Id, opt)
-			var wg sync.WaitGroup
-			for r := range pipelines {
-				if r.Error != nil {
-					if errors.Is(r.Error, context.Canceled) {
-						return
-					} else {
-						slog.Error("error listing project pipelines", "error", r.Error)
-					}
-					continue
-				}
-
-				pipelineID := r.Pipeline.Id
-				wg.Add(1)
-				j.WorkerPool.Submit(func(ctx context.Context) {
-					defer wg.Done()
-					if err := j.export(ctx, pipelineID); err != nil {
-						slog.Error(err.Error())
-					}
-				})
-			}
+			slog.Debug("Exporting project", "id", projectID, "iteration", iteration)
+
+			wg.Add(1)
+			j.WorkerPool.Submit(func(ctx context.Context) {
+				defer wg.Done()
+				j.exportProject(ctx)
+			})
+
+			j.exportProjectPipelines(ctx, &wg)
 
 			wg.Wait()
+			j.lastUpdate = time.Now().UTC()
 		}
 	}
 }
 
-func (j *ProjectExportJob) export(ctx context.Context, pipelineID int64) error {
-	options := tasks.ExportPipelineHierarchyOptions{
-		ProjectID:  j.Config.Id,
-		PipelineID: pipelineID,
+func (j *ProjectExportJob) exportProject(ctx context.Context) {
+	projectID := j.Config.Id
+
+	project, err := j.GitLab.GetProject(ctx, projectID)
+	if err != nil {
+		slog.Error("error fetching project", "project", projectID, "error", err)
+		return
+	} else if !project.LastActivityAt.AsTime().After(j.lastUpdate) {
+		return
+	}
+
+	if err := j.Exporter.ExportProjects(ctx, []*typespb.Project{project}); err != nil {
+		slog.Error(err.Error())
+	}
+}
+
+func (j *ProjectExportJob) exportProjectPipelines(ctx context.Context, wg *sync.WaitGroup) {
+	projectID := j.Config.Id
+
+	opt := gitlab.ListProjectPipelinesOptions{
+		ListOptions: gitlab.ListOptions{
+			PerPage: 100,
+			Page:    1,
+		},
 
-		ExportSections:    j.Config.Export.Sections.Enabled,
-		ExportTestReports: j.Config.Export.TestReports.Enabled,
-		ExportTraces:      j.Config.Export.Traces.Enabled,
-		ExportMetrics:     j.Config.Export.Metrics.Enabled,
+		Scope:        &[]string{"finished"}[0],
+		UpdatedAfter: &j.lastUpdate,
 	}
 
-	return tasks.ExportPipelineHierarchy(ctx, j.GitLab, j.Exporter, options)
+	pipelines := j.GitLab.ListProjectPipelines(ctx, projectID, opt)
+	for r := range pipelines {
+		if r.Error != nil {
+			if errors.Is(r.Error, context.Canceled) {
+				return
+			}
+			slog.Error("error listing project pipelines", "error", r.Error)
+			continue
+		}
+
+		pipelineID := r.Pipeline.Id
+		wg.Add(1)
+		j.WorkerPool.Submit(func(ctx context.Context) {
+			defer wg.Done()
+			err := tasks.ExportPipelineHierarchy(ctx, j.GitLab, j.Exporter, tasks.ExportPipelineHierarchyOptions{
+				ProjectID:  projectID,
+				PipelineID: pipelineID,
+
+				ExportSections:    j.Config.Export.Sections.Enabled,
+				ExportTestReports: j.Config.Export.TestReports.Enabled,
+				ExportTraces:      j.Config.Export.Traces.Enabled,
+				ExportMetrics:     j.Config.Export.Metrics.Enabled,
+			})
+			if err != nil {
+				slog.Error(err.Error())
+			}
+		})
+	}
 }

internal/tasks/projects.go

@@ -0,0 +1,26 @@
+package tasks
+
+import (
+	"context"
+
+	"github.com/cluttrdev/gitlab-exporter/internal/exporter"
+	"github.com/cluttrdev/gitlab-exporter/internal/gitlab"
+	"github.com/cluttrdev/gitlab-exporter/protobuf/typespb"
+)
+
+type ExportProjectOptions struct {
+	ProjectID int64
+}
+
+func ExportProject(ctx context.Context, glab *gitlab.Client, exp *exporter.Exporter, opt ExportProjectOptions) error {
+	p, err := glab.GetProject(ctx, opt.ProjectID)
+	if err != nil {
+		return err
+	}
+
+	if err := exp.ExportProjects(ctx, []*typespb.Project{p}); err != nil {
+		return err
+	}
+
+	return nil
+}