diff --git a/internal/gitlab/client.go b/internal/gitlab/client.go index 542f153..901497a 100644 --- a/internal/gitlab/client.go +++ b/internal/gitlab/client.go @@ -10,6 +10,7 @@ import ( gitlab "github.com/xanzy/go-gitlab" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -156,9 +157,9 @@ func (c *Client) GetPipelineHierarchy(ctx context.Context, projectID int64, pipe }, Pipeline: job.Pipeline, Name: secdat.Name, - StartedAt: convertUnixSeconds(secdat.Start), - FinishedAt: convertUnixSeconds(secdat.End), - Duration: convertDuration(float64(secdat.End - secdat.Start)), + StartedAt: types.ConvertUnixSeconds(secdat.Start), + FinishedAt: types.ConvertUnixSeconds(secdat.End), + Duration: types.ConvertDuration(float64(secdat.End - secdat.Start)), } sections = append(sections, section) @@ -176,7 +177,7 @@ func (c *Client) GetPipelineHierarchy(ctx context.Context, projectID int64, pipe Name: m.Name, Labels: convertLabels(m.Labels), Value: m.Value, - Timestamp: convertUnixMilli(m.Timestamp), + Timestamp: types.ConvertUnixMilli(m.Timestamp), } metrics = append(metrics, metric) } diff --git a/internal/gitlab/gitlab.go b/internal/gitlab/gitlab.go index 15ef39a..418895e 100644 --- a/internal/gitlab/gitlab.go +++ b/internal/gitlab/gitlab.go @@ -3,10 +3,6 @@ package gitlab import ( "fmt" "strconv" - "time" - - durationpb "google.golang.org/protobuf/types/known/durationpb" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) func Ptr[T any](v T) *T { @@ -23,38 +19,3 @@ func parseID(id interface{}) (string, error) { return "", fmt.Errorf("invalid ID type %#v, the ID must be an int or a string", id) } } - -func convertTime(t *time.Time) *timestamppb.Timestamp { - if t == nil { - return nil - } - return timestamppb.New(*t) -} - -func convertUnixSeconds(ts int64) *timestamppb.Timestamp { - return ×tamppb.Timestamp{ - Seconds: ts, - Nanos: 0, - } -} - -func convertUnixMilli(ts int64) *timestamppb.Timestamp { - const msPerSecond int64 = 1_000 - const nsPerMilli int64 = 1_000 - return ×tamppb.Timestamp{ - Seconds: ts / msPerSecond, - Nanos: int32((ts % msPerSecond) * nsPerMilli), - } -} - -func convertUnixNano(ts int64) *timestamppb.Timestamp { - const nsPerSecond int64 = 1_000_000_000 - return ×tamppb.Timestamp{ - Seconds: ts / nsPerSecond, - Nanos: int32(ts % nsPerSecond), - } -} - -func convertDuration(d float64) *durationpb.Duration { - return durationpb.New(time.Duration(d * float64(time.Second))) -} diff --git a/internal/gitlab/jobs.go b/internal/gitlab/jobs.go index 6588b1b..b391beb 100644 --- a/internal/gitlab/jobs.go +++ b/internal/gitlab/jobs.go @@ -4,8 +4,8 @@ import ( "context" gitlab "github.com/xanzy/go-gitlab" - "google.golang.org/protobuf/types/known/timestamppb" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -41,7 +41,7 @@ func (c *Client) ListPipelineJobs(ctx context.Context, projectID int64, pipeline for _, j := range jobs { ch <- ListPipelineJobsResult{ - Job: convertJob(j), + Job: types.ConvertJob(j), } } @@ -87,7 +87,7 @@ func (c *Client) ListPipelineBridges(ctx context.Context, projectID int64, pipel for _, b := range bridges { ch <- ListPipelineBridgesResult{ - Bridge: convertBridge(b), + Bridge: types.ConvertBridge(b), } } @@ -101,131 +101,3 @@ func (c *Client) ListPipelineBridges(ctx context.Context, projectID int64, pipel return ch } - -func convertJob(job *gitlab.Job) *typespb.Job { - artifacts := make([]*typespb.JobArtifacts, 0, len(job.Artifacts)) - for _, a := range job.Artifacts { - artifacts = append(artifacts, &typespb.JobArtifacts{ - Filename: a.Filename, - FileType: a.FileType, - FileFormat: a.FileFormat, - Size: int64(a.Size), - }) - } - - return &typespb.Job{ - Id: int64(job.ID), - Name: job.Name, - Pipeline: &typespb.PipelineReference{ - Id: int64(job.Pipeline.ID), - ProjectId: int64(job.Pipeline.ProjectID), - Ref: job.Pipeline.Ref, - Sha: job.Pipeline.Sha, - Status: job.Pipeline.Status, - }, - Ref: job.Ref, - CreatedAt: convertTime(job.CreatedAt), - StartedAt: convertTime(job.StartedAt), - FinishedAt: convertTime(job.FinishedAt), - ErasedAt: convertTime(job.ErasedAt), - Duration: convertDuration(job.Duration), - QueuedDuration: convertDuration(job.QueuedDuration), - Coverage: job.Coverage, - Stage: job.Stage, - Status: job.Status, - AllowFailure: job.AllowFailure, - FailureReason: job.FailureReason, - Tag: job.Tag, - WebUrl: job.WebURL, - TagList: job.TagList, - - Commit: convertCommit(job.Commit), - Project: convertProject(job.Project), - User: convertUser(job.User), - - Runner: &typespb.JobRunner{ - Id: int64(job.Runner.ID), - Name: job.Runner.Name, - Description: job.Runner.Description, - Active: job.Runner.Active, - IsShared: job.Runner.IsShared, - }, - - Artifacts: artifacts, - ArtifactsFile: &typespb.JobArtifactsFile{ - Filename: job.ArtifactsFile.Filename, - Size: int64(job.ArtifactsFile.Size), - }, - ArtifactsExpireAt: convertTime(job.ArtifactsExpireAt), - } -} - -func convertCommit(commit *gitlab.Commit) *typespb.Commit { - var status string - if commit.Status != nil { - status = string(*commit.Status) - } - return &typespb.Commit{ - Id: commit.ID, - ShortId: commit.ShortID, - ParentIds: commit.ParentIDs, - ProjectId: int64(commit.ProjectID), - AuthorName: commit.AuthorName, - AuthorEmail: commit.AuthorEmail, - AuthoredDate: convertTime(commit.AuthoredDate), - CommitterName: commit.CommitterName, - CommitterEmail: commit.CommitterEmail, - CommittedDate: convertTime(commit.CommittedDate), - CreatedAt: convertTime(commit.CreatedAt), - Title: commit.Title, - Message: commit.Message, - Trailers: commit.Trailers, - Stats: convertCommitStats(commit.Stats), - Status: status, - WebUrl: commit.WebURL, - } -} - -func convertCommitStats(stats *gitlab.CommitStats) *typespb.CommitStats { - if stats == nil { - return nil - } - return &typespb.CommitStats{ - Additions: int64(stats.Additions), - Deletions: int64(stats.Deletions), - Total: int64(stats.Total), - } -} - -func convertBridge(bridge *gitlab.Bridge) *typespb.Bridge { - // account for downstream pipeline creation failures - downstreamPipeline := &typespb.PipelineInfo{ - CreatedAt: ×tamppb.Timestamp{}, - UpdatedAt: ×tamppb.Timestamp{}, - } - if bridge.DownstreamPipeline != nil { - downstreamPipeline = convertPipelineInfo(bridge.DownstreamPipeline) - } - return &typespb.Bridge{ - // Commit: ConvertCommit(bridge.Commit), - Id: int64(bridge.ID), - Name: bridge.Name, - Pipeline: convertPipelineInfo(&bridge.Pipeline), - Ref: bridge.Ref, - CreatedAt: convertTime(bridge.CreatedAt), - StartedAt: convertTime(bridge.StartedAt), - FinishedAt: convertTime(bridge.FinishedAt), - ErasedAt: convertTime(bridge.ErasedAt), - Duration: convertDuration(bridge.Duration), - QueuedDuration: convertDuration(bridge.QueuedDuration), - Coverage: bridge.Coverage, - Stage: bridge.Stage, - Status: bridge.Status, - AllowFailure: bridge.AllowFailure, - FailureReason: bridge.FailureReason, - Tag: bridge.Tag, - WebUrl: bridge.WebURL, - // User: ConvertUser(bridge.User), - DownstreamPipeline: downstreamPipeline, - } -} diff --git a/internal/gitlab/pipelines.go b/internal/gitlab/pipelines.go index d6be845..1a1a684 100644 --- a/internal/gitlab/pipelines.go +++ b/internal/gitlab/pipelines.go @@ -3,10 +3,10 @@ package gitlab import ( "context" "fmt" - "strconv" gitlab "github.com/xanzy/go-gitlab" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -37,7 +37,7 @@ func (c *Client) ListProjectPipelines(ctx context.Context, projectID int64, opt for _, pi := range ps { out <- ListProjectPipelinesResult{ - Pipeline: convertPipelineInfo(pi), + Pipeline: types.ConvertPipelineInfo(pi), } } @@ -58,56 +58,5 @@ func (c *Client) GetPipeline(ctx context.Context, projectID int64, pipelineID in if err != nil { return nil, fmt.Errorf("error getting pipeline: %w", err) } - return convertPipeline(pipeline), nil -} - -func convertPipelineInfo(pipeline *gitlab.PipelineInfo) *typespb.PipelineInfo { - if pipeline == nil { - return nil - } - return &typespb.PipelineInfo{ - Id: int64(pipeline.ID), - Iid: int64(pipeline.IID), - ProjectId: int64(pipeline.ProjectID), - Status: pipeline.Status, - Source: pipeline.Source, - Ref: pipeline.Ref, - Sha: pipeline.SHA, - WebUrl: pipeline.WebURL, - CreatedAt: convertTime(pipeline.CreatedAt), - UpdatedAt: convertTime(pipeline.UpdatedAt), - } -} - -func convertPipeline(pipeline *gitlab.Pipeline) *typespb.Pipeline { - return &typespb.Pipeline{ - Id: int64(pipeline.ID), - Iid: int64(pipeline.IID), - ProjectId: int64(pipeline.ProjectID), - Status: pipeline.Status, - Source: pipeline.Source, - Ref: pipeline.Ref, - Sha: pipeline.SHA, - BeforeSha: pipeline.BeforeSHA, - Tag: pipeline.Tag, - YamlErrors: pipeline.YamlErrors, - CreatedAt: convertTime(pipeline.CreatedAt), - UpdatedAt: convertTime(pipeline.UpdatedAt), - StartedAt: convertTime(pipeline.StartedAt), - FinishedAt: convertTime(pipeline.FinishedAt), - CommittedAt: convertTime(pipeline.CommittedAt), - Duration: convertDuration(float64(pipeline.Duration)), - QueuedDuration: convertDuration(float64(pipeline.QueuedDuration)), - Coverage: convertCoverage(pipeline.Coverage), - WebUrl: pipeline.WebURL, - User: convertBasicUser(pipeline.User), - } -} - -func convertCoverage(coverage string) float64 { - cov, err := strconv.ParseFloat(coverage, 64) - if err != nil { - cov = 0.0 - } - return cov + return types.ConvertPipeline(pipeline), nil } diff --git a/internal/gitlab/projects.go b/internal/gitlab/projects.go index 16a2fce..c3828ff 100644 --- a/internal/gitlab/projects.go +++ b/internal/gitlab/projects.go @@ -7,6 +7,7 @@ import ( gitlab "github.com/xanzy/go-gitlab" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -24,7 +25,7 @@ func (c *Client) ListProjects(ctx context.Context, opt ListProjectsOptions) ([]* } for _, p := range ps { - projects = append(projects, convertProject(p)) + projects = append(projects, types.ConvertProject(p)) } if resp.NextPage == 0 { @@ -46,7 +47,7 @@ func (c *Client) GetProject(ctx context.Context, id int64) (*typespb.Project, er return nil, err } - return convertProject(p), nil + return types.ConvertProject(p), nil } type ListNamespaceProjectsOptions struct { @@ -89,7 +90,7 @@ func (c *Client) ListUserProjects(ctx context.Context, uid interface{}, opt List } for _, p := range ps { - projects = append(projects, convertProject(p)) + projects = append(projects, types.ConvertProject(p)) } if resp.NextPage == 0 { @@ -111,7 +112,7 @@ func (c *Client) ListGroupProjects(ctx context.Context, gid interface{}, opt Lis } for _, p := range ps { - projects = append(projects, convertProject(p)) + projects = append(projects, types.ConvertProject(p)) } if resp.NextPage == 0 { @@ -122,69 +123,3 @@ func (c *Client) ListGroupProjects(ctx context.Context, gid interface{}, opt Lis return projects, nil } - -func convertProject(p *gitlab.Project) *typespb.Project { - return &typespb.Project{ - Id: int64(p.ID), - Name: p.Name, - NameWithNamespace: p.NameWithNamespace, - Path: p.Path, - PathWithNamespace: p.PathWithNamespace, - - CreatedAt: convertTime(p.CreatedAt), - LastActivityAt: convertTime(p.LastActivityAt), - - Namespace: convertProjectNamespace(p.Namespace), - Owner: convertUser(p.Owner), - CreatorId: int64(p.CreatorID), - - Topics: p.Topics, - ForksCount: int64(p.ForksCount), - StarsCount: int64(p.StarCount), - Statistics: convertProjectStatistics(p.Statistics), - OpenIssuesCount: int64(p.OpenIssuesCount), - - Description: p.Description, - - EmptyRepo: p.EmptyRepo, - Archived: p.Archived, - - DefaultBranch: p.DefaultBranch, - Visibility: string(p.Visibility), - WebUrl: p.WebURL, - } -} - -func convertProjectNamespace(n *gitlab.ProjectNamespace) *typespb.ProjectNamespace { - if n == nil { - return nil - } - return &typespb.ProjectNamespace{ - Id: int64(n.ID), - Name: n.Name, - Kind: n.Kind, - Path: n.Path, - FullPath: n.FullPath, - ParentId: int64(n.ParentID), - - AvatarUrl: n.AvatarURL, - WebUrl: n.WebURL, - } -} - -func convertProjectStatistics(stats *gitlab.Statistics) *typespb.ProjectStatistics { - if stats == nil { - return nil - } - return &typespb.ProjectStatistics{ - CommitCount: stats.CommitCount, - StorageSize: stats.StorageSize, - RepositorySize: stats.RepositorySize, - WikiSize: stats.WikiSize, - LfsObjectsSize: stats.LFSObjectsSize, - JobArtifactsSize: stats.JobArtifactsSize, - PackagesSize: stats.PackagesSize, - SnippetsSize: stats.SnippetsSize, - UploadsSize: stats.UploadsSize, - } -} diff --git a/internal/gitlab/reports.go b/internal/gitlab/reports.go index 4df5efd..45cb9bf 100644 --- a/internal/gitlab/reports.go +++ b/internal/gitlab/reports.go @@ -8,6 +8,7 @@ import ( gitlab "github.com/xanzy/go-gitlab" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -29,7 +30,7 @@ func (c *Client) GetPipelineTestReport(ctx context.Context, projectID int64, pip return nil, fmt.Errorf("error getting pipeline test report summary: %w", err) } - testreport, testsuites, testcases := convertTestReport(pipelineID, report) + testreport, testsuites, testcases := types.ConvertTestReport(pipelineID, report) if err := overrideIDs(pipelineID, summary, testreport, testsuites, testcases); err != nil { return nil, fmt.Errorf("error setting test report ids: %w", err) @@ -128,77 +129,3 @@ func overrideIDs(pipelineID int64, summary *PipelineTestReportSummary, report *t return nil } - -func convertTestReport(pipelineID int64, report *gitlab.PipelineTestReport) (*typespb.TestReport, []*typespb.TestSuite, []*typespb.TestCase) { - testreport := &typespb.TestReport{ - Id: testReportID(pipelineID), - PipelineId: pipelineID, - TotalTime: report.TotalTime, - TotalCount: int64(report.TotalCount), - SuccessCount: int64(report.SuccessCount), - FailedCount: int64(report.FailedCount), - SkippedCount: int64(report.SkippedCount), - ErrorCount: int64(report.ErrorCount), - } - - testsuites := make([]*typespb.TestSuite, 0, len(report.TestSuites)) - testcases := []*typespb.TestCase{} - for i, testsuite := range report.TestSuites { - testsuiteID := testSuiteID(testreport.Id, i) - testsuites = append(testsuites, &typespb.TestSuite{ - Id: testsuiteID, - TestreportId: testreport.Id, - PipelineId: pipelineID, - Name: testsuite.Name, - TotalTime: testsuite.TotalTime, - TotalCount: int64(testsuite.TotalCount), - SuccessCount: int64(testsuite.SuccessCount), - FailedCount: int64(testsuite.FailedCount), - SkippedCount: int64(testsuite.SkippedCount), - ErrorCount: int64(testsuite.ErrorCount), - }) - - cases := make([]*typespb.TestCase, 0, len(testsuite.TestCases)) - for j, testcase := range testsuite.TestCases { - cases = append(cases, &typespb.TestCase{ - Id: testCaseID(testsuiteID, j), - TestsuiteId: testsuiteID, - TestreportId: testreport.Id, - PipelineId: pipelineID, - Status: testcase.Status, - Name: testcase.Name, - Classname: testcase.Classname, - File: testcase.File, - ExecutionTime: testcase.ExecutionTime, - SystemOutput: fmt.Sprint(testcase.SystemOutput), - StackTrace: testcase.StackTrace, - AttachmentUrl: testcase.AttachmentURL, - RecentFailures: convertTestCaseRecentFailures(testcase.RecentFailures), - }) - } - testcases = append(testcases, cases...) - } - - return testreport, testsuites, testcases -} - -func convertTestCaseRecentFailures(f *gitlab.RecentFailures) *typespb.TestCase_RecentFailures { - var r typespb.TestCase_RecentFailures - if f != nil { - r.Count = int64(f.Count) - r.BaseBranch = f.BaseBranch - } - return &r -} - -func testReportID(pipelineID int64) string { - return fmt.Sprint(pipelineID) -} - -func testSuiteID(reportID string, suiteIndex int) string { - return fmt.Sprintf("%s-%d", reportID, suiteIndex+1) -} - -func testCaseID(suiteID string, caseIndex int) string { - return fmt.Sprintf("%s-%d", suiteID, caseIndex+1) -} diff --git a/internal/gitlab/sections.go b/internal/gitlab/sections.go index 6a7f7e9..4316dc8 100644 --- a/internal/gitlab/sections.go +++ b/internal/gitlab/sections.go @@ -1,126 +1,17 @@ package gitlab import ( - "bufio" - "bytes" - "context" "errors" "regexp" - "sort" "strconv" - - gitlab "github.com/xanzy/go-gitlab" - - "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) -type ListJobSectionsResult struct { - Section *typespb.Section - Error error -} - -func (c *Client) ListJobSections(ctx context.Context, projectID int64, jobID int64) <-chan ListJobSectionsResult { - ch := make(chan ListJobSectionsResult) - - go func() { - defer close(ch) - - c.RLock() - job, _, err := c.client.Jobs.GetJob(int(projectID), int(jobID), gitlab.WithContext(ctx)) - c.RUnlock() - if err != nil { - ch <- ListJobSectionsResult{ - Error: err, - } - return - } - - c.RLock() - trace, _, err := c.client.Jobs.GetTraceFile(int(projectID), int(jobID), gitlab.WithContext(ctx)) - c.RUnlock() - if err != nil { - ch <- ListJobSectionsResult{ - Error: err, - } - return - } - - data, err := parseSections(trace) - if err != nil { - ch <- ListJobSectionsResult{ - Error: err, - } - return - } - - for secnum, secdat := range data { - section := &typespb.Section{ - Name: secdat.Name, - StartedAt: convertUnixSeconds(secdat.Start), - FinishedAt: convertUnixSeconds(secdat.End), - Duration: convertDuration(float64(secdat.End - secdat.Start)), - } - - section.Id = int64(job.ID*1000 + secnum) - section.Job.Id = int64(job.ID) - section.Job.Name = job.Name - section.Job.Status = job.Status - section.Pipeline.Id = int64(job.Pipeline.ID) - section.Pipeline.ProjectId = int64(job.Pipeline.ProjectID) - section.Pipeline.Ref = job.Pipeline.Ref - section.Pipeline.Sha = job.Pipeline.Sha - section.Pipeline.Status = job.Pipeline.Status - - ch <- ListJobSectionsResult{ - Section: section, - } - } - }() - - return ch -} - type SectionData struct { Name string `json:"name"` Start int64 `json:"start"` End int64 `json:"end"` } -func parseSections(trace *bytes.Reader) ([]SectionData, error) { - sections := []SectionData{} - stack := sectionStack{} - - scanner := bufio.NewScanner(trace) - for scanner.Scan() { - line := scanner.Bytes() - var i, j int - sep := []byte(`section_`) - for { - j = bytes.Index(line[i:], sep) - if j < 0 { - break - } - - marker, ts, name, err := parseSection(line[i:]) - if err != nil { - // TODO: what? - } else if marker == string(sectionMarkerStart) { - stack.Start(ts, name) - } else if marker == string(sectionMarkerEnd) { - sections = append(sections, stack.End(ts, name)...) - } - - i = i + j + 1 - } - } - - sort.SliceStable(sections, func(i, j int) bool { - return sections[i].Start < sections[j].Start - }) - - return sections, nil -} - type sectionStack struct { Sections []SectionData } diff --git a/internal/jobs/catchup.go b/internal/jobs/catchup.go index 294195a..a8a48bd 100644 --- a/internal/jobs/catchup.go +++ b/internal/jobs/catchup.go @@ -13,6 +13,7 @@ import ( "github.com/cluttrdev/gitlab-exporter/internal/exporter" "github.com/cluttrdev/gitlab-exporter/internal/gitlab" "github.com/cluttrdev/gitlab-exporter/internal/tasks" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/pkg/worker" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -165,7 +166,7 @@ func (j *ProjectCatchUpJob) exportProjectMergeRequests(ctx context.Context) { continue } - mergerequests = append(mergerequests, gitlab.ConvertMergeRequest(mr)) + mergerequests = append(mergerequests, types.ConvertMergeRequest(mr)) } if len(mergerequests) == 0 { diff --git a/internal/jobs/export.go b/internal/jobs/export.go index 561db15..627b9eb 100644 --- a/internal/jobs/export.go +++ b/internal/jobs/export.go @@ -3,6 +3,7 @@ package jobs import ( "context" "errors" + "fmt" "log/slog" "sync" "time" @@ -13,6 +14,7 @@ import ( "github.com/cluttrdev/gitlab-exporter/internal/exporter" "github.com/cluttrdev/gitlab-exporter/internal/gitlab" "github.com/cluttrdev/gitlab-exporter/internal/tasks" + "github.com/cluttrdev/gitlab-exporter/internal/types" "github.com/cluttrdev/gitlab-exporter/pkg/worker" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) @@ -129,7 +131,8 @@ func (j *ProjectExportJob) exportProjectPipelines(ctx context.Context, wg *sync. } func (j *ProjectExportJob) exportProjectMergeRequests(ctx context.Context) { - projectID := j.Config.Id + projectID := int(j.Config.Id) + glab := j.GitLab.Client() opt := _gitlab.ListProjectMergeRequestsOptions{ ListOptions: _gitlab.ListOptions{ @@ -149,7 +152,8 @@ func (j *ProjectExportJob) exportProjectMergeRequests(ctx context.Context) { var wg sync.WaitGroup for { - mrs, resp, err := j.GitLab.Client().MergeRequests.ListProjectMergeRequests(int(projectID), &opt, options...) + // get iids of updated merge requests + mrs, resp, err := glab.MergeRequests.ListProjectMergeRequests(projectID, &opt, options...) if err != nil { slog.Error("error fetching project merge requests", "project", projectID, "error", err) break @@ -167,24 +171,22 @@ func (j *ProjectExportJob) exportProjectMergeRequests(ctx context.Context) { opt := _gitlab.GetMergeRequestsOptions{} for _, iid := range iids { - mr, _, err := j.GitLab.Client().MergeRequests.GetMergeRequest(int(projectID), iid, &opt, _gitlab.WithContext(ctx)) + mr, _, err := glab.MergeRequests.GetMergeRequest(projectID, iid, &opt, _gitlab.WithContext(ctx)) if err != nil { if errors.Is(err, context.Canceled) { break } - slog.Error(err.Error()) + slog.Error("error fetching merge request", "project_id", projectID, "iid", iid) continue } - mergerequests = append(mergerequests, gitlab.ConvertMergeRequest(mr)) - } - - if len(mergerequests) == 0 { - return + mergerequests = append(mergerequests, types.ConvertMergeRequest(mr)) } - if err := j.Exporter.ExportMergeRequests(ctx, mergerequests); err != nil { - slog.Error(err.Error()) + if len(mergerequests) > 0 { + if err := j.Exporter.ExportMergeRequests(ctx, mergerequests); err != nil { + slog.Error(fmt.Sprintf("error exporting merge requests: %v", err)) + } } }) @@ -197,5 +199,7 @@ func (j *ProjectExportJob) exportProjectMergeRequests(ctx context.Context) { _gitlab.WithKeysetPaginationParameters(resp.NextLink), } } + + // wait for all paginated exports to finish wg.Wait() } diff --git a/internal/types/jobs.go b/internal/types/jobs.go new file mode 100644 index 0000000..64875c3 --- /dev/null +++ b/internal/types/jobs.go @@ -0,0 +1,136 @@ +package types + +import ( + "github.com/xanzy/go-gitlab" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" +) + +func ConvertJob(job *gitlab.Job) *typespb.Job { + artifacts := make([]*typespb.JobArtifacts, 0, len(job.Artifacts)) + for _, a := range job.Artifacts { + artifacts = append(artifacts, &typespb.JobArtifacts{ + Filename: a.Filename, + FileType: a.FileType, + FileFormat: a.FileFormat, + Size: int64(a.Size), + }) + } + + return &typespb.Job{ + Id: int64(job.ID), + Name: job.Name, + Pipeline: &typespb.PipelineReference{ + Id: int64(job.Pipeline.ID), + ProjectId: int64(job.Pipeline.ProjectID), + Ref: job.Pipeline.Ref, + Sha: job.Pipeline.Sha, + Status: job.Pipeline.Status, + }, + Ref: job.Ref, + CreatedAt: ConvertTime(job.CreatedAt), + StartedAt: ConvertTime(job.StartedAt), + FinishedAt: ConvertTime(job.FinishedAt), + ErasedAt: ConvertTime(job.ErasedAt), + Duration: ConvertDuration(job.Duration), + QueuedDuration: ConvertDuration(job.QueuedDuration), + Coverage: job.Coverage, + Stage: job.Stage, + Status: job.Status, + AllowFailure: job.AllowFailure, + FailureReason: job.FailureReason, + Tag: job.Tag, + WebUrl: job.WebURL, + TagList: job.TagList, + + Commit: convertCommit(job.Commit), + Project: ConvertProject(job.Project), + User: convertUser(job.User), + + Runner: &typespb.JobRunner{ + Id: int64(job.Runner.ID), + Name: job.Runner.Name, + Description: job.Runner.Description, + Active: job.Runner.Active, + IsShared: job.Runner.IsShared, + }, + + Artifacts: artifacts, + ArtifactsFile: &typespb.JobArtifactsFile{ + Filename: job.ArtifactsFile.Filename, + Size: int64(job.ArtifactsFile.Size), + }, + ArtifactsExpireAt: ConvertTime(job.ArtifactsExpireAt), + } +} + +func convertCommit(commit *gitlab.Commit) *typespb.Commit { + var status string + if commit.Status != nil { + status = string(*commit.Status) + } + return &typespb.Commit{ + Id: commit.ID, + ShortId: commit.ShortID, + ParentIds: commit.ParentIDs, + ProjectId: int64(commit.ProjectID), + AuthorName: commit.AuthorName, + AuthorEmail: commit.AuthorEmail, + AuthoredDate: ConvertTime(commit.AuthoredDate), + CommitterName: commit.CommitterName, + CommitterEmail: commit.CommitterEmail, + CommittedDate: ConvertTime(commit.CommittedDate), + CreatedAt: ConvertTime(commit.CreatedAt), + Title: commit.Title, + Message: commit.Message, + Trailers: commit.Trailers, + Stats: convertCommitStats(commit.Stats), + Status: status, + WebUrl: commit.WebURL, + } +} + +func convertCommitStats(stats *gitlab.CommitStats) *typespb.CommitStats { + if stats == nil { + return nil + } + return &typespb.CommitStats{ + Additions: int64(stats.Additions), + Deletions: int64(stats.Deletions), + Total: int64(stats.Total), + } +} + +func ConvertBridge(bridge *gitlab.Bridge) *typespb.Bridge { + // account for downstream pipeline creation failures + downstreamPipeline := &typespb.PipelineInfo{ + CreatedAt: ×tamppb.Timestamp{}, + UpdatedAt: ×tamppb.Timestamp{}, + } + if bridge.DownstreamPipeline != nil { + downstreamPipeline = ConvertPipelineInfo(bridge.DownstreamPipeline) + } + return &typespb.Bridge{ + // Commit: ConvertCommit(bridge.Commit), + Id: int64(bridge.ID), + Name: bridge.Name, + Pipeline: ConvertPipelineInfo(&bridge.Pipeline), + Ref: bridge.Ref, + CreatedAt: ConvertTime(bridge.CreatedAt), + StartedAt: ConvertTime(bridge.StartedAt), + FinishedAt: ConvertTime(bridge.FinishedAt), + ErasedAt: ConvertTime(bridge.ErasedAt), + Duration: ConvertDuration(bridge.Duration), + QueuedDuration: ConvertDuration(bridge.QueuedDuration), + Coverage: bridge.Coverage, + Stage: bridge.Stage, + Status: bridge.Status, + AllowFailure: bridge.AllowFailure, + FailureReason: bridge.FailureReason, + Tag: bridge.Tag, + WebUrl: bridge.WebURL, + // User: ConvertUser(bridge.User), + DownstreamPipeline: downstreamPipeline, + } +} diff --git a/internal/gitlab/merge_requests.go b/internal/types/mergerequest.go similarity index 59% rename from internal/gitlab/merge_requests.go rename to internal/types/mergerequest.go index d58f32b..b54a6fd 100644 --- a/internal/gitlab/merge_requests.go +++ b/internal/types/mergerequest.go @@ -1,70 +1,24 @@ -package gitlab +package types import ( - "context" + "strings" "time" - gitlab "github.com/xanzy/go-gitlab" + "github.com/xanzy/go-gitlab" "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" ) -type ListProjectMergeRequestsOptions struct { - gitlab.ListProjectMergeRequestsOptions - - Paginate bool -} - -func (c *Client) ListProjectMergeRequests(ctx context.Context, id int64, opt ListProjectMergeRequestsOptions) ([]*typespb.MergeRequest, error) { - var mergerequests []*typespb.MergeRequest - - options := []gitlab.RequestOptionFunc{ - gitlab.WithContext(ctx), - } - - for { - mrs, resp, err := c.client.MergeRequests.ListProjectMergeRequests(int(id), &opt.ListProjectMergeRequestsOptions, options...) - if err != nil { - return mergerequests, err - } - - for _, mr := range mrs { - mergerequests = append(mergerequests, ConvertMergeRequest(mr)) - } - - if !opt.Paginate { - break - } - - if opt.ListOptions.Pagination == "keyset" { - if resp.NextLink == "" { - break - } - options = []gitlab.RequestOptionFunc{ - gitlab.WithContext(ctx), - gitlab.WithKeysetPaginationParameters(resp.NextLink), - } - } else { - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - } - - return mergerequests, nil -} - func ConvertMergeRequest(mr *gitlab.MergeRequest) *typespb.MergeRequest { return &typespb.MergeRequest{ Id: int64(mr.ID), Iid: int64(mr.IID), ProjectId: int64(mr.ProjectID), - CreatedAt: convertTime(mr.CreatedAt), - UpdatedAt: convertTime(mr.UpdatedAt), - MergedAt: convertTime(mr.MergedAt), - ClosedAt: convertTime(mr.ClosedAt), + CreatedAt: ConvertTime(mr.CreatedAt), + UpdatedAt: ConvertTime(mr.UpdatedAt), + MergedAt: ConvertTime(mr.MergedAt), + ClosedAt: ConvertTime(mr.ClosedAt), SourceProjectId: int64(mr.SourceProjectID), TargetProjectId: int64(mr.TargetProjectID), @@ -102,7 +56,7 @@ func ConvertMergeRequest(mr *gitlab.MergeRequest) *typespb.MergeRequest { Upvotes: int64(mr.Upvotes), Downvotes: int64(mr.Downvotes), - Pipeline: convertPipelineInfo(mr.Pipeline), + Pipeline: ConvertPipelineInfo(mr.Pipeline), Milestone: convertMilestone(mr.Milestone), @@ -119,7 +73,7 @@ func convertBasicUser(u *gitlab.BasicUser) *typespb.User { Username: u.Username, Name: u.Name, State: u.State, - CreatedAt: convertTime(u.CreatedAt), + CreatedAt: ConvertTime(u.CreatedAt), } } @@ -132,7 +86,7 @@ func convertUser(u *gitlab.User) *typespb.User { Username: u.Username, Name: u.Name, State: u.State, - CreatedAt: convertTime(u.CreatedAt), + CreatedAt: ConvertTime(u.CreatedAt), } } @@ -153,9 +107,9 @@ func convertMilestone(m *gitlab.Milestone) *typespb.Milestone { Iid: int64(m.IID), ProjectId: int64(m.ProjectID), GroupId: int64(m.GroupID), - CreatedAt: convertTime(m.CreatedAt), - UpdatedAt: convertTime(m.UpdatedAt), - StartDate: convertTime((*time.Time)(m.StartDate)), - DueDate: convertTime((*time.Time)(m.DueDate)), + CreatedAt: ConvertTime(m.CreatedAt), + UpdatedAt: ConvertTime(m.UpdatedAt), + StartDate: ConvertTime((*time.Time)(m.StartDate)), + DueDate: ConvertTime((*time.Time)(m.DueDate)), } } diff --git a/internal/types/pipeline.go b/internal/types/pipeline.go new file mode 100644 index 0000000..760e695 --- /dev/null +++ b/internal/types/pipeline.go @@ -0,0 +1,61 @@ +package types + +import ( + "strconv" + + "github.com/xanzy/go-gitlab" + + "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" +) + +func ConvertPipelineInfo(pipeline *gitlab.PipelineInfo) *typespb.PipelineInfo { + if pipeline == nil { + return nil + } + + return &typespb.PipelineInfo{ + Id: int64(pipeline.ID), + Iid: int64(pipeline.IID), + ProjectId: int64(pipeline.ProjectID), + Status: pipeline.Status, + Source: pipeline.Source, + Ref: pipeline.Ref, + Sha: pipeline.SHA, + WebUrl: pipeline.WebURL, + CreatedAt: ConvertTime(pipeline.CreatedAt), + UpdatedAt: ConvertTime(pipeline.UpdatedAt), + } +} + +func ConvertPipeline(pipeline *gitlab.Pipeline) *typespb.Pipeline { + return &typespb.Pipeline{ + Id: int64(pipeline.ID), + Iid: int64(pipeline.IID), + ProjectId: int64(pipeline.ProjectID), + Status: pipeline.Status, + Source: pipeline.Source, + Ref: pipeline.Ref, + Sha: pipeline.SHA, + BeforeSha: pipeline.BeforeSHA, + Tag: pipeline.Tag, + YamlErrors: pipeline.YamlErrors, + CreatedAt: ConvertTime(pipeline.CreatedAt), + UpdatedAt: ConvertTime(pipeline.UpdatedAt), + StartedAt: ConvertTime(pipeline.StartedAt), + FinishedAt: ConvertTime(pipeline.FinishedAt), + CommittedAt: ConvertTime(pipeline.CommittedAt), + Duration: ConvertDuration(float64(pipeline.Duration)), + QueuedDuration: ConvertDuration(float64(pipeline.QueuedDuration)), + Coverage: convertCoverage(pipeline.Coverage), + WebUrl: pipeline.WebURL, + User: convertBasicUser(pipeline.User), + } +} + +func convertCoverage(coverage string) float64 { + cov, err := strconv.ParseFloat(coverage, 64) + if err != nil { + cov = 0.0 + } + return cov +} diff --git a/internal/types/project.go b/internal/types/project.go new file mode 100644 index 0000000..a718957 --- /dev/null +++ b/internal/types/project.go @@ -0,0 +1,73 @@ +package types + +import ( + "github.com/xanzy/go-gitlab" + + "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" +) + +func ConvertProject(p *gitlab.Project) *typespb.Project { + return &typespb.Project{ + Id: int64(p.ID), + Name: p.Name, + NameWithNamespace: p.NameWithNamespace, + Path: p.Path, + PathWithNamespace: p.PathWithNamespace, + + CreatedAt: ConvertTime(p.CreatedAt), + LastActivityAt: ConvertTime(p.LastActivityAt), + + Namespace: convertProjectNamespace(p.Namespace), + Owner: convertUser(p.Owner), + CreatorId: int64(p.CreatorID), + + Topics: p.Topics, + ForksCount: int64(p.ForksCount), + StarsCount: int64(p.StarCount), + Statistics: convertProjectStatistics(p.Statistics), + OpenIssuesCount: int64(p.OpenIssuesCount), + + Description: p.Description, + + EmptyRepo: p.EmptyRepo, + Archived: p.Archived, + + DefaultBranch: p.DefaultBranch, + Visibility: string(p.Visibility), + WebUrl: p.WebURL, + } +} + +func convertProjectNamespace(n *gitlab.ProjectNamespace) *typespb.ProjectNamespace { + if n == nil { + return nil + } + return &typespb.ProjectNamespace{ + Id: int64(n.ID), + Name: n.Name, + Kind: n.Kind, + Path: n.Path, + FullPath: n.FullPath, + ParentId: int64(n.ParentID), + + AvatarUrl: n.AvatarURL, + WebUrl: n.WebURL, + } +} + +func convertProjectStatistics(stats *gitlab.Statistics) *typespb.ProjectStatistics { + if stats == nil { + return nil + } + return &typespb.ProjectStatistics{ + CommitCount: stats.CommitCount, + StorageSize: stats.StorageSize, + RepositorySize: stats.RepositorySize, + WikiSize: stats.WikiSize, + LfsObjectsSize: stats.LFSObjectsSize, + JobArtifactsSize: stats.JobArtifactsSize, + PackagesSize: stats.PackagesSize, + SnippetsSize: stats.SnippetsSize, + UploadsSize: stats.UploadsSize, + } +} diff --git a/internal/types/testreport.go b/internal/types/testreport.go new file mode 100644 index 0000000..b253f75 --- /dev/null +++ b/internal/types/testreport.go @@ -0,0 +1,83 @@ +package types + +import ( + "fmt" + + "github.com/xanzy/go-gitlab" + + "github.com/cluttrdev/gitlab-exporter/protobuf/typespb" +) + +func ConvertTestReport(pipelineID int64, report *gitlab.PipelineTestReport) (*typespb.TestReport, []*typespb.TestSuite, []*typespb.TestCase) { + testreport := &typespb.TestReport{ + Id: testReportID(pipelineID), + PipelineId: pipelineID, + TotalTime: report.TotalTime, + TotalCount: int64(report.TotalCount), + SuccessCount: int64(report.SuccessCount), + FailedCount: int64(report.FailedCount), + SkippedCount: int64(report.SkippedCount), + ErrorCount: int64(report.ErrorCount), + } + + testsuites := make([]*typespb.TestSuite, 0, len(report.TestSuites)) + testcases := []*typespb.TestCase{} + for i, testsuite := range report.TestSuites { + testsuiteID := testSuiteID(testreport.Id, i) + testsuites = append(testsuites, &typespb.TestSuite{ + Id: testsuiteID, + TestreportId: testreport.Id, + PipelineId: pipelineID, + Name: testsuite.Name, + TotalTime: testsuite.TotalTime, + TotalCount: int64(testsuite.TotalCount), + SuccessCount: int64(testsuite.SuccessCount), + FailedCount: int64(testsuite.FailedCount), + SkippedCount: int64(testsuite.SkippedCount), + ErrorCount: int64(testsuite.ErrorCount), + }) + + cases := make([]*typespb.TestCase, 0, len(testsuite.TestCases)) + for j, testcase := range testsuite.TestCases { + cases = append(cases, &typespb.TestCase{ + Id: testCaseID(testsuiteID, j), + TestsuiteId: testsuiteID, + TestreportId: testreport.Id, + PipelineId: pipelineID, + Status: testcase.Status, + Name: testcase.Name, + Classname: testcase.Classname, + File: testcase.File, + ExecutionTime: testcase.ExecutionTime, + SystemOutput: fmt.Sprint(testcase.SystemOutput), + StackTrace: testcase.StackTrace, + AttachmentUrl: testcase.AttachmentURL, + RecentFailures: convertTestCaseRecentFailures(testcase.RecentFailures), + }) + } + testcases = append(testcases, cases...) + } + + return testreport, testsuites, testcases +} + +func convertTestCaseRecentFailures(f *gitlab.RecentFailures) *typespb.TestCase_RecentFailures { + var r typespb.TestCase_RecentFailures + if f != nil { + r.Count = int64(f.Count) + r.BaseBranch = f.BaseBranch + } + return &r +} + +func testReportID(pipelineID int64) string { + return fmt.Sprint(pipelineID) +} + +func testSuiteID(reportID string, suiteIndex int) string { + return fmt.Sprintf("%s-%d", reportID, suiteIndex+1) +} + +func testCaseID(suiteID string, caseIndex int) string { + return fmt.Sprintf("%s-%d", suiteID, caseIndex+1) +} diff --git a/internal/types/types.go b/internal/types/types.go new file mode 100644 index 0000000..9ac22fe --- /dev/null +++ b/internal/types/types.go @@ -0,0 +1,43 @@ +package types + +import ( + "time" + + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func ConvertTime(t *time.Time) *timestamppb.Timestamp { + if t == nil { + return nil + } + return timestamppb.New(*t) +} + +func ConvertUnixSeconds(ts int64) *timestamppb.Timestamp { + return ×tamppb.Timestamp{ + Seconds: ts, + Nanos: 0, + } +} + +func ConvertUnixMilli(ts int64) *timestamppb.Timestamp { + const msPerSecond int64 = 1_000 + const nsPerMilli int64 = 1_000 + return ×tamppb.Timestamp{ + Seconds: ts / msPerSecond, + Nanos: int32((ts % msPerSecond) * nsPerMilli), + } +} + +func ConvertUnixNano(ts int64) *timestamppb.Timestamp { + const nsPerSecond int64 = 1_000_000_000 + return ×tamppb.Timestamp{ + Seconds: ts / nsPerSecond, + Nanos: int32(ts % nsPerSecond), + } +} + +func ConvertDuration(d float64) *durationpb.Duration { + return durationpb.New(time.Duration(d * float64(time.Second))) +}