diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9e0f51e..54b43e10 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,3 +46,5 @@ jobs: - name: Build run: make build + + \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go index 274ef694..a7417ae4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -135,6 +135,13 @@ type Redis struct { // Pull .. type Pull struct { + // Projects configuration + Projects struct { + OnInit bool `default:"true" yaml:"on_init"` + Scheduled bool `default:"true" yaml:"scheduled"` + IntervalSeconds int `default:"1800" validate:"gte=1" yaml:"interval_seconds"` + } `yaml:"projects"` + // ProjectsFromWildcards configuration ProjectsFromWildcards struct { OnInit bool `default:"true" yaml:"on_init"` diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index cb15964a..b8c7c797 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -26,6 +26,10 @@ func TestNew(t *testing.T) { c.Gitlab.BurstableRequestsPerSecond = 5 c.Gitlab.MaximumJobsQueueSize = 1000 + c.Pull.Projects.OnInit = true + c.Pull.Projects.Scheduled = true + c.Pull.Projects.IntervalSeconds = 1800 + c.Pull.ProjectsFromWildcards.OnInit = true c.Pull.ProjectsFromWildcards.Scheduled = true c.Pull.ProjectsFromWildcards.IntervalSeconds = 1800 diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7dcad65a..567aad7e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -55,7 +55,7 @@ func New(ctx context.Context, cfg config.Config, version string) (c Controller, c.TaskController = NewTaskController(ctx, c.Redis, cfg.Gitlab.MaximumJobsQueueSize) c.registerTasks() - c.Store = store.New(ctx, c.Redis, c.Config.Projects) + c.Store = store.New(ctx, c.Redis) if err = c.configureGitlab(cfg.Gitlab, version); err != nil { return @@ -78,6 +78,7 @@ func (c *Controller) registerTasks() { schemas.TaskTypePullEnvironmentsFromProjects: c.TaskHandlerPullEnvironmentsFromProjects, schemas.TaskTypePullMetrics: c.TaskHandlerPullMetrics, schemas.TaskTypePullProject: c.TaskHandlerPullProject, + schemas.TaskTypePullProjects: c.TaskHandlerPullProjects, schemas.TaskTypePullProjectsFromWildcard: c.TaskHandlerPullProjectsFromWildcard, schemas.TaskTypePullProjectsFromWildcards: c.TaskHandlerPullProjectsFromWildcards, schemas.TaskTypePullRefMetrics: c.TaskHandlerPullRefMetrics, diff --git a/pkg/controller/environments_test.go b/pkg/controller/environments_test.go index c34056f6..b04c1132 100644 --- a/pkg/controller/environments_test.go +++ b/pkg/controller/environments_test.go @@ -47,7 +47,7 @@ func TestPullEnvironmentsFromProject(t *testing.T) { }`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Environments.Regexp = "^prod" assert.NoError(t, c.PullEnvironmentsFromProject(ctx, p)) diff --git a/pkg/controller/garbage_collector.go b/pkg/controller/garbage_collector.go index c4a6e3ce..86f39131 100644 --- a/pkg/controller/garbage_collector.go +++ b/pkg/controller/garbage_collector.go @@ -70,7 +70,7 @@ func (c *Controller) GarbageCollectEnvironments(ctx context.Context) error { envProjects := make(map[schemas.Project]bool) for _, env := range storedEnvironments { - p := schemas.NewProject(env.ProjectName) + p := schemas.NewProject(env.ProjectName, []string{}) projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { @@ -296,7 +296,7 @@ func (c *Controller) GarbageCollectMetrics(ctx context.Context) error { if metricLabelRefExists && !metricLabelEnvironmentExists { refKey := schemas.NewRef( - schemas.NewProject(metricLabelProject), + schemas.NewProject(metricLabelProject, []string{}), schemas.RefKind(m.Labels["kind"]), metricLabelRef, ).Key() diff --git a/pkg/controller/garbage_collector_test.go b/pkg/controller/garbage_collector_test.go index 7a4be0ae..047f7155 100644 --- a/pkg/controller/garbage_collector_test.go +++ b/pkg/controller/garbage_collector_test.go @@ -14,10 +14,10 @@ import ( ) func TestGarbageCollectProjects(t *testing.T) { - p1 := schemas.NewProject("cfg/p1") - p2 := schemas.NewProject("cfg/p2") - p3 := schemas.NewProject("wc/p3") - p4 := schemas.NewProject("wc/p4") + p1 := schemas.NewProject("cfg/p1", []string{}) + p2 := schemas.NewProject("cfg/p2", []string{}) + p3 := schemas.NewProject("wc/p3", []string{}) + p4 := schemas.NewProject("wc/p4", []string{}) ctx, c, mux, srv := newTestController(config.Config{ Projects: []config.Project{p1.Project}, @@ -62,7 +62,7 @@ func TestGarbageCollectEnvironments(t *testing.T) { fmt.Fprint(w, `[{"name": "main"}]`) }) - p2 := schemas.NewProject("p2") + p2 := schemas.NewProject("p2", []string{}) p2.Pull.Environments.Enabled = true p2.Pull.Environments.Regexp = "^main$" @@ -103,10 +103,10 @@ func TestGarbageCollectRefs(t *testing.T) { fmt.Fprint(w, `[{"name": "main"}]`) }) - pr1dev := schemas.NewRef(schemas.NewProject("p1"), schemas.RefKindBranch, "dev") - pr1main := schemas.NewRef(schemas.NewProject("p1"), schemas.RefKindBranch, "main") + pr1dev := schemas.NewRef(schemas.NewProject("p1", []string{}), schemas.RefKindBranch, "dev") + pr1main := schemas.NewRef(schemas.NewProject("p1", []string{}), schemas.RefKindBranch, "main") - p2 := schemas.NewProject("p2") + p2 := schemas.NewProject("p2", []string{}) p2.Pull.Environments.Regexp = "^main$" pr2dev := schemas.NewRef(p2, schemas.RefKindBranch, "dev") @@ -133,7 +133,7 @@ func TestGarbageCollectMetrics(t *testing.T) { ctx, c, _, srv := newTestController(config.Config{}) srv.Close() - p1 := schemas.NewProject("p1") + p1 := schemas.NewProject("p1", []string{}) p1.Pull.Pipeline.Jobs.Enabled = true ref1 := schemas.NewRef(p1, schemas.RefKindBranch, "foo") diff --git a/pkg/controller/jobs_test.go b/pkg/controller/jobs_test.go index 545a8eca..50304c59 100644 --- a/pkg/controller/jobs_test.go +++ b/pkg/controller/jobs_test.go @@ -20,7 +20,7 @@ func TestPullRefPipelineJobsMetrics(t *testing.T) { fmt.Fprint(w, `[{"id":1,"created_at":"2016-08-11T11:28:34.085Z","started_at":"2016-08-11T11:28:56.085Z"},{"id":2,"created_at":"2016-08-11T11:28:34.085Z","started_at":"2016-08-11T11:28:58.085Z"}]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = false ref := schemas.NewRef(p, schemas.RefKindBranch, "bar") @@ -42,7 +42,7 @@ func TestPullRefMostRecentJobsMetrics(t *testing.T) { }) ref := schemas.Ref{ - Project: schemas.NewProject("foo"), + Project: schemas.NewProject("foo", []string{}), Name: "bar", LatestJobs: schemas.Jobs{ "bar": { @@ -85,7 +85,7 @@ func TestProcessJobMetrics(t *testing.T) { }, } - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Topics = "first,second" p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `foo-(.*)-bar` diff --git a/pkg/controller/pipelines_test.go b/pkg/controller/pipelines_test.go index a2d644c6..c482ec0c 100644 --- a/pkg/controller/pipelines_test.go +++ b/pkg/controller/pipelines_test.go @@ -40,7 +40,7 @@ func TestPullRefMetricsSucceed(t *testing.T) { }) // Metrics pull shall succeed - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Pipeline.Variables.Enabled = true p.Pull.Pipeline.TestReports.Enabled = true p.Pull.Pipeline.TestReports.TestCases.Enabled = true @@ -130,7 +130,7 @@ func TestPullRefTestReportMetrics(t *testing.T) { }) // Metrics pull shall succeed - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Pipeline.Variables.Enabled = true p.Pull.Pipeline.TestReports.Enabled = true p.Pull.Pipeline.TestReports.TestCases.Enabled = true @@ -281,7 +281,7 @@ func TestPullRefMetricsMergeRequestPipeline(t *testing.T) { }) // Metrics pull shall succeed - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Pipeline.Variables.Enabled = true assert.NoError(t, c.PullRefMetrics( diff --git a/pkg/controller/projects.go b/pkg/controller/projects.go index ff14fed3..d21c1d26 100644 --- a/pkg/controller/projects.go +++ b/pkg/controller/projects.go @@ -10,33 +10,38 @@ import ( ) // PullProject .. -func (c *Controller) PullProject(ctx context.Context, name string, pull config.ProjectPull) error { - gp, err := c.Gitlab.GetProject(ctx, name) +func (c *Controller) PullProject(ctx context.Context, project config.Project) error { + gp, err := c.Gitlab.GetProject(ctx, project.Name) if err != nil { return err } - p := schemas.NewProject(gp.PathWithNamespace) - p.Pull = pull - - projectExists, err := c.Store.ProjectExists(ctx, p.Key()) + projectExists, err := c.Store.ProjectExists(ctx, gp.Key()) if err != nil { return err } + // We need to set the project in the store regardless of it being new or not + // to ensure any project updates e.g. Topics are correctly reflected + if err := c.Store.SetProject(ctx, gp); err != nil { + log.WithContext(ctx). + WithError(err). + Error() + } + if !projectExists { log.WithFields(log.Fields{ - "project-name": p.Name, + "project-name": gp.Name, }).Info("discovered new project") - if err := c.Store.SetProject(ctx, p); err != nil { + if err := c.Store.SetProject(ctx, gp); err != nil { log.WithContext(ctx). WithError(err). Error() } - c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p) - c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p) + c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(gp.Key()), gp) + c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(gp.Key()), gp) } return nil @@ -55,6 +60,14 @@ func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wild return err } + // We need to set the project in the store regardless of it being new or not + // to ensure any project updates e.g. Topics are correctly reflected + if err := c.Store.SetProject(ctx, p); err != nil { + log.WithContext(ctx). + WithError(err). + Error() + } + if !projectExists { log.WithFields(log.Fields{ "wildcard-search": w.Search, @@ -65,12 +78,6 @@ func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wild "project-name": p.Name, }).Info("discovered new project") - if err := c.Store.SetProject(ctx, p); err != nil { - log.WithContext(ctx). - WithError(err). - Error() - } - c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p) c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p) } diff --git a/pkg/controller/projects_test.go b/pkg/controller/projects_test.go index 52522ac4..004175c7 100644 --- a/pkg/controller/projects_test.go +++ b/pkg/controller/projects_test.go @@ -15,19 +15,66 @@ func TestPullProjectsFromWildcard(t *testing.T) { ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() + topicsIdentifier := 0 + mux.HandleFunc("/api/v4/projects", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `[{"id":2,"path_with_namespace":"bar","jobs_enabled":true}]`) + fmt.Fprintf(w, `[{"id":2,"path_with_namespace":"bar","jobs_enabled":true,"topics":["foo%d","bar%d"]}]`, topicsIdentifier, topicsIdentifier) + topicsIdentifier += 1 }) w := config.NewWildcard() assert.NoError(t, c.PullProjectsFromWildcard(ctx, w)) projects, _ := c.Store.Projects(ctx) - p1 := schemas.NewProject("bar") + p1 := schemas.NewProject("bar", []string{"foo0", "bar0"}) + p2 := schemas.NewProject("bar", []string{"foo1", "bar1"}) + + expectedProjects := schemas.Projects{ + p1.Key(): p1, + } + assert.Equal(t, expectedProjects, projects) + + expectedUpdatedProjects := schemas.Projects{ + p2.Key(): p2, + } + + // Pull projects again, which will have topics updated + assert.NoError(t, c.PullProjectsFromWildcard(ctx, w)) + projects, _ = c.Store.Projects(ctx) + assert.Equal(t, expectedUpdatedProjects, projects) +} + +func TestPullProjects(t *testing.T) { + ctx, c, mux, srv := newTestController(config.Config{}) + defer srv.Close() + + topicsIdentifier := 0 + + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%s", "foo%2Fbar"), + func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"id":2,"path_with_namespace":"bar","jobs_enabled":true,"topics":["foo%d","bar%d"]}`, topicsIdentifier, topicsIdentifier) + topicsIdentifier += 1 + }) + + w := config.NewProject("foo/bar") + assert.NoError(t, c.PullProject(ctx, w)) + + projects, _ := c.Store.Projects(ctx) + p1 := schemas.NewProject("bar", []string{"foo0", "bar0"}) + p2 := schemas.NewProject("bar", []string{"foo1", "bar1"}) expectedProjects := schemas.Projects{ p1.Key(): p1, } assert.Equal(t, expectedProjects, projects) + + expectedUpdatedProjects := schemas.Projects{ + p2.Key(): p2, + } + + // Pull projects again, which will have topics updated + assert.NoError(t, c.PullProject(ctx, w)) + projects, _ = c.Store.Projects(ctx) + assert.Equal(t, expectedUpdatedProjects, projects) } diff --git a/pkg/controller/refs.go b/pkg/controller/refs.go index 9b57d04b..acc0985d 100644 --- a/pkg/controller/refs.go +++ b/pkg/controller/refs.go @@ -88,6 +88,12 @@ func (c *Controller) PullRefsFromProject(ctx context.Context, p schemas.Project) return err } + // We need to set the ref in the store regardless of it being new or not + // to ensure any project updates e.g. Topics are correctly reflected + if err = c.Store.SetRef(ctx, ref); err != nil { + return err + } + if !refExists { log.WithFields(log.Fields{ "project-name": ref.Project.Name, @@ -95,10 +101,6 @@ func (c *Controller) PullRefsFromProject(ctx context.Context, p schemas.Project) "ref-kind": ref.Kind, }).Info("discovered new ref") - if err = c.Store.SetRef(ctx, ref); err != nil { - return err - } - c.ScheduleTask(ctx, schemas.TaskTypePullRefMetrics, string(ref.Key()), ref) } } diff --git a/pkg/controller/refs_test.go b/pkg/controller/refs_test.go index 34879306..af9cebbb 100644 --- a/pkg/controller/refs_test.go +++ b/pkg/controller/refs_test.go @@ -30,7 +30,7 @@ func TestGetRefs(t *testing.T) { fmt.Fprint(w, `[{"ref":"refs/merge-requests/1234/head"}]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Refs.Branches.Regexp = `^m` p.Pull.Refs.Tags.Regexp = `^v` p.Pull.Refs.MergeRequests.Enabled = true @@ -53,11 +53,6 @@ func TestPullRefsFromProject(t *testing.T) { ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() - mux.HandleFunc("/api/v4/projects/foo", - func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"name":"foo"}`) - }) - mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/branches"), func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `[{"name":"main"},{"name":"nope"}]`) @@ -68,14 +63,25 @@ func TestPullRefsFromProject(t *testing.T) { fmt.Fprint(w, `[]`) }) - p1 := schemas.NewProject("foo") + p1 := schemas.NewProject("foo", []string{"foo", "bar"}) assert.NoError(t, c.PullRefsFromProject(ctx, p1)) ref1 := schemas.NewRef(p1, schemas.RefKindBranch, "main") - expectedRefs := schemas.Refs{ + expectedRefs1 := schemas.Refs{ ref1.Key(): ref1, } - projectsRefs, _ := c.Store.Refs(ctx) - assert.Equal(t, expectedRefs, projectsRefs) + projectsRefs1, _ := c.Store.Refs(ctx) + assert.Equal(t, expectedRefs1, projectsRefs1) + + p2 := schemas.NewProject("foo", []string{"foo"}) + assert.NoError(t, c.PullRefsFromProject(ctx, p2)) + + ref2 := schemas.NewRef(p2, schemas.RefKindBranch, "main") + expectedRefs2 := schemas.Refs{ + ref2.Key(): ref2, + } + + projectsRefs2, _ := c.Store.Refs(ctx) + assert.Equal(t, expectedRefs2, projectsRefs2) } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index ac6a0be7..9fc66ad0 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -71,11 +71,25 @@ func NewTaskController(ctx context.Context, r *redis.Client, maximumJobsQueueSiz return } -// TaskHandlerPullProject .. -func (c *Controller) TaskHandlerPullProject(ctx context.Context, name string, pull config.ProjectPull) error { - defer c.unqueueTask(ctx, schemas.TaskTypePullProject, name) +func (c *Controller) TaskHandlerPullProjects(ctx context.Context) { + defer c.unqueueTask(ctx, schemas.TaskTypePullProjects, "_") + defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypePullProjects) - return c.PullProject(ctx, name, pull) + log.WithFields( + log.Fields{ + "projects-count": len(c.Config.Projects), + }, + ).Info("scheduling projects pull") + + for id, p := range c.Config.Projects { + c.ScheduleTask(ctx, schemas.TaskTypePullProject, strconv.Itoa(id), p) + } +} + +func (c *Controller) TaskHandlerPullProject(ctx context.Context, project config.Project) error { + defer c.unqueueTask(ctx, schemas.TaskTypePullProjects, project.Name) + + return c.PullProject(ctx, project) } // TaskHandlerPullProjectsFromWildcard .. @@ -319,6 +333,7 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G }() for tt, cfg := range map[schemas.TaskType]config.SchedulerConfig{ + schemas.TaskTypePullProjects: config.SchedulerConfig(pull.Projects), schemas.TaskTypePullProjectsFromWildcards: config.SchedulerConfig(pull.ProjectsFromWildcards), schemas.TaskTypePullEnvironmentsFromProjects: config.SchedulerConfig(pull.EnvironmentsFromProjects), schemas.TaskTypePullRefsFromProjects: config.SchedulerConfig(pull.RefsFromProjects), diff --git a/pkg/controller/webhooks.go b/pkg/controller/webhooks.go index 3faf81ec..ec8d7363 100644 --- a/pkg/controller/webhooks.go +++ b/pkg/controller/webhooks.go @@ -32,7 +32,7 @@ func (c *Controller) processPipelineEvent(ctx context.Context, e goGitlab.Pipeli } c.triggerRefMetricsPull(ctx, schemas.NewRef( - schemas.NewProject(e.Project.PathWithNamespace), + schemas.NewProject(e.Project.PathWithNamespace, []string{}), refKind, refName, )) @@ -60,7 +60,7 @@ func (c *Controller) processJobEvent(ctx context.Context, e goGitlab.JobEvent) { } c.triggerRefMetricsPull(ctx, schemas.NewRef( - schemas.NewProject(project.PathWithNamespace), + schemas.NewProject(project.PathWithNamespace, project.Topics), refKind, refName, )) @@ -88,7 +88,7 @@ func (c *Controller) processPushEvent(ctx context.Context, e goGitlab.PushEvent) } _ = deleteRef(ctx, c.Store, schemas.NewRef( - schemas.NewProject(e.Project.PathWithNamespace), + schemas.NewProject(e.Project.PathWithNamespace, []string{}), refKind, refName, ), "received branch deletion push event from webhook") @@ -117,7 +117,7 @@ func (c *Controller) processTagEvent(ctx context.Context, e goGitlab.TagEvent) { } _ = deleteRef(ctx, c.Store, schemas.NewRef( - schemas.NewProject(e.Project.PathWithNamespace), + schemas.NewProject(e.Project.PathWithNamespace, []string{}), refKind, refName, ), "received tag deletion tag event from webhook") @@ -126,7 +126,7 @@ func (c *Controller) processTagEvent(ctx context.Context, e goGitlab.TagEvent) { func (c *Controller) processMergeEvent(ctx context.Context, e goGitlab.MergeEvent) { ref := schemas.NewRef( - schemas.NewProject(e.Project.PathWithNamespace), + schemas.NewProject(e.Project.PathWithNamespace, []string{}), schemas.RefKindMergeRequest, strconv.Itoa(e.ObjectAttributes.IID), ) @@ -162,7 +162,7 @@ func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) // Let's try to see if the project is configured to export this ref if !refExists { - p := schemas.NewProject(ref.Project.Name) + p := schemas.NewProject(ref.Project.Name, strings.Split(ref.Project.Topics, ",")) projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { @@ -276,7 +276,7 @@ func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env sche } if !envExists { - p := schemas.NewProject(env.ProjectName) + p := schemas.NewProject(env.ProjectName, []string{}) projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { diff --git a/pkg/controller/webhooks_test.go b/pkg/controller/webhooks_test.go index 5e47e5c1..ee19bc41 100644 --- a/pkg/controller/webhooks_test.go +++ b/pkg/controller/webhooks_test.go @@ -14,11 +14,11 @@ func TestTriggerRefMetricsPull(t *testing.T) { srv.Close() ref1 := schemas.Ref{ - Project: schemas.NewProject("group/foo"), + Project: schemas.NewProject("group/foo", []string{"topicFoo"}), Name: "main", } - p2 := schemas.NewProject("group/bar") + p2 := schemas.NewProject("group/bar", []string{"topicBar"}) ref2 := schemas.Ref{ Project: p2, Name: "main", @@ -36,7 +36,7 @@ func TestTriggerEnvironmentMetricsPull(t *testing.T) { ctx, c, _, srv := newTestController(config.Config{}) srv.Close() - p1 := schemas.NewProject("foo/bar") + p1 := schemas.NewProject("foo/bar", []string{}) env1 := schemas.Environment{ ProjectName: p1.Name, Name: "dev", diff --git a/pkg/gitlab/branches_test.go b/pkg/gitlab/branches_test.go index 088feec7..4016d49f 100644 --- a/pkg/gitlab/branches_test.go +++ b/pkg/gitlab/branches_test.go @@ -43,7 +43,7 @@ func TestGetProjectBranches(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) expectedRef := schemas.NewRef(p, schemas.RefKindBranch, "main") refs, err := c.GetProjectBranches(ctx, p) assert.NoError(t, err) diff --git a/pkg/gitlab/environments_test.go b/pkg/gitlab/environments_test.go index e77d0e43..d1184d8d 100644 --- a/pkg/gitlab/environments_test.go +++ b/pkg/gitlab/environments_test.go @@ -53,7 +53,7 @@ func TestGetProjectEnvironments(t *testing.T) { }, ) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Environments.Regexp = "^dev" p.Pull.Environments.ExcludeStopped = false diff --git a/pkg/gitlab/jobs_test.go b/pkg/gitlab/jobs_test.go index 93b28813..7e087b89 100644 --- a/pkg/gitlab/jobs_test.go +++ b/pkg/gitlab/jobs_test.go @@ -16,7 +16,7 @@ func TestListRefPipelineJobs(t *testing.T) { defer server.Close() ref := schemas.Ref{ - Project: schemas.NewProject("foo"), + Project: schemas.NewProject("foo", []string{}), Name: "yay", } @@ -167,7 +167,7 @@ func TestListRefMostRecentJobs(t *testing.T) { } ref := schemas.Ref{ - Project: schemas.NewProject("foo"), + Project: schemas.NewProject("foo", []string{}), Name: "yay", } diff --git a/pkg/gitlab/pipelines_test.go b/pkg/gitlab/pipelines_test.go index a4a8ebc7..9d3875d0 100644 --- a/pkg/gitlab/pipelines_test.go +++ b/pkg/gitlab/pipelines_test.go @@ -25,7 +25,7 @@ func TestGetRefPipeline(t *testing.T) { }) ref := schemas.Ref{ - Project: schemas.NewProject("foo"), + Project: schemas.NewProject("foo", []string{}), Name: "yay", } @@ -71,7 +71,7 @@ func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) { fmt.Fprint(w, `[{"key":"foo","value":"bar"},{"key":"bar","value":"baz"}]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Pipeline.Variables.Enabled = true p.Pull.Pipeline.Variables.Regexp = `[` ref := schemas.Ref{ @@ -135,7 +135,7 @@ func TestGetRefsFromPipelines(t *testing.T) { fmt.Fprint(w, `[{"id":1,"ref":"keep_dev"},{"id":2,"ref":"keep_main"},{"id":3,"ref":"donotkeep_0.0.1"},{"id":4,"ref":"keep_0.0.2"},{"id":5,"ref":"refs/merge-requests/1234/head"}]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) // Branches p.Pull.Refs.Branches.Regexp = `[` // invalid regexp pattern @@ -186,7 +186,7 @@ func TestGetRefPipelineTestReport(t *testing.T) { fmt.Fprint(w, `{"total_time": 5, "total_count": 1, "success_count": 1, "failed_count": 0, "skipped_count": 0, "error_count": 0, "test_suites": [{"name": "Secure", "total_time": 5, "total_count": 1, "success_count": 1, "failed_count": 0, "skipped_count": 0, "error_count": 0, "test_cases": [{"status": "success", "name": "Security Reports can create an auto-remediation MR", "classname": "vulnerability_management_spec", "execution_time": 5, "system_output": null, "stack_trace": null}]}]}`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) ref := schemas.Ref{ Project: p, @@ -244,7 +244,7 @@ func TestGetRefPipelineFailedTestReport(t *testing.T) { fmt.Fprint(w, `{"total_time": 5, "total_count": 2, "success_count": 1, "failed_count": 1, "skipped_count": 0, "error_count": 0, "test_suites": [{"name": "Secure", "total_time": 5, "total_count": 2, "success_count": 1, "failed_count": 1, "skipped_count": 0, "error_count": 0, "test_cases": [{"status": "failed", "name": "Security Reports can create an auto-remediation MR", "classname": "vulnerability_management_spec", "execution_time": 5, "system_output": "Failed message", "stack_trace": null}]}]}`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) ref := schemas.Ref{ Project: p, @@ -320,7 +320,7 @@ func TestGetRefPipelineWithParentChildTestReport(t *testing.T) { fmt.Fprint(w, `[]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Project.Pull.Pipeline.TestReports.FromChildPipelines.Enabled = true @@ -415,7 +415,7 @@ func TestGetRefPipelineWithMultiProjectTestReport(t *testing.T) { fmt.Fprint(w, `[]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Project.Pull.Pipeline.TestReports.FromChildPipelines.Enabled = true @@ -498,7 +498,7 @@ func TestGetRefPipelineWithNoChildrenTestReport(t *testing.T) { fmt.Fprint(w, `[]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Project.Pull.Pipeline.TestReports.FromChildPipelines.Enabled = true diff --git a/pkg/gitlab/projects.go b/pkg/gitlab/projects.go index adc5f1d6..7a8f1c59 100644 --- a/pkg/gitlab/projects.go +++ b/pkg/gitlab/projects.go @@ -16,7 +16,7 @@ import ( ) // GetProject .. -func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project, error) { +func (c *Client) GetProject(ctx context.Context, name string) (schemas.Project, error) { ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProject") defer span.End() span.SetAttributes(attribute.String("project_name", name)) @@ -29,7 +29,7 @@ func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project p, resp, err := c.Projects.GetProject(name, &goGitlab.GetProjectOptions{}, goGitlab.WithContext(ctx)) c.requestsRemaining(resp) - return p, err + return schemas.NewProject(p.PathWithNamespace, p.Topics), err } // ListProjects .. @@ -134,7 +134,7 @@ func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas continue } - p := schemas.NewProject(gp.PathWithNamespace) + p := schemas.NewProject(gp.PathWithNamespace, gp.Topics) p.ProjectParameters = w.ProjectParameters projects = append(projects, p) } diff --git a/pkg/gitlab/projects_test.go b/pkg/gitlab/projects_test.go index 65613478..6b445fae 100644 --- a/pkg/gitlab/projects_test.go +++ b/pkg/gitlab/projects_test.go @@ -18,13 +18,14 @@ func TestGetProject(t *testing.T) { mux.HandleFunc("/api/v4/projects/foo%2Fbar", func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, r.Method, "GET") - _, _ = fmt.Fprint(w, `{"id":1}`) + _, _ = fmt.Fprint(w, `{"id":1,"topics":["foo","bar"]}`) }) p, err := c.GetProject(ctx, "foo/bar") assert.NoError(t, err) require.NotNil(t, p) - assert.Equal(t, 1, p.ID) + // assert.Equal(t, 1, p.ID) + assert.Equal(t, "foo,bar", p.Topics) } func TestListUserProjects(t *testing.T) { @@ -44,13 +45,14 @@ func TestListUserProjects(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/users/%s/projects", w.Owner.Name), func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, r.Method, "GET") - _, _ = fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar"},{"id":2,"path_with_namespace":"bar/baz"}]`) + _, _ = fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar", "topics":["foo","bar"]},{"id":2,"path_with_namespace":"bar/baz"}]`) }) projects, err := c.ListProjects(ctx, w) assert.NoError(t, err) assert.Len(t, projects, 1) assert.Equal(t, "foo/bar", projects[0].Name) + assert.Equal(t, "foo,bar", projects[0].Topics) } func TestListGroupProjects(t *testing.T) { diff --git a/pkg/gitlab/tags_test.go b/pkg/gitlab/tags_test.go index ffeb6346..f9826639 100644 --- a/pkg/gitlab/tags_test.go +++ b/pkg/gitlab/tags_test.go @@ -26,7 +26,7 @@ func TestGetProjectTags(t *testing.T) { fmt.Fprint(w, `[{"name":"foo"},{"name":"bar"}]`) }) - p := schemas.NewProject("foo") + p := schemas.NewProject("foo", []string{}) p.Pull.Refs.Tags.Regexp = `^f` expectedRef := schemas.NewRef(p, schemas.RefKindTag, "foo") diff --git a/pkg/monitor/server/server.go b/pkg/monitor/server/server.go index b342428c..91f3408d 100644 --- a/pkg/monitor/server/server.go +++ b/pkg/monitor/server/server.go @@ -171,6 +171,11 @@ func (s *Server) GetTelemetry(_ *pb.Empty, ts pb.Monitor_GetTelemetryServer) (er return } + if _, ok := s.taskSchedulingMonitoring[schemas.TaskTypePullProjects]; ok { + telemetry.Projects.LastPull = timestamppb.New(s.taskSchedulingMonitoring[schemas.TaskTypePullProjects].Last) + telemetry.Projects.NextPull = timestamppb.New(s.taskSchedulingMonitoring[schemas.TaskTypePullProjects].Next) + } + if _, ok := s.taskSchedulingMonitoring[schemas.TaskTypePullProjectsFromWildcards]; ok { telemetry.Projects.LastPull = timestamppb.New(s.taskSchedulingMonitoring[schemas.TaskTypePullProjectsFromWildcards].Last) telemetry.Projects.NextPull = timestamppb.New(s.taskSchedulingMonitoring[schemas.TaskTypePullProjectsFromWildcards].Next) diff --git a/pkg/schemas/projects.go b/pkg/schemas/projects.go index 66e9bd67..c50038dc 100644 --- a/pkg/schemas/projects.go +++ b/pkg/schemas/projects.go @@ -3,6 +3,7 @@ package schemas import ( "hash/crc32" "strconv" + "strings" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" ) @@ -26,6 +27,9 @@ func (p Project) Key() ProjectKey { } // NewProject .. -func NewProject(name string) Project { - return Project{Project: config.NewProject(name)} +func NewProject(name string, topics []string) Project { + return Project{ + Project: config.NewProject(name), + Topics: strings.Join(topics, ","), + } } diff --git a/pkg/schemas/projects_test.go b/pkg/schemas/projects_test.go index f75b0ff4..d0a69d5e 100644 --- a/pkg/schemas/projects_test.go +++ b/pkg/schemas/projects_test.go @@ -7,5 +7,5 @@ import ( ) func TestProjectKey(t *testing.T) { - assert.Equal(t, ProjectKey("2356372769"), NewProject("foo").Key()) + assert.Equal(t, ProjectKey("2356372769"), NewProject("foo", []string{}).Key()) } diff --git a/pkg/schemas/ref_test.go b/pkg/schemas/ref_test.go index 6607b938..4cb6ff2c 100644 --- a/pkg/schemas/ref_test.go +++ b/pkg/schemas/ref_test.go @@ -8,7 +8,7 @@ import ( func TestRefKey(t *testing.T) { assert.Equal(t, RefKey("1690074537"), NewRef( - NewProject("foo/bar"), + NewProject("foo/bar", []string{}), RefKindBranch, "baz", ).Key()) @@ -22,7 +22,7 @@ func TestRefsCount(t *testing.T) { } func TestRefDefaultLabelsValues(t *testing.T) { - p := NewProject("foo/bar") + p := NewProject("foo/bar", []string{}) p.Topics = "amazing,project" ref := Ref{ Project: p, @@ -48,8 +48,7 @@ func TestRefDefaultLabelsValues(t *testing.T) { } func TestNewRef(t *testing.T) { - p := NewProject("foo/bar") - p.Topics = "bar,baz" + p := NewProject("foo/bar", []string{"bar", "baz"}) p.OutputSparseStatusMetrics = false p.Pull.Pipeline.Jobs.Enabled = true p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = false diff --git a/pkg/schemas/tasks.go b/pkg/schemas/tasks.go index 1be9232c..9b25ce86 100644 --- a/pkg/schemas/tasks.go +++ b/pkg/schemas/tasks.go @@ -7,6 +7,9 @@ const ( // TaskTypePullProject .. TaskTypePullProject TaskType = "PullProject" + // TaskTypePullProject .. + TaskTypePullProjects TaskType = "PullProjects" + // TaskTypePullProjectsFromWildcard .. TaskTypePullProjectsFromWildcard TaskType = "PullProjectsFromWildcard" diff --git a/pkg/store/local_test.go b/pkg/store/local_test.go index aa4fcb1e..b753ff54 100644 --- a/pkg/store/local_test.go +++ b/pkg/store/local_test.go @@ -10,7 +10,7 @@ import ( ) func TestLocalProjectFunctions(t *testing.T) { - p := schemas.NewProject("foo/bar") + p := schemas.NewProject("foo/bar", []string{}) p.OutputSparseStatusMetrics = false l := NewLocalStore() @@ -28,7 +28,7 @@ func TestLocalProjectFunctions(t *testing.T) { assert.True(t, exists) // GetProject should succeed - newProject := schemas.NewProject("foo/bar") + newProject := schemas.NewProject("foo/bar", []string{}) assert.NoError(t, l.GetProject(testCtx, &newProject)) assert.Equal(t, p, newProject) @@ -48,7 +48,7 @@ func TestLocalProjectFunctions(t *testing.T) { assert.False(t, exists) // GetProject should not update the var this time - newProject = schemas.NewProject("foo/bar") + newProject = schemas.NewProject("foo/bar", []string{}) assert.NoError(t, l.GetProject(testCtx, &newProject)) assert.NotEqual(t, p, newProject) } @@ -107,7 +107,7 @@ func TestLocalEnvironmentFunctions(t *testing.T) { } func TestLocalRefFunctions(t *testing.T) { - p := schemas.NewProject("foo/bar") + p := schemas.NewProject("foo/bar", []string{}) p.Topics = "salty" ref := schemas.NewRef( p, @@ -131,7 +131,7 @@ func TestLocalRefFunctions(t *testing.T) { // GetRef should succeed newRef := schemas.Ref{ - Project: schemas.NewProject("foo/bar"), + Project: schemas.NewProject("foo/bar", []string{}), Kind: schemas.RefKindBranch, Name: "sweet", } @@ -156,7 +156,7 @@ func TestLocalRefFunctions(t *testing.T) { // GetRef should not update the var this time newRef = schemas.Ref{ Kind: schemas.RefKindBranch, - Project: schemas.NewProject("foo/bar"), + Project: schemas.NewProject("foo/bar", []string{}), Name: "sweet", } assert.NoError(t, l.GetRef(testCtx, &newRef)) diff --git a/pkg/store/redis_test.go b/pkg/store/redis_test.go index 3fadb93a..81a8e590 100644 --- a/pkg/store/redis_test.go +++ b/pkg/store/redis_test.go @@ -29,7 +29,7 @@ func newTestRedisStore(t *testing.T) (mr *miniredis.Miniredis, r Store) { func TestRedisProjectFunctions(t *testing.T) { _, r := newTestRedisStore(t) - p := schemas.NewProject("foo/bar") + p := schemas.NewProject("foo/bar", []string{}) p.OutputSparseStatusMetrics = false // Set project @@ -45,7 +45,7 @@ func TestRedisProjectFunctions(t *testing.T) { assert.True(t, exists) // GetProject should succeed - newProject := schemas.NewProject("foo/bar") + newProject := schemas.NewProject("foo/bar", []string{}) assert.NoError(t, r.GetProject(testCtx, &newProject)) assert.Equal(t, p, newProject) @@ -65,7 +65,7 @@ func TestRedisProjectFunctions(t *testing.T) { assert.False(t, exists) // GetProject should not update the var this time - newProject = schemas.NewProject("foo/bar") + newProject = schemas.NewProject("foo/bar", []string{}) assert.NoError(t, r.GetProject(testCtx, &newProject)) assert.NotEqual(t, p, newProject) } @@ -127,7 +127,7 @@ func TestRedisEnvironmentFunctions(t *testing.T) { func TestRedisRefFunctions(t *testing.T) { _, r := newTestRedisStore(t) - p := schemas.NewProject("foo/bar") + p := schemas.NewProject("foo/bar", []string{}) p.Topics = "salty" ref := schemas.NewRef( p, @@ -149,7 +149,7 @@ func TestRedisRefFunctions(t *testing.T) { // GetRef should succeed newRef := schemas.Ref{ - Project: schemas.NewProject("foo/bar"), + Project: schemas.NewProject("foo/bar", []string{}), Kind: schemas.RefKindBranch, Name: "sweet", } @@ -174,7 +174,7 @@ func TestRedisRefFunctions(t *testing.T) { // GetRef should not update the var this time newRef = schemas.Ref{ Kind: schemas.RefKindBranch, - Project: schemas.NewProject("foo/bar"), + Project: schemas.NewProject("foo/bar", []string{}), Name: "sweet", } assert.NoError(t, r.GetRef(testCtx, &newRef)) diff --git a/pkg/store/store.go b/pkg/store/store.go index 811096b5..59f652c8 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -4,10 +4,8 @@ import ( "context" "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" - "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" ) @@ -68,9 +66,8 @@ func NewRedisStore(client *redis.Client) Store { func New( ctx context.Context, r *redis.Client, - projects config.Projects, ) (s Store) { - ctx, span := otel.Tracer("gitlab-ci-pipelines-exporter").Start(ctx, "store:New") + _, span := otel.Tracer("gitlab-ci-pipelines-exporter").Start(ctx, "store:New") defer span.End() if r != nil { @@ -79,31 +76,5 @@ func New( s = NewLocalStore() } - // Load all the configured projects in the store - for _, p := range projects { - sp := schemas.Project{Project: p} - - exists, err := s.ProjectExists(ctx, sp.Key()) - if err != nil { - log.WithContext(ctx). - WithFields(log.Fields{ - "project-name": p.Name, - }). - WithError(err). - Error("reading project from the store") - } - - if !exists { - if err = s.SetProject(ctx, sp); err != nil { - log.WithContext(ctx). - WithFields(log.Fields{ - "project-name": p.Name, - }). - WithError(err). - Error("writing project in the store") - } - } - } - return } diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 61ff7014..d0f8e1ac 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -7,7 +7,6 @@ import ( "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" - "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" ) @@ -33,24 +32,12 @@ func TestNewRedisStore(t *testing.T) { } func TestNew(t *testing.T) { - localStore := New(testCtx, nil, config.Projects{}) + localStore := New(testCtx, nil) assert.IsType(t, &Local{}, localStore) redisClient := redis.NewClient(&redis.Options{}) - redisStore := New(testCtx, redisClient, config.Projects{}) + redisStore := New(testCtx, redisClient) assert.IsType(t, &Redis{}, redisStore) - localStore = New(testCtx, nil, config.Projects{ - { - Name: "foo", - }, - { - Name: "foo", - }, - { - Name: "bar", - }, - }) - count, _ := localStore.ProjectsCount(testCtx) - assert.Equal(t, int64(2), count) + New(testCtx, nil) }