diff --git a/configs/filter.json b/configs/filter.json index 1e31d524..ca6bd283 100644 --- a/configs/filter.json +++ b/configs/filter.json @@ -15,7 +15,9 @@ "forks": true, "repos": [ "my-org/repo1", - "my-org/repo2" + "my-org/repo2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, @@ -34,7 +36,9 @@ "forks": true, "projects": [ "my-group/project1", - "my-group/project2" + "my-group/project2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, @@ -53,7 +57,9 @@ "forks": true, "repos": [ "my-org/repo1", - "my-org/repo2" + "my-org/repo2", + "my-org/sub-org-1/**", + "my-org/sub-org-*/**" ] } }, diff --git a/packages/backend/src/utils.test.ts b/packages/backend/src/utils.test.ts index 84d77f21..76c14067 100644 --- a/packages/backend/src/utils.test.ts +++ b/packages/backend/src/utils.test.ts @@ -1,5 +1,62 @@ import { expect, test } from 'vitest'; -import { arraysEqualShallow, isRemotePath } from './utils'; +import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils'; +import { Repository } from './types'; + +const testNames: string[] = [ + "abcdefg/zfmno/ioiwerj/fawdf", + "abcdefg/zfmno/ioiwerj/werw", + "abcdefg/zfmno/ioiwerj/terne", + "abcdefg/zfmno/ioiwerj/asdf45e4r", + "abcdefg/zfmno/ioiwerj/ddee", + "abcdefg/zfmno/ioiwerj/ccdfeee", + "abcdefg/zfmno/sadfaw", + "abcdefg/zfmno/ioiwerj/wwe", + "abcdefg/ieieiowowieu8383/ieckup-e", + "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3" +]; + +const createRepository = (name: string) => ({ + vcs: 'git', + id: name, + name: name, + path: name, + isStale: false, + cloneUrl: name, + branches: [name], + tags: [name] +}); + +test('should filter repos by micromatch pattern', () => { + // bad glob patterns + const unfilteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['/zfmno/']); + expect(unfilteredRepos.length).toBe(10); + expect(unfilteredRepos.map(r => r.name)).toEqual(testNames); + const unfilteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**zfmno**']); + expect(unfilteredRepos1.length).toBe(10); + expect(unfilteredRepos1.map(r => r.name)).toEqual(testNames); + + // good glob patterns + const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/zfmno/**']); + expect(filteredRepos.length).toBe(2); + expect(filteredRepos.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]); + const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), ['**/*fmn*/**']); + expect(filteredRepos1.length).toBe(2); + expect(filteredRepos1.map(r => r.name)).toEqual(["abcdefg/ieieiowowieu8383/ieckup-e", "abcdefg/ieieiowowieu8383/fvas-eer-wwwer3"]); +}); + +test('should filter repos by name exact match', () => { + const filteredRepos = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(1, 9)); + expect(filteredRepos.length).toBe(2); + expect(filteredRepos.map(r => r.name)).toEqual([testNames[0], testNames[9]]); + + const filteredRepos1 = excludeReposByName(testNames.map(n => (createRepository(n))), testNames.slice(3, 5)); + expect(filteredRepos1.length).toBe(8); + expect(filteredRepos1.map(r => r.name)).toEqual([testNames[0], testNames[1], testNames[2], testNames[5], testNames[6], testNames[7], testNames[8], testNames[9]]); + + const filteredRepos2 = excludeReposByName(testNames.map(n => (createRepository(n))), [testNames[0], testNames[7], testNames[9]]); + expect(filteredRepos2.length).toBe(7); + expect(filteredRepos2.map(r => r.name)).toEqual([...testNames.slice(1, 7), testNames[8]]); +}); test('should return true for identical arrays', () => { expect(arraysEqualShallow([1, 2, 3], [1, 2, 3])).toBe(true); diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 8ac58d78..8996dd1e 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,6 +1,7 @@ import { Logger } from "winston"; import { AppContext, Repository } from "./types.js"; import path from 'path'; +import micromatch from "micromatch"; export const measure = async (cb : () => Promise) => { const start = Date.now(); @@ -36,10 +37,10 @@ export const excludeArchivedRepos = (repos: T[], logger?: }); } + export const excludeReposByName = (repos: T[], excludedRepoNames: string[], logger?: Logger) => { - const excludedRepos = new Set(excludedRepoNames); return repos.filter((repo) => { - if (excludedRepos.has(repo.name)) { + if (micromatch.isMatch(repo.name, excludedRepoNames)) { logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.repos contains ${repo.name}`); return false; } @@ -90,4 +91,4 @@ export const arraysEqualShallow = (a?: readonly T[], b?: readonly T[]) => { } return true; -} \ No newline at end of file +} diff --git a/schemas/v2/index.json b/schemas/v2/index.json index 256fd3cd..e71058d2 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -146,11 +146,10 @@ "repos": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "type": "string" }, "default": [], - "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." } }, "additionalProperties": false @@ -238,8 +237,7 @@ "projects": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "type": "string" }, "default": [], "examples": [ @@ -247,7 +245,7 @@ "my-group/my-project" ] ], - "description": "List of individual projects to exclude from syncing. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" + "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/" } }, "additionalProperties": false @@ -336,11 +334,10 @@ "repos": { "type": "array", "items": { - "type": "string", - "pattern": "^[\\w.-]+\\/[\\w.-]+$" + "type": "string" }, "default": [], - "description": "List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." } }, "additionalProperties": false @@ -432,4 +429,4 @@ } }, "additionalProperties": false -} \ No newline at end of file +}