From 9f60988e63f50e395422540e730e190199ccce08 Mon Sep 17 00:00:00 2001 From: Anton Kosiakov Date: Fri, 1 Dec 2017 08:18:21 +0100 Subject: [PATCH] [git] use native modules to find git repositories instead of constrainig by depth Signed-off-by: Anton Kosiakov --- .../application-package/src/rebuild.ts | 2 +- packages/git/package.json | 3 +- .../src/browser/git-repository-provider.ts | 3 +- packages/git/src/common/git-preferences.ts | 10 +- packages/git/src/common/git.ts | 5 - packages/git/src/node/dugite-git.spec.ts | 8 +- packages/git/src/node/dugite-git.ts | 1 - .../src/node/git-locator/git-locator-impl.ts | 151 ++++++++++-------- .../node/git-locator/git-locator-protocol.ts | 1 - yarn.lock | 14 +- 10 files changed, 102 insertions(+), 96 deletions(-) diff --git a/dev-packages/application-package/src/rebuild.ts b/dev-packages/application-package/src/rebuild.ts index c546f6d78fb74..81d5323baa7cd 100644 --- a/dev-packages/application-package/src/rebuild.ts +++ b/dev-packages/application-package/src/rebuild.ts @@ -12,7 +12,7 @@ import cp = require('child_process'); export function rebuild(target: 'electron' | 'browser', modules: string[]) { const nodeModulesPath = path.join(process.cwd(), 'node_modules'); const browserModulesPath = path.join(process.cwd(), '.browser_modules'); - const modulesToProcess = modules || ['node-pty', 'nsfw']; + const modulesToProcess = modules || ['node-pty', 'nsfw', 'find-git-repositories']; if (target === 'electron' && !fs.existsSync(browserModulesPath)) { const dependencies: { diff --git a/packages/git/package.json b/packages/git/package.json index a798107a459b8..ca7126c84749b 100644 --- a/packages/git/package.json +++ b/packages/git/package.json @@ -9,8 +9,7 @@ "@theia/preferences-api": "^0.2.3", "@theia/workspace": "^0.2.3", "dugite-extra": "0.0.1-alpha.15", - "findit2": "^2.2.3", - "ignore": "^3.3.7" + "find-git-repositories": "^0.1.0" }, "publishConfig": { "access": "public" diff --git a/packages/git/src/browser/git-repository-provider.ts b/packages/git/src/browser/git-repository-provider.ts index c892b586758a9..7fd4ee9d2c97b 100644 --- a/packages/git/src/browser/git-repository-provider.ts +++ b/packages/git/src/browser/git-repository-provider.ts @@ -71,8 +71,7 @@ export class GitRepositoryProvider { return; } const repositories = await this.git.repositories(root.uri, { - ...options, - maxDepth: this.preferences['git.locateMaxDepth'] + ...options }); this._allRepositories = repositories; const selectedRepository = this._selectedRepository; diff --git a/packages/git/src/common/git-preferences.ts b/packages/git/src/common/git-preferences.ts index e58225eaa5147..a542f4fbe013c 100644 --- a/packages/git/src/common/git-preferences.ts +++ b/packages/git/src/common/git-preferences.ts @@ -10,22 +10,14 @@ import { createPreferenceProxy, PreferenceProxy, PreferenceService } from '@thei import { PreferenceSchema } from '@theia/preferences-api/lib/common/'; export interface GitConfiguration { - 'git.locateMaxDepth': number } const DefaultGitConfiguration: GitConfiguration = { - "git.locateMaxDepth": 4 }; export const GitPreferenceSchema: PreferenceSchema = { 'type': 'object', - 'properties': { - "git.locateMaxDepth": { - "type": "number", - "default": 4, - "description": "Configure the max depth of the git repository look up" - } - } + 'properties': {} }; export const GitPreferences = Symbol('GitPreferences'); diff --git a/packages/git/src/common/git.ts b/packages/git/src/common/git.ts index 93d6db9c8c3a8..2002f040534a6 100644 --- a/packages/git/src/common/git.ts +++ b/packages/git/src/common/git.ts @@ -141,11 +141,6 @@ export namespace Git { */ readonly maxCount?: number; - /** - * The max depth of repositories look up. - */ - readonly maxDepth: number; - } /** diff --git a/packages/git/src/node/dugite-git.spec.ts b/packages/git/src/node/dugite-git.spec.ts index 6b36fe68fa668..8151b9deff8ac 100644 --- a/packages/git/src/node/dugite-git.spec.ts +++ b/packages/git/src/node/dugite-git.spec.ts @@ -47,7 +47,7 @@ describe('git', async function () { const git = await createGit(); const workspace = await createWorkspace(root); const workspaceRootUri = await workspace.getRoot(); - const repositories = await git.repositories(workspaceRootUri!, { maxDepth: 1, maxCount: 1 }); + const repositories = await git.repositories(workspaceRootUri!, { maxCount: 1 }); expect(repositories.length).to.deep.equal(1); }); @@ -64,7 +64,7 @@ describe('git', async function () { const git = await createGit(); const workspace = await createWorkspace(root); const workspaceRootUri = await workspace.getRoot(); - const repositories = await git.repositories(workspaceRootUri!, { maxDepth: 1 }); + const repositories = await git.repositories(workspaceRootUri!, {}); expect(repositories.map(r => path.basename(FileUri.fsPath(r.localUri))).sort()).to.deep.equal(['A', 'B', 'C']); }); @@ -83,7 +83,7 @@ describe('git', async function () { const git = await createGit(); const workspace = await createWorkspace(path.join(root, 'BASE')); const workspaceRootUri = await workspace.getRoot(); - const repositories = await git.repositories(workspaceRootUri!, { maxDepth: 2 }); + const repositories = await git.repositories(workspaceRootUri!, {}); expect(repositories.map(r => path.basename(FileUri.fsPath(r.localUri))).sort()).to.deep.equal(['A', 'B', 'BASE', 'C']); }); @@ -103,7 +103,7 @@ describe('git', async function () { const git = await createGit(); const workspace = await createWorkspace(path.join(root, 'BASE', 'WS_ROOT')); const workspaceRootUri = await workspace.getRoot(); - const repositories = await git.repositories(workspaceRootUri!, { maxDepth: 2 }); + const repositories = await git.repositories(workspaceRootUri!, {}); const repositoryNames = repositories.map(r => path.basename(FileUri.fsPath(r.localUri))); expect(repositoryNames.shift()).to.equal('BASE'); // The first must be the container repository. expect(repositoryNames.sort()).to.deep.equal(['A', 'B', 'C']); diff --git a/packages/git/src/node/dugite-git.ts b/packages/git/src/node/dugite-git.ts index fc90b2c35ac97..e09612f499b36 100644 --- a/packages/git/src/node/dugite-git.ts +++ b/packages/git/src/node/dugite-git.ts @@ -69,7 +69,6 @@ export class DugiteGit implements Git { return repositories; } for (const repositoryPath of await this.locator.locate(workspaceRootPath, { - ...options, maxCount })) { if (containingPath !== repositoryPath) { diff --git a/packages/git/src/node/git-locator/git-locator-impl.ts b/packages/git/src/node/git-locator/git-locator-impl.ts index e128c59dd13c3..23721418f4453 100644 --- a/packages/git/src/node/git-locator/git-locator-impl.ts +++ b/packages/git/src/node/git-locator/git-locator-impl.ts @@ -5,93 +5,118 @@ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 */ -import * as fs from 'fs'; -import * as paths from 'path'; +import * as fs from 'fs-extra'; +import * as path from 'path'; import { GitLocator, GitLocateOptions } from './git-locator-protocol'; -// tslint:disable:no-console +export type FindGitRepositories = (path: string, progressCb: (repos: string[]) => void) => Promise; +const findGitRepositories: FindGitRepositories = require('find-git-repositories'); -const ignore: () => IgnoreFilter = require('ignore'); -const finder: (path: string) => FindIt = require('findit2'); - -export interface IgnoreFilter { - add(patterns: string | IgnoreFilter): void - ignores(pathname: string): boolean; - depth: number; -} - -export interface FindIt extends NodeJS.EventEmitter { - stop(): void; +export interface GitLocateContext { + maxCount: number + readonly visited: Map } export class GitLocatorImpl implements GitLocator { - protected disposed = false; - dispose(): void { - this.disposed = true; } - locate(path: string, options: GitLocateOptions): Promise { - if (this.disposed) { + async locate(basePath: string, options: GitLocateOptions): Promise { + return await this.doLocate(basePath, { + maxCount: typeof options.maxCount === 'number' ? options.maxCount : -1, + visited: new Map() + }); + } + + protected doLocate(basePath: string, context: GitLocateContext): Promise { + const realBasePath = fs.realpathSync(basePath); + if (context.visited.has(realBasePath)) { return Promise.resolve([]); } - return new Promise(resolve => { - const filters = new Map(); - const repositoryPaths = new Set(); - const emitter = finder(path); - emitter.on('directory', (dir: string, stat: fs.Stats, stop: () => void) => { - const base = paths.basename(dir); - if (base === '.git') { - const dirName = paths.dirname(dir); - try { - const resolvedPath = fs.realpathSync(dirName); - repositoryPaths.add(resolvedPath); - } catch (e) { - console.error(e); + context.visited.set(realBasePath, true); + return new Promise(resolve => { + fs.stat(realBasePath).then(async stat => { + if (stat.isDirectory) { + const progress: string[] = []; + const paths = await findGitRepositories(realBasePath, repositories => { + progress.push(...repositories); + if (context.maxCount >= 0 && progress.length >= context.maxCount) { + resolve(progress.slice(0, context.maxCount).map(GitLocatorImpl.map)); + } + }); + if (context.maxCount >= 0 && paths.length >= context.maxCount) { + resolve(paths.slice(0, context.maxCount).map(GitLocatorImpl.map)); + return; } - if (!!options.maxCount && repositoryPaths.size >= options.maxCount) { - emitter.stop(); - } - stop(); - return; + const repositoryPaths = paths.map(GitLocatorImpl.map); + resolve(this.locateFrom( + newContext => this.generateNested(repositoryPaths, newContext), + context, + repositoryPaths + )); } - if (this.shouldStop(dir, filters, options)) { - stop(); - return; + }, () => []); + }); + } + + protected * generateNested(repositoryPaths: string[], context: GitLocateContext) { + for (const repository of repositoryPaths) { + yield this.locateNested(repository, context); + } + } + protected locateNested(repositoryPath: string, context: GitLocateContext): Promise { + return new Promise(resolve => { + fs.readdir(repositoryPath, async (err, files) => { + if (err) { + console.error(err); + resolve([]); + } else { + resolve(this.locateFrom( + newContext => this.generateRepositories(repositoryPath, files, newContext), + context + )); } - filters.set(dir, this.createFilter(dir, filters)); }); - const complete = () => resolve([...repositoryPaths]); - emitter.once('end', complete); - emitter.once('stop', complete); - emitter.on('error', error => console.error(error)); }); } - protected createFilter(path: string, filters: Map): IgnoreFilter { - const ig = ignore(); - const parent = filters.get(paths.dirname(path)); - ig.depth = parent ? parent.depth + 1 : 0; - if (parent && !fs.existsSync(paths.join(path, '.git'))) { - ig.add(parent); + protected * generateRepositories(repositoryPath: string, files: string[], context: GitLocateContext) { + for (const file of files) { + if (file !== '.git') { + yield this.doLocate(path.join(repositoryPath, file), { + ...context + }); + } } - if (fs.existsSync(paths.join(path, '.gitignore'))) { - ig.add(fs.readFileSync(paths.join(path, '.gitignore')).toString()); - } - return ig; } - protected shouldStop(pathname: string, filters: Map, options: GitLocateOptions): boolean { - if (this.disposed) { - return true; + protected async locateFrom( + generator: (context: GitLocateContext) => IterableIterator>, parentContext: GitLocateContext, initial?: string[] + ): Promise { + const result: string[] = []; + if (initial) { + result.push(...initial); } - const parent = paths.dirname(pathname); - const ig = filters.get(parent); - if (!ig) { - return false; + const context = { + ...parentContext, + maxCount: parentContext.maxCount - result.length + }; + for (const locateRepositories of generator(context)) { + const repositories = await locateRepositories; + result.push(...repositories); + if (context.maxCount >= 0) { + if (result.length >= context.maxCount) { + return result.slice(0, context.maxCount); + } + context.maxCount -= repositories.length; + } } - return ig.depth >= options.maxDepth || ig.ignores(pathname); + return result; + } + + static map(repository: string): string { + return fs.realpathSync(path.dirname(repository)); } } diff --git a/packages/git/src/node/git-locator/git-locator-protocol.ts b/packages/git/src/node/git-locator/git-locator-protocol.ts index 0938bb4b0a71a..cb205eda45d09 100644 --- a/packages/git/src/node/git-locator/git-locator-protocol.ts +++ b/packages/git/src/node/git-locator/git-locator-protocol.ts @@ -9,7 +9,6 @@ import { Disposable } from "vscode-jsonrpc"; export interface GitLocateOptions { readonly maxCount?: number; - readonly maxDepth: number; } export const GitLocator = Symbol('GitLocator'); diff --git a/yarn.lock b/yarn.lock index 11581f1d237bc..8188c9105c5fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2413,6 +2413,12 @@ find-git-exec@0.0.1-alpha.2: dependencies: "@types/node" "^8.0.26" +find-git-repositories@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/find-git-repositories/-/find-git-repositories-0.1.0.tgz#1ac886f0f54a11f5f073bca3bcdfddc03486305a" + dependencies: + nan "^2.0.0" + find-index@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/find-index/-/find-index-0.1.1.tgz#675d358b2ca3892d795a1ab47232f8b6e2e0dde4" @@ -2430,10 +2436,6 @@ find-up@^2.0.0, find-up@^2.1.0: dependencies: locate-path "^2.0.0" -findit2@^2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" - findup-sync@^0.4.2: version "0.4.3" resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-0.4.3.tgz#40043929e7bc60adf0b7f4827c4c6e75a0deca12" @@ -3247,10 +3249,6 @@ ignore-loader@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/ignore-loader/-/ignore-loader-0.1.2.tgz#d81f240376d0ba4f0d778972c3ad25874117a463" -ignore@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.7.tgz#612289bfb3c220e186a58118618d5be8c1bab021" - image-size@~0.5.0: version "0.5.5" resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"