Automate SDK regeneration (Azure#212)

* Add Version class

* Bootstrap packages regeneration

* Fix missing closing brace

* Improve branch handling

* Working version

* Fix branch creation

* Change commit story

* Add version skipping

.scripts/git.ts

@@ -4,17 +4,54 @@
  * license information.
  */
 
-import { Repository, Signature, Merge, Oid, Reference, Cred, StatusFile } from "nodegit";
+import { Repository, Signature, Merge, Oid, Reference, Cred, StatusFile, Reset, Index } from "nodegit";
 import { getLogger } from "./logger";
 import { getCommandLineOptions } from "./commandLine";
 
 export type ValidateFunction = (statuses: StatusFile[]) => boolean;
-export type ValidateEachFunction = (value: StatusFile, index: number, array: StatusFile[]) => boolean;
+export type ValidateEachFunction = (path: string, matchedPatter: string) => number;
+
+export enum BranchLocation {
+    Local = "heads",
+    Remote = "remotes"
+}
+
+export class Branch {
+    static LocalMaster = new Branch("master", BranchLocation.Local);
+    static RemoteMaster = new Branch("master", BranchLocation.Remote);
+
+    constructor(public name: string, public location: BranchLocation, public remote: string = "origin") {
+    }
+
+    shorthand(): string {
+        return `${this.remote}/${this.name}`;
+    }
+
+    fullName(): string {
+        if (this.name.startsWith("refs")) {
+            return this.name;
+        }
+
+        return `refs/${this.location}/${this.remote}/${this.name}`;
+    }
+
+    fullNameWithoutRemote(): string {
+        if (this.name.startsWith("refs")) {
+            return this.name;
+        }
+
+        return `refs/${this.location}/${this.name}`;
+    }
+
+    convertTo(location: BranchLocation): Branch {
+        return new Branch(this.name, location, this.remote);
+    }
+}
 
 const _args = getCommandLineOptions();
 const _logger = getLogger();
 
-const _lockMap = { }
+const _lockMap = {}
 
 function isLocked(repositoryPath: string) {
     const isLocked = _lockMap[repositoryPath];
@@ -86,28 +123,39 @@ export async function validateRepositoryStatus(repository: Repository): Promise<
 export async function getValidatedRepository(repositoryPath: string): Promise<Repository> {
     const repository = await openRepository(repositoryPath);
     await validateRepositoryStatus(repository);
+    await repository.fetchAll();
     return repository;
 }
 
-export async function pull(repository: Repository, branchName: string, origin: string = "origin"): Promise<Oid> {
-    _logger.logTrace(`Pulling "${branchName}" branch from ${origin} origin in ${repository.path()} repository`);
+export async function mergeBranch(repository: Repository, toBranch: Branch, fromBranch: Branch): Promise<Oid> {
+    _logger.logTrace(`Merging "${fromBranch.fullName()}" to "${toBranch.fullName()}" branch in ${repository.path()} repository`);
+    return repository.mergeBranches(toBranch.name, fromBranch.shorthand(), Signature.default(repository), Merge.PREFERENCE.NONE);
+}
+
+export async function mergeMasterIntoBranch(repository: Repository, toBranch: Branch): Promise<Oid> {
+    return mergeBranch(repository, toBranch, Branch.RemoteMaster);
+}
+
+export async function pullBranch(repository: Repository, localBranch: Branch): Promise<Oid> {
+    _logger.logTrace(`Pulling "${localBranch.fullName()}" branch in ${repository.path()} repository`);
 
     await repository.fetchAll();
     _logger.logTrace(`Fetched all successfully`);
 
-    const oid = await repository.mergeBranches(branchName, `${origin}/${branchName}`, Signature.default(repository), Merge.PREFERENCE.NONE);
+    const remoteBranch = new Branch(localBranch.name, BranchLocation.Remote, localBranch.remote);
+    await mergeBranch(repository, localBranch, remoteBranch);
 
     const index = await repository.index();
     if (index.hasConflicts()) {
-        throw new Error(`Conflict while pulling ${branchName} from origin.`);
+        throw new Error(`Conflict while pulling ${remoteBranch.fullName()}`);
     }
 
-    _logger.logTrace(`Merged "${origin}/${branchName}" to "${branchName}" successfully without any conflicts`);
-    return oid;
+    _logger.logTrace(`Merged "${remoteBranch.fullName()}" to "${localBranch.fullName()}" successfully without any conflicts`);
+    return undefined;
 }
 
 export async function pullMaster(repository: Repository): Promise<Oid> {
-    return pull(repository, "master");
+    return pullBranch(repository, Branch.LocalMaster);
 }
 
 export async function createNewBranch(repository: Repository, branchName: string, checkout?: boolean): Promise<Reference> {
@@ -121,10 +169,31 @@ export async function createNewBranch(repository: Repository, branchName: string
         return branchPromise;
     } else {
         const branch = await branchPromise;
-        return checkoutBranch(repository, branch.name());
+        return checkoutBranch(repository, branch.shorthand());
     }
 }
 
+export async function checkoutRemoteBranch(repository: Repository, remoteBranch: Branch): Promise<Reference> {
+    _logger.logTrace(`Checking out "${remoteBranch.fullName()}" remote branch`);
+
+    const branchNames = await repository.getReferenceNames(Reference.TYPE.LISTALL);
+    const localBranch = remoteBranch.convertTo(BranchLocation.Local);
+    const branchExists = branchNames.some(name => name === localBranch.fullNameWithoutRemote());
+    _logger.logTrace(`Branch exists: ${branchExists}`);
+
+    let branchRef: Reference;
+    if (branchExists) {
+        branchRef = await checkoutBranch(repository, remoteBranch.name);
+    } else {
+        branchRef = await createNewBranch(repository, remoteBranch.name, true);
+        const commit = await repository.getReferenceCommit(remoteBranch.name);
+        await Reset.reset(repository, commit as any, Reset.TYPE.HARD, {});
+        await pullBranch(repository, remoteBranch.convertTo(BranchLocation.Local));
+    }
+
+    return branchRef;
+}
+
 function getCurrentDateSuffix(): string {
     const now = new Date();
     return `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}-${now.getMilliseconds()}`;
@@ -135,7 +204,7 @@ export async function createNewUniqueBranch(repository: Repository, branchPrefix
 }
 
 export async function checkoutBranch(repository: Repository, branchName: string | Reference): Promise<Reference> {
-    _logger.logTrace(`Checking out ${branchName} branch`);
+    _logger.logTrace(`Checking out "${branchName}" branch`);
     return repository.checkoutBranch(branchName);
 }
 
@@ -148,26 +217,45 @@ export async function refreshRepository(repository: Repository) {
     return checkoutMaster(repository);
 }
 
-export async function commitSpecificationChanges(repository: Repository, commitMessage: string, validate?: ValidateFunction, validateEach?: ValidateEachFunction): Promise<Oid> {
+export async function commitChanges(repository: Repository, commitMessage: string, validateStatus?: ValidateFunction, validateEach?: string | ValidateEachFunction): Promise<Oid> {
     _logger.logTrace(`Committing changes in "${repository.path()}" repository`);
 
-    const emptyValidate = () => true;
-    validate = validate || emptyValidate;
-    validateEach = validateEach || emptyValidate;
+    validateStatus = validateStatus || ((_) => true);
+    validateEach = validateEach || ((_, __) => 0);
 
     const status = await repository.getStatus();
-
-    if (validate(status) && status.every(validateEach)) {
-        var author = Signature.default(repository);
-        return repository.createCommitOnHead(status.map(el => el.path()), author, author, commitMessage);
-    } else {
+    if (!validateStatus(status)) {
         return Promise.reject("Unknown changes present in the repository");
     }
+
+    const index = await repository.refreshIndex();
+    if (typeof validateEach === "string") {
+        const folderName = validateEach;
+        validateEach = (path, pattern) => {
+             return path.startsWith(folderName) ? 0 : 1;
+        }
+    }
+
+    await index.addAll("*", Index.ADD_OPTION.ADD_CHECK_PATHSPEC, validateEach);
+
+    const entries = index.entries();
+    _logger.logTrace(`Files added to the index ${index.entryCount}: ${JSON.stringify(entries)}`)
+
+    await index.write();
+    const oid = await index.writeTree();
+
+    const head = await repository.getHeadCommit();
+    const author = Signature.default(repository);
+
+    return repository.createCommit("HEAD", author, author, commitMessage, oid, [head]);
 }
 
-export async function pushToNewBranch(repository: Repository, branchName: string): Promise<number> {
+export async function pushBranch(repository: Repository, localBranch: Branch): Promise<number> {
     const remote = await repository.getRemote("origin");
-    return remote.push([`${branchName}:${branchName}`], {
+    const refSpec = `refs/heads/${localBranch.name}:refs/heads/${localBranch.name}`;
+    _logger.logTrace(`Pushing to ${refSpec}`);
+
+    return remote.push([refSpec], {
         callbacks: {
             credentials: function (url, userName) {
                 return Cred.userpassPlaintextNew(getToken(), "x-oauth-basic");
@@ -176,6 +264,11 @@ export async function pushToNewBranch(repository: Repository, branchName: string
     });
 }
 
+export async function commitAndPush(repository: Repository, localBranch: Branch, commitMessage: string, validate?: ValidateFunction, validateEach?: string | ValidateEachFunction) {
+    await commitChanges(repository, commitMessage, validate, validateEach);
+    await pushBranch(repository, localBranch);
+}
+
 export function getToken(): string {
     const token = _args.token || process.env.SDK_GEN_GITHUB_TOKEN;
     _validatePersonalAccessToken(token);
@@ -186,7 +279,7 @@ export function getToken(): string {
 function _validatePersonalAccessToken(token: string): void {
     if (!token) {
         const text =
-        `Github personal access token was not found as a script parameter or as an
+            `Github personal access token was not found as a script parameter or as an
         environmental variable. Please visit https://github.com/settings/tokens,
         generate new token with "repo" scope and pass it with -token switch or set
         it as environmental variable named SDK_GEN_GITHUB_TOKEN.`

.scripts/github.ts

@@ -6,7 +6,7 @@
 
 import * as Octokit from '@octokit/rest'
 import { PullRequestsCreateParams, Response, PullRequestsCreateReviewRequestParams, PullRequestsCreateReviewRequestResponse } from '@octokit/rest';
-import { getToken, createNewUniqueBranch, commitSpecificationChanges, pushToNewBranch, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from './git';
+import { getToken, createNewUniqueBranch, commitChanges, pushBranch,ValidateFunction, ValidateEachFunction, Branch, BranchLocation } from './git';
 import { getLogger } from './logger';
 import { Repository } from 'nodegit';
 
@@ -69,17 +69,18 @@ export async function commitAndCreatePullRequest(
     pullRequestTitle: string,
     pullRequestDescription:string,
     validate?: ValidateFunction,
-    validateEach?: ValidateEachFunction): Promise<string> {
+    validateEach?: string | ValidateEachFunction): Promise<string> {
     await createNewUniqueBranch(repository, `generated/${packageName}`, true);
 
-    await commitSpecificationChanges(repository, commitMessage, validate, validateEach);
-    const newBranch = await repository.getCurrentBranch();
-    _logger.logInfo(`Committed changes successfully on ${newBranch.name()} branch`);
+    await commitChanges(repository, commitMessage, validate, validateEach);
+    const newBranchRef = await repository.getCurrentBranch();
+    const newBranch = new Branch(newBranchRef.name(), BranchLocation.Local);
+    _logger.logInfo(`Committed changes successfully on ${newBranch.name} branch`);
 
-    await pushToNewBranch(repository, newBranch.name());
-    _logger.logInfo(`Pushed changes successfully to ${newBranch.name()} branch`);
+    await pushBranch(repository, newBranch);
+    _logger.logInfo(`Pushed changes successfully to ${newBranch.name} branch`);
 
-    const pullRequestResponse = await createPullRequest(repositoryName, pullRequestTitle, pullRequestDescription, newBranch.name());
+    const pullRequestResponse = await createPullRequest(repositoryName, pullRequestTitle, pullRequestDescription, newBranchRef.name());
     _logger.logInfo(`Created pull request successfully - ${pullRequestResponse.data.html_url}`);
 
     const reviewResponse = await requestPullRequestReview(repositoryName, pullRequestResponse.data.number);

.scripts/gulp.ts

@@ -9,10 +9,11 @@ import { findAzureRestApiSpecsRepositoryPath, findSdkDirectory, saveContentToFil
 import { copyExistingNodeJsReadme, updateTypeScriptReadmeFile, findReadmeTypeScriptMdFilePaths, getPackageNamesFromReadmeTypeScriptMdFileContents, getAbsolutePackageFolderPathFromReadmeFileContents, updateMainReadmeFile, getSinglePackageName } from "./readme";
 import * as fs from "fs";
 import * as path from "path";
+import { Version } from "./version";
 import { contains, npmInstall } from "./common";
 import { execSync } from "child_process";
 import { getLogger } from "./logger";
-import { refreshRepository, getValidatedRepository, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from "./git";
+import { refreshRepository, getValidatedRepository, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction, checkoutBranch, pullBranch, mergeBranch, mergeMasterIntoBranch, commitAndPush, checkoutRemoteBranch, Branch, BranchLocation } from "./git";
 import { commitAndCreatePullRequest } from "./github";
 
 const _logger = getLogger();
@@ -118,9 +119,8 @@ export async function generateTsReadme(packageName: string, sdkType: SdkType): P
     const pullRequestTitle = `Add ${packageName}/${sdkType}/readme.typescript.md`;
     const pullRequestDescription = "Autogenerated";
     const validate: ValidateFunction = statuses => statuses.length == 2;
-    const validateEach: ValidateEachFunction = el => el.path().startsWith(`specification/${packageName}`);
 
-    const pullRequestUrl = await commitAndCreatePullRequest(azureRestApiSpecRepository, packageName, pullRequestTitle, "azure-rest-api-specs", pullRequestTitle, pullRequestDescription, validate, validateEach);
+    const pullRequestUrl = await commitAndCreatePullRequest(azureRestApiSpecRepository, packageName, pullRequestTitle, "azure-rest-api-specs", pullRequestTitle, pullRequestDescription, validate, `specification/${packageName}`);
     await unlockGitRepository(azureRestApiSpecRepository);
 
     return { pullRequestUrl: pullRequestUrl, typescriptReadmePath: typescriptReadmePath };
@@ -143,7 +143,7 @@ export async function generateMissingSdk(azureSdkForJsRepoPath: string, packageN
 
     const azureSdkForJsRepository = await getValidatedRepository(azureSdkForJsRepoPath);
     await refreshRepository(azureSdkForJsRepository);
-    _logger.log(`Refreshed ${azureRestApiSpecsRepositoryPath} repository successfully`);
+    _logger.log(`Refreshed ${azureSdkForJsRepoPath} repository successfully`);
 
     await waitAndLockGitRepository(azureSdkForJsRepository);
     await generateSdk(azureRestApiSpecsRepositoryPath, azureSdkForJsRepoPath, packageName);
@@ -157,9 +157,8 @@ ${_logger.getCapturedText()}
 \`\`\``
 
     const validate: ValidateFunction = changes => changes.length > 0;
-    const validateEach: ValidateEachFunction = el => el.path().startsWith(`packages/${packageName}`);
 
-    const pullRequestUrl = await commitAndCreatePullRequest(azureSdkForJsRepository, packageName, pullRequestTitle, "azure-sdk-for-js", pullRequestTitle, pullRequestDescription, validate, validateEach);
+    const pullRequestUrl = await commitAndCreatePullRequest(azureSdkForJsRepository, packageName, pullRequestTitle, "azure-sdk-for-js", pullRequestTitle, pullRequestDescription, validate, `packages/${packageName}`);
     await unlockGitRepository(azureSdkForJsRepository);
 
     return pullRequestUrl;
@@ -178,3 +177,44 @@ export async function generateAllMissingSdks(azureSdkForJsRepoPath: string, azur
         }
     }
 }
+
+export async function regenerate(branchName: string, packageName: string, azureSdkForJsRepoPath: string, azureRestAPISpecsPath: string, skipVersionBump?: boolean) {
+    const azureSdkForJsRepository = await getValidatedRepository(azureSdkForJsRepoPath);
+    await refreshRepository(azureSdkForJsRepository);
+    _logger.log(`Refreshed ${azureSdkForJsRepository.path()} repository successfully`);
+
+    const remoteBranch = new Branch(branchName, BranchLocation.Remote);
+    await checkoutRemoteBranch(azureSdkForJsRepository, remoteBranch);
+    _logger.log(`Checked out ${branchName} branch`);
+
+    const localBranch = remoteBranch.convertTo(BranchLocation.Local);
+    await mergeMasterIntoBranch(azureSdkForJsRepository, localBranch);
+    _logger.log(`Merged master into ${localBranch.shorthand()} successfully`);
+
+    if (skipVersionBump) {
+        _logger.log("Skip version bump");
+    } else {
+        await bumpMinorVersion(azureSdkForJsRepoPath, packageName);
+        _logger.log(`Successfully updated version in package.json`);
+    }
+
+
+    await generateSdk(azureRestAPISpecsPath, azureSdkForJsRepoPath, packageName)
+    _logger.log(`Generated sdk successfully`);
+
+    await commitAndPush(azureSdkForJsRepository, localBranch, `Regenerated "${packageName}" SDK.`, undefined, `packages/${packageName}`);
+    _logger.log(`Committed and pushed the changes successfully`);
+}
+
+async function bumpMinorVersion(azureSdkForJsRepoPath: string, packageName: string) {
+    const pathToPackageJson = path.resolve(azureSdkForJsRepoPath, "packages", packageName, "package.json");
+    const packageJsonContent = await fs.promises.readFile(pathToPackageJson);
+    const packageJson = JSON.parse(packageJsonContent.toString());
+    const versionString = packageJson.version;
+    const version = Version.parse(versionString);
+    version.bumpMinor();
+    _logger.log(`Updating package.json version from ${versionString} to ${version.toString()}`);
+
+    packageJson.version = version.toString();
+    await saveContentToFile(pathToPackageJson, JSON.stringify(packageJson, undefined, "  "));
+}

.scripts/version.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for
+ * license information.
+ */
+
+export class Version {
+    major: number;
+    minor: number;
+    patch: number;
+    suffix?: string;
+
+    constructor(version: string) {
+        const parts = version.split("-");
+        this.suffix = parts[1];
+
+        const numbers = parts[0].split(".");
+        this.major = Number.parseInt(numbers[0]);
+        this.minor = Number.parseInt(numbers[1]);
+        this.patch = Number.parseInt(numbers[2]);
+    }
+
+    static parse(version: string) {
+        return new Version(version);
+    }
+
+    bumpMajor() {
+        this.major = this.major + 1;
+    }
+
+    bumpMinor() {
+        this.minor = this.minor + 1;
+    }
+
+    bumpPath() {
+        this.patch = this.patch + 1;
+    }
+
+    toString(): string {
+        const suffix = this.suffix ? `-${this.suffix}` : "";
+        return `${this.major}.${this.minor}.${this.patch}${suffix}`;
+    }
+}