From 40861477eafeaeccb58a7fb528be9297a4069606 Mon Sep 17 00:00:00 2001 From: Kamil Pajdzik <41709775+kpajdzik@users.noreply.github.com> Date: Wed, 17 Oct 2018 10:50:20 -0700 Subject: [PATCH] Automate SDK regeneration (#212) * Add Version class * Bootstrap packages regeneration * Fix missing closing brace * Improve branch handling * Working version * Fix branch creation * Change commit story * Add version skipping --- .scripts/git.ts | 141 ++++++++++++++++++++++++++++++++++++-------- .scripts/github.ts | 17 +++--- .scripts/gulp.ts | 52 ++++++++++++++-- .scripts/version.ts | 43 ++++++++++++++ gulpfile.ts | 63 ++++++++++++++++++-- package.json | 1 + 6 files changed, 274 insertions(+), 43 deletions(-) create mode 100644 .scripts/version.ts diff --git a/.scripts/git.ts b/.scripts/git.ts index 78ccc27c157c..f21d6c6e55c4 100644 --- a/.scripts/git.ts +++ b/.scripts/git.ts @@ -4,17 +4,54 @@ * license information. */ -import { Repository, Signature, Merge, Oid, Reference, Cred, StatusFile } from "nodegit"; +import { Repository, Signature, Merge, Oid, Reference, Cred, StatusFile, Reset, Index } from "nodegit"; import { getLogger } from "./logger"; import { getCommandLineOptions } from "./commandLine"; export type ValidateFunction = (statuses: StatusFile[]) => boolean; -export type ValidateEachFunction = (value: StatusFile, index: number, array: StatusFile[]) => boolean; +export type ValidateEachFunction = (path: string, matchedPatter: string) => number; + +export enum BranchLocation { + Local = "heads", + Remote = "remotes" +} + +export class Branch { + static LocalMaster = new Branch("master", BranchLocation.Local); + static RemoteMaster = new Branch("master", BranchLocation.Remote); + + constructor(public name: string, public location: BranchLocation, public remote: string = "origin") { + } + + shorthand(): string { + return `${this.remote}/${this.name}`; + } + + fullName(): string { + if (this.name.startsWith("refs")) { + return this.name; + } + + return `refs/${this.location}/${this.remote}/${this.name}`; + } + + fullNameWithoutRemote(): string { + if (this.name.startsWith("refs")) { + return this.name; + } + + return `refs/${this.location}/${this.name}`; + } + + convertTo(location: BranchLocation): Branch { + return new Branch(this.name, location, this.remote); + } +} const _args = getCommandLineOptions(); const _logger = getLogger(); -const _lockMap = { } +const _lockMap = {} function isLocked(repositoryPath: string) { const isLocked = _lockMap[repositoryPath]; @@ -86,28 +123,39 @@ export async function validateRepositoryStatus(repository: Repository): Promise< export async function getValidatedRepository(repositoryPath: string): Promise { const repository = await openRepository(repositoryPath); await validateRepositoryStatus(repository); + await repository.fetchAll(); return repository; } -export async function pull(repository: Repository, branchName: string, origin: string = "origin"): Promise { - _logger.logTrace(`Pulling "${branchName}" branch from ${origin} origin in ${repository.path()} repository`); +export async function mergeBranch(repository: Repository, toBranch: Branch, fromBranch: Branch): Promise { + _logger.logTrace(`Merging "${fromBranch.fullName()}" to "${toBranch.fullName()}" branch in ${repository.path()} repository`); + return repository.mergeBranches(toBranch.name, fromBranch.shorthand(), Signature.default(repository), Merge.PREFERENCE.NONE); +} + +export async function mergeMasterIntoBranch(repository: Repository, toBranch: Branch): Promise { + return mergeBranch(repository, toBranch, Branch.RemoteMaster); +} + +export async function pullBranch(repository: Repository, localBranch: Branch): Promise { + _logger.logTrace(`Pulling "${localBranch.fullName()}" branch in ${repository.path()} repository`); await repository.fetchAll(); _logger.logTrace(`Fetched all successfully`); - const oid = await repository.mergeBranches(branchName, `${origin}/${branchName}`, Signature.default(repository), Merge.PREFERENCE.NONE); + const remoteBranch = new Branch(localBranch.name, BranchLocation.Remote, localBranch.remote); + await mergeBranch(repository, localBranch, remoteBranch); const index = await repository.index(); if (index.hasConflicts()) { - throw new Error(`Conflict while pulling ${branchName} from origin.`); + throw new Error(`Conflict while pulling ${remoteBranch.fullName()}`); } - _logger.logTrace(`Merged "${origin}/${branchName}" to "${branchName}" successfully without any conflicts`); - return oid; + _logger.logTrace(`Merged "${remoteBranch.fullName()}" to "${localBranch.fullName()}" successfully without any conflicts`); + return undefined; } export async function pullMaster(repository: Repository): Promise { - return pull(repository, "master"); + return pullBranch(repository, Branch.LocalMaster); } export async function createNewBranch(repository: Repository, branchName: string, checkout?: boolean): Promise { @@ -121,10 +169,31 @@ export async function createNewBranch(repository: Repository, branchName: string return branchPromise; } else { const branch = await branchPromise; - return checkoutBranch(repository, branch.name()); + return checkoutBranch(repository, branch.shorthand()); } } +export async function checkoutRemoteBranch(repository: Repository, remoteBranch: Branch): Promise { + _logger.logTrace(`Checking out "${remoteBranch.fullName()}" remote branch`); + + const branchNames = await repository.getReferenceNames(Reference.TYPE.LISTALL); + const localBranch = remoteBranch.convertTo(BranchLocation.Local); + const branchExists = branchNames.some(name => name === localBranch.fullNameWithoutRemote()); + _logger.logTrace(`Branch exists: ${branchExists}`); + + let branchRef: Reference; + if (branchExists) { + branchRef = await checkoutBranch(repository, remoteBranch.name); + } else { + branchRef = await createNewBranch(repository, remoteBranch.name, true); + const commit = await repository.getReferenceCommit(remoteBranch.name); + await Reset.reset(repository, commit as any, Reset.TYPE.HARD, {}); + await pullBranch(repository, remoteBranch.convertTo(BranchLocation.Local)); + } + + return branchRef; +} + function getCurrentDateSuffix(): string { const now = new Date(); return `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}-${now.getMilliseconds()}`; @@ -135,7 +204,7 @@ export async function createNewUniqueBranch(repository: Repository, branchPrefix } export async function checkoutBranch(repository: Repository, branchName: string | Reference): Promise { - _logger.logTrace(`Checking out ${branchName} branch`); + _logger.logTrace(`Checking out "${branchName}" branch`); return repository.checkoutBranch(branchName); } @@ -148,26 +217,45 @@ export async function refreshRepository(repository: Repository) { return checkoutMaster(repository); } -export async function commitSpecificationChanges(repository: Repository, commitMessage: string, validate?: ValidateFunction, validateEach?: ValidateEachFunction): Promise { +export async function commitChanges(repository: Repository, commitMessage: string, validateStatus?: ValidateFunction, validateEach?: string | ValidateEachFunction): Promise { _logger.logTrace(`Committing changes in "${repository.path()}" repository`); - const emptyValidate = () => true; - validate = validate || emptyValidate; - validateEach = validateEach || emptyValidate; + validateStatus = validateStatus || ((_) => true); + validateEach = validateEach || ((_, __) => 0); const status = await repository.getStatus(); - - if (validate(status) && status.every(validateEach)) { - var author = Signature.default(repository); - return repository.createCommitOnHead(status.map(el => el.path()), author, author, commitMessage); - } else { + if (!validateStatus(status)) { return Promise.reject("Unknown changes present in the repository"); } + + const index = await repository.refreshIndex(); + if (typeof validateEach === "string") { + const folderName = validateEach; + validateEach = (path, pattern) => { + return path.startsWith(folderName) ? 0 : 1; + } + } + + await index.addAll("*", Index.ADD_OPTION.ADD_CHECK_PATHSPEC, validateEach); + + const entries = index.entries(); + _logger.logTrace(`Files added to the index ${index.entryCount}: ${JSON.stringify(entries)}`) + + await index.write(); + const oid = await index.writeTree(); + + const head = await repository.getHeadCommit(); + const author = Signature.default(repository); + + return repository.createCommit("HEAD", author, author, commitMessage, oid, [head]); } -export async function pushToNewBranch(repository: Repository, branchName: string): Promise { +export async function pushBranch(repository: Repository, localBranch: Branch): Promise { const remote = await repository.getRemote("origin"); - return remote.push([`${branchName}:${branchName}`], { + const refSpec = `refs/heads/${localBranch.name}:refs/heads/${localBranch.name}`; + _logger.logTrace(`Pushing to ${refSpec}`); + + return remote.push([refSpec], { callbacks: { credentials: function (url, userName) { return Cred.userpassPlaintextNew(getToken(), "x-oauth-basic"); @@ -176,6 +264,11 @@ export async function pushToNewBranch(repository: Repository, branchName: string }); } +export async function commitAndPush(repository: Repository, localBranch: Branch, commitMessage: string, validate?: ValidateFunction, validateEach?: string | ValidateEachFunction) { + await commitChanges(repository, commitMessage, validate, validateEach); + await pushBranch(repository, localBranch); +} + export function getToken(): string { const token = _args.token || process.env.SDK_GEN_GITHUB_TOKEN; _validatePersonalAccessToken(token); @@ -186,7 +279,7 @@ export function getToken(): string { function _validatePersonalAccessToken(token: string): void { if (!token) { const text = - `Github personal access token was not found as a script parameter or as an + `Github personal access token was not found as a script parameter or as an environmental variable. Please visit https://github.com/settings/tokens, generate new token with "repo" scope and pass it with -token switch or set it as environmental variable named SDK_GEN_GITHUB_TOKEN.` diff --git a/.scripts/github.ts b/.scripts/github.ts index 098c12808800..09a7fbb3ec2e 100644 --- a/.scripts/github.ts +++ b/.scripts/github.ts @@ -6,7 +6,7 @@ import * as Octokit from '@octokit/rest' import { PullRequestsCreateParams, Response, PullRequestsCreateReviewRequestParams, PullRequestsCreateReviewRequestResponse } from '@octokit/rest'; -import { getToken, createNewUniqueBranch, commitSpecificationChanges, pushToNewBranch, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from './git'; +import { getToken, createNewUniqueBranch, commitChanges, pushBranch,ValidateFunction, ValidateEachFunction, Branch, BranchLocation } from './git'; import { getLogger } from './logger'; import { Repository } from 'nodegit'; @@ -69,17 +69,18 @@ export async function commitAndCreatePullRequest( pullRequestTitle: string, pullRequestDescription:string, validate?: ValidateFunction, - validateEach?: ValidateEachFunction): Promise { + validateEach?: string | ValidateEachFunction): Promise { await createNewUniqueBranch(repository, `generated/${packageName}`, true); - await commitSpecificationChanges(repository, commitMessage, validate, validateEach); - const newBranch = await repository.getCurrentBranch(); - _logger.logInfo(`Committed changes successfully on ${newBranch.name()} branch`); + await commitChanges(repository, commitMessage, validate, validateEach); + const newBranchRef = await repository.getCurrentBranch(); + const newBranch = new Branch(newBranchRef.name(), BranchLocation.Local); + _logger.logInfo(`Committed changes successfully on ${newBranch.name} branch`); - await pushToNewBranch(repository, newBranch.name()); - _logger.logInfo(`Pushed changes successfully to ${newBranch.name()} branch`); + await pushBranch(repository, newBranch); + _logger.logInfo(`Pushed changes successfully to ${newBranch.name} branch`); - const pullRequestResponse = await createPullRequest(repositoryName, pullRequestTitle, pullRequestDescription, newBranch.name()); + const pullRequestResponse = await createPullRequest(repositoryName, pullRequestTitle, pullRequestDescription, newBranchRef.name()); _logger.logInfo(`Created pull request successfully - ${pullRequestResponse.data.html_url}`); const reviewResponse = await requestPullRequestReview(repositoryName, pullRequestResponse.data.number); diff --git a/.scripts/gulp.ts b/.scripts/gulp.ts index a287267fa8c7..73c1b17fb91d 100644 --- a/.scripts/gulp.ts +++ b/.scripts/gulp.ts @@ -9,10 +9,11 @@ import { findAzureRestApiSpecsRepositoryPath, findSdkDirectory, saveContentToFil import { copyExistingNodeJsReadme, updateTypeScriptReadmeFile, findReadmeTypeScriptMdFilePaths, getPackageNamesFromReadmeTypeScriptMdFileContents, getAbsolutePackageFolderPathFromReadmeFileContents, updateMainReadmeFile, getSinglePackageName } from "./readme"; import * as fs from "fs"; import * as path from "path"; +import { Version } from "./version"; import { contains, npmInstall } from "./common"; import { execSync } from "child_process"; import { getLogger } from "./logger"; -import { refreshRepository, getValidatedRepository, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction } from "./git"; +import { refreshRepository, getValidatedRepository, waitAndLockGitRepository, unlockGitRepository, ValidateFunction, ValidateEachFunction, checkoutBranch, pullBranch, mergeBranch, mergeMasterIntoBranch, commitAndPush, checkoutRemoteBranch, Branch, BranchLocation } from "./git"; import { commitAndCreatePullRequest } from "./github"; const _logger = getLogger(); @@ -118,9 +119,8 @@ export async function generateTsReadme(packageName: string, sdkType: SdkType): P const pullRequestTitle = `Add ${packageName}/${sdkType}/readme.typescript.md`; const pullRequestDescription = "Autogenerated"; const validate: ValidateFunction = statuses => statuses.length == 2; - const validateEach: ValidateEachFunction = el => el.path().startsWith(`specification/${packageName}`); - const pullRequestUrl = await commitAndCreatePullRequest(azureRestApiSpecRepository, packageName, pullRequestTitle, "azure-rest-api-specs", pullRequestTitle, pullRequestDescription, validate, validateEach); + const pullRequestUrl = await commitAndCreatePullRequest(azureRestApiSpecRepository, packageName, pullRequestTitle, "azure-rest-api-specs", pullRequestTitle, pullRequestDescription, validate, `specification/${packageName}`); await unlockGitRepository(azureRestApiSpecRepository); return { pullRequestUrl: pullRequestUrl, typescriptReadmePath: typescriptReadmePath }; @@ -143,7 +143,7 @@ export async function generateMissingSdk(azureSdkForJsRepoPath: string, packageN const azureSdkForJsRepository = await getValidatedRepository(azureSdkForJsRepoPath); await refreshRepository(azureSdkForJsRepository); - _logger.log(`Refreshed ${azureRestApiSpecsRepositoryPath} repository successfully`); + _logger.log(`Refreshed ${azureSdkForJsRepoPath} repository successfully`); await waitAndLockGitRepository(azureSdkForJsRepository); await generateSdk(azureRestApiSpecsRepositoryPath, azureSdkForJsRepoPath, packageName); @@ -157,9 +157,8 @@ ${_logger.getCapturedText()} \`\`\`` const validate: ValidateFunction = changes => changes.length > 0; - const validateEach: ValidateEachFunction = el => el.path().startsWith(`packages/${packageName}`); - const pullRequestUrl = await commitAndCreatePullRequest(azureSdkForJsRepository, packageName, pullRequestTitle, "azure-sdk-for-js", pullRequestTitle, pullRequestDescription, validate, validateEach); + const pullRequestUrl = await commitAndCreatePullRequest(azureSdkForJsRepository, packageName, pullRequestTitle, "azure-sdk-for-js", pullRequestTitle, pullRequestDescription, validate, `packages/${packageName}`); await unlockGitRepository(azureSdkForJsRepository); return pullRequestUrl; @@ -178,3 +177,44 @@ export async function generateAllMissingSdks(azureSdkForJsRepoPath: string, azur } } } + +export async function regenerate(branchName: string, packageName: string, azureSdkForJsRepoPath: string, azureRestAPISpecsPath: string, skipVersionBump?: boolean) { + const azureSdkForJsRepository = await getValidatedRepository(azureSdkForJsRepoPath); + await refreshRepository(azureSdkForJsRepository); + _logger.log(`Refreshed ${azureSdkForJsRepository.path()} repository successfully`); + + const remoteBranch = new Branch(branchName, BranchLocation.Remote); + await checkoutRemoteBranch(azureSdkForJsRepository, remoteBranch); + _logger.log(`Checked out ${branchName} branch`); + + const localBranch = remoteBranch.convertTo(BranchLocation.Local); + await mergeMasterIntoBranch(azureSdkForJsRepository, localBranch); + _logger.log(`Merged master into ${localBranch.shorthand()} successfully`); + + if (skipVersionBump) { + _logger.log("Skip version bump"); + } else { + await bumpMinorVersion(azureSdkForJsRepoPath, packageName); + _logger.log(`Successfully updated version in package.json`); + } + + + await generateSdk(azureRestAPISpecsPath, azureSdkForJsRepoPath, packageName) + _logger.log(`Generated sdk successfully`); + + await commitAndPush(azureSdkForJsRepository, localBranch, `Regenerated "${packageName}" SDK.`, undefined, `packages/${packageName}`); + _logger.log(`Committed and pushed the changes successfully`); +} + +async function bumpMinorVersion(azureSdkForJsRepoPath: string, packageName: string) { + const pathToPackageJson = path.resolve(azureSdkForJsRepoPath, "packages", packageName, "package.json"); + const packageJsonContent = await fs.promises.readFile(pathToPackageJson); + const packageJson = JSON.parse(packageJsonContent.toString()); + const versionString = packageJson.version; + const version = Version.parse(versionString); + version.bumpMinor(); + _logger.log(`Updating package.json version from ${versionString} to ${version.toString()}`); + + packageJson.version = version.toString(); + await saveContentToFile(pathToPackageJson, JSON.stringify(packageJson, undefined, " ")); +} diff --git a/.scripts/version.ts b/.scripts/version.ts new file mode 100644 index 000000000000..51ba11b2ccc9 --- /dev/null +++ b/.scripts/version.ts @@ -0,0 +1,43 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for + * license information. + */ + +export class Version { + major: number; + minor: number; + patch: number; + suffix?: string; + + constructor(version: string) { + const parts = version.split("-"); + this.suffix = parts[1]; + + const numbers = parts[0].split("."); + this.major = Number.parseInt(numbers[0]); + this.minor = Number.parseInt(numbers[1]); + this.patch = Number.parseInt(numbers[2]); + } + + static parse(version: string) { + return new Version(version); + } + + bumpMajor() { + this.major = this.major + 1; + } + + bumpMinor() { + this.minor = this.minor + 1; + } + + bumpPath() { + this.patch = this.patch + 1; + } + + toString(): string { + const suffix = this.suffix ? `-${this.suffix}` : ""; + return `${this.major}.${this.minor}.${this.patch}${suffix}`; + } +} diff --git a/gulpfile.ts b/gulpfile.ts index 1814f435f29b..7a1e9eb68463 100644 --- a/gulpfile.ts +++ b/gulpfile.ts @@ -7,12 +7,13 @@ import { contains, endsWith, npmInstall, npmRunBuild } from "./.scripts/common"; import { getCommandLineOptions } from "./.scripts/commandLine"; import { findAzureRestApiSpecsRepositoryPath, findMissingSdks } from "./.scripts/generateSdks"; -import { generateTsReadme, generateSdk, generateMissingSdk, generateAllMissingSdks } from "./.scripts/gulp"; +import { generateTsReadme, generateSdk, generateMissingSdk, generateAllMissingSdks, regenerate } from "./.scripts/gulp"; import { getPackageNamesFromReadmeTypeScriptMdFileContents, findReadmeTypeScriptMdFilePaths, getAbsolutePackageFolderPathFromReadmeFileContents } from "./.scripts/readme"; -import { getLogger } from "./.scripts/logger"; +import { getLogger, LoggingLevel } from "./.scripts/logger"; import * as fs from "fs"; import * as gulp from "gulp"; import * as path from "path"; +import * as yargs from "yargs"; import { execSync } from "child_process"; const _logger = getLogger(); @@ -20,6 +21,16 @@ const args = getCommandLineOptions(); const azureSDKForJSRepoRoot: string = args["azure-sdk-for-js-repo-root"] || __dirname; const azureRestAPISpecsRoot: string = args["azure-rest-api-specs-root"] || path.resolve(azureSDKForJSRepoRoot, '..', 'azure-rest-api-specs'); +const commonArgv = yargs.options({ + "logging-level": { + alias: ["l", "loggingLevel"], + default: "info", + choices: ["all", "trace", "debug", "info", "warn", "error"], + coerce: (str) => LoggingLevel[str], + } +}).help("?") + .showHelpOnFail(true, "Invalid usage. Run with -? to see help."); + function getPackageFolderPathFromPackageArgument(): string | undefined { let packageFolderPath: string | undefined; @@ -93,8 +104,8 @@ gulp.task("build", () => { // This task is used to generate libraries based on the mappings specified above. gulp.task('codegen', async () => { - _logger.log(`Passed arguments: ${process.argv}`); - await generateSdk(azureRestAPISpecsRoot, azureSDKForJSRepoRoot, args.package); + _logger.log(`Passed arguments: ${process.argv}`); + await generateSdk(azureRestAPISpecsRoot, azureSDKForJSRepoRoot, args.package); }); gulp.task('publish', () => { @@ -213,4 +224,46 @@ gulp.task("generate-all-missing-sdks", async () => { } catch (error) { _logger.logError(error); } -}); \ No newline at end of file +}); + +gulp.task("regenerate", async () => { + return new Promise((resolve, reject) => { + const argv = commonArgv.options({ + "branch": { + alias: "b", + string: true, + required: true, + description: "Name of the AutoPR branch" + }, + "package": { + alias: "p", + string: true, + required: true, + description: "Name of the regenerated package" + }, + "skip-version-bump": { + boolean: true, + description: "Determines if version bumping should be skipped" + }, + "azure-sdk-for-js-root": { + alias: "sdk", + string: true, + default: azureSDKForJSRepoRoot, + description: "Path to the azure-sdk-for-js repository" + }, + "azure-rest-api-specs-root": { + alias: "specs", + string: true, + default: azureRestAPISpecsRoot, + description: "Path to the azure-rest-api-specs repository" + } + }).usage("gulp regenerate --branch 'restapi_auto_daschult/sql'").argv; + + regenerate(argv.branch, argv.package, argv["azure-sdk-for-js-root"], argv["azure-rest-api-specs-root"], argv["skip-version-bump"]) + .then(_ => resolve(), + error => reject(error)) + .catch(error => { + reject(error) + }); + }); +}); diff --git a/package.json b/package.json index fca57abc17b6..82a808a12c07 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@types/minimist": "^1.2.0", "@types/node": "^10.11.4", "@types/nodegit": "^0.22.3", + "@types/yargs": "^12.0.1", "colors": "^1.3.2", "fs": "0.0.1-security", "gulp": "^4.0.0",