From c71e66f7f7a3dc20d2c965349b5e01e15edabf36 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Thu, 19 Aug 2021 11:54:55 -0500 Subject: [PATCH] feat: finish status, add clear/reset --- README.md | 13 ++- messages/source_tracking.js | 24 ++++++ src/commands/source/push.ts | 9 +- src/commands/source/status.ts | 47 +++++----- src/commands/source/tracking/clear.ts | 47 ++++++++++ src/commands/source/tracking/reset.ts | 67 +++++++++++++++ src/shared/localShadowRepo.ts | 30 ++++--- src/shared/remoteSourceTrackingService.ts | 44 +++++----- src/sourceTracking.ts | 100 +++++++++++++++++++++- 9 files changed, 320 insertions(+), 61 deletions(-) create mode 100644 messages/source_tracking.js create mode 100644 src/commands/source/tracking/clear.ts create mode 100644 src/commands/source/tracking/reset.ts diff --git a/README.md b/README.md index 2691de2b..4f2c752e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,17 @@ You should use the class named sourceTracking. ## TODO +can migrate maxRevision.json to its new home + +This code in SourceTracking.ts is making identical queries in parallel, which could be really expensive + +````ts +if (options?.origin === 'remote') { + await this.ensureRemoteTracking(); + const remoteChanges = await this.remoteSourceTrackingService.retrieveUpdates(); + +tracking:clear may not handle errors where it fails to delete local or remote + integration testing Push can have partial successes and needs a proper status code ex: @@ -88,4 +99,4 @@ Push can have partial successes and needs a proper status code ex: "status": "Failed", "success": false } -``` +```` diff --git a/messages/source_tracking.js b/messages/source_tracking.js new file mode 100644 index 00000000..e9b56557 --- /dev/null +++ b/messages/source_tracking.js @@ -0,0 +1,24 @@ +const warning = + 'WARNING: This command deletes or overwrites all existing source tracking files. Use with extreme caution.'; + +module.exports = { + resetDescription: `reset local and remote source tracking + +${warning} + +Resets local and remote source tracking so that the CLI no longer registers differences between your local files and those in the org. When you next run force:source:status, the CLI returns no results, even though conflicts might actually exist. The CLI then resumes tracking new source changes as usual. + +Use the --revision parameter to reset source tracking to a specific revision number of an org source member. To get the revision number, query the SourceMember Tooling API object with the force:data:soql:query command. For example: + $ sfdx force:data:soql:query -q "SELECT MemberName, MemberType, RevisionCounter FROM SourceMember" -t`, + + clearDescription: `clear all local source tracking information + +${warning} + +Clears all local source tracking information. When you next run force:source:status, the CLI displays all local and remote files as changed, and any files with the same name are listed as conflicts.`, + + nopromptDescription: 'do not prompt for source tracking override confirmation', + revisionDescription: 'reset to a specific SourceMember revision counter number', + promptMessage: + 'WARNING: This operation will modify all your local source tracking files. The operation can have unintended consequences on all the force:source commands. Are you sure you want to proceed (y/n)?', +}; diff --git a/src/commands/source/push.ts b/src/commands/source/push.ts index 38f83187..21813175 100644 --- a/src/commands/source/push.ts +++ b/src/commands/source/push.ts @@ -39,7 +39,14 @@ export default class SourcePush extends SfdxCommand { // tracking.getChanges({ origin: 'local', state: 'delete' }), // tracking.getChanges({ origin: 'local', state: 'changed' }), // ]); - const nonDeletes = (await tracking.getChanges({ origin: 'local', state: 'changed' })) + await tracking.ensureLocalTracking(); + const nonDeletes = ( + await Promise.all([ + tracking.getChanges({ origin: 'local', state: 'changed' }), + tracking.getChanges({ origin: 'local', state: 'add' }), + ]) + ) + .flat() .map((change) => change.filenames as string[]) .flat(); const deletes = (await tracking.getChanges({ origin: 'local', state: 'delete' })) diff --git a/src/commands/source/status.ts b/src/commands/source/status.ts index 6e82e71b..6a803228 100644 --- a/src/commands/source/status.ts +++ b/src/commands/source/status.ts @@ -42,18 +42,24 @@ export default class SourceStatus extends SfdxCommand { org: this.org, project: this.project, }); - const outputRows: StatusResult[] = []; + let outputRows: StatusResult[] = []; if (this.flags.local || this.flags.all || (!this.flags.remote && !this.flags.all)) { await tracking.ensureLocalTracking(); - const [localDeletes, localModifies, localAdds] = await Promise.all([ - tracking.getChanges({ origin: 'local', state: 'delete' }), - tracking.getChanges({ origin: 'local', state: 'changed' }), - tracking.getChanges({ origin: 'local', state: 'add' }), - ]); - outputRows.concat(localAdds.map((item) => this.statusResultToOutputRows(item, 'add')).flat()); - outputRows.concat(localModifies.map((item) => this.statusResultToOutputRows(item, 'changed')).flat()); - outputRows.concat(localDeletes.map((item) => this.statusResultToOutputRows(item, 'delete')).flat()); + const [localDeletes, localModifies, localAdds] = ( + await Promise.all([ + tracking.getChanges({ origin: 'local', state: 'delete' }), + tracking.getChanges({ origin: 'local', state: 'changed' }), + tracking.getChanges({ origin: 'local', state: 'add' }), + ]) + ) + // we don't get type/name on local changes unless we request them + .map((changes) => tracking.populateTypesAndNames(changes)); + outputRows = outputRows.concat(localAdds.map((item) => this.statusResultToOutputRows(item, 'add')).flat()); + outputRows = outputRows.concat( + localModifies.map((item) => this.statusResultToOutputRows(item, 'changed')).flat() + ); + outputRows = outputRows.concat(localDeletes.map((item) => this.statusResultToOutputRows(item, 'delete')).flat()); } if (this.flags.remote || this.flags.all || (!this.flags.local && !this.flags.all)) { @@ -62,17 +68,17 @@ export default class SourceStatus extends SfdxCommand { tracking.getChanges({ origin: 'remote', state: 'delete' }), tracking.getChanges({ origin: 'remote', state: 'changed' }), ]); - outputRows.concat(remoteDeletes.map((item) => this.statusResultToOutputRows(item, 'delete')).flat()); - outputRows.concat( + outputRows = outputRows.concat(remoteDeletes.map((item) => this.statusResultToOutputRows(item)).flat()); + outputRows = outputRows.concat( remoteModifies .filter((item) => item.modified) - .map((item) => this.statusResultToOutputRows(item, 'delete')) + .map((item) => this.statusResultToOutputRows(item)) .flat() ); - outputRows.concat( + outputRows = outputRows.concat( remoteModifies .filter((item) => !item.modified) - .map((item) => this.statusResultToOutputRows(item, 'delete')) + .map((item) => this.statusResultToOutputRows(item)) .flat() ); } @@ -86,12 +92,13 @@ export default class SourceStatus extends SfdxCommand { ); } } + this.ux.table(outputRows, { columns: [ { label: 'STATE', key: 'state' }, - { label: 'FULL NAME', key: 'name' }, + { label: 'FULL NAME', key: 'fullName' }, { label: 'TYPE', key: 'type' }, - { label: 'PROJECT PATH', key: 'filenames' }, + { label: 'PROJECT PATH', key: 'filepath' }, ], }); @@ -100,7 +107,7 @@ export default class SourceStatus extends SfdxCommand { } private statusResultToOutputRows(input: ChangeResult, localType?: 'delete' | 'changed' | 'add'): StatusResult[] { - this.logger.debug(input); + this.logger.debug('converting ChangeResult to a row', input); const state = (): string => { if (localType) { @@ -114,12 +121,12 @@ export default class SourceStatus extends SfdxCommand { } return 'Add'; }; - this.logger.debug(state); const baseObject = { - type: input.type || '', + type: input.type || 'TODO', state: `${input.origin} ${state()}`, - fullName: input.name || '', + fullName: input.name || 'TODO', }; + this.logger.debug(baseObject); if (!input.filenames) { return [baseObject]; diff --git a/src/commands/source/tracking/clear.ts b/src/commands/source/tracking/clear.ts new file mode 100644 index 00000000..368df1c0 --- /dev/null +++ b/src/commands/source/tracking/clear.ts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { flags, FlagsConfig, SfdxCommand } from '@salesforce/command'; +import { Messages, Org, SfdxProject } from '@salesforce/core'; +import * as chalk from 'chalk'; +import { SourceTracking } from '../../../sourceTracking'; + +Messages.importMessagesDirectory(__dirname); +const messages: Messages = Messages.loadMessages('@salesforce/source-tracking', 'source_tracking'); + +export type SourceTrackingClearResult = { + clearedFiles: string[]; +}; + +export class SourceTrackingClearCommand extends SfdxCommand { + public static readonly description = messages.getMessage('clearDescription'); + + public static readonly requiresProject = true; + public static readonly requiresUsername = true; + + public static readonly flagsConfig: FlagsConfig = { + noprompt: flags.boolean({ + char: 'p', + description: messages.getMessage('nopromptDescription'), + required: false, + }), + }; + + // valid assertions with ! because requiresProject and requiresUsername + protected org!: Org; + protected project!: SfdxProject; + + public async run(): Promise { + let clearedFiles: string[] = []; + if (this.flags.noprompt || (await this.ux.confirm(chalk.dim(messages.getMessage('promptMessage'))))) { + const sourceTracking = new SourceTracking({ project: this.project, org: this.org }); + clearedFiles = await Promise.all([sourceTracking.clearLocalTracking(), sourceTracking.clearRemoteTracking()]); + this.ux.log('Cleared local tracking files.'); + } + return { clearedFiles }; + } +} diff --git a/src/commands/source/tracking/reset.ts b/src/commands/source/tracking/reset.ts new file mode 100644 index 00000000..1b33503f --- /dev/null +++ b/src/commands/source/tracking/reset.ts @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { flags, FlagsConfig, SfdxCommand } from '@salesforce/command'; +import { Messages, Org, SfdxProject } from '@salesforce/core'; +import * as chalk from 'chalk'; +import { SourceTracking } from '../../../sourceTracking'; + +Messages.importMessagesDirectory(__dirname); +const messages: Messages = Messages.loadMessages('@salesforce/source-tracking', 'source_tracking'); + +export type SourceTrackingResetResult = { + sourceMembersSynced: number; + localPathsSynced: number; +}; + +export class SourceTrackingResetCommand extends SfdxCommand { + public static readonly description = messages.getMessage('resetDescription'); + + public static readonly requiresProject = true; + public static readonly requiresUsername = true; + + public static readonly flagsConfig: FlagsConfig = { + revision: flags.integer({ + char: 'r', + description: messages.getMessage('revisionDescription'), + min: 0, + }), + noprompt: flags.boolean({ + char: 'p', + description: messages.getMessage('nopromptDescription'), + }), + }; + + // valid assertions with ! because requiresProject and requiresUsername + protected org!: Org; + protected project!: SfdxProject; + + public async run(): Promise { + if (this.flags.noprompt || (await this.ux.confirm(chalk.dim(messages.getMessage('promptMessage'))))) { + const sourceTracking = new SourceTracking({ project: this.project, org: this.org }); + + const [remoteResets, localResets] = await Promise.all([ + sourceTracking.resetRemoteTracking(this.flags.revision as number), + sourceTracking.resetLocalTracking(), + ]); + + this.ux.log( + `Reset local tracking files${this.flags.revision ? ` to revision ${this.flags.revision as number}` : ''}.` + ); + + return { + sourceMembersSynced: remoteResets, + localPathsSynced: localResets.length, + }; + } + + return { + sourceMembersSynced: 0, + localPathsSynced: 0, + }; + } +} diff --git a/src/shared/localShadowRepo.ts b/src/shared/localShadowRepo.ts index d93f3d29..59ea55fa 100644 --- a/src/shared/localShadowRepo.ts +++ b/src/shared/localShadowRepo.ts @@ -6,17 +6,17 @@ */ /* eslint-disable no-console */ -import * as path from 'path'; +import { join as pathJoin } from 'path'; import * as fs from 'fs'; import { AsyncCreatable } from '@salesforce/kit'; -import { NamedPackageDir, fs as fsCore, Logger } from '@salesforce/core'; +import { NamedPackageDir, Logger } from '@salesforce/core'; import * as git from 'isomorphic-git'; /** * returns the full path to where we store the shadow repo */ const getGitDir = (orgId: string, projectPath: string): string => { - return path.join(projectPath, '.sfdx', 'orgs', orgId); + return pathJoin(projectPath, '.sfdx', 'orgs', orgId, 'localSourceTracking'); }; const toFilenames = (rows: StatusRow[]): string[] => rows.map((file) => file[FILE] as string); @@ -74,10 +74,14 @@ export class ShadowRepo extends AsyncCreatable { * */ public async gitInit(): Promise { - await fsCore.mkdirp(this.gitDir); + await fs.promises.mkdir(this.gitDir, { recursive: true }); await git.init({ fs, dir: this.projectPath, gitdir: this.gitDir, defaultBranch: 'main' }); } + public async delete(): Promise { + await fs.promises.rm(this.gitDir, { recursive: true, force: true }); + return this.gitDir; + } /** * If the status already exists, return it. Otherwise, set the status before returning. * It's kinda like a cache @@ -95,6 +99,8 @@ export class ShadowRepo extends AsyncCreatable { dir: this.projectPath, gitdir: this.gitDir, filepaths: this.packageDirs.map((dir) => dir.path), + // filter out hidden files + filter: (f) => !f.includes('/.'), }); await this.unStashIgnoreFile(); } @@ -108,6 +114,9 @@ export class ShadowRepo extends AsyncCreatable { return (await this.getStatus()).filter((file) => file[HEAD] !== file[WORKDIR]); } + /** + * returns any change (add, modify, delete) + */ public async getChangedFilenames(): Promise { return toFilenames(await this.getChangedRows()); } @@ -127,6 +136,9 @@ export class ShadowRepo extends AsyncCreatable { return (await this.getStatus()).filter((file) => file[WORKDIR] === 2); } + /** + * returns adds and modifies but not deletes + */ public async getNonDeleteFilenames(): Promise { return toFilenames(await this.getNonDeletes()); } @@ -197,20 +209,14 @@ export class ShadowRepo extends AsyncCreatable { private async stashIgnoreFile(): Promise { if (!this.stashed) { this.stashed = true; - await fs.promises.rename( - path.join(this.projectPath, '.gitignore'), - path.join(this.projectPath, '.BAK.gitignore') - ); + await fs.promises.rename(pathJoin(this.projectPath, '.gitignore'), pathJoin(this.projectPath, '.BAK.gitignore')); } } private async unStashIgnoreFile(): Promise { if (this.stashed) { this.stashed = false; - await fs.promises.rename( - path.join(this.projectPath, '.BAK.gitignore'), - path.join(this.projectPath, '.gitignore') - ); + await fs.promises.rename(pathJoin(this.projectPath, '.BAK.gitignore'), pathJoin(this.projectPath, '.gitignore')); } } } diff --git a/src/shared/remoteSourceTrackingService.ts b/src/shared/remoteSourceTrackingService.ts index 1907c263..e999c704 100644 --- a/src/shared/remoteSourceTrackingService.ts +++ b/src/shared/remoteSourceTrackingService.ts @@ -8,8 +8,8 @@ /* eslint-disable @typescript-eslint/member-ordering */ import * as path from 'path'; -import { join as pathJoin } from 'path'; -import { ConfigFile, fs, Logger, Org, SfdxError, Messages } from '@salesforce/core'; +import * as fs from 'fs'; +import { ConfigFile, Logger, Org, SfdxError, Messages } from '@salesforce/core'; import { Dictionary, Optional } from '@salesforce/ts-types'; import { Duration, env, toNumber } from '@salesforce/kit'; import { retryDecorator } from 'ts-retry-promise'; @@ -117,42 +117,37 @@ export class RemoteSourceTrackingService extends ConfigFile { + const fileToDelete = RemoteSourceTrackingService.getFilePath(orgId); + // the file might not exist, in which case we don't need to delete it + if (fs.existsSync(fileToDelete)) { + await fs.promises.rm(fileToDelete, { recursive: true }); + } + return path.isAbsolute(fileToDelete) ? fileToDelete : path.join(process.cwd(), fileToDelete); + } + /** * Initializes the service with existing remote source tracking data, or sets * the state to begin source tracking of metadata changes in the org. */ public async init(): Promise { - this.options.filePath = pathJoin('orgs', this.options.orgId); - this.options.filename = RemoteSourceTrackingService.getFileName(); this.org = await Org.create({ aliasOrUsername: this.options.username }); this.logger = await Logger.child(this.constructor.name); + this.options.filePath = path.join('orgs', this.org.getOrgId()); + this.options.filename = RemoteSourceTrackingService.getFileName(); try { await super.init(); } catch (err) { - // This error is thrown when the legacy maxRevision.json is read. Transform to the new schema. - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (err.name === 'JsonDataFormatError') { - const filePath = path.join(process.cwd(), this.options.filePath, RemoteSourceTrackingService.getFileName()); - const legacyRevision = await fs.readFile(filePath, 'utf-8'); - this.logger.debug(`Converting legacy maxRevision.json with revision ${legacyRevision} to new schema`); - await fs.writeFile( - filePath, - JSON.stringify({ serverMaxRevisionCounter: parseInt(legacyRevision, 10), sourceMembers: {} }, null, 4) - ); - await super.init(); - } else { - throw SfdxError.wrap(err); - } + throw SfdxError.wrap(err); } const contents = this.getTypedContents(); @@ -244,7 +239,7 @@ export class RemoteSourceTrackingService extends ConfigFile { + public async reset(toRevision?: number): Promise { // Called during a source:tracking:reset this.setServerMaxRevision(0); this.initSourceMembers(); @@ -257,6 +252,7 @@ export class RemoteSourceTrackingService extends ConfigFile getMetadataKey(member.MemberType, member.MemberName)); } // diff --git a/src/sourceTracking.ts b/src/sourceTracking.ts index badf2090..00cde31b 100644 --- a/src/sourceTracking.ts +++ b/src/sourceTracking.ts @@ -72,7 +72,7 @@ export class SourceTracking { if (options?.origin === 'local') { await this.ensureLocalTracking(); if (options.state === 'changed') { - return (await this.localRepo.getNonDeleteFilenames()).map((filename) => ({ + return (await this.localRepo.getModifyFilenames()).map((filename) => ({ filenames: [filename], origin: 'local', })); @@ -157,14 +157,56 @@ export class SourceTracking { */ public async ensureRemoteTracking(): Promise { if (this.remoteSourceTrackingService) { + this.logger.debug('ensureRemoteTracking: remote tracking already exists'); return; } + this.logger.debug('ensureRemoteTracking: remote tracking does not exist yet; getting instance'); this.remoteSourceTrackingService = await RemoteSourceTrackingService.getInstance({ username: this.username, orgId: this.orgId, }); - // loads the status from file so that it's cached - await this.remoteSourceTrackingService.init(); + } + + /** + * Deletes the local tracking shadowRepo + * return the list of files that were in it + */ + public async clearLocalTracking(): Promise { + await this.ensureLocalTracking(); + return this.localRepo.delete(); + } + + /** + * Commits all the local changes so that no changes are present in status + */ + public async resetLocalTracking(): Promise { + await this.ensureLocalTracking(); + const [deletes, nonDeletes] = await Promise.all([ + this.localRepo.getDeleteFilenames(), + this.localRepo.getNonDeleteFilenames(), + ]); + await this.localRepo.commitChanges({ + deletedFiles: deletes, + deployedFiles: nonDeletes, + message: 'via resetLocalTracking', + }); + return [...deletes, ...nonDeletes]; + } + + /** + * Deletes the remote tracking files + */ + public async clearRemoteTracking(): Promise { + return RemoteSourceTrackingService.delete(this.orgId); + } + + /** + * Sets the files to max revision so that no changes appear + */ + public async resetRemoteTracking(serverRevision?: number): Promise { + await this.ensureRemoteTracking(); + const resetMembers = await this.remoteSourceTrackingService.reset(serverRevision); + return resetMembers.length; } /** @@ -229,6 +271,58 @@ export class SourceTracking { return Array.from(elementMap.values()); } + /** + * uses SDR to translate remote metadata records into local file paths + */ + // public async populateFilePaths(elements: ChangeResult[]): Promise { + public populateTypesAndNames(elements: ChangeResult[]): ChangeResult[] { + if (elements.length === 0) { + return []; + } + + this.logger.debug('populateTypesAndNames for change elements', elements); + // component set generated from an filenames on all local changes + const matchingLocalSourceComponentsSet = ComponentSet.fromSource({ + fsPaths: elements + .map((element) => element.filenames) + .flat() + .filter(Boolean) as string[], + }); + + this.logger.debug( + ` local source-backed component set has ${matchingLocalSourceComponentsSet.size.toString()} items from remote` + ); + + // make it simpler to find things later + const elementMap = new Map(); + elements.map((element) => { + element.filenames?.map((filename) => { + elementMap.set(this.ensureRelative(filename), element); + }); + }); + + // iterates the local components and sets their filenames + matchingLocalSourceComponentsSet + .getSourceComponents() + .toArray() + .map((matchingComponent) => { + if (matchingComponent?.fullName && matchingComponent?.type.name) { + const filenamesFromMatchingComponent = [matchingComponent.xml, ...matchingComponent.walkContent()]; + filenamesFromMatchingComponent.map((filename) => { + if (filename && elementMap.has(filename)) { + // add the type/name from the componentSet onto the element + elementMap.set(filename, { + ...(elementMap.get(filename) as ChangeResult), + type: matchingComponent.type.name, + name: matchingComponent.fullName, + }); + } + }); + } + }); + return [...new Set(elementMap.values())]; + } + public async getConflicts(): Promise { // we're going to need have both initialized await Promise.all([this.ensureRemoteTracking(), this.ensureLocalTracking()]);