Skip to content

Commit

Permalink
feat: remote tracknig with UT
Browse files Browse the repository at this point in the history
  • Loading branch information
@mshanemc
mshanemc committed Aug 6, 2021
1 parent 6c2ebb4 commit cb805e5
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 172 deletions.
3 changes: 3 additions & 0 deletions messages/source.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"NonSourceTrackedOrgError": "This command can only be used on orgs that have source tracking enabled, such as sandboxes and scratch orgs."
}
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
"nyc": "^15.1.0",
"prettier": "^2.3.2",
"pretty-quick": "^3.1.1",
"sinon": "^11.1.2",
"sinon": "^10.0.0",
"ts-node": "^10.1.0",
"typescript": "^4.3.5"
},
Expand Down
108 changes: 52 additions & 56 deletions src/shared/remoteSourceTrackingService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@

import * as path from 'path';
import { join as pathJoin } from 'path';
import { ConfigFile, Connection, fs, Logger, Org, SfdxError } from '@salesforce/core';
import { ConfigFile, fs, Logger, Org, SfdxError, Messages } from '@salesforce/core';
import { Dictionary, Optional } from '@salesforce/ts-types';
import { Duration, env, sleep, toNumber } from '@salesforce/kit';
import { Duration, env, toNumber } from '@salesforce/kit';
import { retryDecorator } from 'ts-retry-promise';

export type MemberRevision = {
serverRevisionCounter: number;
Expand All @@ -33,6 +34,11 @@ export type ChangeElement = {
deleted?: boolean;
};

interface Contents {
serverMaxRevisionCounter: number;
sourceMembers: Dictionary<MemberRevision>;
}

export namespace RemoteSourceTrackingService {
// Constructor Options for RemoteSourceTrackingService.
export interface Options extends ConfigFile.Options {
Expand Down Expand Up @@ -84,14 +90,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
private static remoteSourceTrackingServiceDictionary: Dictionary<RemoteSourceTrackingService> = {};
protected logger!: Logger;
private org!: Org;
private readonly FIRST_REVISION_COUNTER_API_VERSION: string = '47.0';
private conn!: Connection;
private currentApiVersion!: string;
private isSourceTrackedOrg = true;
protected contents!: {
serverMaxRevisionCounter: number;
sourceMembers: { [key: string]: MemberRevision };
};

// A short term cache (within the same process) of query results based on a revision.
// Useful for source:pull, which makes 3 of the same queries; during status, building manifests, after pull success.
Expand All @@ -111,7 +110,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
if (!this.remoteSourceTrackingServiceDictionary[options.username]) {
this.remoteSourceTrackingServiceDictionary[options.username] = await RemoteSourceTrackingService.create(options);
}
return this.remoteSourceTrackingServiceDictionary[options.username];
return this.remoteSourceTrackingServiceDictionary[options.username] as RemoteSourceTrackingService;
}

/**
Expand All @@ -132,8 +131,6 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
this.options.filename = RemoteSourceTrackingService.getFileName();
this.org = await Org.create({ aliasOrUsername: this.options.username });
this.logger = await Logger.child(this.constructor.name);
this.conn = this.org.getConnection();
this.currentApiVersion = this.conn.getApiVersion();

try {
await super.init();
Expand Down Expand Up @@ -162,7 +159,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
// for SourceMembers. If a certain error is thrown during the query we won't try to do
// source tracking for this org. Calling querySourceMembersFrom() has the extra benefit
// of caching the query so we don't have to make an identical request in the same process.
await this.querySourceMembersFrom(0);
await this.querySourceMembersFrom({ fromRevision: 0 });

this.logger.debug('Initializing source tracking state');
contents.serverMaxRevisionCounter = 0;
Expand Down Expand Up @@ -254,7 +251,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
if (toRevision != null) {
members = await this.querySourceMembersTo(toRevision);
} else {
members = await this.querySourceMembersFrom(0);
members = await this.querySourceMembersFrom({ fromRevision: 0 });
}

await this.trackSourceMembers(members, true);
Expand All @@ -273,19 +270,19 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
}

private getServerMaxRevision(): number {
return this['contents'].serverMaxRevisionCounter;
return (this['contents'] as Contents).serverMaxRevisionCounter;
}

private setServerMaxRevision(revision = 0): void {
this['contents'].serverMaxRevisionCounter = revision;
(this['contents'] as Contents).serverMaxRevisionCounter = revision;
}

private getSourceMembers(): Dictionary<MemberRevision> {
return this['contents'].sourceMembers;
return (this['contents'] as Contents).sourceMembers;
}

private initSourceMembers(): void {
this['contents'].sourceMembers = {};
(this['contents'] as Contents).sourceMembers = {};
}

// Return a tracked element as MemberRevision data.
Expand All @@ -294,7 +291,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
}

private setMemberRevision(key: string, sourceMember: MemberRevision): void {
this.getContents().sourceMembers[key] = sourceMember;
(this.getContents() as unknown as Contents).sourceMembers[key] = sourceMember;
}

// Adds the given SourceMembers to the list of tracked MemberRevisions, optionally updating
Expand Down Expand Up @@ -439,51 +436,52 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
this.logger.debug(`Computed SourceMember polling timeout of ${pollingTimeout}s`);
}

const pollStartTime = Date.now();
let pollEndTime: number;
let totalPollTime = 0;
const fromRevision = this.getServerMaxRevision();
this.logger.debug(
`Polling for ${memberNames.length} SourceMembers from revision ${fromRevision} with timeout of ${pollingTimeout}s`
);

let pollAttempts = 1;
let pollAttempts = 0;

// we need to keep asking the server for sourceMembers until time runs out OR we find everything in matches
const matches = new Set(memberNames);

retry;
let sourceMembers;
const poll = async (): Promise<SourceMember[]> => {
const allMembers = await this.querySourceMembersFrom(fromRevision, pollAttempts !== 1, false);
pollAttempts += 1;
sourceMembers = await this.querySourceMembersFrom({
fromRevision,
quiet: pollAttempts !== 1,
useCache: false,
});

for (const member of allMembers) {
for (const member of sourceMembers) {
matches.delete(member.MemberName);
}

this.logger.debug(
`[${pollAttempts}] Found ${memberNames.length - matches.size} of ${memberNames.length} SourceMembers`
);
pollEndTime = Date.now();
totalPollTime = Math.round((pollEndTime - pollStartTime) / 1000) || 1;
if (matches.size === 0 || totalPollTime >= pollingTimeout) {
return allMembers;
if (matches.size === 0) {
return sourceMembers;
}

if (matches.size < 20) {
this.logger.debug(`Still looking for SourceMembers: ${[...matches].join(',')}`);
this.logger.debug(`Still looking for SourceMembers: ${Array.from(matches).join(',')}`);
}

await sleep(Duration.seconds(1));
pollAttempts += 1;
return poll();
throw new Error();
};
const sourceMembers = await poll();

if (matches.size === 0) {
this.logger.debug(`Retrieved all SourceMember data after ${totalPollTime}s and ${pollAttempts} attempts`);
} else {
this.logger.warn(`Polling for SourceMembers timed out after ${totalPollTime}s and ${pollAttempts} attempts`);
const pollingFunction = retryDecorator(poll, {
timeout: pollingTimeout * 1000,
delay: 1000,
retries: 'INFINITELY',
});
try {
await pollingFunction();
this.logger.debug(`Retrieved all SourceMember data after ${pollAttempts} attempts`);
} catch {
this.logger.warn(`Polling for SourceMembers timed out after ${pollAttempts} attempts`);
if (matches.size < 51) {
this.logger.debug(`Could not find ${matches.size} SourceMembers: ${[...matches].join(',')}`);
this.logger.debug(`Could not find ${matches.size} SourceMembers: ${Array.from(matches).join(',')}`);
} else {
this.logger.debug(`Could not find SourceMembers for ${matches.size} components`);
}
Expand All @@ -497,7 +495,11 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
await this.trackSourceMembers(sourceMembers, true);
}

private async querySourceMembersFrom(fromRevision?: number, quiet = false, useCache = true): Promise<SourceMember[]> {
private async querySourceMembersFrom({
fromRevision,
quiet = false,
useCache = true,
}: { fromRevision?: number; quiet?: boolean; useCache?: boolean } = {}): Promise<SourceMember[]> {
const rev = fromRevision != null ? fromRevision : this.getServerMaxRevision();

if (useCache) {
Expand All @@ -519,27 +521,21 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking

private async querySourceMembersTo(toRevision: number, quiet = false): Promise<SourceMember[]> {
const query = `SELECT MemberType, MemberName, IsNameObsolete, RevisionCounter FROM SourceMember WHERE RevisionCounter <= ${toRevision}`;
return this.query(query, quiet);
return this.query<SourceMember>(query, quiet);
}

private async query<T>(query: string, quiet = false): Promise<T[]> {
// to switch to using RevisionCounter - apiVersion > 46.0
// set the api version of the connection to 47.0, query, revert api version
if (!this.isSourceTrackedOrg) {
throw SfdxError.create('salesforce-alm', 'source', 'NonSourceTrackedOrgError');
Messages.importMessagesDirectory(__dirname);
this.messages = Messages.loadMessages('@salesforce/source-tracking', 'source');
throw SfdxError.create('@salesforce/source-tracking', 'source', 'NonSourceTrackedOrgError');
}
if (!quiet) {
this.logger.debug(query);
}

let results;
if (parseFloat(this.currentApiVersion) < parseFloat(this.FIRST_REVISION_COUNTER_API_VERSION)) {
this.conn.setApiVersion(this.FIRST_REVISION_COUNTER_API_VERSION);
results = await this.conn.tooling.autoFetchQuery<T>(query);
this.conn.setApiVersion(this.currentApiVersion);
} else {
results = await this.conn.tooling.autoFetchQuery<T>(query);
}
const results = await this.org.getConnection().tooling.autoFetchQuery<T>(query);

return results.records;
}
}
2 changes: 1 addition & 1 deletion test/nuts/scenario.nut.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { TestSession } from '@salesforce/cli-plugins-testkit';
import { fs } from '@salesforce/core';
import { expect } from 'chai';
import { shouldThrow } from '@salesforce/core/lib/testSetup';
import { ShadowRepo } from '../../src/shared/repo';
import { ShadowRepo } from '../../src/shared/localShadowRepo';

describe('end-to-end-test', () => {
let session: TestSession;
Expand Down
Loading

0 comments on commit cb805e5

Please sign in to comment.