Reduce test time by lowering the sample count for perf tests (#1888)

microsoft · Jun 7, 2018 · 870dbfe · 870dbfe
1 parent d4f7799
commit 870dbfe
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/news/3 Code Health/1887.md b/news/3 Code Health/1887.md
@@ -0,0 +1 @@
+Reduce sample count used to capture performance metrics in order to reduce time taken to complete the tests.
diff --git a/src/test/performance/load.perf.test.ts b/src/test/performance/load.perf.test.ts
@@ -18,7 +18,7 @@ suite('Activation Times', () => {
  if (process.env.ACTIVATION_TIMES_LOG_FILE_PATH) {
  const logFile = process.env.ACTIVATION_TIMES_LOG_FILE_PATH;
  const sampleCounter = fs.existsSync(logFile) ? fs.readFileSync(logFile, { encoding: 'utf8' }).toString().split(/\r?\n/g).length : 1;
- if (sampleCounter > 10) {
+ if (sampleCounter > 5) {
  return;
  }
  test(`Capture Extension Activation Times (Version: ${process.env.ACTIVATION_TIMES_EXT_VERSION}, sample: ${sampleCounter})`, async () => {

diff --git a/src/test/performanceTest.ts b/src/test/performanceTest.ts
@@ -38,7 +38,7 @@ class TestRunner {
  await del([path.join(tmpFolder, '**')]);
  await this.extractLatestExtension(publishedExtensionPath);
 
- const timesToLoadEachVersion = 3;
+ const timesToLoadEachVersion = 2;
  const devLogFiles: string[] = [];
  const releaseLogFiles: string[] = [];
  const newAnalysisEngineLogFiles: string[] = [];
@@ -47,20 +47,24 @@ class TestRunner {
  await this.enableNewAnalysisEngine(false);
 
  const devLogFile = path.join(logFilesPath, `dev_loadtimes${i}.txt`);
+ console.log(`Start Performance Tests: Counter ${i}, for Dev version with Jedi`);
  await this.capturePerfTimes(Version.Dev, devLogFile);
  devLogFiles.push(devLogFile);
 
  const releaseLogFile = path.join(logFilesPath, `release_loadtimes${i}.txt`);
+ console.log(`Start Performance Tests: Counter ${i}, for Release version with Jedi`);
  await this.capturePerfTimes(Version.Release, releaseLogFile);
  releaseLogFiles.push(releaseLogFile);
 
  // New Analysis engine.
  await this.enableNewAnalysisEngine(true);
  const newAnalysisEngineLogFile = path.join(logFilesPath, `newAnalysisEngine_loadtimes${i}.txt`);
+ console.log(`Start Performance Tests: Counter ${i}, for Release version with Analysis Engine`);
  await this.capturePerfTimes(Version.Release, newAnalysisEngineLogFile);
  newAnalysisEngineLogFiles.push(newAnalysisEngineLogFile);
  }
 
+ console.log('Compare Performance Results');
  await this.runPerfTest(devLogFiles, releaseLogFiles, newAnalysisEngineLogFiles);
  }
  private async enableNewAnalysisEngine(enable: boolean) {