Use the remapped code coverage summary for code coverage threshold check

config/karma/shared.karma.conf.js

@@ -8,27 +8,15 @@ const logger = require('@blackbaud/skyux-logger');
  * @param {*} config
  */
 function getCoverageThreshold(skyPagesConfig) {
-
-  function getProperty(threshold) {
-    return {
-      global: {
-        statements: threshold,
-        branches: threshold,
-        functions: threshold,
-        lines: threshold
-      }
-    };
-  }
-
   switch (skyPagesConfig.skyux.codeCoverageThreshold) {
     case 'none':
-      return getProperty(0);
+      return 0;
 
     case 'standard':
-      return getProperty(80);
+      return 80;
 
     case 'strict':
-      return getProperty(100);
+      return 100;
   }
 }
 
@@ -47,6 +35,8 @@ function getConfig(config) {
   let testWebpackConfig = require('../webpack/test.webpack.config');
   let remapIstanbul = require('remap-istanbul');
 
+  const utils = require('istanbul').utils;
+
   // See minimist documentation regarding `argv._` https://github.com/substack/minimist
   let skyPagesConfig = require('../sky-pages/sky-pages.config').getSkyPagesConfig(argv._[0]);
 
@@ -58,6 +48,9 @@ function getConfig(config) {
   preprocessors[specBundle] = ['coverage', 'webpack', 'sourcemap'];
   preprocessors[specStyles] = ['webpack'];
 
+  let onWriteReportIndex = -1;
+  let coverageFailed;
+
   config.set({
     basePath: '',
     frameworks: ['jasmine'],
@@ -77,15 +70,91 @@ function getConfig(config) {
     webpack: testWebpackConfig.getWebpackConfig(skyPagesConfig, argv),
     coverageReporter: {
       dir: path.join(process.cwd(), 'coverage'),
-      check: getCoverageThreshold(skyPagesConfig),
       reporters: [
         { type: 'json' },
         { type: 'html' },
         { type: 'text-summary' },
-        { type: 'lcov' }
+        { type: 'lcov' },
+        { type: 'in-memory' }
       ],
       _onWriteReport: function (collector) {
-        return remapIstanbul.remap(collector.getFinalCoverage());
+        onWriteReportIndex++;
+
+        const newCollector = remapIstanbul.remap(collector.getFinalCoverage());
+
+        const threshold = getCoverageThreshold(skyPagesConfig);
+
+        // The karma-coverage library does not use the coverage summary from the remapped source
+        // code, so its built-in code coverage check uses numbers that don't match what's reported
+        // to the user.  This will use the coverage summary generated from the remapped
+        // source code.
+        if (threshold) {
+          // When calling the _onWriteReport() method, karma-coverage loops through each reporter,
+          // then for each reporter loops through each browser.  Since karma-coverage doesn't
+          // supply this method with any information about the reporter or browser for which this
+          // method is being called, we must calculate it by looking at how many times the method
+          // has been called.
+          const browserIndex = Math.floor(onWriteReportIndex / this.reporters.length);
+
+          if (onWriteReportIndex % this.reporters.length === 0) {
+            // The karma-coverage library has moved to the next browser and has started the first
+            // reporter for that browser, so evaluate the code coverage now.
+            const browserName = config.browsers[browserIndex];
+
+            let summaries = [];
+
+            newCollector.files().forEach((file) => {
+              summaries.push(
+                utils.summarizeFileCoverage(
+                  newCollector.fileCoverageFor(file)
+                )
+              );
+            });
+
+            const remapCoverageSummary =
+              utils.mergeSummaryObjects.apply(
+                null,
+                summaries
+              );
+
+            let keys = [
+              'statements',
+              'branches',
+              'lines',
+              'functions'
+            ];
+
+            keys.forEach((key) => {
+              let actual = remapCoverageSummary[key].pct;
+
+              if (actual < threshold) {
+                coverageFailed = true;
+                logger.error(
+                  `Coverage in ${browserName} for ${key} (${actual}%) does not meet ` +
+                  `global threshold (${threshold}%)`
+                );
+              }
+            });
+          }
+        }
+
+        // It's possible the user is running the watch command, so the field we're
+        // using to track the number of calls to _onWriteReport() needs to be reset
+        // after each run.
+        if (onWriteReportIndex === (this.reporters.length * config.browsers.length - 1)) {
+          onWriteReportIndex = -1;
+        }
+
+        return newCollector;
+      },
+
+      _onExit: (done) => {
+        if (coverageFailed) {
+          logger.info('Karma has exited with 1.');
+          process.exit(1);
+        }
+
+        done();
       }
     },
     webpackServer: {

test/config-karma-shared.spec.js

@@ -22,13 +22,8 @@ describe('config karma shared', () => {
     });
 
     mock.reRequire('../config/karma/shared.karma.conf')({
-      set: (config) => {
-        const collector = {
-          getFinalCoverage: () => ({})
-        };
+      set: () => {
         expect(called).toEqual(true);
-        expect(typeof config.coverageReporter._onWriteReport).toEqual('function');
-        expect(config.coverageReporter._onWriteReport(collector)).toBeDefined();
         done();
       }
     });
@@ -59,33 +54,6 @@ describe('config karma shared', () => {
     });
   });
 
-  function checkCodeCoverage(configValue, threshold) {
-
-    mock('../config/sky-pages/sky-pages.config.js', {
-      getSkyPagesConfig: () => ({
-        skyux: {
-          codeCoverageThreshold: configValue
-        }
-      })
-    });
-
-    mock(testConfigFilename, {
-      getWebpackConfig: () => {}
-    });
-
-    mock.reRequire('../config/karma/shared.karma.conf')({
-      set: (config) => {
-        expect(config.coverageReporter.check).toEqual({
-          global: {
-            statements: threshold,
-            branches: threshold,
-            functions: threshold,
-            lines: threshold
-          }
-        });
-      }
-    });
-  }
 
   it('should not add the check property when codeCoverageThreshold is not defined', () => {
     mock('../config/sky-pages/sky-pages.config.js', {
@@ -105,18 +73,6 @@ describe('config karma shared', () => {
     });
   });
 
-  it('should handle codeCoverageThreshold set to "none"', () => {
-    checkCodeCoverage('none', 0);
-  });
-
-  it('should handle codeCoverageThreshold set to "standard"', () => {
-    checkCodeCoverage('standard', 80);
-  });
-
-  it('should handle codeCoverageThreshold set to "strict"', () => {
-    checkCodeCoverage('strict', 100);
-  });
-
   it('should pass the logColor flag to the config', () => {
     mock('@blackbaud/skyux-logger', { logColor: false });
     mock.reRequire('../config/karma/shared.karma.conf')({
@@ -153,4 +109,168 @@ describe('config karma shared', () => {
     });
   });
 
+  describe('code coverage', () => {
+    let errorSpy;
+    let exitSpy;
+    let infoSpy;
+
+    const coverageProps = [
+      'statements',
+      'branches',
+      'lines',
+      'functions'
+    ];
+
+    beforeEach(() => {
+      mock(testConfigFilename, {
+        getWebpackConfig: () => {}
+      });
+
+      errorSpy = jasmine.createSpy('error');
+      infoSpy = jasmine.createSpy('info');
+
+      exitSpy = spyOn(process, 'exit');
+
+      mock('@blackbaud/skyux-logger', {
+        error: errorSpy,
+        info: infoSpy
+      });
+
+      mock('remap-istanbul', {
+        remap: () => {
+          return {
+            fileCoverageFor: () => { },
+            files: () => [
+              'test.js'
+            ]
+          };
+        }
+      });
+    });
+
+    function createMergeSummaryObjectSpy(testPct) {
+      return jasmine.createSpy('mergeSummaryObjects').and.callFake(() => {
+        const summary = {};
+
+        coverageProps.forEach((key) => {
+          summary[key] = {
+            pct: testPct
+          };
+        });
+
+        return summary;
+      });
+    }
+
+    function mockIstanbul(mergeSummaryObjects) {
+      mock('istanbul', {
+        utils: {
+          summarizeFileCoverage: () => {},
+          mergeSummaryObjects
+        }
+      });
+    }
+
+    function mockConfig(codeCoverageThreshold) {
+      mock('../config/sky-pages/sky-pages.config.js', {
+        getSkyPagesConfig: () => ({
+          skyux: {
+            codeCoverageThreshold
+          }
+        })
+      });
+    }
+
+    function resetSpies() {
+      errorSpy.calls.reset();
+      infoSpy.calls.reset();
+      exitSpy.calls.reset();
+    }
+
+    function checkCodeCoverage(thresholdName, threshold, testPct, shouldPass) {
+      const mergeSummaryObjectsSpy = createMergeSummaryObjectSpy(testPct);
+
+      mockIstanbul(mergeSummaryObjectsSpy);
+      mockConfig(thresholdName);
+
+      resetSpies();
+
+      const browsers = ['Chrome', 'Firefox'];
+      const reporters = [
+        { type: 'json' },
+        { type: 'html' }
+      ];
+
+      mock.reRequire('../config/karma/shared.karma.conf')({
+        browsers: browsers,
+        set: (config) => {
+          config.coverageReporter.reporters = reporters;
+
+          const fakeCollector = {
+            getFinalCoverage: () => {
+              return {
+                files: () => []
+              };
+            }
+          };
+
+          // Simulate multiple reporters/browsers the same way that karma-coverage does.
+          reporters.forEach(() => {
+            browsers.forEach(() => {
+              config.coverageReporter._onWriteReport(fakeCollector);
+            });
+          });
+
+          // Code coverage should be evaluated once per browser unless the threshold is 0,
+          // in which case it should not be called at all.
+          expect(mergeSummaryObjectsSpy).toHaveBeenCalledTimes(
+            threshold === 0 ? 0 : browsers.length
+          );
+
+          // Verify the tests pass or fail based on the coverage percentage.
+          const doneSpy = jasmine.createSpy('done');
+
+          config.coverageReporter._onExit(doneSpy);
+
+          if (shouldPass) {
+            expect(exitSpy).not.toHaveBeenCalled();
+            expect(errorSpy).not.toHaveBeenCalled();
+            expect(infoSpy).not.toHaveBeenCalledWith('Karma has exited with 1.');
+          } else {
+            expect(exitSpy).toHaveBeenCalledWith(1);
+
+            browsers.forEach((browserName) => {
+              coverageProps.forEach((key) => {
+                expect(errorSpy).toHaveBeenCalledWith(
+                  `Coverage in ${browserName} for ${key} (${testPct}%) does not meet ` +
+                  `global threshold (${threshold}%)`
+                );
+              });
+            })
+
+            expect(infoSpy).toHaveBeenCalledWith('Karma has exited with 1.');
+          }
+
+          expect(doneSpy).toHaveBeenCalled();
+        }
+      });
+    }
+
+    it('should handle codeCoverageThreshold set to "none"', () => {
+      checkCodeCoverage('none', 0, 0, true);
+      checkCodeCoverage('none', 0, 1, true);
+    });
+
+    it('should handle codeCoverageThreshold set to "standard"', () => {
+      checkCodeCoverage('standard', 80, 79, false);
+      checkCodeCoverage('standard', 80, 80, true);
+      checkCodeCoverage('standard', 80, 81, true);
+    });
+
+    it('should handle codeCoverageThreshold set to "strict"', () => {
+      checkCodeCoverage('strict', 100, 99, false);
+      checkCodeCoverage('strict', 100, 100, true);
+    });
+  });
+
 });