TestResultAnalyzer.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.runtime;

import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.packages.TestSize;
import com.google.devtools.build.lib.packages.TestTimeout;
import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Prints results to the terminal, showing the results of each test target.
 */
@ThreadCompatible
public class TestResultAnalyzer {
  private final TestSummaryOptions summaryOptions;
  private final ExecutionOptions executionOptions;
  private final EventBus eventBus;

  // Store information about potential failures in the presence of --nokeep_going or
  // --notest_keep_going.
  private boolean skipTargetsOnFailure;

  /**
   * @param summaryOptions Parsed test summarization options.
   * @param executionOptions Parsed build/test execution options.
   * @param eventBus For reporting failed to build and cached tests.
   */
  public TestResultAnalyzer(TestSummaryOptions summaryOptions,
                            ExecutionOptions executionOptions,
                            EventBus eventBus) {
    this.summaryOptions = summaryOptions;
    this.executionOptions = executionOptions;
    this.eventBus = eventBus;
    eventBus.register(this);
  }

  @Subscribe
  public void doneBuild(BuildCompleteEvent event) {
    skipTargetsOnFailure = event.getResult().getStopOnFirstFailure();
  }

  /**
   * Prints out the results of the given tests, and returns true if they all passed.
   * Posts any targets which weren't already completed by the listener to the EventBus.
   * Reports all targets on the console via the given notifier.
   * Run at the end of the build, run only once.
   *
   * @param testTargets The list of targets being run
   * @param listener An aggregating listener with intermediate results
   * @param notifier A console notifier to echo results to.
   * @return true if all the tests passed, else false
   */
  public boolean differentialAnalyzeAndReport(
      Collection<ConfiguredTarget> testTargets,
      Collection<ConfiguredTarget> skippedTargets,
      AggregatingTestListener listener,
      TestResultNotifier notifier) {

    Preconditions.checkNotNull(testTargets);
    Preconditions.checkNotNull(listener);
    Preconditions.checkNotNull(notifier);

    // The natural ordering of the summaries defines their output order.
    Set<TestSummary> summaries = Sets.newTreeSet();

    int totalRun = 0; // Number of targets running at least one non-cached test.
    int passCount = 0;

    for (ConfiguredTarget testTarget : testTargets) {
      TestSummary summary = aggregateAndReportSummary(testTarget, listener).build();
      summaries.add(summary);

      // Finished aggregating; build the final console output.
      if (summary.actionRan()) {
        totalRun++;
      }

      if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
        passCount++;
      }
    }

    int summarySize = summaries.size();
    int testTargetsSize = testTargets.size();
    Preconditions.checkState(
        summarySize == testTargetsSize,
        "Unequal sizes: %s vs %s (%s and %s)",
        summarySize,
        testTargetsSize,
        summaries,
        testTargets);

    notifier.notify(summaries, totalRun);
    // skipped targets are not in passCount since they have NO_STATUS
    Set<ConfiguredTarget> testTargetsSet = new HashSet<>(testTargets);
    Set<ConfiguredTarget> skippedTargetsSet = new HashSet<>(skippedTargets);
    return passCount == Sets.difference(testTargetsSet, skippedTargetsSet).size();
  }

  private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
    return status.getNumber() > other.getNumber() ? status : other;
  }

  /**
   * Helper for differential analysis which aggregates the TestSummary
   * for an individual target, reporting runs on the EventBus if necessary.
   */
  private TestSummary.Builder aggregateAndReportSummary(
      ConfiguredTarget testTarget,
      AggregatingTestListener listener) {

    // If already reported by the listener, no work remains for this target.
    TestSummary.Builder summary = listener.getCurrentSummary(testTarget);
    Label testLabel = testTarget.getLabel();
    Preconditions.checkNotNull(summary,
        "%s did not complete test filtering, but has a test result", testLabel);
    if (listener.targetReported(testTarget)) {
      return summary;
    }

    Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget);
    Map<Artifact, TestResult> statusMap = listener.getStatusMap();

    // We will get back multiple TestResult instances if test had to be retried several
    // times before passing. Sharding and multiple runs of the same test without retries
    // will be represented by separate artifacts and will produce exactly one TestResult.
    for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
      // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
      // be null for an unrelated test because we were not able to even try to execute the test.
      // In that case, for tests that were previously passing we return null ( == NO STATUS),
      // because checking if the cached test target is up-to-date would require running the
      // dependency checker transitively.
      TestResult runResult = statusMap.get(testStatus);
      boolean isIncompleteRun = incompleteRuns.contains(testStatus);
      if (runResult == null) {
        summary = markIncomplete(summary);
      } else if (isIncompleteRun) {
        // Only process results which were not recorded by the listener.

        boolean newlyFetched = !statusMap.containsKey(testStatus);
        summary = incrementalAnalyze(summary, runResult);
        if (newlyFetched) {
          eventBus.post(runResult);
        }
        Preconditions.checkState(
            listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun,
            "TestListener changed in differential analysis. Ensure it isn't still registered.");
      }
    }

    // The target was not posted by the listener and must be posted now.
    eventBus.post(summary.build());
    return summary;
  }

  /**
   * Incrementally updates a TestSummary given an existing summary
   * and a new TestResult. Only call on built targets.
   *
   * @param summaryBuilder Existing unbuilt test summary associated with a target.
   * @param result New test result to aggregate into the summary.
   * @return The updated TestSummary.
   */
  public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder,
                                                TestResult result) {
    // Cache retrieval should have been performed already.
    Preconditions.checkNotNull(result);
    Preconditions.checkNotNull(summaryBuilder);
    TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek());

    TransitiveInfoCollection target = existingSummary.getTarget();
    Preconditions.checkNotNull(
        target, "The existing TestSummary must be associated with a target");

    BlazeTestStatus status = existingSummary.getStatus();
    int numCached = existingSummary.numCached();
    int numLocalActionCached = existingSummary.numLocalActionCached();

    // If a test was neither cached locally nor remotely we say action was taken.
    if (!(result.isCached() || result.getData().getRemotelyCached())) {
      summaryBuilder.setActionRan(true);
    } else {
      numCached++;
    }
    
    if (result.isCached()) {
      numLocalActionCached++;
    }
    
    Path coverageData = result.getCoverageData();
    if (coverageData != null) {
      summaryBuilder.addCoverageFiles(Collections.singletonList(coverageData));
    }

    if (!executionOptions.runsPerTestDetectsFlakes) {
      status = aggregateStatus(status, result.getData().getStatus());
    } else {
      int shardNumber = result.getShardNum();
      int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns();
      List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(
          shardNumber, result.getData().getStatus());
      if (singleShardStatuses.size() == runsPerTestForLabel) {
        BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
        int passes = 0;
        for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
          shardStatus = aggregateStatus(shardStatus, runStatusForShard);
          if (TestResult.isBlazeTestStatusPassed(runStatusForShard)) {
            passes++;
          }
        }
        // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass.
        // If all results pass or fail, aggregate the passing/failing shardStatus.
        if (passes == 0 || passes == runsPerTestForLabel) {
          status = aggregateStatus(status, shardStatus);
        } else {
          status = aggregateStatus(status, BlazeTestStatus.FLAKY);
        }
      }
    }

    List<Path> passed = new ArrayList<>();
    if (result.getData().hasPassedLog()) {
      passed.add(result.getTestAction().getTestLog().getPath().getRelative(
          result.getData().getPassedLog()));
    }

    List<Path> failed = new ArrayList<>();
    for (String path : result.getData().getFailedLogsList()) {
      failed.add(result.getTestAction().getTestLog().getPath().getRelative(path));
    }

    summaryBuilder
        .addTestTimes(result.getData().getTestTimesList())
        .addPassedLogs(passed)
        .addFailedLogs(failed)
        .addWarnings(result.getData().getWarningList())
        .collectFailedTests(result.getData().getTestCase())
        .setRanRemotely(result.getData().getIsRemoteStrategy());

    List<String> warnings = new ArrayList<>();
    if (status == BlazeTestStatus.PASSED
        && shouldEmitTestSizeWarningInSummary(
            summaryOptions.testVerboseTimeoutWarnings,
            warnings,
            result.getData().getTestProcessTimesList(),
            target)) {
      summaryBuilder.setWasUnreportedWrongSize(true);
    }

    return summaryBuilder
        .setStatus(status)
        .setNumCached(numCached)
        .setNumLocalActionCached(numLocalActionCached)
        .addWarnings(warnings);
  }

  private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) {
    // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
    // tests with no status and post it here.
    TestSummary summary = summaryBuilder.peek();
    BlazeTestStatus status = summary.getStatus();
    if (skipTargetsOnFailure) {
      status = BlazeTestStatus.NO_STATUS;
    } else if (status != BlazeTestStatus.NO_STATUS) {
      status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
    }

    return summaryBuilder.setStatus(status);
  }

  TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) {
    BlazeTestStatus runStatus =
        blazeHalted
            ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
            : (executionOptions.testCheckUpToDate || skipTargetsOnFailure
                ? BlazeTestStatus.NO_STATUS
                : BlazeTestStatus.FAILED_TO_BUILD);

    return summary.setStatus(runStatus);
  }

  /**
   * Checks whether the specified test timeout could have been smaller or is too small and adds a
   * warning message if verbose is true.
   *
   * <p>Returns true if there was a test with the wrong timeout, but if was not reported.
   */
  private static boolean shouldEmitTestSizeWarningInSummary(
      boolean verbose,
      List<String> warnings,
      List<Long> testTimes,
      TransitiveInfoCollection target) {

    TestTimeout specifiedTimeout =
        target.getProvider(TestProvider.class).getTestParams().getTimeout();
    long maxTimeOfShard = 0;

    for (Long shardTime : testTimes) {
      if (shardTime != null) {
        maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
      }
    }

    int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);

    if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
      TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
      TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
      if (verbose) {
        StringBuilder builder = new StringBuilder(String.format(
            "%s: Test execution time (%.1fs excluding execution overhead) outside of "
            + "range for %s tests. Consider setting timeout=\"%s\"",
            target.getLabel(),
            maxTimeOfShard / 1000.0,
            specifiedTimeout.prettyPrint(),
            expectedTimeout));
        if (expectedSize != null) {
          builder.append(" or size=\"").append(expectedSize).append("\"");
        }
        builder.append(".");
        warnings.add(builder.toString());
        return false;
      }
      return true;
    } else {
      return false;
    }
  }
}