-
-
Notifications
You must be signed in to change notification settings - Fork 645
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
78 additions
and
131 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| #!/usr/bin/env python3 | ||
| import os | ||
| import sys | ||
| import subprocess | ||
| import concurrent.futures | ||
| from collections import Counter | ||
| from typing import List, Dict, Tuple | ||
|
|
||
| NUM_PARALLEL = int(os.cpu_count() * 1.5) | ||
|
|
||
| def find_test_files(root_dir: str) -> List[str]: | ||
| """Find all executable files ending with _test recursively.""" | ||
| test_files = [] | ||
| for root, _, files in os.walk(root_dir): | ||
| for file in files: | ||
| if file.endswith('_test'): | ||
| file_path = os.path.join(root, file) | ||
| if os.access(file_path, os.X_OK): | ||
| test_files.append(file_path) | ||
| return test_files | ||
|
|
||
| def run_single_test(test_path: str) -> int: | ||
| """Run a single test and return its exit code.""" | ||
| try: | ||
| result = subprocess.run([test_path], capture_output=False) | ||
| return result.returncode | ||
| except Exception as e: | ||
| print(f"Error running {test_path}: {e}") | ||
| return -1 | ||
|
|
||
| def run_test_multiple_times(test_path: str, iterations: int = NUM_PARALLEL) -> List[int]: | ||
| """Run a test multiple times in parallel and collect exit codes.""" | ||
| with concurrent.futures.ProcessPoolExecutor() as executor: | ||
| futures = [executor.submit(run_single_test, test_path) for _ in range(iterations)] | ||
| return [f.result() for f in concurrent.futures.as_completed(futures)] | ||
|
|
||
| def analyze_results(test_path: str, exit_codes: List[int]) -> Tuple[bool, Dict[int, int]]: | ||
| """Analyze test results and return if it flaked and error distribution.""" | ||
| error_counts = Counter(code for code in exit_codes if code != 0) | ||
| return bool(error_counts), dict(error_counts) | ||
|
|
||
| def print_flaky_report(test_path: str, error_distribution: Dict[int, int], total_runs: int): | ||
| """Print a report for a flaky test.""" | ||
| print(f"{test_path} flaked!") | ||
| for exit_code, count in error_distribution.items(): | ||
| print(f"* {count}/{total_runs} processes died with exit code {exit_code}") | ||
|
|
||
| def main(directory = "o"): | ||
| test_files = find_test_files(directory) | ||
| for i, test_path in enumerate(test_files): | ||
| print("testing [%d/%d] %s..." % (i, len(test_files), test_path)) | ||
| sys.stdout.flush() | ||
| exit_codes = run_test_multiple_times(test_path) | ||
| is_flaky, error_distribution = analyze_results(test_path, exit_codes) | ||
| if is_flaky: | ||
| print_flaky_report(test_path, error_distribution, len(exit_codes)) | ||
| sys.exit(1) | ||
|
|
||
| if __name__ == "__main__": | ||
| main(*sys.argv[1:]) |