diff --git a/.github/workflows/benchmark_nightly_cpu.yml b/.github/workflows/benchmark_nightly_cpu.yml index 6d91438574..ac8487e416 100644 --- a/.github/workflows/benchmark_nightly_cpu.yml +++ b/.github/workflows/benchmark_nightly_cpu.yml @@ -43,6 +43,21 @@ jobs: with: name: nightly cpu artifact path: /tmp/ts_benchmark + - name: Download benchmark artifacts for auto validation + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + workflow_conclusion: success + if_no_artifact_found: ignore + path: /tmp/ts_artifacts + name: cpu_benchmark_validation + - name: Update benchmark artifacts for auto validation + run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/cpu_benchmark_validation + - name: Upload the updated benchmark artifacts for auto validation + uses: actions/upload-artifact@v2 + with: + name: cpu_benchmark_validation + path: /tmp/ts_artifacts - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} uses: dacbd/create-issue-action@v1 diff --git a/.github/workflows/benchmark_nightly_gpu.yml b/.github/workflows/benchmark_nightly_gpu.yml index 3211a87339..edf7de86b4 100644 --- a/.github/workflows/benchmark_nightly_gpu.yml +++ b/.github/workflows/benchmark_nightly_gpu.yml @@ -43,3 +43,18 @@ jobs: with: name: nightly gpu artifact path: /tmp/ts_benchmark + - name: Download benchmark artifacts for auto validation + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + workflow_conclusion: success + if_no_artifact_found: ignore + path: /tmp/ts_artifacts + name: gpu_benchmark_validation + - name: Update benchmark artifacts for auto validation + run: python benchmarks/utils/update_artifacts.py --output /tmp/ts_artifacts/gpu_benchmark_validation + - name: Upload the updated benchmark artifacts for auto validation + uses: actions/upload-artifact@v2 + with: + name: gpu_benchmark_validation + path: /tmp/ts_artifacts diff --git a/benchmarks/utils/update_artifacts.py b/benchmarks/utils/update_artifacts.py new file mode 100644 index 0000000000..28e41cce7d --- /dev/null +++ b/benchmarks/utils/update_artifacts.py @@ -0,0 +1,129 @@ +import argparse +import os +import shutil + +BENCHMARK_REPORT_PATH = "/tmp/ts_benchmark" +BENCHMARK_ARTIFACTS_PATH = "/tmp/ts_artifacts" +BENCHMARK_REPORT_FILE = "ab_report.csv" +WINDOW_LEN = 8 +WINDOW_START = 0 + +################################################################ +# This is an example directory structure for the artifacts. +# Here, report_id 1 is missing, new report would be added under 1 +# and we would remove report_id 2. +# . +# └── tmp/ +# └── ts_artifacts/ +# ├── 0/ +# │ ├── eager_mode_mnist_w4_b1/ +# │ │ └── ab_report.csv +# │ ├── eager_mode_mnist_w4_b2/ +# │ │ └── ab_report.csv +# │ └── ... +# ├── 2/ +# │ ├── eager_mode_mnist_w4_b1/ +# │ │ └── ab_report.csv +# │ ├── eager_mode_mnist_w4_b2/ +# │ │ └── ab_report.csv +# │ └── ... +# ├── 3/ +# │ ├── eager_mode_mnist_w4_b1/ +# │ │ └── ab_report.csv +# │ ├── eager_mode_mnist_w4_b2/ +# │ │ └── ab_report.csv +# │ └── ... +# ├── ... +# └── 6/ +# ├── eager_mode_mnist_w4_b1/ +# │ └── ab_report.csv +# ├── eager_mode_mnist_w4_b2/ +# │ └── ab_report.csv +# └── ... +################################################################ + + +# Copy BENCHMARK_REPORT_FILE to artifacts +def copy_benchmark_reports(input, output): + + for dir in os.listdir(input): + if os.path.isdir(os.path.join(input, dir)): + new_dir = os.path.join(output, dir) + os.makedirs(new_dir, exist_ok=True) + shutil.copy(os.path.join(input, dir, BENCHMARK_REPORT_FILE), new_dir) + + +# Save new report and delete the oldest report +def update_new_report(input_dir, output_dir, add_report_id, del_report_id): + + # Add new report + new_dir = os.path.join(output_dir, str(add_report_id)) + print("Creating artifacts ", new_dir) + copy_benchmark_reports(input_dir, new_dir) + + # Remove old report + if isinstance(del_report_id, int): + rm_dir = os.path.join(output_dir, str(del_report_id % WINDOW_LEN)) + print("Removing artifacts ", rm_dir) + shutil.rmtree(rm_dir, ignore_errors=True) + + +# Create artifacts for a period of rolling WINDOW_LEN-1 reports +def update_artifacts(input_dir, output_dir): + + # Create a drectory where artifacts will be stored + os.makedirs(output_dir, exist_ok=True) + + # Get the sorted list of existing report_ids + list_dirs = sorted(map(lambda x: int(x), os.listdir(output_dir))) + num_reports = len(list_dirs) + + # Initial case: When they are less than WINDOW_LEN-1 reports + if num_reports < WINDOW_LEN - 1: + add_report_id, del_report_id = num_reports, None + update_new_report(input_dir, output_dir, add_report_id, del_report_id) + return + + # When there are WINDOW_LEN - 1 reports and we want to add the new report + # and remove the oldest report + for i, report_id in enumerate(list_dirs): + + if i != report_id or (i + 1 == WINDOW_LEN - 1): + if i != report_id: + # When report_id has a missing element in sequence + add_report_id, del_report_id = i, report_id + else: + # When report_id WINDOW_LEN-1 is missing + add_report_id, del_report_id = i + 1, (i + 2) % WINDOW_LEN + update_new_report(input_dir, output_dir, add_report_id, del_report_id) + break + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--input_dir", + nargs="?", + help="the dir of a list of model benchmark result subdir ", + const=BENCHMARK_REPORT_PATH, + type=str, + default=BENCHMARK_REPORT_PATH, + ) + + parser.add_argument( + "--output_dir", + nargs="?", + help="the dir of model benchmark artifacts ", + const=BENCHMARK_ARTIFACTS_PATH, + type=str, + default=BENCHMARK_ARTIFACTS_PATH, + ) + + args = parser.parse_args() + + update_artifacts(args.input_dir, args.output_dir) + + +if __name__ == "__main__": + main()