From 93fb0890c9d62ec1391ff57282f7db922f87f064 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 8 Mar 2023 18:55:40 +0530 Subject: [PATCH 01/60] Initial commit --- asv/asv.conf.json | 8 ++++--- asv/benchmarks/benchmark_simulation.py | 30 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 asv/benchmarks/benchmark_simulation.py diff --git a/asv/asv.conf.json b/asv/asv.conf.json index 82b305d0773..78518ef1a8e 100644 --- a/asv/asv.conf.json +++ b/asv/asv.conf.json @@ -3,9 +3,11 @@ "project": "tardis", "project_url": "https://github.com/tardis-sn/tardis", "repo": "https://github.com/tardis-sn/tardis.git", + "branches": ["master"], // for git "matrix": { "numpy": [], - "astropy": [], - "Cython": [] - } + "astropy": [], + "Cython": [] + }, + "environment_type": "conda" } diff --git a/asv/benchmarks/benchmark_simulation.py b/asv/benchmarks/benchmark_simulation.py new file mode 100644 index 00000000000..0773fda64de --- /dev/null +++ b/asv/benchmarks/benchmark_simulation.py @@ -0,0 +1,30 @@ +"""Benchmark the simulation.""" +from tardis.simulation import Simulation +import os +from tardis.io.config_reader import Configuration + +class BenchmarkSimulation: + # params = ["tardis_configv1_verysimple.yml"] + + def setup(self): + filename = "tardis_configv1_verysimple.yml" + dir_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(dir_path, filename) + config = Configuration.from_yaml(path) + config.atom_data = "/home/atharva/workspace/code/tardis-main/tardis/docs/kurucz_cd23_chianti_H_He.h5" + sim = Simulation.from_config(config) + self.config = config + self.sim = sim + + def time_iterate(self): + self.sim.iterate(int(self.config.montecarlo.no_of_packets)) + + def time_advance_state(self): + _ = self.sim.advance_state() + + + + + + + From 8904b1cf69ea23fcf92e85953253e653960c1a1d Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 17 Mar 2023 18:54:58 +0530 Subject: [PATCH 02/60] Comment old benchmarks temporarily --- asv/asv.conf.json | 3 +- asv/benchmarks/benchmark_run_tardis.py | 18 +++ asv/benchmarks/benchmark_simulation.py | 30 ----- asv/benchmarks/benchmarks.py | 112 +++++++++--------- asv/benchmarks/tardis_configv1_verysimple.yml | 49 ++++++++ 5 files changed, 125 insertions(+), 87 deletions(-) create mode 100644 asv/benchmarks/benchmark_run_tardis.py delete mode 100644 asv/benchmarks/benchmark_simulation.py create mode 100644 asv/benchmarks/tardis_configv1_verysimple.yml diff --git a/asv/asv.conf.json b/asv/asv.conf.json index 78518ef1a8e..0357a14b9e3 100644 --- a/asv/asv.conf.json +++ b/asv/asv.conf.json @@ -1,8 +1,9 @@ { "version": 1, "project": "tardis", - "project_url": "https://github.com/tardis-sn/tardis", + "project_url": "https://tardis-sn.github.io/tardis", "repo": "https://github.com/tardis-sn/tardis.git", + "show_commit_url": "https://github.com/tardis-sn/tardis/commit", "branches": ["master"], // for git "matrix": { "numpy": [], diff --git a/asv/benchmarks/benchmark_run_tardis.py b/asv/benchmarks/benchmark_run_tardis.py new file mode 100644 index 00000000000..8e4100834ee --- /dev/null +++ b/asv/benchmarks/benchmark_run_tardis.py @@ -0,0 +1,18 @@ +"""Basic TARDIS Benchmark.""" +import os +from tardis.io.config_reader import Configuration +from tardis import run_tardis + +class Benchmarkruntardis: + """Class to benchmark the run_tardis function. + """ + def setup(self): + filename = "tardis_configv1_verysimple.yml" + dir_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(dir_path, filename) + config = Configuration.from_yaml(path) + config.atom_data = "kurucz_cd23_chianti_H_He.h5" + self.config = config + + def time_run_tardis(self): + sim = run_tardis(self.config) diff --git a/asv/benchmarks/benchmark_simulation.py b/asv/benchmarks/benchmark_simulation.py deleted file mode 100644 index 0773fda64de..00000000000 --- a/asv/benchmarks/benchmark_simulation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Benchmark the simulation.""" -from tardis.simulation import Simulation -import os -from tardis.io.config_reader import Configuration - -class BenchmarkSimulation: - # params = ["tardis_configv1_verysimple.yml"] - - def setup(self): - filename = "tardis_configv1_verysimple.yml" - dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, filename) - config = Configuration.from_yaml(path) - config.atom_data = "/home/atharva/workspace/code/tardis-main/tardis/docs/kurucz_cd23_chianti_H_He.h5" - sim = Simulation.from_config(config) - self.config = config - self.sim = sim - - def time_iterate(self): - self.sim.iterate(int(self.config.montecarlo.no_of_packets)) - - def time_advance_state(self): - _ = self.sim.advance_state() - - - - - - - diff --git a/asv/benchmarks/benchmarks.py b/asv/benchmarks/benchmarks.py index 1e828a7cd27..b9aa4b5cef7 100644 --- a/asv/benchmarks/benchmarks.py +++ b/asv/benchmarks/benchmarks.py @@ -1,66 +1,66 @@ -# Write the benchmarking functions here. -# See "Writing benchmarks" in the asv docs for more information. +# # Write the benchmarking functions here. +# # See "Writing benchmarks" in the asv docs for more information. -import numpy as np -from tardis.tests import montecarlo_test_wrappers as montecarlo +# import numpy as np +# from tardis.tests import montecarlo_test_wrappers as montecarlo -LINE_SIZE = 10000000 +# LINE_SIZE = 10000000 -class TimeSuite: - """ - An example benchmark that times the performance of various kinds - of iterating over dictionaries in Python. - """ +# class TimeSuite: +# """ +# An example benchmark that times the performance of various kinds +# of iterating over dictionaries in Python. +# """ - def setup(self): - self.line = np.arange(LINE_SIZE, 1, -1).astype(np.float64) +# def setup(self): +# self.line = np.arange(LINE_SIZE, 1, -1).astype(np.float64) - def time_binarysearch(self): - for _ in range(LINE_SIZE): - montecarlo.binary_search_wrapper( - self.line, np.random.random() * LINE_SIZE, 0, LINE_SIZE - 1 - ) +# def time_binarysearch(self): +# for _ in range(LINE_SIZE): +# montecarlo.binary_search_wrapper( +# self.line, np.random.random() * LINE_SIZE, 0, LINE_SIZE - 1 +# ) - def time_compute_distance2outer(self): - for _ in range(1000000): - montecarlo.compute_distance2outer_wrapper(0.0, 0.5, 1.0) - montecarlo.compute_distance2outer_wrapper(1.0, 0.5, 1.0) - montecarlo.compute_distance2outer_wrapper(0.3, 1.0, 1.0) - montecarlo.compute_distance2outer_wrapper(0.3, -1.0, 1.0) - montecarlo.compute_distance2outer_wrapper(0.5, 0.0, 1.0) +# def time_compute_distance2outer(self): +# for _ in range(1000000): +# montecarlo.compute_distance2outer_wrapper(0.0, 0.5, 1.0) +# montecarlo.compute_distance2outer_wrapper(1.0, 0.5, 1.0) +# montecarlo.compute_distance2outer_wrapper(0.3, 1.0, 1.0) +# montecarlo.compute_distance2outer_wrapper(0.3, -1.0, 1.0) +# montecarlo.compute_distance2outer_wrapper(0.5, 0.0, 1.0) - def time_compute_distance2inner(self): - for _ in range(1000000): - montecarlo.compute_distance2inner_wrapper(1.5, -1.0, 1.0) - montecarlo.compute_distance2inner_wrapper(0.0, 0.0, 0.0) - montecarlo.compute_distance2inner_wrapper(1.2, -0.7, 1.0) +# def time_compute_distance2inner(self): +# for _ in range(1000000): +# montecarlo.compute_distance2inner_wrapper(1.5, -1.0, 1.0) +# montecarlo.compute_distance2inner_wrapper(0.0, 0.0, 0.0) +# montecarlo.compute_distance2inner_wrapper(1.2, -0.7, 1.0) - def time_compute_distance2line(self): - for _ in range(1000000): - montecarlo.compute_distance2line_wrapper( - 2.20866912e15, - -0.251699059004, - 1.05581082105e15, - 1.06020910733e15, - 1693440.0, - 5.90513983371e-07, - 1.0602263591e15, - 1.06011723237e15, - 2, - ) - montecarlo.compute_distance2line_wrapper( - 2.23434667994e15, - -0.291130548401, - 1.05581082105e15, - 1.06733618121e15, - 1693440.0, - 5.90513983371e-07, - 1.06738407486e15, - 1.06732933961e15, - 3, - ) +# def time_compute_distance2line(self): +# for _ in range(1000000): +# montecarlo.compute_distance2line_wrapper( +# 2.20866912e15, +# -0.251699059004, +# 1.05581082105e15, +# 1.06020910733e15, +# 1693440.0, +# 5.90513983371e-07, +# 1.0602263591e15, +# 1.06011723237e15, +# 2, +# ) +# montecarlo.compute_distance2line_wrapper( +# 2.23434667994e15, +# -0.291130548401, +# 1.05581082105e15, +# 1.06733618121e15, +# 1693440.0, +# 5.90513983371e-07, +# 1.06738407486e15, +# 1.06732933961e15, +# 3, +# ) - def time_compute_distance2electron(self): - for _ in range(1000000): - montecarlo.compute_distance2electron_wrapper(0.0, 0.0, 2.0, 2.0) +# def time_compute_distance2electron(self): +# for _ in range(1000000): +# montecarlo.compute_distance2electron_wrapper(0.0, 0.0, 2.0, 2.0) diff --git a/asv/benchmarks/tardis_configv1_verysimple.yml b/asv/benchmarks/tardis_configv1_verysimple.yml new file mode 100644 index 00000000000..074f7764fa5 --- /dev/null +++ b/asv/benchmarks/tardis_configv1_verysimple.yml @@ -0,0 +1,49 @@ +tardis_config_version: v1.0 + +supernova: + luminosity_requested: 2.8e9 solLum + time_explosion: 13 day + +atom_data: kurucz_atom_pure_simple.h5 + +model: + structure: + type: specific + velocity: + start: 1.1e4 km/s + stop: 2.0e4 km/s + num: 20 + density: + type: branch85_w7 + abundances: + type: uniform + O: 0.19 + Mg: 0.03 + Si: 0.52 + S: 0.19 + Ar: 0.04 + Ca: 0.03 + +plasma: + ionization: lte + excitation: lte + radiative_rates_type: dilute-blackbody + line_interaction_type: macroatom + +montecarlo: + seed: 23111963 + no_of_packets : 2.0e+5 + iterations: 5 + last_no_of_packets: 5.0e+5 + no_of_virtual_packets: 5 + convergence_strategy: + type: damped + damping_constant: 0.5 + threshold: 0.05 + lock_t_inner_cycles: 1 + t_inner_update_exponent: -0.5 + +spectrum: + start: 500 angstrom + stop: 20000 angstrom + num: 10000 From f8fb484d36e5bcbff7101b111ca12cfdcea998fa Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 17 Mar 2023 19:01:01 +0530 Subject: [PATCH 03/60] Add timeout --- asv/benchmarks/benchmark_run_tardis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/asv/benchmarks/benchmark_run_tardis.py b/asv/benchmarks/benchmark_run_tardis.py index 8e4100834ee..15ab092ddcd 100644 --- a/asv/benchmarks/benchmark_run_tardis.py +++ b/asv/benchmarks/benchmark_run_tardis.py @@ -6,6 +6,8 @@ class Benchmarkruntardis: """Class to benchmark the run_tardis function. """ + timeout = 200 + def setup(self): filename = "tardis_configv1_verysimple.yml" dir_path = os.path.dirname(os.path.realpath(__file__)) From ac99c71d52dee27f0d5fe97adf8df3c5947c3447 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 20 Mar 2023 15:09:02 +0530 Subject: [PATCH 04/60] Change the tests workflow to run benchmarks --- .github/workflows/tests.yml | 92 +++++++++++++------ asv/asv.conf.json | 31 ++++++- asv/benchmarks/__init__.py | 0 asv/benchmarks/tardis_configv1_verysimple.yml | 8 +- 4 files changed, 94 insertions(+), 37 deletions(-) create mode 100644 asv/benchmarks/__init__.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf5810b368a..8fe6e03507b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ # # https://tardis-sn.github.io/tardis/development/continuous_integration.html -name: tests +name: tests-modified-benchmark-temp on: push: @@ -27,6 +27,10 @@ defaults: run: shell: bash -l {0} +permissions: + pages: write # to deploy to Pages + id-token: write # to verify the deployment + jobs: build: @@ -47,33 +51,33 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Clone tardis-sn/tardis-refdata - uses: actions/checkout@v2 - with: - repository: tardis-sn/tardis-refdata - path: tardis-refdata - lfs: false - - - name: Create LFS file list - run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - working-directory: tardis-refdata - - - name: Restore LFS cache - uses: actions/cache@v2 - id: lfs-cache - with: - path: tardis-refdata/.git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 - - - name: Git LFS Pull - run: git lfs pull - working-directory: tardis-refdata - if: steps.lfs-cache.outputs.cache-hit != 'true' + # - name: Clone tardis-sn/tardis-refdata + # uses: actions/checkout@v2 + # with: + # repository: tardis-sn/tardis-refdata + # path: tardis-refdata + # lfs: false + + # - name: Create LFS file list + # run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + # working-directory: tardis-refdata + + # - name: Restore LFS cache + # uses: actions/cache@v2 + # id: lfs-cache + # with: + # path: tardis-refdata/.git/lfs + # key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + + # - name: Git LFS Pull + # run: git lfs pull + # working-directory: tardis-refdata + # if: steps.lfs-cache.outputs.cache-hit != 'true' - - name: Git LFS Checkout - run: git lfs checkout - working-directory: tardis-refdata - if: steps.lfs-cache.outputs.cache-hit == 'true' + # - name: Git LFS Checkout + # run: git lfs checkout + # working-directory: tardis-refdata + # if: steps.lfs-cache.outputs.cache-hit == 'true' - name: Setup environment uses: conda-incubator/setup-miniconda@v2 @@ -95,9 +99,39 @@ jobs: - name: Install package run: pip install -e . + + - run: wget https://github.com/tardis-sn/tardis-refdata/raw/master/atom_data/kurucz_cd23_chianti_H_He.h5 + working-directory: asv/benchmarks + + # - name: Run tests + # run: pytest tardis ${{ env.PYTEST_FLAGS }} + - name: Install asv + run: pip install asv + + - run: asv machine --yes + working-directory: asv + + - name: Run benchmarks + run: asv run --verbose + working-directory: asv + + - uses: actions/upload-artifact@v2 + if: always() + with: + name: asv-benchmark-results-ubuntu-latest + path: asv/results + + - run: asv publish + working-directory: asv + + - uses: actions/configure-pages@v1 + - uses: actions/upload-pages-artifact@v1 + with: + path: asv/html - - name: Run tests - run: pytest tardis ${{ env.PYTEST_FLAGS }} + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 - name: Upload to Codecov run: bash <(curl -s https://codecov.io/bash) diff --git a/asv/asv.conf.json b/asv/asv.conf.json index 0357a14b9e3..227989327c8 100644 --- a/asv/asv.conf.json +++ b/asv/asv.conf.json @@ -6,9 +6,32 @@ "show_commit_url": "https://github.com/tardis-sn/tardis/commit", "branches": ["master"], // for git "matrix": { - "numpy": [], - "astropy": [], - "Cython": [] + "pip": [], + "numpy": ["1.19"], + "scipy": ["1.5"], + "pandas": ["1.0"], + "astropy": ["5.0"], + "numba": ["0.53"], + "numexpr": [], + "radioactivedecay": ["0.4.12"], + "networkx": [], + "pygraphviz": [], + "pyyaml": [], + "jsonschema": [], + "hdf5": ["1.10"], + "h5py": [], + "pytables": [], + "requests": [], + "tqdm": [], + "notebook": [], + "matplotlib-base": [], + "ipywidgets": [], + "plotly": [], + "qgrid": ["1.3.1"], + "beautifulsoup4": [], + "lxml": [] }, - "environment_type": "conda" + "environment_type": "conda", + "results_dir": "results", + "html_dir": "html" } diff --git a/asv/benchmarks/__init__.py b/asv/benchmarks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/asv/benchmarks/tardis_configv1_verysimple.yml b/asv/benchmarks/tardis_configv1_verysimple.yml index 074f7764fa5..ce48b3b363f 100644 --- a/asv/benchmarks/tardis_configv1_verysimple.yml +++ b/asv/benchmarks/tardis_configv1_verysimple.yml @@ -32,10 +32,10 @@ plasma: montecarlo: seed: 23111963 - no_of_packets : 2.0e+5 - iterations: 5 - last_no_of_packets: 5.0e+5 - no_of_virtual_packets: 5 + no_of_packets : 2.0e+10 + iterations: 10 + last_no_of_packets: 5.0e+7 + no_of_virtual_packets: 15 convergence_strategy: type: damped damping_constant: 0.5 From d19b17b399e31f7cf6f83b72ab6e4746b268227d Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 27 Mar 2023 13:52:58 +0530 Subject: [PATCH 05/60] Replace ASV config file by template one, make changes there, modify tests workflow --- .github/workflows/tests.yml | 36 ++-- asv.conf.json | 184 ++++++++++++++++++ asv/{benchmarks => }/__init__.py | 0 asv/asv.conf.json | 37 ---- asv/{benchmarks => }/benchmark_run_tardis.py | 2 +- asv/{benchmarks => }/benchmarks.py | 0 .../tardis_configv1_verysimple.yml | 0 tardis_env3.yml | 3 +- 8 files changed, 210 insertions(+), 52 deletions(-) create mode 100644 asv.conf.json rename asv/{benchmarks => }/__init__.py (100%) delete mode 100644 asv/asv.conf.json rename asv/{benchmarks => }/benchmark_run_tardis.py (91%) rename asv/{benchmarks => }/benchmarks.py (100%) rename asv/{benchmarks => data}/tardis_configv1_verysimple.yml (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8fe6e03507b..dcd0ab0779f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,14 +42,19 @@ jobs: label: linux-64 prefix: /usr/share/miniconda3/envs/tardis - - os: macos-latest - label: osx-64 - prefix: /Users/runner/miniconda3/envs/tardis + # - os: macos-latest + # label: osx-64 + # prefix: /Users/runner/miniconda3/envs/tardis name: ${{ matrix.label }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + repository: atharva-2001/tardis + ref: benchmark_simulation + + - run: tree ../ -L 4 # - name: Clone tardis-sn/tardis-refdata # uses: actions/checkout@v2 @@ -101,7 +106,7 @@ jobs: run: pip install -e . - run: wget https://github.com/tardis-sn/tardis-refdata/raw/master/atom_data/kurucz_cd23_chianti_H_He.h5 - working-directory: asv/benchmarks + working-directory: asv/data # - name: Run tests # run: pytest tardis ${{ env.PYTEST_FLAGS }} @@ -109,29 +114,34 @@ jobs: run: pip install asv - run: asv machine --yes - working-directory: asv - name: Run benchmarks - run: asv run --verbose - working-directory: asv + run: asv run --verbose -e - uses: actions/upload-artifact@v2 if: always() with: name: asv-benchmark-results-ubuntu-latest - path: asv/results + path: .asv/results - run: asv publish - working-directory: asv - uses: actions/configure-pages@v1 - uses: actions/upload-pages-artifact@v1 with: - path: asv/html + path: .asv/html - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 + # - name: Deploy to GitHub Pages + # id: deployment + # uses: actions/deploy-pages@v1 + # - name: Deploy asv + - uses: peaceiris/actions-gh-pages@v3 + with: + deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} + publish_dir: .asv/html + destination_dir: asv + user_name: 'TARDIS Bot' + user_email: 'tardis.sn.bot@gmail.com' - name: Upload to Codecov run: bash <(curl -s https://codecov.io/bash) diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 00000000000..dcc240d6d03 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,184 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "tardis", + + // The project's homepage + "project_url": "https://tardis-sn.github.io/tardis", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": ".", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + "build_command": [ + "python setup.py build", + "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + ], + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["benchmark_simulation"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + // "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/tardis-sn/tardis/commit", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + // "pythons": ["2.7", "3.6"], + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + // "conda_channels": ["conda-forge", "defaults"], + + // A conda environment file that is used for environment creation. + "conda_environment_file": "tardis_env3.yml", + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + // "matrix": { + // "req": { + // "numpy": [], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + // }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "asv", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": ".asv/env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": ".asv/results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": ".asv/html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + // "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, +} diff --git a/asv/benchmarks/__init__.py b/asv/__init__.py similarity index 100% rename from asv/benchmarks/__init__.py rename to asv/__init__.py diff --git a/asv/asv.conf.json b/asv/asv.conf.json deleted file mode 100644 index 227989327c8..00000000000 --- a/asv/asv.conf.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 1, - "project": "tardis", - "project_url": "https://tardis-sn.github.io/tardis", - "repo": "https://github.com/tardis-sn/tardis.git", - "show_commit_url": "https://github.com/tardis-sn/tardis/commit", - "branches": ["master"], // for git - "matrix": { - "pip": [], - "numpy": ["1.19"], - "scipy": ["1.5"], - "pandas": ["1.0"], - "astropy": ["5.0"], - "numba": ["0.53"], - "numexpr": [], - "radioactivedecay": ["0.4.12"], - "networkx": [], - "pygraphviz": [], - "pyyaml": [], - "jsonschema": [], - "hdf5": ["1.10"], - "h5py": [], - "pytables": [], - "requests": [], - "tqdm": [], - "notebook": [], - "matplotlib-base": [], - "ipywidgets": [], - "plotly": [], - "qgrid": ["1.3.1"], - "beautifulsoup4": [], - "lxml": [] - }, - "environment_type": "conda", - "results_dir": "results", - "html_dir": "html" -} diff --git a/asv/benchmarks/benchmark_run_tardis.py b/asv/benchmark_run_tardis.py similarity index 91% rename from asv/benchmarks/benchmark_run_tardis.py rename to asv/benchmark_run_tardis.py index 15ab092ddcd..62f9143d0d3 100644 --- a/asv/benchmarks/benchmark_run_tardis.py +++ b/asv/benchmark_run_tardis.py @@ -11,7 +11,7 @@ class Benchmarkruntardis: def setup(self): filename = "tardis_configv1_verysimple.yml" dir_path = os.path.dirname(os.path.realpath(__file__)) - path = os.path.join(dir_path, filename) + path = os.path.join(dir_path, "data", filename) config = Configuration.from_yaml(path) config.atom_data = "kurucz_cd23_chianti_H_He.h5" self.config = config diff --git a/asv/benchmarks/benchmarks.py b/asv/benchmarks.py similarity index 100% rename from asv/benchmarks/benchmarks.py rename to asv/benchmarks.py diff --git a/asv/benchmarks/tardis_configv1_verysimple.yml b/asv/data/tardis_configv1_verysimple.yml similarity index 100% rename from asv/benchmarks/tardis_configv1_verysimple.yml rename to asv/data/tardis_configv1_verysimple.yml diff --git a/tardis_env3.yml b/tardis_env3.yml index f4c7344e0c4..5de9d573d62 100644 --- a/tardis_env3.yml +++ b/tardis_env3.yml @@ -28,7 +28,6 @@ dependencies: - pyyaml - jsonschema - hdf5=1.10 - - h5py - pytables - requests - tqdm @@ -74,3 +73,5 @@ dependencies: # Other - git-lfs + - pip: + - h5py From f25d3691df42afbeb4c5fd1b5493ec957795d4d8 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 29 Mar 2023 17:07:13 +0530 Subject: [PATCH 06/60] Change the tests workflow back --- .github/workflows/benchmarks.yml | 100 ++++++++++++++++++++++++++++ .github/workflows/tests.yml | 110 ++++++++++--------------------- 2 files changed, 133 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/benchmarks.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000000..f48f8a03f15 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,100 @@ +name: benchmarks + +on: + push: + branches: + - '*' + + pull_request: + branches: + - '*' + +defaults: + run: + shell: bash -l {0} + +# permissions: +# pages: write # to deploy to Pages +# id-token: write # to verify the deployment + +jobs: + build: + strategy: + matrix: + include: + + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3/envs/tardis + + # - os: macos-latest + # label: osx-64 + # prefix: /Users/runner/miniconda3/envs/tardis + name: ${{ matrix.label }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + repository: atharva-2001/tardis + ref: benchmark_simulation + + - name: Setup environment + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: tardis + use-mamba: true + + - uses: actions/cache@v2 + with: + path: ${{ matrix.prefix }} + key: conda-${{ matrix.label }}-${{ hashFiles('conda-${{ matrix.label }}.lock') }}-${{ env.CACHE_NUMBER }} + id: cache-conda + + - name: Update environment + run: mamba update -n tardis --file conda-${{ matrix.label }}.lock + if: steps.cache-conda.outputs.cache-hit != 'true' + + - name: Install package + run: pip install -e . + + - run: wget https://github.com/tardis-sn/tardis-refdata/raw/master/atom_data/kurucz_cd23_chianti_H_He.h5 + working-directory: asv/data + + - name: Install asv + run: pip install asv + + - run: asv machine --yes + + - name: Run benchmarks for last 10 commits if not PR + if: github.event_name != 'pull_request_target' + run: | + git log -n 10 --pretty=format:"%H" >> tag_commits.txt + asv run HASHFILE:tag_commits.txt -e + asv publish + + - name: Push results to repo + if: github.event_name != 'pull_request_target' + continue-on-error: true + uses: cpina/github-action-push-to-another-repository@main + env: + API_TOKEN_GITHUB: ${{ secrets.DEPLOY_PAT }} + with: + source-directory: .asv + destination-github-username: atharva-2001 + destination-repository-name: tardis-benchmark-test-results + user-email: aryaatharva18@gmail.com + target-branch: gh-pages + + - name: Compare HEAD with master if PR + if: github.event_name == 'pull_request_target' + run: | + asv continuous --split --show-stderr master 'HEAD' | tee asv-output.log + if grep -q Traceback asv-output.log; then + echo "Some benchmarks have errors!" + + + + + \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dcd0ab0779f..67aed9fdc1a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ # # https://tardis-sn.github.io/tardis/development/continuous_integration.html -name: tests-modified-benchmark-temp +name: tests on: push: @@ -27,10 +27,6 @@ defaults: run: shell: bash -l {0} -permissions: - pages: write # to deploy to Pages - id-token: write # to verify the deployment - jobs: build: @@ -42,47 +38,42 @@ jobs: label: linux-64 prefix: /usr/share/miniconda3/envs/tardis - # - os: macos-latest - # label: osx-64 - # prefix: /Users/runner/miniconda3/envs/tardis + - os: macos-latest + label: osx-64 + prefix: /Users/runner/miniconda3/envs/tardis name: ${{ matrix.label }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + + - name: Clone tardis-sn/tardis-refdata + uses: actions/checkout@v2 with: - repository: atharva-2001/tardis - ref: benchmark_simulation - - - run: tree ../ -L 4 - - # - name: Clone tardis-sn/tardis-refdata - # uses: actions/checkout@v2 - # with: - # repository: tardis-sn/tardis-refdata - # path: tardis-refdata - # lfs: false - - # - name: Create LFS file list - # run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - # working-directory: tardis-refdata - - # - name: Restore LFS cache - # uses: actions/cache@v2 - # id: lfs-cache - # with: - # path: tardis-refdata/.git/lfs - # key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 - - # - name: Git LFS Pull - # run: git lfs pull - # working-directory: tardis-refdata - # if: steps.lfs-cache.outputs.cache-hit != 'true' + repository: tardis-sn/tardis-refdata + path: tardis-refdata + lfs: false + + - name: Create LFS file list + run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + working-directory: tardis-refdata + + - name: Restore LFS cache + uses: actions/cache@v2 + id: lfs-cache + with: + path: tardis-refdata/.git/lfs + key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + + - name: Git LFS Pull + run: git lfs pull + working-directory: tardis-refdata + if: steps.lfs-cache.outputs.cache-hit != 'true' - # - name: Git LFS Checkout - # run: git lfs checkout - # working-directory: tardis-refdata - # if: steps.lfs-cache.outputs.cache-hit == 'true' + - name: Git LFS Checkout + run: git lfs checkout + working-directory: tardis-refdata + if: steps.lfs-cache.outputs.cache-hit == 'true' - name: Setup environment uses: conda-incubator/setup-miniconda@v2 @@ -104,44 +95,9 @@ jobs: - name: Install package run: pip install -e . - - - run: wget https://github.com/tardis-sn/tardis-refdata/raw/master/atom_data/kurucz_cd23_chianti_H_He.h5 - working-directory: asv/data - # - name: Run tests - # run: pytest tardis ${{ env.PYTEST_FLAGS }} - - name: Install asv - run: pip install asv - - - run: asv machine --yes - - - name: Run benchmarks - run: asv run --verbose -e - - - uses: actions/upload-artifact@v2 - if: always() - with: - name: asv-benchmark-results-ubuntu-latest - path: .asv/results - - - run: asv publish - - - uses: actions/configure-pages@v1 - - uses: actions/upload-pages-artifact@v1 - with: - path: .asv/html - - # - name: Deploy to GitHub Pages - # id: deployment - # uses: actions/deploy-pages@v1 - # - name: Deploy asv - - uses: peaceiris/actions-gh-pages@v3 - with: - deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} - publish_dir: .asv/html - destination_dir: asv - user_name: 'TARDIS Bot' - user_email: 'tardis.sn.bot@gmail.com' + - name: Run tests + run: pytest tardis ${{ env.PYTEST_FLAGS }} - name: Upload to Codecov - run: bash <(curl -s https://codecov.io/bash) + run: bash <(curl -s https://codecov.io/bash) \ No newline at end of file From a2d20e2bf8b52cad668b3f56f65263e54d90e8ad Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 30 Mar 2023 16:27:12 +0530 Subject: [PATCH 07/60] Change token --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f48f8a03f15..e6e01bbec17 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -79,7 +79,7 @@ jobs: continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: - API_TOKEN_GITHUB: ${{ secrets.DEPLOY_PAT }} + API_TOKEN_GITHUB: ${{ secrets.DEPLOY_PAT_TEMP}} with: source-directory: .asv destination-github-username: atharva-2001 From 8942ba3f6a29c7293b638ac33471c14256299962 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 7 Apr 2023 17:41:27 +0530 Subject: [PATCH 08/60] reduce the number of packets --- .github/workflows/benchmarks.yml | 1 + asv/data/tardis_configv1_verysimple.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e6e01bbec17..de857f8a2d4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -37,6 +37,7 @@ jobs: with: repository: atharva-2001/tardis ref: benchmark_simulation + fetch-depth: 10 - name: Setup environment uses: conda-incubator/setup-miniconda@v2 diff --git a/asv/data/tardis_configv1_verysimple.yml b/asv/data/tardis_configv1_verysimple.yml index ce48b3b363f..fc73d697f2b 100644 --- a/asv/data/tardis_configv1_verysimple.yml +++ b/asv/data/tardis_configv1_verysimple.yml @@ -32,9 +32,9 @@ plasma: montecarlo: seed: 23111963 - no_of_packets : 2.0e+10 + no_of_packets : 2.0e+2 iterations: 10 - last_no_of_packets: 5.0e+7 + last_no_of_packets: 5.0e+1 no_of_virtual_packets: 15 convergence_strategy: type: damped From 7d64f2e545fb687b9be360d336ec14478556b275 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 7 Apr 2023 18:05:11 +0530 Subject: [PATCH 09/60] change config again --- .github/workflows/benchmarks.yml | 2 +- asv/data/tardis_configv1_verysimple.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index de857f8a2d4..2c3be961ace 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -37,7 +37,7 @@ jobs: with: repository: atharva-2001/tardis ref: benchmark_simulation - fetch-depth: 10 + # fetch-depth: 10 - name: Setup environment uses: conda-incubator/setup-miniconda@v2 diff --git a/asv/data/tardis_configv1_verysimple.yml b/asv/data/tardis_configv1_verysimple.yml index fc73d697f2b..d5ef243450d 100644 --- a/asv/data/tardis_configv1_verysimple.yml +++ b/asv/data/tardis_configv1_verysimple.yml @@ -35,7 +35,7 @@ montecarlo: no_of_packets : 2.0e+2 iterations: 10 last_no_of_packets: 5.0e+1 - no_of_virtual_packets: 15 + no_of_virtual_packets: 1 convergence_strategy: type: damped damping_constant: 0.5 From 28eb478eb256357157ec9517fb5e216263592d40 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 7 Apr 2023 18:28:17 +0530 Subject: [PATCH 10/60] Change config again --- asv/data/tardis_configv1_verysimple.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/asv/data/tardis_configv1_verysimple.yml b/asv/data/tardis_configv1_verysimple.yml index d5ef243450d..22e1c9ac30b 100644 --- a/asv/data/tardis_configv1_verysimple.yml +++ b/asv/data/tardis_configv1_verysimple.yml @@ -32,9 +32,9 @@ plasma: montecarlo: seed: 23111963 - no_of_packets : 2.0e+2 - iterations: 10 - last_no_of_packets: 5.0e+1 + no_of_packets : 400 + iterations: 2 + last_no_of_packets: -1 no_of_virtual_packets: 1 convergence_strategy: type: damped @@ -46,4 +46,4 @@ montecarlo: spectrum: start: 500 angstrom stop: 20000 angstrom - num: 10000 + num: 200 From 8277f7b842050179e27238477da7047d2ac0708c Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 16:12:21 +0530 Subject: [PATCH 11/60] Remove environment --- .github/workflows/benchmarks.yml | 48 +++++++++++++++++--------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 2c3be961ace..edef0f822e4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -39,29 +39,33 @@ jobs: ref: benchmark_simulation # fetch-depth: 10 - - name: Setup environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: tardis - use-mamba: true + # - name: Setup environment + # uses: conda-incubator/setup-miniconda@v2 + # with: + # miniforge-variant: Mambaforge + # miniforge-version: latest + # activate-environment: tardis + # use-mamba: true - - uses: actions/cache@v2 - with: - path: ${{ matrix.prefix }} - key: conda-${{ matrix.label }}-${{ hashFiles('conda-${{ matrix.label }}.lock') }}-${{ env.CACHE_NUMBER }} - id: cache-conda + # - uses: actions/cache@v2 + # with: + # path: ${{ matrix.prefix }} + # key: conda-${{ matrix.label }}-${{ hashFiles('conda-${{ matrix.label }}.lock') }}-${{ env.CACHE_NUMBER }} + # id: cache-conda - - name: Update environment - run: mamba update -n tardis --file conda-${{ matrix.label }}.lock - if: steps.cache-conda.outputs.cache-hit != 'true' + # - name: Update environment + # run: mamba update -n tardis --file conda-${{ matrix.label }}.lock + # if: steps.cache-conda.outputs.cache-hit != 'true' + + # - name: Install package + # run: pip install -e . + + - run: wget https://raw.githubusercontent.com/tardis-sn/tardis/83cd1f13088e95e087c97370af04bda6cf9956b7/.ci-helpers/download_reference_data.sh + # working-directory: asv/data - - name: Install package - run: pip install -e . + - run: bash .ci-helpers/download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 - - run: wget https://github.com/tardis-sn/tardis-refdata/raw/master/atom_data/kurucz_cd23_chianti_H_He.h5 - working-directory: asv/data + - run: mv tardis-refdata/atom-data/kurucz_cd23_chianti_H_He.h5 asv/data/ - name: Install asv run: pip install asv @@ -69,14 +73,14 @@ jobs: - run: asv machine --yes - name: Run benchmarks for last 10 commits if not PR - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request' run: | git log -n 10 --pretty=format:"%H" >> tag_commits.txt asv run HASHFILE:tag_commits.txt -e asv publish - name: Push results to repo - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request' continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: @@ -89,7 +93,7 @@ jobs: target-branch: gh-pages - name: Compare HEAD with master if PR - if: github.event_name == 'pull_request_target' + if: github.event_name == 'pull_request' run: | asv continuous --split --show-stderr master 'HEAD' | tee asv-output.log if grep -q Traceback asv-output.log; then From 02395bea66dd61bc7eb710359e8a3ffb51bb4883 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 16:55:09 +0530 Subject: [PATCH 12/60] Fix paths --- .github/workflows/benchmarks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index edef0f822e4..82ba8345cae 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -63,9 +63,11 @@ jobs: - run: wget https://raw.githubusercontent.com/tardis-sn/tardis/83cd1f13088e95e087c97370af04bda6cf9956b7/.ci-helpers/download_reference_data.sh # working-directory: asv/data - - run: bash .ci-helpers/download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 + - run: bash download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 + + - run: tree ./tardis-refdata -L 2 - - run: mv tardis-refdata/atom-data/kurucz_cd23_chianti_H_He.h5 asv/data/ + - run: mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 asv/data/ - name: Install asv run: pip install asv From 8e89893b3f31ad8c06fc69321fb54337112ee7b3 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 17:08:23 +0530 Subject: [PATCH 13/60] Display comment --- .github/workflows/benchmarks.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 82ba8345cae..4e25930a54f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -97,9 +97,19 @@ jobs: - name: Compare HEAD with master if PR if: github.event_name == 'pull_request' run: | - asv continuous --split --show-stderr master 'HEAD' | tee asv-output.log - if grep -q Traceback asv-output.log; then - echo "Some benchmarks have errors!" + asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} | tee asv-output.log + if grep -q Traceback asv-output.log; then + echo "Some benchmarks have errors!" + fi + + - name: Post comment + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v2 + with: + token: ${{ secrets.API_TOKEN_GITHUB }} + issue-number: ${{ github.event.pull_request.number }} + # edit-mode: replace + body-file: asv-output.log From ddc2de6f8b52ab8cd1313be5f0fc35de20fa61b4 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 17:12:08 +0530 Subject: [PATCH 14/60] Try PAT --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 4e25930a54f..bce416cb8c3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -106,7 +106,7 @@ jobs: if: github.event_name == 'pull_request' uses: peter-evans/create-or-update-comment@v2 with: - token: ${{ secrets.API_TOKEN_GITHUB }} + token: ${{ secrets.DEPLOY_PAT_TEMP }} issue-number: ${{ github.event.pull_request.number }} # edit-mode: replace body-file: asv-output.log From aa31c299b64a016994364d6699f0cd78cebb2c19 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 17:16:57 +0530 Subject: [PATCH 15/60] Bot token --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index bce416cb8c3..d60216092d6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -106,7 +106,7 @@ jobs: if: github.event_name == 'pull_request' uses: peter-evans/create-or-update-comment@v2 with: - token: ${{ secrets.DEPLOY_PAT_TEMP }} + token: ${{ secrets.BOT_TOKEN}} issue-number: ${{ github.event.pull_request.number }} # edit-mode: replace body-file: asv-output.log From 1b8530c09e6e26b3f29ee60a229c41ffe43672e4 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 13 Apr 2023 17:40:21 +0530 Subject: [PATCH 16/60] asv continuous fails sporadically, continue-on-error temporarily --- .github/workflows/benchmarks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d60216092d6..4d85e7931e8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -96,6 +96,7 @@ jobs: - name: Compare HEAD with master if PR if: github.event_name == 'pull_request' + continue-on-error: true # TODO: step failed sporadically while testing run: | asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} | tee asv-output.log if grep -q Traceback asv-output.log; then From e333e422f30b575f72eac8ecd5d3cb6966268f19 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 09:30:10 +0530 Subject: [PATCH 17/60] Rename asv directory to benchmarks --- asv.conf.json | 2 +- {asv => benchmarks}/__init__.py | 0 {asv => benchmarks}/benchmark_run_tardis.py | 0 {asv => benchmarks}/benchmarks.py | 0 {asv => benchmarks}/data/tardis_configv1_verysimple.yml | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename {asv => benchmarks}/__init__.py (100%) rename {asv => benchmarks}/benchmark_run_tardis.py (100%) rename {asv => benchmarks}/benchmarks.py (100%) rename {asv => benchmarks}/data/tardis_configv1_verysimple.yml (100%) diff --git a/asv.conf.json b/asv.conf.json index dcc240d6d03..cd498a96955 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -136,7 +136,7 @@ // The directory (relative to the current directory) that benchmarks are // stored in. If not provided, defaults to "benchmarks" - "benchmark_dir": "asv", + "benchmark_dir": "benchmarks", // The directory (relative to the current directory) to cache the Python // environments in. If not provided, defaults to "env" diff --git a/asv/__init__.py b/benchmarks/__init__.py similarity index 100% rename from asv/__init__.py rename to benchmarks/__init__.py diff --git a/asv/benchmark_run_tardis.py b/benchmarks/benchmark_run_tardis.py similarity index 100% rename from asv/benchmark_run_tardis.py rename to benchmarks/benchmark_run_tardis.py diff --git a/asv/benchmarks.py b/benchmarks/benchmarks.py similarity index 100% rename from asv/benchmarks.py rename to benchmarks/benchmarks.py diff --git a/asv/data/tardis_configv1_verysimple.yml b/benchmarks/data/tardis_configv1_verysimple.yml similarity index 100% rename from asv/data/tardis_configv1_verysimple.yml rename to benchmarks/data/tardis_configv1_verysimple.yml From fa3a44d958b2664791d50ce970ab9f3282bdf163 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 09:32:36 +0530 Subject: [PATCH 18/60] Try running for 5 commits --- .github/workflows/benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 4d85e7931e8..94522b33bbc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -74,10 +74,10 @@ jobs: - run: asv machine --yes - - name: Run benchmarks for last 10 commits if not PR - if: github.event_name != 'pull_request' + - name: Run benchmarks for last 5 commits if not PR + # if: github.event_name != 'pull_request' run: | - git log -n 10 --pretty=format:"%H" >> tag_commits.txt + git log -n 5 --pretty=format:"%H" >> tag_commits.txt asv run HASHFILE:tag_commits.txt -e asv publish From e3aa5655b2752d11cb7260d76a0ac8548653449f Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 09:44:46 +0530 Subject: [PATCH 19/60] Change branch on my workflow --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 94522b33bbc..ad714b2cbdc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v2 with: repository: atharva-2001/tardis - ref: benchmark_simulation + ref: benchmark_base # fetch-depth: 10 # - name: Setup environment From a4d1526d34a8ded15bcc93135c20fc7751fb2676 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 09:51:03 +0530 Subject: [PATCH 20/60] asv -> benchmarks --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ad714b2cbdc..987ef077c7f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -67,7 +67,7 @@ jobs: - run: tree ./tardis-refdata -L 2 - - run: mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 asv/data/ + - run: mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 benchmarks/data/ - name: Install asv run: pip install asv From 140128dd0db04440a775d128d1e569827b3ae122 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 09:55:58 +0530 Subject: [PATCH 21/60] Clean up benchmarks workflow --- .github/workflows/benchmarks.yml | 46 ++++++-------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 987ef077c7f..b2d4f163f72 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -30,6 +30,7 @@ jobs: # - os: macos-latest # label: osx-64 # prefix: /Users/runner/miniconda3/envs/tardis + name: ${{ matrix.label }} runs-on: ${{ matrix.os }} steps: @@ -37,47 +38,21 @@ jobs: with: repository: atharva-2001/tardis ref: benchmark_base - # fetch-depth: 10 - # - name: Setup environment - # uses: conda-incubator/setup-miniconda@v2 - # with: - # miniforge-variant: Mambaforge - # miniforge-version: latest - # activate-environment: tardis - # use-mamba: true - - # - uses: actions/cache@v2 - # with: - # path: ${{ matrix.prefix }} - # key: conda-${{ matrix.label }}-${{ hashFiles('conda-${{ matrix.label }}.lock') }}-${{ env.CACHE_NUMBER }} - # id: cache-conda - - # - name: Update environment - # run: mamba update -n tardis --file conda-${{ matrix.label }}.lock - # if: steps.cache-conda.outputs.cache-hit != 'true' - - # - name: Install package - # run: pip install -e . - - - run: wget https://raw.githubusercontent.com/tardis-sn/tardis/83cd1f13088e95e087c97370af04bda6cf9956b7/.ci-helpers/download_reference_data.sh - # working-directory: asv/data - - - run: bash download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 - - - run: tree ./tardis-refdata -L 2 - - - run: mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 benchmarks/data/ - + - name: Download Atom Data + run: | + wget https://raw.githubusercontent.com/tardis-sn/tardis/83cd1f13088e95e087c97370af04bda6cf9956b7/.ci-helpers/download_reference_data.sh + bash download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 + mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 benchmarks/data/ + - name: Install asv run: pip install asv - - run: asv machine --yes - - name: Run benchmarks for last 5 commits if not PR # if: github.event_name != 'pull_request' run: | git log -n 5 --pretty=format:"%H" >> tag_commits.txt + asv machine --yes asv run HASHFILE:tag_commits.txt -e asv publish @@ -109,10 +84,5 @@ jobs: with: token: ${{ secrets.BOT_TOKEN}} issue-number: ${{ github.event.pull_request.number }} - # edit-mode: replace body-file: asv-output.log - - - - \ No newline at end of file From fdb38fbabe78f3a94efc013372baa1428dddbad3 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 10:26:02 +0530 Subject: [PATCH 22/60] Move publish to separate step --- .github/workflows/benchmarks.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b2d4f163f72..fabe6160589 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -30,7 +30,7 @@ jobs: # - os: macos-latest # label: osx-64 # prefix: /Users/runner/miniconda3/envs/tardis - + name: ${{ matrix.label }} runs-on: ${{ matrix.os }} steps: @@ -54,7 +54,8 @@ jobs: git log -n 5 --pretty=format:"%H" >> tag_commits.txt asv machine --yes asv run HASHFILE:tag_commits.txt -e - asv publish + + - run: asv publish - name: Push results to repo if: github.event_name != 'pull_request' From 693fa349e2cab569a78c9b48524eec360d0e6525 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 10:41:00 +0530 Subject: [PATCH 23/60] do not show logs and progress bars --- benchmarks/benchmark_run_tardis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark_run_tardis.py b/benchmarks/benchmark_run_tardis.py index 62f9143d0d3..b59292b486d 100644 --- a/benchmarks/benchmark_run_tardis.py +++ b/benchmarks/benchmark_run_tardis.py @@ -17,4 +17,4 @@ def setup(self): self.config = config def time_run_tardis(self): - sim = run_tardis(self.config) + sim = run_tardis(self.config, log_level="ERROR", show_progress_bars=False) From 66e0b7a62b9ee0178ef3c3c66bca1b27af3a6f64 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 10:43:31 +0530 Subject: [PATCH 24/60] Change branch in asv config file --- asv.conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv.conf.json b/asv.conf.json index cd498a96955..c23f40d58b4 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -30,7 +30,7 @@ // List of branches to benchmark. If not provided, defaults to "master" // (for git) or "default" (for mercurial). - "branches": ["benchmark_simulation"], // for git + "branches": ["benchmark_base"], // for git // "branches": ["default"], // for mercurial // The DVCS being used. If not set, it will be automatically From 828e5b89c009c79fd6904b4b4032af32e47f3250 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 10:52:52 +0530 Subject: [PATCH 25/60] Fix commits in asv cont. --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index fabe6160589..3c5e6edd7b6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -74,7 +74,7 @@ jobs: if: github.event_name == 'pull_request' continue-on-error: true # TODO: step failed sporadically while testing run: | - asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} | tee asv-output.log + asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log if grep -q Traceback asv-output.log; then echo "Some benchmarks have errors!" fi From c0bdfb91b4060a2aa11c43810966d3bfd5a8bca1 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 14 Apr 2023 10:54:52 +0530 Subject: [PATCH 26/60] Set target directory in push-to-another-repo --- .github/workflows/benchmarks.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 3c5e6edd7b6..ad124978d41 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -58,7 +58,7 @@ jobs: - run: asv publish - name: Push results to repo - if: github.event_name != 'pull_request' + # if: github.event_name != 'pull_request' continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: @@ -68,7 +68,8 @@ jobs: destination-github-username: atharva-2001 destination-repository-name: tardis-benchmark-test-results user-email: aryaatharva18@gmail.com - target-branch: gh-pages + target-branch: main + target-directory: .asv - name: Compare HEAD with master if PR if: github.event_name == 'pull_request' From 04bc7dded5f0523d6d573eb496f7c915f78dd70f Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 17 Apr 2023 12:24:19 +0530 Subject: [PATCH 27/60] Change trigger to pull request and push on master --- .github/workflows/benchmarks.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ad124978d41..906ffa59a56 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,13 +1,10 @@ name: benchmarks on: - push: - branches: - - '*' - pull_request: + push: branches: - - '*' + - master defaults: run: @@ -33,6 +30,7 @@ jobs: name: ${{ matrix.label }} runs-on: ${{ matrix.os }} + # if: contains(github.event.label.name, 'benchmark') steps: - uses: actions/checkout@v2 with: @@ -55,7 +53,11 @@ jobs: asv machine --yes asv run HASHFILE:tag_commits.txt -e - - run: asv publish + - name: Generate Graphs and HTML + run: asv publish + + - name: Delete env files + run: rm -r .asv/env - name: Push results to repo # if: github.event_name != 'pull_request' @@ -87,4 +89,11 @@ jobs: token: ${{ secrets.BOT_TOKEN}} issue-number: ${{ github.event.pull_request.number }} body-file: asv-output.log + + - name: Save results artifact + uses: actions/upload-artifact@v3 + if: always() + with: + name: asv-benchmark-results-${{ runner.os }} + path: .asv/results \ No newline at end of file From 3a41f339ff018cc9ee2560e46c86477ba50bf388 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 17 Apr 2023 18:48:20 +0530 Subject: [PATCH 28/60] Delete commented permissions --- .github/workflows/benchmarks.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 906ffa59a56..ad55044f3f1 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -10,16 +10,11 @@ defaults: run: shell: bash -l {0} -# permissions: -# pages: write # to deploy to Pages -# id-token: write # to verify the deployment - jobs: build: strategy: matrix: include: - - os: ubuntu-latest label: linux-64 prefix: /usr/share/miniconda3/envs/tardis @@ -88,7 +83,7 @@ jobs: with: token: ${{ secrets.BOT_TOKEN}} issue-number: ${{ github.event.pull_request.number }} - body-file: asv-output.log + body-file: asv-output.log - name: Save results artifact uses: actions/upload-artifact@v3 From ec27009de5b05a06db5d1c119357b6125f76a201 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 17 Apr 2023 19:12:01 +0530 Subject: [PATCH 29/60] Modify the comment, trigger via [bench] word --- .github/workflows/benchmarks.yml | 37 ++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index ad55044f3f1..d0332e53ee3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,6 +2,7 @@ name: benchmarks on: pull_request: + types: [labeled, synchronize] push: branches: - master @@ -25,7 +26,7 @@ jobs: name: ${{ matrix.label }} runs-on: ${{ matrix.os }} - # if: contains(github.event.label.name, 'benchmark') + if: "!contains(github.event.head_commit.message, 'bench')" steps: - uses: actions/checkout@v2 with: @@ -46,7 +47,11 @@ jobs: run: | git log -n 5 --pretty=format:"%H" >> tag_commits.txt asv machine --yes - asv run HASHFILE:tag_commits.txt -e + asv run HASHFILE:tag_commits.txt -e | tee asv-output.log + if grep -q Traceback asv-output.log; then + echo "Some benchmarks have errors!" + exit 1 + fi - name: Generate Graphs and HTML run: asv publish @@ -54,7 +59,7 @@ jobs: - name: Delete env files run: rm -r .asv/env - - name: Push results to repo + - name: Push results to results repository # if: github.event_name != 'pull_request' continue-on-error: true uses: cpina/github-action-push-to-another-repository@main @@ -75,15 +80,39 @@ jobs: asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log if grep -q Traceback asv-output.log; then echo "Some benchmarks have errors!" + exit 1 fi + - name: Output logs for comment + id: asv_logs + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "text<<$EOF" >> $GITHUB_OUTPUT + echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + - name: Post comment if: github.event_name == 'pull_request' uses: peter-evans/create-or-update-comment@v2 with: token: ${{ secrets.BOT_TOKEN}} issue-number: ${{ github.event.pull_request.number }} - body-file: asv-output.log + body: | + *\*beep\* \*bop\** + Hi human, + I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). + Here are the logs produced by ASV. + Results can also be downloaded as artifacts [**here**](${{ env.URL }}). +
+ + ``` + ${{join(steps.asv_logs.outputs.*, '\n')}} + ${{ steps.asv_logs.outputs.LOGS }} + ``` +
+ env: + URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true + - name: Save results artifact uses: actions/upload-artifact@v3 From 255ab0a09d7a412b82872f11a66db49175bad945 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 12:08:38 +0530 Subject: [PATCH 30/60] Send comments via github actions --- .github/workflows/benchmarks.yml | 39 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d0332e53ee3..33378fe318f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -91,25 +91,32 @@ jobs: echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT - - name: Post comment - if: github.event_name == 'pull_request' - uses: peter-evans/create-or-update-comment@v2 + - name: Write comment + uses: actions/github-script@v6 with: - token: ${{ secrets.BOT_TOKEN}} - issue-number: ${{ github.event.pull_request.number }} - body: | - *\*beep\* \*bop\** - Hi human, - I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). - Here are the logs produced by ASV. - Results can also be downloaded as artifacts [**here**](${{ env.URL }}). -
- - ``` + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: ` + *\*beep\* \*bop\** + Hi human, + I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). + Here are the logs produced by ASV. + Results can also be downloaded as artifacts [**here**](${{ env.URL }}). + +
+ 
+              
               ${{join(steps.asv_logs.outputs.*, '\n')}}
               ${{ steps.asv_logs.outputs.LOGS }}
-              ```
-
+ + +
+ + ` + }) env: URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true From 05565a6a7ac91f167cbf8de081c3b500f82a9f10 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 12:49:28 +0530 Subject: [PATCH 31/60] Download atom data via URL --- .github/workflows/benchmarks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 33378fe318f..e52063151a1 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -35,9 +35,8 @@ jobs: - name: Download Atom Data run: | - wget https://raw.githubusercontent.com/tardis-sn/tardis/83cd1f13088e95e087c97370af04bda6cf9956b7/.ci-helpers/download_reference_data.sh - bash download_reference_data.sh atom_data/kurucz_cd23_chianti_H_He.h5 - mv ./tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 benchmarks/data/ + REPO_URL="https://dev.azure.com/tardis-sn/TARDIS/_apis/git/repositories/tardis-refdata" + wget -q "$REPO_URL/items?path=atom_data/kurucz_cd23_chianti_H_He.h5&resolveLfs=true" -O benchmarks/data/kurucz_cd23_chianti_H_He.h5 - name: Install asv run: pip install asv From 1788944b0edd157cdf022e28a4a1198de9615abd Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 12:55:20 +0530 Subject: [PATCH 32/60] Atom data version --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e52063151a1..a5aa39dac19 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -36,8 +36,8 @@ jobs: - name: Download Atom Data run: | REPO_URL="https://dev.azure.com/tardis-sn/TARDIS/_apis/git/repositories/tardis-refdata" - wget -q "$REPO_URL/items?path=atom_data/kurucz_cd23_chianti_H_He.h5&resolveLfs=true" -O benchmarks/data/kurucz_cd23_chianti_H_He.h5 - + wget -q "$REPO_URL/items?path=atom_data/kurucz_cd23_chianti_H_He.h5&versionType=branch&version=master&resolveLfs=true" -O benchmarks/data/kurucz_cd23_chianti_H_He.h5 + - name: Install asv run: pip install asv From 20299f13bf02c8d366c701710b9f43ddc1ff16cd Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 15:46:46 +0530 Subject: [PATCH 33/60] Use Actions instead of github scripts --- .github/workflows/benchmarks.yml | 39 +++++++++++++------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a5aa39dac19..b5ce962857d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -90,32 +90,25 @@ jobs: echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT - - name: Write comment - uses: actions/github-script@v6 + - name: Post comment + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v2 with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: ` - *\*beep\* \*bop\** - Hi human, - I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). - Here are the logs produced by ASV. - Results can also be downloaded as artifacts [**here**](${{ env.URL }}). - -
- 
-              
+          token:  ${{ secrets.BOT_TOKEN}}
+          issue-number: ${{ github.event.pull_request.number }} 
+          body: |
+            *\*beep\* \*bop\**
+            Hi human, 
+            I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). 
+            Here are the logs produced by ASV.
+            Results can also be downloaded as artifacts [**here**](${{ env.URL }}). 
+            

+
+              ```
               ${{join(steps.asv_logs.outputs.*, '\n')}}
               ${{ steps.asv_logs.outputs.LOGS }}
-              
-
-
- - ` - }) + ``` + env: URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true From 4c046e94cbbe9c1a3a36a267c2f6be8a1fd1d017 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 15:52:57 +0530 Subject: [PATCH 34/60] Use bot token --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b5ce962857d..a26daa08fce 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -63,7 +63,7 @@ jobs: continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: - API_TOKEN_GITHUB: ${{ secrets.DEPLOY_PAT_TEMP}} + API_TOKEN_GITHUB: ${{ secrets.BOT_TOKEN}} with: source-directory: .asv destination-github-username: atharva-2001 From 4b88612d8b8762ae65f493dc739c9e9d15295f0b Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Tue, 18 Apr 2023 16:08:22 +0530 Subject: [PATCH 35/60] change triggers [skip ci] --- .github/workflows/benchmarks.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a26daa08fce..4b5dff8643f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,11 +1,17 @@ name: benchmarks on: - pull_request: - types: [labeled, synchronize] push: branches: - master + pull_request_target: + branches: + - master + types: + - opened + - reopened + - synchronize + - labeled # run-benchmarks label required defaults: run: @@ -19,14 +25,14 @@ jobs: - os: ubuntu-latest label: linux-64 prefix: /usr/share/miniconda3/envs/tardis - # - os: macos-latest # label: osx-64 # prefix: /Users/runner/miniconda3/envs/tardis - name: ${{ matrix.label }} runs-on: ${{ matrix.os }} - if: "!contains(github.event.head_commit.message, 'bench')" + if: github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'run-benchmarks') steps: - uses: actions/checkout@v2 with: @@ -42,13 +48,13 @@ jobs: run: pip install asv - name: Run benchmarks for last 5 commits if not PR - # if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request_target' run: | git log -n 5 --pretty=format:"%H" >> tag_commits.txt asv machine --yes - asv run HASHFILE:tag_commits.txt -e | tee asv-output.log - if grep -q Traceback asv-output.log; then - echo "Some benchmarks have errors!" + asv run HASHFILE:tag_commits.txt | tee asv-output.log + if grep -q failed asv-output.log; then + echo "Some benchmarks have failed!" exit 1 fi @@ -59,7 +65,7 @@ jobs: run: rm -r .asv/env - name: Push results to results repository - # if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request_target' continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: @@ -73,7 +79,7 @@ jobs: target-directory: .asv - name: Compare HEAD with master if PR - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request_target' continue-on-error: true # TODO: step failed sporadically while testing run: | asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log From 202dcba236c9e5dc0cdae8a821b7e9cfdae150ed Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 16:28:21 +0530 Subject: [PATCH 36/60] github context: check --- .github/workflows/tests.yml | 57 +++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67aed9fdc1a..2436a57b280 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,34 +46,37 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - - name: Clone tardis-sn/tardis-refdata - uses: actions/checkout@v2 - with: - repository: tardis-sn/tardis-refdata - path: tardis-refdata - lfs: false - - - name: Create LFS file list - run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - working-directory: tardis-refdata - - - name: Restore LFS cache - uses: actions/cache@v2 - id: lfs-cache - with: - path: tardis-refdata/.git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 - - - name: Git LFS Pull - run: git lfs pull - working-directory: tardis-refdata - if: steps.lfs-cache.outputs.cache-hit != 'true' + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" + # - name: Clone tardis-sn/tardis-refdata + # uses: actions/checkout@v2 + # with: + # repository: tardis-sn/tardis-refdata + # path: tardis-refdata + # lfs: false + + # - name: Create LFS file list + # run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + # working-directory: tardis-refdata + + # - name: Restore LFS cache + # uses: actions/cache@v2 + # id: lfs-cache + # with: + # path: tardis-refdata/.git/lfs + # key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + + # - name: Git LFS Pull + # run: git lfs pull + # working-directory: tardis-refdata + # if: steps.lfs-cache.outputs.cache-hit != 'true' - - name: Git LFS Checkout - run: git lfs checkout - working-directory: tardis-refdata - if: steps.lfs-cache.outputs.cache-hit == 'true' + # - name: Git LFS Checkout + # run: git lfs checkout + # working-directory: tardis-refdata + # if: steps.lfs-cache.outputs.cache-hit == 'true' - name: Setup environment uses: conda-incubator/setup-miniconda@v2 From dfe913a7c03a719b6101942eac243c9160d7bf2f Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 18:38:23 +0530 Subject: [PATCH 37/60] pull_request_target to pull_request --- .github/workflows/benchmarks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 4b5dff8643f..0779789a805 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -4,7 +4,7 @@ on: push: branches: - master - pull_request_target: + pull_request: branches: - master types: @@ -48,7 +48,7 @@ jobs: run: pip install asv - name: Run benchmarks for last 5 commits if not PR - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request' run: | git log -n 5 --pretty=format:"%H" >> tag_commits.txt asv machine --yes @@ -65,7 +65,7 @@ jobs: run: rm -r .asv/env - name: Push results to results repository - if: github.event_name != 'pull_request_target' + if: github.event_name != 'pull_request' continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: @@ -79,7 +79,7 @@ jobs: target-directory: .asv - name: Compare HEAD with master if PR - if: github.event_name == 'pull_request_target' + if: github.event_name == 'pull_request' continue-on-error: true # TODO: step failed sporadically while testing run: | asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log From 40c48cf88532b86630d401a277e032e67793cdbb Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 18:40:14 +0530 Subject: [PATCH 38/60] change branch to master --- .github/workflows/benchmarks.yml | 2 +- asv.conf.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0779789a805..c2cfbc44f75 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 with: repository: atharva-2001/tardis - ref: benchmark_base + ref: master - name: Download Atom Data run: | diff --git a/asv.conf.json b/asv.conf.json index c23f40d58b4..234a312750d 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -30,7 +30,7 @@ // List of branches to benchmark. If not provided, defaults to "master" // (for git) or "default" (for mercurial). - "branches": ["benchmark_base"], // for git + "branches": ["master"], // for git // "branches": ["default"], // for mercurial // The DVCS being used. If not set, it will be automatically From f9b47187a8d0ce23b6f50d6da7d7798b4a2ab040 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 18:43:38 +0530 Subject: [PATCH 39/60] asv publish conditional statement --- .github/workflows/benchmarks.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c2cfbc44f75..001a8f40d9e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -19,20 +19,21 @@ defaults: jobs: build: + if: github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'run-benchmarks') strategy: matrix: include: - os: ubuntu-latest label: linux-64 prefix: /usr/share/miniconda3/envs/tardis - # - os: macos-latest - # label: osx-64 - # prefix: /Users/runner/miniconda3/envs/tardis + - os: macos-latest + label: osx-64 + prefix: /Users/runner/miniconda3/envs/tardis + name: ${{ matrix.label }} runs-on: ${{ matrix.os }} - if: github.event_name == 'push' || - github.event_name == 'workflow_dispatch' || - contains(github.event.pull_request.labels.*.name, 'run-benchmarks') steps: - uses: actions/checkout@v2 with: @@ -59,6 +60,7 @@ jobs: fi - name: Generate Graphs and HTML + if: github.event_name != 'pull_request' run: asv publish - name: Delete env files From 0bc5a916cf36343f3e72f5fbceee9a5fd2b76362 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 18:48:06 +0530 Subject: [PATCH 40/60] move delete env files --- .github/workflows/benchmarks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 001a8f40d9e..7c13a6d2003 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -62,9 +62,6 @@ jobs: - name: Generate Graphs and HTML if: github.event_name != 'pull_request' run: asv publish - - - name: Delete env files - run: rm -r .asv/env - name: Push results to results repository if: github.event_name != 'pull_request' @@ -120,6 +117,8 @@ jobs: env: URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true + - name: Delete env files + run: rm -r .asv/env - name: Save results artifact uses: actions/upload-artifact@v3 From 91d15b2e52fab78723d28024c7b93f3602778a20 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 19:01:23 +0530 Subject: [PATCH 41/60] revert temporary changes made in tests workflow and the env file --- .github/workflows/tests.yml | 59 ++++++++++++++++++------------------- tardis_env3.yml | 3 +- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2436a57b280..bf5810b368a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,37 +46,34 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJSON(github) }} - run: echo "$GITHUB_CONTEXT" - # - name: Clone tardis-sn/tardis-refdata - # uses: actions/checkout@v2 - # with: - # repository: tardis-sn/tardis-refdata - # path: tardis-refdata - # lfs: false - - # - name: Create LFS file list - # run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - # working-directory: tardis-refdata - - # - name: Restore LFS cache - # uses: actions/cache@v2 - # id: lfs-cache - # with: - # path: tardis-refdata/.git/lfs - # key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 - - # - name: Git LFS Pull - # run: git lfs pull - # working-directory: tardis-refdata - # if: steps.lfs-cache.outputs.cache-hit != 'true' + + - name: Clone tardis-sn/tardis-refdata + uses: actions/checkout@v2 + with: + repository: tardis-sn/tardis-refdata + path: tardis-refdata + lfs: false + + - name: Create LFS file list + run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + working-directory: tardis-refdata + + - name: Restore LFS cache + uses: actions/cache@v2 + id: lfs-cache + with: + path: tardis-refdata/.git/lfs + key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + + - name: Git LFS Pull + run: git lfs pull + working-directory: tardis-refdata + if: steps.lfs-cache.outputs.cache-hit != 'true' - # - name: Git LFS Checkout - # run: git lfs checkout - # working-directory: tardis-refdata - # if: steps.lfs-cache.outputs.cache-hit == 'true' + - name: Git LFS Checkout + run: git lfs checkout + working-directory: tardis-refdata + if: steps.lfs-cache.outputs.cache-hit == 'true' - name: Setup environment uses: conda-incubator/setup-miniconda@v2 @@ -103,4 +100,4 @@ jobs: run: pytest tardis ${{ env.PYTEST_FLAGS }} - name: Upload to Codecov - run: bash <(curl -s https://codecov.io/bash) \ No newline at end of file + run: bash <(curl -s https://codecov.io/bash) diff --git a/tardis_env3.yml b/tardis_env3.yml index 5de9d573d62..f4c7344e0c4 100644 --- a/tardis_env3.yml +++ b/tardis_env3.yml @@ -28,6 +28,7 @@ dependencies: - pyyaml - jsonschema - hdf5=1.10 + - h5py - pytables - requests - tqdm @@ -73,5 +74,3 @@ dependencies: # Other - git-lfs - - pip: - - h5py From 1242151abe936a08c1642660227497f94ffc51d2 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 19:19:05 +0530 Subject: [PATCH 42/60] change to fetch-depth:0 --- .github/workflows/benchmarks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7c13a6d2003..1d56c3119ec 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -37,8 +37,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - repository: atharva-2001/tardis - ref: master + fetch-depth: 0 - name: Download Atom Data run: | From 62db512585730497024fc5388daf425e2d7a15de Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 19 Apr 2023 19:32:31 +0530 Subject: [PATCH 43/60] asv machine accept all questions --- .github/workflows/benchmarks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1d56c3119ec..6c0f9f35c9b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -47,11 +47,13 @@ jobs: - name: Install asv run: pip install asv + - name: Accpet all asv questions + run: asv machine --yes + - name: Run benchmarks for last 5 commits if not PR if: github.event_name != 'pull_request' run: | git log -n 5 --pretty=format:"%H" >> tag_commits.txt - asv machine --yes asv run HASHFILE:tag_commits.txt | tee asv-output.log if grep -q failed asv-output.log; then echo "Some benchmarks have failed!" From 0ac99ae5dd564eb5262b483d4f4efdfa6b4a51c7 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 20 Apr 2023 15:48:57 +0530 Subject: [PATCH 44/60] Only one os is sufficient --- .github/workflows/benchmarks.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6c0f9f35c9b..00e1b76b599 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -22,18 +22,7 @@ jobs: if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') - strategy: - matrix: - include: - - os: ubuntu-latest - label: linux-64 - prefix: /usr/share/miniconda3/envs/tardis - - os: macos-latest - label: osx-64 - prefix: /Users/runner/miniconda3/envs/tardis - - name: ${{ matrix.label }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: From d6e9756382a7fd25be0c3db673f4f28d990b9811 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 20 Apr 2023 17:51:08 +0530 Subject: [PATCH 45/60] Check if benchmarks failed without showing stderr --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 00e1b76b599..f322a9766da 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -71,8 +71,8 @@ jobs: if: github.event_name == 'pull_request' continue-on-error: true # TODO: step failed sporadically while testing run: | - asv continuous --show-stderr ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log - if grep -q Traceback asv-output.log; then + asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log + if grep -q failed asv-output.log; then echo "Some benchmarks have errors!" exit 1 fi From 2e08f7c830c3c28cf9c22bec9459c6528aad0e25 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 20 Apr 2023 19:48:02 +0530 Subject: [PATCH 46/60] change destination repo for results --- .github/workflows/benchmarks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f322a9766da..d0f6e3c28de 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -61,9 +61,9 @@ jobs: API_TOKEN_GITHUB: ${{ secrets.BOT_TOKEN}} with: source-directory: .asv - destination-github-username: atharva-2001 - destination-repository-name: tardis-benchmark-test-results - user-email: aryaatharva18@gmail.com + destination-github-username: tardis-sn + destination-repository-name: tardis-benchmarks + user-email: tardis.sn.bot@gmail.com target-branch: main target-directory: .asv From d53d5aad20a640110dfb2a41209546fbadd7179d Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 20 Apr 2023 20:09:55 +0530 Subject: [PATCH 47/60] pull request on any branch --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d0f6e3c28de..1a541a77df5 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -6,7 +6,7 @@ on: - master pull_request: branches: - - master + - '*' types: - opened - reopened From 09a7a5f913588aaaf9c8d3907c712db2960bd36c Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 17:41:14 +0530 Subject: [PATCH 48/60] Prevent tests from running on forks --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf5810b368a..c82fc3fe10b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,7 @@ defaults: jobs: build: - + if: github.repository_owner == 'tardis-sn' strategy: matrix: include: From 434473f97d9c8d0d8c4b147e91c3757a1164a9af Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 18:03:16 +0530 Subject: [PATCH 49/60] ccache --- .github/workflows/benchmarks.yml | 33 ++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1a541a77df5..fea2ca7c2dc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -23,6 +23,9 @@ jobs: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest + env: + CCACHE_BASEDIR: "${{ github.workspace }}" + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 with: @@ -33,9 +36,24 @@ jobs: REPO_URL="https://dev.azure.com/tardis-sn/TARDIS/_apis/git/repositories/tardis-refdata" wget -q "$REPO_URL/items?path=atom_data/kurucz_cd23_chianti_H_He.h5&versionType=branch&version=master&resolveLfs=true" -O benchmarks/data/kurucz_cd23_chianti_H_He.h5 - - name: Install asv - run: pip install asv + - name: Install asv and ccache + run: | + pip install asv + sudo apt-get update -y && sudo apt-get install -y ccache + sudo /usr/sbin/update-ccache-symlinks + echo "/usr/lib/ccache" >> $GITHUB_PATH + + - name: Prepare ccache + run: | + ccache -p + ccache -z + - name: Restore ccache + uses: actions/cache/restore@v3 + with: + path: .ccache + key: ccache + - name: Accpet all asv questions run: asv machine --yes @@ -109,6 +127,17 @@ jobs: - name: Delete env files run: rm -r .asv/env + + + - name: Save .ccache + uses: actions/cache/save@v3 + id: ccache + with: + path: .ccache + key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + + - run: ccache -s + if: always() - name: Save results artifact uses: actions/upload-artifact@v3 From ff3c7c255847eb2f77ff3a27e706661e1b6ad78e Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 18:04:35 +0530 Subject: [PATCH 50/60] token --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index fea2ca7c2dc..f66de012ac6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -76,7 +76,7 @@ jobs: continue-on-error: true uses: cpina/github-action-push-to-another-repository@main env: - API_TOKEN_GITHUB: ${{ secrets.BOT_TOKEN}} + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB}} with: source-directory: .asv destination-github-username: tardis-sn @@ -107,7 +107,7 @@ jobs: if: github.event_name == 'pull_request' uses: peter-evans/create-or-update-comment@v2 with: - token: ${{ secrets.BOT_TOKEN}} + token: ${{ secrets.API_TOKEN_GITHUB}} issue-number: ${{ github.event.pull_request.number }} body: | *\*beep\* \*bop\** From 50725c9c1cdd6d2610d25d9fb519fd236cb5d693 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 18:16:59 +0530 Subject: [PATCH 51/60] env problem --- tardis_env3.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tardis_env3.yml b/tardis_env3.yml index f4c7344e0c4..e1809ff0f48 100644 --- a/tardis_env3.yml +++ b/tardis_env3.yml @@ -74,3 +74,5 @@ dependencies: # Other - git-lfs + - pip: + - h5py \ No newline at end of file From 2c9c79a099ed9a823a718554e59bc411417fe45e Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 18:43:06 +0530 Subject: [PATCH 52/60] run agaain From b8c73e377dfe4fee45da9f2d803cbcdb4d6b9a5d Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Wed, 26 Apr 2023 18:44:20 +0530 Subject: [PATCH 53/60] ccache not saved --- .github/workflows/benchmarks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f66de012ac6..dea8aa6c3a3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -129,12 +129,12 @@ jobs: run: rm -r .asv/env - - name: Save .ccache + - name: Save ccache uses: actions/cache/save@v3 id: ccache with: path: .ccache - key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1 + key: ccache - run: ccache -s if: always() From 4dba80004cad36c2b4d7d67e26257edf3a00aa40 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 27 Apr 2023 10:17:31 +0530 Subject: [PATCH 54/60] Run workflows From caf65951e666c42d6d6c8ffe136f2a2715050554 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Thu, 27 Apr 2023 11:30:11 +0530 Subject: [PATCH 55/60] save all asv data as artifact for testing --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index dea8aa6c3a3..8a532076a08 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -144,5 +144,5 @@ jobs: if: always() with: name: asv-benchmark-results-${{ runner.os }} - path: .asv/results + path: .asv \ No newline at end of file From a864018b03d1903cc5c2526fce9e8d503f970625 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 28 Apr 2023 15:53:56 +0530 Subject: [PATCH 56/60] compare --- .github/workflows/benchmarks.yml | 48 +++++++++++--------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8a532076a08..d6b14000dda 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -23,9 +23,6 @@ jobs: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest - env: - CCACHE_BASEDIR: "${{ github.workspace }}" - CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 with: @@ -36,24 +33,9 @@ jobs: REPO_URL="https://dev.azure.com/tardis-sn/TARDIS/_apis/git/repositories/tardis-refdata" wget -q "$REPO_URL/items?path=atom_data/kurucz_cd23_chianti_H_He.h5&versionType=branch&version=master&resolveLfs=true" -O benchmarks/data/kurucz_cd23_chianti_H_He.h5 - - name: Install asv and ccache - run: | - pip install asv - sudo apt-get update -y && sudo apt-get install -y ccache - sudo /usr/sbin/update-ccache-symlinks - echo "/usr/lib/ccache" >> $GITHUB_PATH - - - name: Prepare ccache - run: | - ccache -p - ccache -z + - name: Install asv + run: pip install asv - - name: Restore ccache - uses: actions/cache/restore@v3 - with: - path: .ccache - key: ccache - - name: Accpet all asv questions run: asv machine --yes @@ -95,6 +77,9 @@ jobs: exit 1 fi + - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} + - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} --only-changed + - name: Output logs for comment id: asv_logs run: | @@ -103,12 +88,22 @@ jobs: echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT + - name: Find Comment + if: github.event_name == 'pull_request' + uses: peter-evans/find-comment@v1 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'tardis-bot' + body-includes: I ran benchmarks as you asked + - name: Post comment if: github.event_name == 'pull_request' uses: peter-evans/create-or-update-comment@v2 with: token: ${{ secrets.API_TOKEN_GITHUB}} issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.fc.outputs.comment-id }} body: | *\*beep\* \*bop\** Hi human, @@ -127,22 +122,11 @@ jobs: - name: Delete env files run: rm -r .asv/env - - - - name: Save ccache - uses: actions/cache/save@v3 - id: ccache - with: - path: .ccache - key: ccache - - - run: ccache -s - if: always() - name: Save results artifact uses: actions/upload-artifact@v3 if: always() with: name: asv-benchmark-results-${{ runner.os }} - path: .asv + path: .asv/results \ No newline at end of file From 2af16594c32adabce66914a8de79c0fdca6dcc5a Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 28 Apr 2023 15:58:22 +0530 Subject: [PATCH 57/60] checkkout v3 --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d6b14000dda..7222391b6c4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -24,7 +24,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 From 420692aa45a7052251ad8528407e2bdf98368d05 Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Fri, 28 Apr 2023 16:55:29 +0530 Subject: [PATCH 58/60] bench more --- .github/workflows/benchmarks.yml | 49 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7222391b6c4..d60c46c2040 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -71,22 +71,35 @@ jobs: if: github.event_name == 'pull_request' continue-on-error: true # TODO: step failed sporadically while testing run: | - asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-output.log + asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-cont-output.log if grep -q failed asv-output.log; then echo "Some benchmarks have errors!" exit 1 fi - - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} - - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} --only-changed + - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | tee asv-compare-output.log + - run: asv compare ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} --only-changed | tee asv-compare-changed-output.log - - name: Output logs for comment - id: asv_logs - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "text<<$EOF" >> $GITHUB_OUTPUT - echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT + - name: asv output 1 + id: asv_pr_vs_master + uses: juliangruber/read-file-action@v1.0.0 + with: + path: asv-compare-output.log + + - name: asv output 2 + id: asv_pr_vs_master_changed + uses: juliangruber/read-file-action@v1.0.0 + with: + path: asv-compare-changed-output.log + + # - name: Output logs for comment + # id: asv_logs + # run: | + # EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + # echo "text<<$EOF" >> $GITHUB_OUTPUT + # echo "$(cat asv-output.log)" >> $GITHUB_OUTPUT + # echo "$EOF" >> $GITHUB_OUTPUT + - name: Find Comment if: github.event_name == 'pull_request' @@ -111,10 +124,13 @@ jobs: Here are the logs produced by ASV. Results can also be downloaded as artifacts [**here**](${{ env.URL }}).
- + + ```diff + ${{ steps.asv_pr_vs_master.outputs.content }} ``` - ${{join(steps.asv_logs.outputs.*, '\n')}} - ${{ steps.asv_logs.outputs.LOGS }} + + ```diff + ${{ steps.asv_pr_vs_master_changed.outputs.content }} ```
env: @@ -128,5 +144,10 @@ jobs: if: always() with: name: asv-benchmark-results-${{ runner.os }} - path: .asv/results + path: | + .asv/results + asv-cont-output.log + asv-compare-output.log + asv-compare-changed-output.log + \ No newline at end of file From 0a4d97a6d633627bfe52c4c50048ae3b659216cb Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 1 May 2023 17:45:27 +0530 Subject: [PATCH 59/60] Significantly changed benchmarks --- .github/workflows/benchmarks.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d60c46c2040..35e7c6ba5ec 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -123,14 +123,17 @@ jobs: I ran benchmarks as you asked comparing master (${{ github.event.pull_request.base.sha }}) and the latest commit (${{ github.event.pull_request.head.sha }}). Here are the logs produced by ASV. Results can also be downloaded as artifacts [**here**](${{ env.URL }}). + Significantly changed benchmarks:
- ```diff - ${{ steps.asv_pr_vs_master.outputs.content }} + ${{ steps.asv_pr_vs_master_changed.outputs.content }} ``` - +
+ + All benchmarks: +
```diff - ${{ steps.asv_pr_vs_master_changed.outputs.content }} + ${{ steps.asv_pr_vs_master.outputs.content }} ```
env: From 9a11894d62e32a142c1635326bde36dc5cdb4b2c Mon Sep 17 00:00:00 2001 From: atharva-2001 Date: Mon, 1 May 2023 18:14:21 +0530 Subject: [PATCH 60/60] needs spaces --- .github/workflows/benchmarks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 35e7c6ba5ec..4429f7b0695 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -125,16 +125,20 @@ jobs: Results can also be downloaded as artifacts [**here**](${{ env.URL }}). Significantly changed benchmarks:
+ ```diff ${{ steps.asv_pr_vs_master_changed.outputs.content }} ``` +
All benchmarks:
+ ```diff ${{ steps.asv_pr_vs_master.outputs.content }} ``` +
env: URL: https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}?check_suite_focus=true