Merge pull request #175 from P403n1x87/ci/data-validation

ci: add data validation workflow

.github/workflows/tests.yml

@@ -225,3 +225,56 @@ jobs:
  python -m pip install -r test/requirements.txt
  python -m pytest --ignore=test\cunit --pastebin=failed --no-flaky-report -sr a
  deactivate
+
+ validation:
+ runs-on: ubuntu-20.04
+
+ needs: build-linux
+
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+
+ env:
+ AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }}
+
+ name: Data validation with Python ${{ matrix.python-version }}
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Install build dependencies
+ run: |
+ sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev
+
+ - name: Install Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}-dev
+
+ - name: Install Python 3.10
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.10"
+
+ - name: Compile Austin
+ run: |
+ autoreconf --install
+ ./configure --enable-debug-symbols true
+ make
+
+ - name: Install runtime dependencies
+ run: |
+ python3.10 -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pip
+ pip install -r scripts/requirements-val.txt
+ deactivate
+
+ - name: Run data validation
+ run: |
+ ulimit -c unlimited
+
+ source .venv/bin/activate
+ python scripts/validation.py --ignore-errors
+ deactivate

scripts/benchmark.py

@@ -7,19 +7,15 @@
 from textwrap import wrap
 import typing as t
 from argparse import ArgumentParser
-from itertools import product
 from math import floor, log
 from pathlib import Path
 
 from scipy.stats import ttest_ind
 
-sys.path.insert(0, str(Path(__file__).parent.parent))
+from common import download_release
+
+from test.utils import metadata, target
 
-import tarfile
-from io import BytesIO
-from test.utils import Variant, metadata, target
-from urllib.error import HTTPError
-from urllib.request import urlopen
 
 VERSIONS = ("3.4.1", "3.5.0", "dev")
 SCENARIOS = [
@@ -99,38 +95,6 @@ def get_stats(output: str) -> t.Optional[dict]:
  return None
 
 
-def download_release(version: str, dest: Path, variant_name: str = "austin") -> Variant:
- if version == "dev":
- return Variant(f"src/{variant_name}")
-
- binary_dest = dest / version
- binary = binary_dest / variant_name
-
- if not binary.exists():
- prefix = "https://github.com/p403n1x87/austin/releases/download/"
- for flavour, v in product({"-gnu", ""}, {"", "v"}):
- try:
- with urlopen(
- f"{prefix}v{version}/{variant_name}-{v}{version}{flavour}-linux-amd64.tar.xz"
- ) as stream:
- buffer = BytesIO(stream.read())
- binary_dest.mkdir(parents=True, exist_ok=True)
- tar = tarfile.open(fileobj=buffer, mode="r:xz")
- tar.extract(variant_name, str(binary_dest))
- except HTTPError:
- continue
- break
- else:
- raise RuntimeError(f"Could not download Austin version {version}")
-
- variant = Variant(str(binary))
-
- out = variant("-V").stdout
- assert f"{variant_name} {version}" in out, (f"{variant_name} {version}", out)
-
- return variant
-
-
 class Outcome:
  def __init__(self, data: list[float]) -> None:
  self.data = data

scripts/common.py

@@ -0,0 +1,59 @@
+import sys
+from itertools import product
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import tarfile
+from io import BytesIO
+from test.utils import Variant
+from urllib.error import HTTPError
+from urllib.request import urlopen
+import json
+
+
+def get_latest_release() -> str:
+ with urlopen(
+ "https://api.github.com/repos/p403n1x87/austin/releases/latest"
+ ) as stream:
+ return json.loads(stream.read().decode("utf-8"))["tag_name"].strip("v")
+
+
+def download_release(version: str, dest: Path, variant_name: str = "austin") -> Variant:
+ if version == "dev":
+ return Variant(f"src/{variant_name}")
+
+ binary_dest = dest / version
+ binary = binary_dest / variant_name
+
+ if not binary.exists():
+ prefix = "https://github.com/p403n1x87/austin/releases/download/"
+ for flavour, v in product({"-gnu", ""}, {"", "v"}):
+ try:
+ with urlopen(
+ f"{prefix}v{version}/{variant_name}-{v}{version}{flavour}-linux-amd64.tar.xz"
+ ) as stream:
+ buffer = BytesIO(stream.read())
+ binary_dest.mkdir(parents=True, exist_ok=True)
+ tar = tarfile.open(fileobj=buffer, mode="r:xz")
+ tar.extract(variant_name, str(binary_dest))
+ except HTTPError:
+ continue
+ break
+ else:
+ raise RuntimeError(f"Could not download Austin version {version}")
+
+ variant = Variant(str(binary))
+
+ out = variant("-V").stdout
+ assert f"{variant_name} {version}" in out, (f"{variant_name} {version}", out)
+
+ return variant
+
+
+def download_latest(dest: Path, variant_name: str = "austin") -> Variant:
+ return download_release(get_latest_release(), dest, variant_name)
+
+
+def get_dev(variant_name: str = "austin") -> Variant:
+ return download_release("dev", None, variant_name)

scripts/requirements-val.txt

@@ -0,0 +1,3 @@
+austin-python~=1.5
+numpy
+scipy