From 523b8d324dbc97247228518d5bbced1ea3346ebb Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 09:18:00 -0500 Subject: [PATCH 01/16] migrate the core of the code to 3.9 minimum and not 3.8 --- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 17 ++++------------- src/pyhf/contrib/utils.py | 21 +++++++-------------- src/pyhf/mixins.py | 3 ++- src/pyhf/modifiers/staterror.py | 3 +-- src/pyhf/parameters/paramsets.py | 4 +--- src/pyhf/pdf.py | 4 ++-- src/pyhf/readxml.py | 14 ++++---------- src/pyhf/schema/loader.py | 9 +-------- src/pyhf/schema/validator.py | 3 ++- src/pyhf/schema/variables.py | 8 +------- src/pyhf/tensor/numpy_backend.py | 3 ++- src/pyhf/typing.py | 9 +++------ src/pyhf/utils.py | 9 +-------- 14 files changed, 35 insertions(+), 80 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93861a557c..34312e5657 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,14 +58,14 @@ repos: hooks: - &mypy id: mypy - name: mypy with Python 3.8 + name: mypy with Python 3.9 files: src additional_dependencies: ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging'] - args: ["--python-version=3.8"] + args: ["--python-version=3.9"] - <<: *mypy - name: mypy with Python 3.12 - args: ["--python-version=3.12"] + name: mypy with Python 3.13 + args: ["--python-version=3.13"] - repo: https://github.com/codespell-project/codespell rev: v2.3.0 diff --git a/pyproject.toml b/pyproject.toml index c72568b2ac..629c3ee83e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dynamic = ["version"] description = "pure-Python HistFactory implementation with tensors and autodiff" readme = "README.rst" license = { text = "Apache-2.0" } # SPDX short identifier -requires-python = ">=3.8" +requires-python = ">=3.9" authors = [ { name = "Lukas Heinrich", email = "lukas.heinrich@cern.ch" }, { name = "Matthew Feickert", email = "matthew.feickert@cern.ch" }, @@ -35,11 +35,11 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Physics", @@ -68,18 +68,9 @@ Homepage = "https://github.com/scikit-hep/pyhf" [project.optional-dependencies] shellcomplete = ["click_completion"] -# TODO: 'tensorflow' supports all platform_machine for tensorflow v2.16.1+ -# but TensorFlow only supports python_version 3.8 up through tensorflow v2.13.1. -# So until Python 3.8 support is dropped, split requirments on python_version -# before and after 3.9. # NOTE: macos x86 support is deprecated from tensorflow v2.17.0 onwards. tensorflow = [ - # python == 3.8 - "tensorflow>=2.7.0; python_version < '3.9' and platform_machine != 'arm64'", # c.f. PR #1962, #2452 - "tensorflow-macos>=2.7.0; python_version < '3.9' and platform_machine == 'arm64' and platform_system == 'Darwin'", # c.f. PR #2119, #2452 - "tensorflow-probability>=0.11.0; python_version < '3.9'", # c.f. PR #1657, #2452 - # python >= 3.9 - "tensorflow-probability[tf]>=0.24.0; python_version >= '3.9'" # c.f. PR #2452 + "tensorflow-probability[tf]>=0.24.0" # c.f. PR #2452 ] torch = ["torch>=1.10.0"] # c.f. PR #1657 jax = [ @@ -246,7 +237,7 @@ exclude_also = [ [tool.mypy] files = "src" -python_version = "3.12" +python_version = "3.13" warn_unused_configs = true strict = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] diff --git a/src/pyhf/contrib/utils.py b/src/pyhf/contrib/utils.py index 7fa9550242..a507fbc4b4 100644 --- a/src/pyhf/contrib/utils.py +++ b/src/pyhf/contrib/utils.py @@ -80,13 +80,7 @@ def download(archive_url, output_directory, force=False, compress=False): with open(output_directory, "wb") as archive: archive.write(response.content) else: - # Support for file-like objects for tarfile.is_tarfile was added - # in Python 3.9, so as pyhf is currently Python 3.8+ then can't - # do tarfile.is_tarfile(BytesIO(response.content)). - # Instead, just use a 'try except' block to determine if the - # archive is a valid tarfile. - # TODO: Simplify after pyhf is Python 3.9+ only - try: + if tarfile.is_tarfile(BytesIO(response.content)): # Use transparent compression to allow for .tar or .tar.gz with tarfile.open( mode="r:*", fileobj=BytesIO(response.content) @@ -97,13 +91,7 @@ def download(archive_url, output_directory, force=False, compress=False): archive.extractall(output_directory, filter="data") else: archive.extractall(output_directory) - except tarfile.ReadError: - if not zipfile.is_zipfile(BytesIO(response.content)): - raise exceptions.InvalidArchive( - f"The archive downloaded from {archive_url} is not a tarfile" - + " or a zipfile and so can not be opened as one." - ) - + elif zipfile.is_zipfile(BytesIO(response.content)): output_directory = Path(output_directory) if output_directory.exists(): rmtree(output_directory) @@ -129,6 +117,11 @@ def download(archive_url, output_directory, force=False, compress=False): # from creation time rmtree(output_directory) _tmp_path.replace(output_directory) + else: + raise exceptions.InvalidArchive( + f"The archive downloaded from {archive_url} is not a tarfile" + + " or a zipfile and so can not be opened as one." + ) except ModuleNotFoundError: log.error( diff --git a/src/pyhf/mixins.py b/src/pyhf/mixins.py index 0314188cc1..4855e95101 100644 --- a/src/pyhf/mixins.py +++ b/src/pyhf/mixins.py @@ -1,7 +1,8 @@ from __future__ import annotations import logging -from typing import Any, Sequence +from typing import Any +from collections.abc import Sequence from pyhf.typing import Channel diff --git a/src/pyhf/modifiers/staterror.py b/src/pyhf/modifiers/staterror.py index a6d6d499c5..bb4e9f3f3e 100644 --- a/src/pyhf/modifiers/staterror.py +++ b/src/pyhf/modifiers/staterror.py @@ -1,5 +1,4 @@ import logging -from typing import List import pyhf from pyhf import events @@ -10,7 +9,7 @@ log = logging.getLogger(__name__) -def required_parset(sigmas, fixed: List[bool]): +def required_parset(sigmas, fixed: list[bool]): n_parameters = len(sigmas) return { 'paramset_type': 'constrained_by_normal', diff --git a/src/pyhf/parameters/paramsets.py b/src/pyhf/parameters/paramsets.py index 2562c89305..3a59d4a1e8 100644 --- a/src/pyhf/parameters/paramsets.py +++ b/src/pyhf/parameters/paramsets.py @@ -1,5 +1,3 @@ -from typing import List - import pyhf __all__ = [ @@ -29,7 +27,7 @@ def __init__(self, **kwargs): ) @property - def suggested_fixed(self) -> List[bool]: + def suggested_fixed(self) -> list[bool]: if isinstance(self._suggested_fixed, bool): return [self._suggested_fixed] * self.n_parameters return self._suggested_fixed diff --git a/src/pyhf/pdf.py b/src/pyhf/pdf.py index ca051d1652..b8cfe612bc 100644 --- a/src/pyhf/pdf.py +++ b/src/pyhf/pdf.py @@ -2,7 +2,7 @@ import copy import logging -from typing import List, Union +from typing import Union import pyhf.parameters import pyhf @@ -406,7 +406,7 @@ def param_set(self, name): """ return self.par_map[name]['paramset'] - def suggested_fixed(self) -> List[bool]: + def suggested_fixed(self) -> list[bool]: """ Identify the fixed parameters in the model. diff --git a/src/pyhf/readxml.py b/src/pyhf/readxml.py index a694dab292..52612ae082 100644 --- a/src/pyhf/readxml.py +++ b/src/pyhf/readxml.py @@ -4,16 +4,10 @@ from typing import ( IO, Callable, - Iterable, - List, - MutableMapping, - MutableSequence, - Sequence, - Set, - Tuple, Union, cast, ) +from collections.abc import Iterable, MutableMapping, MutableSequence, Sequence import xml.etree.ElementTree as ET from pathlib import Path @@ -46,8 +40,8 @@ log = logging.getLogger(__name__) -FileCacheType = MutableMapping[str, Tuple[Union[IO[str], IO[bytes]], Set[str]]] -MountPathType = Iterable[Tuple[Path, Path]] +FileCacheType = MutableMapping[str, tuple[Union[IO[str], IO[bytes]], set[str]]] +MountPathType = Iterable[tuple[Path, Path]] ResolverType = Callable[[str], Path] __FILECACHE__: FileCacheType = {} @@ -99,7 +93,7 @@ def extract_error(hist: uproot.behaviors.TH1.TH1) -> list[float]: """ variance = hist.variances() if hist.weighted else hist.to_numpy()[0] - return cast(List[float], np.sqrt(variance).tolist()) + return cast(list[float], np.sqrt(variance).tolist()) def import_root_histogram( diff --git a/src/pyhf/schema/loader.py b/src/pyhf/schema/loader.py index 920766c4dc..0f0001faef 100644 --- a/src/pyhf/schema/loader.py +++ b/src/pyhf/schema/loader.py @@ -1,15 +1,8 @@ from pathlib import Path -import sys import json import pyhf.exceptions from pyhf.schema import variables - -# importlib.resources.as_file wasn't added until Python 3.9 -# c.f. https://docs.python.org/3.9/library/importlib.html#importlib.resources.as_file -if sys.version_info >= (3, 9): - from importlib import resources -else: - import importlib_resources as resources +from importlib import resources def load_schema(schema_id: str): diff --git a/src/pyhf/schema/validator.py b/src/pyhf/schema/validator.py index 2540a3d002..69230d7410 100644 --- a/src/pyhf/schema/validator.py +++ b/src/pyhf/schema/validator.py @@ -1,6 +1,7 @@ import numbers from pathlib import Path -from typing import Mapping, Union +from typing import Union +from collections.abc import Mapping import jsonschema diff --git a/src/pyhf/schema/variables.py b/src/pyhf/schema/variables.py index 80c0a0dd06..d02cc6b322 100644 --- a/src/pyhf/schema/variables.py +++ b/src/pyhf/schema/variables.py @@ -1,11 +1,5 @@ -import sys +from importlib import resources -# importlib.resources.as_file wasn't added until Python 3.9 -# c.f. https://docs.python.org/3.9/library/importlib.html#importlib.resources.as_file -if sys.version_info >= (3, 9): - from importlib import resources -else: - import importlib_resources as resources schemas = resources.files('pyhf') / "schemas" SCHEMA_CACHE = {} diff --git a/src/pyhf/tensor/numpy_backend.py b/src/pyhf/tensor/numpy_backend.py index 5221800a3d..98d29a185d 100644 --- a/src/pyhf/tensor/numpy_backend.py +++ b/src/pyhf/tensor/numpy_backend.py @@ -3,7 +3,8 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Callable, Generic, Mapping, Sequence, TypeVar, Union +from typing import TYPE_CHECKING, Callable, Generic, TypeVar, Union +from collections.abc import Mapping, Sequence import numpy as np diff --git a/src/pyhf/typing.py b/src/pyhf/typing.py index 19dd36c485..f3e4e5784c 100644 --- a/src/pyhf/typing.py +++ b/src/pyhf/typing.py @@ -2,14 +2,12 @@ from typing import ( Any, Literal, - MutableSequence, Protocol, - Sequence, SupportsIndex, - Tuple, TypedDict, Union, ) +from collections.abc import MutableSequence, Sequence __all__ = ( "Channel", @@ -35,10 +33,9 @@ ) -# TODO: Switch to os.PathLike[str] once Python 3.8 support dropped -PathOrStr = Union[str, "os.PathLike[str]"] +PathOrStr = Union[str, os.PathLike[str]] -Shape = Tuple[int, ...] +Shape = tuple[int, ...] ShapeLike = Union[SupportsIndex, Sequence[SupportsIndex]] diff --git a/src/pyhf/utils.py b/src/pyhf/utils.py index c9ad5d0185..ea2fdfa99a 100644 --- a/src/pyhf/utils.py +++ b/src/pyhf/utils.py @@ -4,14 +4,7 @@ import hashlib from gettext import gettext -import sys - -# importlib.resources.as_file wasn't added until Python 3.9 -# c.f. https://docs.python.org/3.9/library/importlib.html#importlib.resources.as_file -if sys.version_info >= (3, 9): - from importlib import resources -else: - import importlib_resources as resources +from importlib import resources __all__ = [ "EqDelimStringParamType", From af60b9b6e145336af722fa1fe1a165e065710aff Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 09:19:40 -0500 Subject: [PATCH 02/16] migrate more to 3.9+ only --- .github/workflows/bump-version.yml | 4 ++-- .github/workflows/ci-windows.yml | 2 +- .github/workflows/ci.yml | 24 +++++++------------ .github/workflows/dependencies-head.yml | 12 +++++----- .github/workflows/docs.yml | 2 +- .../workflows/lower-bound-requirements.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/publish-package.yml | 4 ++-- .github/workflows/release_tests.yml | 6 ++--- tests/contrib/test_viz.py | 5 ---- 10 files changed, 25 insertions(+), 38 deletions(-) diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index e574a2ab34..c602ffc018 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -195,11 +195,11 @@ jobs: echo "steps.script.outputs.old_tag=v${current_tag}" echo "old_tag=v${current_tag}" >> $GITHUB_OUTPUT - - name: Set up Python 3.12 + - name: Set up Python 3.13 if: success() uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install Python dependencies run: | diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index b625754baf..d4cff442ff 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 668a315959..9f56a516c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,13 +30,13 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] include: - os: macos-latest - python-version: '3.12' + python-version: '3.13' # Intel runner - os: macos-13 - python-version: '3.12' + python-version: '3.13' steps: - uses: actions/checkout@v4 @@ -47,18 +47,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies - if: matrix.python-version != '3.8' run: | python -m pip install uv uv pip install --system --upgrade ".[all,test]" - # c.f. https://github.com/astral-sh/uv/issues/2062 - - name: Install dependencies (Python 3.8) - if: matrix.python-version == '3.8' - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade ".[all,test]" - - name: List installed Python packages run: python -m pip list @@ -97,7 +89,7 @@ jobs: coverage xml - name: Report contrib coverage with Codecov - if: github.event_name != 'schedule' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' + if: github.event_name != 'schedule' && matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v5 with: fail_ci_if_error: true @@ -107,17 +99,17 @@ jobs: - name: Test docstring examples with doctest # TODO: Don't currently try to match amd64 and arm64 floating point for docs, but will in the future. - if: matrix.python-version == '3.12' && matrix.os != 'macos-latest' + if: matrix.python-version == '3.13' && matrix.os != 'macos-latest' run: coverage run --data-file=.coverage-doctest --module pytest src/ README.rst - name: Coverage report for doctest only - if: matrix.python-version == '3.12' && matrix.os != 'macos-latest' + if: matrix.python-version == '3.13' && matrix.os != 'macos-latest' run: | coverage report --data-file=.coverage-doctest coverage xml --data-file=.coverage-doctest -o doctest-coverage.xml - name: Report doctest coverage with Codecov - if: github.event_name != 'schedule' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' + if: github.event_name != 'schedule' && matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v5 with: fail_ci_if_error: true @@ -126,6 +118,6 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Run benchmarks - if: github.event_name == 'schedule' && matrix.python-version == '3.12' + if: github.event_name == 'schedule' && matrix.python-version == '3.13' run: | pytest --benchmark-sort=mean tests/benchmarks/test_benchmark.py diff --git a/.github/workflows/dependencies-head.yml b/.github/workflows/dependencies-head.yml index c714392d81..0742036e50 100644 --- a/.github/workflows/dependencies-head.yml +++ b/.github/workflows/dependencies-head.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, macos-13] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 @@ -79,7 +79,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 @@ -105,7 +105,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 @@ -130,7 +130,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 @@ -168,7 +168,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c7a99fda4d..a802b904c9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install Python dependencies run: | diff --git a/.github/workflows/lower-bound-requirements.yml b/.github/workflows/lower-bound-requirements.yml index 5bdd837c68..81130f861f 100644 --- a/.github/workflows/lower-bound-requirements.yml +++ b/.github/workflows/lower-bound-requirements.yml @@ -17,7 +17,7 @@ jobs: matrix: os: [ubuntu-latest] # minimum supported Python - python-version: ['3.8'] + python-version: ['3.9'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 52b6fb1af4..8e06f6e6d7 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 7125f082e5..259e56b66b 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -41,10 +41,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install python-build and twine run: | diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 0569562704..362b019b37 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -21,13 +21,13 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] include: - os: macos-latest - python-version: '3.12' + python-version: '3.13' # Intel runner - os: macos-13 - python-version: '3.12' + python-version: '3.13' fail-fast: false steps: diff --git a/tests/contrib/test_viz.py b/tests/contrib/test_viz.py index f8f760931f..f7c0b29427 100644 --- a/tests/contrib/test_viz.py +++ b/tests/contrib/test_viz.py @@ -1,5 +1,4 @@ import json -import sys import matplotlib import matplotlib.pyplot as plt @@ -68,10 +67,6 @@ def test_plot_results(datadir): @pytest.mark.mpl_image_compare -@pytest.mark.xfail( - sys.version_info < (3, 8), - reason="baseline image generated with matplotlib v3.6.0 which is Python 3.8+", -) def test_plot_results_no_axis(datadir): data = json.load(datadir.joinpath("hypotest_results.json").open(encoding="utf-8")) From d9bb85c6d34e81b2eb2054caf0ef5a70ba500035 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 10:15:34 -0500 Subject: [PATCH 03/16] fix typehints in 3.13 --- pyproject.toml | 1 + src/pyhf/readxml.py | 2 +- src/pyhf/tensor/numpy_backend.py | 17 ++++++++++------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 629c3ee83e..250987e44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,6 +242,7 @@ warn_unused_configs = true strict = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true +plugins = "numpy.typing.mypy_plugin" [[tool.mypy.overrides]] module = [ diff --git a/src/pyhf/readxml.py b/src/pyhf/readxml.py index 52612ae082..2ad5fe6642 100644 --- a/src/pyhf/readxml.py +++ b/src/pyhf/readxml.py @@ -216,7 +216,7 @@ def process_sample( modtag.attrib.get('HistoPath', ''), modtag.attrib['HistoName'], ) - staterr = np.multiply(extstat, data).tolist() + staterr = cast(list[float], np.multiply(extstat, data).tolist()) if not staterr: raise RuntimeError('cannot determine stat error.') modifier_staterror: StatError = { diff --git a/src/pyhf/tensor/numpy_backend.py b/src/pyhf/tensor/numpy_backend.py index 98d29a185d..423d54c152 100644 --- a/src/pyhf/tensor/numpy_backend.py +++ b/src/pyhf/tensor/numpy_backend.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Callable, Generic, TypeVar, Union +from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, Union from collections.abc import Mapping, Sequence import numpy as np @@ -27,7 +27,7 @@ log = logging.getLogger(__name__) -class _BasicPoisson: +class _BasicPoisson(Generic[T]): def __init__(self, rate: Tensor[T]): self.rate = rate @@ -39,7 +39,7 @@ def log_prob(self, value: NDArray[np.number[T]]) -> ArrayLike: return tensorlib.poisson_logpdf(value, self.rate) -class _BasicNormal: +class _BasicNormal(Generic[T]): def __init__(self, loc: Tensor[T], scale: Tensor[T]): self.loc = loc self.scale = scale @@ -199,9 +199,12 @@ def conditional( """ return true_callable() if predicate else false_callable() - def tolist(self, tensor_in: Tensor[T] | list[T]) -> list[T]: + def tolist( + self, tensor_in: Tensor[T] | list[T] + ) -> int | float | complex | list[T] | list[Any]: try: - return tensor_in.tolist() # type: ignore[union-attr,no-any-return] + # unused-ignore for [no-any-return] in python 3.9 + return tensor_in.tolist() # type: ignore[union-attr,no-any-return,unused-ignore] except AttributeError: if isinstance(tensor_in, list): return tensor_in @@ -551,7 +554,7 @@ def normal_cdf( """ return norm.cdf(x, loc=mu, scale=sigma) # type: ignore[no-any-return] - def poisson_dist(self, rate: Tensor[T]) -> _BasicPoisson: + def poisson_dist(self, rate: Tensor[T]) -> _BasicPoisson[T]: r""" The Poisson distribution with rate parameter :code:`rate`. @@ -572,7 +575,7 @@ def poisson_dist(self, rate: Tensor[T]) -> _BasicPoisson: """ return _BasicPoisson(rate) - def normal_dist(self, mu: Tensor[T], sigma: Tensor[T]) -> _BasicNormal: + def normal_dist(self, mu: Tensor[T], sigma: Tensor[T]) -> _BasicNormal[T]: r""" The Normal distribution with mean :code:`mu` and standard deviation :code:`sigma`. From ff2715c68c103f3d50d48410ad289d0fccab56a8 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 10:25:11 -0500 Subject: [PATCH 04/16] don't do 3.13 yet --- .github/workflows/bump-version.yml | 4 ++-- .github/workflows/ci-windows.yml | 2 +- .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/dependencies-head.yml | 12 ++++++------ .github/workflows/docs.yml | 2 +- .github/workflows/notebooks.yml | 2 +- .github/workflows/publish-package.yml | 4 ++-- .github/workflows/release_tests.yml | 6 +++--- pyproject.toml | 3 +-- 9 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index c602ffc018..e574a2ab34 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -195,11 +195,11 @@ jobs: echo "steps.script.outputs.old_tag=v${current_tag}" echo "old_tag=v${current_tag}" >> $GITHUB_OUTPUT - - name: Set up Python 3.13 + - name: Set up Python 3.12 if: success() uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: '3.12' - name: Install Python dependencies run: | diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index d4cff442ff..b625754baf 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f56a516c1..ed42f3670b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,13 +30,13 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] include: - os: macos-latest - python-version: '3.13' + python-version: '3.12' # Intel runner - os: macos-13 - python-version: '3.13' + python-version: '3.12' steps: - uses: actions/checkout@v4 @@ -89,7 +89,7 @@ jobs: coverage xml - name: Report contrib coverage with Codecov - if: github.event_name != 'schedule' && matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' + if: github.event_name != 'schedule' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v5 with: fail_ci_if_error: true @@ -99,17 +99,17 @@ jobs: - name: Test docstring examples with doctest # TODO: Don't currently try to match amd64 and arm64 floating point for docs, but will in the future. - if: matrix.python-version == '3.13' && matrix.os != 'macos-latest' + if: matrix.python-version == '3.12' && matrix.os != 'macos-latest' run: coverage run --data-file=.coverage-doctest --module pytest src/ README.rst - name: Coverage report for doctest only - if: matrix.python-version == '3.13' && matrix.os != 'macos-latest' + if: matrix.python-version == '3.12' && matrix.os != 'macos-latest' run: | coverage report --data-file=.coverage-doctest coverage xml --data-file=.coverage-doctest -o doctest-coverage.xml - name: Report doctest coverage with Codecov - if: github.event_name != 'schedule' && matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' + if: github.event_name != 'schedule' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v5 with: fail_ci_if_error: true @@ -118,6 +118,6 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Run benchmarks - if: github.event_name == 'schedule' && matrix.python-version == '3.13' + if: github.event_name == 'schedule' && matrix.python-version == '3.12' run: | pytest --benchmark-sort=mean tests/benchmarks/test_benchmark.py diff --git a/.github/workflows/dependencies-head.yml b/.github/workflows/dependencies-head.yml index 0742036e50..c714392d81 100644 --- a/.github/workflows/dependencies-head.yml +++ b/.github/workflows/dependencies-head.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, macos-13] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 @@ -79,7 +79,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 @@ -105,7 +105,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 @@ -130,7 +130,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 @@ -168,7 +168,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a802b904c9..c7a99fda4d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: '3.12' - name: Install Python dependencies run: | diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 8e06f6e6d7..52b6fb1af4 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.13'] + python-version: ['3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 259e56b66b..7125f082e5 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -41,10 +41,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.13 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: '3.13' + python-version: '3.12' - name: Install python-build and twine run: | diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 362b019b37..4a2b910dda 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -21,13 +21,13 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] include: - os: macos-latest - python-version: '3.13' + python-version: '3.12' # Intel runner - os: macos-13 - python-version: '3.13' + python-version: '3.12' fail-fast: false steps: diff --git a/pyproject.toml b/pyproject.toml index 250987e44f..67e13ac0f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Physics", @@ -237,7 +236,7 @@ exclude_also = [ [tool.mypy] files = "src" -python_version = "3.13" +python_version = "3.12" warn_unused_configs = true strict = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] From 2e621ffc34b99e6077a0ccdeccfd4f9cfc4822fa Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 10:25:47 -0500 Subject: [PATCH 05/16] no 3.13 --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34312e5657..fbfc223415 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,8 +64,8 @@ repos: ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging'] args: ["--python-version=3.9"] - <<: *mypy - name: mypy with Python 3.13 - args: ["--python-version=3.13"] + name: mypy with Python 3.12 + args: ["--python-version=3.12"] - repo: https://github.com/codespell-project/codespell rev: v2.3.0 From 95276cb544bb6a1de909629e48ff0247423d4ce6 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 10:32:51 -0500 Subject: [PATCH 06/16] fix up pre-commit by dropping 3.9 check for mypy --- .pre-commit-config.yaml | 5 +++-- src/pyhf/tensor/numpy_backend.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbfc223415..a68228bb17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,14 +55,15 @@ repos: - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.13.0 # check the oldest and newest supported Pythons + # except skip python 3.9 for numpy, due to poor typing hooks: - &mypy id: mypy - name: mypy with Python 3.9 + name: mypy with Python 3.10 files: src additional_dependencies: ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging'] - args: ["--python-version=3.9"] + args: ["--python-version=3.10"] - <<: *mypy name: mypy with Python 3.12 args: ["--python-version=3.12"] diff --git a/src/pyhf/tensor/numpy_backend.py b/src/pyhf/tensor/numpy_backend.py index 423d54c152..677434cab5 100644 --- a/src/pyhf/tensor/numpy_backend.py +++ b/src/pyhf/tensor/numpy_backend.py @@ -203,8 +203,7 @@ def tolist( self, tensor_in: Tensor[T] | list[T] ) -> int | float | complex | list[T] | list[Any]: try: - # unused-ignore for [no-any-return] in python 3.9 - return tensor_in.tolist() # type: ignore[union-attr,no-any-return,unused-ignore] + return tensor_in.tolist() # type: ignore[union-attr] except AttributeError: if isinstance(tensor_in, list): return tensor_in From 2dd854ac1f8cdcb73357cfb531c77fbac748db6b Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 11:12:18 -0500 Subject: [PATCH 07/16] add more links to ignroe for now --- docs/conf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index dddeb31188..cd8ed89991 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -529,6 +529,15 @@ def setup(app): # and ReadTheDocs need to reference them r'https://github.com/scikit-hep/pyhf/releases/tag/.*', r'https://pyhf.readthedocs.io/en/.*', + # the following are 403s as they map to journals.aps.org or academic.oup.com + r'https://doi.org/10.1093/ptep/ptad144', + r'https://doi.org/10.1103/PhysRevD.104.055017', + r'https://doi.org/10.1103/PhysRevD.107.095021', + r'https://doi.org/10.1103/PhysRevD.108.016002', + r'https://doi.org/10.1103/PhysRevD.106.032005', + r'https://doi.org/10.1103/PhysRevLett.127.181802', + r'https://doi.org/10.1103/PhysRevLett.130.231801', + r'https://doi.org/10.1103/PhysRevLett.131.211802', ] linkcheck_retries = 50 From 1bbbbc4aafafc3918da92362c40c8bba66b30de4 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 11:20:30 -0500 Subject: [PATCH 08/16] put a lower-bound on tensorflow because of tfp issues --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 67e13ac0f7..1a6dd78aed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ keywords = [ "physics", "pytorch", "scipy", - "tensorflow", + "tensorflow >= 2.18", # see https://github.com/tensorflow/probability/issues/1854 ] classifiers = [ "Development Status :: 4 - Beta", From e0ebad4eb4201e3e53aca5b439f735f2389ff687 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 11:38:00 -0500 Subject: [PATCH 09/16] wrrong place --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1a6dd78aed..0bd9bd9cf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ keywords = [ "physics", "pytorch", "scipy", - "tensorflow >= 2.18", # see https://github.com/tensorflow/probability/issues/1854 + "tensorflow", ] classifiers = [ "Development Status :: 4 - Beta", @@ -69,6 +69,7 @@ Homepage = "https://github.com/scikit-hep/pyhf" shellcomplete = ["click_completion"] # NOTE: macos x86 support is deprecated from tensorflow v2.17.0 onwards. tensorflow = [ + "tensorflow >= 2.18", # see https://github.com/tensorflow/probability/issues/1854 "tensorflow-probability[tf]>=0.24.0" # c.f. PR #2452 ] torch = ["torch>=1.10.0"] # c.f. PR #1657 From e0366e5ffd317dac06b9dba04b086da87234d274 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 11:49:03 -0500 Subject: [PATCH 10/16] restrict tfp --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0bd9bd9cf1..cb17bd3cf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ Homepage = "https://github.com/scikit-hep/pyhf" shellcomplete = ["click_completion"] # NOTE: macos x86 support is deprecated from tensorflow v2.17.0 onwards. tensorflow = [ - "tensorflow >= 2.18", # see https://github.com/tensorflow/probability/issues/1854 - "tensorflow-probability[tf]>=0.24.0" # c.f. PR #2452 + # see https://github.com/tensorflow/probability/issues/1854 + "tensorflow-probability[tf]>=0.24.0,!=0.25.0" # c.f. PR #2452 ] torch = ["torch>=1.10.0"] # c.f. PR #1657 jax = [ From 104b4faf4df0bf410c8d6fa6c5a3d92c7da80ea5 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 12:09:10 -0500 Subject: [PATCH 11/16] cast all masks first --- src/pyhf/modifiers/histosys.py | 13 +++++++++---- src/pyhf/modifiers/lumi.py | 14 ++++++++++---- src/pyhf/modifiers/normfactor.py | 14 ++++++++++---- src/pyhf/modifiers/normsys.py | 14 ++++++++++---- src/pyhf/modifiers/shapefactor.py | 11 +++++++---- src/pyhf/modifiers/shapesys.py | 11 +++++++---- src/pyhf/modifiers/staterror.py | 11 +++++++---- 7 files changed, 60 insertions(+), 28 deletions(-) diff --git a/src/pyhf/modifiers/histosys.py b/src/pyhf/modifiers/histosys.py index c76170e36b..938444a8a9 100644 --- a/src/pyhf/modifiers/histosys.py +++ b/src/pyhf/modifiers/histosys.py @@ -101,6 +101,8 @@ class histosys_combined: def __init__( self, modifiers, pdfconfig, builder_data, interpcode='code0', batch_size=None ): + default_backend = pyhf.default_backend + self.batch_size = batch_size self.interpcode = interpcode assert self.interpcode in ['code0', 'code2', 'code4p'] @@ -128,10 +130,13 @@ def __init__( ] for m in keys ] - self._histosys_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._histosys_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) if histosys_mods: self.interpolator = getattr(interpolators, self.interpcode)( diff --git a/src/pyhf/modifiers/lumi.py b/src/pyhf/modifiers/lumi.py index f8697b7bee..748c1fb972 100644 --- a/src/pyhf/modifiers/lumi.py +++ b/src/pyhf/modifiers/lumi.py @@ -1,5 +1,6 @@ import logging +import pyhf from pyhf import get_backend, events from pyhf.parameters import ParamViewer @@ -60,6 +61,8 @@ class lumi_combined: op_code = 'multiplication' def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): + default_backend = pyhf.default_backend + self.batch_size = batch_size keys = [f'{mtype}/{m}' for m, mtype in modifiers] @@ -72,10 +75,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): ) self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, lumi_mods) - self._lumi_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._lumi_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) self._precompute() events.subscribe('tensorlib_changed')(self._precompute) diff --git a/src/pyhf/modifiers/normfactor.py b/src/pyhf/modifiers/normfactor.py index fb723664f0..6639530121 100644 --- a/src/pyhf/modifiers/normfactor.py +++ b/src/pyhf/modifiers/normfactor.py @@ -1,5 +1,6 @@ import logging +import pyhf from pyhf import get_backend, events from pyhf.parameters import ParamViewer @@ -58,6 +59,8 @@ class normfactor_combined: op_code = 'multiplication' def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): + default_backend = pyhf.default_backend + self.batch_size = batch_size keys = [f'{mtype}/{m}' for m, mtype in modifiers] @@ -72,10 +75,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): parfield_shape, pdfconfig.par_map, normfactor_mods ) - self._normfactor_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._normfactor_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) self._precompute() events.subscribe('tensorlib_changed')(self._precompute) diff --git a/src/pyhf/modifiers/normsys.py b/src/pyhf/modifiers/normsys.py index e536bfe7a4..96cf1ff5c0 100644 --- a/src/pyhf/modifiers/normsys.py +++ b/src/pyhf/modifiers/normsys.py @@ -1,5 +1,6 @@ import logging +import pyhf from pyhf import get_backend, events from pyhf import interpolators from pyhf.parameters import ParamViewer @@ -72,6 +73,8 @@ class normsys_combined: def __init__( self, modifiers, pdfconfig, builder_data, interpcode='code1', batch_size=None ): + default_backend = pyhf.default_backend + self.interpcode = interpcode assert self.interpcode in ['code1', 'code4'] @@ -97,10 +100,13 @@ def __init__( ] for m in keys ] - self._normsys_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._normsys_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) if normsys_mods: self.interpolator = getattr(interpolators, self.interpcode)( diff --git a/src/pyhf/modifiers/shapefactor.py b/src/pyhf/modifiers/shapefactor.py index 64b0db2a48..2579dbe4be 100644 --- a/src/pyhf/modifiers/shapefactor.py +++ b/src/pyhf/modifiers/shapefactor.py @@ -138,10 +138,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): parfield_shape, pdfconfig.par_map, shapefactor_mods ) - self._shapefactor_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._shapefactor_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) global_concatenated_bin_indices = [ [[j for c in pdfconfig.channels for j in range(pdfconfig.channel_nbins[c])]] diff --git a/src/pyhf/modifiers/shapesys.py b/src/pyhf/modifiers/shapesys.py index c18ac90149..6accb5bb68 100644 --- a/src/pyhf/modifiers/shapesys.py +++ b/src/pyhf/modifiers/shapesys.py @@ -112,10 +112,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): parfield_shape, pdfconfig.par_map, self._shapesys_mods ) - self._shapesys_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._shapesys_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) self.__shapesys_info = default_backend.astensor( [ [ diff --git a/src/pyhf/modifiers/staterror.py b/src/pyhf/modifiers/staterror.py index bb4e9f3f3e..18c15a7830 100644 --- a/src/pyhf/modifiers/staterror.py +++ b/src/pyhf/modifiers/staterror.py @@ -149,10 +149,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None): parfield_shape, pdfconfig.par_map, self._staterr_mods ) - self._staterror_mask = [ - [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] - for m in keys - ] + self._staterror_mask = default_backend.astensor( + [ + [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples] + for m in keys + ], + dtype='bool', + ) global_concatenated_bin_indices = [ [[j for c in pdfconfig.channels for j in range(pdfconfig.channel_nbins[c])]] ] From 40ecdc798ed79af41730126dc1ce760fbc9ee6ca Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 12:12:32 -0500 Subject: [PATCH 12/16] migrate get_backend --- src/pyhf/cli/infer.py | 3 ++- src/pyhf/infer/__init__.py | 2 +- src/pyhf/infer/calculators.py | 2 +- src/pyhf/infer/intervals/upper_limits.py | 2 +- src/pyhf/infer/mle.py | 2 +- src/pyhf/infer/test_statistics.py | 2 +- src/pyhf/modifiers/lumi.py | 3 ++- src/pyhf/modifiers/normfactor.py | 3 ++- src/pyhf/modifiers/normsys.py | 3 ++- src/pyhf/optimize/opt_jax.py | 2 +- src/pyhf/optimize/opt_numpy.py | 2 +- src/pyhf/optimize/opt_pytorch.py | 2 +- src/pyhf/optimize/opt_tflow.py | 2 +- src/pyhf/probability.py | 2 +- 14 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/pyhf/cli/infer.py b/src/pyhf/cli/infer.py index f2b0dce107..38649d48b2 100644 --- a/src/pyhf/cli/infer.py +++ b/src/pyhf/cli/infer.py @@ -9,7 +9,8 @@ from pyhf.infer import hypotest from pyhf.infer import mle from pyhf.workspace import Workspace -from pyhf import get_backend, set_backend, optimize +from pyhf.tensor.manager import get_backend, set_backend +from pyhf import optimize log = logging.getLogger(__name__) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 058a7460d6..d75516f61c 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -1,7 +1,7 @@ """Inference for Statistical Models.""" from pyhf.infer import utils -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf import exceptions diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index f9922c6d89..f0066b501d 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -9,7 +9,7 @@ """ from pyhf.infer.mle import fixed_poi_fit -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf.infer import utils import tqdm diff --git a/src/pyhf/infer/intervals/upper_limits.py b/src/pyhf/infer/intervals/upper_limits.py index 6b86d586fe..ece45201ba 100644 --- a/src/pyhf/infer/intervals/upper_limits.py +++ b/src/pyhf/infer/intervals/upper_limits.py @@ -3,7 +3,7 @@ import numpy as np from scipy.optimize import toms748 -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf.infer import hypotest __all__ = ["linear_grid_scan", "toms748_scan", "upper_limit"] diff --git a/src/pyhf/infer/mle.py b/src/pyhf/infer/mle.py index c269eb47c8..7c0420f076 100644 --- a/src/pyhf/infer/mle.py +++ b/src/pyhf/infer/mle.py @@ -1,6 +1,6 @@ """Module for Maximum Likelihood Estimation.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf.exceptions import UnspecifiedPOI __all__ = ["fit", "fixed_poi_fit", "twice_nll"] diff --git a/src/pyhf/infer/test_statistics.py b/src/pyhf/infer/test_statistics.py index 97b6babe79..245e13cd35 100644 --- a/src/pyhf/infer/test_statistics.py +++ b/src/pyhf/infer/test_statistics.py @@ -1,4 +1,4 @@ -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf.infer.mle import fixed_poi_fit, fit from pyhf.exceptions import UnspecifiedPOI diff --git a/src/pyhf/modifiers/lumi.py b/src/pyhf/modifiers/lumi.py index 748c1fb972..7b2279f779 100644 --- a/src/pyhf/modifiers/lumi.py +++ b/src/pyhf/modifiers/lumi.py @@ -1,7 +1,8 @@ import logging import pyhf -from pyhf import get_backend, events +from pyhf.tensor.manager import get_backend +from pyhf import events from pyhf.parameters import ParamViewer log = logging.getLogger(__name__) diff --git a/src/pyhf/modifiers/normfactor.py b/src/pyhf/modifiers/normfactor.py index 6639530121..547c76d031 100644 --- a/src/pyhf/modifiers/normfactor.py +++ b/src/pyhf/modifiers/normfactor.py @@ -1,7 +1,8 @@ import logging import pyhf -from pyhf import get_backend, events +from pyhf.tensor.manager import get_backend +from pyhf import events from pyhf.parameters import ParamViewer log = logging.getLogger(__name__) diff --git a/src/pyhf/modifiers/normsys.py b/src/pyhf/modifiers/normsys.py index 96cf1ff5c0..ee319d5187 100644 --- a/src/pyhf/modifiers/normsys.py +++ b/src/pyhf/modifiers/normsys.py @@ -1,7 +1,8 @@ import logging import pyhf -from pyhf import get_backend, events +from pyhf.tensor.manager import get_backend +from pyhf import events from pyhf import interpolators from pyhf.parameters import ParamViewer diff --git a/src/pyhf/optimize/opt_jax.py b/src/pyhf/optimize/opt_jax.py index 5567678844..be14632123 100644 --- a/src/pyhf/optimize/opt_jax.py +++ b/src/pyhf/optimize/opt_jax.py @@ -1,6 +1,6 @@ """JAX Backend Function Shim.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf.tensor.common import _TensorViewer import jax import logging diff --git a/src/pyhf/optimize/opt_numpy.py b/src/pyhf/optimize/opt_numpy.py index 8f3d4178cf..7032c9e748 100644 --- a/src/pyhf/optimize/opt_numpy.py +++ b/src/pyhf/optimize/opt_numpy.py @@ -1,6 +1,6 @@ """Numpy Backend Function Shim.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend from pyhf import exceptions diff --git a/src/pyhf/optimize/opt_pytorch.py b/src/pyhf/optimize/opt_pytorch.py index 55b204ff41..7e0fafe9ca 100644 --- a/src/pyhf/optimize/opt_pytorch.py +++ b/src/pyhf/optimize/opt_pytorch.py @@ -1,6 +1,6 @@ """PyTorch Backend Function Shim.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend import torch diff --git a/src/pyhf/optimize/opt_tflow.py b/src/pyhf/optimize/opt_tflow.py index 178bc332ac..76647b084b 100644 --- a/src/pyhf/optimize/opt_tflow.py +++ b/src/pyhf/optimize/opt_tflow.py @@ -1,6 +1,6 @@ """Tensorflow Backend Function Shim.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend import tensorflow as tf diff --git a/src/pyhf/probability.py b/src/pyhf/probability.py index 0cc0330272..3368ee90ed 100644 --- a/src/pyhf/probability.py +++ b/src/pyhf/probability.py @@ -1,6 +1,6 @@ """The probability density function module.""" -from pyhf import get_backend +from pyhf.tensor.manager import get_backend __all__ = ["Independent", "Normal", "Poisson", "Simultaneous"] From 9688401703944203363a343c56b44d4931c1a5bb Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 18:54:28 -0500 Subject: [PATCH 13/16] fix doctest --- src/pyhf/infer/calculators.py | 4 ++-- src/pyhf/probability.py | 4 ++-- src/pyhf/tensor/numpy_backend.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index f0066b501d..ed8eaaf9c6 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -121,7 +121,7 @@ def cdf(self, value): >>> pyhf.set_backend("numpy") >>> bkg_dist = pyhf.infer.calculators.AsymptoticTestStatDistribution(0.0) >>> bkg_dist.cdf(0.0) - 0.5 + np.float64(0.5) Args: value (:obj:`float`): The test statistic value. @@ -620,7 +620,7 @@ def expected_value(self, nsigma): >>> samples = normal.sample((100,)) >>> dist = pyhf.infer.calculators.EmpiricalDistribution(samples) >>> dist.expected_value(nsigma=1) - 6.15094381... + np.float64(6.15094381...) >>> import pyhf >>> import numpy.random as random diff --git a/src/pyhf/probability.py b/src/pyhf/probability.py index 3368ee90ed..8505757840 100644 --- a/src/pyhf/probability.py +++ b/src/pyhf/probability.py @@ -171,10 +171,10 @@ def log_prob(self, value): >>> independent = pyhf.probability.Independent(poissons) >>> values = pyhf.tensorlib.astensor([8.0, 9.0]) >>> independent.log_prob(values) - -4.26248380... + np.float64(-4.26248380...) >>> broadcast_value = pyhf.tensorlib.astensor([11.0]) >>> independent.log_prob(broadcast_value) - -4.34774364... + np.float64(-4.34774364...) Args: value (:obj:`tensor` or :obj:`float`): The value at which to evaluate the distribution diff --git a/src/pyhf/tensor/numpy_backend.py b/src/pyhf/tensor/numpy_backend.py index 677434cab5..b7ca445a70 100644 --- a/src/pyhf/tensor/numpy_backend.py +++ b/src/pyhf/tensor/numpy_backend.py @@ -320,7 +320,7 @@ def percentile( >>> pyhf.set_backend("numpy") >>> a = pyhf.tensorlib.astensor([[10, 7, 4], [3, 2, 1]]) >>> pyhf.tensorlib.percentile(a, 50) - 3.5 + np.float64(3.5) >>> pyhf.tensorlib.percentile(a, 50, axis=1) array([7., 2.]) @@ -384,7 +384,7 @@ def simple_broadcast(self, *args: Sequence[Tensor[T]]) -> Sequence[Tensor[T]]: ... pyhf.tensorlib.astensor([1]), ... pyhf.tensorlib.astensor([2, 3, 4]), ... pyhf.tensorlib.astensor([5, 6, 7])) - [array([1., 1., 1.]), array([2., 3., 4.]), array([5., 6., 7.])] + (array([1., 1., 1.]), array([2., 3., 4.]), array([5., 6., 7.])) Args: args (Array of Tensors): Sequence of arrays @@ -467,7 +467,7 @@ def poisson(self, n: Tensor[T], lam: Tensor[T]) -> ArrayLike: >>> import pyhf >>> pyhf.set_backend("numpy") >>> pyhf.tensorlib.poisson(5., 6.) - 0.16062314... + np.float64(0.16062314...) >>> values = pyhf.tensorlib.astensor([5., 9.]) >>> rates = pyhf.tensorlib.astensor([6., 8.]) >>> pyhf.tensorlib.poisson(values, rates) @@ -510,7 +510,7 @@ def normal(self, x: Tensor[T], mu: Tensor[T], sigma: Tensor[T]) -> ArrayLike: >>> import pyhf >>> pyhf.set_backend("numpy") >>> pyhf.tensorlib.normal(0.5, 0., 1.) - 0.35206532... + np.float64(0.35206532...) >>> values = pyhf.tensorlib.astensor([0.5, 2.0]) >>> means = pyhf.tensorlib.astensor([0., 2.3]) >>> sigmas = pyhf.tensorlib.astensor([1., 0.8]) @@ -538,7 +538,7 @@ def normal_cdf( >>> import pyhf >>> pyhf.set_backend("numpy") >>> pyhf.tensorlib.normal_cdf(0.8) - 0.78814460... + np.float64(0.78814460...) >>> values = pyhf.tensorlib.astensor([0.8, 2.0]) >>> pyhf.tensorlib.normal_cdf(values) array([0.7881446 , 0.97724987]) From 589d571a1a1516c3d842cca14cd7f812c1f61d05 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 19:42:49 -0500 Subject: [PATCH 14/16] verbose installation to understand why differences exist --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed42f3670b..44c47a27db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install uv - uv pip install --system --upgrade ".[all,test]" + uv pip install --system --upgrade --verbose ".[all,test]" - name: List installed Python packages run: python -m pip list From 37234d3c851d7f2794c08bef993146a0a58e423a Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 20:04:51 -0500 Subject: [PATCH 15/16] fix up again more with prints/list... --- src/pyhf/infer/calculators.py | 8 ++++---- src/pyhf/probability.py | 8 ++++---- src/pyhf/tensor/numpy_backend.py | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index ed8eaaf9c6..04ac58aec1 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -120,8 +120,8 @@ def cdf(self, value): >>> import pyhf >>> pyhf.set_backend("numpy") >>> bkg_dist = pyhf.infer.calculators.AsymptoticTestStatDistribution(0.0) - >>> bkg_dist.cdf(0.0) - np.float64(0.5) + >>> print(bkg_dist.cdf(0.0)) + 0.5 Args: value (:obj:`float`): The test statistic value. @@ -619,8 +619,8 @@ def expected_value(self, nsigma): >>> normal = pyhf.probability.Normal(mean, std) >>> samples = normal.sample((100,)) >>> dist = pyhf.infer.calculators.EmpiricalDistribution(samples) - >>> dist.expected_value(nsigma=1) - np.float64(6.15094381...) + >>> print(dist.expected_value(nsigma=1)) + 6.15094381... >>> import pyhf >>> import numpy.random as random diff --git a/src/pyhf/probability.py b/src/pyhf/probability.py index 8505757840..ed00c205f4 100644 --- a/src/pyhf/probability.py +++ b/src/pyhf/probability.py @@ -170,11 +170,11 @@ def log_prob(self, value): >>> poissons = pyhf.probability.Poisson(rates) >>> independent = pyhf.probability.Independent(poissons) >>> values = pyhf.tensorlib.astensor([8.0, 9.0]) - >>> independent.log_prob(values) - np.float64(-4.26248380...) + >>> print(independent.log_prob(values)) + -4.26248380... >>> broadcast_value = pyhf.tensorlib.astensor([11.0]) - >>> independent.log_prob(broadcast_value) - np.float64(-4.34774364...) + >>> print(independent.log_prob(broadcast_value)) + -4.34774364... Args: value (:obj:`tensor` or :obj:`float`): The value at which to evaluate the distribution diff --git a/src/pyhf/tensor/numpy_backend.py b/src/pyhf/tensor/numpy_backend.py index b7ca445a70..8135d15068 100644 --- a/src/pyhf/tensor/numpy_backend.py +++ b/src/pyhf/tensor/numpy_backend.py @@ -319,8 +319,8 @@ def percentile( >>> import pyhf >>> pyhf.set_backend("numpy") >>> a = pyhf.tensorlib.astensor([[10, 7, 4], [3, 2, 1]]) - >>> pyhf.tensorlib.percentile(a, 50) - np.float64(3.5) + >>> print(pyhf.tensorlib.percentile(a, 50)) + 3.5 >>> pyhf.tensorlib.percentile(a, 50, axis=1) array([7., 2.]) @@ -380,11 +380,11 @@ def simple_broadcast(self, *args: Sequence[Tensor[T]]) -> Sequence[Tensor[T]]: >>> import pyhf >>> pyhf.set_backend("numpy") - >>> pyhf.tensorlib.simple_broadcast( + >>> list(pyhf.tensorlib.simple_broadcast( ... pyhf.tensorlib.astensor([1]), ... pyhf.tensorlib.astensor([2, 3, 4]), - ... pyhf.tensorlib.astensor([5, 6, 7])) - (array([1., 1., 1.]), array([2., 3., 4.]), array([5., 6., 7.])) + ... pyhf.tensorlib.astensor([5, 6, 7]))) + [array([1., 1., 1.]), array([2., 3., 4.]), array([5., 6., 7.])] Args: args (Array of Tensors): Sequence of arrays @@ -466,8 +466,8 @@ def poisson(self, n: Tensor[T], lam: Tensor[T]) -> ArrayLike: >>> import pyhf >>> pyhf.set_backend("numpy") - >>> pyhf.tensorlib.poisson(5., 6.) - np.float64(0.16062314...) + >>> print(pyhf.tensorlib.poisson(5., 6.)) + 0.16062314... >>> values = pyhf.tensorlib.astensor([5., 9.]) >>> rates = pyhf.tensorlib.astensor([6., 8.]) >>> pyhf.tensorlib.poisson(values, rates) @@ -509,8 +509,8 @@ def normal(self, x: Tensor[T], mu: Tensor[T], sigma: Tensor[T]) -> ArrayLike: >>> import pyhf >>> pyhf.set_backend("numpy") - >>> pyhf.tensorlib.normal(0.5, 0., 1.) - np.float64(0.35206532...) + >>> print(pyhf.tensorlib.normal(0.5, 0., 1.)) + 0.35206532... >>> values = pyhf.tensorlib.astensor([0.5, 2.0]) >>> means = pyhf.tensorlib.astensor([0., 2.3]) >>> sigmas = pyhf.tensorlib.astensor([1., 0.8]) @@ -537,8 +537,8 @@ def normal_cdf( >>> import pyhf >>> pyhf.set_backend("numpy") - >>> pyhf.tensorlib.normal_cdf(0.8) - np.float64(0.78814460...) + >>> print(pyhf.tensorlib.normal_cdf(0.8)) + 0.78814460... >>> values = pyhf.tensorlib.astensor([0.8, 2.0]) >>> pyhf.tensorlib.normal_cdf(values) array([0.7881446 , 0.97724987]) From 42a5249004bbcd1401c6eaeba83399ae6a761ff6 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Sat, 21 Dec 2024 20:05:17 -0500 Subject: [PATCH 16/16] drop verbose --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44c47a27db..ed42f3670b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install uv - uv pip install --system --upgrade --verbose ".[all,test]" + uv pip install --system --upgrade ".[all,test]" - name: List installed Python packages run: python -m pip list