chore: Drop Python 3.8, fix issues with CI

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         include:
           - os: macos-latest
             python-version: '3.12'
@@ -47,18 +47,10 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
-      if: matrix.python-version != '3.8'
       run: |
         python -m pip install uv
         uv pip install --system --upgrade ".[all,test]"
 
-      # c.f. https://github.com/astral-sh/uv/issues/2062
-    - name: Install dependencies (Python 3.8)
-      if: matrix.python-version == '3.8'
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install --upgrade ".[all,test]"
-
     - name: List installed Python packages
       run: python -m pip list
 

.github/workflows/lower-bound-requirements.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         # minimum supported Python
-        python-version: ['3.8']
+        python-version: ['3.9']
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/release_tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         include:
           - os: macos-latest
             python-version: '3.12'

.pre-commit-config.yaml

@@ -55,14 +55,15 @@ repos:
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.13.0
     # check the oldest and newest supported Pythons
+    # except skip python 3.9 for numpy, due to poor typing
     hooks:
       - &mypy
         id: mypy
-        name: mypy with Python 3.8
+        name: mypy with Python 3.10
         files: src
         additional_dependencies:
           ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging']
-        args: ["--python-version=3.8"]
+        args: ["--python-version=3.10"]
       - <<: *mypy
         name: mypy with Python 3.12
         args: ["--python-version=3.12"]

docs/conf.py

@@ -529,6 +529,15 @@ def setup(app):
     # and ReadTheDocs need to reference them
     r'https://github.com/scikit-hep/pyhf/releases/tag/.*',
     r'https://pyhf.readthedocs.io/en/.*',
+    # the following are 403s as they map to journals.aps.org or academic.oup.com
+    r'https://doi.org/10.1093/ptep/ptad144',
+    r'https://doi.org/10.1103/PhysRevD.104.055017',
+    r'https://doi.org/10.1103/PhysRevD.107.095021',
+    r'https://doi.org/10.1103/PhysRevD.108.016002',
+    r'https://doi.org/10.1103/PhysRevD.106.032005',
+    r'https://doi.org/10.1103/PhysRevLett.127.181802',
+    r'https://doi.org/10.1103/PhysRevLett.130.231801',
+    r'https://doi.org/10.1103/PhysRevLett.131.211802',
 ]
 linkcheck_retries = 50
 

pyproject.toml

@@ -11,7 +11,7 @@ dynamic = ["version"]
 description = "pure-Python HistFactory implementation with tensors and autodiff"
 readme = "README.rst"
 license = { text = "Apache-2.0" }  # SPDX short identifier
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
     { name = "Lukas Heinrich", email = "lukas.heinrich@cern.ch" },
     { name = "Matthew Feickert", email = "matthew.feickert@cern.ch" },
@@ -35,7 +35,6 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -68,18 +67,10 @@ Homepage = "https://github.com/scikit-hep/pyhf"
 
 [project.optional-dependencies]
 shellcomplete = ["click_completion"]
-# TODO: 'tensorflow' supports all platform_machine for tensorflow v2.16.1+
-# but TensorFlow only supports python_version 3.8 up through tensorflow v2.13.1.
-# So until Python 3.8 support is dropped, split requirments on python_version
-# before and after 3.9.
 # NOTE: macos x86 support is deprecated from tensorflow v2.17.0 onwards.
 tensorflow = [
-    # python == 3.8
-    "tensorflow>=2.7.0; python_version < '3.9' and platform_machine != 'arm64'",  # c.f. PR #1962, #2452
-    "tensorflow-macos>=2.7.0; python_version < '3.9' and platform_machine == 'arm64' and platform_system == 'Darwin'",  # c.f. PR #2119, #2452
-    "tensorflow-probability>=0.11.0; python_version < '3.9'",  # c.f. PR #1657, #2452
-    # python >= 3.9
-    "tensorflow-probability[tf]>=0.24.0; python_version >= '3.9'"  # c.f. PR #2452
+    # see https://github.com/tensorflow/probability/issues/1854
+    "tensorflow-probability[tf]>=0.24.0,!=0.25.0"  # c.f. PR #2452
 ]
 torch = ["torch>=1.10.0"]  # c.f. PR #1657
 jax = [
@@ -251,6 +242,7 @@ warn_unused_configs = true
 strict = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 warn_unreachable = true
+plugins = "numpy.typing.mypy_plugin"
 
 [[tool.mypy.overrides]]
 module = [

src/pyhf/cli/infer.py

@@ -9,7 +9,8 @@
 from pyhf.infer import hypotest
 from pyhf.infer import mle
 from pyhf.workspace import Workspace
-from pyhf import get_backend, set_backend, optimize
+from pyhf.tensor.manager import get_backend, set_backend
+from pyhf import optimize
 
 log = logging.getLogger(__name__)
 

src/pyhf/contrib/utils.py

@@ -80,13 +80,7 @@
                 with open(output_directory, "wb") as archive:
                     archive.write(response.content)
             else:
-                # Support for file-like objects for tarfile.is_tarfile was added
-                # in Python 3.9, so as pyhf is currently Python 3.8+ then can't
-                # do tarfile.is_tarfile(BytesIO(response.content)).
-                # Instead, just use a 'try except' block to determine if the
-                # archive is a valid tarfile.
-                # TODO: Simplify after pyhf is Python 3.9+ only
-                try:
+                if tarfile.is_tarfile(BytesIO(response.content)):
                     # Use transparent compression to allow for .tar or .tar.gz
                     with tarfile.open(
                         mode="r:*", fileobj=BytesIO(response.content)
@@ -97,13 +91,7 @@
                             archive.extractall(output_directory, filter="data")
                         else:
                             archive.extractall(output_directory)
-                except tarfile.ReadError:
-                    if not zipfile.is_zipfile(BytesIO(response.content)):
-                        raise exceptions.InvalidArchive(
-                            f"The archive downloaded from {archive_url} is not a tarfile"
-                            + " or a zipfile and so can not be opened as one."
-                        )
-
+                elif zipfile.is_zipfile(BytesIO(response.content)):
                     output_directory = Path(output_directory)
                     if output_directory.exists():
                         rmtree(output_directory)
@@ -129,6 +117,11 @@
                         # from creation time
                         rmtree(output_directory)
                         _tmp_path.replace(output_directory)
+                else:
+                    raise exceptions.InvalidArchive(
+                        f"The archive downloaded from {archive_url} is not a tarfile"
+                        + " or a zipfile and so can not be opened as one."
+                    )
 
 except ModuleNotFoundError:
     log.error(

src/pyhf/infer/__init__.py

@@ -1,7 +1,7 @@
 """Inference for Statistical Models."""
 
 from pyhf.infer import utils
-from pyhf import get_backend
+from pyhf.tensor.manager import get_backend
 from pyhf import exceptions
 
 

src/pyhf/infer/calculators.py

@@ -9,7 +9,7 @@
 """
 
 from pyhf.infer.mle import fixed_poi_fit
-from pyhf import get_backend
+from pyhf.tensor.manager import get_backend
 from pyhf.infer import utils
 import tqdm
 
@@ -120,7 +120,7 @@ def cdf(self, value):
             >>> import pyhf
             >>> pyhf.set_backend("numpy")
             >>> bkg_dist = pyhf.infer.calculators.AsymptoticTestStatDistribution(0.0)
-            >>> bkg_dist.cdf(0.0)
+            >>> print(bkg_dist.cdf(0.0))
             0.5
 
         Args:
@@ -619,7 +619,7 @@ def expected_value(self, nsigma):
             >>> normal = pyhf.probability.Normal(mean, std)
             >>> samples = normal.sample((100,))
             >>> dist = pyhf.infer.calculators.EmpiricalDistribution(samples)
-            >>> dist.expected_value(nsigma=1)
+            >>> print(dist.expected_value(nsigma=1))
             6.15094381...
 
             >>> import pyhf

src/pyhf/infer/intervals/upper_limits.py

@@ -3,7 +3,7 @@
 import numpy as np
 from scipy.optimize import toms748
 
-from pyhf import get_backend
+from pyhf.tensor.manager import get_backend
 from pyhf.infer import hypotest
 
 __all__ = ["linear_grid_scan", "toms748_scan", "upper_limit"]

src/pyhf/infer/mle.py

@@ -1,6 +1,6 @@
 """Module for Maximum Likelihood Estimation."""
 
-from pyhf import get_backend
+from pyhf.tensor.manager import get_backend
 from pyhf.exceptions import UnspecifiedPOI
 
 __all__ = ["fit", "fixed_poi_fit", "twice_nll"]

src/pyhf/infer/test_statistics.py

@@ -1,4 +1,4 @@
-from pyhf import get_backend
+from pyhf.tensor.manager import get_backend
 from pyhf.infer.mle import fixed_poi_fit, fit
 from pyhf.exceptions import UnspecifiedPOI
 

src/pyhf/mixins.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import logging
-from typing import Any, Sequence
+from typing import Any
+from collections.abc import Sequence
 
 from pyhf.typing import Channel
 

src/pyhf/modifiers/histosys.py

@@ -101,6 +101,8 @@ class histosys_combined:
     def __init__(
         self, modifiers, pdfconfig, builder_data, interpcode='code0', batch_size=None
     ):
+        default_backend = pyhf.default_backend
+
         self.batch_size = batch_size
         self.interpcode = interpcode
         assert self.interpcode in ['code0', 'code2', 'code4p']
@@ -128,10 +130,13 @@ def __init__(
             ]
             for m in keys
         ]
-        self._histosys_mask = [
-            [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
-            for m in keys
-        ]
+        self._histosys_mask = default_backend.astensor(
+            [
+                [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
+                for m in keys
+            ],
+            dtype='bool',
+        )
 
         if histosys_mods:
             self.interpolator = getattr(interpolators, self.interpcode)(

src/pyhf/modifiers/lumi.py

@@ -1,6 +1,8 @@
 import logging
 
-from pyhf import get_backend, events
+import pyhf
+from pyhf.tensor.manager import get_backend
+from pyhf import events
 from pyhf.parameters import ParamViewer
 
 log = logging.getLogger(__name__)
@@ -60,6 +62,8 @@ class lumi_combined:
     op_code = 'multiplication'
 
     def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
+        default_backend = pyhf.default_backend
+
         self.batch_size = batch_size
 
         keys = [f'{mtype}/{m}' for m, mtype in modifiers]
@@ -72,10 +76,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
         )
         self.param_viewer = ParamViewer(parfield_shape, pdfconfig.par_map, lumi_mods)
 
-        self._lumi_mask = [
-            [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
-            for m in keys
-        ]
+        self._lumi_mask = default_backend.astensor(
+            [
+                [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
+                for m in keys
+            ],
+            dtype='bool',
+        )
         self._precompute()
         events.subscribe('tensorlib_changed')(self._precompute)
 

src/pyhf/modifiers/normfactor.py

@@ -1,6 +1,8 @@
 import logging
 
-from pyhf import get_backend, events
+import pyhf
+from pyhf.tensor.manager import get_backend
+from pyhf import events
 from pyhf.parameters import ParamViewer
 
 log = logging.getLogger(__name__)
@@ -58,6 +60,8 @@ class normfactor_combined:
     op_code = 'multiplication'
 
     def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
+        default_backend = pyhf.default_backend
+
         self.batch_size = batch_size
 
         keys = [f'{mtype}/{m}' for m, mtype in modifiers]
@@ -72,10 +76,13 @@ def __init__(self, modifiers, pdfconfig, builder_data, batch_size=None):
             parfield_shape, pdfconfig.par_map, normfactor_mods
         )
 
-        self._normfactor_mask = [
-            [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
-            for m in keys
-        ]
+        self._normfactor_mask = default_backend.astensor(
+            [
+                [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
+                for m in keys
+            ],
+            dtype='bool',
+        )
         self._precompute()
         events.subscribe('tensorlib_changed')(self._precompute)
 

src/pyhf/modifiers/normsys.py

@@ -1,6 +1,8 @@
 import logging
 
-from pyhf import get_backend, events
+import pyhf
+from pyhf.tensor.manager import get_backend
+from pyhf import events
 from pyhf import interpolators
 from pyhf.parameters import ParamViewer
 
@@ -72,6 +74,8 @@ class normsys_combined:
     def __init__(
         self, modifiers, pdfconfig, builder_data, interpcode='code1', batch_size=None
     ):
+        default_backend = pyhf.default_backend
+
         self.interpcode = interpcode
         assert self.interpcode in ['code1', 'code4']
 
@@ -97,10 +101,13 @@ def __init__(
             ]
             for m in keys
         ]
-        self._normsys_mask = [
-            [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
-            for m in keys
-        ]
+        self._normsys_mask = default_backend.astensor(
+            [
+                [[builder_data[m][s]['data']['mask']] for s in pdfconfig.samples]
+                for m in keys
+            ],
+            dtype='bool',
+        )
 
         if normsys_mods:
             self.interpolator = getattr(interpolators, self.interpcode)(