Add support for Python 3.11

.github/workflows/manage-python-cache.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Check out repository
         uses: actions/checkout@v4

.github/workflows/test-daily-integration.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Clear free space
         run: |

.github/workflows/test_scenarios.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Clear free space
         run: |

.github/workflows/update-requirements.yml

@@ -31,8 +31,8 @@ jobs:
         # PyTorch has to be installed manually in a separate step with --no-cache-dir
         # to avoid pip getting killed because PyTorch is too big
         # See: https://stackoverflow.com/a/54329850
-        run: pip install torch==2.2.2 torchvision==0.17.2 --index-url https://download.pytorch.org/whl/cu121 --no-cache-dir
-      - name: Upgrade dependencies for Dependabot alerts
+        run: pip install torch==2.2.2 torchvision==0.17.2 --index-url https://download.pytorch.org/whl/cpu --no-cache-dir
+      - name: Resolve and install requirements
         run: pip install -e .[all,dev]
       - name: Verify dependencies
         run: pip check

install-dev.sh

@@ -4,6 +4,7 @@
 
 set -e
 
+pip install --upgrade setuptools
 # On Mac OS, skip installing pytorch with CUDA because CUDA is not supported
 if [[ $OSTYPE != 'darwin'* ]]; then
   # Manually install pytorch with `--no-cache-dir` to avoid pip getting killed: https://stackoverflow.com/a/54329850

requirements.txt

@@ -88,12 +88,12 @@ google-api-core==2.19.2
 google-api-python-client==2.144.0
 google-auth==2.34.0
 google-auth-httplib2==0.2.0
-google-auth-oauthlib==0.4.6
+google-auth-oauthlib==0.5.3
 google-cloud-aiplatform==1.60.0
 google-cloud-bigquery==3.25.0
 google-cloud-core==2.4.1
 google-cloud-resource-manager==1.12.3
-google-cloud-storage==2.9.0
+google-cloud-storage==2.18.2
 google-cloud-translate==3.11.3
 google-crc32c==1.6.0
 google-pasta==0.2.0
@@ -102,7 +102,7 @@ googleapis-common-protos==1.63.1
 grapheme==0.6.0
 grpc-google-iam-v1==0.13.0
 grpcio==1.66.1
-grpcio-status==1.48.2
+grpcio-status==1.62.3
 gunicorn==20.1.0
 h11==0.14.0
 h5py==3.11.0
@@ -113,7 +113,6 @@ httpx==0.25.2
 httpx-sse==0.4.0
 huggingface-hub==0.23.5
 humanfriendly==10.0
-humanize==4.10.0
 icetk==0.0.4
 identify==2.6.0
 idna==3.8
@@ -132,7 +131,7 @@ joblib==1.4.2
 jsonschema==4.23.0
 jsonschema-specifications==2023.12.1
 jstyleson==0.0.2
-keras==2.11.0
+keras==3.5.0
 keras-tuner==1.4.7
 kiwisolver==1.4.7
 kt-legacy==1.0.5
@@ -143,7 +142,7 @@ latex==0.7.0
 lazy_loader==0.4
 libclang==18.1.1
 lightning-utilities==0.11.7
-llvmlite==0.39.1
+llvmlite==0.42.0
 lpips==0.1.4
 lxml==5.3.0
 Mako==1.3.5
@@ -172,15 +171,15 @@ nltk==3.8.1
 nncf==2.12.0
 nodeenv==1.9.1
 NudeNet==2.0.9
-numba==0.56.4
-numpy==1.23.5
+numba==0.59.1
+numpy==1.26.4
 oauthlib==3.2.2
 omegaconf==2.3.0
-onnx==1.12.0
+onnx==1.16.2
 onnxruntime==1.19.2
 open_clip_torch==2.26.1
 openai==1.44.0
-OpenCC==1.1.6
+OpenCC==1.1.7
 opencv-python==4.7.0.72
 opencv-python-headless==4.10.0.84
 openvino==2024.3.0
@@ -198,7 +197,6 @@ pandas==2.2.2
 param==2.1.1
 parameterized==0.9.0
 pathspec==0.12.1
-pathtools==0.1.2
 pdf2image==1.16.3
 pillow==10.4.0
 platformdirs==4.2.2
@@ -208,7 +206,7 @@ pre-commit==2.20.0
 preshed==3.0.9
 progressbar2==4.5.0
 proto-plus==1.24.0
-protobuf==3.19.6
+protobuf==4.25.4
 psutil==6.0.0
 pyarrow==17.0.0
 pyarrow-hotfix==0.6
@@ -284,11 +282,11 @@ surge-api==1.1.4
 sympy==1.11.1
 tabulate==0.9.0
 tempdir==0.7.1
-tensorboard==2.11.2
-tensorboard-data-server==0.6.1
+tensorboard==2.17.0
+tensorboard-data-server==0.7.2
 tensorboard-plugin-wit==1.8.1
-tensorflow==2.11.1
-tensorflow-estimator==2.11.0
+tensorflow==2.17.0
+tensorflow-estimator==2.15.0
 tensorflow-io-gcs-filesystem==0.37.1
 tensorstore==0.1.65
 termcolor==2.4.0
@@ -321,16 +319,16 @@ Unidecode==1.3.6
 uritemplate==4.1.1
 urllib3==1.26.20
 virtualenv==20.26.3
-wandb==0.13.11
+wandb==0.16.6
 wasabi==1.1.3
 wcwidth==0.2.13
 weasel==0.4.1
 websocket-client==1.3.3
 Werkzeug==3.0.4
-wrapt==1.16.0
+wrapt==1.14.0
 wsproto==1.2.0
 xlrd==2.0.1
 xxhash==3.5.0
 yarl==1.9.11
 zipp==3.20.1
-zstandard==0.18.0
+zstandard==0.18.0

setup.cfg

@@ -9,13 +9,16 @@ long_description_content_type = text/markdown
 keywords = language models benchmarking
 license = Apache License 2.0
 classifiers =
+    Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     License :: OSI Approved :: Apache Software License
 url = https://github.com/stanford-crfm/helm
 
 [options]
-python_requires = >=3.8,<3.11
+python_requires = >=3.9,<3.12
 package_dir =
     =src
 packages = find:
@@ -72,7 +75,7 @@ scenarios =
 
 metrics =
     google-api-python-client~=2.64  # For perspective_api_client via toxicity_metrics
-    numba~=0.56.4  # For copyright_metrics
+    numba~=0.56  # For copyright_metrics
     pytrec_eval==0.5  # For ranking_metrics
     sacrebleu~=2.2.1  # For disinformation_metrics, machine_translation_metrics
 
@@ -247,24 +250,22 @@ heim =
     # HEIM metrics
     autokeras~=1.0.20
     clip-anytorch~=2.5.0
-    google-cloud-storage~=2.9.0
+    google-cloud-storage~=2.9
     lpips~=0.1.4
     multilingual-clip~=1.0.10
     NudeNet~=2.0.9
     opencv-python~=4.7.0.68
     pytorch-fid~=0.3.0
-    tensorflow~=2.11.1
+    tensorflow~=2.11
     timm~=0.6.12
     torch-fidelity~=0.3.0
     torchmetrics~=0.11.1
 
     # Transitive dependency of NudeNet
-    # This needs to pinned to 0.21.0, which is the only version
-    # that provides wheels for all of HELM's supported Python versions
-    # i.e. Python 3.8, 3.9, 3.10
-    # https://github.com/scikit-image/scikit-image/pull/4920
-    # TODO(#1913): Upgrade to 0.22 after removing support for Python 3.8
-    scikit-image~=0.21.0
+    # This needs to be a version that provides wheels for all Python versions
+    # supported by crfm-helm i.e. Python 3.9, 3.10, 3.11, 3.12
+    # Disallow version 0.23.* because it has no Python 3.9 wheels.
+    scikit-image>=0.22,==0.*,!=0.23.*
 
     # Shared image dependencies
     crfm-helm[images]