fix current failures in pytest when run locally

CONTRIBUTING.md

@@ -12,7 +12,7 @@ We welcome contributions to `guidance`, and this document exists to provide usef
 
 The quickest way to get started is to run (in a fresh environment):
 ```bash
-pip install -e .[all,test]
+pip install -e .[all,test,bench]
 ```
 which should bring in all of the basic required dependencies.
 Note that if you want to use GPU acceleration, then you will need to do whatever is required to allow `torch` and `llama-cpp` to access your GPU too.
@@ -32,16 +32,16 @@ However, if you have your own means of installing Rust and CUDA, you should be a
 
 ## Running Tests
 
-Because we run tests on GPU-equipped machines and also tests which call LLM endpoints, approval is required before our GitHub workflows will run on external Pull Requests.
-To run a basic test suite locally, we suggest:
+To run a basic test suite locally:
 ```bash
-python -m pytest -m "not (needs_credentials or use_gpu or server)" ./tests/
+python -m pytest ./tests/
 ```
 which runs our basic test suite.
 Where an LLM is required, this will default to using GPT2 on the CPU.
+
 To change that default, run
 ```bash
-python -m pytest -m "not (needs_credentials or use_gpu or server)" --selected_model <MODELNAME> ./tests/
+python -m pytest --selected_model <MODELNAME> ./tests/
 ```
 where `<MODELNAME>` is taken from the `AVAILABLE_MODELS` dictionary defined in `_llms_for_testing.py`.
 
@@ -68,7 +68,6 @@ If your model requires credentials, then those will need to be added to our GitH
 The endpoint itself (and any other required information) should be configured as environment variables too.
 When the test runs, the environment variables will be set, and can then be used to configure the model as required.
 See `test_azureai_openai.py` for examples of this being done.
-The tests should also be marked as `needs_credentials` - if this is needed for the entire module, then `pytestmark` can be used - see `test_azureai_openai.py` again for this.
 
 The environment variables and secrets will also need to be configured in the `ci_tests.yml` file.
 

tests/bench/test_powerlift.py

@@ -11,7 +11,6 @@ def test_retrieve_langchain_err(monkeypatch):
         _ = list(gen)
 
 @pytest.mark.skip("Waiting on CI upgrades. Need access to env var LANGCHAIN_API_KEY.")
-# @pytest.mark.needs_credentials
 def test_retrieve_langchain_basic():
     with tempfile.TemporaryDirectory() as tmp_dir:
         # Run once

tests/need_credentials/test_azureai_openai.py

@@ -9,13 +9,13 @@
 from guidance import assistant, gen, models, system, user
 
 from ..model_specific import common_chat_testing
-from ..utils import env_or_fail
+from ..utils import env_or_skip
 
 
 @pytest.fixture(scope="function")
 def azureai_chat_model(rate_limiter):
-    azureai_endpoint = env_or_fail("AZUREAI_CHAT_ENDPOINT")
-    model = env_or_fail("AZUREAI_CHAT_MODEL")
+    azureai_endpoint = env_or_skip("AZUREAI_CHAT_ENDPOINT")
+    model = env_or_skip("AZUREAI_CHAT_MODEL")
 
     print(f"{azureai_endpoint=}")
     print(f"{model=}")
@@ -46,8 +46,8 @@ def test_azureai_openai_chat_longer_2(azureai_chat_model):
 
 
 def test_azureai_openai_chat_alt_args(rate_limiter):
-    azureai_endpoint = env_or_fail("AZUREAI_CHAT_ENDPOINT")
-    model = env_or_fail("AZUREAI_CHAT_MODEL")
+    azureai_endpoint = env_or_skip("AZUREAI_CHAT_ENDPOINT")
+    model = env_or_skip("AZUREAI_CHAT_MODEL")
 
     parsed_url = urlparse(azureai_endpoint)
     parsed_query = parse_qs(parsed_url.query)
@@ -71,8 +71,8 @@ def test_azureai_openai_chat_alt_args(rate_limiter):
 
 
 def test_azureai_openai_completion_smoke(rate_limiter):
-    azureai_endpoint = env_or_fail("AZUREAI_COMPLETION_ENDPOINT")
-    model = env_or_fail("AZUREAI_COMPLETION_MODEL")
+    azureai_endpoint = env_or_skip("AZUREAI_COMPLETION_ENDPOINT")
+    model = env_or_skip("AZUREAI_COMPLETION_MODEL")
 
     print(f"endpoint: {' '.join(azureai_endpoint)}")
     print(f"model: {' '.join(model)}")
@@ -95,8 +95,8 @@ def test_azureai_openai_completion_smoke(rate_limiter):
 
 
 def test_azureai_openai_completion_alt_args(rate_limiter):
-    azureai_endpoint = env_or_fail("AZUREAI_COMPLETION_ENDPOINT")
-    model = env_or_fail("AZUREAI_COMPLETION_MODEL")
+    azureai_endpoint = env_or_skip("AZUREAI_COMPLETION_ENDPOINT")
+    model = env_or_skip("AZUREAI_COMPLETION_MODEL")
 
     parsed_url = urlparse(azureai_endpoint)
     parsed_query = parse_qs(parsed_url.query)

tests/need_credentials/test_chat_templates.py

@@ -5,7 +5,7 @@
 
 from guidance.chat import CHAT_TEMPLATE_CACHE
 
-from ..utils import env_or_fail
+from ..utils import env_or_skip
 
 
 @pytest.mark.parametrize(
@@ -26,7 +26,7 @@
 def test_popular_models_in_cache(model_id: str, should_pass: bool):
     # This test simply checks to make sure the chat_templates haven't changed, and that they're still in our cache.
     # If this fails, the models have had their templates updated, and we need to fix the cache manually.
-    hf_token = env_or_fail("HF_TOKEN")
+    hf_token = env_or_skip("HF_TOKEN")
 
     tokenizer = transformers.AutoTokenizer.from_pretrained(
         model_id, token=hf_token, trust_remote_code=True
@@ -56,7 +56,7 @@ def test_popular_models_in_cache(model_id: str, should_pass: bool):
     ],
 )
 def test_chat_format_smoke(model_id: str):
-    hf_token = env_or_fail("HF_TOKEN")
+    hf_token = env_or_skip("HF_TOKEN")
 
     tokenizer = transformers.AutoTokenizer.from_pretrained(
         model_id, token=hf_token, trust_remote_code=True
@@ -97,7 +97,7 @@ def test_chat_format_smoke(model_id: str):
     ],
 )
 def test_chat_format_smoke_with_system(model_id: str):
-    hf_token = env_or_fail("HF_TOKEN")
+    hf_token = env_or_skip("HF_TOKEN")
 
     tokenizer = transformers.AutoTokenizer.from_pretrained(
         model_id, token=hf_token, trust_remote_code=True

tests/need_credentials/test_tokenizers.py

@@ -19,9 +19,15 @@ class TestAuthenticatedTransformerTokenizers(BaseTestTransformerTokenizers):
         TRANSFORMER_MODELS,
     )
     def test_smoke(self, model_name: str):
-        self.base_smoke(model_name)
+        try:
+            self.base_smoke(model_name)
+        except OSError:
+            pytest.skip("HuggingFace raises OSError if user is not authenticated.")
 
     @pytest.mark.parametrize("model_name", TRANSFORMER_MODELS)
     @pytest.mark.parametrize("target_string", TOKENIZER_ROUND_TRIP_STRINGS)
     def test_string_roundtrip(self, model_name: str, target_string: str):
-        self.base_string_roundtrip(model_name, target_string)
+        try:
+            self.base_string_roundtrip(model_name, target_string)
+        except OSError:
+            pytest.skip("HuggingFace raises OSError if user is not authenticated.")

tests/notebooks/test_notebooks.py

@@ -7,6 +7,8 @@
 import papermill as pm
 import pytest
 
+from ..utils import env_or_skip
+
 BASE_NB_PATH = pathlib.Path("./notebooks").absolute()
 
 
@@ -23,7 +25,7 @@ class TestTutorials:
     BASE_TUTORIAL_PATH = BASE_NB_PATH / "tutorials"
 
     def test_chat(self, rate_limiter):
-        azureai_endpoint = os.getenv("AZUREAI_CHAT_ENDPOINT", None)
+        azureai_endpoint = env_or_skip("AZUREAI_CHAT_ENDPOINT")
 
         parsed_url = urlparse(azureai_endpoint)
         parsed_query = parse_qs(parsed_url.query)
@@ -54,7 +56,7 @@ class TestModels:
     BASE_MODEL_PATH = BASE_NB_PATH / "api_examples" / "models"
 
     def test_azure_openai(self, rate_limiter):
-        azureai_endpoint = os.getenv("AZUREAI_CHAT_ENDPOINT", None)
+        azureai_endpoint = env_or_skip("AZUREAI_CHAT_ENDPOINT")
 
         parsed_url = urlparse(azureai_endpoint)
         parsed_query = parse_qs(parsed_url.query)

tests/utils.py

@@ -15,10 +15,16 @@
 def env_or_fail(var_name: str) -> str:
     env_value = os.getenv(var_name, None)
 
-    assert env_value is not None, f"Env '{var_name}' not found"
+    assert env_value is not None, f"Env '{var_name}' not found."
 
     return env_value
 
+def env_or_skip(var_name: str) -> str:
+    env_value = os.getenv(var_name, None)
+    if env_value is None:
+        pytest.skip(f"Env '{var_name}' not found.")
+    return env_value
+
 
 def get_model(model_name, caching=False, **kwargs):
     """Get an LLM by name."""