diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd7bb40..a213983 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: test_pip_install: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ['3.11', '3.12'] name: Test with pip install ${{ matrix.python-version }} @@ -32,7 +33,7 @@ jobs: pytest: runs-on: ${{ matrix.os }} strategy: - max-parallel: 1 + fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ['3.11', '3.12'] @@ -48,24 +49,20 @@ jobs: run: | pip install ".[test]" - - name: install Spice - if: matrix.os != 'windows-latest' + - name: Install Spice (https://install.spiceai.org) (Linux) + if: matrix.os == 'ubuntu-latest' env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - function curl() { - if [ -z "$GH_TOKEN" ] - then - command curl -H "Accept: application/vnd.github.v3.raw" \ - $@ - else - command curl -H "Accept: application/vnd.github.v3.raw" \ - -H "Authorization: token $GH_TOKEN" \ - $@ - fi - } curl https://install.spiceai.org | /bin/bash echo "$HOME/.spice/bin" >> $GITHUB_PATH + $HOME/.spice/bin/spice install + + - name: Install Spice (https://install.spiceai.org) (MacOS) + if: matrix.os == 'macos-latest' + run: | + brew install spiceai/spiceai/spice + brew install spiceai/spiceai/spiced - name: install Spice (Windows) if: matrix.os == 'windows-latest' @@ -84,7 +81,7 @@ jobs: spice init spice_qs cd spice_qs spice add spiceai/quickstart - spice run &> spice.log & + spiced &> spice.log & # time to initialize added dataset sleep 10 @@ -109,8 +106,7 @@ jobs: - name: Stop spice and check logs working-directory: spice_qs - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-latest' && always() run: | - sleep 10 - killall spice + killall spice || true cat spice.log diff --git a/.gitignore b/.gitignore index 95cab01..f672ab8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ .pytest_cache build dist -spicepy.egg-info \ No newline at end of file +spicepy.egg-info +.env \ No newline at end of file diff --git a/spicepy/_client.py b/spicepy/_client.py index 6236ae4..535ab61 100644 --- a/spicepy/_client.py +++ b/spicepy/_client.py @@ -5,7 +5,13 @@ from typing import Dict, Union import certifi -from pyarrow._flight import FlightCallOptions, FlightClient, Ticket # pylint: disable=E0611 + +# pylint: disable=E0611 +from pyarrow._flight import ( + FlightCallOptions, + FlightClient, + Ticket, +) from ._http import HttpRequests from . import config @@ -53,20 +59,32 @@ def read_cert(self, tls_root_cert): class _SpiceFlight: + @staticmethod + def _user_agent(): + # headers kwargs claim to support Tuple[str, str], but it's actually Tuple[bytes, bytes] :| + # Open issue in Arrow: https://github.com/apache/arrow/issues/35288 + return (str.encode("x-spice-user-agent"), str.encode(config.SPICE_USER_AGENT)) + def __init__(self, grpc: str, api_key: str, tls_root_certs): self._flight_client = flight.connect(grpc, tls_root_certs=tls_root_certs) self._api_key = api_key - self._flight_options = flight.FlightCallOptions() + self.headers = [_SpiceFlight._user_agent()] + self._flight_options = flight.FlightCallOptions( + headers=self.headers, timeout=DEFAULT_QUERY_TIMEOUT_SECS + ) self._authenticate() def _authenticate(self): if self._api_key is not None: - self.headers = [self._flight_client.authenticate_basic_token("", self._api_key)] + self.headers = [ + self._flight_client.authenticate_basic_token("", self._api_key), + _SpiceFlight._user_agent(), + ] self._flight_options = flight.FlightCallOptions( headers=self.headers, timeout=DEFAULT_QUERY_TIMEOUT_SECS ) else: - self.headers = [] + self.headers = [_SpiceFlight._user_agent()] self._flight_options = flight.FlightCallOptions( headers=self.headers, timeout=DEFAULT_QUERY_TIMEOUT_SECS ) diff --git a/spicepy/_http.py b/spicepy/_http.py index 2f28788..ac44664 100644 --- a/spicepy/_http.py +++ b/spicepy/_http.py @@ -5,14 +5,19 @@ from requests.adapters import HTTPAdapter, Retry from .error import SpiceAIError +from .config import SPICE_USER_AGENT -HttpMethod = Literal['POST', 'GET', 'PUT', 'HEAD', 'POST'] +HttpMethod = Literal["POST", "GET", "PUT", "HEAD", "POST"] class HttpRequests: def __init__(self, base_url: str, headers: Dict[str, str]) -> None: self.session = self._create_session(headers) + + # set the x-spice-user-agent header + self.session.headers["X-Spice-User-Agent"] = SPICE_USER_AGENT + self.base_url = base_url def send_request( diff --git a/spicepy/config.py b/spicepy/config.py index 2fe1969..62afba9 100644 --- a/spicepy/config.py +++ b/spicepy/config.py @@ -1,8 +1,31 @@ import os +import platform +from importlib.metadata import version DEFAULT_FLIGHT_URL = os.environ.get("SPICE_FLIGHT_URL", "grpc+tls://flight.spiceai.io") -DEFAULT_FIRECACHE_URL = os.environ.get("SPICE_FIRECACHE_URL", "grpc+tls://firecache.spiceai.io") +DEFAULT_FIRECACHE_URL = os.environ.get( + "SPICE_FIRECACHE_URL", "grpc+tls://firecache.spiceai.io" +) DEFAULT_HTTP_URL = os.environ.get("SPICE_HTTP_URL", "https://data.spiceai.io") -DEFAULT_LOCAL_FLIGHT_URL = os.environ.get("SPICE_LOCAL_FLIGHT_URL", "grpc://localhost:50051") -DEFAULT_LOCAL_HTTP_URL = os.environ.get("SPICE_LOCAL_HTTP_URL", "http://localhost:3000 ") +DEFAULT_LOCAL_FLIGHT_URL = os.environ.get( + "SPICE_LOCAL_FLIGHT_URL", "grpc://localhost:50051" +) +DEFAULT_LOCAL_HTTP_URL = os.environ.get( + "SPICE_LOCAL_HTTP_URL", "http://localhost:3000 " +) + + +def get_user_agent(): + package_version = version("spicepy") + system = platform.system() + release = platform.release() + arch = platform.machine() + if arch == "AMD64": + arch = "x86_64" + + system_info = f"{system}/{release} {arch}" + return f"spicepy {package_version} ({system_info})" + + +SPICE_USER_AGENT = get_user_agent() diff --git a/tests/test_main.py b/tests/test_main.py index f450c50..05cbcd5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,8 +1,9 @@ - import os import time +import re import pytest from spicepy import Client +from spicepy.config import SPICE_USER_AGENT # Skip cloud tests if TEST_SPICE_CLOUD is not set to true @@ -13,16 +14,20 @@ def skip_cloud(): def get_cloud_client(): api_key = os.environ["API_KEY"] - return Client( - api_key=api_key, - flight_url="grpc+tls://flight.spiceai.io" - ) + return Client(api_key=api_key, flight_url="grpc+tls://flight.spiceai.io") def get_local_client(): return Client(flight_url="grpc://localhost:50051") +def test_user_agent_is_populated(): + # use a regex to match the expected user agent string + matching_regex = r"spicepy \d+\.\d+\.\d+ \((Linux|Windows|Darwin)/[\d\w\.\-\_]+ (x86_64|aarch64|i386|arm64)\)" + + assert re.match(matching_regex, SPICE_USER_AGENT) + + @skip_cloud() def test_flight_recent_blocks(): client = get_cloud_client()