From 4df68b5b720541690ad7b63c3183dc530161d8e3 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 22 May 2025 16:49:24 -0400 Subject: [PATCH 1/7] feat: `get_obstore_store` function for creating Obstore store with Open PC credentials --- planetary_computer/__init__.py | 2 ++ planetary_computer/_obstore.py | 58 ++++++++++++++++++++++++++++++++++ pyproject.toml | 3 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 planetary_computer/_obstore.py diff --git a/planetary_computer/__init__.py b/planetary_computer/__init__.py index 6088948..43432b2 100644 --- a/planetary_computer/__init__.py +++ b/planetary_computer/__init__.py @@ -13,12 +13,14 @@ ) from planetary_computer.settings import set_subscription_key from planetary_computer._adlfs import get_adlfs_filesystem, get_container_client +from planetary_computer._obstore import get_obstore_store from planetary_computer.version import __version__ __all__ = [ "get_adlfs_filesystem", "get_container_client", + "get_obstore_store", "set_subscription_key", "sign_asset", "sign_assets", diff --git a/planetary_computer/_obstore.py b/planetary_computer/_obstore.py new file mode 100644 index 0000000..b9ccef7 --- /dev/null +++ b/planetary_computer/_obstore.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from planetary_computer.sas import get_token + +if TYPE_CHECKING: + import sys + + from obstore.store import ( + AzureStore, + AzureConfig, + ClientConfig, + RetryConfig, + AzureSASToken, + ) + + if sys.version_info >= (3, 11): + from typing import Unpack + else: + from typing_extensions import Unpack + + +def get_obstore_store( # type: ignore[misc] # overlap with kwargs + account_name: str, + container_name: str, + *, + prefix: str | None = None, + config: AzureConfig | None = None, + client_options: ClientConfig | None = None, + retry_config: RetryConfig | None = None, + **kwargs: Unpack[AzureConfig], # type: ignore # noqa: PGH003 (container_name key overlaps with positional arg) +) -> AzureStore: + try: + import obstore + except ImportError as e: + raise ImportError( + "'planetary_computer.get_obstore_store' requires " + "the optional dependency 'obstore'." + ) from e + + def credential_provider() -> AzureSASToken: + token = get_token(account_name, container_name) + return { + "sas_token": token.token, + "expires_at": token.expiry, + } + + return obstore.store.AzureStore( + account_name=account_name, + container_name=container_name, + prefix=prefix, + config=config, + client_options=client_options, + retry_config=retry_config, + credential_provider=credential_provider, + **kwargs, # type: ignore # (container_name key overlaps with positional arg) + ) diff --git a/pyproject.toml b/pyproject.toml index a2bbc21..9ac7ee2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dev = [ "pytest", "responses", ] +obstore = ["obstore"] [project.scripts] planetarycomputer = "planetary_computer.scripts.cli:app" @@ -42,4 +43,4 @@ planetarycomputer = "planetary_computer.scripts.cli:app" include-package-data = false [tool.setuptools.dynamic] -version = {attr = "planetary_computer.version.__version__"} \ No newline at end of file +version = {attr = "planetary_computer.version.__version__"} From d5a6ef575073c91154726d9afacb0c8137a9eebb Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 28 May 2025 14:23:36 -0400 Subject: [PATCH 2/7] Install obstore in ci tests --- scripts/cibuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cibuild b/scripts/cibuild index bd40609..e77ffc3 100755 --- a/scripts/cibuild +++ b/scripts/cibuild @@ -19,7 +19,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then else # Install/upgrade dependencies python -m pip install --upgrade pip - pip install -e .[adlfs,azure,dev] + pip install -e .[adlfs,azure,dev,obstore] ./scripts/test fi From 22c97b5ec51884ba85f2b034706d4a33007e328b Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 4 Jun 2025 15:38:18 -0400 Subject: [PATCH 3/7] Add default credential_provider argument --- planetary_computer/_obstore.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/planetary_computer/_obstore.py b/planetary_computer/_obstore.py index b9ccef7..a719a41 100644 --- a/planetary_computer/_obstore.py +++ b/planetary_computer/_obstore.py @@ -8,11 +8,12 @@ import sys from obstore.store import ( - AzureStore, AzureConfig, + AzureCredentialProvider, + AzureSASToken, + AzureStore, ClientConfig, RetryConfig, - AzureSASToken, ) if sys.version_info >= (3, 11): @@ -21,7 +22,7 @@ from typing_extensions import Unpack -def get_obstore_store( # type: ignore[misc] # overlap with kwargs +def get_obstore( # type: ignore[misc] # overlap with kwargs account_name: str, container_name: str, *, @@ -29,6 +30,7 @@ def get_obstore_store( # type: ignore[misc] # overlap with kwargs config: AzureConfig | None = None, client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, + credential_provider: AzureCredentialProvider | None = None, **kwargs: Unpack[AzureConfig], # type: ignore # noqa: PGH003 (container_name key overlaps with positional arg) ) -> AzureStore: try: @@ -39,13 +41,15 @@ def get_obstore_store( # type: ignore[misc] # overlap with kwargs "the optional dependency 'obstore'." ) from e - def credential_provider() -> AzureSASToken: + def default_credential_provider() -> AzureSASToken: token = get_token(account_name, container_name) return { "sas_token": token.token, "expires_at": token.expiry, } + credential_provider = credential_provider or default_credential_provider + return obstore.store.AzureStore( account_name=account_name, container_name=container_name, From 5afe1512a6e9b299a7dc6ef82078506842e2cd98 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 12 Jun 2025 14:03:29 -0400 Subject: [PATCH 4/7] Remove Python 3.8 from CI --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 807cb41..6cfc27b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From dd04899d37dccbcbf6d71c703d7b2c41be0fdbea Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 12 Jun 2025 16:15:59 -0400 Subject: [PATCH 5/7] lint --- planetary_computer/__init__.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/planetary_computer/__init__.py b/planetary_computer/__init__.py index 43432b2..fc2990b 100644 --- a/planetary_computer/__init__.py +++ b/planetary_computer/__init__.py @@ -13,14 +13,14 @@ ) from planetary_computer.settings import set_subscription_key from planetary_computer._adlfs import get_adlfs_filesystem, get_container_client -from planetary_computer._obstore import get_obstore_store +from planetary_computer._obstore import get_obstore from planetary_computer.version import __version__ __all__ = [ "get_adlfs_filesystem", "get_container_client", - "get_obstore_store", + "get_obstore", "set_subscription_key", "sign_asset", "sign_assets", diff --git a/pyproject.toml b/pyproject.toml index 9ac7ee2..eb99ff0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "planetary-computer" authors = [{name = "microsoft", email = "planetarycomputer@microsoft.com"}] description = "Planetary Computer SDK for Python" -requires-python = ">=3.7" +requires-python = ">=3.9" dependencies = [ "click>=7.1", "pydantic>=1.7.3", From 0b806ad7ac63ed3bcaf7ad590c4c49f65994cffc Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Tue, 17 Jun 2025 15:10:11 -0400 Subject: [PATCH 6/7] lint --- planetary_computer/_obstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetary_computer/_obstore.py b/planetary_computer/_obstore.py index a719a41..2dc5958 100644 --- a/planetary_computer/_obstore.py +++ b/planetary_computer/_obstore.py @@ -31,7 +31,7 @@ def get_obstore( # type: ignore[misc] # overlap with kwargs client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, credential_provider: AzureCredentialProvider | None = None, - **kwargs: Unpack[AzureConfig], # type: ignore # noqa: PGH003 (container_name key overlaps with positional arg) + **kwargs: Unpack[AzureConfig], # type: ignore # noqa: PGH003 (container_name key overlaps with positional arg) # noqa: E501 ) -> AzureStore: try: import obstore From 59dea6ff0fcf4fb801f57de87987e9f2695d66ec Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Tue, 17 Jun 2025 16:04:48 -0400 Subject: [PATCH 7/7] try again to fix lint --- planetary_computer/_obstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetary_computer/_obstore.py b/planetary_computer/_obstore.py index 2dc5958..d0d9613 100644 --- a/planetary_computer/_obstore.py +++ b/planetary_computer/_obstore.py @@ -31,7 +31,7 @@ def get_obstore( # type: ignore[misc] # overlap with kwargs client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, credential_provider: AzureCredentialProvider | None = None, - **kwargs: Unpack[AzureConfig], # type: ignore # noqa: PGH003 (container_name key overlaps with positional arg) # noqa: E501 + **kwargs: Unpack[AzureConfig], # type: ignore # noqa ) -> AzureStore: try: import obstore