From ae50862dc47d3fac103754d1da6d978af118d847 Mon Sep 17 00:00:00 2001 From: mostaphaRoudsari Date: Tue, 22 Oct 2024 15:57:31 -0400 Subject: [PATCH 1/3] feat: support Python 3.12 --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/tests.yaml | 2 +- dev-requirements.txt | 12 ++++++------ pollination_dsl/common.py | 36 ++++++++++++++++++++++++++++-------- requirements.txt | 2 +- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a45622c..00e12c8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ jobs: - name: set up Python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.12" - name: set up node # we need node for for semantic release uses: actions/setup-node@v2.1.2 with: @@ -44,7 +44,7 @@ jobs: - name: set up Python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.12" - name: install dependencies run: | pip install -U . diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1dbfc6b..771f942 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -17,7 +17,7 @@ jobs: - name: set up Python uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.12" - name: install python dependencies run: | python -m pip install --upgrade pip diff --git a/dev-requirements.txt b/dev-requirements.txt index 38e5d76..fb83259 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,10 @@ -pytest==6.2.4 -Sphinx==5.3.0 +pytest==8.3.2 +Sphinx==8.0.2 sphinx-bootstrap-theme==0.8.1 sphinxcontrib-fulltoc==1.2.0 -sphinxcontrib-websupport==1.2.4 -twine==3.4.1 -wheel==0.38.1 -setuptools==65.5.1 +sphinxcontrib-websupport==2.0.0 +twine==5.1.1 +wheel==0.44.0 +setuptools==75.1.0 jinja2==3.0.3 markupsafe==2.0.1 diff --git a/pollination_dsl/common.py b/pollination_dsl/common.py index 822ea53..9c5fd1f 100644 --- a/pollination_dsl/common.py +++ b/pollination_dsl/common.py @@ -94,6 +94,7 @@ def get_requirement_version(package_name, dependency_name): # try to get it from meta data package_data = importlib_metadata.metadata(package_name) req_dists: List[str] = package_data.get_all('Requires-Dist') or [] + version = -1 for package in req_dists: try: if '(' in package: @@ -101,12 +102,28 @@ def get_requirement_version(package_name, dependency_name): else: name, version = package.strip().split() except ValueError as e: - print( - f'Failed to parse the dependency version for {dependency_name} ' - f'from {package}. The version will not be set:\n{str(e)}' - ) - requirements[dependency_name] = '' - else: + # Python 3.12 + if 'not enough values to unpack' in str(e): + if '=' in package: + name, version = package.strip().split('=') + elif '>' in package: + name, version = package.strip().split('>') + elif '<' in package: + name, version = package.strip().split('<') + else: + print( + f'Failed to parse the dependency version for {dependency_name} ' + f'from {package}. The version will not be set:\n{str(e)}' + ) + requirements[dependency_name] = '' + else: + print( + f'Failed to parse the dependency version for {dependency_name} ' + f'from {package}. The version will not be set:\n{str(e)}' + ) + requirements[dependency_name] = '' + + if version != -1: version = \ version.replace('=', '').replace('>', '').replace('<', '') \ .replace(')', '').strip() @@ -146,7 +163,10 @@ def get_docker_image_from_dependency(package, dependency, owner, alias=None): image_name = alias if alias else dependency try: image_version = get_requirement_version(package, dependency) - image_id = f'{owner}/{image_name}:{image_version}' + if not image_version: + image_id = f'{owner}/{image_name}' + else: + image_id = f'{owner}/{image_name}:{image_version}' except (FileNotFoundError, AssertionError) as error: # this should not happen if the package is installed correctly # but Python has so many ways to store requirements based on how the package @@ -155,7 +175,7 @@ def get_docker_image_from_dependency(package, dependency, owner, alias=None): f'Failed to pinpoint the version for {dependency} as a dependency for' f' {package}. Will set the docker version to latest.\n{error}' ) - image_id = f'{owner}/{image_name}:latest' + image_id = f'{owner}/{image_name}' return image_id diff --git a/requirements.txt b/requirements.txt index a6f0a67..48608ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -queenbee-pollination==0.7.5 +queenbee-pollination==0.7.10 queenbee-local>=0.6.0 importlib-metadata>=6.6.0 From 64361c4633aa295fa7c345b863d8513a61441f72 Mon Sep 17 00:00:00 2001 From: mostaphaRoudsari Date: Tue, 22 Oct 2024 15:58:40 -0400 Subject: [PATCH 2/3] fix: switch to pollination.solutions --- pollination_dsl/cli.py | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pollination_dsl/cli.py b/pollination_dsl/cli.py index 55923cd..5742d1f 100644 --- a/pollination_dsl/cli.py +++ b/pollination_dsl/cli.py @@ -78,7 +78,7 @@ def translate_recipe(ctx, recipe_name, target_folder, queenbee, no_exit=False): from queenbee_luigi.recipe import Recipe except ImportError: # try pollination endpoint - url = 'https://utilities.pollination.cloud/to-luigi-archive' + url = 'https://utilities.pollination.solutions/to-luigi-archive' token = os.getenv('QB_POLLINATION_TOKEN') assert token is not None, \ 'Pollination token is not set. Use QB_POLLINATION_TOKEN to set it as ' \ @@ -116,14 +116,14 @@ def translate_recipe(ctx, recipe_name, target_folder, queenbee, no_exit=False): @click.argument('package_name') @click.option( '-e', '--endpoint', help='Endpoint to push the resource.', show_default=True, - default='https://api.pollination.cloud' + default='https://api.pollination.solutions' ) # TODO: Add better support for mapping different dependencies to different sources. For # now it is all set to the same value which is fine for our use cases. @click.option( '-src', '--source', help='A link to replace the source for dependencies. This value ' 'will overwrite the source value in recipe\'s dependencies files. By default it ' - 'will be set to https://api.pollination.cloud/registries' + 'will be set to https://api.pollination.solutions/registries' ) @click.option( '--public/--private', help='Indicate if the recipe or plugin should be created as ' @@ -206,7 +206,7 @@ def push_resource( # overwite resources in dependencies if resource_type == 'recipe': - source = source or 'https://api.pollination.cloud/registries' + source = source or 'https://api.pollination.solutions/registries' # update the value for source in dependencies.yaml file dep_file = pathlib.Path(folder, 'dependencies.yaml') data = yaml.safe_load(dep_file.read_bytes()) diff --git a/setup.py b/setup.py index eddcb4c..5edbb85 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ use_scm_version=True, setup_requires=['setuptools_scm'], author="Pollination", - author_email="info@pollination.cloud", + author_email="info@pollination.solutions", description="A Python DSL to create Pollination recipes and plugins.", long_description=long_description, long_description_content_type="text/markdown", From 385224937fa7229dd0df23771c78e0a049cc097b Mon Sep 17 00:00:00 2001 From: mostaphaRoudsari Date: Tue, 22 Oct 2024 16:03:41 -0400 Subject: [PATCH 3/3] deps: bump Jinja2 to 3.1 to be compatible with Sphinx 8.0.2 --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fb83259..26de85a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,5 +6,5 @@ sphinxcontrib-websupport==2.0.0 twine==5.1.1 wheel==0.44.0 setuptools==75.1.0 -jinja2==3.0.3 +jinja2==3.1 markupsafe==2.0.1