From b70a6c19cca6f78ed3b13234dcd3402ee4bf8948 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 8 Jun 2022 13:10:41 +0200 Subject: [PATCH] Relax glean_parser version requirement to "all compatible releases" In pip `~=V.N` is equivalent to `>= V.N, == V.*` So we essentially allow everything from the specified minor version upwards until the next major version (excluded). As long as glean_parser is careful to not introduce breaking changes before that this will simplify work a lot by allowing small bug fixes without requiring subsequent Glean releases to bump versions all the time. This also relaxed the version check in the Glean Gradle Plugin. We always upgrade if in online mode. We allow semver-compatible versions in offline mode (we do the semver check ourselves to avoid more dependencies). --- CHANGELOG.md | 2 + Makefile | 2 +- bin/update-glean-parser-version.sh | 9 +-- glean-core/ios/sdk_generator.sh | 4 +- glean-core/python/glean/__init__.py | 8 ++- glean-core/python/requirements_dev.txt | 1 + glean-core/python/setup.py | 3 +- .../GleanGradlePlugin.groovy | 62 +++++++++++++------ 8 files changed, 61 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f3e73d8b..9ee47bb568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ [Full changelog](https://github.com/mozilla/glean/compare/v51.1.0...main) +* General + * Relax `glean_parser` version requirement. All "compatible releases" are now allowed ([#2086](https://github.com/mozilla/glean/pull/2086)) * Kotlin * BUGFIX: Re-enable correctly connecting `glean.validation.foreground_count` again ([#2153](https://github.com/mozilla/glean/pull/2153)) diff --git a/Makefile b/Makefile index ce645ae6e4..a405ec96a7 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ python-docs: build-python ## Build the Python documentation .PHONY: docs rust-docs swift-docs metrics-docs: python-setup ## Build the internal metrics documentation - $(GLEAN_PYENV)/bin/pip install glean_parser==6.1.2 + $(GLEAN_PYENV)/bin/pip install glean_parser~=6.1 $(GLEAN_PYENV)/bin/glean_parser translate --allow-reserved \ -f markdown \ -o ./docs/user/user/collected-metrics \ diff --git a/bin/update-glean-parser-version.sh b/bin/update-glean-parser-version.sh index a32fc01639..eb016eb80b 100755 --- a/bin/update-glean-parser-version.sh +++ b/bin/update-glean-parser-version.sh @@ -27,18 +27,19 @@ if [ -z "$1" ]; then fi NEW_VERSION="$1" +NEW_VERSION_MAJOR_MINOR="$(echo "$NEW_VERSION" | awk -F'.' '{print $1"."$2}')" # Update the version in glean-core/ios/sdk_generator.sh FILE=glean-core/ios/sdk_generator.sh run $SED -i.bak -E \ - -e "s/^GLEAN_PARSER_VERSION=[0-9.]+/GLEAN_PARSER_VERSION=${NEW_VERSION}/" \ + -e "s/^GLEAN_PARSER_VERSION=[0-9.]+/GLEAN_PARSER_VERSION=${NEW_VERSION_MAJOR_MINOR}/" \ "${WORKSPACE_ROOT}/${FILE}" run rm "${WORKSPACE_ROOT}/${FILE}.bak" # Update the version in glean-core/python/setup.py FILE=glean-core/python/setup.py run $SED -i.bak -E \ - -e "s/\"glean_parser==[0-9.]+\"/\"glean_parser==${NEW_VERSION}\"/" \ + -e "s/\"glean_parser~=[0-9.]+\"/\"glean_parser~=${NEW_VERSION_MAJOR_MINOR}\"/" \ "${WORKSPACE_ROOT}/${FILE}" run rm "${WORKSPACE_ROOT}/${FILE}.bak" @@ -52,7 +53,7 @@ run rm "${WORKSPACE_ROOT}/${FILE}.bak" # update the version in gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy FILE=gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy run $SED -i.bak -E \ - -e "s/GLEAN_PARSER_VERSION = \"[0-9.]+\"/GLEAN_PARSER_VERSION = \"${NEW_VERSION}\"/" \ + -e "s/GLEAN_PARSER_VERSION = \"[0-9.]+\"/GLEAN_PARSER_VERSION = \"${NEW_VERSION_MAJOR_MINOR}\"/" \ "${WORKSPACE_ROOT}/${FILE}" run rm "${WORKSPACE_ROOT}/${FILE}.bak" @@ -80,6 +81,6 @@ run rm "${WORKSPACE_ROOT}/${FILE}.bak" # update the version in the Makefile FILE=Makefile run $SED -i.bak -E \ - -e "s/glean_parser==[0-9.]+/glean_parser==${NEW_VERSION}/" \ + -e "s/glean_parser~=[0-9.]+/glean_parser~=${NEW_VERSION_MAJOR_MINOR}/" \ "${WORKSPACE_ROOT}/${FILE}" run rm "${WORKSPACE_ROOT}/${FILE}.bak" diff --git a/glean-core/ios/sdk_generator.sh b/glean-core/ios/sdk_generator.sh index 698675d08f..840c528f5d 100755 --- a/glean-core/ios/sdk_generator.sh +++ b/glean-core/ios/sdk_generator.sh @@ -25,7 +25,7 @@ set -e -GLEAN_PARSER_VERSION=6.1.2 +GLEAN_PARSER_VERSION=6.1 # CMDNAME is used in the usage text below. # shellcheck disable=SC2034 @@ -159,7 +159,7 @@ VENVDIR="${SOURCE_ROOT}/.venv" # We need at least pip 20.3 for Big Sur support, see https://pip.pypa.io/en/stable/news/#id48 # Latest pip is 21.0.1 "${VENVDIR}"/bin/pip install "pip>=20.3" -"${VENVDIR}"/bin/pip install --upgrade glean_parser==$GLEAN_PARSER_VERSION +"${VENVDIR}"/bin/pip install --upgrade "glean_parser~=$GLEAN_PARSER_VERSION" # Run the glinter # Turn its warnings into warnings visible in Xcode (but don't do for the success message) diff --git a/glean-core/python/glean/__init__.py b/glean-core/python/glean/__init__.py index 54b05f6d20..3e73655d4a 100644 --- a/glean-core/python/glean/__init__.py +++ b/glean-core/python/glean/__init__.py @@ -9,6 +9,7 @@ from pkg_resources import get_distribution, DistributionNotFound +from semver import VersionInfo # type: ignore import glean_parser # type: ignore @@ -31,11 +32,14 @@ GLEAN_PARSER_VERSION = "6.1.2" +parser_version = VersionInfo.parse(GLEAN_PARSER_VERSION) +parser_version_next_major = parser_version.bump_major() -if glean_parser.__version__ != GLEAN_PARSER_VERSION: +current_parser = VersionInfo.parse(glean_parser.__version__) +if current_parser < parser_version or current_parser >= parser_version_next_major: warnings.warn( - f"glean_sdk expected glean_parser v{GLEAN_PARSER_VERSION}, " + f"glean_sdk expected glean_parser ~= v{GLEAN_PARSER_VERSION}, " f"found v{glean_parser.__version__}", Warning, ) diff --git a/glean-core/python/requirements_dev.txt b/glean-core/python/requirements_dev.txt index 92c202eebc..a46f026fb0 100644 --- a/glean-core/python/requirements_dev.txt +++ b/glean-core/python/requirements_dev.txt @@ -10,6 +10,7 @@ pip pytest-localserver==0.5.1 pytest-runner==5.3.1 pytest==7.0.1 +semver==2.13.0 setuptools-git==1.2 twine==3.8.0 types-pkg_resources==0.1.3 diff --git a/glean-core/python/setup.py b/glean-core/python/setup.py index 0e4c3b342b..5bd295966b 100644 --- a/glean-core/python/setup.py +++ b/glean-core/python/setup.py @@ -59,7 +59,8 @@ version = "51.1.0" requirements = [ - "glean_parser==6.1.2", + "semver>=2.13.0", + "glean_parser~=6.1", ] # The environment variable `GLEAN_BUILD_VARIANT` can be set to `debug` or `release` diff --git a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy index 6d3a596b21..0cf0fcec33 100644 --- a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy +++ b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy @@ -41,7 +41,7 @@ class GleanMetricsYamlTransform extends ArtifactTransform { @SuppressWarnings("GrPackage") class GleanPlugin implements Plugin { // The version of glean_parser to install from PyPI. - private String GLEAN_PARSER_VERSION = "6.1.2" + private String GLEAN_PARSER_VERSION = "6.1" // The version of Miniconda is explicitly specified. // Miniconda3-4.5.12 is known to not work on Windows. private String MINICONDA_VERSION = "4.5.11" @@ -71,25 +71,47 @@ except ImportError: else: found_version = getattr(module, '__version__') -if found_version != expected_version: - if not offline: - if 'git' in expected_version: - target=expected_version - else: - target=f'{module_name}=={expected_version}' - - subprocess.check_call([ - sys.executable, - '-m', - 'pip', - 'install', - '--upgrade', - target - ]) +if not offline: + # When running in online mode, we always install. + # If it is installed this is essentially a no-op, + # otherwise it installs/upgrades. + if 'git' in expected_version: + target=expected_version else: - print(f'Using Python environment at {sys.executable},') - print(f'expected glean_parser version {expected_version}, found {found_version}.') + target=f'{module_name}~={expected_version}' + + subprocess.check_call([ + sys.executable, + '-m', + 'pip', + 'install', + '--upgrade', + target + ]) +else: + error_text = f''' + Using Python environment at {sys.executable}, + expected glean_parser version ~={expected_version}, found {found_version}. + Please remove the Python environment, then prepare the package wheels for use: + + mkdir -p glean-wheels + cd glean-wheels + pip download glean_parser~={expected_version} + ''' + + if found_version is None: + print(error_text) sys.exit(1) + else: + # We check MAJOR.MINOR only + expected_ver = expected_version.split(".") + expected_maj, expected_min = int(expected_ver[0]), int(expected_ver[1]) + current_ver = found_version.split(".") + current_maj, current_min = int(current_ver[0]), int(current_ver[1]) + + if current_maj > expected_maj or current_maj < expected_maj or (current_maj == expected_maj and current_min < expected_min): + print(error_text) + sys.exit(1) try: subprocess.check_call([ sys.executable, @@ -384,7 +406,7 @@ except: } project.logger.warn("Building in offline mode, therefore, Glean is using a supplied Python at ${pythonBinary}") - project.logger.warn("The Python binary can be overridden GLEAN_PYTHON env var.") + project.logger.warn("The Python binary can be overridden with the GLEAN_PYTHON env var.") commandLine pythonBinary args "-m" @@ -458,7 +480,7 @@ except: if (parserVersion.matches("git.+")) { conda "Miniconda3", "Miniconda3-${MINICONDA_VERSION}", "64", [parserVersion] } else { - conda "Miniconda3", "Miniconda3-${MINICONDA_VERSION}", "64", ["glean_parser==${parserVersion}"] + conda "Miniconda3", "Miniconda3-${MINICONDA_VERSION}", "64", ["glean_parser~=${parserVersion}"] } } File envDir = new File(