From aa70623ec71283608a0c4a88fdcc04aad537bfbf Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Wed, 13 Mar 2024 13:51:56 +0100 Subject: [PATCH 1/4] Silence stdout/stderr during Pants setup. --- .github/workflows/ci.yml | 3 +- CHANGES.md | 5 +++ Cargo.lock | 2 +- Cargo.toml | 2 +- tools/src/scie_pants/install_pants.py | 47 +++++++++++++++++---------- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a7c4e6..67fe9bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,9 @@ name: CI on: push: + # Ignore non top-level branches. branches-ignore: - - dependabot/** + - '*/**' pull_request: defaults: run: diff --git a/CHANGES.md b/CHANGES.md index 5dc9672..ec380fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # Release Notes +## 0.10.8 + +Redirect pants install messages to a `pants-install.log` file in pants venv directory to not pollute +stdout/stderr. + ## 0.10.7 This release upgrades `pex` to `v2.1.163` and the bootstrap Python to `3.9.18`. diff --git a/Cargo.lock b/Cargo.lock index 8127adf..96774b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,7 +573,7 @@ dependencies = [ [[package]] name = "scie-pants" -version = "0.10.7" +version = "0.10.8" dependencies = [ "anyhow", "dirs", diff --git a/Cargo.toml b/Cargo.toml index f0ddbae..2af549a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "scie-pants" description = "Protects your Pants from the elements." -version = "0.10.7" +version = "0.10.8" edition = "2021" authors = [ "John Sirois ", diff --git a/tools/src/scie_pants/install_pants.py b/tools/src/scie_pants/install_pants.py index 4846b11..21f0b78 100644 --- a/tools/src/scie_pants/install_pants.py +++ b/tools/src/scie_pants/install_pants.py @@ -120,23 +120,36 @@ def install_pants_from_pex( " or file an issue on GitHub: https://github.com/pantsbuild/pants/issues/new/choose.\n\n" f"Exception:\n\n{e}" ) - subprocess.run( - args=[ - sys.executable, - pants_pex.name, - "venv", - "--prompt", - prompt, - "--compile", - "--pip", - "--collisions-ok", - "--no-emit-warnings", # Silence `PEXWarning: You asked for --pip ...` - "--disable-cache", - str(venv_dir), - ], - env={"PEX_TOOLS": "1"}, - check=True, - ) + try: + pants_venv_result = subprocess.run( + args=[ + sys.executable, + pants_pex.name, + "venv", + "--prompt", + prompt, + "--compile", + "--pip", + "--collisions-ok", + "--no-emit-warnings", # Silence `PEXWarning: You asked for --pip ...` + "--disable-cache", + str(venv_dir), + ], + env={"PEX_TOOLS": "1"}, + check=True, + capture_output=True, + ) + with open(str(venv_dir / "pants-install.log"), "a") as fp: + if pants_venv_result.stdout: + print(pants_venv_result.stdout, file=fp) + if pants_venv_result.stderr: + print(pants_venv_result.stderr, file=fp) + except subprocess.CalledProcessError as e: + fatal( + f"Failed to create Pants virtual environment.\n{e}\n\n" + f"STDOUT:\n{e.stdout}\n\n" + f"STDERR:\n{e.stderr}\n\n" + ) if extra_requirements: venv_pip_install( From 1bd66c300195588a254526feedf0265b80b32e5c Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Wed, 13 Mar 2024 14:32:57 +0100 Subject: [PATCH 2/4] Drop support for `PANTS_SHA`. --- CHANGES.md | 9 +++-- Cargo.lock | 2 +- Cargo.toml | 2 +- package/scie-pants.toml | 6 --- package/src/test.rs | 49 +++---------------------- src/main.rs | 34 ++--------------- tools/src/scie_pants/configure_pants.py | 13 +------ tools/src/scie_pants/pants_version.py | 34 +---------------- 8 files changed, 19 insertions(+), 130 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ec380fa..5d4aff4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Release Notes +## 0.11.0 + +Drop support for `PANTS_SHA` which was deprecated in `0.10.0`. + ## 0.10.8 Redirect pants install messages to a `pants-install.log` file in pants venv directory to not pollute @@ -50,9 +54,8 @@ It also supports fetching Pants PEXes from behind a firewall. ## 0.10.0 This release deprecates support for running against an arbitrary Pants commit using -`PANTS_SHA=abc123... pants ...`. Pants no longer -publishes the artifacts required for this for new commits, and so this is becoming less and less -useful. To replace use of `PANTS_SHA`, do one of: +`PANTS_SHA=abc123... pants ...`. Pants no longer publishes the artifacts required for this for new +commits, and so this is becoming less and less useful. To replace use of `PANTS_SHA`, do one of: - Use a released version of Pants. - Run pants from sources (for example: `PANTS_SOURCE=/path/to/pants-checkout pants ...`). diff --git a/Cargo.lock b/Cargo.lock index 96774b1..bea2671 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,7 +573,7 @@ dependencies = [ [[package]] name = "scie-pants" -version = "0.10.8" +version = "0.11.0" dependencies = [ "anyhow", "dirs", diff --git a/Cargo.toml b/Cargo.toml index 2af549a..cfd8d6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "scie-pants" description = "Protects your Pants from the elements." -version = "0.10.8" +version = "0.11.0" edition = "2021" authors = [ "John Sirois ", diff --git a/package/scie-pants.toml b/package/scie-pants.toml index e55aa38..ac6e856 100644 --- a/package/scie-pants.toml +++ b/package/scie-pants.toml @@ -56,9 +56,6 @@ name = "pants" # appropriate by the default "Boot" one above) # description = "Runs a hermetic Pants installation." exe = "{scie.bindings.install:PANTS_CLIENT_EXE}" -args = [ - "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}", -] [lift.commands.env.default] PANTS_BUILDROOT_OVERRIDE = "{scie.bindings.configure:PANTS_BUILDROOT_OVERRIDE}" @@ -90,7 +87,6 @@ __import__("debugpy.server.cli").server.cli.main() "127.0.0.1:5678", "--wait-for-client", "{scie.bindings.install:VIRTUAL_ENV}/bin/pants", - "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}", ] [lift.commands.env.default] @@ -183,8 +179,6 @@ args = [ "{ptex}", "--pants-version", "{scie.env.PANTS_VERSION}", - "--pants-sha", - "{scie.env.PANTS_SHA}", "--pants-config", "{scie.env.PANTS_TOML}", "--github-api-bearer-token", diff --git a/package/src/test.rs b/package/src/test.rs index 08c4f51..da18342 100644 --- a/package/src/test.rs +++ b/package/src/test.rs @@ -102,31 +102,13 @@ pub(crate) fn run_integration_tests( test_tools_pex_reproducibility(workspace_root, tools_pex_path, tools_pex_mismatch_warn); test_pants_bootstrap_tools(scie_pants_scie); - // TODO(John Sirois): The --no-pantsd here works around a fairly prevalent Pants crash on - // Linux x86_64 along the lines of the following, but sometimes varying: - // >> Verifying PANTS_SHA is respected - // Bootstrapping Pants 2.14.0a0+git8e381dbf using cpython 3.9.15 - // Installing pantsbuild.pants==2.14.0a0+git8e381dbf into a virtual environment at /home/runner/.cache/nce/67f27582b3729c677922eb30c5c6e210aa54badc854450e735ef41cf25ac747f/bindings/venvs/2.14.0a0+git8e381dbf - // New virtual environment successfully created at /home/runner/.cache/nce/67f27582b3729c677922eb30c5c6e210aa54badc854450e735ef41cf25ac747f/bindings/venvs/2.14.0a0+git8e381dbf. - // 18:11:53.75 [INFO] Initializing scheduler... - // 18:11:53.97 [INFO] Scheduler initialized. - // 2.14.0a0+git8e381dbf - // Fatal Python error: PyGILState_Release: thread state 0x7efe18001140 must be current when releasing - // Python runtime state: finalizing (tstate=0x1f4b810) - // - // Thread 0x00007efe30b75540 (most recent call first): - // - // Error: Command "/home/runner/work/scie-pants/scie-pants/dist/scie-pants-linux-x86_64" "--no-verify-config" "-V" failed with exit code: None - if matches!(*CURRENT_PLATFORM, Platform::LinuxX86_64) { - log!(Color::Yellow, "Turning off pantsd for remaining tests."); - env::set_var("PANTS_PANTSD", "False"); - } + log!(Color::Yellow, "Turning off pantsd for remaining tests."); + env::set_var("PANTS_PANTSD", "False"); - test_pants_shas(scie_pants_scie); test_python_repos_repos(scie_pants_scie); test_initialize_new_pants_project(scie_pants_scie); test_set_pants_version(scie_pants_scie); - test_ignore_empty_pants_version_pants_sha(scie_pants_scie); + test_ignore_empty_pants_version(scie_pants_scie); test_pants_from_pex_version(scie_pants_scie); test_pants_from_bad_pex_version(scie_pants_scie); @@ -341,26 +323,6 @@ fn test_pants_bootstrap_tools(scie_pants_scie: &Path) { .unwrap(); } -fn test_pants_shas(scie_pants_scie: &Path) { - for sha in [ - // initial - "8e381dbf90cae57c5da2b223c577b36ca86cace9", - // native-client added to wheel - "558d843549204bbe49c351d00cdf23402da262c1", - ] { - integration_test!("Verifying significant PANTS_SHA: {sha}"); - let existing_project_dir = create_tempdir().unwrap(); - touch(&existing_project_dir.path().join("pants.toml")).unwrap(); - execute( - Command::new(scie_pants_scie) - .current_dir(existing_project_dir.path()) - .env("PANTS_SHA", sha) - .args(["--no-verify-config", "-V"]), - ) - .unwrap(); - } -} - fn test_python_repos_repos(scie_pants_scie: &Path) { integration_test!( "Verifying --python-repos-repos is used prior to Pants 2.13 (no warnings should be \ @@ -403,8 +365,8 @@ fn test_set_pants_version(scie_pants_scie: &Path) { .unwrap(); } -fn test_ignore_empty_pants_version_pants_sha(scie_pants_scie: &Path) { - integration_test!("Verifying ignoring PANTS_SHA and PANTS_VERSION when set to empty string"); +fn test_ignore_empty_pants_version(scie_pants_scie: &Path) { + integration_test!("Verifying ignoring PANTS_VERSION when set to empty string"); let tmpdir = create_tempdir().unwrap(); @@ -421,7 +383,6 @@ fn test_ignore_empty_pants_version_pants_sha(scie_pants_scie: &Path) { let output = execute( Command::new(scie_pants_scie) .arg("-V") - .env("PANTS_SHA", "") .env("PANTS_VERSION", "") .current_dir(&tmpdir) .stdout(Stdio::piped()), diff --git a/src/main.rs b/src/main.rs index b75fecf..306a3f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ use std::ffi::{OsStr, OsString}; use std::fmt::Debug; use std::path::PathBuf; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use build_root::BuildRoot; use log::{info, trace}; use logging_timer::{time, timer, Level}; @@ -78,7 +78,7 @@ impl Process { fn env_version(env_var_name: &str) -> Result> { let raw_version = env::var_os(env_var_name).unwrap_or(OsString::new()); if raw_version.len() == 0 { - // setting PANTS_VERSION= or PANTS_SHA= behaves the same as not setting them + // setting PANTS_VERSION= behaves the same as not setting it Ok(None) } else { Ok(Some(raw_version.into_string().map_err(|raw| { @@ -171,37 +171,11 @@ fn get_pants_process() -> Result { (None, None, None, false) }; - let env_pants_sha = env_version("PANTS_SHA")?; let env_pants_version = env_version("PANTS_VERSION")?; - if let Some(pants_sha) = &env_pants_sha { - // when support for PANTS_SHA is fully removed, PANTS_SHA_FIND_LINKS can be removed too - eprintln!( - "\ -DEPRECATED: Support for PANTS_SHA=... will be removed in a future version of the `pants` launcher. - -The artifacts for PANTS_SHA are no longer published for new commits. This invocation set PANTS_SHA={pants_sha}. - -To resolve, do one of: -- Use a released version of Pants. -- Run pants from sources (for example: `PANTS_SOURCE=/path/to/pants-checkout pants ...`). -- If these are not appropriate, let us know what you're using it for: . -" - ); - - if let Some(pants_version) = &env_pants_version { - bail!( - "Both PANTS_SHA={pants_sha} and PANTS_VERSION={pants_version} were set. \ - Please choose one.", - ) - } - } - let pants_version = if let Some(env_version) = env_pants_version { Some(env_version) - } else if env_pants_sha.is_none() { - configured_pants_version.clone() } else { - None + configured_pants_version.clone() }; if delegate_bootstrap && pants_version.is_none() { @@ -264,7 +238,7 @@ To resolve, do one of: env.push(("_PANTS_VERSION_OVERRIDE".into(), version.clone().into())); } env.push(("PANTS_VERSION".into(), version.into())); - } else if env_pants_sha.is_none() { + } else { // Ensure the install binding always re-runs when no Pants version is found so that the // the user can be prompted with configuration options. env.push(( diff --git a/tools/src/scie_pants/configure_pants.py b/tools/src/scie_pants/configure_pants.py index cec0e5d..f8c87f8 100644 --- a/tools/src/scie_pants/configure_pants.py +++ b/tools/src/scie_pants/configure_pants.py @@ -16,7 +16,6 @@ from scie_pants.log import fatal, info, init_logging, warn from scie_pants.pants_version import ( determine_latest_stable_version, - determine_sha_version, determine_tag_version, ) from scie_pants.ptex import Ptex @@ -59,7 +58,6 @@ def prompt_for_pants_config() -> Path | None: def main() -> NoReturn: parser = ArgumentParser() get_ptex = Ptex.add_options(parser) - parser.add_argument("--pants-sha", help="The Pants sha to install (trumps --version)") parser.add_argument("--pants-version", help="The Pants version to install") parser.add_argument("--pants-config", help="The path of the pants.toml file") parser.add_argument( @@ -82,13 +80,7 @@ def main() -> NoReturn: finalizers = [] newly_created_build_root = None pants_config = Path(options.pants_config) if options.pants_config else None - if options.pants_sha: - resolve_info = determine_sha_version( - ptex=ptex, sha=options.pants_sha, find_links_dir=find_links_dir - ) - assert resolve_info.sha_version is not None - version = resolve_info.sha_version - elif options.pants_version: + if options.pants_version: resolve_info = determine_tag_version( ptex=ptex, pants_version=options.pants_version, @@ -125,9 +117,6 @@ def main() -> NoReturn: with open(env_file, "a") as fp: if resolve_info.find_links: print(f"FIND_LINKS={resolve_info.find_links}", file=fp) - # This can be removed once we stop supporting PANTS_SHA: - # NB. this is added unconditionally because it gets set as an argument - print(f"PANTS_SHA_FIND_LINKS={resolve_info.pants_find_links_option(version)}", file=fp) if newly_created_build_root: print(f"PANTS_BUILDROOT_OVERRIDE={newly_created_build_root}", file=fp) print(f"PANTS_VERSION={version}", file=fp) diff --git a/tools/src/scie_pants/pants_version.py b/tools/src/scie_pants/pants_version.py index be3ec68..29925bc 100644 --- a/tools/src/scie_pants/pants_version.py +++ b/tools/src/scie_pants/pants_version.py @@ -31,25 +31,8 @@ @dataclass(frozen=True) class ResolveInfo: stable_version: Version - sha_version: Version | None find_links: str | None - def pants_find_links_option(self, pants_version_selected: Version) -> str: - # We only want to add the find-links repo for PANTS_SHA invocations so that plugins can - # resolve Pants the only place it can be found in that case - our ~private - # binaries.pantsbuild.org S3 find-links bucket. - operator = "-" if pants_version_selected == self.stable_version else "+" - option_name = ( - "repos" - if self.stable_version in SpecifierSet("<2.14.0", prereleases=True) - else "find-links" - ) - value = f"'{self.find_links}'" if self.find_links else "" - - # we usually pass a no-op, e.g. --python-repos-find-links=-[], because this is only used for - # PANTS_SHA support that is now deprecated and will be removed - return f"--python-repos-{option_name}={operator}[{value}]" - def determine_find_links( ptex: Ptex, @@ -91,7 +74,6 @@ def determine_find_links( return ResolveInfo( stable_version=Version(pants_version), - sha_version=sha_version, find_links=f"file://{find_links_file}", ) @@ -101,7 +83,7 @@ def determine_tag_version( ) -> ResolveInfo: stable_version = Version(pants_version) if stable_version >= PANTS_PEX_GITHUB_RELEASE_VERSION: - return ResolveInfo(stable_version, sha_version=None, find_links=None) + return ResolveInfo(stable_version, find_links=None) tag = f"release_{pants_version}" @@ -149,20 +131,6 @@ def determine_tag_version( ) -def determine_sha_version(ptex: Ptex, sha: str, find_links_dir: Path) -> ResolveInfo: - version_file_url = ( - f"https://raw.githubusercontent.com/pantsbuild/pants/{sha}/src/python/pants/VERSION" - ) - pants_version = ptex.fetch_text(version_file_url).strip() - return determine_find_links( - ptex, - pants_version, - sha, - find_links_dir, - include_nonrelease_pants_distributions_in_findlinks=True, - ) - - def determine_latest_stable_version( ptex: Ptex, pants_config: Path, find_links_dir: Path, github_api_bearer_token: str | None = None ) -> tuple[Callable[[], None], ResolveInfo]: From 20665f1750e8866455ae293baf099cc1685aafa0 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Mon, 18 Mar 2024 11:07:35 +0100 Subject: [PATCH 3/4] Remove dead code. --- tools/src/scie_pants/pants_version.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/src/scie_pants/pants_version.py b/tools/src/scie_pants/pants_version.py index 29925bc..d297dcf 100644 --- a/tools/src/scie_pants/pants_version.py +++ b/tools/src/scie_pants/pants_version.py @@ -39,7 +39,6 @@ def determine_find_links( pants_version: str, sha: str, find_links_dir: Path, - include_nonrelease_pants_distributions_in_findlinks: bool, ) -> ResolveInfo: abbreviated_sha = sha[:8] sha_version = Version(f"{pants_version}+git{abbreviated_sha}") @@ -62,14 +61,6 @@ def determine_find_links( f"{os.linesep}".encode() ) fp.flush() - if include_nonrelease_pants_distributions_in_findlinks: - pantsbuild_pants_find_links = ( - "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/" - f"{sha}/{urllib.parse.quote(str(sha_version))}/index.html" - ) - ptex.fetch_to_fp(pantsbuild_pants_find_links, fp) - fp.flush() - ptex.fetch_to_fp("https://wheels.pantsbuild.org/simple/", fp) return ResolveInfo( @@ -127,7 +118,6 @@ def determine_tag_version( pants_version, commit_sha, find_links_dir, - include_nonrelease_pants_distributions_in_findlinks=False, ) From 0dd5bb4589442cf827ef4f02ab3126898a2901e9 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Mon, 18 Mar 2024 15:00:51 +0100 Subject: [PATCH 4/4] Purge pants setup of 2.19.1 each run. --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67fe9bc..64b502d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: run: cargo clippy --locked --all - name: Unit Tests run: cargo test --all - - name: Setup Python 3.9 + - name: Setup Python 3.9 (Ubuntu only) if: ${{ matrix.os == 'ubuntu-22.04' }} uses: actions/setup-python@v4 with: @@ -54,9 +54,12 @@ jobs: with: path: ${{ env.SCIE_PANTS_DEV_CACHE }} key: ${{ matrix.os }}-scie-pants-v6 - - name: Build, Package & Integration Tests + - name: Build, Package & Integration Tests (MacOS) if: ${{ matrix.os == 'macOS-10.15-X64' || matrix.os == 'macOS-11-ARM64'}} run: | + # Clean up the science cache for Pants 2.19.1 setup to ensure it's bootstrapped each run. + rm -rf ~/Library/Caches/nce/*/bindings/venv/2.19.1 + # TODO(John Sirois): Kill --tools-pex-mismatch-warn: # https://github.com/pantsbuild/scie-pants/issues/2 # @@ -69,7 +72,7 @@ jobs: # PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN=${{ secrets.GITHUB_TOKEN }} \ cargo run -p package -- test --check --tools-pex-mismatch-warn - - name: Build, Package & Integration Tests + - name: Build, Package & Integration Tests (Ubuntu) if: ${{ matrix.os == 'ubuntu-22.04' }} run: | cargo run -p package -- --dest-dir dist/ tools