From d3080add06689edba4468fdd936a3710a72b5833 Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 17:47:03 +0200 Subject: [PATCH 1/8] wire namespace cloud. --- .github/workflows/pants.yaml | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 9b8cab4..329437e 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -5,25 +5,22 @@ name: Pants -on: [push, pull_request] +on: [push, workflow_dispatch] jobs: - org-check: - name: Check GitHub Organization - if: ${{ github.repository_owner == 'pantsbuild' }} - runs-on: ubuntu-24.04 - steps: - - name: Noop - run: "true" build: name: Perform CI Checks - needs: org-check - runs-on: ubuntu-24.04 + runs-on: + # Replace with your profile label https://namespace.so/docs/features/faster-github-actions#using-runner-profiles + - namespace-profile-default strategy: matrix: python-version: [3.9] steps: - uses: actions/checkout@v4 + - name: Set up remote cache + run: | + nsc pants cache setup --pants-toml=/tmp/pants.toml - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -31,22 +28,13 @@ jobs: # This action bootstraps pants and manages 2-3 GHA caches. # See: github.com/pantsbuild/actions/tree/main/init-pants/ with: - # v0 makes it easy to bust the cache if needed - # just increase the integer to start with a fresh cache gha-cache-key: v0 - # The Python backend uses named_caches for Pip/PEX state, - # so it is appropriate to invalidate on lockfile changes. named-caches-hash: ${{ hashFiles('python-default.lock') }} - # If you're not using a fine-grained remote caching service (see https://www.pantsbuild.org/docs/remote-caching), - # then you may also want to preserve the local Pants cache (lmdb_store). However this must invalidate for - # changes to any file that can affect the build, so may not be practical in larger repos. - # A remote cache service integrates with Pants's fine-grained invalidation and avoids these problems. - cache-lmdb-store: 'true' # defaults to 'false' - # Note that named_caches and lmdb_store falls back to partial restore keys which - # may give a useful partial result that will save time over completely clean state, - # but will cause the cache entry to grow without bound over time. - # See https://www.pantsbuild.org/stable/docs/using-pants/using-pants-in-ci for tips on how to periodically clean it up. - # Alternatively you change gha-cache-key to ignore old caches. + pants-ci-config: | + pants.toml + pants.ci.toml + /tmp/pants.toml + - name: Check BUILD files run: | pants tailor --check update-build-files --check :: From 1485dc47bb07742eee02d5b447ba00fd87b27bc0 Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 17:49:08 +0200 Subject: [PATCH 2/8] remove ci pants --- .github/workflows/pants.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 329437e..5e954a2 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -32,7 +32,6 @@ jobs: named-caches-hash: ${{ hashFiles('python-default.lock') }} pants-ci-config: | pants.toml - pants.ci.toml /tmp/pants.toml - name: Check BUILD files From a4b9ddb72843211928d5161fcb86f9ec776f912c Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 17:50:37 +0200 Subject: [PATCH 3/8] Try array syntax --- .github/workflows/pants.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 5e954a2..edcbb29 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -30,9 +30,7 @@ jobs: with: gha-cache-key: v0 named-caches-hash: ${{ hashFiles('python-default.lock') }} - pants-ci-config: | - pants.toml - /tmp/pants.toml + pants-ci-config: "['pants.toml', '/tmp/pants.toml']" - name: Check BUILD files run: | From 059170efe56840b5589665d08ff5d33196a60d41 Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 17:58:09 +0200 Subject: [PATCH 4/8] Update README. --- README.md | 189 ++---------------------------------------------------- 1 file changed, 4 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index 3cc3c1d..253b022 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,7 @@ -# example-python -An example repository to demonstrate Python support in Pants. +# pantbuild-example-python -See [pantsbuild.org](https://www.pantsbuild.org/docs) for much more detailed documentation. - -This is only one possible way of laying out your project with Pants. See -[pantsbuild.org/docs/source-roots#examples](https://www.pantsbuild.org/docs/source-roots#examples) for some other -example layouts. - -# Running Pants - -You run Pants goals using the `pants` launcher binary, which will bootstrap the -version of Pants configured for this repo if necessary. - -See [here](https://www.pantsbuild.org/docs/installation) for how to install the `pants` binary. - -> :question: Running with Apple Silicon and/or macOS? You will want to make changes to the `search_path` and -`interpreter_constraints` values in `pants.toml` before running `pants` - there is guidance in `pants.toml` -for those settings. - -Use `pants --version` to see the version of Pants configured for the repo (which you can also find -in `pants.toml`). - -# Goals - -Pants commands are called _goals_. You can get a list of goals with - -``` -pants help goals -``` - -# Targets - -Targets are a way of setting metadata for some part of your code, such as timeouts for tests and -entry points for binaries. Targets have types like `python_source`, `resources`, and -`pex_binary`. They are defined in `BUILD` files. - -Pants goals can be invoked on targets or directly on source files (which is often more intuitive and convenient). -In the latter case, Pants locates target metadata for the source files as needed. - -## File specifications - -Invoking goals on files is straightforward, e.g., - -``` -pants test helloworld/greet/greeting_test.py -``` - -You can use globs: - -``` -pants lint helloworld/greet/*.py -``` - -But note that these will be expanded by your shell, so this is equivalent to having used - -``` -pants lint helloworld/greet/__init__.py helloworld/greet/greeting.py helloworld/greet/greeting_test.py -``` - -If you want Pants itself to expand the globs (which is sometimes necessary), you must quote them in the shell: - -``` -pants lint 'helloworld/greet/*.py' -``` - -You can run on all changed files: - -``` -pants --changed-since=HEAD lint -``` - -You can run on all changed files, and any of their "dependents": - -``` -pants --changed-since=HEAD --changed-dependents=transitive test -``` - -## Target specifications - -Targets are referenced on the command line using their address, of the form `path/to/dir:name`, e.g., - -``` -pants lint helloworld/greet:lib -``` - -You can glob over all targets in a directory with a single trailing `:`, or over all targets in a directory -and all its subdirectories with a double trailing `::`, e.g., - -``` -pants lint helloworld:: -``` +An example repository to demonstrate Python support in Pants with remote caching in Namespace. -## Globbing semantics - -When you glob over files or targets, Pants knows to ignore ones that aren't relevant to the requested goal. -For example, if you run the `test` goal over a set of files that includes non-test files, Pants will just ignore -those, rather than error. So you can safely do things like - -``` -pants test :: -``` - -To run all tests. - -# Example Goals - -Try these out in this repo! - -## List targets - -``` -pants list :: # All targets. -pants list 'helloworld/**/*.py' # Just targets containing Python code. -``` - -## Run linters and formatters - -``` -pants lint :: -pants fmt helloworld/greet:: -``` - -## Run MyPy - -``` -pants check :: -``` - -## Run tests - -``` -pants test :: # Run all tests in the repo. -pants test --output=all :: # Run all tests in the repo and view pytest output even for tests that passed (you can set this permanently in pants.toml). -pants test helloworld/translator:tests # Run all the tests in this target. -pants test helloworld/translator/translator_test.py # Run just the tests in this file. -pants test helloworld/translator/translator_test.py -- -k test_unknown_phrase # Run just this one test by passing through pytest args. -``` - -## Create a PEX binary - -The `package` goal requires specifying a target which can be packaged. In this case, the there is a `pex_binary` target with the name `pex_binary` in the `helloworld/BUILD` file. - -``` -pants package helloworld:pex_binary -``` - -The pex file is output to `dist/helloworld/pex_binary.pex` and can be executed directly. - -## Run a binary directly - -``` -pants run helloworld/main.py -``` - -## Open a REPL - -``` -pants repl helloworld/greet:lib # The REPL will have all relevant code and dependencies on its sys.path. -pants repl --shell=ipython helloworld/greet:lib --no-pantsd # To use IPython, you must disable Pantsd for now. -``` - -## Build a wheel / generate `setup.py` - -This will build both a `.whl` bdist and a `.tar.gz` sdist. - -``` -pants package helloworld/translator:dist -``` - -## Count lines of code - -``` -pants count-loc '**/*' -``` - -## Generate or update a lockfile containing the dependencies - -``` -pants generate-lockfiles --resolve=python-default -``` - - -## Create virtualenv for IDE integration +See [pantsbuild.org](https://www.pantsbuild.org/docs) for much more detailed documentation. -``` -pants export --resolve=python-default -``` +This example is based on [pantsbuild/example-python](https://github.com/pantsbuild/example-python). From 159a3aec4240b887e38f0810268e3d4a506f1507 Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 18:02:23 +0200 Subject: [PATCH 5/8] also run on arm --- .github/workflows/pants.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index edcbb29..17127c5 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -13,6 +13,7 @@ jobs: runs-on: # Replace with your profile label https://namespace.so/docs/features/faster-github-actions#using-runner-profiles - namespace-profile-default + - namespace-profile-default-arm64 strategy: matrix: python-version: [3.9] From 151896ff90cc58eceacd2f6462fd4ee97329b11a Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 18:04:39 +0200 Subject: [PATCH 6/8] Arm runners from matrix --- .github/workflows/pants.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 17127c5..2e9129a 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -10,13 +10,15 @@ on: [push, workflow_dispatch] jobs: build: name: Perform CI Checks - runs-on: - # Replace with your profile label https://namespace.so/docs/features/faster-github-actions#using-runner-profiles - - namespace-profile-default - - namespace-profile-default-arm64 strategy: matrix: + runner: + # Replace with your profile label https://namespace.so/docs/features/faster-github-actions#using-runner-profiles + - namespace-profile-default + - namespace-profile-default-arm64 python-version: [3.9] + fail-fast: false + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 - name: Set up remote cache From c86b21d7ee75ed13d872f6a24b426f9247f9409c Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Mon, 26 May 2025 18:13:03 +0200 Subject: [PATCH 7/8] split artifact logs per runner --- .github/workflows/pants.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 2e9129a..b96cab0 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -50,8 +50,8 @@ jobs: pants package :: pants run helloworld/:pex_binary - name: Upload pants log - uses: actions/upload-artifact@v4 + uses: namespace-actions/upload-artifact@v1 with: - name: pants-log + name: pants-log-${{ matrix.runner }} path: .pants.d/workdir/pants.log if: always() # We want the log even on failures. From b2fc7c00eb9e7aae284a65f41455b4ef2aedf05a Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Tue, 27 May 2025 10:04:19 +0200 Subject: [PATCH 8/8] Set up arm and amd examples. --- .github/workflows/pants.yaml | 46 +++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index b96cab0..234b034 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -14,26 +14,49 @@ jobs: matrix: runner: # Replace with your profile label https://namespace.so/docs/features/faster-github-actions#using-runner-profiles - - namespace-profile-default - - namespace-profile-default-arm64 + - namespace-profile-amd64-with-caching + - namespace-profile-arm64-with-caching python-version: [3.9] + setup: + - manual + - init-pants-action fail-fast: false runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 + - name: Set up remote cache run: | - nsc pants cache setup --pants-toml=/tmp/pants.toml + nsc pants cache setup --pants-toml=pants.cache.toml + + - name: Set up local cache + uses: namespacelabs/nscloud-cache-action@v1 + with: + path: | + ~/.cache/pants/named_caches + ~/.cache/pants/lmdb_store + ~/.cache/nce + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + + - name: Limit LMDB memory usage + if: runner.arch == 'ARM64' + run: echo "PANTS_LOCAL_STORE_PROCESSES_MAX_SIZE_BYTES=536870912" >> $GITHUB_ENV + - uses: pantsbuild/actions/init-pants@v8 - # This action bootstraps pants and manages 2-3 GHA caches. - # See: github.com/pantsbuild/actions/tree/main/init-pants/ + if: matrix.setup == 'init-pants-action' with: gha-cache-key: v0 named-caches-hash: ${{ hashFiles('python-default.lock') }} - pants-ci-config: "['pants.toml', '/tmp/pants.toml']" + pants-ci-config: "['pants.toml', 'pants.ci.toml', 'pants.cache.toml']" + + - name: Install Pants manually + if: matrix.setup == 'manual' + run: | + curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash + echo "PANTS_CONFIG_FILES=['pants.toml','pants.ci.toml','pants.cache.toml']" >> $GITHUB_ENV - name: Check BUILD files run: | @@ -49,9 +72,10 @@ jobs: # We also smoke test that our release process will work by running `package`. pants package :: pants run helloworld/:pex_binary - - name: Upload pants log - uses: namespace-actions/upload-artifact@v1 + + - name: Breakpoint + uses: namespacelabs/breakpoint-action@v0 + if: failure() with: - name: pants-log-${{ matrix.runner }} - path: .pants.d/workdir/pants.log - if: always() # We want the log even on failures. + duration: 15m + authorized-users: edganiukov,hugosantos,n-g,htr,nichtverstehen,gmichelo