Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
benjyw committed Oct 13, 2022
1 parent c2166c5 commit 81ee8f9
Show file tree
Hide file tree
Showing 4 changed files with 203 additions and 74 deletions.
113 changes: 65 additions & 48 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ jobs:
env:
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Bootstrap Pants, test and lint Rust (Linux-x86_64)
needs:
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -147,11 +147,11 @@ jobs:
env:
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Bootstrap Pants, test Rust (macOS11-x86_64)
needs:
- docs_only_check
- classify_changes
runs-on:
- macos-11
steps:
Expand Down Expand Up @@ -274,11 +274,11 @@ jobs:
env:
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Build wheels and fs_util (Linux-x86_64)
needs:
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -359,11 +359,11 @@ jobs:
env:
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Build wheels and fs_util (macOS10-15-x86_64)
needs:
- docs_only_check
- classify_changes
runs-on:
- macOS-10.15-X64
steps:
Expand Down Expand Up @@ -452,11 +452,11 @@ jobs:
ARCHFLAGS: -arch arm64
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Bootstrap Pants, build wheels and fs_util (macOS11-ARM64)
needs:
- docs_only_check
- classify_changes
runs-on:
- macOS-11-ARM64
steps:
Expand Down Expand Up @@ -582,11 +582,11 @@ jobs:
env:
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Build wheels and fs_util (macOS11-x86_64)
needs:
- docs_only_check
- classify_changes
runs-on:
- macos-11
steps:
Expand Down Expand Up @@ -688,11 +688,15 @@ jobs:
labels: category:new feature, category:user api change, category:plugin api
change, category:performance, category:bugfix, category:documentation, category:internal
mode: exactly
docs_only_check:
classify_changes:
if: github.repository_owner == 'pantsbuild'
name: Check for docs-only change
name: Classify changes
outputs:
docs_only: ${{ steps.docs_only_check.outputs.docs_only }}
ci_config: ${{ steps.classify.outputs.ci_config }}
docs: ${{ steps.classify.outputs.docs }}
docs_only: ${{ steps.classify.outputs.docs_only }}
release: ${{ steps.classify.outputs.release }}
rust: ${{ steps.classify.outputs.rust }}
runs-on:
- ubuntu-20.04
steps:
Expand All @@ -702,21 +706,34 @@ jobs:
fetch-depth: 10
- id: files
name: Get changed files
uses: tj-actions/changed-files@v32.0.0
uses: tj-actions/changed-files@v32
with:
files_ignore: docs/**|build-support/bin/generate_user_list.py
files_ignore_separator: '|'
- id: docs_only_check
if: steps.files.outputs.any_changed != 'true'
name: Check for docs-only changes
run: echo '::set-output name=docs_only::DOCS_ONLY'
separator: '|'
- id: classify
name: Classify changed files
run: " affected=$(python build-support/bin/classify_changed_files.py\
\ '${{ steps.files.outputs.all_modified_files }}')\n\
\ echo \"Affected:\n${affected}\"\n \
\ if [[ \"${affected}\" == \"docs\" ]]; then\n \
\ \"echo '::set-output name=docs_only::true'\"\n \
\ fi\n for i in ${affected}; do\n \
\ if [[ \"${i}\" == \"docs\" ]]; then\n \
\ \"echo '::set-output name=docs::true'\"\n \
\ fi\n if [[ \"${i}\" == \"rust\" ]]; then\n \
\ \"echo '::set-output name=rust::true'\"\n \
\ fi\n if [[ \"${i}\" == \"release\"\
\ ]]; then\n \"echo '::set-output name=release::true'\"\
\n fi\n if [[ \"${i}\" ==\
\ \"ci_config\" ]]; then\n \"echo '::set-output\
\ name=ci_config::true'\"\n fi\n \
\ done\n"
lint_python:
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Lint Python and Shell
needs:
- bootstrap_pants_linux_x86_64
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -793,10 +810,10 @@ jobs:
\ == \"true\" ]]; then\n echo \"Merge OK\"\n exit 0\nelse\n echo\
\ \"Merge NOT OK\"\n exit 1\nfi\n"
set_merge_ok_docs_only:
if: needs.docs_only_check.outputs.docs_only == 'DOCS_ONLY'
if: needs.classify_changes.outputs.docs_only == true
name: Set Merge OK
needs:
- docs_only_check
- classify_changes
- check_labels
outputs:
merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
Expand All @@ -806,10 +823,10 @@ jobs:
- id: set_merge_ok
run: echo '::set-output name=merge_ok::true'
set_merge_ok_not_docs_only:
if: needs.docs_only_check.outputs.docs_only != 'DOCS_ONLY'
if: needs.classify_changes.outputs.docs_only != true
name: Set Merge OK
needs:
- docs_only_check
- classify_changes
- check_labels
- bootstrap_pants_linux_x86_64
- bootstrap_pants_macos11_x86_64
Expand All @@ -818,7 +835,7 @@ jobs:
- build_wheels_macos11_arm64
- build_wheels_macos11_x86_64
- check_labels
- docs_only_check
- classify_changes
- lint_python
- test_python_linux_x86_64_0
- test_python_linux_x86_64_1
Expand All @@ -832,12 +849,12 @@ jobs:
- id: set_merge_ok
run: echo '::set-output name=merge_ok::true'
test_python_linux_x86_64_0:
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Test Python (Linux-x86_64) Shard 0/3
needs:
- bootstrap_pants_linux_x86_64
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -923,12 +940,12 @@ jobs:
- '3.7'
timeout-minutes: 90
test_python_linux_x86_64_1:
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Test Python (Linux-x86_64) Shard 1/3
needs:
- bootstrap_pants_linux_x86_64
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -1014,12 +1031,12 @@ jobs:
- '3.7'
timeout-minutes: 90
test_python_linux_x86_64_2:
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Test Python (Linux-x86_64) Shard 2/3
needs:
- bootstrap_pants_linux_x86_64
- docs_only_check
- classify_changes
runs-on:
- ubuntu-20.04
steps:
Expand Down Expand Up @@ -1107,12 +1124,12 @@ jobs:
test_python_macos11_x86_64:
env:
ARCHFLAGS: -arch x86_64
if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only
!= 'DOCS_ONLY')
if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only
!= true)
name: Test Python (macOS11-x86_64)
needs:
- bootstrap_pants_macos11_x86_64
- docs_only_check
- classify_changes
runs-on:
- macos-11
steps:
Expand Down
67 changes: 67 additions & 0 deletions build-support/bin/classify_changed_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import enum
import fnmatch
import sys
from collections import defaultdict

# This script may be run in CI before Pants is bootstrapped, so it must be kept simple
# and runnable without `./pants run`.


class Affected(enum.Enum):
docs = "docs"
rust = "rust"
release = "release"
ci_config = "ci_config"
other = "other"


_docs_globs = ["docs/*", "build-support/bin/generate_user_list.py"]
_rust_globs = ["src/rust/engine/*", "rust-toolchain", "build-support/bin/rust/*"]
_release_globs = [
"pants.toml",
"src/python/pants/VERSION",
"src/python/pants/notes/*",
"src/python/pants/init/BUILD",
"build-support/bin/release.sh",
"build-support/bin/release_helper.py",
]
_ci_config_globs = [
"build-support/bin/classify_changed_files.py",
"build-support/bin/generate_github_workflows.py",
]


_affected_to_globs = {
Affected.docs: _docs_globs,
Affected.rust: _rust_globs,
Affected.release: _release_globs,
Affected.ci_config: _ci_config_globs,
}


def classify(changed_files: list[str]) -> set[Affected]:
classified: dict[Affected, set[str]] = defaultdict(set)
for affected, globs in _affected_to_globs.items():
for pattern in globs:
classified[affected].update(fnmatch.filter(changed_files, pattern))
ret = {k for k, v in classified.items() if v}
if set(changed_files) - set().union(*classified.values()):
ret.add(Affected.other)
return ret


def main() -> None:
if len(sys.argv) < 2:
return
affecteds = classify(sys.argv[1].split("|"))
for affected in sorted([a.name for a in affecteds]):
print(affected)


if __name__ == "__main__":
main()
24 changes: 24 additions & 0 deletions build-support/bin/classify_changed_files_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import pytest
from classify_changed_files import Affected, classify


@pytest.mark.parametrize(
["changed_files", "expected"],
(
[["docs/path/to/some/doc", "docs/path/to/some/other/doc"], {Affected.docs}],
[["src/rust/engine/path/to/file.rs"], {Affected.rust}],
[["src/python/pants/VERSION"], {Affected.release}],
[["build-support/bin/generate_github_workflows.py"], {Affected.ci_config}],
[["src/python/pants/whatever.py"], {Affected.other}],
[["docs/path/to/some/doc", "rust-toolchain"], {Affected.docs, Affected.rust}],
[
["docs/path/to/some/doc", "rust-toolchain", "src/python/pants/whatever.py"],
{Affected.docs, Affected.rust, Affected.other},
],
),
)
def test_classification(changed_files, expected):
assert classify(changed_files) == expected
Loading

0 comments on commit 81ee8f9

Please sign in to comment.