diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml index 14b036e5fb6..ef90879a939 100644 --- a/.github/workflows/audit.yaml +++ b/.github/workflows/audit.yaml @@ -5,7 +5,7 @@ jobs: audit: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' runs-on: ubuntu-latest steps: - name: Check out code diff --git a/.github/workflows/cancel.yaml b/.github/workflows/cancel.yaml index 309b7c88f53..39d762a4fa1 100644 --- a/.github/workflows/cancel.yaml +++ b/.github/workflows/cancel.yaml @@ -5,7 +5,7 @@ jobs: cancel: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.9.1 diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index 330ddf9d0da..0454171c6d3 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -11,7 +11,7 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Bootstrap Pants, test and lint Rust (Linux-x86_64) runs-on: - ubuntu-20.04 @@ -153,7 +153,7 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Bootstrap Pants, test Rust (macOS10-x86_64) runs-on: - macos-10.15 @@ -286,7 +286,7 @@ jobs: ARCHFLAGS: -arch arm64 PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Bootstrap Pants, build wheels and fs_util (macOS11-ARM64) runs-on: - self-hosted @@ -402,7 +402,7 @@ jobs: src/python/pants/engine/internals/native_engine.so.metadata' - if: (github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')) - && (${{ github.repository_owner == 'pantsbuild' }}) + && (github.repository_owner == 'pantsbuild') name: Build wheels run: USE_PY39=true arch -arm64 ./build-support/bin/release.sh build-wheels - if: github.event_name == 'push' @@ -420,7 +420,7 @@ jobs: - '3.9' timeout-minutes: 60 check_labels: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Ensure PR has a category label runs-on: - ubuntu-20.04 @@ -436,7 +436,7 @@ jobs: change, category:performance, category:bugfix, category:documentation, category:internal mode: exactly lint_python: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Lint Python and Shell needs: bootstrap_pants_linux_x86_64 runs-on: @@ -503,7 +503,7 @@ jobs: - '3.9' timeout-minutes: 30 test_python_linux_x86_64_0: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Test Python (Linux-x86_64) Shard 0/3 needs: bootstrap_pants_linux_x86_64 runs-on: @@ -591,7 +591,7 @@ jobs: - '3.9' timeout-minutes: 90 test_python_linux_x86_64_1: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Test Python (Linux-x86_64) Shard 1/3 needs: bootstrap_pants_linux_x86_64 runs-on: @@ -679,7 +679,7 @@ jobs: - '3.9' timeout-minutes: 90 test_python_linux_x86_64_2: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Test Python (Linux-x86_64) Shard 2/3 needs: bootstrap_pants_linux_x86_64 runs-on: @@ -769,7 +769,7 @@ jobs: test_python_macos_x86_64: env: ARCHFLAGS: -arch x86_64 - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Test Python (macOS10-x86_64) needs: bootstrap_pants_macos_x86_64 runs-on: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index faa36ada709..1221866492e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,8 +11,8 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Bootstrap Pants, test and lint Rust (Linux-x86_64) needs: - check_labels @@ -156,8 +156,8 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Bootstrap Pants, test Rust (macOS10-x86_64) needs: - check_labels @@ -292,8 +292,8 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Build wheels and fs_util (Linux-x86_64) needs: - check_labels @@ -378,8 +378,8 @@ jobs: ARCHFLAGS: -arch arm64 PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Bootstrap Pants, build wheels and fs_util (macOS11-ARM64) needs: - check_labels @@ -498,7 +498,7 @@ jobs: src/python/pants/engine/internals/native_engine.so.metadata' - if: (github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]')) - && (${{ github.repository_owner == 'pantsbuild' }}) + && (github.repository_owner == 'pantsbuild') name: Build wheels run: USE_PY39=true arch -arm64 ./build-support/bin/release.sh build-wheels - if: github.event_name == 'push' @@ -519,8 +519,8 @@ jobs: env: PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Build wheels and fs_util (macOS10-x86_64) needs: - check_labels @@ -619,7 +619,7 @@ jobs: run: ./build-support/bin/deploy_to_s3.py timeout-minutes: 80 check_labels: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Ensure PR has a category label runs-on: - ubuntu-20.04 @@ -635,29 +635,30 @@ jobs: change, category:performance, category:bugfix, category:documentation, category:internal mode: exactly docs_only_check: - if: ${{ github.repository_owner == 'pantsbuild' }} + if: github.repository_owner == 'pantsbuild' name: Check for docs-only change outputs: docs_only: ${{ steps.docs_only_check.outputs.docs_only }} runs-on: - ubuntu-20.04 steps: - - continue-on-error: true - id: files + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - id: files name: Get changed files - uses: jitterbit/get-changed-files@v1 + uses: tj-actions/changed-files@v23.1 with: - format: json + files_ignore: docs/** + files_ignore_separator: '|' - id: docs_only_check + if: steps.files.outputs.any_changed != 'true' name: Check for docs-only changes - run: "readarray -t all_files <<<\"$(jq -r '.[]' <<<'${{ steps.files.outputs.all\ - \ }}')\"\nDOCS_ONLY=1\nfor file in ${all_files[@]}; do\n if [[ ${file}\ - \ != docs/* ]]; then\n DOCS_ONLY=0\n fi\ndone\nif [[ ${DOCS_ONLY}\ - \ == 1 ]]; then\n echo \"::set-output name=docs_only::1\"\nelse\n echo\ - \ \"::set-output name=docs_only::0\"\nfi\n" + run: echo '::set-output name=docs_only::DOCS_ONLY' lint_python: - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Lint Python and Shell needs: - bootstrap_pants_linux_x86_64 @@ -726,7 +727,7 @@ jobs: - '3.7' timeout-minutes: 30 merge_ok_docs_only: - if: needs.docs_only_check.outputs.docs_only == 1 + if: needs.docs_only_check.outputs.docs_only == 'DOCS_ONLY' name: Merge OK needs: - docs_only_check @@ -735,7 +736,7 @@ jobs: steps: - run: echo 'Merge OK' merge_ok_not_docs_only: - if: needs.docs_only_check.outputs.docs_only == 0 + if: needs.docs_only_check.outputs.docs_only != 'DOCS_ONLY' name: Merge OK needs: - docs_only_check @@ -756,8 +757,8 @@ jobs: steps: - run: echo 'Merge OK' test_python_linux_x86_64_0: - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Test Python (Linux-x86_64) Shard 0/3 needs: - bootstrap_pants_linux_x86_64 @@ -847,8 +848,8 @@ jobs: - '3.7' timeout-minutes: 90 test_python_linux_x86_64_1: - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Test Python (Linux-x86_64) Shard 1/3 needs: - bootstrap_pants_linux_x86_64 @@ -938,8 +939,8 @@ jobs: - '3.7' timeout-minutes: 90 test_python_linux_x86_64_2: - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Test Python (Linux-x86_64) Shard 2/3 needs: - bootstrap_pants_linux_x86_64 @@ -1031,8 +1032,8 @@ jobs: test_python_macos_x86_64: env: ARCHFLAGS: -arch x86_64 - if: (${{ github.repository_owner == 'pantsbuild' }}) && needs.docs_only_check.outputs.docs_only - == 0 + if: (github.repository_owner == 'pantsbuild') && (needs.docs_only_check.outputs.docs_only + != 'DOCS_ONLY') name: Test Python (macOS10-x86_64) needs: - bootstrap_pants_macos_x86_64 diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index 629bfc474e5..7b607acdb1b 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -64,16 +64,24 @@ class Platform(Enum): DISABLE_REMOTE_CACHE_ENV = {"PANTS_REMOTE_CACHE_READ": "false", "PANTS_REMOTE_CACHE_WRITE": "false"} -IS_PANTS_OWNER = "${{ github.repository_owner == 'pantsbuild' }}" +IS_PANTS_OWNER = "github.repository_owner == 'pantsbuild'" # ---------------------------------------------------------------------- # Actions # ---------------------------------------------------------------------- +_DOCS_ONLY_TEXT = "DOCS_ONLY" + + +def _docs_only_cond(docs_only: bool) -> str: + op = "==" if docs_only else "!=" + return f"needs.docs_only_check.outputs.docs_only {op} '{_DOCS_ONLY_TEXT}'" + def is_docs_only() -> Jobs: """Check if this change only involves docs.""" linux_x86_64_helper = Helper(Platform.LINUX_X86_64) + docs_files = ["docs/**"] return { "docs_only_check": { "name": "Check for docs-only change", @@ -81,35 +89,20 @@ def is_docs_only() -> Jobs: "if": IS_PANTS_OWNER, "outputs": {"docs_only": "${{ steps.docs_only_check.outputs.docs_only }}"}, "steps": [ + *checkout(get_commit_msg=False), { "id": "files", "name": "Get changed files", - "uses": "jitterbit/get-changed-files@v1", - # Workaround for https://github.com/jitterbit/get-changed-files/issues/11 - "continue-on-error": True, - "with": { - "format": "json", - }, + "uses": "tj-actions/changed-files@v23.1", + "with": {"files_ignore_separator": "|", "files_ignore": "|".join(docs_files)}, }, { "id": "docs_only_check", "name": "Check for docs-only changes", - "run": dedent( - """\ - readarray -t all_files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.all }}')" - DOCS_ONLY=1 - for file in ${all_files[@]}; do - if [[ ${file} != docs/* ]]; then - DOCS_ONLY=0 - fi - done - if [[ ${DOCS_ONLY} == 1 ]]; then - echo "::set-output name=docs_only::1" - else - echo "::set-output name=docs_only::0" - fi - """ - ), + # Note that if no changes were detected in the step above, the string + # may be empty, not 'false', so we check for != 'true'. + "if": "steps.files.outputs.any_changed != 'true'", + "run": f"echo '::set-output name=docs_only::{_DOCS_ONLY_TEXT}'", }, ], }, @@ -139,7 +132,7 @@ def ensure_category_label() -> Sequence[Step]: ] -def checkout(*, containerized: bool = False) -> Sequence[Step]: +def checkout(*, containerized: bool = False, get_commit_msg: bool = True) -> Sequence[Step]: """Get prior commits and the commit message.""" steps = [ # See https://github.community/t/accessing-commit-message-in-pull-request-event/17158/8 @@ -162,36 +155,37 @@ def checkout(*, containerized: bool = False) -> Sequence[Step]: "run": 'git config --global safe.directory "$GITHUB_WORKSPACE"', } ) - steps.extend( - [ - # For a push event, the commit we care about is HEAD itself. - # This CI currently only runs on PRs, so this is future-proofing. - { - "name": "Get commit message for branch builds", - "if": "github.event_name == 'push'", - "run": dedent( - """\ - echo "COMMIT_MESSAGE<> $GITHUB_ENV - echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - """ - ), - }, - # For a pull_request event, the commit we care about is the second parent of the merge - # commit. This CI currently only runs on PRs, so this is future-proofing. - { - "name": "Get commit message for PR builds", - "if": "github.event_name == 'pull_request'", - "run": dedent( - """\ - echo "COMMIT_MESSAGE<> $GITHUB_ENV - echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - """ - ), - }, - ] - ) + if get_commit_msg: + steps.extend( + [ + # For a push event, the commit we care about is HEAD itself. + # This CI currently only runs on PRs, so this is future-proofing. + { + "name": "Get commit message for branch builds", + "if": "github.event_name == 'push'", + "run": dedent( + """\ + echo "COMMIT_MESSAGE<> $GITHUB_ENV + echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + """ + ), + }, + # For a pull_request event, the commit we care about is the second parent of the + # merge commit. This CI currently only runs on PRs, so this is future-proofing. + { + "name": "Get commit message for PR builds", + "if": "github.event_name == 'pull_request'", + "run": dedent( + """\ + echo "COMMIT_MESSAGE<> $GITHUB_ENV + echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + """ + ), + }, + ] + ) return steps @@ -884,7 +878,7 @@ def merge_ok(needs: list[str], docs_only: bool) -> Jobs: key: { "name": "Merge OK", "runs-on": Helper(Platform.LINUX_X86_64).runs_on(), - "if": f"needs.docs_only_check.outputs.docs_only == {'1' if docs_only else '0'}", + "if": _docs_only_cond(docs_only), # If in the future we have any docs-related checks, we can make both "Merge OK" # jobs depend on them here (we have to do both since some changes may modify docs # as well as code, and so are not "docs only"). @@ -897,7 +891,7 @@ def merge_ok(needs: list[str], docs_only: bool) -> Jobs: def generate() -> dict[Path, str]: """Generate all YAML configs with repo-relative paths.""" - docs_only_cond = "needs.docs_only_check.outputs.docs_only == 0" + not_docs_only = _docs_only_cond(docs_only=False) pr_jobs = test_workflow_jobs([PYTHON37_VERSION], cron=False) pr_jobs.update(**is_docs_only()) for key, val in pr_jobs.items(): @@ -909,7 +903,7 @@ def generate() -> dict[Path, str]: needs.extend(["check_labels", "docs_only_check"]) val["needs"] = needs if_cond = val.get("if") - val["if"] = docs_only_cond if if_cond is None else f"({if_cond}) && {docs_only_cond}" + val["if"] = not_docs_only if if_cond is None else f"({if_cond}) && ({not_docs_only})" pr_jobs.update(merge_ok(needs=list(pr_jobs.keys()), docs_only=False)) pr_jobs.update(merge_ok(needs=[], docs_only=True))