diff --git a/.github/actions/detect-versions/action.yml b/.github/actions/detect-versions/action.yml new file mode 100644 index 000000000..f431c6d4f --- /dev/null +++ b/.github/actions/detect-versions/action.yml @@ -0,0 +1,63 @@ +name: Detect Versions +description: Detect truss versions +outputs: + version_changed: + description: Did the version change + value: ${{ steps.versions.outputs.version_changed }} + new_version: + description: Value of new version + value: ${{ steps.versions.outputs.new_version }} + new_base_image_version: + description: new base image version + value: ${{ steps.versions.outputs.new_base_image_version }} + build_base_images: + description: should build base images + value: ${{ steps.versions.outputs.build_base_images }} + release_version: + description: should release version + value: ${{ steps.versions.outputs.release_version }} + is_prerelease_version: + description: is it a prerelease version + value: ${{ steps.versions.outputs.is_prerelease_version }} +runs: + using: "composite" + steps: + - run: curl -sSL https://install.python-poetry.org | python3 - + shell: bash + - id: versions + shell: bash + run: | + NEW_VERSION=$(poetry version | awk '{print $2}') + NEW_BASE_IMAGE_VERSION=$(grep TRUSS_BASE_IMAGE_VERSION_TAG truss/contexts/image_builder/util.py | cut -d'=' -f2 | cut -d'"' -f2) + + git checkout HEAD^1 -- pyproject.toml truss/contexts/image_builder/util.py + OLD_VERSION=$(poetry version | awk '{print $2}') + OLD_BASE_IMAGE_VERSION=$(grep TRUSS_BASE_IMAGE_VERSION_TAG truss/contexts/image_builder/util.py | cut -d'=' -f2 | cut -d'"' -f2) + + # Put back things into place + git checkout HEAD -- pyproject.toml truss/contexts/image_builder/util.py + + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "new_base_image_version=$NEW_BASE_IMAGE_VERSION" >> $GITHUB_OUTPUT + + if [[ "$NEW_VERSION" != "$OLD_VERSION" ]]; then + echo "version_changed=true" >> $GITHUB_OUTPUT + if [[ "$NEW_VERSION" == *"dev"* ]]; then + echo "release_version=false" >> $GITHUB_OUTPUT + else + echo "release_version=true" >> $GITHUB_OUTPUT + if [[ "$NEW_VERSION" == *"rc"* ]]; then + echo "is_prerelease_version=true" >> $GITHUB_OUTPUT + else + echo "is_prerelease_version=false" >> $GITHUB_OUTPUT + fi + fi + else + echo "version_changed=false" >> $GITHUB_OUTPUT + fi + + if [[ "$NEW_BASE_IMAGE_VERSION" != "$OLD_BASE_IMAGE_VERSION" ]]; then + echo "build_base_images=true" >> $GITHUB_OUTPUT + else + echo "build_base_images=false" >> $GITHUB_OUTPUT + fi diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 465935d21..5b716e759 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,7 +11,57 @@ concurrency: cancel-in-progress: false jobs: - integration_tests: + detect-version-changed: + runs-on: ubuntu-20.04 + outputs: + version_changed: ${{ steps.versions.outputs.version_changed }} + new_version: ${{ steps.versions.outputs.new_version }} + new_base_image_version: ${{ steps.versions.outputs.new_base_image_version }} + build_base_images: ${{ steps.versions.outputs.build_base_images }} + release_version: ${{ steps.versions.outputs.release_version }} + is_prerelease_version: ${{ steps.versions.outputs.is_prerelease_version }} + steps: + - uses: actions/checkout@v3 + with: + # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another + token: ${{secrets.BASETENBOT_GITHUB_TOKEN}} + fetch-depth: 2 + - uses: ./.github/actions/detect-versions/ + id: versions + build-and-push-truss-base-images-if-needed: + needs: [detect-version-changed] + if: needs.detect-version-changed.outputs.build_base_images == 'true' + runs-on: ubuntu-20.04 + strategy: + matrix: + python_version: ["3.8", "3.9", "3.10", "3.11"] + use_gpu: ["y", "n"] + job_type: ["server", "training"] + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python/ + - run: poetry install + - shell: bash + run: | + poetry run bin/generate_base_images.py \ + --use-gpu ${{ matrix.use_gpu }} \ + --python-version ${{ matrix.python_version }} \ + --job-type ${{ matrix.job_type }} \ + --version-tag ${{ needs.detect-version-changed.outputs.new_base_image_version }} \ + --skip-login --push + + integration-tests: + needs: [detect-version-changed, build-and-push-truss-base-images-if-needed] + if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }} runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -19,8 +69,6 @@ jobs: split_group: ["1", "2", "3", "4", "5"] steps: - uses: actions/checkout@v3 - with: - lfs: true - uses: ./.github/actions/setup-python/ - run: poetry install - - run: poetry run pytest truss/tests -m 'integration' --splits 5 --group ${{ matrix.split_group }} + - run: poetry run pytest truss/tests -m 'integration' --splits 5 --group ${{ matrix.split_group }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a24cf5aa2..d4a15ab45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,97 +20,17 @@ jobs: release_version: ${{ steps.versions.outputs.release_version }} is_prerelease_version: ${{ steps.versions.outputs.is_prerelease_version }} steps: - - uses: actions/checkout@v3 - with: - # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another - token: ${{secrets.BASETENBOT_GITHUB_TOKEN}} - fetch-depth: 2 - - run: curl -sSL https://install.python-poetry.org | python3 - - shell: bash - - id: versions - run: | - NEW_VERSION=$(poetry version | awk '{print $2}') - NEW_BASE_IMAGE_VERSION=$(grep TRUSS_BASE_IMAGE_VERSION_TAG truss/contexts/image_builder/util.py | cut -d'=' -f2 | cut -d'"' -f2) - - git checkout HEAD^1 -- pyproject.toml truss/contexts/image_builder/util.py - OLD_VERSION=$(poetry version | awk '{print $2}') - OLD_BASE_IMAGE_VERSION=$(grep TRUSS_BASE_IMAGE_VERSION_TAG truss/contexts/image_builder/util.py | cut -d'=' -f2 | cut -d'"' -f2) - - # Put back things into place - git checkout HEAD -- pyproject.toml truss/contexts/image_builder/util.py - - echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT - echo "new_base_image_version=$NEW_BASE_IMAGE_VERSION" >> $GITHUB_OUTPUT - - if [[ "$NEW_VERSION" != "$OLD_VERSION" ]]; then - echo "version_changed=true" >> $GITHUB_OUTPUT - if [[ "$NEW_VERSION" == *"dev"* ]]; then - echo "release_version=false" >> $GITHUB_OUTPUT - else - echo "release_version=true" >> $GITHUB_OUTPUT - if [[ "$NEW_VERSION" == *"rc"* ]]; then - echo "is_prerelease_version=true" >> $GITHUB_OUTPUT - else - echo "is_prerelease_version=false" >> $GITHUB_OUTPUT - fi - fi - else - echo "version_changed=false" >> $GITHUB_OUTPUT - fi - - if [[ "$NEW_BASE_IMAGE_VERSION" != "$OLD_BASE_IMAGE_VERSION" ]]; then - echo "build_base_images=true" >> $GITHUB_OUTPUT - else - echo "build_base_images=false" >> $GITHUB_OUTPUT - fi - build-and-push-truss-base-images-if-needed: - needs: [detect-version-changed] - if: needs.detect-version-changed.outputs.build_base_images == 'true' - runs-on: ubuntu-20.04 - strategy: - matrix: - python_version: ["3.8", "3.9", "3.10", "3.11"] - use_gpu: ["y", "n"] - job_type: ["server", "training"] - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python/ - - run: poetry install - - shell: bash - run: | - poetry run bin/generate_base_images.py \ - --use-gpu ${{ matrix.use_gpu }} \ - --python-version ${{ matrix.python_version }} \ - --job-type ${{ matrix.job_type }} \ - --version-tag ${{ needs.detect-version-changed.outputs.new_base_image_version }} \ - --skip-login --push - - integration-tests: - needs: [detect-version-changed, build-and-push-truss-base-images-if-needed] - if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }} - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - split_group: ["1", "2", "3", "4", "5"] - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python/ - - run: poetry install - - run: poetry run pytest truss/tests -m 'integration' --splits 5 --group ${{ matrix.split_group }} + - uses: actions/checkout@v3 + with: + # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another + token: ${{secrets.BASETENBOT_GITHUB_TOKEN}} + fetch-depth: 2 + - uses: ./.github/actions/detect-versions/ + id: versions build-n-push-context-builder-image: - needs: [integration-tests, detect-version-changed] - if: ${{ !failure() && !cancelled() && needs.detect-version-changed.outputs.release_version == 'true' && (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') }} + needs: [detect-version-changed] + if: ${{ !failure() && !cancelled() && needs.detect-version-changed.outputs.release_version == 'true' }} runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 @@ -141,8 +61,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} publish-to-pypi: - needs: [integration-tests, detect-version-changed] - if: ${{ !failure() && !cancelled() && needs.detect-version-changed.outputs.release_version == 'true' && (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') }} + needs: [detect-version-changed] + if: ${{ !failure() && !cancelled() && needs.detect-version-changed.outputs.release_version == 'true' }} runs-on: ubuntu-20.04 steps: - name: "Git tag release"