From 07e4f432082ca40f42ece66f9f10864656f8155e Mon Sep 17 00:00:00 2001 From: Alessandro De Maria Date: Sun, 10 Sep 2023 11:52:42 +0000 Subject: [PATCH] All proposed changes for visibility --- .github/workflows/docker-image.yml | 76 -------- .github/workflows/documentation.yml | 4 + .github/workflows/pex-build-upload.yml | 2 - .github/workflows/python-black.yml | 14 -- .github/workflows/python-pip-publish.yml | 4 + .github/workflows/test-build-publish.yml | 229 +++++++++++++++++++++++ .github/workflows/test.yml | 46 ----- Dockerfile | 40 ++-- 8 files changed, 261 insertions(+), 154 deletions(-) delete mode 100644 .github/workflows/docker-image.yml delete mode 100644 .github/workflows/python-black.yml create mode 100644 .github/workflows/test-build-publish.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 795ef6d2e..000000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: Docker Build and Push -on: - push: - paths-ignore: - - 'docs/**' - workflow_run: - workflows: - - Python Tests - types: - - completed - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout kapitan recursively - uses: actions/checkout@master - with: - submodules: recursive - - name: Strip git ref prefix from tag version and store in TAG_VERSION - run: | - echo "TAG_VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV - echo "REF_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV - - - name: Strip full version and just keep major part in MAJOR_VERSION VAR - run: | - echo "MAJOR_VERSION=${TAG_VERSION:0:4}" >> $GITHUB_ENV - - # Printing versions needs to be a separate step, - # as they aren't set during the previous two steps - - name: Print Versions - run: | - echo ${{ env.TAG_VERSION }} - echo ${{ env.MAJOR_VERSION }} - echo ${{ env.REF_NAME }} - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Setup QEMU and Buildx to build multi-platform image - # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build PR/versioned tags - if: github.ref != 'refs/heads/master' - uses: docker/build-push-action@v4 - with: - tags: kapicorp/kapitan:${{ format('{0}', env.REF_NAME ) }} - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} # push image only on non-pull_requests - file: Dockerfile - # TODO push and tag as latest if release (and not RC) - - name: Build latest tag - uses: docker/build-push-action@v4 - if: github.ref == 'refs/heads/master' - with: - tags: kapicorp/kapitan:${{ format('{0}', env.REF_NAME ) }},kapicorp/kapitan:latest - platforms: linux/amd64,linux/arm64 - file: Dockerfile - - name: Test Dockerfile in current ref - run: | - [ ${{ env.REF_NAME }} == "master" ] && tagname="latest" || tagname=${{ env.REF_NAME }} - docker run -t --rm kapicorp/kapitan:${tagname} --version - - name: Build major version tag - uses: docker/build-push-action@v4 - if: startsWith(github.ref, 'refs/tags/') - with: - tags: kapicorp/kapitan:${{ format('{0}', env.MAJOR_VERSION ) }} - platforms: linux/amd64,linux/arm64 - file: Dockerfile diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 2c217e7b1..d118aa793 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,6 +1,10 @@ --- name: Build and deploy documentation on merge to master +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + on: push: branches: diff --git a/.github/workflows/pex-build-upload.yml b/.github/workflows/pex-build-upload.yml index 498ba62a8..599336c83 100644 --- a/.github/workflows/pex-build-upload.yml +++ b/.github/workflows/pex-build-upload.yml @@ -2,8 +2,6 @@ name: PEX Build and Upload on: - release: - types: [created] workflow_run: workflows: - Upload Python Package diff --git a/.github/workflows/python-black.yml b/.github/workflows/python-black.yml deleted file mode 100644 index 1084d9854..000000000 --- a/.github/workflows/python-black.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: Python Lint -on: - push: - branches: - - master - pull_request: -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - - uses: psf/black@main diff --git a/.github/workflows/python-pip-publish.yml b/.github/workflows/python-pip-publish.yml index 14bc850c6..9e73909fe 100644 --- a/.github/workflows/python-pip-publish.yml +++ b/.github/workflows/python-pip-publish.yml @@ -1,6 +1,10 @@ --- name: Upload Python Package +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + on: release: types: [created] diff --git a/.github/workflows/test-build-publish.yml b/.github/workflows/test-build-publish.yml new file mode 100644 index 000000000..d369822af --- /dev/null +++ b/.github/workflows/test-build-publish.yml @@ -0,0 +1,229 @@ +--- +name: Test, Build and Publish docker image +run-name: Docker Build for ${{ github.actor }} on branch ${{ github.ref_name }} + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +on: + push: + branches: + - master + - main + - test/* + paths-ignore: + - 'docs/**' + - 'requirements.docs.txt' + - 'mkdocs.yml' + - 'CNAME' + + release: + types: [created] + + pull_request: + paths-ignore: + - 'docs/**' + - 'requirements.docs.txt' + - 'mkdocs.yml' + - 'CNAME' + +jobs: + lint: + name: linter + runs-on: ubuntu-latest + if: success() || failure() # Continue running if other jobs fail + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + - uses: psf/black@main + + test: + name: python ${{ matrix.python-version }} tests + runs-on: ubuntu-latest + if: success() || failure() # Continue running if other jobs fail + strategy: + matrix: + python-version: [3.8, 3.9] + + steps: + - name: Checkout kapitan recursively + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: ${{ matrix.python-version }} + + - name: Install testing dependencies + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157 + sudo apt-get -qq update + sudo apt-get install -y gnupg2 git curl + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + sudo ./get_helm.sh + pip3 install --editable ".[test]" + pip3 install coverage black + - name: Run tests + run: |- + make test && make test_coverage + + build: + name: build ${{ matrix.platform }} image + if: success() || failure() # Continue running if other jobs fail + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + steps: + - name: Checkout kapitan recursively + uses: actions/checkout@v4 + with: + submodules: recursive + + # Setup QEMU and Buildx to build multi-platform image + # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Builds docker image and allow scoped caching + - name: build Kapitan Image + uses: docker/build-push-action@v4 + with: + push: False + platforms: ${{ matrix.platform }} + load: True + file: Dockerfile + tags: local-test-${{ matrix.platform }} + cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }} + + - name: Test Kapitan for ${{ matrix.platform }} + run: | + docker run -t --rm local-test-${{ matrix.platform }} --version + + publish: + name: publish platform images + # Only starts if everything else is successful + needs: [lint, test, build] + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + steps: + - name: Checkout kapitan recursively + uses: actions/checkout@v4 + with: + submodules: recursive + + # Setup QEMU and Buildx to build multi-platform image + # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}} + if: env.DOCKERHUB_USERNAME != null + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}} + with: + # list of Docker images to use as base name for tags + images: | + name=${{ env.DOCKERHUB_USERNAME }}/kapitan + # generate Docker tags based on the following events/attributes + tags: | + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + flavor: | + suffix=-${{ matrix.platform }} + + - name: Build and push by digest + id: push-digest + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}} + if: env.DOCKERHUB_USERNAME != null + uses: docker/build-push-action@v4 + with: + platforms: ${{ matrix.platform }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{steps.meta.output.labels}} + cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }} + + build-multi-architecture: + name: combine platform images + needs: + - publish + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # Setup QEMU and Buildx to build multi-platform image + # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}} + if: env.DOCKERHUB_USERNAME != null + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}} + with: + # list of Docker images to use as base name for tags + images: | + name=${{ env.DOCKERHUB_USERNAME }}/kapitan + # generate Docker tags based on the following events/attributes + tags: | + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - uses: int128/docker-manifest-create-action@v1 + with: + tags: ${{ steps.meta.outputs.tags }} + builder: buildx + suffixes: | + -linux-amd64 + -linux-arm64 \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 03143898c..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -name: Python Tests -on: - push: - paths-ignore: - - 'docs/**' - branches: - - master - pull_request: - paths-ignore: - - 'docs/**' - workflow_run: - workflows: - - Python Lint - types: - - completed - -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9] - - steps: - - name: Checkout kapitan recursively - uses: actions/checkout@master - with: - submodules: recursive - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install testing dependencies - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157 - sudo apt-get -qq update - sudo apt-get install -y gnupg2 git curl - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - sudo ./get_helm.sh - pip3 install --editable ".[test]" - pip3 install coverage black - - name: Run tests - run: |- - make test && make test_coverage diff --git a/Dockerfile b/Dockerfile index b555844fe..9c6990cb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,51 @@ # Build the virtualenv for Kapitan FROM python:3.8-slim AS python-builder - ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH:-amd64} RUN mkdir /kapitan WORKDIR /kapitan -COPY ./kapitan ./kapitan -COPY ./MANIFEST.in ./MANIFEST.in -COPY ./pyproject.toml ./pyproject.toml -COPY ./README.md ./README.md - -ENV PATH="/opt/venv/bin:${PATH}" - RUN apt-get update \ && apt-get install --no-install-recommends -y \ curl \ build-essential +ENV POETRY_VERSION=1.4.0 +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:/usr/local/go/bin:${PATH}" +RUN python -m venv $VIRTUAL_ENV \ + && pip install --upgrade pip yq wheel poetry==$POETRY_VERSION + # Install Go (for go-jsonnet) RUN curl -fsSL -o go.tar.gz https://go.dev/dl/go1.17.3.linux-${TARGETARCH}.tar.gz \ && tar -C /usr/local -xzf go.tar.gz \ && rm go.tar.gz -RUN python -m venv /opt/venv \ - && pip install --upgrade pip yq wheel \ - && export PATH=$PATH:/usr/local/go/bin \ - && pip install --editable .[test] \ - && pip install .[gojsonnet] - # Install Helm RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ && chmod 700 get_helm.sh \ && HELM_INSTALL_DIR=/opt/venv/bin ./get_helm.sh --no-sudo \ && rm get_helm.sh + +COPY ./MANIFEST.in ./MANIFEST.in +COPY ./pyproject.toml ./pyproject.toml +COPY ./poetry.lock ./poetry.lock +COPY ./README.md ./README.md + +# Installs and caches dependencies +RUN poetry install --no-root --extras=gojsonnet + +COPY ./kapitan ./kapitan + +RUN pip install --editable .[test] \ + && pip install .[gojsonnet] + + # Final image with virtualenv built in previous step FROM python:3.8-slim -COPY --from=python-builder /opt/venv /opt/venv - ENV PATH="/opt/venv/bin:${PATH}" ENV HELM_CACHE_HOME=".cache/helm" ENV SEARCHPATH="/src" @@ -58,6 +64,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && useradd --create-home --no-log-init --user-group kapitan +COPY --from=python-builder /opt/venv /opt/venv + USER kapitan ENTRYPOINT ["kapitan"]