[release v1.2] Backport v1.6

.github/workflows/pythonbuild.yml

@@ -65,6 +65,7 @@ jobs:
  - flytekit-deck-standard
  - flytekit-dolt
  - flytekit-duckdb
+ - flytekit-envd
  - flytekit-greatexpectations
  - flytekit-hive
  - flytekit-k8s-pod
@@ -156,19 +157,3 @@ jobs:
  uses: ludeeus/action-shellcheck@master
  with:
  ignore_paths: boilerplate
-
- docs:
- runs-on: ubuntu-latest
- steps:
- - name: Fetch the code
- uses: actions/checkout@v3
- - name: Set up Python 3.9
- uses: actions/setup-python@v4
- with:
- python-version: 3.9
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip==21.2.4 setuptools wheel
- pip install -r doc-requirements.txt
- - name: Build the documentation
- run: make -C docs html

.github/workflows/pythonpublish.yml

@@ -47,11 +47,28 @@ jobs:
  run: |
  make -C plugins build_all_plugins
  make -C plugins publish_all_plugins
- # Added sleep because PYPI take some time in publish
- - name: Sleep for 180 seconds
- uses: jakejarvis/wait-action@master
- with:
- time: '180s'
+ - name: Sleep until pypi is available
+ id: pypiwait
+ run: |
+ # from refs/tags/v1.2.3 get 1.2.3 and make sure it's not an empty string
+ VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+ if [ -z "$VERSION" ]
+ then
+ echo "No tagged version found, exiting"
+ exit 1
+ fi
+ LINK="https://pypi.org/project/flytekit/${VERSION}"
+ for i in {1..60}; do
+ if curl -L -I -s -f ${LINK} >/dev/null; then
+ echo "Found pypi"
+ exit 0
+ else
+ echo "Did not find - Retrying in 10 seconds..."
+ sleep 10
+ fi
+ done
+ exit 1
+ shell: bash
  outputs:
  version: ${{ steps.bump.outputs.version }}
 
@@ -120,6 +137,91 @@ jobs:
  tags: ${{ steps.sqlalchemy-names.outputs.tags }}
  build-args: |
  VERSION=${{ needs.deploy.outputs.version }}
- file: ./plugins/flytekit-sqlalchemy/Dockerfile.py${{ matrix.python-version }}
+ PYTHON_VERSION=${{ matrix.python-version }}
+ file: ./plugins/flytekit-sqlalchemy/Dockerfile
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ build-and-push-external-plugin-service-images:
+ runs-on: ubuntu-latest
+ needs: deploy
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ fetch-depth: "0"
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Login to GitHub Container Registry
+ if: ${{ github.event_name == 'release' }}
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+ password: "${{ secrets.FLYTE_BOT_PAT }}"
+ - name: Prepare External Plugin Service Image Names
+ id: external-plugin-service-names
+ uses: docker/metadata-action@v3
+ with:
+ images: |
+ ghcr.io/${{ github.repository_owner }}/external-plugin-service
+ tags: |
+ latest
+ ${{ github.sha }}
+ ${{ needs.deploy.outputs.version }}
+ - name: Push External Plugin Service Image to GitHub Registry
+ uses: docker/build-push-action@v2
+ with:
+ context: "."
+ platforms: linux/arm64, linux/amd64
+ push: ${{ github.event_name == 'release' }}
+ tags: ${{ steps.external-plugin-service-names.outputs.tags }}
+ build-args: |
+ VERSION=${{ needs.deploy.outputs.version }}
+ file: ./Dockerfile.external-plugin-service
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ build-and-push-spark-images:
+ runs-on: ubuntu-latest
+ needs: deploy
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ fetch-depth: "0"
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Login to GitHub Container Registry
+ if: ${{ github.event_name == 'release' }}
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+ password: "${{ secrets.FLYTE_BOT_PAT }}"
+ - name: Prepare Spark Image Names
+ id: spark-names
+ uses: docker/metadata-action@v3
+ with:
+ images: |
+ ghcr.io/${{ github.repository_owner }}/flytekit
+ tags: |
+ spark-latest
+ spark-${{ github.sha }}
+ spark-${{ needs.deploy.outputs.version }}
+ - name: Push Spark Image to GitHub Registry
+ uses: docker/build-push-action@v2
+ with:
+ context: "./plugins/flytekit-spark/"
+ platforms: linux/arm64, linux/amd64
+ push: ${{ github.event_name == 'release' }}
+ tags: ${{ steps.spark-names.outputs.tags }}
+ build-args: |
+ VERSION=${{ needs.deploy.outputs.version }}
+ file: ./plugins/flytekit-spark/Dockerfile
  cache-from: type=gha
  cache-to: type=gha,mode=max

.gitignore

@@ -32,3 +32,4 @@ htmlcov
 *.ipynb
 *dat
 docs/source/_tags/
+.hypothesis

.pre-commit-config.yaml

@@ -22,3 +22,7 @@ repos:
  rev: v0.8.0.4
  hooks:
  - id: shellcheck
+- repo: https://github.com/conorfalvey/check_pdb_hook
+ rev: 0.0.9
+ hooks:
+ - id: check_pdb_hook

.readthedocs.yml

@@ -9,6 +9,8 @@ build:
  os: ubuntu-20.04
  tools:
  python: "3.9"
+ apt_packages:
+ - graphviz
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

Dockerfile

@@ -16,8 +16,7 @@ RUN apt-get update && apt-get install build-essential -y
 RUN pip install -U flytekit==$VERSION \
  flytekitplugins-pod==$VERSION \
  flytekitplugins-deck-standard==$VERSION \
- flytekitplugins-data-fsspec[aws]==$VERSION \
- flytekitplugins-data-fsspec[gcp]==$VERSION \
+ flytekitplugins-envd==$VERSION \
  scikit-learn
 
 RUN useradd -u 1000 flytekit

Dockerfile.dev

@@ -0,0 +1,32 @@
+# This Dockerfile is here to help with end-to-end testing
+# From flytekit
+# $ docker build -f Dockerfile.dev --build-arg PYTHON_VERSION=3.10 -t localhost:30000/flytekittest:someversion .
+# $ docker push localhost:30000/flytekittest:someversion
+# From your test user code
+# $ pyflyte run --image localhost:30000/flytekittest:someversion
+
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}-slim-buster
+
+MAINTAINER Flyte Team <users@flyte.org>
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
+
+WORKDIR /root
+
+ARG VERSION
+
+RUN apt-get update && apt-get install build-essential vim -y
+
+COPY . /flytekit
+
+# Pod tasks should be exposed in the default image
+RUN pip install -e /flytekit
+RUN pip install -e /flytekit/plugins/flytekit-k8s-pod
+RUN pip install -e /flytekit/plugins/flytekit-deck-standard
+RUN pip install scikit-learn
+
+ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"
+
+RUN useradd -u 1000 flytekit
+RUN chown flytekit: /root
+USER flytekit

Dockerfile.external-plugin-service

@@ -0,0 +1,9 @@
+FROM python:3.9-slim-buster
+
+MAINTAINER Flyte Team <users@flyte.org>
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
+
+ARG VERSION
+RUN pip install -U flytekit==$VERSION flytekitplugins-bigquery==$VERSION
+
+CMD pyflyte serve --port 8000

Makefile

@@ -26,6 +26,7 @@ setup: install-piptools ## Install requirements
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
+ autoflake --remove-all-unused-imports --ignore-init-module-imports --ignore-pass-after-docstring --in-place -r flytekit plugins tests
  pre-commit run black --all-files || true
  pre-commit run isort --all-files || true
 

dev-requirements.in

@@ -2,6 +2,7 @@
 
 git+https://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte
 coverage[toml]
+hypothesis
 joblib
 mock
 pytest
@@ -21,3 +22,7 @@ tensorflow==2.8.1; platform_machine!='arm64' or platform_system!='Darwin'
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1
 scikit-learn
+types-protobuf<4
+types-croniter
+types-mock
+autoflake

doc-requirements.in

@@ -1,6 +1,7 @@
 .
 -e file:.#egg=flytekit
 
+grpcio<=1.49.1
 git+https://github.com/flyteorg/furo@main
 sphinx
 sphinx-gallery
@@ -11,7 +12,7 @@ sphinx-autoapi
 sphinx-copybutton
 sphinx_fontawesome
 sphinx-panels
-sphinxcontrib-yt
+sphinxcontrib-youtube
 cryptography
 google-api-core[grpc]
 scikit-learn