Refactored end-to-end tests fully orchestrated by pytest (#1094)

.github/workflows/complete.yml

@@ -156,3 +156,54 @@ jobs:
       - name: copy to gs
         run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/
 
+  test-end-to-end:
+    runs-on: [self-hosted]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+      - uses: stCarolas/setup-maven@v3
+        with:
+          maven-version: 3.6.3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - name: install
+        run: |
+          apt-get update && apt-get install -y redis-server postgresql libpq-dev
+          make build-java-no-tests REVISION=develop
+          python -m pip install --upgrade pip setuptools wheel
+          make install-python
+          python -m pip install -qr tests/requirements.txt
+      - name: run tests
+        run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop"
+
+  test-end-to-end-gcp:
+    runs-on: [self-hosted]
+    env:
+      DISABLE_SERVICE_FIXTURES: true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+      - uses: stCarolas/setup-maven@v3
+        with:
+          maven-version: 3.6.3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - name: install
+        run: |
+          apt-get update && apt-get install -y redis-server postgresql libpq-dev
+          make build-java-no-tests REVISION=develop
+          python -m pip install --upgrade pip setuptools wheel
+          make install-python
+          python -m pip install -qr tests/requirements.txt
+      - name: run tests
+        run: >
+          su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/
+          --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e
+          --dataproc-project kf-feast --dataproc-region us-central1
+          --redis-url 10.128.0.105:6379 --redis-cluster --kafka-brokers 10.128.0.103:9094"

.prow/config.yaml

@@ -141,118 +141,6 @@ presubmits:
       - image: golang:1.13
         command: ["infra/scripts/test-golang-sdk.sh"]
 
-  - name: test-end-to-end
-    decorate: true
-    always_run: true
-    spec:
-      containers:
-      - image: maven:3.6-jdk-11
-        command: ["infra/scripts/test-end-to-end.sh"]
-        resources:
-          requests:
-            cpu: "6"
-            memory: "6144Mi"
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/gcloud/service-account.json
-        volumeMounts:
-        - mountPath: /etc/gcloud/service-account.json
-          name: service-account
-          readOnly: true
-          subPath: service-account.json
-      volumes:
-      - name: service-account
-        secret:
-          secretName: feast-service-account
-    skip_branches:
-    - ^v0\.(3|4)-branch$
-
-  - name: test-end-to-end-auth
-    decorate: true
-    always_run: true
-    spec:
-      containers:
-      - image: maven:3.6-jdk-11
-        command: ["infra/scripts/test-end-to-end.sh", "True"]
-        resources:
-          requests:
-            cpu: "6"
-            memory: "6144Mi"
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/gcloud/service-account.json
-        volumeMounts:
-        - mountPath: /etc/gcloud/service-account.json
-          name: service-account
-          readOnly: true
-          subPath: service-account.json
-      volumes:
-      - name: service-account
-        secret:
-          secretName: feast-service-account
-    skip_branches:
-    - ^v0\.(3|4)-branch$
-
-  - name: test-end-to-end-redis-cluster
-    decorate: true
-    always_run: true
-    spec:
-      containers:
-      - image: maven:3.6-jdk-11
-        command: ["infra/scripts/test-end-to-end-redis-cluster.sh"]
-        resources:
-          requests:
-            cpu: "6"
-            memory: "6144Mi"
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/gcloud/service-account.json
-        volumeMounts:
-        - mountPath: /etc/gcloud/service-account.json
-          name: service-account
-          readOnly: true
-          subPath: service-account.json
-      volumes:
-      - name: service-account
-        secret:
-          secretName: feast-service-account
-    skip_branches:
-    - ^v0\.(3|4)-branch$
-
-  - name: test-end-to-end-java-8
-    decorate: true
-    always_run: true
-    spec:
-      containers:
-        - image: maven:3.6-jdk-8
-          command: ["infra/scripts/test-end-to-end.sh"]
-          resources:
-            requests:
-              cpu: "6"
-              memory: "6144Mi"
-    branches:
-    - ^v0\.(3|4)-branch$
-
-  - name: test-end-to-end-batch-java-8
-    decorate: true
-    always_run: true
-    spec:
-      volumes:
-        - name: service-account
-          secret:
-            secretName: feast-service-account
-      containers:
-        - image: maven:3.6-jdk-8
-          command: ["infra/scripts/test-end-to-end-batch.sh"]
-          resources:
-            requests:
-              cpu: "6"
-              memory: "6144Mi"
-          volumeMounts:
-            - name: service-account
-              mountPath: "/etc/service-account"
-    branches:
-    - ^v0\.(3|4)-branch$
 
 postsubmits:
   feast-dev/feast:

Makefile

@@ -70,7 +70,7 @@ compile-protos-python: install-python-ci-dependencies
 	cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --grpc_python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/third_party/grpc/health/v1/*.proto
 
 install-python: compile-protos-python
-	cd sdk/python; python setup.py develop
+	python -m pip install -e sdk/python
 
 test-python:
 	pytest --verbose --color=yes sdk/python/tests
@@ -90,10 +90,10 @@ lint-python:
 	cd ${ROOT_DIR}/sdk/python; flake8 feast/ tests/
 	cd ${ROOT_DIR}/sdk/python; black --check feast tests
 
-	cd ${ROOT_DIR}/tests/e2e; mypy .
-	cd ${ROOT_DIR}/tests/e2e; isort . --check-only
-	cd ${ROOT_DIR}/tests/e2e; flake8 .
-	cd ${ROOT_DIR}/tests/e2e; black --check .
+	cd ${ROOT_DIR}/tests; mypy e2e
+	cd ${ROOT_DIR}/tests; isort e2e --check-only
+	cd ${ROOT_DIR}/tests; flake8 e2e
+	cd ${ROOT_DIR}/tests; black --check e2e
 
 # Go SDK
 

infra/scripts/helm/kafka-values.tpl.yaml

@@ -0,0 +1,18 @@
+externalAccess:
+  enabled: true
+  service:
+    loadBalancerIPs:
+      - $feast_kafka_ip
+    annotations:
+      cloud.google.com/load-balancer-type: Internal
+    loadBalancerSourceRanges:
+      - 10.0.0.0/8
+      - 172.16.0.0/12
+      - 192.168.0.0/16
+
+persistence:
+  enabled: false
+
+zookeeper:
+  persistence:
+    enabled: false

infra/scripts/helm/redis-cluster-values.tpl.yaml

@@ -0,0 +1,17 @@
+cluster:
+  nodes: 3
+  replicas: 0
+  externalAccess:
+    enabled: true
+    service:
+      annotations:
+        cloud.google.com/load-balancer-type: Internal
+      loadBalancerIP:
+        - $feast_redis_1_ip
+        - $feast_redis_2_ip
+        - $feast_redis_3_ip
+
+persistence:
+  enabled: false
+
+usePassword: false

infra/scripts/setup-common-functions.sh

@@ -9,136 +9,6 @@ install_test_tools() {
   apt-get -y install wget netcat kafkacat build-essential
 }
 
-install_gcloud_sdk() {
-  print_banner "Installing Google Cloud SDK"
-  if [[ ! $(command -v gsutil) ]]; then
-    CURRENT_DIR=$(dirname "$BASH_SOURCE")
-    . "${CURRENT_DIR}"/install-google-cloud-sdk.sh
-  fi
-
-  export GOOGLE_APPLICATION_CREDENTIALS
-  gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
-}
-
-install_and_start_local_redis() {
-  print_banner "Installing and tarting Redis at localhost:6379"
-  # Allow starting serving in this Maven Docker image. Default set to not allowed.
-  echo "exit 0" >/usr/sbin/policy-rc.d
-  apt-get -y install redis-server >/var/log/redis.install.log
-  redis-server --daemonize yes
-  redis-cli ping
-}
-
-install_and_start_local_redis_cluster() {
-  print_banner "Installing Redis at localhost:6379"
-  echo "exit 0" >/usr/sbin/policy-rc.d
-  ${SCRIPTS_DIR}/setup-redis-cluster.sh
-  redis-cli -c -p 7000 ping
-}
-
-install_and_start_local_postgres() {
-  print_banner "Installing and starting Postgres at localhost:5432"
-  apt-get -y install postgresql >/var/log/postgresql.install.log
-  service postgresql start
-  # Initialize with database: 'postgres', user: 'postgres', password: 'password'
-  cat <<EOF >/tmp/update-postgres-role.sh
-psql -c "ALTER USER postgres PASSWORD 'password';"
-EOF
-  chmod +x /tmp/update-postgres-role.sh
-  su -s /bin/bash -c /tmp/update-postgres-role.sh postgres
-  export PGPASSWORD=password
-  pg_isready
-}
-
-install_and_start_local_zookeeper_and_kafka() {
-  print_banner "Installing and starting Zookeeper at localhost:2181 and Kafka at localhost:9092"
-  wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz
-  mv kafka_2.12-2.3.0/ /tmp/kafka
-
-  nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &>/var/log/zookeeper.log 2>&1 &
-  ${SCRIPTS_DIR}/wait-for-it.sh localhost:2181 --timeout=20
-  tail -n10 /var/log/zookeeper.log
-
-  nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &>/var/log/kafka.log 2>&1 &
-  ${SCRIPTS_DIR}/wait-for-it.sh localhost:9092 --timeout=40
-  tail -n10 /var/log/kafka.log
-  kafkacat -b localhost:9092 -L
-}
-
-build_feast_core_and_serving() {
-  print_banner "Building Feast Core and Feast Serving"
-  infra/scripts/download-maven-cache.sh \
-    --archive-uri gs://feast-templocation-kf-feast/.m2.2020-08-19.tar \
-    --output-dir /root/
-
-  # Build jars for Feast
-  mvn --quiet --batch-mode -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean package
-
-  ls -lh core/target/*jar
-  ls -lh serving/target/*jar
-  ls -lh job-controller/target/*jar
-}
-
-start_feast_core() {
-  print_banner "Starting Feast Core"
-
-  if [ -n "$1" ]; then
-    echo "Custom Spring application.yml location provided: $1"
-    export CONFIG_ARG="--spring.config.location=classpath:/application.yml,file://$1"
-  fi
-
-  nohup java -jar core/target/feast-core-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-core.log &
-  ${SCRIPTS_DIR}/wait-for-it.sh localhost:6565 --timeout=90
-
-  tail -n10 /var/log/feast-core.log
-  nc -w2 localhost 6565 </dev/null
-}
-
-start_feast_jobcontroller() {
-  print_banner "Starting Feast Job Controller"
-
-  if [ -n "$1" ]; then
-    echo "Custom Spring application.yml location provided: $1"
-    export CONFIG_ARG="--spring.config.location=classpath:/application.yml,file://$1"
-  fi
-
-  nohup java -jar job-controller/target/feast-job-controller-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-jobcontroller.log &
-  ${SCRIPTS_DIR}/wait-for-it.sh localhost:6570 --timeout=90
-
-  tail -n10 /var/log/feast-jobcontroller.log
-  nc -w2 localhost 6570 </dev/null
-}
-
-start_feast_serving() {
-  print_banner "Starting Feast Online Serving"
-
-  if [ -n "$1" ]; then
-    echo "Custom Spring application.yml location provided: $1"
-    export CONFIG_ARG="--spring.config.location=classpath:/application.yml,file://$1"
-  fi
-
-  nohup java -jar serving/target/feast-serving-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-serving-online.log &
-  ${SCRIPTS_DIR}/wait-for-it.sh localhost:6566 --timeout=60
-
-  tail -n100 /var/log/feast-serving-online.log
-  nc -w2 localhost 6566 </dev/null
-}
-
-install_python_with_miniconda_and_feast_sdk() {
-  print_banner "Installing Python 3.7 with Miniconda and Feast SDK"
-  # Install Python 3.7 with Miniconda
-  wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \
-    -O /tmp/miniconda.sh
-  bash /tmp/miniconda.sh -b -p /root/miniconda -f
-  /root/miniconda/bin/conda init
-  source ~/.bashrc
-
-  # Install Feast Python SDK and test requirements
-  make compile-protos-python
-  pip install -qe sdk/python
-  pip install -qr tests/e2e/requirements.txt
-}
-
 print_banner() {
   echo "
 ============================================================