return e2e to prow (#1100)

Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
feast-dev · Oct 27, 2020 · 7ba60f0 · 7ba60f0
1 parent ec0d65f
commit 7ba60f0
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 52 deletions.
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -155,55 +155,3 @@ jobs:
         run: make build-java-no-tests REVISION=${GITHUB_SHA}
       - name: copy to gs
         run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/
-
-  test-end-to-end:
-    runs-on: [self-hosted]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v1
-        with:
-          java-version: '11'
-      - uses: stCarolas/setup-maven@v3
-        with:
-          maven-version: 3.6.3
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - name: install
-        run: |
-          apt-get update && apt-get install -y redis-server postgresql libpq-dev
-          make build-java-no-tests REVISION=develop
-          python -m pip install --upgrade pip setuptools wheel
-          make install-python
-          python -m pip install -qr tests/requirements.txt
-      - name: run tests
-        run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop"
-
-  test-end-to-end-gcp:
-    runs-on: [self-hosted]
-    env:
-      DISABLE_SERVICE_FIXTURES: true
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v1
-        with:
-          java-version: '11'
-      - uses: stCarolas/setup-maven@v3
-        with:
-          maven-version: 3.6.3
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - name: install
-        run: |
-          apt-get update && apt-get install -y redis-server postgresql libpq-dev
-          make build-java-no-tests REVISION=develop
-          python -m pip install --upgrade pip setuptools wheel
-          make install-python
-          python -m pip install -qr tests/requirements.txt
-      - name: run tests
-        run: >
-          su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/
-          --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e
-          --dataproc-project kf-feast --dataproc-region us-central1
-          --redis-url 10.128.0.105:6379 --redis-cluster --kafka-brokers 10.128.0.103:9094"
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -141,6 +141,54 @@ presubmits:
       - image: golang:1.13
         command: ["infra/scripts/test-golang-sdk.sh"]
 
+  - name: test-end-to-end
+    decorate: true
+    always: true
+    spec:
+      containers:
+      - image: gcr.io/kf-feast/feast-ci:latest
+        command: ["infra/scripts/test-end-to-end.sh"]
+        resources:
+          requests:
+            cpu: "6"
+            memory: "6144Mi"
+        env:
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /etc/gcloud/service-account.json
+        volumeMounts:
+          - mountPath: /etc/gcloud/service-account.json
+            name: service-account
+            readOnly: true
+            subPath: service-account.json
+      volumes:
+        - name: service-account
+          secret:
+            secretName: feast-service-account
+
+  - name: test-end-to-end-gcp
+    decorate: true
+    always: true
+    spec:
+      containers:
+      - image: gcr.io/kf-feast/feast-ci:latest
+        command: [ "infra/scripts/test-end-to-end-gcp.sh" ]
+      resources:
+        requests:
+          cpu: "6"
+          memory: "6144Mi"
+        env:
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /etc/gcloud/service-account.json
+        volumeMounts:
+          - mountPath: /etc/gcloud/service-account.json
+            name: service-account
+            readOnly: true
+            subPath: service-account.json
+      volumes:
+        - name: service-account
+          secret:
+            secretName: feast-service-account
+
 
 postsubmits:
   feast-dev/feast:

diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh
@@ -0,0 +1,14 @@
+export DISABLE_SERVICE_FIXTURES=1
+
+apt-get update && apt-get install -y redis-server postgresql libpq-dev
+
+make build-java-no-tests REVISION=develop
+
+python -m pip install --upgrade pip setuptools wheel
+make install-python
+python -m pip install -qr tests/requirements.txt
+
+su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ \
+          --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e \
+          --dataproc-project kf-feast --dataproc-region us-central1 \
+          --redis-url 10.128.0.105:6379 --redis-cluster --kafka-brokers 10.128.0.103:9094"
diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
@@ -0,0 +1,8 @@
+apt-get update && apt-get install -y redis-server postgresql libpq-dev
+
+make build-java-no-tests REVISION=develop
+python -m pip install --upgrade pip setuptools wheel
+make install-python
+python -m pip install -qr tests/requirements.txt
+
+su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop"