Merge pull request feast-dev#12 from farfetch-external/algattik/merge…

…_2020-06-01 Rebase to Feast v0.5.0
Yanson · Jun 3, 2020 · 1ae84df · 1ae84df
2 parents a7621a5 + 14e6e83
commit 1ae84df
Show file tree

Hide file tree

Showing 374 changed files with 14,593 additions and 9,474 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,3 @@
 docs
+!docs/coverage
 charts
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,8 +1,8 @@
 # Each line is a file pattern followed by one or more owners.
 # https://help.github.com/en/articles/about-code-owners
 
-*           @zhilingc @pradithya @woop @davidheryanto @khorshuheng
-/core/      @zhilingc @pradithya
-/ingestion/ @zhilingc @pradithya
-/serving/   @zhilingc @pradithya
-/cli/       @zhilingc @pradithya
+*           @zhilingc @woop @davidheryanto @khorshuheng @pyalex
+/core/      @zhilingc
+/ingestion/ @zhilingc
+/serving/   @zhilingc
+/cli/       @zhilingc
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,8 +1,8 @@
 <!--  Thanks for sending a pull request!  Here are some tips for you:
 
-1. Ensure that your code follows our code conventions: https://github.com/gojek/feast/blob/master/docs/contributing.md#code-conventions
-2. Run unit tests and ensure that they are passing: https://github.com/gojek/feast/blob/master/docs/contributing.md#running-unit-tests
-3. If your change introduces any API changes, make sure to update the integration tests scripts here: https://github.com/gojek/feast/tree/master/tests/e2e
+1. Ensure that your code follows our code conventions: https://github.com/feast-dev/feast/blob/master/docs/contributing.md#code-conventions
+2. Run unit tests and ensure that they are passing: https://github.com/feast-dev/feast/blob/master/docs/contributing.md#running-unit-tests
+3. If your change introduces any API changes, make sure to update the integration tests scripts here: https://github.com/feast-dev/feast/tree/master/tests/e2e
 4. Make sure documentation is updated for your PR!
 5. Make sure you have signed the CLA https://cla.developers.google.com/clas
 

diff --git a/.github/workflows/integration-tests.yml → .github/workflows/docker_build_test_push.yml b/.github/workflows/integration-tests.yml → .github/workflows/docker_build_test_push.yml
@@ -25,67 +25,39 @@ jobs:
     # Checkout sources from Git
     - uses: actions/checkout@v2
 
-    # Build base CI image, with dependencies such as Go and Python.
-    # This image seldom changes, so thanks to layer caching this usually completes in seconds.
-    - name: Build CI image
-      uses: docker/build-push-action@v1
-      with:
-        username: ${{ secrets.CONTAINERREGISTRY_USERNAME }}
-        password: ${{ secrets.CONTAINERREGISTRY_PASSWORD }}
-        repository: ${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci
-        registry: ${{ secrets.CONTAINERREGISTRY_URL }}
-        dockerfile: infra/docker/ci/Dockerfile
-        tags: v${{ github.sha }}
-
     # Cache the Maven repository across runs.
     - name: Cache Maven repository
       id: cache-maven
       uses: actions/cache@v1
       with:
         path: .m2
-        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        key: ${{ runner.os }}-mavenr-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
-          ${{ runner.os }}-maven-
+          ${{ runner.os }}-mavenr-
 
     # Generate the Maven cache if needed by running a throwaway build of Feast.
     # --user: run as the host user, to avoid writing .m2 files to the host as root.
     # -e MAVEN_CONFIG: See https://github.com/carlossg/docker-maven#running-as-non-root
     # -v $PWD:/build: mount the current directory to write .m2 files onto the host.
     - name: Generate Maven cache
       if: steps.cache-maven.outputs.cache-hit != 'true'
-      run: docker run --rm --user $(id -u):$(id -g) -e MAVEN_CONFIG=/build/.m2 -v $PWD:/build $IMAGE bash -c "cd /build && mvn -Dmaven.repo.local=/build/.m2/repository -Dgpg.skip=true -B clean verify -DskipTests"
-      env:
-        IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci:v${{ github.sha }}
+      run: docker run --rm --user $(id -u):$(id -g) -e MAVEN_CONFIG=/build/.m2 -v $PWD:/build maven:3.6-jdk-11 bash -c "cp -R /build /tmp && mvn -f /tmp/build -Dmaven.repo.local=/build/.m2/repository -Dgpg.skip=true -B clean verify -DskipTests"
 
     # Build images in parallel.
-    - name: Build Feast runtime and end-to-end test images
-      run: |
-        set -x
-        docker-compose -f infra/docker-compose/docker-compose.yml build --parallel end-to-end-tests core online-serving
+    - name: Build Feast runtime images
+      run: docker-compose -f infra/docker-compose/docker-compose.yml -f infra/docker-compose/docker-compose.online.yml build --parallel core online-serving
       env:
         FEAST_VERSION: v${{ github.sha }}
-        FEAST_CI_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci
         FEAST_CORE_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-core
         FEAST_SERVING_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-serving
 
-    # "docker-compose up" starts docker containers.
-    # "--abort-on-container-exit" ensures that the containers are shutdown when end-to-end
-    # tests complete, and that the task exit status indicates failure if there are test failures.  
-    # Images fetched from a repository, such as Kafka, will not be built, so this
-    # step will only build the Feast Core image.
-    - name: Run Feast end-to-end test with Docker compose
-      run: |
-        set -x
-        docker-compose -f infra/docker-compose/docker-compose.yml up --abort-on-container-exit end-to-end-tests
+    - name: test docker compose
+      run: ./infra/scripts/test-docker-compose.sh
       env:
         COMPOSE_PROJECT_NAME: feast
         FEAST_VERSION: v${{ github.sha }}
-        FEAST_CI_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci
         FEAST_CORE_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-core
         FEAST_SERVING_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-serving
-        FEAST_CORE_CONFIG: direct-runner.yml
-        FEAST_ONLINE_SERVING_CONFIG: online-serving.yml
-        FEAST_ONLINE_STORE_CONFIG: redis-store.yml
 
     # Login to Azure Container Registry.
     - name: Login to Azure Container Registry

diff --git a/.github/workflows/docker_compose_tests.yml b/.github/workflows/docker_compose_tests.yml
@@ -0,0 +1,15 @@
+name: docker compose tests
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  basic-redis-e2e-tests-docker-compose:
+    runs-on: ubuntu-latest
+    name: basic redis e2e tests on docker compose
+    steps:
+      - uses: actions/checkout@v2
+      - name: test docker compose
+        run: ./infra/scripts/test-docker-compose.sh
diff --git a/.github/workflows/unit-tests.yml → .github/workflows/unit_tests.yml b/.github/workflows/unit-tests.yml → .github/workflows/unit_tests.yml
@@ -8,22 +8,26 @@ jobs:
     container: gcr.io/kf-feast/feast-ci:latest
     name: unit test java
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions/cache@v1
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: test java
-        run: make test-java
+        run: make test-java-with-coverage
+      - uses: actions/upload-artifact@v2
+        with:
+          name: java-coverage-report
+          path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/
 
   unit-test-python:
     runs-on: ubuntu-latest
     container: gcr.io/kf-feast/feast-ci:latest
     name: unit test python
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: install python
         run: make install-python
       - name: test python
@@ -34,7 +38,7 @@ jobs:
     container: gcr.io/kf-feast/feast-ci:latest
     name: unit test go
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: install dependencies
         run: make compile-protos-go
       - name: test go

diff --git a/.gitignore b/.gitignore
@@ -179,11 +179,12 @@ dmypy.json
 .flattened-pom.xml
 
 sdk/python/docs/html
-sdk/python/feast/core/
-sdk/python/feast/serving/
-sdk/python/feast/storage/
-sdk/python/feast/types/
-sdk/python/tensorflow_metadata
+
+# Generated python code
+*_pb2.py
+*_pb2.pyi
+*_pb2_grpc.py
+
 
 ### Farfetch ###
 

diff --git a/.helmdocsignore b/.helmdocsignore
@@ -0,0 +1,6 @@
+infra/charts/feast/charts/postgresql
+infra/charts/feast/charts/kafka
+infra/charts/feast/charts/redis
+infra/charts/feast/charts/prometheus-statsd-exporter
+infra/charts/feast/charts/prometheus
+infra/charts/feast/charts/grafana
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -41,7 +41,7 @@ deck:
 tide:
   queries:
   - repos:
-    - gojek/feast
+    - feast-dev/feast
     labels:
     - lgtm
     - approved
@@ -51,16 +51,17 @@ tide:
     - do-not-merge/invalid-owners-file
     - do-not-merge/work-in-progress
     - needs-rebase
+    - needs-kind
   merge_method:
-    gojek/feast: squash
+    feast-dev/feast: squash
   blocker_label: merge-blocker
   squash_label: tide/squash
 
 # presubmits and postsubmits configure ProwJobs:
 # https://github.com/kubernetes/test-infra/blob/6571843b1aa7bd6cf577a7a8b9e9971241f424d5/prow/jobs.md
 
 presubmits:
-  gojek/feast:
+  feast-dev/feast:
   - name: test-core-and-ingestion
     decorate: true
     spec:
@@ -142,6 +143,7 @@ presubmits:
 
   - name: test-end-to-end
     decorate: true
+    always_run: true
     spec:
       containers:
       - image: maven:3.6-jdk-11
@@ -153,6 +155,20 @@ presubmits:
     skip_branches:
     - ^v0\.(3|4)-branch$
 
+  - name: test-end-to-end-redis-cluster
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+        - image: maven:3.6-jdk-11
+          command: ["infra/scripts/test-end-to-end-redis-cluster.sh"]
+          resources:
+            requests:
+              cpu: "6"
+              memory: "6144Mi"
+    skip_branches:
+      - ^v0\.(3|4)-branch$
+
   - name: test-end-to-end-java-8
     decorate: true
     always_run: true
@@ -169,6 +185,7 @@ presubmits:
 
   - name: test-end-to-end-batch
     decorate: true
+    always_run: true
     spec:
       volumes:
       - name: service-account
@@ -187,6 +204,27 @@ presubmits:
     skip_branches:
     - ^v0\.(3|4)-branch$
 
+  - name: test-end-to-end-batch-fs-update
+    decorate: true
+    always_run: false
+    spec:
+      volumes:
+        - name: service-account
+          secret:
+            secretName: feast-service-account
+      containers:
+        - image: maven:3.6-jdk-11
+          command: ["infra/scripts/test-end-to-end-batch.sh", "-m", "fs_update"]
+          resources:
+            requests:
+              cpu: "6"
+              memory: "6144Mi"
+          volumeMounts:
+            - name: service-account
+              mountPath: "/etc/service-account"
+    skip_branches:
+      - ^v0\.(3|4)-branch$
+
   - name: test-end-to-end-batch-java-8
     decorate: true
     always_run: true
@@ -208,8 +246,79 @@ presubmits:
     branches:
     - ^v0\.(3|4)-branch$
 
+  - name: test-end-to-end-batch-dataflow
+    decorate: true
+    always_run: false
+    spec:
+      volumes:
+      - name: service-account-df
+        secret:
+          secretName: feast-e2e-service-account
+      - name: docker-socket
+          hostPath:
+            path: /var/run/docker.sock
+      containers:
+      - image: google/cloud-sdk:273.0.0
+        command: ["infra/scripts/test-end-to-end-batch-dataflow.sh"]
+        resources:
+          requests:
+            cpu: "6"
+            memory: "6144Mi"
+        volumeMounts:
+        - name: service-account-df
+          mountPath: "/etc/service-account-df"
+        - name: docker-socket
+          mountPath: /var/run/docker.sock
+        securityContext:
+          privileged: true
+    skip_branches:
+    - ^v0\.(3|4)-branch$
+
+  - name: publish-docker-images
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+      - image: google/cloud-sdk:273.0.0
+        command:
+        - bash
+        - -c
+        - |
+          infra/scripts/download-maven-cache.sh \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --output-dir $PWD/
+
+          infra/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-core \
+            --tag ${PULL_PULL_SHA:1} \
+            --file infra/docker/core/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+          infra/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-serving \
+            --tag ${PULL_PULL_SHA:1} \
+            --file infra/docker/serving/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+        volumeMounts:
+        - name: docker-socket
+          mountPath: /var/run/docker.sock
+        - name: service-account
+          mountPath: /etc/gcloud/service-account.json
+          subPath: service-account.json
+          readOnly: true
+        securityContext:
+          privileged: true
+      volumes:
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+      - name: service-account
+        secret:
+          secretName: feast-service-account
+
 postsubmits:
-  gojek/feast:
+  feast-dev/feast:
   - name: publish-python-sdk
     decorate: true
     spec:
@@ -219,7 +328,7 @@ postsubmits:
         - sh
         - -c
         - |
-          infra/scripts/publish-python-sdk.sh \
+          make compile-protos-python && infra/scripts/publish-python-sdk.sh \
             --directory-path sdk/python --repository pypi
         volumeMounts:
         - name: pypirc

diff --git a/.prow/plugins.yaml b/.prow/plugins.yaml
@@ -1,5 +1,5 @@
 plugins:
-  gojek/feast:
+  feast-dev/feast:
   - approve      
   - assign
   - help
@@ -12,14 +12,22 @@ plugins:
   - wip         
   - trigger
   - config-updater
+  - require-matching-label
 
 config_updater:
   maps:
     .prow/config.yaml:
       name: config
 
 external_plugins:
-  gojek/feast:
+  feast-dev/feast:
   - name: needs-rebase
     events:
     - pull_request
+
+require_matching_label:
+- missing_label: needs-kind
+  org: feast-dev
+  repo: feast
+  prs: true
+  regexp: ^kind/