Fix Docker Compose Test timeouts (#728)

* Fix poll interface of docker compose interval causing timeout errors in test

The poll interval of Feast core  was set at 10ms.
As a result, feast core hogged CPU time from the rest of test code/serving,
resulting in nodeterministic timeouts. This changes the poll interval of
Feast Core in the Docker Compose test to 30s

* Remove old v0.4 unused config from docker-compose setup

* Update batch-serving.yml serving config to v0.5 format

* Split out core specific config in docker-compose's .env.sample into core.yml

* Add github badge for docker-compose tests status  in README

README.md

@@ -1,10 +1,12 @@
 # Feast - Feature Store for Machine Learning
 
 [![Unit Tests](https://github.com/feast-dev/feast/workflows/unit%20tests/badge.svg?branch=master)](https://github.com/feast-dev/feast/actions?query=workflow%3A%22unit+tests%22+branch%3Amaster)
+![Docker Compose Tests](https://github.com/feast-dev/feast/workflows/docker%20compose%20tests/badge.svg?branch=master)
 [![Code Standards](https://github.com/feast-dev/feast/workflows/code%20standards/badge.svg?branch=master)](https://github.com/feast-dev/feast/actions?query=workflow%3A%22code+standards%22+branch%3Amaster)
 [![Docs latest](https://img.shields.io/badge/Docs-latest-blue.svg)](https://docs.feast.dev/)
 [![GitHub Release](https://img.shields.io/github/release/feast-dev/feast.svg?style=flat)](https://github.com/feast-dev/feast/releases)
 
+
 ## Overview
 
 Feast (Feature Store) is a tool for managing and serving machine learning features. Feast is the bridge between models and data.

infra/docker-compose/.env.sample

@@ -5,21 +5,17 @@ FEAST_REPOSITORY_VERSION=v0.5-branch
 
 # Feast Core
 FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
-FEAST_CORE_CONFIG=direct-runner.yml
+FEAST_CORE_CONFIG=core.yml
 FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
-FEAST_STREAM_OPTIONS_BOOTSTRAPSERVERS=kafka:9092
-FEAST_JOBS_POLLING_INTERVAL_MILLISECONDS=10
 
-# Feast Serving Batch (BigQuery)
+# Feast Serving
+FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
+# Feast Serving - Batch (BigQuery)
 FEAST_BATCH_SERVING_CONFIG=batch-serving.yml
-FEAST_BATCH_STORE_CONFIG=bq-store.yml
 FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
 FEAST_BATCH_JOB_STAGING_LOCATION=gs://your-gcs-bucket/staging
-
-# Feast Serving Online (Redis)
-FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
+# Feast Serving - Online (Redis)
 FEAST_ONLINE_SERVING_CONFIG=online-serving.yml
-FEAST_ONLINE_STORE_CONFIG=redis-store.yml
 
 # Jupyter
-FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
+FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json

infra/docker-compose/core/core.yml

@@ -0,0 +1,14 @@
+feast:
+  jobs:
+    polling_interval_milliseconds: 30000
+    job_update_timeout_seconds: 240
+    active_runner: direct
+    runners:
+      - name: direct
+        type: DirectRunner
+        options: {}
+  stream:
+    type: kafka
+    options:
+      topic: feast-features
+      bootstrapServers: kafka:9092

infra/docker-compose/docker-compose.batch.yml

@@ -5,7 +5,6 @@ services:
     image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./serving/${FEAST_BATCH_SERVING_CONFIG}:/etc/feast/application.yml
-      - ./serving/${FEAST_BATCH_STORE_CONFIG}:/etc/feast/store.yml
       - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
     depends_on:
       - redis
@@ -26,4 +25,4 @@ services:
   redis:
     image: redis:5-alpine
     ports:
-      - "6379:6379"
+      - "6379:6379"

infra/docker-compose/docker-compose.online.yml

@@ -5,7 +5,6 @@ services:
     image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./serving/${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml
-      - ./serving/${FEAST_ONLINE_STORE_CONFIG}:/etc/feast/store.yml
     depends_on:
       - redis
     ports:
@@ -20,4 +19,4 @@ services:
   redis:
     image: redis:5-alpine
     ports:
-      - "6379:6379"
+      - "6379:6379"

infra/docker-compose/docker-compose.yml

@@ -9,8 +9,6 @@ services:
     environment:
       DB_HOST: db
       GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
-      FEAST_STREAM_OPTIONS_BOOTSTRAPSERVERS: ${FEAST_STREAM_OPTIONS_BOOTSTRAPSERVERS}
-      FEAST_JOBS_POLLING_INTERVAL_MILLISECONDS: ${FEAST_JOBS_POLLING_INTERVAL_MILLISECONDS}
     restart: on-failure
     depends_on:
       - db
@@ -66,4 +64,4 @@ services:
     environment:
       POSTGRES_PASSWORD: password
     ports:
-      - "5432:5432"
+      - "5432:5432"

infra/docker-compose/serving/batch-serving.yml

@@ -1,14 +1,20 @@
 feast:
   core-host: core
-  core-grpc-port: 6565
-  store:
-    config-path: /etc/feast/store.yml
-
-  jobs:
-    store-type: REDIS
-    store-options:
-      host: redis
-      port: 6379
-
-grpc:
-  port: 6567
+  active-store: batch
+  stores:
+    - name: historical
+      type: BIGQUERY
+      # Changes required for batch serving to work
+      # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options
+      config:
+        project_id: my_project
+        dataset_id: my_dataset
+        staging_location: gs://mybucket/myprefix
+        initial_retry_delay_seconds: 1
+        total_timeout_seconds: 21600
+      subscriptions:
+        - name: "*"
+          project: "*"
+  job_store:
+    redis_host: redis
+    redis_port: 6379