Update ingestion to write feature metrics for validation

.prow/scripts/test-end-to-end-batch.sh

@@ -248,7 +248,7 @@ ORIGINAL_DIR=$(pwd)
 cd tests/e2e
 
 set +e
-pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest bigquery_batch_retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
 TEST_EXIT_CODE=$?
 
 if [[ ${TEST_EXIT_CODE} != 0 ]]; then

.prow/scripts/test-end-to-end.sh

@@ -28,7 +28,7 @@ echo "
 Installing Redis at localhost:6379
 ============================================================
 "
-# Allow starting serving in this Maven Docker image. Default set to not allowed.
+# Allow starting Linux services in this Maven Docker image. Default set to not allowed.
 echo "exit 0" > /usr/sbin/policy-rc.d
 apt-get -y install redis-server > /var/log/redis.install.log
 redis-server --daemonize yes
@@ -57,15 +57,65 @@ Installing Kafka at localhost:9092
 ============================================================
 "
 wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz
-mv kafka_2.12-2.3.0/ /tmp/kafka
-nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 &
+mv kafka_2.12-2.3.0/ /opt/kafka
+nohup /opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 &
 sleep 5
 tail -n10 /var/log/zookeeper.log
-nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 &
+nohup /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties &> /var/log/kafka.log 2>&1 &
 sleep 20
 tail -n10 /var/log/kafka.log
 kafkacat -b localhost:9092 -L
 
+echo "
+============================================================
+Installing Telegraf with StatsD input and Prometheus output
+Installing Prometheus that scrapes metrics from Telegraf
+============================================================
+"
+
+echo "Downloading Telegraf ..."
+wget -O- https://dl.influxdata.com/telegraf/releases/telegraf-1.13.2_linux_amd64.tar.gz | tar xz && \
+mv telegraf /opt/telegraf
+
+cat <<EOF > /tmp/telegraf.conf
+[agent]
+  interval = "10s"
+  round_interval = true
+
+[[inputs.statsd]]
+  protocol = "udp"
+  service_address = ":8125"
+  datadog_extensions = true
+  delete_counters = false
+
+[[outputs.prometheus_client]]
+  listen = ":9273"
+  collectors_exclude = ["gocollector", "process"]
+EOF
+nohup /opt/telegraf/usr/bin/telegraf --config /tmp/telegraf.conf &> /var/log/telegraf.log &
+
+echo "Downloading Prometheus ..."
+wget -qO- https://github.com/prometheus/prometheus/releases/download/v2.15.2/prometheus-2.15.2.linux-amd64.tar.gz | tar xz
+mv prometheus-2.15.2.linux-amd64 /opt/prometheus
+
+cat <<EOF > /tmp/prometheus.yml
+global:
+  scrape_interval:     10s
+  evaluation_interval: 10s
+
+scrape_configs:
+- job_name: "prometheus"
+  static_configs:
+  - targets: ["localhost:9273"]
+EOF
+nohup /opt/prometheus/prometheus --config.file=/tmp/prometheus.yml &> /var/log/prometheus.log &
+
+sleep 3
+echo -e "\nTelegraf logs:"
+tail -n5 /var/log/telegraf.log
+echo -e "\nPrometheus logs:"
+tail -n5 /var/log/prometheus.log
+
 if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
   echo "
   ============================================================
@@ -105,7 +155,10 @@ feast:
     updates:
       timeoutSeconds: 240
     metrics:
-      enabled: false
+      enabled: true
+      type: statsd
+      host: localhost
+      port: 8125
 
   stream:
     type: kafka
@@ -184,7 +237,6 @@ grpc:
 spring:
   main:
     web-environment: false
-
 EOF
 
 nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \
@@ -201,7 +253,7 @@ Installing Python 3.7 with Miniconda and Feast SDK
 "
 # Install Python 3.7 with Miniconda
 wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \
-   -O /tmp/miniconda.sh
+  -O /tmp/miniconda.sh
 bash /tmp/miniconda.sh -b -p /root/miniconda -f
 /root/miniconda/bin/conda init
 source ~/.bashrc
@@ -222,7 +274,8 @@ ORIGINAL_DIR=$(pwd)
 cd tests/e2e
 
 set +e
-pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest basic_ingest_redis_serving.py --prometheus_server_url=http://localhost:9090 \
+  --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
 TEST_EXIT_CODE=$?
 
 if [[ ${TEST_EXIT_CODE} != 0 ]]; then

core/src/main/java/feast/core/http/HealthController.java

@@ -29,7 +29,9 @@
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
-/** Web http for pod health-check endpoints. */
+/**
+ * Web http for pod health-check endpoints.
+ */
 @Slf4j
 @RestController
 public class HealthController {
@@ -64,7 +66,7 @@ public ResponseEntity healthz() {
       return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
           .body("Unable to establish connection with DB");
     } catch (SQLException e) {
-      log.error("Unable to reach DB: {}", e);
+      log.error(String.format("Unable to reach DB: %s", e.getMessage()));
       return ResponseEntity.status(INTERNAL_SERVER_ERROR).body(e.getMessage());
     }
   }

core/src/main/java/feast/core/model/Field.java

@@ -19,6 +19,7 @@
 import feast.core.FeatureSetProto.EntitySpec;
 import feast.core.FeatureSetProto.FeatureSpec;
 import feast.types.ValueProto.ValueType;
+import java.util.Arrays;
 import java.util.Objects;
 import javax.persistence.Column;
 import javax.persistence.Embeddable;
@@ -216,15 +217,38 @@ public Field(EntitySpec entitySpec) {
   }
 
   @Override
-  public boolean equals(Object o) {
-    if (this == o) {
+  public boolean equals(Object thatObject) {
+    if (this == thatObject) {
       return true;
     }
-    if (o == null || getClass() != o.getClass()) {
+
+    if (thatObject == null || this.getClass() != thatObject.getClass()) {
       return false;
     }
-    Field field = (Field) o;
-    return name.equals(field.getName()) && type.equals(field.getType());
+
+    Field that = (Field) thatObject;
+
+    // "this" field is equal to "that" field if all the properties (except version) have the same values.
+    // Note that Objects.equals(a,b) handles "null" String as well.
+    return Objects.equals(this.name, that.name) &&
+        Objects.equals(this.type, that.type) &&
+        Objects.equals(this.project, that.project) &&
+        Arrays.equals(this.presence, that.presence) &&
+        Arrays.equals(this.groupPresence, that.groupPresence) &&
+        Arrays.equals(this.shape, that.shape) &&
+        Arrays.equals(this.valueCount, that.valueCount) &&
+        Objects.equals(this.domain, that.domain) &&
+        Arrays.equals(this.intDomain, that.intDomain) &&
+        Arrays.equals(this.floatDomain, that.floatDomain) &&
+        Arrays.equals(this.stringDomain, that.stringDomain) &&
+        Arrays.equals(this.boolDomain, that.boolDomain) &&
+        Arrays.equals(this.structDomain, that.structDomain) &&
+        Arrays.equals(this.naturalLanguageDomain, that.naturalLanguageDomain) &&
+        Arrays.equals(this.imageDomain, that.imageDomain) &&
+        Arrays.equals(this.midDomain, that.midDomain) &&
+        Arrays.equals(this.urlDomain, that.urlDomain) &&
+        Arrays.equals(this.timeDomain, that.timeDomain) &&
+        Arrays.equals(this.timeOfDayDomain, that.timeOfDayDomain);
   }
 
   @Override

core/src/test/java/feast/core/service/SpecServiceTest.java

@@ -596,7 +596,8 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
 
     // appliedFeatureSpecs needs to be sorted because the list returned by specService may not
     // follow the order in the request
-    List<FeatureSpec> appliedFeatureSpecs = new ArrayList<>(appliedFeatureSetSpec.getFeaturesList());
+    List<FeatureSpec> appliedFeatureSpecs = new ArrayList<>(
+        appliedFeatureSetSpec.getFeaturesList());
     appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName));
 
     assertEquals(appliedEntitySpecs.size(), entitySpecs.size());
@@ -611,6 +612,31 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
     }
   }
 
+  @Test
+  public void applyFeatureSetShouldUpdateExistingFeatureSetWhenConstraintsAreUpdated()
+      throws InvalidProtocolBufferException {
+    FeatureSetProto.FeatureSet existingFeatureSet = featureSets.get(2).toProto();
+    assertThat("Existing feature set has version 3",
+        existingFeatureSet.getSpec().getVersion() == 3);
+    assertThat("Existing feature set has at least 1 feature",
+        existingFeatureSet.getSpec().getFeaturesList().size() > 0);
+
+    // New FeatureSetSpec with IntDomain
+    FeatureSpec newFeatureSpec = existingFeatureSet.getSpec().getFeatures(0).toBuilder()
+        .setIntDomain(IntDomain.newBuilder().setMin(5)).build();
+    FeatureSetSpec newFeatureSetSpec = existingFeatureSet.getSpec().toBuilder()
+        .setFeatures(0, newFeatureSpec).build();
+    FeatureSetProto.FeatureSet newFeatureSet = existingFeatureSet.toBuilder()
+        .setSpec(newFeatureSetSpec).build();
+
+    ApplyFeatureSetResponse response = specService.applyFeatureSet(newFeatureSet);
+    assertEquals("Response should have CREATED status", Status.CREATED, response.getStatus());
+    assertEquals("Response FeatureSet should have new version",
+        4, response.getFeatureSet().getSpec().getVersion());
+    assertEquals("Response should have IntDomain value set",
+        5, response.getFeatureSet().getSpec().getFeatures(0).getIntDomain().getMin());
+  }
+
   @Test
   public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException {
     when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0)));

examples/basic/basic.ipynb

@@ -500,7 +500,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.4"
+   "version": "3.7.6"
   },
   "pycharm": {
    "stem_cell": {

examples/data_validation/bikeshare_stations.csv

@@ -0,0 +1,97 @@
+station_id,name,status,latitude,longitude,location
+3793,Rio Grande & 28th,active,30.29333,-97.74412,"(30.29333, -97.74412)"
+3291,11th & San Jacinto,active,30.27193,-97.73854,"(30.27193, -97.73854)"
+4058,Hollow Creek & Barton Hills,active,30.26139,-97.77234,"(30.26139, -97.77234)"
+3797,21st & University,active,30.28354,-97.73953,"(30.28354, -97.73953)"
+3838,Nueces & 26th,active,30.29068,-97.74292,"(30.29068, -97.74292)"
+2544,East 6th & Pedernales St.,active,30.25895,-97.71475,"(30.25895, -97.71475)"
+3390,Brazos & 6th,active,30.26754,-97.74154,"(30.26754, -97.74154)"
+2823,Capital Metro HQ - East 5th at Broadway,active,30.2563,-97.71007,"(30.2563, -97.71007)"
+2711,Barton Springs @ Kinney Ave,active,30.262,-97.76118,"(30.262, -97.76118)"
+3685,Henderson & 9th,active,30.27217,-97.75246,"(30.27217, -97.75246)"
+4052,Rosewood & Angelina,active,30.26888,-97.72431,"(30.26888, -97.72431)"
+3621,Nueces & 3rd,active,30.26697,-97.74929,"(30.26697, -97.74929)"
+3686,Sterzing at Barton Springs,active,30.26406,-97.76385,"(30.26406, -97.76385)"
+3684,Congress & Cesar Chavez,active,30.26332,-97.74508,"(30.26332, -97.74508)"
+4055,11th & Salina,active,30.26638,-97.7214,"(30.26638, -97.7214)"
+2566,Pfluger Bridge @ W 2nd Street,active,30.26717,-97.75484,"(30.26717, -97.75484)"
+4059,Nash Hernandez @ RBJ South,active,30.25189,-97.73323,"(30.25189, -97.73323)"
+3635,13th & San Antonio,active,30.27616,-97.74488,"(30.27616, -97.74488)"
+3513,South Congress & Barton Springs at the Austin American-Statesman,active,30.25839,-97.74592,"(30.25839, -97.74592)"
+4050,5th & Campbell,active,30.27489,-97.76483,"(30.27489, -97.76483)"
+2499,City Hall / Lavaca & 2nd,active,30.26476,-97.74678,"(30.26476, -97.74678)"
+2568,East 11th St. at Victory Grill,active,30.26896,-97.72843,"(30.26896, -97.72843)"
+2575,Riverside @ S. Lamar,active,30.26446,-97.75665,"(30.26446, -97.75665)"
+2502,Barton Springs & Riverside,active,30.2587,-97.74872,"(30.2587, -97.74872)"
+2563,Davis at Rainey Street,active,30.26019,-97.73845,"(30.26019, -97.73845)"
+4061,Lakeshore @ Austin Hostel,active,30.24472,-97.72336,"(30.24472, -97.72336)"
+2707,Rainey St @ Cummings,active,30.25579,-97.73982,"(30.25579, -97.73982)"
+2549,Long Center @ South 1st & Riverside,active,30.25941,-97.74971,"(30.25941, -97.74971)"
+2561,State Capitol Visitors Garage @ San Jacinto & 12th,active,30.27336,-97.73805,"(30.27336, -97.73805)"
+2494,2nd & Congress,active,30.26408,-97.74355,"(30.26408, -97.74355)"
+2570,South Congress & Academy,active,30.25226,-97.74854,"(30.25226, -97.74854)"
+2540,17th & Guadalupe,active,30.27974,-97.74254,"(30.27974, -97.74254)"
+2498,Convention Center / 4th St. @ MetroRail,active,30.26483,-97.739,"(30.26483, -97.739)"
+3794,Dean Keeton & Speedway,active,30.28953,-97.73695,"(30.28953, -97.73695)"
+2547,Guadalupe & 21st,active,30.28395,-97.74198,"(30.28395, -97.74198)"
+2501,5th & Bowie,active,30.2696,-97.75332,"(30.2696, -97.75332)"
+2572,Barton Springs Pool,active,30.26452,-97.7712,"(30.26452, -97.7712)"
+2567,Palmer Auditorium,active,30.25971,-97.75346,"(30.25971, -97.75346)"
+3799,23rd & San Jacinto @ DKR Stadium,active,30.2856,-97.7335,"(30.2856, -97.7335)"
+3660,Medina & East 6th,active,30.26455,-97.73165,"(30.26455, -97.73165)"
+3791,Lake Austin & Enfield,active,30.29439,-97.78375,"(30.29439, -97.78375)"
+4057,6th & Chalmers,active,30.26269,-97.72438,"(30.26269, -97.72438)"
+2537,West & 6th St.,active,30.27041,-97.75046,"(30.27041, -97.75046)"
+2503,South Congress & James,active,30.25103,-97.74926,"(30.25103, -97.74926)"
+3792,22nd & Pearl,active,30.2853,-97.7467,"(30.2853, -97.7467)"
+3293,East 2nd & Pedernales,active,30.25542,-97.71665,"(30.25542, -97.71665)"
+4062,Lakeshore & Pleasant Valley,active,30.24258,-97.71726,"(30.24258, -97.71726)"
+2571,Red River & 8th Street,active,30.26854,-97.73646,"(30.26854, -97.73646)"
+3798,21st & Speedway @PCL,active,30.283,-97.7375,"(30.283, -97.7375)"
+2548,UT West Mall @ Guadalupe,active,30.28576,-97.74181,"(30.28576, -97.74181)"
+3795,Dean Keeton & Whitis,active,30.2898,-97.74041,"(30.2898, -97.74041)"
+2542,Plaza Saltillo,active,30.26217,-97.72743,"(30.26217, -97.72743)"
+3619,6th & Congress,active,30.26822,-97.74285,"(30.26822, -97.74285)"
+4048,South Congress @ Bouldin Creek,active,30.25495,-97.74755,"(30.25495, -97.74755)"
+3790,Lake Austin Blvd @ Deep Eddy,active,30.27807,-97.77272,"(30.27807, -97.77272)"
+2495,4th & Congress,active,30.26634,-97.74378,"(30.26634, -97.74378)"
+3841,23rd & Rio Grande,active,30.28728,-97.74495,"(30.28728, -97.74495)"
+2562,San Jacinto & 8th Street,active,30.26912,-97.73986,"(30.26912, -97.73986)"
+2822,East 6th at Robert Martinez,active,30.26032,-97.71899,"(30.26032, -97.71899)"
+4047,8th & Lavaca,active,30.27059,-97.74441,"(30.27059, -97.74441)"
+3292,East 4th & Chicon,active,30.25987,-97.72373,"(30.25987, -97.72373)"
+2497,Capitol Station / Congress & 11th,active,30.2726,-97.74127,"(30.2726, -97.74127)"
+3377,MoPac Pedestrian Bridge @ Veterans Drive,active,30.27466,-97.77028,"(30.27466, -97.77028)"
+3687,Boardwalk West,active,30.25457,-97.74258,"(30.25457, -97.74258)"
+2504,South Congress & Elizabeth,active,30.24891,-97.75019,"(30.24891, -97.75019)"
+1001,OFFICE/Main/Shop/Repair,active,30.27186,-97.73997,"(30.27186, -97.73997)"
+2574,Zilker Park,active,30.2659,-97.76822,"(30.2659, -97.76822)"
+2569,East 11th St. & San Marcos,active,30.26968,-97.73074,"(30.26968, -97.73074)"
+2539,Convention Center / 3rd & Trinity,active,30.26426,-97.74023,"(30.26426, -97.74023)"
+3294,Lavaca & 6th,active,30.26911,-97.7462,"(30.26911, -97.7462)"
+2496,8th & Congress,active,30.2698,-97.74186,"(30.2698, -97.74186)"
+2565,Trinity & 6th Street,active,30.26735,-97.73933,"(30.26735, -97.73933)"
+4054,Rosewood & Chicon,active,30.26969,-97.71873,"(30.26969, -97.71873)"
+2552,3rd & West,active,30.2678,-97.75189,"(30.2678, -97.75189)"
+3455,Republic Square @ 5th & Guadalupe,active,30.26753,-97.74805,"(30.26753, -97.74805)"
+4060,Red River/Cesar Chavez @ The Fairmont,active,30.26212,-97.73815,"(30.26212, -97.73815)"
+4051,10th & Red River,active,30.27024,-97.73578,"(30.27024, -97.73578)"
+2546,ACC - West & 12th Street,closed,30.27624,-97.74831,"(30.27624, -97.74831)"
+1008,Nueces @ 3rd,closed,30.26694,-97.74939,"(30.26694, -97.74939)"
+2550,Republic Square @ Guadalupe & 4th St.,closed,30.26774,-97.74692,"(30.26774, -97.74692)"
+2538,Bullock Museum @ Congress & MLK,closed,30.28039,-97.73809,"(30.28039, -97.73809)"
+1002,6th & Navasota St.,closed,30.26383,-97.72864,"(30.26383, -97.72864)"
+2541,State Capitol @ 14th & Colorado,closed,30.27654,-97.74155,"(30.27654, -97.74155)"
+2545,ACC - Rio Grande & 12th,closed,30.27595,-97.74739,"(30.27595, -97.74739)"
+3464,Pease Park,closed,30.28118,-97.75219,"(30.28118, -97.75219)"
+2712,Toomey Rd @ South Lamar,closed,30.26304,-97.75824,"(30.26304, -97.75824)"
+2576,Rainey @ River St,closed,30.25802,-97.7391,"(30.25802, -97.7391)"
+3381,East 7th & Pleasant Valley,closed,30.26025,-97.71002,"(30.26025, -97.71002)"
+1006,Zilker Park West,closed,30.26587,-97.76826,"(30.26587, -97.76826)"
+2500,Republic Square,closed,30.26751,-97.74802,"(30.26751, -97.74802)"
+2536,Waller & 6th St.,closed,30.26461,-97.73049,"(30.26461, -97.73049)"
+1007,Lavaca & 6th,closed,30.26889,-97.74525,"(30.26889, -97.74525)"
+1003,8th & Guadalupe,closed,30.27106,-97.74563,"(30.27106, -97.74563)"
+2564,5th & San Marcos,closed,30.26416,-97.73289,"(30.26416, -97.73289)"
+1004,Red River & LBJ Library,closed,30.2848,-97.72756,"(30.2848, -97.72756)"
+1005,State Parking Garage @ Brazos & 18th,closed,30.27907,-97.73715,"(30.27907, -97.73715)"