diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index 268bd248c1..dc5365d371 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -248,7 +248,7 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest bigquery_batch_retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index c436d2f690..6c9da06306 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -28,7 +28,7 @@ echo " Installing Redis at localhost:6379 ============================================================ " -# Allow starting serving in this Maven Docker image. Default set to not allowed. +# Allow starting Linux services in this Maven Docker image. Default set to not allowed. echo "exit 0" > /usr/sbin/policy-rc.d apt-get -y install redis-server > /var/log/redis.install.log redis-server --daemonize yes @@ -57,15 +57,65 @@ Installing Kafka at localhost:9092 ============================================================ " wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz -mv kafka_2.12-2.3.0/ /tmp/kafka -nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & +mv kafka_2.12-2.3.0/ /opt/kafka +nohup /opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & sleep 5 tail -n10 /var/log/zookeeper.log -nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & +nohup /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & sleep 20 tail -n10 /var/log/kafka.log kafkacat -b localhost:9092 -L +echo " +============================================================ +Installing Telegraf with StatsD input and Prometheus output +Installing Prometheus that scrapes metrics from Telegraf +============================================================ +" + +echo "Downloading Telegraf ..." +wget -O- https://dl.influxdata.com/telegraf/releases/telegraf-1.13.2_linux_amd64.tar.gz | tar xz && \ +mv telegraf /opt/telegraf + +cat < /tmp/telegraf.conf +[agent] + interval = "10s" + round_interval = true + +[[inputs.statsd]] + protocol = "udp" + service_address = ":8125" + datadog_extensions = true + delete_counters = false + +[[outputs.prometheus_client]] + listen = ":9273" + collectors_exclude = ["gocollector", "process"] +EOF +nohup /opt/telegraf/usr/bin/telegraf --config /tmp/telegraf.conf &> /var/log/telegraf.log & + +echo "Downloading Prometheus ..." +wget -qO- https://github.com/prometheus/prometheus/releases/download/v2.15.2/prometheus-2.15.2.linux-amd64.tar.gz | tar xz +mv prometheus-2.15.2.linux-amd64 /opt/prometheus + +cat < /tmp/prometheus.yml +global: + scrape_interval: 10s + evaluation_interval: 10s + +scrape_configs: +- job_name: "prometheus" + static_configs: + - targets: ["localhost:9273"] +EOF +nohup /opt/prometheus/prometheus --config.file=/tmp/prometheus.yml &> /var/log/prometheus.log & + +sleep 3 +echo -e "\nTelegraf logs:" +tail -n5 /var/log/telegraf.log +echo -e "\nPrometheus logs:" +tail -n5 /var/log/prometheus.log + if [[ ${SKIP_BUILD_JARS} != "true" ]]; then echo " ============================================================ @@ -105,7 +155,10 @@ feast: updates: timeoutSeconds: 240 metrics: - enabled: false + enabled: true + type: statsd + host: localhost + port: 8125 stream: type: kafka @@ -184,7 +237,6 @@ grpc: spring: main: web-environment: false - EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ @@ -201,7 +253,7 @@ Installing Python 3.7 with Miniconda and Feast SDK " # Install Python 3.7 with Miniconda wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \ - -O /tmp/miniconda.sh + -O /tmp/miniconda.sh bash /tmp/miniconda.sh -b -p /root/miniconda -f /root/miniconda/bin/conda init source ~/.bashrc @@ -222,7 +274,8 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest basic_ingest_redis_serving.py --prometheus_server_url=http://localhost:9090 \ + --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then diff --git a/core/src/main/java/feast/core/http/HealthController.java b/core/src/main/java/feast/core/http/HealthController.java index 2451ed793e..3efe6e6e8c 100644 --- a/core/src/main/java/feast/core/http/HealthController.java +++ b/core/src/main/java/feast/core/http/HealthController.java @@ -29,7 +29,9 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; -/** Web http for pod health-check endpoints. */ +/** + * Web http for pod health-check endpoints. + */ @Slf4j @RestController public class HealthController { @@ -64,7 +66,7 @@ public ResponseEntity healthz() { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body("Unable to establish connection with DB"); } catch (SQLException e) { - log.error("Unable to reach DB: {}", e); + log.error(String.format("Unable to reach DB: %s", e.getMessage())); return ResponseEntity.status(INTERNAL_SERVER_ERROR).body(e.getMessage()); } } diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index edb0a73acb..0067b48e3a 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -19,6 +19,7 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.types.ValueProto.ValueType; +import java.util.Arrays; import java.util.Objects; import javax.persistence.Column; import javax.persistence.Embeddable; @@ -216,15 +217,38 @@ public Field(EntitySpec entitySpec) { } @Override - public boolean equals(Object o) { - if (this == o) { + public boolean equals(Object thatObject) { + if (this == thatObject) { return true; } - if (o == null || getClass() != o.getClass()) { + + if (thatObject == null || this.getClass() != thatObject.getClass()) { return false; } - Field field = (Field) o; - return name.equals(field.getName()) && type.equals(field.getType()); + + Field that = (Field) thatObject; + + // "this" field is equal to "that" field if all the properties (except version) have the same values. + // Note that Objects.equals(a,b) handles "null" String as well. + return Objects.equals(this.name, that.name) && + Objects.equals(this.type, that.type) && + Objects.equals(this.project, that.project) && + Arrays.equals(this.presence, that.presence) && + Arrays.equals(this.groupPresence, that.groupPresence) && + Arrays.equals(this.shape, that.shape) && + Arrays.equals(this.valueCount, that.valueCount) && + Objects.equals(this.domain, that.domain) && + Arrays.equals(this.intDomain, that.intDomain) && + Arrays.equals(this.floatDomain, that.floatDomain) && + Arrays.equals(this.stringDomain, that.stringDomain) && + Arrays.equals(this.boolDomain, that.boolDomain) && + Arrays.equals(this.structDomain, that.structDomain) && + Arrays.equals(this.naturalLanguageDomain, that.naturalLanguageDomain) && + Arrays.equals(this.imageDomain, that.imageDomain) && + Arrays.equals(this.midDomain, that.midDomain) && + Arrays.equals(this.urlDomain, that.urlDomain) && + Arrays.equals(this.timeDomain, that.timeDomain) && + Arrays.equals(this.timeOfDayDomain, that.timeOfDayDomain); } @Override diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index c533f593e3..1f52ef5529 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -596,7 +596,8 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints() // appliedFeatureSpecs needs to be sorted because the list returned by specService may not // follow the order in the request - List appliedFeatureSpecs = new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); + List appliedFeatureSpecs = new ArrayList<>( + appliedFeatureSetSpec.getFeaturesList()); appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName)); assertEquals(appliedEntitySpecs.size(), entitySpecs.size()); @@ -611,6 +612,31 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints() } } + @Test + public void applyFeatureSetShouldUpdateExistingFeatureSetWhenConstraintsAreUpdated() + throws InvalidProtocolBufferException { + FeatureSetProto.FeatureSet existingFeatureSet = featureSets.get(2).toProto(); + assertThat("Existing feature set has version 3", + existingFeatureSet.getSpec().getVersion() == 3); + assertThat("Existing feature set has at least 1 feature", + existingFeatureSet.getSpec().getFeaturesList().size() > 0); + + // New FeatureSetSpec with IntDomain + FeatureSpec newFeatureSpec = existingFeatureSet.getSpec().getFeatures(0).toBuilder() + .setIntDomain(IntDomain.newBuilder().setMin(5)).build(); + FeatureSetSpec newFeatureSetSpec = existingFeatureSet.getSpec().toBuilder() + .setFeatures(0, newFeatureSpec).build(); + FeatureSetProto.FeatureSet newFeatureSet = existingFeatureSet.toBuilder() + .setSpec(newFeatureSetSpec).build(); + + ApplyFeatureSetResponse response = specService.applyFeatureSet(newFeatureSet); + assertEquals("Response should have CREATED status", Status.CREATED, response.getStatus()); + assertEquals("Response FeatureSet should have new version", + 4, response.getFeatureSet().getSpec().getVersion()); + assertEquals("Response should have IntDomain value set", + 5, response.getFeatureSet().getSpec().getFeatures(0).getIntDomain().getMin()); + } + @Test public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException { when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0))); diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb index 94fc82f2ce..0360fe05a3 100644 --- a/examples/basic/basic.ipynb +++ b/examples/basic/basic.ipynb @@ -500,7 +500,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.6" }, "pycharm": { "stem_cell": { diff --git a/examples/data_validation/bikeshare_stations.csv b/examples/data_validation/bikeshare_stations.csv new file mode 100644 index 0000000000..ef9f056518 --- /dev/null +++ b/examples/data_validation/bikeshare_stations.csv @@ -0,0 +1,97 @@ +station_id,name,status,latitude,longitude,location +3793,Rio Grande & 28th,active,30.29333,-97.74412,"(30.29333, -97.74412)" +3291,11th & San Jacinto,active,30.27193,-97.73854,"(30.27193, -97.73854)" +4058,Hollow Creek & Barton Hills,active,30.26139,-97.77234,"(30.26139, -97.77234)" +3797,21st & University,active,30.28354,-97.73953,"(30.28354, -97.73953)" +3838,Nueces & 26th,active,30.29068,-97.74292,"(30.29068, -97.74292)" +2544,East 6th & Pedernales St.,active,30.25895,-97.71475,"(30.25895, -97.71475)" +3390,Brazos & 6th,active,30.26754,-97.74154,"(30.26754, -97.74154)" +2823,Capital Metro HQ - East 5th at Broadway,active,30.2563,-97.71007,"(30.2563, -97.71007)" +2711,Barton Springs @ Kinney Ave,active,30.262,-97.76118,"(30.262, -97.76118)" +3685,Henderson & 9th,active,30.27217,-97.75246,"(30.27217, -97.75246)" +4052,Rosewood & Angelina,active,30.26888,-97.72431,"(30.26888, -97.72431)" +3621,Nueces & 3rd,active,30.26697,-97.74929,"(30.26697, -97.74929)" +3686,Sterzing at Barton Springs,active,30.26406,-97.76385,"(30.26406, -97.76385)" +3684,Congress & Cesar Chavez,active,30.26332,-97.74508,"(30.26332, -97.74508)" +4055,11th & Salina,active,30.26638,-97.7214,"(30.26638, -97.7214)" +2566,Pfluger Bridge @ W 2nd Street,active,30.26717,-97.75484,"(30.26717, -97.75484)" +4059,Nash Hernandez @ RBJ South,active,30.25189,-97.73323,"(30.25189, -97.73323)" +3635,13th & San Antonio,active,30.27616,-97.74488,"(30.27616, -97.74488)" +3513,South Congress & Barton Springs at the Austin American-Statesman,active,30.25839,-97.74592,"(30.25839, -97.74592)" +4050,5th & Campbell,active,30.27489,-97.76483,"(30.27489, -97.76483)" +2499,City Hall / Lavaca & 2nd,active,30.26476,-97.74678,"(30.26476, -97.74678)" +2568,East 11th St. at Victory Grill,active,30.26896,-97.72843,"(30.26896, -97.72843)" +2575,Riverside @ S. Lamar,active,30.26446,-97.75665,"(30.26446, -97.75665)" +2502,Barton Springs & Riverside,active,30.2587,-97.74872,"(30.2587, -97.74872)" +2563,Davis at Rainey Street,active,30.26019,-97.73845,"(30.26019, -97.73845)" +4061,Lakeshore @ Austin Hostel,active,30.24472,-97.72336,"(30.24472, -97.72336)" +2707,Rainey St @ Cummings,active,30.25579,-97.73982,"(30.25579, -97.73982)" +2549,Long Center @ South 1st & Riverside,active,30.25941,-97.74971,"(30.25941, -97.74971)" +2561,State Capitol Visitors Garage @ San Jacinto & 12th,active,30.27336,-97.73805,"(30.27336, -97.73805)" +2494,2nd & Congress,active,30.26408,-97.74355,"(30.26408, -97.74355)" +2570,South Congress & Academy,active,30.25226,-97.74854,"(30.25226, -97.74854)" +2540,17th & Guadalupe,active,30.27974,-97.74254,"(30.27974, -97.74254)" +2498,Convention Center / 4th St. @ MetroRail,active,30.26483,-97.739,"(30.26483, -97.739)" +3794,Dean Keeton & Speedway,active,30.28953,-97.73695,"(30.28953, -97.73695)" +2547,Guadalupe & 21st,active,30.28395,-97.74198,"(30.28395, -97.74198)" +2501,5th & Bowie,active,30.2696,-97.75332,"(30.2696, -97.75332)" +2572,Barton Springs Pool,active,30.26452,-97.7712,"(30.26452, -97.7712)" +2567,Palmer Auditorium,active,30.25971,-97.75346,"(30.25971, -97.75346)" +3799,23rd & San Jacinto @ DKR Stadium,active,30.2856,-97.7335,"(30.2856, -97.7335)" +3660,Medina & East 6th,active,30.26455,-97.73165,"(30.26455, -97.73165)" +3791,Lake Austin & Enfield,active,30.29439,-97.78375,"(30.29439, -97.78375)" +4057,6th & Chalmers,active,30.26269,-97.72438,"(30.26269, -97.72438)" +2537,West & 6th St.,active,30.27041,-97.75046,"(30.27041, -97.75046)" +2503,South Congress & James,active,30.25103,-97.74926,"(30.25103, -97.74926)" +3792,22nd & Pearl,active,30.2853,-97.7467,"(30.2853, -97.7467)" +3293,East 2nd & Pedernales,active,30.25542,-97.71665,"(30.25542, -97.71665)" +4062,Lakeshore & Pleasant Valley,active,30.24258,-97.71726,"(30.24258, -97.71726)" +2571,Red River & 8th Street,active,30.26854,-97.73646,"(30.26854, -97.73646)" +3798,21st & Speedway @PCL,active,30.283,-97.7375,"(30.283, -97.7375)" +2548,UT West Mall @ Guadalupe,active,30.28576,-97.74181,"(30.28576, -97.74181)" +3795,Dean Keeton & Whitis,active,30.2898,-97.74041,"(30.2898, -97.74041)" +2542,Plaza Saltillo,active,30.26217,-97.72743,"(30.26217, -97.72743)" +3619,6th & Congress,active,30.26822,-97.74285,"(30.26822, -97.74285)" +4048,South Congress @ Bouldin Creek,active,30.25495,-97.74755,"(30.25495, -97.74755)" +3790,Lake Austin Blvd @ Deep Eddy,active,30.27807,-97.77272,"(30.27807, -97.77272)" +2495,4th & Congress,active,30.26634,-97.74378,"(30.26634, -97.74378)" +3841,23rd & Rio Grande,active,30.28728,-97.74495,"(30.28728, -97.74495)" +2562,San Jacinto & 8th Street,active,30.26912,-97.73986,"(30.26912, -97.73986)" +2822,East 6th at Robert Martinez,active,30.26032,-97.71899,"(30.26032, -97.71899)" +4047,8th & Lavaca,active,30.27059,-97.74441,"(30.27059, -97.74441)" +3292,East 4th & Chicon,active,30.25987,-97.72373,"(30.25987, -97.72373)" +2497,Capitol Station / Congress & 11th,active,30.2726,-97.74127,"(30.2726, -97.74127)" +3377,MoPac Pedestrian Bridge @ Veterans Drive,active,30.27466,-97.77028,"(30.27466, -97.77028)" +3687,Boardwalk West,active,30.25457,-97.74258,"(30.25457, -97.74258)" +2504,South Congress & Elizabeth,active,30.24891,-97.75019,"(30.24891, -97.75019)" +1001,OFFICE/Main/Shop/Repair,active,30.27186,-97.73997,"(30.27186, -97.73997)" +2574,Zilker Park,active,30.2659,-97.76822,"(30.2659, -97.76822)" +2569,East 11th St. & San Marcos,active,30.26968,-97.73074,"(30.26968, -97.73074)" +2539,Convention Center / 3rd & Trinity,active,30.26426,-97.74023,"(30.26426, -97.74023)" +3294,Lavaca & 6th,active,30.26911,-97.7462,"(30.26911, -97.7462)" +2496,8th & Congress,active,30.2698,-97.74186,"(30.2698, -97.74186)" +2565,Trinity & 6th Street,active,30.26735,-97.73933,"(30.26735, -97.73933)" +4054,Rosewood & Chicon,active,30.26969,-97.71873,"(30.26969, -97.71873)" +2552,3rd & West,active,30.2678,-97.75189,"(30.2678, -97.75189)" +3455,Republic Square @ 5th & Guadalupe,active,30.26753,-97.74805,"(30.26753, -97.74805)" +4060,Red River/Cesar Chavez @ The Fairmont,active,30.26212,-97.73815,"(30.26212, -97.73815)" +4051,10th & Red River,active,30.27024,-97.73578,"(30.27024, -97.73578)" +2546,ACC - West & 12th Street,closed,30.27624,-97.74831,"(30.27624, -97.74831)" +1008,Nueces @ 3rd,closed,30.26694,-97.74939,"(30.26694, -97.74939)" +2550,Republic Square @ Guadalupe & 4th St.,closed,30.26774,-97.74692,"(30.26774, -97.74692)" +2538,Bullock Museum @ Congress & MLK,closed,30.28039,-97.73809,"(30.28039, -97.73809)" +1002,6th & Navasota St.,closed,30.26383,-97.72864,"(30.26383, -97.72864)" +2541,State Capitol @ 14th & Colorado,closed,30.27654,-97.74155,"(30.27654, -97.74155)" +2545,ACC - Rio Grande & 12th,closed,30.27595,-97.74739,"(30.27595, -97.74739)" +3464,Pease Park,closed,30.28118,-97.75219,"(30.28118, -97.75219)" +2712,Toomey Rd @ South Lamar,closed,30.26304,-97.75824,"(30.26304, -97.75824)" +2576,Rainey @ River St,closed,30.25802,-97.7391,"(30.25802, -97.7391)" +3381,East 7th & Pleasant Valley,closed,30.26025,-97.71002,"(30.26025, -97.71002)" +1006,Zilker Park West,closed,30.26587,-97.76826,"(30.26587, -97.76826)" +2500,Republic Square,closed,30.26751,-97.74802,"(30.26751, -97.74802)" +2536,Waller & 6th St.,closed,30.26461,-97.73049,"(30.26461, -97.73049)" +1007,Lavaca & 6th,closed,30.26889,-97.74525,"(30.26889, -97.74525)" +1003,8th & Guadalupe,closed,30.27106,-97.74563,"(30.27106, -97.74563)" +2564,5th & San Marcos,closed,30.26416,-97.73289,"(30.26416, -97.73289)" +1004,Red River & LBJ Library,closed,30.2848,-97.72756,"(30.2848, -97.72756)" +1005,State Parking Garage @ Brazos & 18th,closed,30.27907,-97.73715,"(30.27907, -97.73715)" diff --git a/examples/data_validation/working_with_schema.ipynb b/examples/data_validation/working_with_schema.ipynb new file mode 100644 index 0000000000..bc436824bc --- /dev/null +++ b/examples/data_validation/working_with_schema.ipynb @@ -0,0 +1,736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Validation in Feast\n", + "\n", + "Feast allows users to specify a **schema** that specifies the value, shape and presence constraints \n", + "of the features they are ingesting. This schema is compatible with the schema defined in Tensorflow\n", + "metadata.\n", + "\n", + "https://github.com/tensorflow/metadata/blob/master/tensorflow_metadata/proto/v0/schema.proto.\n", + "\n", + "An existing Tensorflow metadata schema can be imported into Feast. Subsequently, Feast can\n", + "check that the features ingested follow the schema provided. In Feast v0.5, feature\n", + "value domains and presence will be validated during ingestion.\n", + "\n", + "For more information regarding Tensorflow data validation, please check these documentations:\n", + "- https://www.tensorflow.org/tfx/data_validation/get_started\n", + "- https://colab.research.google.com/github/tensorflow/tfx/blob/master/docs/tutorials/data_validation/tfdv_basic.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Importing Tensorflow metadata schema to Feast" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from feast import Client, FeatureSet\n", + "import tensorflow_data_validation as tfdv\n", + "from google.protobuf import text_format\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "# Sample data from BigQuery public dataset: bikeshare stations\n", + "# https://cloud.google.com/bigquery/public-data\n", + "wget https://raw.githubusercontent.com/davidheryanto/feast/update-ingestion-metrics-for-validation/examples/data_validation/bikeshare_stations.csv\n", + "ls *.csv" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
station_idnamestatuslatitudelongitudelocation
03793Rio Grande & 28thactive30.29333-97.74412(30.29333, -97.74412)
1329111th & San Jacintoactive30.27193-97.73854(30.27193, -97.73854)
24058Hollow Creek & Barton Hillsactive30.26139-97.77234(30.26139, -97.77234)
\n", + "
" + ], + "text/plain": [ + " station_id name status latitude longitude \\\n", + "0 3793 Rio Grande & 28th active 30.29333 -97.74412 \n", + "1 3291 11th & San Jacinto active 30.27193 -97.73854 \n", + "2 4058 Hollow Creek & Barton Hills active 30.26139 -97.77234 \n", + "\n", + " location \n", + "0 (30.29333, -97.74412) \n", + "1 (30.27193, -97.73854) \n", + "2 (30.26139, -97.77234) " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.read_csv(\"bikeshare_stations.csv\").head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cat < bikeshare_stations_feature_set.yaml\n", + "\n", + "spec:\n", + " name: bikeshare_stations\n", + " entities:\n", + " - name: station_id\n", + " valueType: INT64\n", + " features:\n", + " - name: name\n", + " valueType: STRING\n", + " - name: status\n", + " valueType: STRING\n", + " - name: latitude\n", + " valueType: FLOAT\n", + " - name: longitude\n", + " valueType: FLOAT\n", + " - name: location\n", + " valueType: STRING\n", + " maxAge: 3600s\n", + "\n", + "EOF" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"spec\": {\n", + " \"name\": \"bikeshare_stations\",\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"station_id\",\n", + " \"valueType\": \"INT64\"\n", + " }\n", + " ],\n", + " \"features\": [\n", + " {\n", + " \"name\": \"name\",\n", + " \"valueType\": \"STRING\"\n", + " },\n", + " {\n", + " \"name\": \"status\",\n", + " \"valueType\": \"STRING\"\n", + " },\n", + " {\n", + " \"name\": \"latitude\",\n", + " \"valueType\": \"FLOAT\"\n", + " },\n", + " {\n", + " \"name\": \"longitude\",\n", + " \"valueType\": \"FLOAT\"\n", + " },\n", + " {\n", + " \"name\": \"location\",\n", + " \"valueType\": \"STRING\"\n", + " }\n", + " ],\n", + " \"maxAge\": \"3600s\"\n", + " },\n", + " \"meta\": {\n", + " \"createdTimestamp\": \"1970-01-01T00:00:00Z\"\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "# Create a FeatureSet bikeshare_stations\n", + "bikeshare_stations_feature_set = FeatureSet.from_yaml(\"bikeshare_stations_feature_set.yaml\")\n", + "print(bikeshare_stations_feature_set)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n", + "/home/dheryanto/miniconda3/envs/feast/lib/python3.7/site-packages/tensorflow_data_validation/arrow/arrow_util.py:214: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", + " types.FeaturePath([column_name]), column.data.chunk(0), weights):\n", + "WARNING:apache_beam.io.tfrecordio:Couldn't find python-snappy so the implementation of _TFRecordUtil._masked_crc32c is not as fast as it could be.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/dheryanto/miniconda3/envs/feast/lib/python3.7/site-packages/tensorflow_data_validation/utils/stats_gen_lib.py:366: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use eager execution and: \n", + "`tf.data.TFRecordDataset(path)`\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/dheryanto/miniconda3/envs/feast/lib/python3.7/site-packages/tensorflow_data_validation/utils/stats_gen_lib.py:366: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use eager execution and: \n", + "`tf.data.TFRecordDataset(path)`\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TypePresenceValencyDomain
Feature name
'station_id'INTrequired-
'name'BYTESrequired-
'status'STRINGrequired'status'
'latitude'FLOATrequired-
'longitude'FLOATrequired-
'location'BYTESrequired-
\n", + "
" + ], + "text/plain": [ + " Type Presence Valency Domain\n", + "Feature name \n", + "'station_id' INT required -\n", + "'name' BYTES required -\n", + "'status' STRING required 'status'\n", + "'latitude' FLOAT required -\n", + "'longitude' FLOAT required -\n", + "'location' BYTES required -" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Values
Domain
'status''active', 'closed'
\n", + "
" + ], + "text/plain": [ + " Values\n", + "Domain \n", + "'status' 'active', 'closed'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Use Tensorflow Data Validation (tfdv) to create a schema from the csv\n", + "train_stats = tfdv.generate_statistics_from_csv(data_location=\"bikeshare_stations.csv\")\n", + "schema = tfdv.infer_schema(statistics=train_stats, max_string_domain_size=10)\n", + "tfdv.display_schema(schema=schema)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"spec\": {\n", + " \"name\": \"bikeshare_stations\",\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"station_id\",\n", + " \"valueType\": \"INT64\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " }\n", + " ],\n", + " \"features\": [\n", + " {\n", + " \"name\": \"name\",\n", + " \"valueType\": \"STRING\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"status\",\n", + " \"valueType\": \"STRING\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " \"stringDomain\": {\n", + " \"name\": \"status\",\n", + " \"value\": [\n", + " \"active\",\n", + " \"closed\"\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"latitude\",\n", + " \"valueType\": \"FLOAT\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"longitude\",\n", + " \"valueType\": \"FLOAT\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"location\",\n", + " \"valueType\": \"STRING\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " }\n", + " ],\n", + " \"maxAge\": \"3600s\"\n", + " },\n", + " \"meta\": {\n", + " \"createdTimestamp\": \"1970-01-01T00:00:00Z\"\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "# Import the schema into the FeatureSet\n", + "bikeshare_stations_feature_set.import_tfx_schema(schema)\n", + "print(bikeshare_stations_feature_set)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that the FeatureSet has imported the schema, Prometheus metrics will be exported during ingestion, which\n", + "can be used to check if the features ingested fulfill the requirements." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Exporting Tensorflow metadata schema from Feast\n", + "\n", + "The following scenario is for users who have created a FeatureSet and used Feast to ingest features. During training,\n", + "they want to run batch validation using Tensorflow data validation utility. Rather than attempting to recreate the\n", + "schema from scratch, users can export the existing one from the FeatureSet.\n", + "\n", + "This ensures that the schema that is currently applied for Feast ingestion will be consistent to the one used in\n", + "batch validation with Tensorflow data validation." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TypePresenceValencyDomain
Feature name
'name'BYTESrequired-
'status'STRINGrequired'status'
'latitude'FLOATrequired-
'longitude'FLOATrequired-
'location'BYTESrequired-
'station_id'INTrequired-
\n", + "
" + ], + "text/plain": [ + " Type Presence Valency Domain\n", + "Feature name \n", + "'name' BYTES required - \n", + "'status' STRING required 'status'\n", + "'latitude' FLOAT required - \n", + "'longitude' FLOAT required - \n", + "'location' BYTES required - \n", + "'station_id' INT required - " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Values
Domain
'status''active', 'closed'
\n", + "
" + ], + "text/plain": [ + " Values\n", + "Domain \n", + "'status' 'active', 'closed'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "exported_tfx_schema = bikeshare_stations_feature_set.export_tfx_schema()\n", + "tfdv.display_schema(exported_tfx_schema)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/infra/charts/feast/charts/feast-core/charts/grafana-4.6.3.tgz b/infra/charts/feast/charts/feast-core/charts/grafana-4.6.3.tgz new file mode 100644 index 0000000000..279d7f509b Binary files /dev/null and b/infra/charts/feast/charts/feast-core/charts/grafana-4.6.3.tgz differ diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-10.4.0.tgz b/infra/charts/feast/charts/feast-core/charts/prometheus-10.4.0.tgz new file mode 100644 index 0000000000..23712b6c8c Binary files /dev/null and b/infra/charts/feast/charts/feast-core/charts/prometheus-10.4.0.tgz differ diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml index 0f9de1e953..fc27a752bb 100644 --- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml +++ b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml @@ -9,6 +9,4 @@ metadata: heritage: {{ .Release.Service }} data: statsd_mappings.yaml: | -# -# defaults: -# ttl: "45s" \ No newline at end of file +{{- toYaml (index .Values "statsd_mappings.yaml") | nindent 4 }} diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml index f2d523771e..fe18b3c171 100644 --- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml +++ b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml @@ -3,26 +3,36 @@ image: tag: v0.12.1 pullPolicy: IfNotPresent -service: +metricsService: + type: ClusterIP + port: 9102 annotations: {} labels: {} clusterIP: "" - ## List of IP addresses at which the alertmanager service is available - ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips - ## externalIPs: [] loadBalancerIP: "" loadBalancerSourceRanges: [] - servicePort: 80 - type: ClusterIP - metricsPort: 9102 - statsdPort: 9125 + +statsdService: + type: ClusterIP + port: 9125 + protocol: UDP + annotations: {} + labels: {} + clusterIP: "" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + +statsd_mappings.yaml: + defaults: + ttl: 60s statsdexporter: podAnnotations: - extraArgs: {} - # - --persistence.file=data-perst + extraArgs: + statsd.event-flush-interval: 1000ms resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/infra/charts/feast/charts/feast-core/requirements.lock b/infra/charts/feast/charts/feast-core/requirements.lock new file mode 100644 index 0000000000..b183bb35a2 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/requirements.lock @@ -0,0 +1,18 @@ +dependencies: +- name: postgresql + repository: https://kubernetes-charts.storage.googleapis.com + version: 6.5.5 +- name: kafka + repository: http://storage.googleapis.com/kubernetes-charts-incubator + version: 0.20.1 +- name: prometheus-statsd-exporter + repository: "" + version: 0.1.2 +- name: prometheus + repository: https://kubernetes-charts.storage.googleapis.com + version: 10.4.0 +- name: grafana + repository: https://kubernetes-charts.storage.googleapis.com + version: 4.6.3 +digest: sha256:ce18529b8328a9075d567b949f45e701e3691b275dbc3bcb9f7c7f4052b8fa95 +generated: "2020-02-09T12:30:30.358843813+08:00" diff --git a/infra/docker-compose/core/direct-runner.yml b/infra/docker-compose/core/direct-runner.yml index f9123480bb..1504a94c70 100644 --- a/infra/docker-compose/core/direct-runner.yml +++ b/infra/docker-compose/core/direct-runner.yml @@ -1,3 +1,10 @@ feast: jobs: runner: DirectRunner + updates: + timeoutSeconds: 20 + metrics: + enabled: true + type: statsd + host: statsd-exporter + port: 9125 diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 27d82efc3c..bcd509a10d 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -19,7 +19,7 @@ services: - java - -jar - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yaml + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml online-serving: image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION} @@ -106,4 +106,29 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 db: - image: postgres:12-alpine \ No newline at end of file + image: postgres:12-alpine + + statsd-exporter: + image: prom/statsd-exporter:v0.14.1 + ports: + - "9125:9125/udp" + - "9102:9102" + + prometheus: + image: prom/prometheus:v2.15.2 + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + depends_on: + - statsd-exporter + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:6.5.3 + volumes: + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboards:/etc/grafana/provisioning/dashboards + depends_on: + - prometheus + ports: + - "3000:3000" diff --git a/infra/docker-compose/grafana/dashboards/dashboard.yaml b/infra/docker-compose/grafana/dashboards/dashboard.yaml new file mode 100644 index 0000000000..2e2dee6c1a --- /dev/null +++ b/infra/docker-compose/grafana/dashboards/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: +- name: 'Prometheus' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/infra/docker-compose/grafana/dashboards/feast-ingestion.json b/infra/docker-compose/grafana/dashboards/feast-ingestion.json new file mode 100644 index 0000000000..3d7630f2bf --- /dev/null +++ b/infra/docker-compose/grafana/dashboards/feast-ingestion.json @@ -0,0 +1,1497 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "iteration": 1580253818906, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "repeat": null, + "title": "Constraint Violation", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(feast_ingestion_feature_value_max - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) feast_ingestion_feature_value_domain_max) > 0", + "hide": false, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "(feast_ingestion_feature_value_domain_min - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) feast_ingestion_feature_value_min) > 0", + "hide": false, + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Domain Value", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_presence_min_count - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) increase(feast_ingestion_feature_value_presence_count[5m]) > 0", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_presence_min_fraction - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) increase(feast_ingestion_feature_value_presence_count[5m]) / (\nincrease(feast_ingestion_feature_value_presence_count[5m]) +\nincrease(feast_ingestion_feature_value_missing_count[5m])\n) > 0", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": -1, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 14, + "panels": [], + "repeat": "feature", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 8 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 8 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 25, + "panels": [], + "repeat": null, + "repeatIteration": 1580253818906, + "repeatPanelId": 14, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 2, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 5, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 7, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 29, + "panels": [], + "repeat": null, + "repeatIteration": 1580253818906, + "repeatPanelId": 14, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 2, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 5, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 7, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "", + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "project1", + "value": "project1" + }, + "datasource": null, + "definition": "label_values(feast_project_name)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "project", + "options": [], + "query": "label_values(feast_project_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "entity1 + feature1 + feature2", + "value": [ + "entity1", + "feature1", + "feature2" + ] + }, + "datasource": null, + "definition": "label_values(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\"},feast_feature_name)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "feature", + "options": [], + "query": "label_values(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\"},feast_feature_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "redis", + "value": "redis" + }, + "datasource": null, + "definition": "label_values(feast_store)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "store", + "options": [], + "query": "label_values(feast_store)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Feast Features Dashboard", + "version": 1 +} diff --git a/infra/docker-compose/grafana/datasources/prometheus.yaml b/infra/docker-compose/grafana/datasources/prometheus.yaml new file mode 100644 index 0000000000..a833442116 --- /dev/null +++ b/infra/docker-compose/grafana/datasources/prometheus.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + isDefault: true \ No newline at end of file diff --git a/infra/docker-compose/prometheus/prometheus.yml b/infra/docker-compose/prometheus/prometheus.yml new file mode 100644 index 0000000000..dd90102d39 --- /dev/null +++ b/infra/docker-compose/prometheus/prometheus.yml @@ -0,0 +1,30 @@ +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: + - localhost:9090 + - statsd-exporter:9102 \ No newline at end of file diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index 41af5f9bb4..8ac82c71f3 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -47,10 +47,12 @@ public class ImportJob { // Tag for main output containing Feature Row that has been successfully processed. - private static final TupleTag FEATURE_ROW_OUT = new TupleTag() {}; + private static final TupleTag FEATURE_ROW_OUT = new TupleTag() { + }; // Tag for deadletter output containing elements and error messages from invalid input/transform. - private static final TupleTag DEADLETTER_OUT = new TupleTag() {}; + private static final TupleTag DEADLETTER_OUT = new TupleTag() { + }; private static final Logger log = org.slf4j.LoggerFactory.getLogger(ImportJob.class); /** @@ -88,14 +90,13 @@ public static PipelineResult runPipeline(ImportOptions options) List subscribedFeatureSets = SpecUtil.getSubscribedFeatureSets(store.getSubscriptionsList(), featureSets); - // Generate tags by key - Map featureSetsByKey = new HashMap<>(); - subscribedFeatureSets.stream() - .forEach( - fs -> { - String ref = getFeatureSetReference(fs); - featureSetsByKey.put(ref, fs); - }); + // featureSetsByRef is a map of FeatureSet string reference to FeatureSet object. + // FeatureSet reference follows this format: PROJECT/FEATURE_SET_NAME:VERSION + Map featureSetsByRef = new HashMap<>(); + subscribedFeatureSets.forEach(fs -> { + String ref = getFeatureSetReference(fs); + featureSetsByRef.put(ref, fs); + }); // TODO: make the source part of the job initialisation options Source source = subscribedFeatureSets.get(0).getSpec().getSource(); @@ -121,7 +122,7 @@ public static PipelineResult runPipeline(ImportOptions options) .get(FEATURE_ROW_OUT) .apply( ValidateFeatureRows.newBuilder() - .setFeatureSets(featureSetsByKey) + .setFeatureSets(featureSetsByRef) .setSuccessTag(FEATURE_ROW_OUT) .setFailureTag(DEADLETTER_OUT) .build()); @@ -131,7 +132,7 @@ public static PipelineResult runPipeline(ImportOptions options) .get(FEATURE_ROW_OUT) .apply( "WriteFeatureRowToStore", - WriteToStore.newBuilder().setFeatureSets(featureSetsByKey).setStore(store).build()); + WriteToStore.newBuilder().setFeatureSets(featureSetsByRef).setStore(store).build()); // Step 4. Write FailedElements to a dead letter table in BigQuery. if (options.getDeadLetterTableSpec() != null) { @@ -161,6 +162,7 @@ public static PipelineResult runPipeline(ImportOptions options) .setStoreName(store.getName()) .setSuccessTag(FEATURE_ROW_OUT) .setFailureTag(DEADLETTER_OUT) + .setFeatureSetByRef(featureSetsByRef) .build()); } diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java index b299bb47e5..fca98532d1 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java @@ -24,8 +24,11 @@ import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.Validation.Required; -/** Options passed to Beam to influence the job's execution environment */ +/** + * Options passed to Beam to influence the job's execution environment + */ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, DirectOptions { + @Required @Description( "JSON string representation of the FeatureSet that the import job will process." @@ -60,12 +63,12 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, /** * @param deadLetterTableSpec (Optional) BigQuery table for storing elements that failed to be - * processed. Table spec must follow this format PROJECT_ID:DATASET_ID.PROJECT_ID + * processed. Table spec must follow this format PROJECT_ID:DATASET_ID.PROJECT_ID */ void setDeadLetterTableSpec(String deadLetterTableSpec); // TODO: expound - @Description("MetricsAccumulator exporter type to instantiate.") + @Description("MetricsAccumulator exporter type to instantiate. Currently supported type: statsd.") @Default.String("none") String getMetricsExporterType(); @@ -83,4 +86,14 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, int getStatsdPort(); void setStatsdPort(int StatsdPort); + + @Description( + "The fixed window size in seconds at which ingestion metrics are collected and aggregated. " + + "Metrics in Feast are sent via StatsD and are later scraped by Prometheus at a regular interval. " + + "The window size here should be higher than Prometheus scrope interval, otherwise gauge metrics may " + + "miss scraping because Prometheus scrape frequency is too slow.") + @Default.Integer(20) + int getWindowSizeForMetrics(); + + void setWindowSizeForMetrics(int durationInSeconds); } diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java index 43f314aa86..aad7e8b37e 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java @@ -17,15 +17,23 @@ package feast.ingestion.transform.metrics; import com.google.auto.value.AutoValue; +import feast.core.FeatureSetProto; +import feast.core.FeatureSetProto.FeatureSet; import feast.ingestion.options.ImportOptions; import feast.ingestion.values.FailedElement; import feast.types.FeatureRowProto.FeatureRow; +import java.util.Map; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TupleTag; +import org.joda.time.Duration; @AutoValue public abstract class WriteMetricsTransform extends PTransform { @@ -36,6 +44,8 @@ public abstract class WriteMetricsTransform extends PTransform getFailureTag(); + public abstract Map getFeatureSetByRef(); + public static Builder newBuilder() { return new AutoValue_WriteMetricsTransform.Builder(); } @@ -49,13 +59,18 @@ public abstract static class Builder { public abstract Builder setFailureTag(TupleTag failureTag); + public abstract Builder setFeatureSetByRef( + Map featureSetByRef); + public abstract WriteMetricsTransform build(); } @Override public PDone expand(PCollectionTuple input) { ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); - switch (options.getMetricsExporterType()) { + assert options.getMetricsExporterType() != null; + + switch (options.getMetricsExporterType().trim().toLowerCase()) { case "statsd": input .get(getFailureTag()) @@ -70,13 +85,24 @@ public PDone expand(PCollectionTuple input) { input .get(getSuccessTag()) + .apply("FixedWindowForMetricsCollection", + Window.into(FixedWindows + .of(Duration.standardSeconds(options.getWindowSizeForMetrics())))) + .apply("MapToFeatureRowByRef", ParDo.of(new DoFn>() { + @ProcessElement + public void processElement(ProcessContext c) { + c.output(KV.of(c.element().getFeatureSet(), c.element())); + } + })) + .apply("GroupByFeatureRef", GroupByKey.create()) .apply( - "WriteRowMetrics", + "WriteFeatureRowMetrics", ParDo.of( WriteRowMetricsDoFn.newBuilder() .setStatsdHost(options.getStatsdHost()) .setStatsdPort(options.getStatsdPort()) .setStoreName(getStoreName()) + .setFeatureSetByRef(getFeatureSetByRef()) .build())); return PDone.in(input.getPipeline()); @@ -89,7 +115,8 @@ public PDone expand(PCollectionTuple input) { ParDo.of( new DoFn() { @ProcessElement - public void processElement(ProcessContext c) {} + public void processElement(ProcessContext c) { + } })); return PDone.in(input.getPipeline()); } diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java index db2d1acd6d..00079125e1 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java @@ -17,27 +17,43 @@ package feast.ingestion.transform.metrics; import com.google.auto.value.AutoValue; +import com.google.protobuf.util.Timestamps; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; import com.timgroup.statsd.StatsDClientException; +import feast.core.FeatureSetProto; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.EntitySpec.DomainInfoCase; +import feast.core.FeatureSetProto.EntitySpec.PresenceConstraintsCase; +import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSpec; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; import feast.types.ValueProto.Value.ValCase; +import java.util.DoubleSummaryStatistics; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; import org.slf4j.Logger; +import org.tensorflow.metadata.v0.FeaturePresence; +import org.tensorflow.metadata.v0.FloatDomain; +import org.tensorflow.metadata.v0.IntDomain; @AutoValue -public abstract class WriteRowMetricsDoFn extends DoFn { +public abstract class WriteRowMetricsDoFn extends DoFn>, Void> { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(WriteRowMetricsDoFn.class); + private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(WriteRowMetricsDoFn.class); - private final String METRIC_PREFIX = "feast_ingestion"; - private final String STORE_TAG_KEY = "feast_store"; - private final String FEATURE_SET_PROJECT_TAG_KEY = "feast_project_name"; - private final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; - private final String FEATURE_SET_VERSION_TAG_KEY = "feast_featureSet_version"; - private final String FEATURE_TAG_KEY = "feast_feature_name"; - private final String INGESTION_JOB_NAME_KEY = "ingestion_job_name"; + private static final String METRIC_PREFIX = "feast_ingestion"; + private static final String STORE_TAG_KEY = "feast_store"; + private static final String FEATURE_SET_PROJECT_TAG_KEY = "feast_project_name"; + private static final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; + private static final String FEATURE_SET_VERSION_TAG_KEY = "feast_featureSet_version"; + private static final String FEATURE_NAME_TAG_KEY = "feast_feature_name"; + private static final String INGESTION_JOB_NAME_KEY = "feast_ingestion_job_name"; public abstract String getStoreName(); @@ -45,16 +61,9 @@ public abstract class WriteRowMetricsDoFn extends DoFn { public abstract int getStatsdPort(); - public static WriteRowMetricsDoFn create( - String newStoreName, String newStatsdHost, int newStatsdPort) { - return newBuilder() - .setStoreName(newStoreName) - .setStatsdHost(newStatsdHost) - .setStatsdPort(newStatsdPort) - .build(); - } + public abstract Map getFeatureSetByRef(); - public StatsDClient statsd; + private StatsDClient statsd; public static Builder newBuilder() { return new AutoValue_WriteRowMetricsDoFn.Builder(); @@ -69,79 +78,298 @@ public abstract static class Builder { public abstract Builder setStatsdPort(int statsdPort); + public abstract Builder setFeatureSetByRef( + Map featureSetByRef); + public abstract WriteRowMetricsDoFn build(); } @Setup public void setup() { - statsd = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); + try { + statsd = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); + } catch (StatsDClientException e) { + LOG.warn("Failed to create StatsD client"); + } } @ProcessElement public void processElement(ProcessContext c) { + if (statsd == null) { + LOG.warn( + "No StatsD client available, maybe it failed to initialize. No FeatureRow metrics will be sent."); + return; + } - try { - FeatureRow row = c.element(); - long eventTimestamp = com.google.protobuf.util.Timestamps.toMillis(row.getEventTimestamp()); + String featureSetRef = c.element().getKey(); + if (featureSetRef == null) { + return; + } + + if (!getFeatureSetByRef().containsKey(featureSetRef)) { + // FeatureRow has a reference not known by the ImportJob. Skip sending metrics. + return; + } - String[] split = row.getFeatureSet().split(":"); - String featureSetProject = split[0].split("/")[0]; - String featureSetName = split[0].split("/")[1]; - String featureSetVersion = split[1]; + String[] colonSplits = featureSetRef.split(":"); + if (colonSplits.length < 1) { + LOG.warn( + "FeatureRow has an invalid FeatureSet reference: " + featureSetRef + + ". Expected format: PROJECT/FEATURE_SET:VERSION"); + return; + } + + String[] slashSplits = colonSplits[0].split("/"); + if (slashSplits.length < 2) { + LOG.warn( + "FeatureRow has an invalid FeatureSet reference: " + featureSetRef + + ". Expected format: PROJECT/FEATURE_SET:VERSION"); + return; + } - statsd.histogram( - "feature_row_lag_ms", - System.currentTimeMillis() - eventTimestamp, + String featureSetProject = slashSplits[0]; + String featureSetName = slashSplits[1]; + String featureSetVersion = colonSplits[1]; + + FeatureSet featureSet = getFeatureSetByRef().get(featureSetRef); + Map entityNameToSpec = createEntityNameToSpecMap(featureSet); + Map featureNameToSpec = createFeatureNameToSpecMap(featureSet); + Map fieldNameToMissingCount = new HashMap<>(); + Map fieldNameToValueStat = new HashMap<>(); + + String[] tags = new String[]{ + STORE_TAG_KEY + ":" + getStoreName(), + FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, + FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, + FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, + INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName() + }; + + for (FeatureRow row : c.element().getValue()) { + // All features in a FeatueRow have the same timestamp so lag metrics are recorded per row basis. + long eventTimestamp = Timestamps.toMillis(row.getEventTimestamp()); + statsd.histogram("feature_row_lag_ms", System.currentTimeMillis() - eventTimestamp, tags); + statsd.histogram("feature_row_event_time_epoch_ms", eventTimestamp, tags); + statsd.count("feature_row_ingested_count", 1, tags); + + // Feature value, count and constraint metrics for each feature/entity in a FeatureRow. + for (Field field : row.getFieldsList()) { + String fieldName = field.getName(); + + // Ensure the map objects have properly initialized value for every key. + if (!fieldNameToMissingCount.containsKey(fieldName)) { + fieldNameToMissingCount.put(fieldName, 0); + } + if (!fieldNameToValueStat.containsKey(fieldName)) { + fieldNameToValueStat.put(fieldName, new DoubleSummaryStatistics()); + } + + tags = new String[]{ + STORE_TAG_KEY + ":" + getStoreName(), + FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, + FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, + FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, + INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName(), + FEATURE_NAME_TAG_KEY + ":" + fieldName, + }; + + Value val = field.getValue(); + ValCase valCase = val.getValCase(); + DoubleSummaryStatistics valueStat = fieldNameToValueStat.get(fieldName); + + switch (valCase) { + case INT32_VAL: + valueStat.accept(val.getInt32Val()); + fieldNameToValueStat.put(fieldName, valueStat); + writeConstraintMetrics(entityNameToSpec, featureNameToSpec, fieldName, valCase, tags); + break; + case INT64_VAL: + valueStat.accept(val.getInt64Val()); + fieldNameToValueStat.put(fieldName, valueStat); + writeConstraintMetrics(entityNameToSpec, featureNameToSpec, fieldName, valCase, tags); + break; + case DOUBLE_VAL: + valueStat.accept(val.getDoubleVal()); + fieldNameToValueStat.put(fieldName, valueStat); + writeConstraintMetrics(entityNameToSpec, featureNameToSpec, fieldName, valCase, tags); + break; + case FLOAT_VAL: + valueStat.accept(val.getFloatVal()); + fieldNameToValueStat.put(fieldName, valueStat); + writeConstraintMetrics(entityNameToSpec, featureNameToSpec, fieldName, valCase, tags); + break; + case BOOL_VAL: + valueStat.accept(val.getBoolVal() ? 1 : 0); + fieldNameToValueStat.put(fieldName, valueStat); + break; + case BYTES_VAL: + case STRING_VAL: + case BYTES_LIST_VAL: + case FLOAT_LIST_VAL: + case STRING_LIST_VAL: + case INT32_LIST_VAL: + case INT64_LIST_VAL: + case DOUBLE_LIST_VAL: + case BOOL_LIST_VAL: + valueStat.accept(Double.NaN); + fieldNameToValueStat.put(fieldName, valueStat); + break; + case VAL_NOT_SET: + Integer oldCount = fieldNameToMissingCount.get(fieldName); + fieldNameToMissingCount.put(fieldName, oldCount + 1); + break; + } + } + } + + for (Entry entry : fieldNameToMissingCount.entrySet()) { + String fieldName = entry.getKey(); + Integer missingCount = entry.getValue(); + tags = new String[]{ STORE_TAG_KEY + ":" + getStoreName(), FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); + INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName(), + FEATURE_NAME_TAG_KEY + ":" + fieldName, + }; + statsd.count("feature_value_missing_count", missingCount, tags); + } - statsd.histogram( - "feature_row_event_time_epoch_ms", - eventTimestamp, + for (Entry entry : fieldNameToValueStat.entrySet()) { + String fieldName = entry.getKey(); + DoubleSummaryStatistics valueStat = entry.getValue(); + tags = new String[]{ STORE_TAG_KEY + ":" + getStoreName(), FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); + INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName(), + FEATURE_NAME_TAG_KEY + ":" + fieldName, + }; - for (Field field : row.getFieldsList()) { - if (!field.getValue().getValCase().equals(ValCase.VAL_NOT_SET)) { - statsd.histogram( - "feature_value_lag_ms", - System.currentTimeMillis() - eventTimestamp, - STORE_TAG_KEY + ":" + getStoreName(), - FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, - FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, - FEATURE_TAG_KEY + ":" + field.getName(), - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); - } else { - statsd.count( - "feature_value_missing_count", - 1, - STORE_TAG_KEY + ":" + getStoreName(), - FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, - FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, - FEATURE_TAG_KEY + ":" + field.getName(), - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); - } + // valueStat.getMin() or getMax() should only return finite values. + // (non-finite values can be returned if there is no element or there is an element that is + // not a number. No metric should be sent in such case) + if (Double.isFinite(valueStat.getMin())) { + // Statsd gauge will asssign a delta instead of the actual value, if there is a sign in + // the value. For e.g. if the value is negative, a delta will be assigned. For this reason, + // the gauge value is set to zero beforehand. + // https://github.com/statsd/statsd/blob/master/docs/metric_types.md#gauges + statsd.gauge("feature_value_min", 0, tags); + statsd.gauge("feature_value_min", valueStat.getMin(), tags); } + if (Double.isFinite(valueStat.getMax())) { + statsd.gauge("feature_value_max", 0, tags); + statsd.gauge("feature_value_max", valueStat.getMax(), tags); + } + statsd.count("feature_value_presence_count", valueStat.getCount(), tags); + } + } - statsd.count( - "feature_row_ingested_count", - 1, - STORE_TAG_KEY + ":" + getStoreName(), - FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, - FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_SET_VERSION_TAG_KEY + ":" + featureSetVersion, - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); + // Record the acceptable value and count for each feature according to the spec. + // These can be used to compare against the actual values in the dashboarding / alerting tool. + private void writeConstraintMetrics(Map entityNameToSpec, + Map featureNameToSpec, String fieldName, ValCase valCase, + String[] tags) { + switch (valCase) { + case INT32_VAL: + case INT64_VAL: + if (entityNameToSpec.containsKey(fieldName)) { + EntitySpec entitySpec = entityNameToSpec.get(fieldName); + if (entitySpec.getDomainInfoCase().equals(DomainInfoCase.INT_DOMAIN)) { + IntDomain intDomain = entitySpec.getIntDomain(); + statsd.gauge("feature_value_domain_min", 0, tags); + statsd.gauge("feature_value_domain_min", intDomain.getMin(), tags); + statsd.gauge("feature_value_domain_max", 0, tags); + statsd.gauge("feature_value_domain_max", intDomain.getMax(), tags); + } + if (entitySpec.getPresenceConstraintsCase().equals(PresenceConstraintsCase.PRESENCE)) { + FeaturePresence presence = entitySpec.getPresence(); + statsd.gauge("feature_presence_min_fraction", Math.max(presence.getMinFraction(), 0), + tags); + statsd.gauge("feature_presence_min_count", Math.max(presence.getMinCount(), 0), tags); + } + } else if (featureNameToSpec.containsKey(fieldName)) { + FeatureSpec featureSpec = featureNameToSpec.get(fieldName); + if (featureSpec.getDomainInfoCase().equals(FeatureSpec.DomainInfoCase.INT_DOMAIN)) { + IntDomain intDomain = featureSpec.getIntDomain(); + statsd.gauge("feature_value_domain_min", 0, tags); + statsd.gauge("feature_value_domain_min", intDomain.getMin(), tags); + statsd.gauge("feature_value_domain_max", 0, tags); + statsd.gauge("feature_value_domain_max", intDomain.getMax(), tags); + } + if (featureSpec.getPresenceConstraintsCase() + .equals(FeatureSpec.PresenceConstraintsCase.PRESENCE)) { + FeaturePresence presence = featureSpec.getPresence(); + statsd.gauge("feature_presence_min_fraction", Math.max(presence.getMinFraction(), 0), + tags); + statsd.gauge("feature_presence_min_count", Math.max(presence.getMinCount(), 0), tags); + } + } + break; + case DOUBLE_VAL: + case FLOAT_VAL: + if (entityNameToSpec.containsKey(fieldName)) { + EntitySpec entitySpec = entityNameToSpec.get(fieldName); + if (entitySpec.getDomainInfoCase().equals(DomainInfoCase.FLOAT_DOMAIN)) { + FloatDomain floatDomain = entitySpec.getFloatDomain(); + statsd.gauge("feature_value_domain_min", 0, tags); + statsd.gauge("feature_value_domain_min", floatDomain.getMin(), tags); + statsd.gauge("feature_value_domain_max", 0, tags); + statsd.gauge("feature_value_domain_max", floatDomain.getMax(), tags); + } + if (entitySpec.getPresenceConstraintsCase().equals(PresenceConstraintsCase.PRESENCE)) { + FeaturePresence presence = entitySpec.getPresence(); + statsd.gauge("feature_presence_min_fraction", Math.max(presence.getMinFraction(), 0), + tags); + statsd.gauge("feature_presence_min_count", Math.max(presence.getMinCount(), 0), tags); + } + } else if (featureNameToSpec.containsKey(fieldName)) { + FeatureSpec featureSpec = featureNameToSpec.get(fieldName); + if (featureSpec.getDomainInfoCase().equals(FeatureSpec.DomainInfoCase.FLOAT_DOMAIN)) { + FloatDomain floatDomain = featureSpec.getFloatDomain(); + statsd.gauge("feature_value_domain_min", 0, tags); + statsd.gauge("feature_value_domain_min", floatDomain.getMin(), tags); + statsd.gauge("feature_value_domain_max", 0, tags); + statsd.gauge("feature_value_domain_max", floatDomain.getMax(), tags); + } + if (featureSpec.getPresenceConstraintsCase() + .equals(FeatureSpec.PresenceConstraintsCase.PRESENCE)) { + FeaturePresence presence = featureSpec.getPresence(); + statsd.gauge("feature_presence_min_fraction", Math.max(presence.getMinFraction(), 0), + tags); + statsd.gauge("feature_presence_min_count", Math.max(presence.getMinCount(), 0), tags); + } + } + break; + default: + break; + } - } catch (StatsDClientException e) { - log.warn("Unable to push metrics to server", e); + } + + private Map createFeatureNameToSpecMap(FeatureSet featureSet) { + Map featureSpecByName = new HashMap<>(); + if (featureSet == null) { + return featureSpecByName; + } + + featureSet.getSpec().getFeaturesList().forEach(featureSpec -> { + featureSpecByName.put(featureSpec.getName(), featureSpec); + }); + return featureSpecByName; + } + + private Map createEntityNameToSpecMap(FeatureSet featureSet) { + Map entitySpecByName = new HashMap<>(); + if (featureSet == null) { + return entitySpecByName; } + + featureSet.getSpec().getEntitiesList().forEach(entitySpec -> { + entitySpecByName.put(entitySpec.getName(), entitySpec); + }); + return entitySpecByName; } } diff --git a/ingestion/src/main/resources/grafana-dashboard.json b/ingestion/src/main/resources/grafana-dashboard.json new file mode 100644 index 0000000000..3758818c86 --- /dev/null +++ b/ingestion/src/main/resources/grafana-dashboard.json @@ -0,0 +1,1497 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "iteration": 1580253818906, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "repeat": null, + "title": "Constraint Violation", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(feast_ingestion_feature_value_max - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) feast_ingestion_feature_value_domain_max) > 0", + "hide": false, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "(feast_ingestion_feature_value_domain_min - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) feast_ingestion_feature_value_min) > 0", + "hide": false, + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Domain Value", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_presence_min_count - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) increase(feast_ingestion_feature_value_presence_count[5m]) > 0", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_presence_min_fraction - on(feast_project_name,feast_feature_name,feast_store,feast_ingestion_job_name) increase(feast_ingestion_feature_value_presence_count[5m]) / (\nincrease(feast_ingestion_feature_value_presence_count[5m]) +\nincrease(feast_ingestion_feature_value_missing_count[5m])\n) > 0", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": -1, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 14, + "panels": [], + "repeat": "feature", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 8 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 8 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "scopedVars": { + "feature": { + "selected": true, + "text": "entity1", + "value": "entity1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 25, + "panels": [], + "repeat": null, + "repeatIteration": 1580253818906, + "repeatPanelId": 14, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 2, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 5, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 7, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature1", + "value": "feature1" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 29, + "panels": [], + "repeat": null, + "repeatIteration": 1580253818906, + "repeatPanelId": 14, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "title": "Feature Stats", + "type": "row" + }, + { + "aliasColors": { + "max": "super-light-red", + "max_domain": "light-red", + "max_val": "dark-orange", + "min": "super-light-green", + "min_domain": "light-green", + "min_val": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 2, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "feast_ingestion_feature_value_min{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_val", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_value_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_val", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_value_domain_min{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "min_domain", + "refId": "C" + }, + { + "expr": "feast_ingestion_feature_value_domain_max{feast_project_name=\"[[project]]\",feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "max_domain", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Feature Value - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "count": "dark-green", + "count over selected range": "dark-yellow", + "feature_presence_min_count": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 5, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}[$__range])", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "A" + }, + { + "expr": "feast_ingestion_feature_presence_min_count{feast_project_name=\"[[project]]\", feast_feature_name=~\"[[feature]]\",feast_store=\"[[store]]\"}", + "legendFormat": "feature_presence_min_count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Count - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "decimals": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fraction": "dark-green", + "min_fraction": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "repeatIteration": 1580253818906, + "repeatPanelId": 7, + "repeatedByRow": true, + "scopedVars": { + "feature": { + "selected": true, + "text": "feature2", + "value": "feature2" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) / (\nincrease(feast_ingestion_feature_value_presence_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range]) + \nincrease(feast_ingestion_feature_value_missing_count{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}[$__range])\n)", + "legendFormat": "fraction", + "refId": "B" + }, + { + "expr": "feast_ingestion_feature_presence_min_fraction{feast_feature_name=~\"[[feature]]\", feast_project_name=\"[[project]]\"}", + "legendFormat": "min_fraction", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Presence Fraction - [[feature]]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "", + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "project1", + "value": "project1" + }, + "datasource": null, + "definition": "label_values(feast_project_name)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "project", + "options": [], + "query": "label_values(feast_project_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "entity1 + feature1 + feature2", + "value": [ + "entity1", + "feature1", + "feature2" + ] + }, + "datasource": null, + "definition": "label_values(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\"},feast_feature_name)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "feature", + "options": [], + "query": "label_values(feast_ingestion_feature_value_presence_count{feast_project_name=\"[[project]]\"},feast_feature_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "redis", + "value": "redis" + }, + "datasource": null, + "definition": "label_values(feast_store)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "store", + "options": [], + "query": "label_values(feast_store)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Feast Features Dashboard", + "version": 1 +} \ No newline at end of file diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index 24293c6faa..d878ea111d 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -35,8 +35,10 @@ message FeatureRow { // will use to perform joins, determine latest values, and coalesce rows. google.protobuf.Timestamp event_timestamp = 3; - // Complete reference to the featureSet this featureRow belongs to, in the form of - // featureSetName:version. This value will be used by the feast ingestion job to filter - // rows, and write the values to the correct tables. + // Complete reference to the featureSet this featureRow belongs to, in the form of: + // [project]/[feature_set_name]:[version] + // + // FeatureSet reference will be used by the Feast ingestion job to filter + // rows and write the values to the correct tables. string feature_set = 6; } \ No newline at end of file diff --git a/sdk/python/feast/core/CoreService_pb2.pyi b/sdk/python/feast/core/CoreService_pb2.pyi index 645226982a..24130ee870 100644 --- a/sdk/python/feast/core/CoreService_pb2.pyi +++ b/sdk/python/feast/core/CoreService_pb2.pyi @@ -36,20 +36,27 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + class GetFeatureSetRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int def __init__(self, *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, + version : typing___Optional[builtin___int] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetRequest: ... + def FromString(cls, s: builtin___bytes) -> GetFeatureSetRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -68,14 +75,14 @@ class GetFeatureSetResponse(google___protobuf___message___Message): feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSet] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetResponse: ... + def FromString(cls, s: builtin___bytes) -> GetFeatureSetResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... class ListFeatureSetsRequest(google___protobuf___message___Message): @@ -93,7 +100,7 @@ class ListFeatureSetsRequest(google___protobuf___message___Message): feature_set_version : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsRequest.Filter: ... + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsRequest.Filter: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -110,14 +117,14 @@ class ListFeatureSetsRequest(google___protobuf___message___Message): filter : typing___Optional[ListFeatureSetsRequest.Filter] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsRequest: ... + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... class ListFeatureSetsResponse(google___protobuf___message___Message): @@ -131,7 +138,7 @@ class ListFeatureSetsResponse(google___protobuf___message___Message): feature_sets : typing___Optional[typing___Iterable[feast___core___FeatureSet_pb2___FeatureSet]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsResponse: ... + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -150,7 +157,7 @@ class ListStoresRequest(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListStoresRequest.Filter: ... + def FromString(cls, s: builtin___bytes) -> ListStoresRequest.Filter: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -167,14 +174,14 @@ class ListStoresRequest(google___protobuf___message___Message): filter : typing___Optional[ListStoresRequest.Filter] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListStoresRequest: ... + def FromString(cls, s: builtin___bytes) -> ListStoresRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... class ListStoresResponse(google___protobuf___message___Message): @@ -188,7 +195,7 @@ class ListStoresResponse(google___protobuf___message___Message): store : typing___Optional[typing___Iterable[feast___core___Store_pb2___Store]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListStoresResponse: ... + def FromString(cls, s: builtin___bytes) -> ListStoresResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -207,36 +214,36 @@ class ApplyFeatureSetRequest(google___protobuf___message___Message): feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSet] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ApplyFeatureSetRequest: ... + def FromString(cls, s: builtin___bytes) -> ApplyFeatureSetRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... class ApplyFeatureSetResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Status(int): + class Status(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> ApplyFeatureSetResponse.Status: ... + def Value(cls, name: builtin___str) -> 'ApplyFeatureSetResponse.Status': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[ApplyFeatureSetResponse.Status]: ... + def values(cls) -> typing___List['ApplyFeatureSetResponse.Status']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, ApplyFeatureSetResponse.Status]]: ... - NO_CHANGE = typing___cast(ApplyFeatureSetResponse.Status, 0) - CREATED = typing___cast(ApplyFeatureSetResponse.Status, 1) - ERROR = typing___cast(ApplyFeatureSetResponse.Status, 2) - NO_CHANGE = typing___cast(ApplyFeatureSetResponse.Status, 0) - CREATED = typing___cast(ApplyFeatureSetResponse.Status, 1) - ERROR = typing___cast(ApplyFeatureSetResponse.Status, 2) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'ApplyFeatureSetResponse.Status']]: ... + NO_CHANGE = typing___cast('ApplyFeatureSetResponse.Status', 0) + CREATED = typing___cast('ApplyFeatureSetResponse.Status', 1) + ERROR = typing___cast('ApplyFeatureSetResponse.Status', 2) + NO_CHANGE = typing___cast('ApplyFeatureSetResponse.Status', 0) + CREATED = typing___cast('ApplyFeatureSetResponse.Status', 1) + ERROR = typing___cast('ApplyFeatureSetResponse.Status', 2) status = ... # type: ApplyFeatureSetResponse.Status @@ -249,14 +256,14 @@ class ApplyFeatureSetResponse(google___protobuf___message___Message): status : typing___Optional[ApplyFeatureSetResponse.Status] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ApplyFeatureSetResponse: ... + def FromString(cls, s: builtin___bytes) -> ApplyFeatureSetResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",u"status"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set",u"status",b"status"]) -> None: ... class GetFeastCoreVersionRequest(google___protobuf___message___Message): @@ -265,7 +272,7 @@ class GetFeastCoreVersionRequest(google___protobuf___message___Message): def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeastCoreVersionRequest: ... + def FromString(cls, s: builtin___bytes) -> GetFeastCoreVersionRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... @@ -278,7 +285,7 @@ class GetFeastCoreVersionResponse(google___protobuf___message___Message): version : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeastCoreVersionResponse: ... + def FromString(cls, s: builtin___bytes) -> GetFeastCoreVersionResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -297,34 +304,34 @@ class UpdateStoreRequest(google___protobuf___message___Message): store : typing___Optional[feast___core___Store_pb2___Store] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> UpdateStoreRequest: ... + def FromString(cls, s: builtin___bytes) -> UpdateStoreRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"store"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> None: ... class UpdateStoreResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Status(int): + class Status(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> UpdateStoreResponse.Status: ... + def Value(cls, name: builtin___str) -> 'UpdateStoreResponse.Status': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[UpdateStoreResponse.Status]: ... + def values(cls) -> typing___List['UpdateStoreResponse.Status']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, UpdateStoreResponse.Status]]: ... - NO_CHANGE = typing___cast(UpdateStoreResponse.Status, 0) - UPDATED = typing___cast(UpdateStoreResponse.Status, 1) - NO_CHANGE = typing___cast(UpdateStoreResponse.Status, 0) - UPDATED = typing___cast(UpdateStoreResponse.Status, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'UpdateStoreResponse.Status']]: ... + NO_CHANGE = typing___cast('UpdateStoreResponse.Status', 0) + UPDATED = typing___cast('UpdateStoreResponse.Status', 1) + NO_CHANGE = typing___cast('UpdateStoreResponse.Status', 0) + UPDATED = typing___cast('UpdateStoreResponse.Status', 1) status = ... # type: UpdateStoreResponse.Status @@ -337,14 +344,14 @@ class UpdateStoreResponse(google___protobuf___message___Message): status : typing___Optional[UpdateStoreResponse.Status] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> UpdateStoreResponse: ... + def FromString(cls, s: builtin___bytes) -> UpdateStoreResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"status",u"store"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"status",b"status",u"store",b"store"]) -> None: ... class CreateProjectRequest(google___protobuf___message___Message): @@ -356,7 +363,7 @@ class CreateProjectRequest(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> CreateProjectRequest: ... + def FromString(cls, s: builtin___bytes) -> CreateProjectRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -370,7 +377,7 @@ class CreateProjectResponse(google___protobuf___message___Message): def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> CreateProjectResponse: ... + def FromString(cls, s: builtin___bytes) -> CreateProjectResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... @@ -383,7 +390,7 @@ class ArchiveProjectRequest(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ArchiveProjectRequest: ... + def FromString(cls, s: builtin___bytes) -> ArchiveProjectRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -397,7 +404,7 @@ class ArchiveProjectResponse(google___protobuf___message___Message): def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ArchiveProjectResponse: ... + def FromString(cls, s: builtin___bytes) -> ArchiveProjectResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... @@ -407,7 +414,7 @@ class ListProjectsRequest(google___protobuf___message___Message): def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListProjectsRequest: ... + def FromString(cls, s: builtin___bytes) -> ListProjectsRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... @@ -420,7 +427,7 @@ class ListProjectsResponse(google___protobuf___message___Message): projects : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ListProjectsResponse: ... + def FromString(cls, s: builtin___bytes) -> ListProjectsResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): diff --git a/sdk/python/feast/core/FeatureSet_pb2.py b/sdk/python/feast/core/FeatureSet_pb2.py index 991220ccae..f265061002 100644 --- a/sdk/python/feast/core/FeatureSet_pb2.py +++ b/sdk/python/feast/core/FeatureSet_pb2.py @@ -18,6 +18,7 @@ from feast.core import Source_pb2 as feast_dot_core_dot_Source__pb2 from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from tensorflow_metadata.proto.v0 import schema_pb2 as tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -25,9 +26,9 @@ package='feast.core', syntax='proto3', serialized_options=_b('\n\nfeast.coreB\017FeatureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x1b\x66\x65\x61st/core/FeatureSet.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x17\x66\x65\x61st/core/Source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"`\n\nFeatureSet\x12(\n\x04spec\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12(\n\x04meta\x18\x02 \x01(\x0b\x32\x1a.feast.core.FeatureSetMeta\"\xe5\x01\n\x0e\x46\x65\x61tureSetSpec\x12\x0f\n\x07project\x18\x07 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\x12(\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x16.feast.core.EntitySpec\x12)\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x17.feast.core.FeatureSpec\x12*\n\x07max_age\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\"\n\x06source\x18\x06 \x01(\x0b\x32\x12.feast.core.Source\"K\n\nEntitySpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\"L\n\x0b\x46\x65\x61tureSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\"u\n\x0e\x46\x65\x61tureSetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x06status\x18\x02 \x01(\x0e\x32\x1c.feast.core.FeatureSetStatus*L\n\x10\x46\x65\x61tureSetStatus\x12\x12\n\x0eSTATUS_INVALID\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x10\n\x0cSTATUS_READY\x10\x02\x42N\n\nfeast.coreB\x0f\x46\x65\x61tureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_pb=_b('\n\x1b\x66\x65\x61st/core/FeatureSet.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x17\x66\x65\x61st/core/Source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a)tensorflow_metadata/proto/v0/schema.proto\"`\n\nFeatureSet\x12(\n\x04spec\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12(\n\x04meta\x18\x02 \x01(\x0b\x32\x1a.feast.core.FeatureSetMeta\"\xe5\x01\n\x0e\x46\x65\x61tureSetSpec\x12\x0f\n\x07project\x18\x07 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\x12(\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x16.feast.core.EntitySpec\x12)\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x17.feast.core.FeatureSpec\x12*\n\x07max_age\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\"\n\x06source\x18\x06 \x01(\x0b\x32\x12.feast.core.Source\"\xbf\x08\n\nEntitySpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12;\n\x08presence\x18\x03 \x01(\x0b\x32\'.tensorflow.metadata.v0.FeaturePresenceH\x00\x12L\n\x0egroup_presence\x18\x04 \x01(\x0b\x32\x32.tensorflow.metadata.v0.FeaturePresenceWithinGroupH\x00\x12\x33\n\x05shape\x18\x05 \x01(\x0b\x32\".tensorflow.metadata.v0.FixedShapeH\x01\x12\x39\n\x0bvalue_count\x18\x06 \x01(\x0b\x32\".tensorflow.metadata.v0.ValueCountH\x01\x12\x10\n\x06\x64omain\x18\x07 \x01(\tH\x02\x12\x37\n\nint_domain\x18\x08 \x01(\x0b\x32!.tensorflow.metadata.v0.IntDomainH\x02\x12;\n\x0c\x66loat_domain\x18\t \x01(\x0b\x32#.tensorflow.metadata.v0.FloatDomainH\x02\x12=\n\rstring_domain\x18\n \x01(\x0b\x32$.tensorflow.metadata.v0.StringDomainH\x02\x12\x39\n\x0b\x62ool_domain\x18\x0b \x01(\x0b\x32\".tensorflow.metadata.v0.BoolDomainH\x02\x12=\n\rstruct_domain\x18\x0c \x01(\x0b\x32$.tensorflow.metadata.v0.StructDomainH\x02\x12P\n\x17natural_language_domain\x18\r \x01(\x0b\x32-.tensorflow.metadata.v0.NaturalLanguageDomainH\x02\x12;\n\x0cimage_domain\x18\x0e \x01(\x0b\x32#.tensorflow.metadata.v0.ImageDomainH\x02\x12\x37\n\nmid_domain\x18\x0f \x01(\x0b\x32!.tensorflow.metadata.v0.MIDDomainH\x02\x12\x37\n\nurl_domain\x18\x10 \x01(\x0b\x32!.tensorflow.metadata.v0.URLDomainH\x02\x12\x39\n\x0btime_domain\x18\x11 \x01(\x0b\x32\".tensorflow.metadata.v0.TimeDomainH\x02\x12\x45\n\x12time_of_day_domain\x18\x12 \x01(\x0b\x32\'.tensorflow.metadata.v0.TimeOfDayDomainH\x02\x42\x16\n\x14presence_constraintsB\x0c\n\nshape_typeB\r\n\x0b\x64omain_info\"\xc0\x08\n\x0b\x46\x65\x61tureSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12;\n\x08presence\x18\x03 \x01(\x0b\x32\'.tensorflow.metadata.v0.FeaturePresenceH\x00\x12L\n\x0egroup_presence\x18\x04 \x01(\x0b\x32\x32.tensorflow.metadata.v0.FeaturePresenceWithinGroupH\x00\x12\x33\n\x05shape\x18\x05 \x01(\x0b\x32\".tensorflow.metadata.v0.FixedShapeH\x01\x12\x39\n\x0bvalue_count\x18\x06 \x01(\x0b\x32\".tensorflow.metadata.v0.ValueCountH\x01\x12\x10\n\x06\x64omain\x18\x07 \x01(\tH\x02\x12\x37\n\nint_domain\x18\x08 \x01(\x0b\x32!.tensorflow.metadata.v0.IntDomainH\x02\x12;\n\x0c\x66loat_domain\x18\t \x01(\x0b\x32#.tensorflow.metadata.v0.FloatDomainH\x02\x12=\n\rstring_domain\x18\n \x01(\x0b\x32$.tensorflow.metadata.v0.StringDomainH\x02\x12\x39\n\x0b\x62ool_domain\x18\x0b \x01(\x0b\x32\".tensorflow.metadata.v0.BoolDomainH\x02\x12=\n\rstruct_domain\x18\x0c \x01(\x0b\x32$.tensorflow.metadata.v0.StructDomainH\x02\x12P\n\x17natural_language_domain\x18\r \x01(\x0b\x32-.tensorflow.metadata.v0.NaturalLanguageDomainH\x02\x12;\n\x0cimage_domain\x18\x0e \x01(\x0b\x32#.tensorflow.metadata.v0.ImageDomainH\x02\x12\x37\n\nmid_domain\x18\x0f \x01(\x0b\x32!.tensorflow.metadata.v0.MIDDomainH\x02\x12\x37\n\nurl_domain\x18\x10 \x01(\x0b\x32!.tensorflow.metadata.v0.URLDomainH\x02\x12\x39\n\x0btime_domain\x18\x11 \x01(\x0b\x32\".tensorflow.metadata.v0.TimeDomainH\x02\x12\x45\n\x12time_of_day_domain\x18\x12 \x01(\x0b\x32\'.tensorflow.metadata.v0.TimeOfDayDomainH\x02\x42\x16\n\x14presence_constraintsB\x0c\n\nshape_typeB\r\n\x0b\x64omain_info\"u\n\x0e\x46\x65\x61tureSetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x06status\x18\x02 \x01(\x0e\x32\x1c.feast.core.FeatureSetStatus*L\n\x10\x46\x65\x61tureSetStatus\x12\x12\n\x0eSTATUS_INVALID\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x10\n\x0cSTATUS_READY\x10\x02\x42N\n\nfeast.coreB\x0f\x46\x65\x61tureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') , - dependencies=[feast_dot_types_dot_Value__pb2.DESCRIPTOR,feast_dot_core_dot_Source__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) + dependencies=[feast_dot_types_dot_Value__pb2.DESCRIPTOR,feast_dot_core_dot_Source__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2.DESCRIPTOR,]) _FEATURESETSTATUS = _descriptor.EnumDescriptor( name='FeatureSetStatus', @@ -50,8 +51,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=762, - serialized_end=838, + serialized_start=2831, + serialized_end=2907, ) _sym_db.RegisterEnumDescriptor(_FEATURESETSTATUS) @@ -95,8 +96,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=158, - serialized_end=254, + serialized_start=201, + serialized_end=297, ) @@ -168,8 +169,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=257, - serialized_end=486, + serialized_start=300, + serialized_end=529, ) @@ -194,6 +195,118 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='presence', full_name='feast.core.EntitySpec.presence', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='group_presence', full_name='feast.core.EntitySpec.group_presence', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shape', full_name='feast.core.EntitySpec.shape', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value_count', full_name='feast.core.EntitySpec.value_count', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='domain', full_name='feast.core.EntitySpec.domain', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_domain', full_name='feast.core.EntitySpec.int_domain', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_domain', full_name='feast.core.EntitySpec.float_domain', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='string_domain', full_name='feast.core.EntitySpec.string_domain', index=9, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_domain', full_name='feast.core.EntitySpec.bool_domain', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='struct_domain', full_name='feast.core.EntitySpec.struct_domain', index=11, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='natural_language_domain', full_name='feast.core.EntitySpec.natural_language_domain', index=12, + number=13, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='image_domain', full_name='feast.core.EntitySpec.image_domain', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mid_domain', full_name='feast.core.EntitySpec.mid_domain', index=14, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='url_domain', full_name='feast.core.EntitySpec.url_domain', index=15, + number=16, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_domain', full_name='feast.core.EntitySpec.time_domain', index=16, + number=17, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_of_day_domain', full_name='feast.core.EntitySpec.time_of_day_domain', index=17, + number=18, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -205,9 +318,18 @@ syntax='proto3', extension_ranges=[], oneofs=[ + _descriptor.OneofDescriptor( + name='presence_constraints', full_name='feast.core.EntitySpec.presence_constraints', + index=0, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='shape_type', full_name='feast.core.EntitySpec.shape_type', + index=1, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='domain_info', full_name='feast.core.EntitySpec.domain_info', + index=2, containing_type=None, fields=[]), ], - serialized_start=488, - serialized_end=563, + serialized_start=532, + serialized_end=1619, ) @@ -232,6 +354,118 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='presence', full_name='feast.core.FeatureSpec.presence', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='group_presence', full_name='feast.core.FeatureSpec.group_presence', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shape', full_name='feast.core.FeatureSpec.shape', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value_count', full_name='feast.core.FeatureSpec.value_count', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='domain', full_name='feast.core.FeatureSpec.domain', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_domain', full_name='feast.core.FeatureSpec.int_domain', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_domain', full_name='feast.core.FeatureSpec.float_domain', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='string_domain', full_name='feast.core.FeatureSpec.string_domain', index=9, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_domain', full_name='feast.core.FeatureSpec.bool_domain', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='struct_domain', full_name='feast.core.FeatureSpec.struct_domain', index=11, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='natural_language_domain', full_name='feast.core.FeatureSpec.natural_language_domain', index=12, + number=13, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='image_domain', full_name='feast.core.FeatureSpec.image_domain', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mid_domain', full_name='feast.core.FeatureSpec.mid_domain', index=14, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='url_domain', full_name='feast.core.FeatureSpec.url_domain', index=15, + number=16, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_domain', full_name='feast.core.FeatureSpec.time_domain', index=16, + number=17, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_of_day_domain', full_name='feast.core.FeatureSpec.time_of_day_domain', index=17, + number=18, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -243,9 +477,18 @@ syntax='proto3', extension_ranges=[], oneofs=[ + _descriptor.OneofDescriptor( + name='presence_constraints', full_name='feast.core.FeatureSpec.presence_constraints', + index=0, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='shape_type', full_name='feast.core.FeatureSpec.shape_type', + index=1, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='domain_info', full_name='feast.core.FeatureSpec.domain_info', + index=2, containing_type=None, fields=[]), ], - serialized_start=565, - serialized_end=641, + serialized_start=1622, + serialized_end=2710, ) @@ -282,8 +525,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=643, - serialized_end=760, + serialized_start=2712, + serialized_end=2829, ) _FEATURESET.fields_by_name['spec'].message_type = _FEATURESETSPEC @@ -293,7 +536,133 @@ _FEATURESETSPEC.fields_by_name['max_age'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _FEATURESETSPEC.fields_by_name['source'].message_type = feast_dot_core_dot_Source__pb2._SOURCE _ENTITYSPEC.fields_by_name['value_type'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM +_ENTITYSPEC.fields_by_name['presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCE +_ENTITYSPEC.fields_by_name['group_presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCEWITHINGROUP +_ENTITYSPEC.fields_by_name['shape'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FIXEDSHAPE +_ENTITYSPEC.fields_by_name['value_count'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._VALUECOUNT +_ENTITYSPEC.fields_by_name['int_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._INTDOMAIN +_ENTITYSPEC.fields_by_name['float_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FLOATDOMAIN +_ENTITYSPEC.fields_by_name['string_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRINGDOMAIN +_ENTITYSPEC.fields_by_name['bool_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._BOOLDOMAIN +_ENTITYSPEC.fields_by_name['struct_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRUCTDOMAIN +_ENTITYSPEC.fields_by_name['natural_language_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._NATURALLANGUAGEDOMAIN +_ENTITYSPEC.fields_by_name['image_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._IMAGEDOMAIN +_ENTITYSPEC.fields_by_name['mid_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._MIDDOMAIN +_ENTITYSPEC.fields_by_name['url_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._URLDOMAIN +_ENTITYSPEC.fields_by_name['time_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEDOMAIN +_ENTITYSPEC.fields_by_name['time_of_day_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEOFDAYDOMAIN +_ENTITYSPEC.oneofs_by_name['presence_constraints'].fields.append( + _ENTITYSPEC.fields_by_name['presence']) +_ENTITYSPEC.fields_by_name['presence'].containing_oneof = _ENTITYSPEC.oneofs_by_name['presence_constraints'] +_ENTITYSPEC.oneofs_by_name['presence_constraints'].fields.append( + _ENTITYSPEC.fields_by_name['group_presence']) +_ENTITYSPEC.fields_by_name['group_presence'].containing_oneof = _ENTITYSPEC.oneofs_by_name['presence_constraints'] +_ENTITYSPEC.oneofs_by_name['shape_type'].fields.append( + _ENTITYSPEC.fields_by_name['shape']) +_ENTITYSPEC.fields_by_name['shape'].containing_oneof = _ENTITYSPEC.oneofs_by_name['shape_type'] +_ENTITYSPEC.oneofs_by_name['shape_type'].fields.append( + _ENTITYSPEC.fields_by_name['value_count']) +_ENTITYSPEC.fields_by_name['value_count'].containing_oneof = _ENTITYSPEC.oneofs_by_name['shape_type'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['domain']) +_ENTITYSPEC.fields_by_name['domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['int_domain']) +_ENTITYSPEC.fields_by_name['int_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['float_domain']) +_ENTITYSPEC.fields_by_name['float_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['string_domain']) +_ENTITYSPEC.fields_by_name['string_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['bool_domain']) +_ENTITYSPEC.fields_by_name['bool_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['struct_domain']) +_ENTITYSPEC.fields_by_name['struct_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['natural_language_domain']) +_ENTITYSPEC.fields_by_name['natural_language_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['image_domain']) +_ENTITYSPEC.fields_by_name['image_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['mid_domain']) +_ENTITYSPEC.fields_by_name['mid_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['url_domain']) +_ENTITYSPEC.fields_by_name['url_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['time_domain']) +_ENTITYSPEC.fields_by_name['time_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['time_of_day_domain']) +_ENTITYSPEC.fields_by_name['time_of_day_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] _FEATURESPEC.fields_by_name['value_type'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM +_FEATURESPEC.fields_by_name['presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCE +_FEATURESPEC.fields_by_name['group_presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCEWITHINGROUP +_FEATURESPEC.fields_by_name['shape'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FIXEDSHAPE +_FEATURESPEC.fields_by_name['value_count'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._VALUECOUNT +_FEATURESPEC.fields_by_name['int_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._INTDOMAIN +_FEATURESPEC.fields_by_name['float_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FLOATDOMAIN +_FEATURESPEC.fields_by_name['string_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRINGDOMAIN +_FEATURESPEC.fields_by_name['bool_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._BOOLDOMAIN +_FEATURESPEC.fields_by_name['struct_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRUCTDOMAIN +_FEATURESPEC.fields_by_name['natural_language_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._NATURALLANGUAGEDOMAIN +_FEATURESPEC.fields_by_name['image_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._IMAGEDOMAIN +_FEATURESPEC.fields_by_name['mid_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._MIDDOMAIN +_FEATURESPEC.fields_by_name['url_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._URLDOMAIN +_FEATURESPEC.fields_by_name['time_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEDOMAIN +_FEATURESPEC.fields_by_name['time_of_day_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEOFDAYDOMAIN +_FEATURESPEC.oneofs_by_name['presence_constraints'].fields.append( + _FEATURESPEC.fields_by_name['presence']) +_FEATURESPEC.fields_by_name['presence'].containing_oneof = _FEATURESPEC.oneofs_by_name['presence_constraints'] +_FEATURESPEC.oneofs_by_name['presence_constraints'].fields.append( + _FEATURESPEC.fields_by_name['group_presence']) +_FEATURESPEC.fields_by_name['group_presence'].containing_oneof = _FEATURESPEC.oneofs_by_name['presence_constraints'] +_FEATURESPEC.oneofs_by_name['shape_type'].fields.append( + _FEATURESPEC.fields_by_name['shape']) +_FEATURESPEC.fields_by_name['shape'].containing_oneof = _FEATURESPEC.oneofs_by_name['shape_type'] +_FEATURESPEC.oneofs_by_name['shape_type'].fields.append( + _FEATURESPEC.fields_by_name['value_count']) +_FEATURESPEC.fields_by_name['value_count'].containing_oneof = _FEATURESPEC.oneofs_by_name['shape_type'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['domain']) +_FEATURESPEC.fields_by_name['domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['int_domain']) +_FEATURESPEC.fields_by_name['int_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['float_domain']) +_FEATURESPEC.fields_by_name['float_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['string_domain']) +_FEATURESPEC.fields_by_name['string_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['bool_domain']) +_FEATURESPEC.fields_by_name['bool_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['struct_domain']) +_FEATURESPEC.fields_by_name['struct_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['natural_language_domain']) +_FEATURESPEC.fields_by_name['natural_language_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['image_domain']) +_FEATURESPEC.fields_by_name['image_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['mid_domain']) +_FEATURESPEC.fields_by_name['mid_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['url_domain']) +_FEATURESPEC.fields_by_name['url_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['time_domain']) +_FEATURESPEC.fields_by_name['time_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['time_of_day_domain']) +_FEATURESPEC.fields_by_name['time_of_day_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] _FEATURESETMETA.fields_by_name['created_timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _FEATURESETMETA.fields_by_name['status'].enum_type = _FEATURESETSTATUS DESCRIPTOR.message_types_by_name['FeatureSet'] = _FEATURESET diff --git a/sdk/python/feast/core/FeatureSet_pb2.pyi b/sdk/python/feast/core/FeatureSet_pb2.pyi index c663c70c68..fa3c626a41 100644 --- a/sdk/python/feast/core/FeatureSet_pb2.pyi +++ b/sdk/python/feast/core/FeatureSet_pb2.pyi @@ -29,6 +29,24 @@ from google.protobuf.timestamp_pb2 import ( Timestamp as google___protobuf___timestamp_pb2___Timestamp, ) +from tensorflow_metadata.proto.v0.schema_pb2 import ( + BoolDomain as tensorflow_metadata___proto___v0___schema_pb2___BoolDomain, + FeaturePresence as tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence, + FeaturePresenceWithinGroup as tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup, + FixedShape as tensorflow_metadata___proto___v0___schema_pb2___FixedShape, + FloatDomain as tensorflow_metadata___proto___v0___schema_pb2___FloatDomain, + ImageDomain as tensorflow_metadata___proto___v0___schema_pb2___ImageDomain, + IntDomain as tensorflow_metadata___proto___v0___schema_pb2___IntDomain, + MIDDomain as tensorflow_metadata___proto___v0___schema_pb2___MIDDomain, + NaturalLanguageDomain as tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain, + StringDomain as tensorflow_metadata___proto___v0___schema_pb2___StringDomain, + StructDomain as tensorflow_metadata___proto___v0___schema_pb2___StructDomain, + TimeDomain as tensorflow_metadata___proto___v0___schema_pb2___TimeDomain, + TimeOfDayDomain as tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain, + URLDomain as tensorflow_metadata___proto___v0___schema_pb2___URLDomain, + ValueCount as tensorflow_metadata___proto___v0___schema_pb2___ValueCount, +) + from typing import ( Iterable as typing___Iterable, List as typing___List, @@ -36,6 +54,7 @@ from typing import ( Text as typing___Text, Tuple as typing___Tuple, cast as typing___cast, + overload as typing___overload, ) from typing_extensions import ( @@ -43,24 +62,31 @@ from typing_extensions import ( ) -class FeatureSetStatus(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + +class FeatureSetStatus(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> FeatureSetStatus: ... + def Value(cls, name: builtin___str) -> 'FeatureSetStatus': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[FeatureSetStatus]: ... + def values(cls) -> typing___List['FeatureSetStatus']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, FeatureSetStatus]]: ... - STATUS_INVALID = typing___cast(FeatureSetStatus, 0) - STATUS_PENDING = typing___cast(FeatureSetStatus, 1) - STATUS_READY = typing___cast(FeatureSetStatus, 2) -STATUS_INVALID = typing___cast(FeatureSetStatus, 0) -STATUS_PENDING = typing___cast(FeatureSetStatus, 1) -STATUS_READY = typing___cast(FeatureSetStatus, 2) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'FeatureSetStatus']]: ... + STATUS_INVALID = typing___cast('FeatureSetStatus', 0) + STATUS_PENDING = typing___cast('FeatureSetStatus', 1) + STATUS_READY = typing___cast('FeatureSetStatus', 2) +STATUS_INVALID = typing___cast('FeatureSetStatus', 0) +STATUS_PENDING = typing___cast('FeatureSetStatus', 1) +STATUS_READY = typing___cast('FeatureSetStatus', 2) class FeatureSet(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -77,21 +103,21 @@ class FeatureSet(google___protobuf___message___Message): meta : typing___Optional[FeatureSetMeta] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureSet: ... + def FromString(cls, s: builtin___bytes) -> FeatureSet: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"meta",u"spec"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"meta",u"spec"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"meta",u"spec"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> None: ... class FeatureSetSpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int @property def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[EntitySpec]: ... @@ -109,60 +135,200 @@ class FeatureSetSpec(google___protobuf___message___Message): *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, + version : typing___Optional[builtin___int] = None, entities : typing___Optional[typing___Iterable[EntitySpec]] = None, features : typing___Optional[typing___Iterable[FeatureSpec]] = None, max_age : typing___Optional[google___protobuf___duration_pb2___Duration] = None, source : typing___Optional[feast___core___Source_pb2___Source] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureSetSpec: ... + def FromString(cls, s: builtin___bytes) -> FeatureSetSpec: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"max_age",u"source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age",u"source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"entities",u"features",u"max_age",u"name",u"project",u"source",u"version"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"source",b"source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"source",b"source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"entities",b"entities",u"features",b"features",u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"source",b"source",u"version",b"version"]) -> None: ... class EntitySpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text value_type = ... # type: feast___types___Value_pb2___ValueType.Enum + domain = ... # type: typing___Text + + @property + def presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence: ... + + @property + def group_presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup: ... + + @property + def shape(self) -> tensorflow_metadata___proto___v0___schema_pb2___FixedShape: ... + + @property + def value_count(self) -> tensorflow_metadata___proto___v0___schema_pb2___ValueCount: ... + + @property + def int_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___IntDomain: ... + + @property + def float_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___FloatDomain: ... + + @property + def string_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StringDomain: ... + + @property + def bool_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___BoolDomain: ... + + @property + def struct_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StructDomain: ... + + @property + def natural_language_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain: ... + + @property + def image_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___ImageDomain: ... + + @property + def mid_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___MIDDomain: ... + + @property + def url_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___URLDomain: ... + + @property + def time_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeDomain: ... + + @property + def time_of_day_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain: ... def __init__(self, *, name : typing___Optional[typing___Text] = None, value_type : typing___Optional[feast___types___Value_pb2___ValueType.Enum] = None, + presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence] = None, + group_presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup] = None, + shape : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FixedShape] = None, + value_count : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ValueCount] = None, + domain : typing___Optional[typing___Text] = None, + int_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___IntDomain] = None, + float_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FloatDomain] = None, + string_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StringDomain] = None, + bool_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___BoolDomain] = None, + struct_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StructDomain] = None, + natural_language_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain] = None, + image_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ImageDomain] = None, + mid_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___MIDDomain] = None, + url_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___URLDomain] = None, + time_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeDomain] = None, + time_of_day_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> EntitySpec: ... + def FromString(cls, s: builtin___bytes) -> EntitySpec: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value_type"]) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",u"domain",u"domain_info",u"float_domain",u"group_presence",u"image_domain",u"int_domain",u"mid_domain",u"natural_language_domain",u"presence",u"presence_constraints",u"shape",u"shape_type",u"string_domain",u"struct_domain",u"time_domain",u"time_of_day_domain",u"url_domain",u"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",u"domain",u"domain_info",u"float_domain",u"group_presence",u"image_domain",u"int_domain",u"mid_domain",u"name",u"natural_language_domain",u"presence",u"presence_constraints",u"shape",u"shape_type",u"string_domain",u"struct_domain",u"time_domain",u"time_of_day_domain",u"url_domain",u"value_count",u"value_type"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value_type",b"value_type"]) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"name",b"name",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count",u"value_type",b"value_type"]) -> None: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"domain_info",b"domain_info"]) -> typing_extensions___Literal["domain","int_domain","float_domain","string_domain","bool_domain","struct_domain","natural_language_domain","image_domain","mid_domain","url_domain","time_domain","time_of_day_domain"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"presence_constraints",b"presence_constraints"]) -> typing_extensions___Literal["presence","group_presence"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"shape_type",b"shape_type"]) -> typing_extensions___Literal["shape","value_count"]: ... class FeatureSpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text value_type = ... # type: feast___types___Value_pb2___ValueType.Enum + domain = ... # type: typing___Text + + @property + def presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence: ... + + @property + def group_presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup: ... + + @property + def shape(self) -> tensorflow_metadata___proto___v0___schema_pb2___FixedShape: ... + + @property + def value_count(self) -> tensorflow_metadata___proto___v0___schema_pb2___ValueCount: ... + + @property + def int_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___IntDomain: ... + + @property + def float_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___FloatDomain: ... + + @property + def string_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StringDomain: ... + + @property + def bool_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___BoolDomain: ... + + @property + def struct_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StructDomain: ... + + @property + def natural_language_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain: ... + + @property + def image_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___ImageDomain: ... + + @property + def mid_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___MIDDomain: ... + + @property + def url_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___URLDomain: ... + + @property + def time_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeDomain: ... + + @property + def time_of_day_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain: ... def __init__(self, *, name : typing___Optional[typing___Text] = None, value_type : typing___Optional[feast___types___Value_pb2___ValueType.Enum] = None, + presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence] = None, + group_presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup] = None, + shape : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FixedShape] = None, + value_count : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ValueCount] = None, + domain : typing___Optional[typing___Text] = None, + int_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___IntDomain] = None, + float_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FloatDomain] = None, + string_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StringDomain] = None, + bool_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___BoolDomain] = None, + struct_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StructDomain] = None, + natural_language_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain] = None, + image_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ImageDomain] = None, + mid_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___MIDDomain] = None, + url_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___URLDomain] = None, + time_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeDomain] = None, + time_of_day_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureSpec: ... + def FromString(cls, s: builtin___bytes) -> FeatureSpec: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value_type"]) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",u"domain",u"domain_info",u"float_domain",u"group_presence",u"image_domain",u"int_domain",u"mid_domain",u"natural_language_domain",u"presence",u"presence_constraints",u"shape",u"shape_type",u"string_domain",u"struct_domain",u"time_domain",u"time_of_day_domain",u"url_domain",u"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",u"domain",u"domain_info",u"float_domain",u"group_presence",u"image_domain",u"int_domain",u"mid_domain",u"name",u"natural_language_domain",u"presence",u"presence_constraints",u"shape",u"shape_type",u"string_domain",u"struct_domain",u"time_domain",u"time_of_day_domain",u"url_domain",u"value_count",u"value_type"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value_type",b"value_type"]) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"name",b"name",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count",u"value_type",b"value_type"]) -> None: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"domain_info",b"domain_info"]) -> typing_extensions___Literal["domain","int_domain","float_domain","string_domain","bool_domain","struct_domain","natural_language_domain","image_domain","mid_domain","url_domain","time_domain","time_of_day_domain"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"presence_constraints",b"presence_constraints"]) -> typing_extensions___Literal["presence","group_presence"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"shape_type",b"shape_type"]) -> typing_extensions___Literal["shape","value_count"]: ... class FeatureSetMeta(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -177,12 +343,12 @@ class FeatureSetMeta(google___protobuf___message___Message): status : typing___Optional[FeatureSetStatus] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureSetMeta: ... + def FromString(cls, s: builtin___bytes) -> FeatureSetMeta: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"created_timestamp",u"status"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp",u"status",b"status"]) -> None: ... diff --git a/sdk/python/feast/core/Source_pb2.pyi b/sdk/python/feast/core/Source_pb2.pyi index 0521ac34f8..fec428a07a 100644 --- a/sdk/python/feast/core/Source_pb2.pyi +++ b/sdk/python/feast/core/Source_pb2.pyi @@ -22,22 +22,29 @@ from typing_extensions import ( ) -class SourceType(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + +class SourceType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> SourceType: ... + def Value(cls, name: builtin___str) -> 'SourceType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[SourceType]: ... + def values(cls) -> typing___List['SourceType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, SourceType]]: ... - INVALID = typing___cast(SourceType, 0) - KAFKA = typing___cast(SourceType, 1) -INVALID = typing___cast(SourceType, 0) -KAFKA = typing___cast(SourceType, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'SourceType']]: ... + INVALID = typing___cast('SourceType', 0) + KAFKA = typing___cast('SourceType', 1) +INVALID = typing___cast('SourceType', 0) +KAFKA = typing___cast('SourceType', 1) class Source(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -52,14 +59,14 @@ class Source(google___protobuf___message___Message): kafka_source_config : typing___Optional[KafkaSourceConfig] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Source: ... + def FromString(cls, s: builtin___bytes) -> Source: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",u"source_config"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",u"source_config"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"kafka_source_config",u"source_config",u"type"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config",u"type",b"type"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"source_config",b"source_config"]) -> typing_extensions___Literal["kafka_source_config"]: ... @@ -74,7 +81,7 @@ class KafkaSourceConfig(google___protobuf___message___Message): topic : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> KafkaSourceConfig: ... + def FromString(cls, s: builtin___bytes) -> KafkaSourceConfig: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): diff --git a/sdk/python/feast/core/Store_pb2.py b/sdk/python/feast/core/Store_pb2.py index 716a597b9a..9adf197b0a 100644 --- a/sdk/python/feast/core/Store_pb2.py +++ b/sdk/python/feast/core/Store_pb2.py @@ -20,7 +20,7 @@ package='feast.core', syntax='proto3', serialized_options=_b('\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x16\x66\x65\x61st/core/Store.proto\x12\nfeast.core\"\xca\x04\n\x05Store\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.feast.core.Store.StoreType\x12\x35\n\rsubscriptions\x18\x04 \x03(\x0b\x32\x1e.feast.core.Store.Subscription\x12\x35\n\x0credis_config\x18\x0b \x01(\x0b\x32\x1d.feast.core.Store.RedisConfigH\x00\x12;\n\x0f\x62igquery_config\x18\x0c \x01(\x0b\x32 .feast.core.Store.BigQueryConfigH\x00\x12=\n\x10\x63\x61ssandra_config\x18\r \x01(\x0b\x32!.feast.core.Store.CassandraConfigH\x00\x1a)\n\x0bRedisConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x1a\x38\n\x0e\x42igQueryConfig\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\ndataset_id\x18\x02 \x01(\t\x1a-\n\x0f\x43\x61ssandraConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x1a>\n\x0cSubscription\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"@\n\tStoreType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05REDIS\x10\x01\x12\x0c\n\x08\x42IGQUERY\x10\x02\x12\r\n\tCASSANDRA\x10\x03\x42\x08\n\x06\x63onfigBI\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_pb=_b('\n\x16\x66\x65\x61st/core/Store.proto\x12\nfeast.core\"\xfb\x04\n\x05Store\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.feast.core.Store.StoreType\x12\x35\n\rsubscriptions\x18\x04 \x03(\x0b\x32\x1e.feast.core.Store.Subscription\x12\x35\n\x0credis_config\x18\x0b \x01(\x0b\x32\x1d.feast.core.Store.RedisConfigH\x00\x12;\n\x0f\x62igquery_config\x18\x0c \x01(\x0b\x32 .feast.core.Store.BigQueryConfigH\x00\x12=\n\x10\x63\x61ssandra_config\x18\r \x01(\x0b\x32!.feast.core.Store.CassandraConfigH\x00\x1aZ\n\x0bRedisConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x1a\n\x12initial_backoff_ms\x18\x03 \x01(\x05\x12\x13\n\x0bmax_retries\x18\x04 \x01(\x05\x1a\x38\n\x0e\x42igQueryConfig\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\ndataset_id\x18\x02 \x01(\t\x1a-\n\x0f\x43\x61ssandraConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x1a>\n\x0cSubscription\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"@\n\tStoreType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05REDIS\x10\x01\x12\x0c\n\x08\x42IGQUERY\x10\x02\x12\r\n\tCASSANDRA\x10\x03\x42\x08\n\x06\x63onfigBI\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') ) @@ -50,8 +50,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=551, - serialized_end=615, + serialized_start=600, + serialized_end=664, ) _sym_db.RegisterEnumDescriptor(_STORE_STORETYPE) @@ -77,6 +77,20 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='initial_backoff_ms', full_name='feast.core.Store.RedisConfig.initial_backoff_ms', index=2, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='max_retries', full_name='feast.core.Store.RedisConfig.max_retries', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -90,7 +104,7 @@ oneofs=[ ], serialized_start=339, - serialized_end=380, + serialized_end=429, ) _STORE_BIGQUERYCONFIG = _descriptor.Descriptor( @@ -126,8 +140,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=382, - serialized_end=438, + serialized_start=431, + serialized_end=487, ) _STORE_CASSANDRACONFIG = _descriptor.Descriptor( @@ -163,8 +177,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=440, - serialized_end=485, + serialized_start=489, + serialized_end=534, ) _STORE_SUBSCRIPTION = _descriptor.Descriptor( @@ -207,8 +221,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=487, - serialized_end=549, + serialized_start=536, + serialized_end=598, ) _STORE = _descriptor.Descriptor( @@ -277,7 +291,7 @@ index=0, containing_type=None, fields=[]), ], serialized_start=39, - serialized_end=625, + serialized_end=674, ) _STORE_REDISCONFIG.containing_type = _STORE diff --git a/sdk/python/feast/core/Store_pb2.pyi b/sdk/python/feast/core/Store_pb2.pyi index 541bcd329b..049969de4a 100644 --- a/sdk/python/feast/core/Store_pb2.pyi +++ b/sdk/python/feast/core/Store_pb2.pyi @@ -27,47 +27,58 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + class Store(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class StoreType(int): + class StoreType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> Store.StoreType: ... + def Value(cls, name: builtin___str) -> 'Store.StoreType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[Store.StoreType]: ... + def values(cls) -> typing___List['Store.StoreType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, Store.StoreType]]: ... - INVALID = typing___cast(Store.StoreType, 0) - REDIS = typing___cast(Store.StoreType, 1) - BIGQUERY = typing___cast(Store.StoreType, 2) - CASSANDRA = typing___cast(Store.StoreType, 3) - INVALID = typing___cast(Store.StoreType, 0) - REDIS = typing___cast(Store.StoreType, 1) - BIGQUERY = typing___cast(Store.StoreType, 2) - CASSANDRA = typing___cast(Store.StoreType, 3) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'Store.StoreType']]: ... + INVALID = typing___cast('Store.StoreType', 0) + REDIS = typing___cast('Store.StoreType', 1) + BIGQUERY = typing___cast('Store.StoreType', 2) + CASSANDRA = typing___cast('Store.StoreType', 3) + INVALID = typing___cast('Store.StoreType', 0) + REDIS = typing___cast('Store.StoreType', 1) + BIGQUERY = typing___cast('Store.StoreType', 2) + CASSANDRA = typing___cast('Store.StoreType', 3) class RedisConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... host = ... # type: typing___Text - port = ... # type: int + port = ... # type: builtin___int + initial_backoff_ms = ... # type: builtin___int + max_retries = ... # type: builtin___int def __init__(self, *, host : typing___Optional[typing___Text] = None, - port : typing___Optional[int] = None, + port : typing___Optional[builtin___int] = None, + initial_backoff_ms : typing___Optional[builtin___int] = None, + max_retries : typing___Optional[builtin___int] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Store.RedisConfig: ... + def FromString(cls, s: builtin___bytes) -> Store.RedisConfig: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"host",u"port"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"host",u"initial_backoff_ms",u"max_retries",u"port"]) -> None: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"host",b"host",u"port",b"port"]) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"host",b"host",u"initial_backoff_ms",b"initial_backoff_ms",u"max_retries",b"max_retries",u"port",b"port"]) -> None: ... class BigQueryConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -80,7 +91,7 @@ class Store(google___protobuf___message___Message): dataset_id : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Store.BigQueryConfig: ... + def FromString(cls, s: builtin___bytes) -> Store.BigQueryConfig: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -91,15 +102,15 @@ class Store(google___protobuf___message___Message): class CassandraConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... host = ... # type: typing___Text - port = ... # type: int + port = ... # type: builtin___int def __init__(self, *, host : typing___Optional[typing___Text] = None, - port : typing___Optional[int] = None, + port : typing___Optional[builtin___int] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Store.CassandraConfig: ... + def FromString(cls, s: builtin___bytes) -> Store.CassandraConfig: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -120,7 +131,7 @@ class Store(google___protobuf___message___Message): version : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Store.Subscription: ... + def FromString(cls, s: builtin___bytes) -> Store.Subscription: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -153,13 +164,13 @@ class Store(google___protobuf___message___Message): cassandra_config : typing___Optional[Store.CassandraConfig] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Store: ... + def FromString(cls, s: builtin___bytes) -> Store: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",u"cassandra_config",u"config",u"redis_config"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",u"cassandra_config",u"config",u"redis_config"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"bigquery_config",u"cassandra_config",u"config",u"name",u"redis_config",u"subscriptions",u"type"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"redis_config",b"redis_config"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"redis_config",b"redis_config"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"name",b"name",u"redis_config",b"redis_config",u"subscriptions",b"subscriptions",u"type",b"type"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"config",b"config"]) -> typing_extensions___Literal["redis_config","bigquery_config","cassandra_config"]: ... diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 795758bc41..2e50ccb3d4 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from tensorflow_metadata.proto.v0 import schema_pb2 -from feast.value_type import ValueType from feast.core.FeatureSet_pb2 import EntitySpec as EntityProto -from feast.types import Value_pb2 as ValueTypeProto from feast.field import Field +from feast.types import Value_pb2 as ValueTypeProto +from feast.value_type import ValueType class Entity(Field): @@ -29,17 +30,42 @@ def to_proto(self) -> EntityProto: Returns EntitySpec object """ value_type = ValueTypeProto.ValueType.Enum.Value(self.dtype.name) - return EntityProto(name=self.name, value_type=value_type) + return EntityProto( + name=self.name, + value_type=value_type, + presence=self.presence, + group_presence=self.group_presence, + shape=self.shape, + value_count=self.value_count, + domain=self.domain, + int_domain=self.int_domain, + float_domain=self.float_domain, + string_domain=self.string_domain, + bool_domain=self.bool_domain, + struct_domain=self.struct_domain, + natural_language_domain=self.natural_language_domain, + image_domain=self.image_domain, + mid_domain=self.mid_domain, + url_domain=self.url_domain, + time_domain=self.time_domain, + time_of_day_domain=self.time_of_day_domain, + ) @classmethod - def from_proto(cls, entity_proto: EntityProto): + def from_proto(cls, entity_proto: EntityProto, schema: schema_pb2.Schema = None): """ Creates a Feast Entity object from its Protocol Buffer representation Args: entity_proto: EntitySpec protobuf object + schema: Schema from Tensorflow metadata, will be used to reference domain + defined at the schema level Returns: Entity object """ - return cls(name=entity_proto.name, dtype=ValueType(entity_proto.value_type)) + entity = cls(name=entity_proto.name, dtype=ValueType(entity_proto.value_type)) + entity.update_presence_constraints(entity_proto) + entity.update_shape_type(entity_proto) + entity.update_domain_info(entity_proto, schema) + return entity diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index c7e3d7af8b..de63ff68d6 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from tensorflow_metadata.proto.v0 import schema_pb2 -from feast.value_type import ValueType from feast.core.FeatureSet_pb2 import FeatureSpec as FeatureProto -from feast.types import Value_pb2 as ValueTypeProto from feast.field import Field +from feast.types import Value_pb2 as ValueTypeProto +from feast.value_type import ValueType class Feature(Field): @@ -24,9 +25,43 @@ class Feature(Field): def to_proto(self) -> FeatureProto: """Converts Feature object to its Protocol Buffer representation""" value_type = ValueTypeProto.ValueType.Enum.Value(self.dtype.name) - return FeatureProto(name=self.name, value_type=value_type) + return FeatureProto( + name=self.name, + value_type=value_type, + presence=self.presence, + group_presence=self.group_presence, + shape=self.shape, + value_count=self.value_count, + domain=self.domain, + int_domain=self.int_domain, + float_domain=self.float_domain, + string_domain=self.string_domain, + bool_domain=self.bool_domain, + struct_domain=self.struct_domain, + natural_language_domain=self.natural_language_domain, + image_domain=self.image_domain, + mid_domain=self.mid_domain, + url_domain=self.url_domain, + time_domain=self.time_domain, + time_of_day_domain=self.time_of_day_domain, + ) @classmethod - def from_proto(cls, feature_proto: FeatureProto): - """Converts Protobuf Feature to its SDK equivalent""" - return cls(name=feature_proto.name, dtype=ValueType(feature_proto.value_type)) + def from_proto(cls, feature_proto: FeatureProto, schema: schema_pb2.Schema = None): + """ + + Args: + feature_proto: FeatureSpec protobuf object + schema: Schema from Tensorflow metadata, will be used to reference domain + defined at the schema level + + Returns: + Feature object + """ + feature = cls( + name=feature_proto.name, dtype=ValueType(feature_proto.value_type) + ) + feature.update_presence_constraints(feature_proto) + feature.update_shape_type(feature_proto) + feature.update_domain_info(feature_proto, schema) + return feature diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index c47c51e5a2..9f0e7ec418 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -11,14 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import warnings from collections import OrderedDict from typing import Dict from typing import List, Optional import pandas as pd import pyarrow as pa +from google.protobuf import json_format +from google.protobuf.duration_pb2 import Duration +from google.protobuf.json_format import MessageToJson +from google.protobuf.message import Message +from pandas.api.types import is_datetime64_ns_dtype +from pyarrow.lib import TimestampType +from tensorflow_metadata.proto.v0 import schema_pb2 + +from feast.value_type import ValueType from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto @@ -29,15 +37,6 @@ from feast.type_map import DATETIME_COLUMN from feast.type_map import pa_to_feast_value_type from feast.type_map import python_type_to_feast_value_type -from google.protobuf import json_format -from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto -from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto -from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto -from google.protobuf.duration_pb2 import Duration -from feast.type_map import python_type_to_feast_value_type -from google.protobuf.json_format import MessageToJson -from pandas.api.types import is_datetime64_ns_dtype -from pyarrow.lib import TimestampType class FeatureSet: @@ -663,6 +662,92 @@ def is_valid(self): if len(self.entities) == 0: raise ValueError(f"No entities found in feature set {self.name}") + def import_tfx_schema(self, schema: schema_pb2.Schema): + """ + Updates presence_constraints, shape_type and domain_info for all fields + (features and entities) in the FeatureSet from schema in the Tensorflow metadata. + + Args: + schema: Schema from Tensorflow metadata + + Returns: + None + + """ + for feature_from_tfx_schema in schema.feature: + if feature_from_tfx_schema.name in self._fields.keys(): + field = self._fields[feature_from_tfx_schema.name] + field.update_presence_constraints(feature_from_tfx_schema) + field.update_shape_type(feature_from_tfx_schema) + field.update_domain_info(feature_from_tfx_schema, schema) + else: + warnings.warn( + f"The provided schema contains feature name '{feature_from_tfx_schema.name}' " + f"that does not exist in the FeatureSet '{self.name}' in Feast" + ) + + def export_tfx_schema(self) -> schema_pb2.Schema: + """ + Create a Tensorflow metadata schema from a FeatureSet. + + Returns: + Tensorflow metadata schema. + + """ + schema = schema_pb2.Schema() + + # List of attributes to copy from fields in the FeatureSet to feature in + # Tensorflow metadata schema where the attribute name is the same. + attributes_to_copy_from_field_to_feature = [ + "name", + "presence", + "group_presence", + "shape", + "value_count", + "domain", + "int_domain", + "float_domain", + "string_domain", + "bool_domain", + "struct_domain", + "_natural_language_domain", + "image_domain", + "mid_domain", + "url_domain", + "time_domain", + "time_of_day_domain", + ] + + for _, field in self._fields.items(): + feature = schema_pb2.Feature() + for attr in attributes_to_copy_from_field_to_feature: + if getattr(field, attr) is None: + # This corresponds to an unset member in the proto Oneof field. + continue + if issubclass(type(getattr(feature, attr)), Message): + # Proto message field to copy is an "embedded" field, so MergeFrom() + # method must be used. + getattr(feature, attr).MergeFrom(getattr(field, attr)) + elif issubclass(type(getattr(feature, attr)), (int, str, bool)): + # Proto message field is a simple Python type, so setattr() + # can be used. + setattr(feature, attr, getattr(field, attr)) + else: + warnings.warn( + f"Attribute '{attr}' cannot be copied from Field " + f"'{field.name}' in FeatureSet '{self.name}' to a " + f"Feature in the Tensorflow metadata schema, because" + f"the type is neither a Protobuf message or Python " + f"int, str and bool" + ) + # "type" attr is handled separately because the attribute name is different + # ("dtype" in field and "type" in Feature) and "type" in Feature is only + # a subset of "dtype". + feature.type = field.dtype.to_tfx_schema_feature_type() + schema.feature.append(feature) + + return schema + @classmethod def from_yaml(cls, yml: str): """ diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 2efd4587ff..19a1d0d6a2 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings + +from tensorflow_metadata.proto.v0 import schema_pb2 from feast.value_type import ValueType @@ -26,6 +29,22 @@ def __init__(self, name: str, dtype: ValueType): if not isinstance(dtype, ValueType): raise ValueError("dtype is not a valid ValueType") self._dtype = dtype + self._presence = None + self._group_presence = None + self._shape = None + self._value_count = None + self._domain = None + self._int_domain = None + self._float_domain = None + self._string_domain = None + self._bool_domain = None + self._struct_domain = None + self._natural_language_domain = None + self._image_domain = None + self._mid_domain = None + self._url_domain = None + self._time_domain = None + self._time_of_day_domain = None def __eq__(self, other): if self.name != other.name or self.dtype != other.dtype: @@ -46,6 +65,300 @@ def dtype(self) -> ValueType: """ return self._dtype + @property + def presence(self) -> schema_pb2.FeaturePresence: + """ + Getter for presence of this field + """ + return self._presence + + @presence.setter + def presence(self, presence: schema_pb2.FeaturePresence): + if not isinstance(presence, schema_pb2.FeaturePresence): + raise TypeError("presence must be of FeaturePresence type") + self._clear_presence_constraints() + self._presence = presence + + @property + def group_presence(self) -> schema_pb2.FeaturePresenceWithinGroup: + """ + Getter for group_presence of this field + """ + return self._group_presence + + @group_presence.setter + def group_presence(self, group_presence: schema_pb2.FeaturePresenceWithinGroup): + if not isinstance(group_presence, schema_pb2.FeaturePresenceWithinGroup): + raise TypeError("group_presence must be of FeaturePresenceWithinGroup type") + self._clear_presence_constraints() + self._group_presence = group_presence + + @property + def shape(self) -> schema_pb2.FixedShape: + """ + Getter for shape of this field + """ + return self._shape + + @shape.setter + def shape(self, shape: schema_pb2.FixedShape): + if not isinstance(shape, schema_pb2.FixedShape): + raise TypeError("shape must be of FixedShape type") + self._clear_shape_type() + self._shape = shape + + @property + def value_count(self) -> schema_pb2.ValueCount: + """ + Getter for value_count of this field + """ + return self._value_count + + @value_count.setter + def value_count(self, value_count: schema_pb2.ValueCount): + if not isinstance(value_count, schema_pb2.ValueCount): + raise TypeError("value_count must be of ValueCount type") + self._clear_shape_type() + self._value_count = value_count + + @property + def domain(self) -> str: + """ + Getter for domain of this field + """ + return self._domain + + @domain.setter + def domain(self, domain: str): + if not isinstance(domain, str): + raise TypeError("domain must be of str type") + self._clear_domain_info() + self._domain = domain + + @property + def int_domain(self) -> schema_pb2.IntDomain: + """ + Getter for int_domain of this field + """ + return self._int_domain + + @int_domain.setter + def int_domain(self, int_domain: schema_pb2.IntDomain): + if not isinstance(int_domain, schema_pb2.IntDomain): + raise TypeError("int_domain must be of IntDomain type") + self._clear_domain_info() + self._int_domain = int_domain + + @property + def float_domain(self) -> schema_pb2.FloatDomain: + """ + Getter for float_domain of this field + """ + return self._float_domain + + @float_domain.setter + def float_domain(self, float_domain: schema_pb2.FloatDomain): + if not isinstance(float_domain, schema_pb2.FloatDomain): + raise TypeError("float_domain must be of FloatDomain type") + self._clear_domain_info() + self._float_domain = float_domain + + @property + def string_domain(self) -> schema_pb2.StringDomain: + """ + Getter for string_domain of this field + """ + return self._string_domain + + @string_domain.setter + def string_domain(self, string_domain: schema_pb2.StringDomain): + if not isinstance(string_domain, schema_pb2.StringDomain): + raise TypeError("string_domain must be of StringDomain type") + self._clear_domain_info() + self._string_domain = string_domain + + @property + def bool_domain(self) -> schema_pb2.BoolDomain: + """ + Getter for bool_domain of this field + """ + return self._bool_domain + + @bool_domain.setter + def bool_domain(self, bool_domain: schema_pb2.BoolDomain): + if not isinstance(bool_domain, schema_pb2.BoolDomain): + raise TypeError("bool_domain must be of BoolDomain type") + self._clear_domain_info() + self._bool_domain = bool_domain + + @property + def struct_domain(self) -> schema_pb2.StructDomain: + """ + Getter for struct_domain of this field + """ + return self._struct_domain + + @struct_domain.setter + def struct_domain(self, struct_domain: schema_pb2.StructDomain): + if not isinstance(struct_domain, schema_pb2.StructDomain): + raise TypeError("struct_domain must be of StructDomain type") + self._clear_domain_info() + self._struct_domain = struct_domain + + @property + def natural_language_domain(self) -> schema_pb2.NaturalLanguageDomain: + """ + Getter for natural_language_domain of this field + """ + return self._natural_language_domain + + @natural_language_domain.setter + def natural_language_domain( + self, natural_language_domain: schema_pb2.NaturalLanguageDomain + ): + if not isinstance(natural_language_domain, schema_pb2.NaturalLanguageDomain): + raise TypeError( + "natural_language_domain must be of NaturalLanguageDomain type" + ) + self._clear_domain_info() + self._natural_language_domain = natural_language_domain + + @property + def image_domain(self) -> schema_pb2.ImageDomain: + """ + Getter for image_domain of this field + """ + return self._image_domain + + @image_domain.setter + def image_domain(self, image_domain: schema_pb2.ImageDomain): + if not isinstance(image_domain, schema_pb2.ImageDomain): + raise TypeError("image_domain must be of ImageDomain type") + self._clear_domain_info() + self._image_domain = image_domain + + @property + def mid_domain(self) -> schema_pb2.MIDDomain: + """ + Getter for mid_domain of this field + """ + return self._mid_domain + + @mid_domain.setter + def mid_domain(self, mid_domain: schema_pb2.MIDDomain): + if not isinstance(mid_domain, schema_pb2.MIDDomain): + raise TypeError("mid_domain must be of MIDDomain type") + self._clear_domain_info() + self._mid_domain = mid_domain + + @property + def url_domain(self) -> schema_pb2.URLDomain: + """ + Getter for url_domain of this field + """ + return self._url_domain + + @url_domain.setter + def url_domain(self, url_domain: schema_pb2.URLDomain): + if not isinstance(url_domain, schema_pb2.URLDomain): + raise TypeError("url_domain must be of URLDomain type") + self._clear_domain_info() + self.url_domain = url_domain + + @property + def time_domain(self) -> schema_pb2.TimeDomain: + """ + Getter for time_domain of this field + """ + return self._time_domain + + @time_domain.setter + def time_domain(self, time_domain: schema_pb2.TimeDomain): + if not isinstance(time_domain, schema_pb2.TimeDomain): + raise TypeError("time_domain must be of TimeDomain type") + self._clear_domain_info() + self._time_domain = time_domain + + @property + def time_of_day_domain(self) -> schema_pb2.TimeOfDayDomain: + """ + Getter for time_of_day_domain of this field + """ + return self._time_of_day_domain + + @time_of_day_domain.setter + def time_of_day_domain(self, time_of_day_domain) -> schema_pb2.TimeOfDayDomain: + if not isinstance(time_of_day_domain, schema_pb2.TimeOfDayDomain): + raise TypeError("time_of_day_domain must be of TimeOfDayDomain type") + self._clear_domain_info() + self._time_of_day_domain = time_of_day_domain + + def update_presence_constraints(self, feature: schema_pb2.Feature): + presence_constraints_case = feature.WhichOneof("presence_constraints") + if presence_constraints_case == "presence": + self.presence = feature.presence + elif presence_constraints_case == "group_presence": + self.group_presence = feature.group_presence + + def update_shape_type(self, feature: schema_pb2.Feature): + shape_type_case = feature.WhichOneof("shape_type") + if shape_type_case == "shape": + self.shape = feature.shape + elif shape_type_case == "value_count": + self.value_count = feature.value_count + + def update_domain_info( + self, feature: schema_pb2.Feature, schema: schema_pb2.Schema = None + ): + domain_info_case = feature.WhichOneof("domain_info") + if domain_info_case == "domain": + domain_ref = feature.domain + if schema is None: + warnings.warn( + f"Schema is not provided so domain '{domain_ref}' cannot be " + f"referenced and domain for field '{self.name}' will not be updated." + ) + else: + domain_ref_to_string_domain = {d.name: d for d in schema.string_domain} + domain_ref_to_float_domain = {d.name: d for d in schema.float_domain} + domain_ref_to_int_domain = {d.name: d for d in schema.int_domain} + + if domain_ref in domain_ref_to_string_domain: + self.string_domain = domain_ref_to_string_domain[domain_ref] + elif domain_ref in domain_ref_to_float_domain: + self.float_domain = domain_ref_to_float_domain[domain_ref] + elif domain_ref in domain_ref_to_int_domain: + self.int_domain = domain_ref_to_int_domain[domain_ref] + else: + raise ValueError( + f"Reference to a domain '{domain_ref}' is missing in the schema. " + f"Please check the string_domain, float_domain and int_domain" + f"fields in the schema of your Tensorflow metadata, making sure" + f"that the domain referenced exists." + ) + elif domain_info_case == "int_domain": + self.int_domain = feature.int_domain + elif domain_info_case == "float_domain": + self.float_domain = feature.float_domain + elif domain_info_case == "string_domain": + self.string_domain = feature.string_domain + elif domain_info_case == "bool_domain": + self.bool_domain = feature.bool_domain + elif domain_info_case == "struct_domain": + self.struct_domain = feature.struct_domain + elif domain_info_case == "natural_language_domain": + self.natural_language_domain = feature.natural_language_domain + elif domain_info_case == "image_domain": + self.image_domain = feature.image_domain + elif domain_info_case == "mid_domain": + self.mid_domain = feature.mid_domain + elif domain_info_case == "url_domain": + self.url_domain = feature.url_domain + elif domain_info_case == "time_domain": + self.time_domain = feature.time_domain + elif domain_info_case == "time_of_day_domain": + self.time_of_day_domain = feature.time_of_day_domain + def to_proto(self): """ Unimplemented to_proto method for a field. This should be extended. @@ -57,3 +370,25 @@ def from_proto(self, proto): Unimplemented from_proto method for a field. This should be extended. """ pass + + def _clear_presence_constraints(self): + self._presence = None + self._group_presence = None + + def _clear_shape_type(self): + self._shape = None + self._value_count = None + + def _clear_domain_info(self): + self._domain = None + self._int_domain = None + self._float_domain = None + self._string_domain = None + self._bool_domain = None + self._struct_domain = None + self._natural_language_domain = None + self._image_domain = None + self._mid_domain = None + self._url_domain = None + self._time_domain = None + self._time_of_day_domain = None diff --git a/sdk/python/feast/loaders/yaml.py b/sdk/python/feast/loaders/yaml.py index 130a71a3d0..624bc47d49 100644 --- a/sdk/python/feast/loaders/yaml.py +++ b/sdk/python/feast/loaders/yaml.py @@ -57,7 +57,8 @@ def _get_yaml_contents(yml: str) -> str: yml_content = yml else: raise Exception( - f"Invalid YAML provided. Please provide either a file path or YAML string: ${yml}" + f"Invalid YAML provided. Please provide either a file path or YAML string.\n" + f"Provided YAML: {yml}" ) return yml_content diff --git a/sdk/python/feast/serving/ServingService_pb2.pyi b/sdk/python/feast/serving/ServingService_pb2.pyi index e10245d6c7..6fab5e46c9 100644 --- a/sdk/python/feast/serving/ServingService_pb2.pyi +++ b/sdk/python/feast/serving/ServingService_pb2.pyi @@ -42,79 +42,86 @@ from typing_extensions import ( ) -class FeastServingType(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + +class FeastServingType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> FeastServingType: ... + def Value(cls, name: builtin___str) -> 'FeastServingType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[FeastServingType]: ... + def values(cls) -> typing___List['FeastServingType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, FeastServingType]]: ... - FEAST_SERVING_TYPE_INVALID = typing___cast(FeastServingType, 0) - FEAST_SERVING_TYPE_ONLINE = typing___cast(FeastServingType, 1) - FEAST_SERVING_TYPE_BATCH = typing___cast(FeastServingType, 2) -FEAST_SERVING_TYPE_INVALID = typing___cast(FeastServingType, 0) -FEAST_SERVING_TYPE_ONLINE = typing___cast(FeastServingType, 1) -FEAST_SERVING_TYPE_BATCH = typing___cast(FeastServingType, 2) - -class JobType(int): + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'FeastServingType']]: ... + FEAST_SERVING_TYPE_INVALID = typing___cast('FeastServingType', 0) + FEAST_SERVING_TYPE_ONLINE = typing___cast('FeastServingType', 1) + FEAST_SERVING_TYPE_BATCH = typing___cast('FeastServingType', 2) +FEAST_SERVING_TYPE_INVALID = typing___cast('FeastServingType', 0) +FEAST_SERVING_TYPE_ONLINE = typing___cast('FeastServingType', 1) +FEAST_SERVING_TYPE_BATCH = typing___cast('FeastServingType', 2) + +class JobType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> JobType: ... + def Value(cls, name: builtin___str) -> 'JobType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[JobType]: ... + def values(cls) -> typing___List['JobType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, JobType]]: ... - JOB_TYPE_INVALID = typing___cast(JobType, 0) - JOB_TYPE_DOWNLOAD = typing___cast(JobType, 1) -JOB_TYPE_INVALID = typing___cast(JobType, 0) -JOB_TYPE_DOWNLOAD = typing___cast(JobType, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'JobType']]: ... + JOB_TYPE_INVALID = typing___cast('JobType', 0) + JOB_TYPE_DOWNLOAD = typing___cast('JobType', 1) +JOB_TYPE_INVALID = typing___cast('JobType', 0) +JOB_TYPE_DOWNLOAD = typing___cast('JobType', 1) -class JobStatus(int): +class JobStatus(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> JobStatus: ... + def Value(cls, name: builtin___str) -> 'JobStatus': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[JobStatus]: ... + def values(cls) -> typing___List['JobStatus']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, JobStatus]]: ... - JOB_STATUS_INVALID = typing___cast(JobStatus, 0) - JOB_STATUS_PENDING = typing___cast(JobStatus, 1) - JOB_STATUS_RUNNING = typing___cast(JobStatus, 2) - JOB_STATUS_DONE = typing___cast(JobStatus, 3) -JOB_STATUS_INVALID = typing___cast(JobStatus, 0) -JOB_STATUS_PENDING = typing___cast(JobStatus, 1) -JOB_STATUS_RUNNING = typing___cast(JobStatus, 2) -JOB_STATUS_DONE = typing___cast(JobStatus, 3) - -class DataFormat(int): + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'JobStatus']]: ... + JOB_STATUS_INVALID = typing___cast('JobStatus', 0) + JOB_STATUS_PENDING = typing___cast('JobStatus', 1) + JOB_STATUS_RUNNING = typing___cast('JobStatus', 2) + JOB_STATUS_DONE = typing___cast('JobStatus', 3) +JOB_STATUS_INVALID = typing___cast('JobStatus', 0) +JOB_STATUS_PENDING = typing___cast('JobStatus', 1) +JOB_STATUS_RUNNING = typing___cast('JobStatus', 2) +JOB_STATUS_DONE = typing___cast('JobStatus', 3) + +class DataFormat(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> DataFormat: ... + def Value(cls, name: builtin___str) -> 'DataFormat': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[DataFormat]: ... + def values(cls) -> typing___List['DataFormat']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, DataFormat]]: ... - DATA_FORMAT_INVALID = typing___cast(DataFormat, 0) - DATA_FORMAT_AVRO = typing___cast(DataFormat, 1) -DATA_FORMAT_INVALID = typing___cast(DataFormat, 0) -DATA_FORMAT_AVRO = typing___cast(DataFormat, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'DataFormat']]: ... + DATA_FORMAT_INVALID = typing___cast('DataFormat', 0) + DATA_FORMAT_AVRO = typing___cast('DataFormat', 1) +DATA_FORMAT_INVALID = typing___cast('DataFormat', 0) +DATA_FORMAT_AVRO = typing___cast('DataFormat', 1) class GetFeastServingInfoRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -122,7 +129,7 @@ class GetFeastServingInfoRequest(google___protobuf___message___Message): def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeastServingInfoRequest: ... + def FromString(cls, s: builtin___bytes) -> GetFeastServingInfoRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... @@ -139,7 +146,7 @@ class GetFeastServingInfoResponse(google___protobuf___message___Message): job_staging_location : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetFeastServingInfoResponse: ... + def FromString(cls, s: builtin___bytes) -> GetFeastServingInfoResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -151,7 +158,7 @@ class FeatureReference(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int @property def max_age(self) -> google___protobuf___duration_pb2___Duration: ... @@ -160,18 +167,18 @@ class FeatureReference(google___protobuf___message___Message): *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, + version : typing___Optional[builtin___int] = None, max_age : typing___Optional[google___protobuf___duration_pb2___Duration] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureReference: ... + def FromString(cls, s: builtin___bytes) -> FeatureReference: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"max_age"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"max_age",u"name",u"project",u"version"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... class GetOnlineFeaturesRequest(google___protobuf___message___Message): @@ -191,14 +198,14 @@ class GetOnlineFeaturesRequest(google___protobuf___message___Message): value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest.EntityRow.FieldsEntry: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest.EntityRow.FieldsEntry: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... @@ -214,17 +221,17 @@ class GetOnlineFeaturesRequest(google___protobuf___message___Message): fields : typing___Optional[typing___Mapping[typing___Text, feast___types___Value_pb2___Value]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest.EntityRow: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest.EntityRow: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"entity_timestamp",u"fields"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp",u"fields",b"fields"]) -> None: ... - omit_entities_in_response = ... # type: bool + omit_entities_in_response = ... # type: builtin___bool @property def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[FeatureReference]: ... @@ -236,10 +243,10 @@ class GetOnlineFeaturesRequest(google___protobuf___message___Message): *, features : typing___Optional[typing___Iterable[FeatureReference]] = None, entity_rows : typing___Optional[typing___Iterable[GetOnlineFeaturesRequest.EntityRow]] = None, - omit_entities_in_response : typing___Optional[bool] = None, + omit_entities_in_response : typing___Optional[builtin___bool] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -262,14 +269,14 @@ class GetBatchFeaturesRequest(google___protobuf___message___Message): dataset_source : typing___Optional[DatasetSource] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetBatchFeaturesRequest: ... + def FromString(cls, s: builtin___bytes) -> GetBatchFeaturesRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",u"features"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"features",b"features"]) -> None: ... class GetOnlineFeaturesResponse(google___protobuf___message___Message): @@ -289,14 +296,14 @@ class GetOnlineFeaturesResponse(google___protobuf___message___Message): value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse.FieldValues.FieldsEntry: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse.FieldValues.FieldsEntry: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... @@ -308,7 +315,7 @@ class GetOnlineFeaturesResponse(google___protobuf___message___Message): fields : typing___Optional[typing___Mapping[typing___Text, feast___types___Value_pb2___Value]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse.FieldValues: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse.FieldValues: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -325,7 +332,7 @@ class GetOnlineFeaturesResponse(google___protobuf___message___Message): field_values : typing___Optional[typing___Iterable[GetOnlineFeaturesResponse.FieldValues]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse: ... + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -344,14 +351,14 @@ class GetBatchFeaturesResponse(google___protobuf___message___Message): job : typing___Optional[Job] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetBatchFeaturesResponse: ... + def FromString(cls, s: builtin___bytes) -> GetBatchFeaturesResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... class GetJobRequest(google___protobuf___message___Message): @@ -365,14 +372,14 @@ class GetJobRequest(google___protobuf___message___Message): job : typing___Optional[Job] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetJobRequest: ... + def FromString(cls, s: builtin___bytes) -> GetJobRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... class GetJobResponse(google___protobuf___message___Message): @@ -386,14 +393,14 @@ class GetJobResponse(google___protobuf___message___Message): job : typing___Optional[Job] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> GetJobResponse: ... + def FromString(cls, s: builtin___bytes) -> GetJobResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... class Job(google___protobuf___message___Message): @@ -415,7 +422,7 @@ class Job(google___protobuf___message___Message): data_format : typing___Optional[DataFormat] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Job: ... + def FromString(cls, s: builtin___bytes) -> Job: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -436,7 +443,7 @@ class DatasetSource(google___protobuf___message___Message): data_format : typing___Optional[DataFormat] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> DatasetSource.FileSource: ... + def FromString(cls, s: builtin___bytes) -> DatasetSource.FileSource: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -453,13 +460,13 @@ class DatasetSource(google___protobuf___message___Message): file_source : typing___Optional[DatasetSource.FileSource] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> DatasetSource: ... + def FromString(cls, s: builtin___bytes) -> DatasetSource: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",u"file_source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",u"file_source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",u"file_source"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> typing_extensions___Literal["file_source"]: ... diff --git a/sdk/python/feast/storage/Redis_pb2.pyi b/sdk/python/feast/storage/Redis_pb2.pyi index 717aae79db..9bea087885 100644 --- a/sdk/python/feast/storage/Redis_pb2.pyi +++ b/sdk/python/feast/storage/Redis_pb2.pyi @@ -27,6 +27,12 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int + + class RedisKey(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... feature_set = ... # type: typing___Text @@ -40,7 +46,7 @@ class RedisKey(google___protobuf___message___Message): entities : typing___Optional[typing___Iterable[feast___types___Field_pb2___Field]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> RedisKey: ... + def FromString(cls, s: builtin___bytes) -> RedisKey: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): diff --git a/sdk/python/feast/types/FeatureRowExtended_pb2.pyi b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi index 4f3d02c8ee..8c4109a75f 100644 --- a/sdk/python/feast/types/FeatureRowExtended_pb2.pyi +++ b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi @@ -26,6 +26,12 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int + + class Error(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... cause = ... # type: typing___Text @@ -41,7 +47,7 @@ class Error(google___protobuf___message___Message): stack_trace : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Error: ... + def FromString(cls, s: builtin___bytes) -> Error: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -51,25 +57,25 @@ class Error(google___protobuf___message___Message): class Attempt(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - attempts = ... # type: int + attempts = ... # type: builtin___int @property def error(self) -> Error: ... def __init__(self, *, - attempts : typing___Optional[int] = None, + attempts : typing___Optional[builtin___int] = None, error : typing___Optional[Error] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Attempt: ... + def FromString(cls, s: builtin___bytes) -> Attempt: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"error"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"error"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"attempts",u"error"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"error",b"error"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"error",b"error"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"attempts",b"attempts",u"error",b"error"]) -> None: ... class FeatureRowExtended(google___protobuf___message___Message): @@ -91,12 +97,12 @@ class FeatureRowExtended(google___protobuf___message___Message): first_seen : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureRowExtended: ... + def FromString(cls, s: builtin___bytes) -> FeatureRowExtended: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"first_seen",u"last_attempt",u"row"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"first_seen",u"last_attempt",u"row"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"first_seen",u"last_attempt",u"row"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> None: ... diff --git a/sdk/python/feast/types/FeatureRow_pb2.pyi b/sdk/python/feast/types/FeatureRow_pb2.pyi index 9bf745f913..e634f46486 100644 --- a/sdk/python/feast/types/FeatureRow_pb2.pyi +++ b/sdk/python/feast/types/FeatureRow_pb2.pyi @@ -31,6 +31,12 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int + + class FeatureRow(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... feature_set = ... # type: typing___Text @@ -48,12 +54,12 @@ class FeatureRow(google___protobuf___message___Message): feature_set : typing___Optional[typing___Text] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FeatureRow: ... + def FromString(cls, s: builtin___bytes) -> FeatureRow: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"event_timestamp",u"feature_set",u"fields"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp",u"feature_set",b"feature_set",u"fields",b"fields"]) -> None: ... diff --git a/sdk/python/feast/types/Field_pb2.pyi b/sdk/python/feast/types/Field_pb2.pyi index 1305503fab..b5e6c1f609 100644 --- a/sdk/python/feast/types/Field_pb2.pyi +++ b/sdk/python/feast/types/Field_pb2.pyi @@ -22,6 +22,12 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int + + class Field(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text @@ -35,12 +41,12 @@ class Field(google___protobuf___message___Message): value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Field: ... + def FromString(cls, s: builtin___bytes) -> Field: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value",b"value"]) -> None: ... diff --git a/sdk/python/feast/types/Value_pb2.pyi b/sdk/python/feast/types/Value_pb2.pyi index d8b8a73dd3..5ead403ad9 100644 --- a/sdk/python/feast/types/Value_pb2.pyi +++ b/sdk/python/feast/types/Value_pb2.pyi @@ -27,68 +27,75 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str + + class ValueType(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Enum(int): + class Enum(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> ValueType.Enum: ... + def Value(cls, name: builtin___str) -> 'ValueType.Enum': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[ValueType.Enum]: ... + def values(cls) -> typing___List['ValueType.Enum']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, ValueType.Enum]]: ... - INVALID = typing___cast(ValueType.Enum, 0) - BYTES = typing___cast(ValueType.Enum, 1) - STRING = typing___cast(ValueType.Enum, 2) - INT32 = typing___cast(ValueType.Enum, 3) - INT64 = typing___cast(ValueType.Enum, 4) - DOUBLE = typing___cast(ValueType.Enum, 5) - FLOAT = typing___cast(ValueType.Enum, 6) - BOOL = typing___cast(ValueType.Enum, 7) - BYTES_LIST = typing___cast(ValueType.Enum, 11) - STRING_LIST = typing___cast(ValueType.Enum, 12) - INT32_LIST = typing___cast(ValueType.Enum, 13) - INT64_LIST = typing___cast(ValueType.Enum, 14) - DOUBLE_LIST = typing___cast(ValueType.Enum, 15) - FLOAT_LIST = typing___cast(ValueType.Enum, 16) - BOOL_LIST = typing___cast(ValueType.Enum, 17) - INVALID = typing___cast(ValueType.Enum, 0) - BYTES = typing___cast(ValueType.Enum, 1) - STRING = typing___cast(ValueType.Enum, 2) - INT32 = typing___cast(ValueType.Enum, 3) - INT64 = typing___cast(ValueType.Enum, 4) - DOUBLE = typing___cast(ValueType.Enum, 5) - FLOAT = typing___cast(ValueType.Enum, 6) - BOOL = typing___cast(ValueType.Enum, 7) - BYTES_LIST = typing___cast(ValueType.Enum, 11) - STRING_LIST = typing___cast(ValueType.Enum, 12) - INT32_LIST = typing___cast(ValueType.Enum, 13) - INT64_LIST = typing___cast(ValueType.Enum, 14) - DOUBLE_LIST = typing___cast(ValueType.Enum, 15) - FLOAT_LIST = typing___cast(ValueType.Enum, 16) - BOOL_LIST = typing___cast(ValueType.Enum, 17) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'ValueType.Enum']]: ... + INVALID = typing___cast('ValueType.Enum', 0) + BYTES = typing___cast('ValueType.Enum', 1) + STRING = typing___cast('ValueType.Enum', 2) + INT32 = typing___cast('ValueType.Enum', 3) + INT64 = typing___cast('ValueType.Enum', 4) + DOUBLE = typing___cast('ValueType.Enum', 5) + FLOAT = typing___cast('ValueType.Enum', 6) + BOOL = typing___cast('ValueType.Enum', 7) + BYTES_LIST = typing___cast('ValueType.Enum', 11) + STRING_LIST = typing___cast('ValueType.Enum', 12) + INT32_LIST = typing___cast('ValueType.Enum', 13) + INT64_LIST = typing___cast('ValueType.Enum', 14) + DOUBLE_LIST = typing___cast('ValueType.Enum', 15) + FLOAT_LIST = typing___cast('ValueType.Enum', 16) + BOOL_LIST = typing___cast('ValueType.Enum', 17) + INVALID = typing___cast('ValueType.Enum', 0) + BYTES = typing___cast('ValueType.Enum', 1) + STRING = typing___cast('ValueType.Enum', 2) + INT32 = typing___cast('ValueType.Enum', 3) + INT64 = typing___cast('ValueType.Enum', 4) + DOUBLE = typing___cast('ValueType.Enum', 5) + FLOAT = typing___cast('ValueType.Enum', 6) + BOOL = typing___cast('ValueType.Enum', 7) + BYTES_LIST = typing___cast('ValueType.Enum', 11) + STRING_LIST = typing___cast('ValueType.Enum', 12) + INT32_LIST = typing___cast('ValueType.Enum', 13) + INT64_LIST = typing___cast('ValueType.Enum', 14) + DOUBLE_LIST = typing___cast('ValueType.Enum', 15) + FLOAT_LIST = typing___cast('ValueType.Enum', 16) + BOOL_LIST = typing___cast('ValueType.Enum', 17) def __init__(self, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> ValueType: ... + def FromString(cls, s: builtin___bytes) -> ValueType: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... class Value(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - bytes_val = ... # type: bytes + bytes_val = ... # type: builtin___bytes string_val = ... # type: typing___Text - int32_val = ... # type: int - int64_val = ... # type: int - double_val = ... # type: float - float_val = ... # type: float - bool_val = ... # type: bool + int32_val = ... # type: builtin___int + int64_val = ... # type: builtin___int + double_val = ... # type: builtin___float + float_val = ... # type: builtin___float + bool_val = ... # type: builtin___bool @property def bytes_list_val(self) -> BytesList: ... @@ -113,13 +120,13 @@ class Value(google___protobuf___message___Message): def __init__(self, *, - bytes_val : typing___Optional[bytes] = None, + bytes_val : typing___Optional[builtin___bytes] = None, string_val : typing___Optional[typing___Text] = None, - int32_val : typing___Optional[int] = None, - int64_val : typing___Optional[int] = None, - double_val : typing___Optional[float] = None, - float_val : typing___Optional[float] = None, - bool_val : typing___Optional[bool] = None, + int32_val : typing___Optional[builtin___int] = None, + int64_val : typing___Optional[builtin___int] = None, + double_val : typing___Optional[builtin___float] = None, + float_val : typing___Optional[builtin___float] = None, + bool_val : typing___Optional[builtin___bool] = None, bytes_list_val : typing___Optional[BytesList] = None, string_list_val : typing___Optional[StringList] = None, int32_list_val : typing___Optional[Int32List] = None, @@ -129,27 +136,27 @@ class Value(google___protobuf___message___Message): bool_list_val : typing___Optional[BoolList] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Value: ... + def FromString(cls, s: builtin___bytes) -> Value: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",u"bool_val",u"bytes_list_val",u"bytes_val",u"double_list_val",u"double_val",u"float_list_val",u"float_val",u"int32_list_val",u"int32_val",u"int64_list_val",u"int64_val",u"string_list_val",u"string_val",u"val"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",u"bool_val",u"bytes_list_val",u"bytes_val",u"double_list_val",u"double_val",u"float_list_val",u"float_val",u"int32_list_val",u"int32_val",u"int64_list_val",u"int64_val",u"string_list_val",u"string_val",u"val"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"bool_list_val",u"bool_val",u"bytes_list_val",u"bytes_val",u"double_list_val",u"double_val",u"float_list_val",u"float_val",u"int32_list_val",u"int32_val",u"int64_list_val",u"int64_val",u"string_list_val",u"string_val",u"val"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> bool: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> builtin___bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"val",b"val"]) -> typing_extensions___Literal["bytes_val","string_val","int32_val","int64_val","double_val","float_val","bool_val","bytes_list_val","string_list_val","int32_list_val","int64_list_val","double_list_val","float_list_val","bool_list_val"]: ... class BytesList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bytes] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bytes] def __init__(self, *, - val : typing___Optional[typing___Iterable[bytes]] = None, + val : typing___Optional[typing___Iterable[builtin___bytes]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> BytesList: ... + def FromString(cls, s: builtin___bytes) -> BytesList: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -166,7 +173,7 @@ class StringList(google___protobuf___message___Message): val : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> StringList: ... + def FromString(cls, s: builtin___bytes) -> StringList: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -176,14 +183,14 @@ class StringList(google___protobuf___message___Message): class Int32List(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] def __init__(self, *, - val : typing___Optional[typing___Iterable[int]] = None, + val : typing___Optional[typing___Iterable[builtin___int]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Int32List: ... + def FromString(cls, s: builtin___bytes) -> Int32List: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -193,14 +200,14 @@ class Int32List(google___protobuf___message___Message): class Int64List(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] def __init__(self, *, - val : typing___Optional[typing___Iterable[int]] = None, + val : typing___Optional[typing___Iterable[builtin___int]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> Int64List: ... + def FromString(cls, s: builtin___bytes) -> Int64List: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -210,14 +217,14 @@ class Int64List(google___protobuf___message___Message): class DoubleList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] def __init__(self, *, - val : typing___Optional[typing___Iterable[float]] = None, + val : typing___Optional[typing___Iterable[builtin___float]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> DoubleList: ... + def FromString(cls, s: builtin___bytes) -> DoubleList: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -227,14 +234,14 @@ class DoubleList(google___protobuf___message___Message): class FloatList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] def __init__(self, *, - val : typing___Optional[typing___Iterable[float]] = None, + val : typing___Optional[typing___Iterable[builtin___float]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> FloatList: ... + def FromString(cls, s: builtin___bytes) -> FloatList: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): @@ -244,14 +251,14 @@ class FloatList(google___protobuf___message___Message): class BoolList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bool] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bool] def __init__(self, *, - val : typing___Optional[typing___Iterable[bool]] = None, + val : typing___Optional[typing___Iterable[builtin___bool]] = None, ) -> None: ... @classmethod - def FromString(cls, s: bytes) -> BoolList: ... + def FromString(cls, s: builtin___bytes) -> BoolList: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): diff --git a/sdk/python/feast/value_type.py b/sdk/python/feast/value_type.py index df315480ce..687dccc7b7 100644 --- a/sdk/python/feast/value_type.py +++ b/sdk/python/feast/value_type.py @@ -14,6 +14,8 @@ import enum +from tensorflow_metadata.proto.v0 import schema_pb2 + class ValueType(enum.Enum): """ @@ -35,3 +37,24 @@ class ValueType(enum.Enum): DOUBLE_LIST = 15 FLOAT_LIST = 16 BOOL_LIST = 17 + + def to_tfx_schema_feature_type(self) -> schema_pb2.FeatureType: + if self.value in [ + ValueType.BYTES.value, + ValueType.STRING.value, + ValueType.BOOL.value, + ValueType.BYTES_LIST.value, + ValueType.STRING_LIST.value, + ValueType.INT32_LIST.value, + ValueType.INT64_LIST.value, + ValueType.DOUBLE_LIST.value, + ValueType.FLOAT_LIST.value, + ValueType.BOOL_LIST.value, + ]: + return schema_pb2.FeatureType.BYTES + elif self.value in [ValueType.INT32.value, ValueType.INT64.value]: + return schema_pb2.FeatureType.INT + elif self.value in [ValueType.DOUBLE.value, ValueType.FLOAT.value]: + return schema_pb2.FeatureType.FLOAT + else: + return schema_pb2.FeatureType.TYPE_UNKNOWN diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index d0fdd76e49..c6890bb3f7 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -1,30 +1,30 @@ -Click==7.* -google-api-core==1.* -google-auth==1.* -google-cloud-bigquery==1.* -google-cloud-bigquery-storage==0.* -google-cloud-storage==1.* +Click>=7.* +google-api-core>=1.* +google-auth>=1.* +google-cloud-bigquery>=1.* +google-cloud-bigquery-storage>=0.* +google-cloud-storage>=1.* google-resumable-media>=0.5 -googleapis-common-protos==1.* -grpcio==1.* +googleapis-common-protos>=1.* +grpcio>=1.* numpy -mock==2.0.0 -pandas==0.* -protobuf==3.* +mock>=2.0.0 +pandas>=0.* +protobuf>=3.* pytest pytest-mock pytest-timeout -PyYAML==5.1.* -fastavro==0.* -grpcio-testing==1.* -pytest-ordering==0.6.* +PyYAML>=5.1.* +fastavro>=0.* +grpcio-testing>=1.26* +pytest-ordering>=0.6.* pyarrow Sphinx sphinx-rtd-theme -toml==0.10.* -tqdm==4.* +toml>=0.10.* +tqdm>=4.* confluent_kafka google -pandavro==1.5.* -kafka-python==1.* -tabulate==0.8.* \ No newline at end of file +pandavro>=1.5.* +kafka-python>=1.* +tabulate>=0.8.* \ No newline at end of file diff --git a/sdk/python/setup.py b/sdk/python/setup.py index d0b37ad941..7d09af95f1 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -24,28 +24,28 @@ REQUIRES_PYTHON = ">=3.6.0" REQUIRED = [ - "Click==7.*", - "google-api-core==1.14.*", - "google-auth==1.6.*", - "google-cloud-bigquery==1.18.*", - "google-cloud-storage==1.20.*", - "google-cloud-core==1.0.*", - "googleapis-common-protos==1.*", - "google-cloud-bigquery-storage==0.7.*", - "grpcio==1.*", - "pandas==0.*", - "pandavro==1.5.*", + "Click>=7.*", + "google-api-core>=1.14.*", + "google-auth>=1.6.*", + "google-cloud-bigquery>=1.18.*", + "google-cloud-storage>=1.20.*", + "google-cloud-core>=1.0.*", + "googleapis-common-protos>=1.*", + "google-cloud-bigquery-storage>=0.7.*", + "grpcio>=1.*", + "pandas>=0.*", + "pandavro>=1.5.*", "protobuf>=3.10", - "PyYAML==5.1.*", - "fastavro==0.*", - "kafka-python==1.*", - "tabulate==0.8.*", - "toml==0.10.*", - "tqdm==4.*", + "PyYAML>=5.1.*", + "fastavro>=0.*", + "kafka-python>=1.*", + "tabulate>=0.8.*", + "toml>=0.10.*", + "tqdm>=4.*", "pyarrow>=0.15.1", - "numpy", - "google", - "confluent_kafka", + "numpy>=1.15.0", + "confluent_kafka>=1.3.0", + "tensorflow_metadata>=0.21.0", ] # README file from Feast repo root directory diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml new file mode 100644 index 0000000000..daa0a35f0a --- /dev/null +++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml @@ -0,0 +1,81 @@ +spec: + name: bikeshare + entities: + - name: station_id + valueType: INT64 + intDomain: + min: 1 + max: 5000 + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + features: + - name: location + valueType: STRING + stringDomain: + name: location + value: + - (30.24258, -97.71726) + - (30.24472, -97.72336) + - (30.24891, -97.75019) + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + - name: name + valueType: STRING + stringDomain: + name: name + value: + - 10th & Red River + - 11th & Salina + - 11th & San Jacinto + - 13th & San Antonio + - 17th & Guadalupe + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + - name: status + valueType: STRING + stringDomain: + name: status + value: + - "active" + - "closed" + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + - name: latitude + valueType: DOUBLE + floatDomain: + min: 100.0 + max: 105.0 + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + - name: longitude + valueType: DOUBLE + floatDomain: + min: 102.0 + max: 105.0 + presence: + minFraction: 1.0 + minCount: 1 + shape: + dim: + - size: 1 + maxAge: 3600s diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json new file mode 100644 index 0000000000..e7a886053c --- /dev/null +++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json @@ -0,0 +1,136 @@ +{ + "feature": [ + { + "name": "location", + "type": "BYTES", + "domain": "location", + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + }, + { + "name": "name", + "type": "BYTES", + "domain": "name", + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + }, + { + "name": "status", + "type": "BYTES", + "domain": "status", + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + }, + { + "name": "latitude", + "type": "FLOAT", + "float_domain": { + "min": 100.0, + "max": 105.0 + }, + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + }, + { + "name": "longitude", + "type": "FLOAT", + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "float_domain": { + "min": 102.0, + "max": 105.0 + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + }, + { + "name": "station_id", + "type": "INT", + "presence": { + "minFraction": 1.0, + "minCount": "1" + }, + "int_domain": { + "min": 1, + "max": 5000 + }, + "shape": { + "dim": [ + { + "size": "1" + } + ] + } + } + ], + "stringDomain": [ + { + "name": "location", + "value": [ + "(30.24258, -97.71726)", + "(30.24472, -97.72336)", + "(30.24891, -97.75019)" + ] + }, + { + "name": "name", + "value": [ + "10th & Red River", + "11th & Salina", + "11th & San Jacinto", + "13th & San Antonio", + "17th & Guadalupe" + ] + }, + { + "name": "status", + "value": [ + "active", + "closed" + ] + } + ] +} \ No newline at end of file diff --git a/sdk/python/tests/test_feature_set.py b/sdk/python/tests/test_feature_set.py index 57d7a8f810..0aaac8fc81 100644 --- a/sdk/python/tests/test_feature_set.py +++ b/sdk/python/tests/test_feature_set.py @@ -11,21 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import pathlib +from concurrent import futures from datetime import datetime +import grpc +import pandas as pd +import pytest import pytz +from google.protobuf import json_format +from tensorflow_metadata.proto.v0 import schema_pb2 +import dataframes +import feast.core.CoreService_pb2_grpc as Core +from feast.client import Client from feast.entity import Entity from feast.feature_set import FeatureSet, Feature from feast.value_type import ValueType -from feast.client import Client -import pandas as pd -import pytest -from concurrent import futures -import grpc from feast_core_server import CoreServicer -import feast.core.CoreService_pb2_grpc as Core -import dataframes CORE_URL = "core.feast.local" SERVING_URL = "serving.feast.local" @@ -167,3 +170,94 @@ def test_add_features_from_df_success( ) assert len(my_feature_set.features) == feature_count assert len(my_feature_set.entities) == entity_count + + def test_import_tfx_schema(self): + tests_folder = pathlib.Path(__file__).parent + test_input_schema_json = open( + tests_folder / "data" / "tensorflow_metadata" / "bikeshare_schema.json" + ).read() + test_input_schema = schema_pb2.Schema() + json_format.Parse(test_input_schema_json, test_input_schema) + + feature_set = FeatureSet( + name="bikeshare", + entities=[Entity(name="station_id", dtype=ValueType.INT64),], + features=[ + Feature(name="name", dtype=ValueType.STRING), + Feature(name="status", dtype=ValueType.STRING), + Feature(name="latitude", dtype=ValueType.FLOAT), + Feature(name="longitude", dtype=ValueType.FLOAT), + Feature(name="location", dtype=ValueType.STRING), + ], + ) + + # Before update + for entity in feature_set.entities: + assert entity.presence is None + assert entity.shape is None + for feature in feature_set.features: + assert feature.presence is None + assert feature.shape is None + assert feature.string_domain is None + assert feature.float_domain is None + assert feature.int_domain is None + + feature_set.import_tfx_schema(test_input_schema) + + # After update + for entity in feature_set.entities: + assert entity.presence is not None + assert entity.shape is not None + for feature in feature_set.features: + assert feature.presence is not None + assert feature.shape is not None + if feature.name in ["location", "name", "status"]: + assert feature.string_domain is not None + elif feature.name in ["latitude", "longitude"]: + assert feature.float_domain is not None + elif feature.name in ["station_id"]: + assert feature.int_domain is not None + + def test_export_tfx_schema(self): + tests_folder = pathlib.Path(__file__).parent + test_input_feature_set = FeatureSet.from_yaml( + str( + tests_folder + / "data" + / "tensorflow_metadata" + / "bikeshare_feature_set.yaml" + ) + ) + + expected_schema_json = open( + tests_folder / "data" / "tensorflow_metadata" / "bikeshare_schema.json" + ).read() + expected_schema = schema_pb2.Schema() + json_format.Parse(expected_schema_json, expected_schema) + make_tfx_schema_domain_info_inline(expected_schema) + + actual_schema = test_input_feature_set.export_tfx_schema() + + assert len(actual_schema.feature) == len(expected_schema.feature) + for actual, expected in zip(actual_schema.feature, expected_schema.feature): + assert actual.SerializeToString() == expected.SerializeToString() + + +def make_tfx_schema_domain_info_inline(schema): + # Copy top-level domain info defined in the schema to inline definition. + # One use case is in FeatureSet which does not have access to the top-level domain + # info. + domain_ref_to_string_domain = {d.name: d for d in schema.string_domain} + domain_ref_to_float_domain = {d.name: d for d in schema.float_domain} + domain_ref_to_int_domain = {d.name: d for d in schema.int_domain} + + for feature in schema.feature: + domain_info_case = feature.WhichOneof("domain_info") + if domain_info_case == "domain": + domain_ref = feature.domain + if domain_ref in domain_ref_to_string_domain: + feature.string_domain.MergeFrom(domain_ref_to_string_domain[domain_ref]) + elif domain_ref in domain_ref_to_float_domain: + feature.float_domain.MergeFrom(domain_ref_to_float_domain[domain_ref]) + elif domain_ref in domain_ref_to_int_domain: + feature.int_domain.MergeFrom(domain_ref_to_int_domain[domain_ref]) diff --git a/tests/e2e/basic/cust_trans_fs.yaml b/tests/e2e/basic/cust_trans_fs.yaml index 14d46794a6..bcc2bad090 100644 --- a/tests/e2e/basic/cust_trans_fs.yaml +++ b/tests/e2e/basic/cust_trans_fs.yaml @@ -4,9 +4,36 @@ spec: entities: - name: customer_id valueType: INT64 + intDomain: + min: 1000 + max: 100005 + presence: + minFraction: 0.99 + minCount: 1 + shape: + dim: + - size: 1 features: - name: daily_transactions valueType: FLOAT + floatDomain: + min: 0 + max: 1 + presence: + minFraction: 0.90 + minCount: 1 + shape: + dim: + - size: 1 - name: total_transactions valueType: FLOAT + floatDomain: + min: 500 + max: 520 + presence: + minFraction: 0.90 + minCount: 1 + shape: + dim: + - size: 1 maxAge: 3600s diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/basic_ingest_redis_serving.py similarity index 64% rename from tests/e2e/basic-ingest-redis-serving.py rename to tests/e2e/basic_ingest_redis_serving.py index 1aeccfa5a3..4dcb130f3c 100644 --- a/tests/e2e/basic-ingest-redis-serving.py +++ b/tests/e2e/basic_ingest_redis_serving.py @@ -1,47 +1,49 @@ -import pytest -import math +import os import random +import tempfile +import uuid +from datetime import datetime, timedelta + +import math +import numpy as np +import pandas as pd +import pytest +import pytz +import requests import time +from feast.client import Client from feast.entity import Entity +from feast.feature import Feature +from feast.feature_set import FeatureSet from feast.serving.ServingService_pb2 import ( GetOnlineFeaturesRequest, GetOnlineFeaturesResponse, ) -from feast.types.Value_pb2 import Value as Value -from feast.client import Client -from feast.feature_set import FeatureSet from feast.type_map import ValueType +from feast.types.Value_pb2 import Value as Value from google.protobuf.duration_pb2 import Duration -from datetime import datetime -import pytz - -import pandas as pd -import numpy as np -import tempfile -import os -from feast.feature import Feature -import uuid +import warnings FLOAT_TOLERANCE = 0.00001 -PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6] +PROJECT_NAME = "basic_" + uuid.uuid4().hex.upper()[0:6] + -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def core_url(pytestconfig): return pytestconfig.getoption("core_url") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def serving_url(pytestconfig): return pytestconfig.getoption("serving_url") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def allow_dirty(pytestconfig): - return True if pytestconfig.getoption( - "allow_dirty").lower() == "true" else False + return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def client(core_url, serving_url, allow_dirty): # Get client for core and serving client = Client(core_url=core_url, serving_url=serving_url) @@ -59,13 +61,12 @@ def client(core_url, serving_url, allow_dirty): return client -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def basic_dataframe(): offset = random.randint(1000, 100000) # ensure a unique key space is used return pd.DataFrame( { - "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(5)], + "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(5)], "customer_id": [offset + inc for inc in range(5)], "daily_transactions": [np.random.rand() for _ in range(5)], "total_transactions": [512 for _ in range(5)], @@ -128,10 +129,7 @@ def test_basic_retrieve_online_success(client, basic_dataframe): } ) ], - feature_refs=[ - "daily_transactions", - "total_transactions", - ], + feature_refs=["daily_transactions", "total_transactions",], ) # type: GetOnlineFeaturesResponse if response is None: @@ -139,11 +137,10 @@ def test_basic_retrieve_online_success(client, basic_dataframe): returned_daily_transactions = float( response.field_values[0] - .fields[PROJECT_NAME + "/daily_transactions"] - .float_val + .fields[PROJECT_NAME + "/daily_transactions"] + .float_val ) - sent_daily_transactions = float( - basic_dataframe.iloc[0]["daily_transactions"]) + sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) if math.isclose( sent_daily_transactions, @@ -153,18 +150,16 @@ def test_basic_retrieve_online_success(client, basic_dataframe): break -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def all_types_dataframe(): return pd.DataFrame( { - "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(3)], + "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(3)], "user_id": [1001, 1002, 1003], "int32_feature": [np.int32(1), np.int32(2), np.int32(3)], "int64_feature": [np.int64(1), np.int64(2), np.int64(3)], "float_feature": [np.float(0.1), np.float(0.2), np.float(0.3)], - "double_feature": [np.float64(0.1), np.float64(0.2), - np.float64(0.3)], + "double_feature": [np.float64(0.1), np.float64(0.2), np.float64(0.3)], "string_feature": ["one", "two", "three"], "bytes_feature": [b"one", b"two", b"three"], "bool_feature": [True, False, False], @@ -226,8 +221,7 @@ def test_all_types_register_feature_set_success(client): Feature(name="float_list_feature", dtype=ValueType.FLOAT_LIST), Feature(name="int64_list_feature", dtype=ValueType.INT64_LIST), Feature(name="int32_list_feature", dtype=ValueType.INT32_LIST), - Feature(name="string_list_feature", - dtype=ValueType.STRING_LIST), + Feature(name="string_list_feature", dtype=ValueType.STRING_LIST), Feature(name="bytes_list_feature", dtype=ValueType.BYTES_LIST), ], max_age=Duration(seconds=3600), @@ -273,8 +267,11 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( - fields={"user_id": Value( - int64_val=all_types_dataframe.iloc[0]["user_id"])} + fields={ + "user_id": Value( + int64_val=all_types_dataframe.iloc[0]["user_id"] + ) + } ) ], feature_refs=[ @@ -297,11 +294,10 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): if response is None: continue - returned_float_list = ( response.field_values[0] - .fields[PROJECT_NAME+"/float_list_feature"] - .float_list_val.val + .fields[PROJECT_NAME + "/float_list_feature"] + .float_list_val.val ) sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] @@ -312,15 +308,14 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): break -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def large_volume_dataframe(): ROW_COUNT = 100000 offset = random.randint(1000000, 10000000) # ensure a unique key space customer_data = pd.DataFrame( { "datetime": [ - datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(ROW_COUNT) + datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(ROW_COUNT) ], "customer_id": [offset + inc for inc in range(ROW_COUNT)], "daily_transactions_large": [np.random.rand() for _ in range(ROW_COUNT)], @@ -334,7 +329,8 @@ def large_volume_dataframe(): @pytest.mark.run(order=30) def test_large_volume_register_feature_set_success(client): cust_trans_fs_expected = FeatureSet.from_yaml( - "large_volume/cust_trans_large_fs.yaml") + "large_volume/cust_trans_large_fs.yaml" + ) # Register feature set client.apply(cust_trans_fs_expected) @@ -342,8 +338,7 @@ def test_large_volume_register_feature_set_success(client): # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(10) - cust_trans_fs_actual = client.get_feature_set( - name="customer_transactions_large") + cust_trans_fs_actual = client.get_feature_set(name="customer_transactions_large") assert cust_trans_fs_actual == cust_trans_fs_expected @@ -378,16 +373,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value( - int64_val=large_volume_dataframe.iloc[0][ - "customer_id"] + int64_val=large_volume_dataframe.iloc[0]["customer_id"] ) } ) ], - feature_refs=[ - "daily_transactions_large", - "total_transactions_large", - ], + feature_refs=["daily_transactions_large", "total_transactions_large",], ) # type: GetOnlineFeaturesResponse if response is None: @@ -395,11 +386,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): returned_daily_transactions = float( response.field_values[0] - .fields[PROJECT_NAME + "/daily_transactions_large"] - .float_val + .fields[PROJECT_NAME + "/daily_transactions_large"] + .float_val ) sent_daily_transactions = float( - large_volume_dataframe.iloc[0]["daily_transactions_large"]) + large_volume_dataframe.iloc[0]["daily_transactions_large"] + ) if math.isclose( sent_daily_transactions, @@ -409,49 +401,47 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): break -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def all_types_parquet_file(): COUNT = 20000 df = pd.DataFrame( { "datetime": [datetime.utcnow() for _ in range(COUNT)], - "customer_id": [np.int32(random.randint(0, 10000)) for _ in - range(COUNT)], - "int32_feature_parquet": [np.int32(random.randint(0, 10000)) for _ in - range(COUNT)], - "int64_feature_parquet": [np.int64(random.randint(0, 10000)) for _ in - range(COUNT)], + "customer_id": [np.int32(random.randint(0, 10000)) for _ in range(COUNT)], + "int32_feature_parquet": [ + np.int32(random.randint(0, 10000)) for _ in range(COUNT) + ], + "int64_feature_parquet": [ + np.int64(random.randint(0, 10000)) for _ in range(COUNT) + ], "float_feature_parquet": [np.float(random.random()) for _ in range(COUNT)], - "double_feature_parquet": [np.float64(random.random()) for _ in - range(COUNT)], - "string_feature_parquet": ["one" + str(random.random()) for _ in - range(COUNT)], + "double_feature_parquet": [ + np.float64(random.random()) for _ in range(COUNT) + ], + "string_feature_parquet": [ + "one" + str(random.random()) for _ in range(COUNT) + ], "bytes_feature_parquet": [b"one" for _ in range(COUNT)], "int32_list_feature_parquet": [ np.array([1, 2, 3, random.randint(0, 10000)], dtype=np.int32) - for _ - in range(COUNT) + for _ in range(COUNT) ], "int64_list_feature_parquet": [ np.array([1, random.randint(0, 10000), 3, 4], dtype=np.int64) - for _ - in range(COUNT) + for _ in range(COUNT) ], "float_list_feature_parquet": [ - np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32) for - _ - in range(COUNT) + np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32) + for _ in range(COUNT) ], "double_list_feature_parquet": [ - np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64) for - _ - in range(COUNT) + np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64) + for _ in range(COUNT) ], "string_list_feature_parquet": [ - np.array(["one", "two" + str(random.random()), "three"]) for _ - in - range(COUNT) + np.array(["one", "two" + str(random.random()), "three"]) + for _ in range(COUNT) ], "bytes_list_feature_parquet": [ np.array([b"one", b"two", b"three"]) for _ in range(COUNT) @@ -462,7 +452,7 @@ def all_types_parquet_file(): # TODO: Boolean list is not being tested. # https://github.com/gojek/feast/issues/341 - file_path = os.path.join(tempfile.mkdtemp(), 'all_types.parquet') + file_path = os.path.join(tempfile.mkdtemp(), "all_types.parquet") df.to_parquet(file_path, allow_truncated_timestamps=True) return file_path @@ -472,7 +462,8 @@ def all_types_parquet_file(): def test_all_types_parquet_register_feature_set_success(client): # Load feature set from file all_types_parquet_expected = FeatureSet.from_yaml( - "all_types_parquet/all_types_parquet.yaml") + "all_types_parquet/all_types_parquet.yaml" + ) # Register feature set client.apply(all_types_parquet_expected) @@ -496,11 +487,128 @@ def test_all_types_parquet_register_feature_set_success(client): @pytest.mark.timeout(600) @pytest.mark.run(order=41) -def test_all_types_infer_register_ingest_file_success(client, - all_types_parquet_file): +def test_all_types_infer_register_ingest_file_success(client, all_types_parquet_file): # Get feature set all_types_fs = client.get_feature_set(name="all_types_parquet") # Ingest user embedding data - client.ingest(feature_set=all_types_fs, source=all_types_parquet_file, - force_update=True) + client.ingest( + feature_set=all_types_fs, source=all_types_parquet_file, force_update=True + ) + + +@pytest.mark.run(order=42) +def test_basic_metrics(pytestconfig, basic_dataframe): + if not pytestconfig.getoption("prometheus_server_url"): + warnings.warn( + "Skipping 'test_basic_metrics' because 'prometheus_server_url' argument" + "is not provided" + ) + return + + feature_set = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") + project_name = PROJECT_NAME + feature_set_name = "customer_transactions" + + range_query_endpoint = ( + f"{pytestconfig.getoption('prometheus_server_url')}/api/v1/query_range" + ) + promql_queries = [ + "feast_ingestion_feature_value_min", + "feast_ingestion_feature_value_max", + "feast_ingestion_feature_value_domain_min", + "feast_ingestion_feature_value_domain_max", + "feast_ingestion_feature_value_presence_count", + "feast_ingestion_feature_value_missing_count", + "feast_ingestion_feature_presence_min_fraction", + "feast_ingestion_feature_presence_min_count", + ] + # "datetime" is the timestamp for the FeatureRow, not a feature in the DataFrame + feature_names = list(c for c in basic_dataframe.columns if c != "datetime") + + for query in promql_queries: + for feature_name in feature_names: + query_with_label_filter = f'{query}{{feast_feature_name="{feature_name}"}}' + resp = requests.post( + range_query_endpoint, + data={ + "query": query_with_label_filter, + "start": int((datetime.now() - timedelta(minutes=30)).timestamp()), + "end": int(datetime.now().timestamp()), + "step": "15s", + }, + ) + assert resp.status_code == 200 + for item in resp.json()["data"]["result"]: + metric = item["metric"] + values = item["values"] + + if ( + metric.get("feast_project_name", "") != project_name + or metric.get("feast_featureSet_name", "") != feature_set_name + ): + continue + + assert len(values) > 0 + # Values item in Prometheus is a tuple of (timestamp, value). + # Only last_value is tested here because the assertions are checking + # for the min, max and count and using the last values make the test + # more deterministic. + last_value_tuple = values[len(values) - 1] + assert len(last_value_tuple) == 2 + last_value = last_value_tuple[1] + + if query == "feast_ingestion_feature_value_min": + assert math.isclose( + float(last_value), + basic_dataframe[feature_name].min(), + abs_tol=FLOAT_TOLERANCE, + ) + elif query == "feast_ingestion_feature_value_max": + assert math.isclose( + float(last_value), + basic_dataframe[feature_name].max(), + abs_tol=FLOAT_TOLERANCE, + ) + elif query == "feast_ingestion_feature_value_domain_min": + if feature_name == "customer_id": + assert ( + int(last_value) + == feature_set.fields[feature_name].int_domain.min + ) + elif feature_name in ["daily_transactions", "total_transactions"]: + assert math.isclose( + float(last_value), + feature_set.fields[feature_name].float_domain.min, + abs_tol=FLOAT_TOLERANCE, + ) + elif query == "feast_ingestion_feature_value_domain_max": + if feature_name == "customer_id": + assert ( + int(last_value) + == feature_set.fields[feature_name].int_domain.max + ) + elif feature_name in ["daily_transactions", "total_transactions"]: + assert math.isclose( + float(last_value), + feature_set.fields[feature_name].float_domain.max, + abs_tol=FLOAT_TOLERANCE, + ) + # basic_dataframe has no UNSET values, hence the assertions + # for "feast_ingestion_feature_value_presence_count" and + # "feast_ingestion_feature_value_missing_count" + elif query == "feast_ingestion_feature_value_presence_count": + assert int(last_value) == basic_dataframe[feature_name].size + elif query == "feast_ingestion_feature_value_missing_count": + assert int(last_value) == 0 + elif query == "feast_ingestion_feature_presence_min_fraction": + assert math.isclose( + float(last_value), + feature_set.fields[feature_name].presence.min_fraction, + abs_tol=FLOAT_TOLERANCE, + ) + elif query == "feast_ingestion_feature_presence_min_count": + assert ( + int(last_value) + == feature_set.fields[feature_name].presence.min_count + ) diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bigquery_batch_retrieval.py similarity index 100% rename from tests/e2e/bq-batch-retrieval.py rename to tests/e2e/bigquery_batch_retrieval.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8ea472b662..96a9c377d6 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -3,3 +3,6 @@ def pytest_addoption(parser): parser.addoption("--serving_url", action="store", default="localhost:6566") parser.addoption("--allow_dirty", action="store", default="False") parser.addoption("--gcs_path", action="store", default="gs://feast-templocation-kf-feast/") + # If prometheus_server_url is not empty, then the e2e test will validate the + # prometheus metrics written by Feast. Example value: http://localhost:9090 + parser.addoption("--prometheus_server_url", action="store", default="")