diff --git a/.prow/config.yaml b/.prow/config.yaml index 1d79fdb3d2..9e4fbd4289 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -274,7 +274,7 @@ presubmits: - name: test-end-to-end-batch-dataflow decorate: true - always_run: false + always_run: true spec: volumes: - name: service-account-df @@ -311,7 +311,7 @@ presubmits: - -c - | infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \ --output-dir $PWD/ infra/scripts/publish-docker-image.sh \ @@ -440,7 +440,7 @@ postsubmits: - -c - | infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \ --output-dir $PWD/ if [ $PULL_BASE_REF == "master" ]; then diff --git a/core/pom.xml b/core/pom.xml index 716d8fdd39..8be1fda2e0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -209,12 +209,12 @@ com.google.api-client google-api-client - 1.27.0 + 1.30.9 com.google.apis google-api-services-dataflow - v1b3-rev266-1.25.0 + v1b3-rev20200305-1.30.9 org.hibernate diff --git a/infra/scripts/test-end-to-end-batch-dataflow.sh b/infra/scripts/test-end-to-end-batch-dataflow.sh index e138a75bde..06a35d2c5b 100755 --- a/infra/scripts/test-end-to-end-batch-dataflow.sh +++ b/infra/scripts/test-end-to-end-batch-dataflow.sh @@ -203,7 +203,7 @@ Export required environment variables ============================================================ " -export TEMP_BUCKET=$TEMP_BUCKET +export TEMP_BUCKET=$TEMP_BUCKET/$HELM_RELEASE_NAME/$(date +%s) export DATASET_NAME=$DATASET_NAME export GCLOUD_PROJECT=$GCLOUD_PROJECT export GCLOUD_NETWORK=$GCLOUD_NETWORK diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index 60f09cb416..e81a0ddd6c 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -62,6 +62,7 @@ feast: jobs: polling_interval_milliseconds: 10000 active_runner: direct + consolidate-jobs-per-source: true runners: - name: direct type: DirectRunner diff --git a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml index 48231face6..2dadd247a9 100644 --- a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml +++ b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml @@ -20,7 +20,7 @@ feast-core: specsAckTopic: $SPECS_TOPIC-ack jobs: active_runner: dataflow - + consolidate-jobs-per-source: true runners: - name: dataflow type: DataflowRunner diff --git a/ingestion/pom.xml b/ingestion/pom.xml index de50789a67..08223ebd77 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -70,6 +70,16 @@ com.google.cloud.bigquery.vendor + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + diff --git a/tests/e2e/bq/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py index a998f907d3..209e1405cc 100644 --- a/tests/e2e/bq/bq-batch-retrieval.py +++ b/tests/e2e/bq/bq-batch-retrieval.py @@ -243,7 +243,7 @@ def test_batch_get_historical_features_with_gs_path(client, gcs_path): uri = urlparse(gcs_path) bucket = uri.hostname ts = int(time.time()) - remote_path = str(uri.path).strip("/") + f"{ts}/{file_name}" + remote_path = str(uri.path).lstrip("/") + f"{ts}/{file_name}" # Upload file to gcs storage_client = storage.Client(project=None)