Exclude dependencies signatures from IngestionJob package (#879)

* fix dataflow e2e * consolidate jobs in e2e * dataflow tests with consolidated jobs * dummy
feast-dev · Jul 14, 2020 · 121cf20 · 121cf20
1 parent 4e80613
commit 121cf20
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 8 deletions.
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -274,7 +274,7 @@ presubmits:
 
   - name: test-end-to-end-batch-dataflow
     decorate: true
-    always_run: false
+    always_run: true
     spec:
       volumes:
       - name: service-account-df
@@ -311,7 +311,7 @@ presubmits:
         - -c
         - |
           infra/scripts/download-maven-cache.sh \
-            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \
             --output-dir $PWD/
 
           infra/scripts/publish-docker-image.sh \
@@ -440,7 +440,7 @@ postsubmits:
         - -c
         - |
           infra/scripts/download-maven-cache.sh \
-            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \
             --output-dir $PWD/
 
           if [ $PULL_BASE_REF == "master" ]; then

diff --git a/core/pom.xml b/core/pom.xml
@@ -209,12 +209,12 @@
         <dependency>
             <groupId>com.google.api-client</groupId>
             <artifactId>google-api-client</artifactId>
-            <version>1.27.0</version>
+            <version>1.30.9</version>
         </dependency>
         <dependency>
             <groupId>com.google.apis</groupId>
             <artifactId>google-api-services-dataflow</artifactId>
-            <version>v1b3-rev266-1.25.0</version>
+            <version>v1b3-rev20200305-1.30.9</version>
         </dependency>
         <dependency>
             <groupId>org.hibernate</groupId>

diff --git a/infra/scripts/test-end-to-end-batch-dataflow.sh b/infra/scripts/test-end-to-end-batch-dataflow.sh
@@ -203,7 +203,7 @@ Export required environment variables
 ============================================================
 "
 
-export TEMP_BUCKET=$TEMP_BUCKET
+export TEMP_BUCKET=$TEMP_BUCKET/$HELM_RELEASE_NAME/$(date +%s)
 export DATASET_NAME=$DATASET_NAME
 export GCLOUD_PROJECT=$GCLOUD_PROJECT
 export GCLOUD_NETWORK=$GCLOUD_NETWORK

diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh
@@ -62,6 +62,7 @@ feast:
   jobs:
     polling_interval_milliseconds: 10000
     active_runner: direct
+    consolidate-jobs-per-source: true
     runners:
       - name: direct
         type: DirectRunner

diff --git a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml
@@ -20,7 +20,7 @@ feast-core:
           specsAckTopic: $SPECS_TOPIC-ack
       jobs:
         active_runner: dataflow
-
+        consolidate-jobs-per-source: true
         runners:
         - name: dataflow
           type: DataflowRunner

diff --git a/ingestion/pom.xml b/ingestion/pom.xml
@@ -70,6 +70,16 @@
                   <shadedPattern>com.google.cloud.bigquery.vendor</shadedPattern>
                 </relocation>
               </relocations>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
             </configuration>
           </execution>
         </executions>

diff --git a/tests/e2e/bq/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py
@@ -243,7 +243,7 @@ def test_batch_get_historical_features_with_gs_path(client, gcs_path):
     uri = urlparse(gcs_path)
     bucket = uri.hostname
     ts = int(time.time())
-    remote_path = str(uri.path).strip("/") + f"{ts}/{file_name}"
+    remote_path = str(uri.path).lstrip("/") + f"{ts}/{file_name}"
 
     # Upload file to gcs
     storage_client = storage.Client(project=None)