add python test

Signed-off-by: Danny Chiao <danny@tecton.ai>
feast-dev · Aug 11, 2022 · 55ee9f0 · 55ee9f0
1 parent 4f93f6f
commit 55ee9f0
Show file tree

Hide file tree

Showing 6 changed files with 132 additions and 7 deletions.
diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml
@@ -95,10 +95,46 @@ jobs:
           java-version: '11'
           java-package: jdk
           architecture: x64
-      - uses: actions/setup-python@v2
+      - name: Setup Python (to call feast apply)
+        uses: actions/setup-python@v2
+        id: setup-python
         with:
-          python-version: '3.8'
-          architecture: 'x64'
+          python-version: 3.8
+          architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.0
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=21.3.1,<22.1"
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ steps.pip-cache.outputs.dir }}
+            /opt/hostedtoolcache/Python
+            /Users/runner/hostedtoolcache/Python
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        run: |
+          sudo apt update
+          sudo apt install -y -V ca-certificates lsb-release wget
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y -V libarrow-dev
+      - name: Install Python dependencies
+        run: make install-python-ci-dependencies
       - uses: actions/cache@v2
         with:
           path: ~/.m2/repository

diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml
@@ -129,6 +129,46 @@ jobs:
           aws-region: us-west-2
       - name: Use AWS CLI
         run: aws sts get-caller-identity
+      - name: Setup Python (to call feast apply)
+        uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: 3.8
+          architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.0
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=21.3.1,<22.1"
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ steps.pip-cache.outputs.dir }}
+            /opt/hostedtoolcache/Python
+            /Users/runner/hostedtoolcache/Python
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        run: |
+          sudo apt update
+          sudo apt install -y -V ca-certificates lsb-release wget
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y -V libarrow-dev
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
       - name: Run integration tests
         run:  make test-java-integration
       - name: Save report

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -167,7 +167,7 @@ jobs:
       - name: Install apache-arrow on macos
         if: matrix.os == 'macOS-latest'
         run: brew install apache-arrow
-      - name: Install dependencies
+      - name: Install Python dependencies
         run: make install-python-ci-dependencies
       - name: Setup Redis Cluster
         run: |

diff --git a/sdk/python/feast/diff/registry_diff.py b/sdk/python/feast/diff/registry_diff.py
@@ -144,8 +144,8 @@ def diff_registry_objects(
                 continue
             elif getattr(current_spec, _field.name) != getattr(new_spec, _field.name):
                 if _field.name == "user_defined_function":
-                    current_spec = cast(OnDemandFeatureViewSpec, current_proto.spec)
-                    new_spec = cast(OnDemandFeatureViewSpec, new_proto.spec)
+                    current_spec = cast(OnDemandFeatureViewSpec, current_spec)
+                    new_spec = cast(OnDemandFeatureViewSpec, new_spec)
                     current_udf = current_spec.user_defined_function
                     new_udf = new_spec.user_defined_function
                     for _udf_field in current_udf.DESCRIPTOR.fields:

diff --git a/sdk/python/tests/unit/diff/test_registry_diff.py b/sdk/python/tests/unit/diff/test_registry_diff.py
@@ -1,9 +1,14 @@
+import pandas as pd
+
+from feast import Field
 from feast.diff.registry_diff import (
     diff_registry_objects,
     tag_objects_for_keep_delete_update_add,
 )
 from feast.entity import Entity
 from feast.feature_view import FeatureView
+from feast.on_demand_feature_view import on_demand_feature_view
+from feast.types import String
 from tests.utils.data_source_test_creator import prep_file_source
 
 
@@ -89,3 +94,48 @@ def test_diff_registry_objects_feature_views(simple_dataset_1):
         assert feast_object_diffs.feast_object_property_diffs[0].val_declared == {
             "when": "after"
         }
+
+
+def test_diff_odfv(simple_dataset_1):
+    with prep_file_source(df=simple_dataset_1, timestamp_field="ts_1") as file_source:
+        entity = Entity(name="id", join_keys=["id"])
+        fv = FeatureView(
+            name="fv2",
+            entities=[entity],
+            source=file_source,
+            tags={"when": "before"},
+        )
+
+        @on_demand_feature_view(
+            sources=[fv],
+            schema=[Field(name="first_char", dtype=String)],
+        )
+        def pre_changed(inputs: pd.DataFrame) -> pd.DataFrame:
+            df = pd.DataFrame()
+            df["first_char"] = inputs["string_col"].str[:1].astype("string")
+            return df
+
+        @on_demand_feature_view(
+            sources=[fv],
+            schema=[Field(name="first_char", dtype=String)],
+        )
+        def post_changed(inputs: pd.DataFrame) -> pd.DataFrame:
+            df = pd.DataFrame()
+            df["first_char"] = inputs["string_col"].str[:1].astype("string") + "hi"
+            return df
+
+        feast_object_diffs = diff_registry_objects(
+            pre_changed, pre_changed, "on demand feature view"
+        )
+        assert len(feast_object_diffs.feast_object_property_diffs) == 0
+
+        feast_object_diffs = diff_registry_objects(
+            pre_changed, post_changed, "on demand feature view"
+        )
+
+        # Note that user_defined_function.body is excluded because it always changes (dill is non-deterministic), even
+        # if no code is changed
+        assert len(feast_object_diffs.feast_object_property_diffs) == 3
+        assert feast_object_diffs.feast_object_property_diffs[0].property_name == "name"
+        assert feast_object_diffs.feast_object_property_diffs[1].property_name == "user_defined_function.name"
+        assert feast_object_diffs.feast_object_property_diffs[2].property_name == "user_defined_function.body_text"
diff --git a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py
@@ -11,7 +11,6 @@
 from feast.feature_view import FeatureView
 from feast.field import Field
 from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
-from feast.on_demand_feature_view import on_demand_feature_view
 from feast.repo_config import RepoConfig
 from feast.types import Array, Bytes, Int64, String
 from tests.utils.data_source_test_creator import prep_file_source