feat: Add possibility to define feature_store.yaml path with env vari…

…able (feast-dev#3231)

* Add possibility to define feature_store.yaml path with environment variable

Signed-off-by: arcabucero <arcabucero@gmail.com>

* Add possibility to define feature_store.yaml path with environment variable for aws and lambda_engine

Signed-off-by: arcabucero <arcabucero@gmail.com>

* Add test for overriding feature_store.yaml path with environment variable

Signed-off-by: arcabucero <arcabucero@gmail.com>

Signed-off-by: arcabucero <arcabucero@gmail.com>

sdk/python/feast/cli.py

@@ -99,7 +99,7 @@ def cli(
     ctx.obj["FS_YAML_FILE"] = (
         Path(feature_store_yaml).absolute()
         if feature_store_yaml
-        else ctx.obj["CHDIR"] / "feature_store.yaml"
+        else utils.get_default_yaml_file_path(ctx.obj["CHDIR"])
     )
     try:
         level = getattr(logging, log_level.upper())

sdk/python/feast/constants.py

@@ -23,6 +23,9 @@
 # feature_store.yaml environment variable name for remote feature server
 FEATURE_STORE_YAML_ENV_NAME: str = "FEATURE_STORE_YAML_BASE64"
 
+# feature_store.yaml path environment variable name
+FEAST_FS_YAML_FILE_PATH_ENV_NAME: str = "FEAST_FS_YAML_FILE_PATH"
+
 # Environment variable for registry
 REGISTRY_ENV_NAME: str = "REGISTRY_BASE64"
 

sdk/python/feast/feature_store.py

@@ -160,7 +160,7 @@ def __init__(
             self.config = load_repo_config(self.repo_path, fs_yaml_file)
         else:
             self.config = load_repo_config(
-                self.repo_path, Path(self.repo_path) / "feature_store.yaml"
+                self.repo_path, utils.get_default_yaml_file_path(self.repo_path)
             )
 
         registry_config = self.config.get_registry_config()

sdk/python/feast/infra/aws.py

@@ -8,6 +8,7 @@
 
 from colorama import Fore, Style
 
+from feast import utils
 from feast.constants import (
     AWS_LAMBDA_FEATURE_SERVER_IMAGE,
     AWS_LAMBDA_FEATURE_SERVER_REPOSITORY,
@@ -113,7 +114,10 @@ def _deploy_feature_server(self, project: str, image_uri: str):
 
         if not self.repo_config.repo_path:
             raise RepoConfigPathDoesNotExist()
-        with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f:
+
+        with open(
+            utils.get_default_yaml_file_path(self.repo_config.repo_path), "rb"
+        ) as f:
             config_bytes = f.read()
             config_base64 = base64.b64encode(config_bytes).decode()
 

sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py

@@ -10,6 +10,7 @@
 from pydantic import StrictStr
 from tqdm import tqdm
 
+from feast import utils
 from feast.batch_feature_view import BatchFeatureView
 from feast.constants import FEATURE_STORE_YAML_ENV_NAME
 from feast.entity import Entity
@@ -137,7 +138,7 @@ def __init__(
         )
         repo_path = self.repo_config.repo_path
         assert repo_path
-        feature_store_path = repo_path / "feature_store.yaml"
+        feature_store_path = utils.get_default_yaml_file_path(repo_path)
         self.feature_store_base64 = str(
             base64.b64encode(bytes(feature_store_path.read_text(), "UTF-8")), "UTF-8"
         )

sdk/python/feast/utils.py

@@ -1,6 +1,8 @@
+import os
 import typing
 from collections import defaultdict
 from datetime import datetime
+from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 import pandas as pd
@@ -9,6 +11,7 @@
 from dateutil.tz import tzlocal
 from pytz import utc
 
+from feast.constants import FEAST_FS_YAML_FILE_PATH_ENV_NAME
 from feast.entity import Entity
 from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
 from feast.protos.feast.types.Value_pb2 import Value as ValueProto
@@ -51,6 +54,14 @@ def maybe_local_tz(t: datetime) -> datetime:
         return t
 
 
+def get_default_yaml_file_path(repo_path: Path) -> Path:
+    if FEAST_FS_YAML_FILE_PATH_ENV_NAME in os.environ:
+        yaml_path = os.environ[FEAST_FS_YAML_FILE_PATH_ENV_NAME]
+        return Path(yaml_path)
+    else:
+        return repo_path / "feature_store.yaml"
+
+
 def _get_requested_feature_views_to_features_dict(
     feature_refs: List[str],
     feature_views: List["FeatureView"],

sdk/python/tests/unit/cli/test_cli.py

@@ -1,7 +1,9 @@
+import os
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
 from textwrap import dedent
+from unittest import mock
 
 from assertpy import assertpy
 
@@ -85,6 +87,21 @@ def test_3rd_party_registry_store_with_fs_yaml_override() -> None:
         assertpy.assert_that(return_code).is_equal_to(0)
 
 
+def test_3rd_party_registry_store_with_fs_yaml_override_by_env_var() -> None:
+    runner = CliRunner()
+
+    fs_yaml_file = "test_fs.yaml"
+    with setup_third_party_registry_store_repo(
+        "foo.registry_store.FooRegistryStore", fs_yaml_file_name=fs_yaml_file
+    ) as repo_path:
+        custom_yaml_path = os.path.join(repo_path, fs_yaml_file)
+        with mock.patch.dict(
+            "os.environ", {"FEAST_FS_YAML_FILE_PATH": custom_yaml_path}, clear=True
+        ):
+            return_code, output = runner.run_with_output(["apply"], cwd=repo_path)
+            assertpy.assert_that(return_code).is_equal_to(0)
+
+
 @contextmanager
 def setup_third_party_provider_repo(provider_name: str):
     with tempfile.TemporaryDirectory() as repo_dir_name: