Add e2e test for purview registry and rbac registry (#689)

* Add e2e test for purview registry and rbac registry

* Add purview and rbac env e2e to registry tests

* Fix merge issue

feathr_project/test/test_fixture.py

@@ -161,17 +161,17 @@ def kafka_test_setup(config_path: str):
  return client
 
 def registry_test_setup(config_path: str):
-
-
- # use a new project name every time to make sure all features are registered correctly
+ # Use a new project name every time to make sure all features are registered correctly
+ # Project name example: feathr_ci_registry_2022_09_24_01_02_30
  now = datetime.now()
- os.environ["project_config__project_name"] =  ''.join(['feathr_ci_registry','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) 
+ os.environ["project_config__project_name"] = f'feathr_ci_registry_{str(now)[:19].replace(" ", "_").replace(":", "_").replace("-", "_")}'
 
  client = FeathrClient(config_path=config_path, project_registry_tag={"for_test_purpose":"true"})
  request_anchor, agg_anchor, derived_feature_list = generate_entities()
 
  client.build_features(anchor_list=[agg_anchor, request_anchor], derived_feature_list=derived_feature_list)
  return client
+
 def registry_test_setup_partially(config_path: str):
  """Register a partial of a project. Will call `generate_entities()` and register only the first anchor feature.
  """

feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml

@@ -0,0 +1,49 @@
+api_version: 1
+project_config:
+ project_name: 'project_feathr_integration_test'
+ required_environment_variables:
+ optional_environment_variables:
+
+offline_store:
+ adls:
+ adls_enabled: false
+ wasb:
+ wasb_enabled: false
+ s3:
+ s3_enabled: false
+ jdbc:
+ jdbc_enabled: false
+ snowflake:
+ snowflake_enabled: false
+
+spark_config:
+ spark_cluster: 'databricks'
+ spark_result_output_parts: '1'
+ azure_synapse:
+ dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'
+ pool_name: 'spark3'
+ workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace'
+ executor_size: 'Small'
+ executor_num: 1
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+ databricks:
+ workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/'
+ workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db'
+ config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}
+ work_dir: 'dbfs:/feathr_getting_started'
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+
+online_store:
+ redis:
+ host: 'feathrazuretest3redis.redis.cache.windows.net'
+ port: 6380
+ ssl_enabled: True
+
+feature_registry:
+ api_endpoint: "https://feathr-registry-purview.azurewebsites.net/api/v1"
+
+monitoring:
+ database:
+ sql:
+ url: 'jdbc:postgresql://featuremonitoring.postgres.database.azure.com:5432/postgres'
+ user: "demo"

feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml

@@ -0,0 +1,49 @@
+api_version: 1
+project_config:
+ project_name: 'project_feathr_integration_test'
+ required_environment_variables:
+ optional_environment_variables:
+
+offline_store:
+ adls:
+ adls_enabled: false
+ wasb:
+ wasb_enabled: false
+ s3:
+ s3_enabled: false
+ jdbc:
+ jdbc_enabled: false
+ snowflake:
+ snowflake_enabled: false
+
+spark_config:
+ spark_cluster: 'databricks'
+ spark_result_output_parts: '1'
+ azure_synapse:
+ dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'
+ pool_name: 'spark3'
+ workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace'
+ executor_size: 'Small'
+ executor_num: 1
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+ databricks:
+ workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/'
+ workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db'
+ config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}
+ work_dir: 'dbfs:/feathr_getting_started'
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+
+online_store:
+ redis:
+ host: 'feathrazuretest3redis.redis.cache.windows.net'
+ port: 6380
+ ssl_enabled: True
+
+feature_registry:
+ api_endpoint: "https://feathr-registry-purview-rbac.azurewebsites.net/api/v1"
+
+monitoring:
+ database:
+ sql:
+ url: 'jdbc:postgresql://featuremonitoring.postgres.database.azure.com:5432/postgres'
+ user: "demo"

feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml

@@ -0,0 +1,49 @@
+api_version: 1
+project_config:
+ project_name: 'project_feathr_integration_test'
+ required_environment_variables:
+ optional_environment_variables:
+
+offline_store:
+ adls:
+ adls_enabled: false
+ wasb:
+ wasb_enabled: false
+ s3:
+ s3_enabled: false
+ jdbc:
+ jdbc_enabled: false
+ snowflake:
+ snowflake_enabled: false
+
+spark_config:
+ spark_cluster: 'databricks'
+ spark_result_output_parts: '1'
+ azure_synapse:
+ dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'
+ pool_name: 'spark3'
+ workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace'
+ executor_size: 'Small'
+ executor_num: 1
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+ databricks:
+ workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/'
+ workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db'
+ config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}
+ work_dir: 'dbfs:/feathr_getting_started'
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+
+online_store:
+ redis:
+ host: 'feathrazuretest3redis.redis.cache.windows.net'
+ port: 6380
+ ssl_enabled: True
+
+feature_registry:
+ api_endpoint: "https://feathr-registry-sql.azurewebsites.net/api/v1"
+
+monitoring:
+ database:
+ sql:
+ url: 'jdbc:postgresql://featuremonitoring.postgres.database.azure.com:5432/postgres'
+ user: "demo"

feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml

@@ -0,0 +1,49 @@
+api_version: 1
+project_config:
+ project_name: 'project_feathr_integration_test'
+ required_environment_variables:
+ optional_environment_variables:
+
+offline_store:
+ adls:
+ adls_enabled: false
+ wasb:
+ wasb_enabled: false
+ s3:
+ s3_enabled: false
+ jdbc:
+ jdbc_enabled: false
+ snowflake:
+ snowflake_enabled: false
+
+spark_config:
+ spark_cluster: 'databricks'
+ spark_result_output_parts: '1'
+ azure_synapse:
+ dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'
+ pool_name: 'spark3'
+ workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace'
+ executor_size: 'Small'
+ executor_num: 1
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+ databricks:
+ workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/'
+ workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db'
+ config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}
+ work_dir: 'dbfs:/feathr_getting_started'
+ feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar"
+
+online_store:
+ redis:
+ host: 'feathrazuretest3redis.redis.cache.windows.net'
+ port: 6380
+ ssl_enabled: True
+
+feature_registry:
+ api_endpoint: "https://feathr-registry-sql-rbac.azurewebsites.net/api/v1"
+
+monitoring:
+ database:
+ sql:
+ url: 'jdbc:postgresql://featuremonitoring.postgres.database.azure.com:5432/postgres'
+ user: "demo"