Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FSTORE-332] implement method for get_or_create_feature_view #813

Merged
merged 23 commits into from
Oct 19, 2022
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
2329257
remove unused import
davitbzh Jul 13, 2022
e38ca9c
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 13, 2022
10ad8b9
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 13, 2022
16b2e52
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 15, 2022
63b812b
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 18, 2022
9fdd925
Merge remote-tracking branch 'upstream/master'
davitbzh Aug 4, 2022
3be3d58
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 7, 2022
6873412
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 14, 2022
c033dad
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 15, 2022
1b3d076
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 15, 2022
738ce21
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 19, 2022
afe2459
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 20, 2022
f972c0d
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 22, 2022
3145a76
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 26, 2022
fc39ad2
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 28, 2022
b750dd4
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 3, 2022
0274f1b
get_or_create_feature_view
davitbzh Oct 3, 2022
cc20170
Update java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java
davitbzh Oct 4, 2022
d1cd2bc
Update java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java
davitbzh Oct 4, 2022
f8a08dc
address comments
davitbzh Oct 4, 2022
255c5aa
WIP
davitbzh Oct 14, 2022
31f2af1
java client
davitbzh Oct 14, 2022
cc48812
description
davitbzh Oct 15, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 35 additions & 4 deletions java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.logicalclocks.hsfs;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.logicalclocks.hsfs.constructor.Query;
import com.logicalclocks.hsfs.engine.FeatureViewEngine;
import com.logicalclocks.hsfs.engine.SparkEngine;
import com.logicalclocks.hsfs.metadata.FeatureGroupApi;
Expand Down Expand Up @@ -253,14 +254,14 @@ public FeatureGroup.FeatureGroupBuilder createFeatureGroup() {
}

public FeatureGroup getOrCreateFeatureGroup(String name, Integer version) throws IOException, FeatureStoreException {
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, null,
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, null,
null, null, false, null, null, null);
}

public FeatureGroup getOrCreateFeatureGroup(String name, Integer version, List<String> primaryKeys,
boolean onlineEnabled, String eventTime)
throws IOException, FeatureStoreException {
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, primaryKeys,
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, primaryKeys,
null, null, onlineEnabled, null, null, eventTime);
}

Expand All @@ -270,7 +271,7 @@ public FeatureGroup getOrCreateFeatureGroup(String name, Integer version,
boolean onlineEnabled,
String eventTime) throws IOException, FeatureStoreException {

return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, primaryKeys,
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, null, primaryKeys,
partitionKeys, null, onlineEnabled, null, null, eventTime);
}

Expand All @@ -281,7 +282,7 @@ public FeatureGroup getOrCreateFeatureGroup(String name, Integer version, String
StatisticsConfig statisticsConfig, String eventTime)
throws IOException, FeatureStoreException {

return featureGroupApi.getOrCreateFeatureGroup(this, name, version, description, primaryKeys,
return featureGroupApi.getOrCreateFeatureGroup(this, name, version, description, primaryKeys,
partitionKeys, hudiPrecombineKey, onlineEnabled, timeTravelFormat, statisticsConfig, eventTime);
}

Expand Down Expand Up @@ -338,6 +339,36 @@ public FeatureView.FeatureViewBuilder createFeatureView() {
return new FeatureView.FeatureViewBuilder(this);
}

/**
* Get feature view metadata object or create a new one if it doesn't exist. This method doesn't update
* existing feature view metadata object.
*
* @param name name of the feature view
* @param query Query object
* @param version version of the feature view
* @return FeatureView
*/
public FeatureView getOrCreateFeatureView(String name, Query query, Integer version)
throws FeatureStoreException, IOException {
return featureViewEngine.getOrCreateFeatureView(this, name, version, query, null, null);
}

/**
* Get feature view metadata object or create a new one if it doesn't exist. This method doesn't update
* existing feature view metadata object.
*
* @param name name of the feature view
* @param query Query object
* @param version version of the feature view
* @param description description of the feature view
* @param labels list of label features
* @return FeatureView
*/
public FeatureView getOrCreateFeatureView(String name, Query query, Integer version, String description,
List<String> labels) throws FeatureStoreException, IOException {
return featureViewEngine.getOrCreateFeatureView(this, name, version, query, description, labels);
}

/**
* Get a feature view object from the selected feature store.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -398,4 +398,23 @@ public Map<String, Object> getTags(FeatureView featureView, Integer trainingData
throws FeatureStoreException, IOException {
return tagsApi.get(featureView, trainingDataVersion);
}

public FeatureView getOrCreateFeatureView(FeatureStore featureStore, String name, Integer version, Query query,
String description, List<String> labels)
throws FeatureStoreException, IOException {
FeatureView featureView = null;
try {
featureView = get(featureStore, name, version);
} catch (IOException | FeatureStoreException e) {
if (e.getMessage().contains("Error: 404") && e.getMessage().contains("\"errorCode\":270181")) {
featureView = new FeatureView.FeatureViewBuilder(featureStore)
.name(name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

description is missing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ahh, thanks. good catch

.version(version)
.query(query)
.labels(labels)
.build();
}
}
return featureView;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public <S> void insert(StreamFeatureGroup streamFeatureGroup, S featureData,
public <S> StreamFeatureGroup saveFeatureGroupMetaData(StreamFeatureGroup featureGroup, List<String> partitionKeys,
String hudiPrecombineKey, Map<String, String> writeOptions,
JobConfiguration sparkJobConfiguration, S featureData)
throws FeatureStoreException, IOException, ParseException {
throws FeatureStoreException, IOException {

if (featureGroup.getFeatures() == null) {
featureGroup.setFeatures(utils
Expand Down
50 changes: 50 additions & 0 deletions python/hsfs/feature_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -963,6 +963,56 @@ def create_feature_view(
)
return self._feature_view_engine.save(feat_view)

def get_or_create_feature_view(
self,
name: str,
query: Query,
version: int,
kennethmhc marked this conversation as resolved.
Show resolved Hide resolved
description: Optional[str] = "",
labels: Optional[List[str]] = [],
transformation_functions: Optional[Dict[str, TransformationFunction]] = {},
):
"""Get feature view metadata object or create a new one if it doesn't exist. This method doesn't update
existing feature view metadata object.

# Arguments
name: Name of the feature view to create.
query: Feature store `Query`.
version: Version of the feature view to create.
description: A string describing the contents of the feature view to
improve discoverability for Data Scientists, defaults to empty string
`""`.
labels: A list of feature names constituting the prediction label/feature of
the feature view. When replaying a `Query` during model inference,
the label features can be omitted from the feature vector retrieval.
Defaults to `[]`, no label.
transformation_functions: A dictionary mapping tansformation functions to
to the features they should be applied to before writing out the
vector and at inference time. Defaults to `{}`, no
transformations.

# Returns:
`FeatureView`: The feature view metadata object.
"""

try:
return self._feature_view_engine.get(name, version)
except exceptions.RestAPIError as e:
if (
kennethmhc marked this conversation as resolved.
Show resolved Hide resolved
e.response.json().get("errorCode", "") == 270181
and e.response.status_code == 404
):
return self.create_feature_view(
name=name,
query=query,
version=version,
description=description,
labels=labels,
transformation_functions=transformation_functions,
)
else:
raise e

def get_feature_view(self, name: str, version: int = None):
"""Get a feature view entity from the feature store.

Expand Down
32 changes: 15 additions & 17 deletions python/tests/fixtures/generate_backend_fixtures.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -354,17 +354,12 @@
" fg_2.save(df_2)\n",
" \n",
" # fv\n",
"\n",
" try:\n",
" self.feature_view = fs.get_feature_view('fv_test')\n",
" except RestAPIError:\n",
" query = fg_1.select_all().join(fg_2.select_all())\n",
" \n",
" self.feature_view = fs.create_feature_view(\n",
" name='fv_test',\n",
" query=query,\n",
" version=1\n",
" )\n",
" query = fg_1.select_all().join(fg_2.select_all())\n",
" self.feature_view = fs.get_or_create_feature_view(\n",
" name='fv_test',\n",
" query=query,\n",
" version=1\n",
" )\n",
"\n",
" def call(self):\n",
" fs.get_feature_view('fv_test')\n",
Expand Down Expand Up @@ -559,20 +554,23 @@
],
"metadata": {
"kernelspec": {
"display_name": "PySpark",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "pysparkkernel"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "python",
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "pyspark",
"pygments_lexer": "python3"
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
}