Replace double create with save for training datasets (#38)

logicalclocks · Apr 28, 2020 · 279ade5 · 279ade5
1 parent b3fd685
commit 279ade5
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 14 deletions.
diff --git a/java/pom.xml b/java/pom.xml
@@ -13,6 +13,7 @@
         <maven.compiler.target>1.8</maven.compiler.target>
         <guava.version>14.0.1</guava.version>
         <httpclient.version>4.5.6</httpclient.version>
+        <httpcore.version>4.4.13</httpcore.version>
         <slf4j.version>1.7.16</slf4j.version>
         <log4j.version>1.2.17</log4j.version>
         <handy.version>2.1.8</handy.version>
@@ -89,6 +90,13 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>${httpcore.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -202,4 +210,4 @@
     </repositories>
 
 
-</project>
+</project>
diff --git a/java/src/main/java/com/logicalclocks/hsfs/MainClass.java b/java/src/main/java/com/logicalclocks/hsfs/MainClass.java
@@ -55,7 +55,7 @@ public static void main(String[] args) throws Exception {
         .splits(splits)
         .build();
 
-    td.create(query);
+    td.save(query);
 
     SparkEngine.getInstance().getSparkSession().close();
   }

diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java
@@ -107,8 +107,8 @@ public TrainingDataset(@NonNull String name, @NonNull Integer version, String de
    * @throws FeatureStoreException
    * @throws IOException
    */
-  public void create(Query query) throws FeatureStoreException, IOException {
-    create(query, null);
+  public void save(Query query) throws FeatureStoreException, IOException {
+    save(query, null);
   }
 
   /**
@@ -117,8 +117,8 @@ public void create(Query query) throws FeatureStoreException, IOException {
    * @throws FeatureStoreException
    * @throws IOException
    */
-  public void create(Dataset<Row> dataset) throws FeatureStoreException, IOException {
-    create(dataset, null);
+  public void save(Dataset<Row> dataset) throws FeatureStoreException, IOException {
+    save(dataset, null);
   }
 
   /**
@@ -128,8 +128,8 @@ public void create(Dataset<Row> dataset) throws FeatureStoreException, IOExcepti
    * @throws FeatureStoreException
    * @throws IOException
    */
-  public void create(Query query, Map<String, String> writeOptions) throws FeatureStoreException, IOException {
-    trainingDatasetEngine.create(this, query.read(), writeOptions);
+  public void save(Query query, Map<String, String> writeOptions) throws FeatureStoreException, IOException {
+    trainingDatasetEngine.save(this, query.read(), writeOptions);
   }
 
   /**
@@ -139,9 +139,9 @@ public void create(Query query, Map<String, String> writeOptions) throws Feature
    * @throws FeatureStoreException
    * @throws IOException
    */
-  public void create(Dataset<Row> dataset, Map<String, String> writeOptions)
+  public void save(Dataset<Row> dataset, Map<String, String> writeOptions)
       throws FeatureStoreException, IOException {
-    trainingDatasetEngine.create(this, dataset, writeOptions);
+    trainingDatasetEngine.save(this, dataset, writeOptions);
   }
 
   /**

diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java
@@ -49,7 +49,7 @@ public class TrainingDatasetEngine {
    * @throws FeatureStoreException
    * @throws IOException
    */
-  public void create(TrainingDataset trainingDataset, Dataset<Row> dataset,
+  public void save(TrainingDataset trainingDataset, Dataset<Row> dataset,
                      Map<String, String> userWriteOptions)
       throws FeatureStoreException, IOException {
     // TODO(Fabio): make sure we can implement the serving part as well

diff --git a/python/hsfs/core/training_dataset_engine.py b/python/hsfs/core/training_dataset_engine.py
@@ -27,7 +27,7 @@ def __init__(self, feature_store_id):
             feature_store_id
         )
 
-    def create(self, training_dataset, feature_dataframe, user_write_options):
+    def save(self, training_dataset, feature_dataframe, user_write_options):
         self._training_dataset_api.post(training_dataset)
 
         write_options = engine.get_instance().write_options(

diff --git a/python/hsfs/training_dataset.py b/python/hsfs/training_dataset.py
@@ -96,7 +96,7 @@ def __init__(
             self._splits = splits
             self._training_dataset_type = training_dataset_type
 
-    def create(self, features, write_options={}):
+    def save(self, features, write_options={}):
         # TODO: Decide if we want to have potentially dangerous defaults like {}
         if isinstance(features, query.Query):
             feature_dataframe = features.read()
@@ -106,7 +106,7 @@ def create(self, features, write_options={}):
             )
 
         self._features = engine.get_instance().parse_schema(feature_dataframe)
-        self._training_dataset_engine.create(self, feature_dataframe, write_options)
+        self._training_dataset_engine.save(self, feature_dataframe, write_options)
         return self
 
     def insert(self, features, overwrite, write_options={}):