From cd4ba6a1698030ebff5984dd2a5a4dd574aaec0d Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Tue, 28 Jul 2020 00:54:22 +0200 Subject: [PATCH 1/3] \#65 add checkstyle to java apis --- java/pom.xml | 24 ++ .../com/logicalclocks/hsfs/FeatureGroup.java | 21 +- .../com/logicalclocks/hsfs/FeatureStore.java | 37 ++- .../java/com/logicalclocks/hsfs/FsQuery.java | 1 + .../hsfs/HopsworksConnection.java | 19 +- .../java/com/logicalclocks/hsfs/JoinType.java | 1 + .../logicalclocks/hsfs/StorageConnector.java | 4 +- .../logicalclocks/hsfs/TrainingDataset.java | 99 +++--- .../hsfs/metadata/AuthorizationHandler.java | 12 +- .../hsfs/metadata/Credentials.java | 3 +- .../hsfs/metadata/HopsworksClient.java | 18 +- .../metadata/HopsworksExternalClient.java | 14 +- .../metadata/HopsworksHostnameVerifier.java | 1 + .../metadata/HopsworksInternalClient.java | 17 +- .../hsfs/metadata/InternalException.java | 1 + .../com/logicalclocks/hsfs/metadata/Join.java | 1 + .../logicalclocks/hsfs/metadata/Query.java | 42 ++- .../metadata/{RestDTO.java => RestDto.java} | 4 +- .../com/logicalclocks/hsfs/metadata/Tags.java | 2 +- .../logicalclocks/hsfs/metadata/TagsApi.java | 68 ++-- .../hsfs/metadata/TrainingDatasetApi.java | 9 +- .../logicalclocks/hsfs/util/Constants.java | 1 + java/src/main/resources/checkstyle.xml | 310 ++++++++++++++++++ 23 files changed, 536 insertions(+), 173 deletions(-) rename java/src/main/java/com/logicalclocks/hsfs/metadata/{RestDTO.java => RestDto.java} (94%) create mode 100644 java/src/main/resources/checkstyle.xml diff --git a/java/pom.xml b/java/pom.xml index 143405bfee..b42224b697 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -192,6 +192,30 @@ + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.1 + + + validate + validate + + check + + + + + src/main/resources/checkstyle.xml + true + true + true + true + + src/main/java + + + diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java index 45ca2335aa..62eaa61b19 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -176,9 +177,9 @@ public void delete() throws FeatureStoreException, IOException { } /** - * Add a tag without value to the feature group + * Add a tag without value to the feature group. * - * @param name: name of the tag + * @param name name of the tag * @throws FeatureStoreException * @throws IOException */ @@ -187,10 +188,10 @@ public void addTag(String name) throws FeatureStoreException, IOException { } /** - * Add name/value tag to the feature group + * Add name/value tag to the feature group. * - * @param name: name of the tag - * @param value: value of the tag + * @param name name of the tag + * @param value value of the tag * @throws FeatureStoreException * @throws IOException */ @@ -199,7 +200,7 @@ public void addTag(String name, String value) throws FeatureStoreException, IOEx } /** - * Get all tags of the feature group + * Get all tags of the feature group. * * @return map of all tags from name to value * @throws FeatureStoreException @@ -211,9 +212,9 @@ public Map getTag() throws FeatureStoreException, IOException { } /** - * Get a single tag value of the feature group + * Get a single tag value of the feature group. * - * @param name: name of tha tag + * @param name name of tha tag * @return string value of the tag * @throws FeatureStoreException * @throws IOException @@ -224,9 +225,9 @@ public Map getTag(String name) throws FeatureStoreException, IOE } /** - * Delete a tag of the feature group + * Delete a tag of the feature group. * - * @param name: name of the tag to be deleted + * @param name name of the tag to be deleted * @throws FeatureStoreException * @throws IOException */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java index e79eb42301..8b1969f9b6 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java @@ -13,11 +13,10 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Strings; -import com.logicalclocks.hsfs.engine.FeatureGroupEngine; import com.logicalclocks.hsfs.engine.SparkEngine; import com.logicalclocks.hsfs.metadata.FeatureGroupApi; import com.logicalclocks.hsfs.metadata.StorageConnectorApi; @@ -60,9 +59,10 @@ public FeatureStore() throws FeatureStoreException { } /** - * Get a feature group object from the feature store - * @param name: the name of the feature group - * @param version: the version of the feature group + * Get a feature group object from the feature store. + * + * @param name the name of the feature group + * @param version the version of the feature group * @return * @throws FeatureStoreException */ @@ -72,8 +72,9 @@ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer versi } /** - * Get a feature group object with default version `1` from the feature store - * @param name: the name of the feature group + * Get a feature group object with default version `1` from the feature store. + * + * @param name the name of the feature group * @return * @throws FeatureStoreException */ @@ -103,9 +104,10 @@ public TrainingDataset.TrainingDatasetBuilder createTrainingDataset() { } /** - * Get a training dataset object from the selected feature store - * @param name: name of the training dataset - * @param version: version to get + * Get a training dataset object from the selected feature store. + * + * @param name name of the training dataset + * @param version version to get * @return * @throws FeatureStoreException * @throws IOException @@ -116,8 +118,9 @@ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer } /** - * Get a training dataset object with the default version `1` from the selected feature store - * @param name: name of the training dataset + * Get a training dataset object with the default version `1` from the selected feature store. + * + * @param name name of the training dataset * @return * @throws FeatureStoreException * @throws IOException @@ -131,10 +134,10 @@ public TrainingDataset getTrainingDataset(String name) throws FeatureStoreExcept @Override public String toString() { return "FeatureStore{" + - "id=" + id + - ", name='" + name + '\'' + - ", projectId=" + projectId + - ", featureGroupApi=" + featureGroupApi + - '}'; + "id=" + id + + ", name='" + name + '\'' + + ", projectId=" + projectId + + ", featureGroupApi=" + featureGroupApi + + '}'; } } diff --git a/java/src/main/java/com/logicalclocks/hsfs/FsQuery.java b/java/src/main/java/com/logicalclocks/hsfs/FsQuery.java index fa30731a0c..64c302b18e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FsQuery.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FsQuery.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java index e832e218f0..ce2859d6c5 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java +++ b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.google.common.base.Strings; @@ -58,7 +59,7 @@ public class HopsworksConnection implements Closeable { private String certPath; @Getter - private String APIKeyFilePath; + private String ApiKeyFilePath; private FeatureStoreApi featureStoreApi = new FeatureStoreApi(); @@ -69,7 +70,7 @@ public class HopsworksConnection implements Closeable { @Builder public HopsworksConnection(String host, int port, String project, Region region, SecretStore secretStore, boolean hostnameVerification, String trustStorePath, - String certPath, String APIKeyFilePath) throws IOException, FeatureStoreException { + String certPath, String ApiKeyFilePath) throws IOException, FeatureStoreException { this.host = host; this.port = port; this.project = project; @@ -78,29 +79,31 @@ public HopsworksConnection(String host, int port, String project, Region region, this.hostnameVerification = hostnameVerification; this.trustStorePath = trustStorePath; this.certPath = certPath; - this.APIKeyFilePath = APIKeyFilePath; + this.ApiKeyFilePath = ApiKeyFilePath; HopsworksClient hopsworksClient = HopsworksClient.setupHopsworksClient(host, port, region, secretStore, - hostnameVerification, trustStorePath, APIKeyFilePath); + hostnameVerification, trustStorePath, this.ApiKeyFilePath); projectObj = getProject(); hopsworksClient.downloadCredentials(projectObj, certPath); } /** - * Retrieve the project feature store + * Retrieve the project feature store. + * * @return * @throws IOException * @throws FeatureStoreException */ - public FeatureStore getFeatureStore() throws IOException, FeatureStoreException{ + public FeatureStore getFeatureStore() throws IOException, FeatureStoreException { return getFeatureStore(project + Constants.FEATURESTORE_SUFFIX); } /** * Retrieve a feature store based on name. The feature store needs to be shared with * the connection's project. - * @param name - * @return + * + * @param name the name of the feature store to get the handle for + * @return FeatureStore * @throws IOException * @throws FeatureStoreException */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/JoinType.java b/java/src/main/java/com/logicalclocks/hsfs/JoinType.java index 7d696c0f98..50c881b185 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/JoinType.java +++ b/java/src/main/java/com/logicalclocks/hsfs/JoinType.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public enum JoinType { diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java b/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java index e1dd005e13..5a0afb5a03 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java +++ b/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.logicalclocks.hsfs.util.Constants; @@ -26,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.Stream; @AllArgsConstructor @NoArgsConstructor @@ -53,7 +53,7 @@ public class StorageConnector { @Getter @Setter private StorageConnectorType storageConnectorType; - public Map getSparkOptions() throws FeatureStoreException{ + public Map getSparkOptions() throws FeatureStoreException { List args = Arrays.stream(arguments.split(",")) .map(arg -> arg.split("=")) .collect(Collectors.toList()); diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java index 35cf1a6951..057c9d767f 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java +++ b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java @@ -102,8 +102,9 @@ public TrainingDataset(@NonNull String name, Integer version, String description } /** - * Create the training dataset based on the content of the feature store query - * @param query + * Create the training dataset based on the content of the feature store query. + * + * @param query the query to save as training dataset * @throws FeatureStoreException * @throws IOException */ @@ -112,8 +113,9 @@ public void save(Query query) throws FeatureStoreException, IOException { } /** - * Create the training dataset based on teh content of the dataset - * @param dataset + * Create the training dataset based on teh content of the dataset. + * + * @param dataset the dataset to save as training dataset * @throws FeatureStoreException * @throws IOException */ @@ -122,9 +124,10 @@ public void save(Dataset dataset) throws FeatureStoreException, IOException } /** - * Create the training dataset based on the content of the feature store query - * @param query - * @param writeOptions: options to pass to the Spark write operation + * Create the training dataset based on the content of the feature store query. + * + * @param query the query to save as training dataset + * @param writeOptions options to pass to the Spark write operation * @throws FeatureStoreException * @throws IOException */ @@ -133,9 +136,10 @@ public void save(Query query, Map writeOptions) throws FeatureSt } /** - * Create the training dataset based on teh content of the dataset - * @param dataset - * @param writeOptions: options to pass to the Spark write operation + * Create the training dataset based on teh content of the dataset. + * + * @param dataset the dataset to save as training dataset + * @param writeOptions options to pass to the Spark write operation * @throws FeatureStoreException * @throws IOException */ @@ -145,9 +149,10 @@ public void save(Dataset dataset, Map writeOptions) } /** - * Insert the content of the feature store query in the training dataset - * @param query - * @param overwrite: true to overwrite the current content of the training dataset + * Insert the content of the feature store query in the training dataset. + * + * @param query the query to write as training dataset + * @param overwrite true to overwrite the current content of the training dataset * @throws FeatureStoreException * @throws IOException */ @@ -156,9 +161,10 @@ public void insert(Query query, boolean overwrite) throws FeatureStoreException, } /** - * Insert the content of the dataset in the training dataset - * @param dataset - * @param overwrite: true to overwrite the current content of the training dataset + * Insert the content of the dataset in the training dataset. + * + * @param dataset the dataset to write as training dataset + * @param overwrite true to overwrite the current content of the training dataset * @throws FeatureStoreException * @throws IOException */ @@ -167,10 +173,11 @@ public void insert(Dataset dataset, boolean overwrite) throws FeatureStoreE } /** - * Insert the content of the feature store query in the training dataset - * @param query - * @param overwrite: true to overwrite the current content of the training dataset - * @param writeOptions: options to pass to the Spark write operation + * Insert the content of the feature store query in the training dataset. + * + * @param query the query to execute to generate the training dataset + * @param overwrite true to overwrite the current content of the training dataset + * @param writeOptions options to pass to the Spark write operation * @throws FeatureStoreException * @throws IOException */ @@ -181,10 +188,11 @@ public void insert(Query query, boolean overwrite, Map writeOpti } /** - * Insert the content of the dataset in the training dataset - * @param dataset - * @param overwrite: true to overwrite the current content of the training dataset - * @param writeOptions: options to pass to the Spark write operation + * Insert the content of the dataset in the training dataset. + * + * @param dataset the spark dataframe to write as training dataset + * @param overwrite true to overwrite the current content of the training dataset + * @param writeOptions options to pass to the Spark write operation * @throws FeatureStoreException * @throws IOException */ @@ -195,7 +203,8 @@ public void insert(Dataset dataset, boolean overwrite, Map } /** - * Read the content (all splits if multiple available) of the training dataset + * Read the content (all splits if multiple available) of the training dataset. + * * @return */ public Dataset read() { @@ -203,8 +212,9 @@ public Dataset read() { } /** - * Read the content (all splits if multiple available) of the training dataset - * @param readOptions: options to pass to the Spark read operation + * Read the content (all splits if multiple available) of the training dataset. + * + * @param readOptions options to pass to the Spark read operation * @return */ public Dataset read(Map readOptions) { @@ -212,8 +222,9 @@ public Dataset read(Map readOptions) { } /** - * Read all a single split from the training dataset - * @param split: the split name + * Read all a single split from the training dataset. + * + * @param split the split name * @return */ public Dataset read(String split) { @@ -222,9 +233,10 @@ public Dataset read(String split) { /** - * Read a single split from the training dataset - * @param split: the split name - * @param readOptions: options to pass to the Spark read operation + * Read a single split from the training dataset. + * + * @param split the split name + * @param readOptions options to pass to the Spark read operation * @return */ public Dataset read(String split, Map readOptions) { @@ -232,7 +244,8 @@ public Dataset read(String split, Map readOptions) { } /** - * Show numRows from the training dataset (across all splits) + * Show numRows from the training dataset (across all splits). + * * @param numRows */ public void show(int numRows) { @@ -240,9 +253,9 @@ public void show(int numRows) { } /** - * Add a tag without value to the training dataset + * Add a tag without value to the training dataset. * - * @param name: name of the tag + * @param name name of the tag * @throws FeatureStoreException * @throws IOException */ @@ -251,10 +264,10 @@ public void addTag(String name) throws FeatureStoreException, IOException { } /** - * Add name/value tag to the training dataset + * Add name/value tag to the training dataset. * - * @param name: name of the tag - * @param value: value of the tag + * @param name name of the tag + * @param value value of the tag * @throws FeatureStoreException * @throws IOException */ @@ -263,7 +276,7 @@ public void addTag(String name, String value) throws FeatureStoreException, IOEx } /** - * Get all tags of the training dataset + * Get all tags of the training dataset. * * @return map of all tags from name to value * @throws FeatureStoreException @@ -275,9 +288,9 @@ public Map getTag() throws FeatureStoreException, IOException { } /** - * Get a single tag value of the training dataset + * Get a single tag value of the training dataset. * - * @param name: name of tha tag + * @param name name of tha tag * @return string value of the tag * @throws FeatureStoreException * @throws IOException @@ -288,9 +301,9 @@ public Map getTag(String name) throws FeatureStoreException, IOE } /** - * Delete a tag of the training dataset + * Delete a tag of the training dataset. * - * @param name: name of the tag to be deleted + * @param name name of the tag to be deleted * @throws FeatureStoreException * @throws IOException */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java index dd13fdf727..83ddb641e7 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import org.apache.http.HttpResponse; @@ -28,7 +29,8 @@ public class AuthorizationHandler implements ResponseHandler { private ResponseHandler originalResponseHandler; - AuthorizationHandler(ResponseHandler originalResponseHandler) { this.originalResponseHandler = originalResponseHandler; + AuthorizationHandler(ResponseHandler originalResponseHandler) { + this.originalResponseHandler = originalResponseHandler; } AuthorizationHandler() { @@ -39,11 +41,11 @@ public T handleResponse(HttpResponse response) throws ClientProtocolException, I if (response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { throw new UnauthorizedException(); } else if (response.getStatusLine().getStatusCode() / 100 == 4) { - throw new IOException("Error: " + response.getStatusLine().getStatusCode() + - EntityUtils.toString(response.getEntity(), Charset.defaultCharset())); + throw new IOException("Error: " + response.getStatusLine().getStatusCode() + + EntityUtils.toString(response.getEntity(), Charset.defaultCharset())); } else if (response.getStatusLine().getStatusCode() / 100 == 5) { - throw new InternalException("Error: " + response.getStatusLine().getStatusCode() + - EntityUtils.toString(response.getEntity(), Charset.defaultCharset())); + throw new InternalException("Error: " + response.getStatusLine().getStatusCode() + + EntityUtils.toString(response.getEntity(), Charset.defaultCharset())); } if (originalResponseHandler != null) { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java index 8473fa2eb2..0a7bdf8a1e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import lombok.AllArgsConstructor; @@ -23,9 +24,7 @@ public class Credentials { private String kStore; - private String tStore; - private String password; public String getkStore() { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java index c5e858b91a..ed5327c923 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java @@ -40,8 +40,8 @@ public class HopsworksClient { - public final static String API_PATH = "/hopsworks-api/api"; - public final static String PROJECT_PATH = API_PATH + "/project{/projectId}"; + public static final String API_PATH = "/hopsworks-api/api"; + public static final String PROJECT_PATH = API_PATH + "/project{/projectId}"; private static HopsworksClient hopsworksClientInstance = null; private static final Logger LOGGER = LoggerFactory.getLogger(HopsworksClient.class); @@ -60,9 +60,9 @@ public static void setInstance(HopsworksClient instance) { hopsworksClientInstance = instance; } - public synchronized static HopsworksClient setupHopsworksClient(String host, int port, Region region, - SecretStore secretStore, boolean hostnameVerification, - String trustStorePath, String APIKeyFilePath) + public static synchronized HopsworksClient setupHopsworksClient(String host, int port, Region region, + SecretStore secretStore, boolean hostnameVerification, + String trustStorePath, String ApiKeyFilePath) throws FeatureStoreException { if (hopsworksClientInstance != null) { return hopsworksClientInstance; @@ -74,7 +74,7 @@ public synchronized static HopsworksClient setupHopsworksClient(String host, int hopsworksHttpClient = new HopsworksInternalClient(); } else { hopsworksHttpClient = new HopsworksExternalClient(host, port, region, - secretStore, hostnameVerification, trustStorePath, APIKeyFilePath); + secretStore, hostnameVerification, trustStorePath, ApiKeyFilePath); } } catch (Exception e) { throw new FeatureStoreException("Could not setup Hopsworks client", e); @@ -127,11 +127,11 @@ public BaseHandler(Class cls, ObjectMapper objectMapper) { @Override public T handleResponse(HttpResponse response) throws ClientProtocolException, IOException { - String responseJSON = EntityUtils.toString(response.getEntity(), Charset.defaultCharset()); + String responseJson = EntityUtils.toString(response.getEntity(), Charset.defaultCharset()); if (response.getStatusLine().getStatusCode() / 100 == 2) { - return objectMapper.readValue(responseJSON, cls); + return objectMapper.readValue(responseJson, cls); } else { - HopsworksErrorClass error = objectMapper.readValue(responseJSON, HopsworksErrorClass.class); + HopsworksErrorClass error = objectMapper.readValue(responseJson, HopsworksErrorClass.class); LOGGER.info("Request error: " + response.getStatusLine().getStatusCode() + " " + error); throw new ClientProtocolException("Request error: " + response.getStatusLine().getStatusCode() + " " + error); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java index 9f92eb7cfc..04faccfcc3 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java @@ -73,7 +73,7 @@ public class HopsworksExternalClient implements HopsworksHttpClient { public HopsworksExternalClient(String host, int port, Region region, SecretStore secretStore, boolean hostnameVerification, - String trustStorePath, String APIKeyFilePath) + String trustStorePath, String ApiKeyFilepath) throws IOException, FeatureStoreException, KeyStoreException, CertificateException, NoSuchAlgorithmException, KeyManagementException { @@ -89,7 +89,7 @@ public HopsworksExternalClient(String host, int port, Region region, .setKeepAliveStrategy((httpResponse, httpContext) -> 30 * 1000) .build(); - apiKey = readAPIKey(secretStore, region, APIKeyFilePath); + apiKey = readAPIKey(secretStore, region, ApiKeyFilepath); } public HopsworksExternalClient(CloseableHttpClient httpClient, HttpHost httpHost) { @@ -131,15 +131,15 @@ private Registry createConnectionFactory(HttpHost httpH * - FIle on the local file system * @param secretStore * @param region - * @param APIKeyFilePath + * @param ApiKeyFilepath * @return * @throws IOException * @throws FeatureStoreException */ - public String readAPIKey(SecretStore secretStore, Region region, String APIKeyFilePath) + public String readAPIKey(SecretStore secretStore, Region region, String ApiKeyFilepath) throws IOException, FeatureStoreException { - if (!Strings.isNullOrEmpty(APIKeyFilePath)) { - return FileUtils.readFileToString(Paths.get(APIKeyFilePath).toFile()); + if (!Strings.isNullOrEmpty(ApiKeyFilepath)) { + return FileUtils.readFileToString(Paths.get(ApiKeyFilepath).toFile()); } switch (secretStore) { @@ -148,7 +148,7 @@ public String readAPIKey(SecretStore secretStore, Region region, String APIKeyFi case SECRET_MANAGER: return readAPIKeySecretManager(region, "api-key"); default: - throw new FeatureStoreException("APIKeyFilePath needs to be set for local mode"); + throw new FeatureStoreException("ApiKeyFilepath needs to be set for local mode"); } } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java index a532e90e06..5f754a42a6 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import javax.net.ssl.HostnameVerifier; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java index 4350f27890..c6826c2721 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.logicalclocks.hsfs.FeatureStoreException; @@ -52,12 +53,12 @@ public class HopsworksInternalClient implements HopsworksHttpClient { - public final static String REST_ENDPOINT_SYS = "hopsworks.restendpoint"; + public static final String REST_ENDPOINT_SYS = "hopsworks.restendpoint"; private static final Logger LOGGER = LoggerFactory.getLogger(HopsworksInternalClient.class.getName()); - private final static String DOMAIN_CA_TRUSTSTORE = "hopsworks.domain.truststore"; - private final static String TOKEN_PATH = "token.jwt"; + private static final String DOMAIN_CA_TRUSTSTORE = "hopsworks.domain.truststore"; + private static final String TOKEN_PATH = "token.jwt"; private PoolingHttpClientConnectionManager connectionPool = null; @@ -81,7 +82,7 @@ public HopsworksInternalClient() throws IOException, KeyStoreException, Certific .setKeepAliveStrategy((httpResponse, httpContext) -> 30 * 1000) .build(); - refreshJWT(); + refreshJwt(); } private Registry createConnectionFactory() @@ -105,7 +106,7 @@ private Registry createConnectionFactory() .build(); } - public void refreshJWT() throws FeatureStoreException { + public void refreshJwt() throws FeatureStoreException { try (FileChannel fc = FileChannel.open(Paths.get(TOKEN_PATH), StandardOpenOption.READ)) { FileLock fileLock = fc.tryLock(0, Long.MAX_VALUE, true); try { @@ -119,8 +120,8 @@ public void refreshJWT() throws FeatureStoreException { } //If could not acquire lock in reasonable time, throw exception if (fileLock == null) { - throw new FeatureStoreException("Could not read jwt token from local container, possibly another process has" + - " acquired the lock"); + throw new FeatureStoreException("Could not read jwt token from local container, possibly another process has" + + " acquired the lock"); } ByteBuffer buf = ByteBuffer.allocateDirect(512); fc.read(buf); @@ -148,7 +149,7 @@ public T handleRequest(HttpRequest request, ResponseHandler responseHandl return httpClient.execute(httpHost, request, authHandler); } catch (UnauthorizedException e) { // re-read the jwt and try one more time - refreshJWT(); + refreshJwt(); request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwt); return httpClient.execute(httpHost, request, authHandler); } catch (InternalException e) { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java index 927470316d..125a914458 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import org.apache.http.client.ClientProtocolException; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Join.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Join.java index 3317680492..00c277d5cb 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Join.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Join.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.logicalclocks.hsfs.Feature; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Query.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Query.java index 8bf66d112b..1324302c06 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Query.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Query.java @@ -13,9 +13,15 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; -import com.logicalclocks.hsfs.*; +import com.logicalclocks.hsfs.Feature; +import com.logicalclocks.hsfs.FeatureGroup; +import com.logicalclocks.hsfs.FeatureStoreException; +import com.logicalclocks.hsfs.JoinType; +import com.logicalclocks.hsfs.Storage; +import com.logicalclocks.hsfs.StorageConnector; import com.logicalclocks.hsfs.engine.SparkEngine; import lombok.Getter; import lombok.Setter; @@ -60,19 +66,11 @@ public Query join(Query subquery, List on) { return joinFeatures(subquery, on.stream().map(Feature::new).collect(Collectors.toList()), JoinType.INNER); } - public Query joinFeatures(Query subquery, List on) { - return joinFeatures(subquery, on, JoinType.INNER); - } - public Query join(Query subquery, List leftOn, List rightOn) { return joinFeatures(subquery, leftOn.stream().map(Feature::new).collect(Collectors.toList()), rightOn.stream().map(Feature::new).collect(Collectors.toList()), JoinType.INNER); } - public Query joinFeatures(Query subquery, List leftOn, List rightOn) { - return joinFeatures(subquery, leftOn, rightOn, JoinType.INNER); - } - public Query join(Query subquery, JoinType joinType) { joins.add(new Join(subquery, joinType)); return this; @@ -83,17 +81,25 @@ public Query join(Query subquery, List on, JoinType joinType) { return this; } - public Query joinFeatures(Query subquery, List on, JoinType joinType) { - joins.add(new Join(subquery, on, joinType)); - return this; - } - public Query join(Query subquery, List leftOn, List rightOn, JoinType joinType) { joins.add(new Join(subquery, leftOn.stream().map(Feature::new).collect(Collectors.toList()), rightOn.stream().map(Feature::new).collect(Collectors.toList()), joinType)); return this; } + public Query joinFeatures(Query subquery, List on) { + return joinFeatures(subquery, on, JoinType.INNER); + } + + public Query joinFeatures(Query subquery, List leftOn, List rightOn) { + return joinFeatures(subquery, leftOn, rightOn, JoinType.INNER); + } + + public Query joinFeatures(Query subquery, List on, JoinType joinType) { + joins.add(new Join(subquery, on, joinType)); + return this; + } + public Query joinFeatures(Query subquery, List leftOn, List rightOn, JoinType joinType) { joins.add(new Join(subquery, leftOn, rightOn, joinType)); return this; @@ -103,10 +109,6 @@ public Dataset read() throws FeatureStoreException, IOException { return read(Storage.OFFLINE); } - public void show(int numRows) throws FeatureStoreException, IOException { - show(Storage.OFFLINE, numRows); - } - public Dataset read(Storage storage) throws FeatureStoreException, IOException { if (storage == null) { throw new FeatureStoreException("Storage not supported"); @@ -128,6 +130,10 @@ public Dataset read(Storage storage) throws FeatureStoreException, IOExcept } } + public void show(int numRows) throws FeatureStoreException, IOException { + show(Storage.OFFLINE, numRows); + } + public void show(Storage storage, int numRows) throws FeatureStoreException, IOException { read(storage).show(numRows); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDTO.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java similarity index 94% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/RestDTO.java rename to java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java index 795f902b9a..5991b5c83f 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDTO.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java @@ -22,11 +22,11 @@ import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -public abstract class RestDTO { +public abstract class RestDto { protected List items = new ArrayList<>(); - public RestDTO() { + public RestDto() { } public List getItems() { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java index 461aca06b6..43fe90501b 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) -public class Tags extends RestDTO { +public class Tags extends RestDto { private String name; private String value; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java index 4de332329d..8f603dd8b8 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; @@ -50,19 +51,19 @@ public TagsApi(@NonNull EntityEndpointType entityType) { } private void add(Integer projectId, Integer featurestoreId, Integer entityId, String name, String value) - throws FeatureStoreException, IOException { + throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = getInstance(); String pathTemplate = PROJECT_PATH - + FeatureStoreApi.FEATURE_STORE_PATH - + TAGS_PATH; + + FeatureStoreApi.FEATURE_STORE_PATH + + TAGS_PATH; UriTemplate uriTemplate = UriTemplate.fromTemplate(pathTemplate) - .set("projectId", projectId) - .set("fsId", featurestoreId) - .set("entityType", entityType.getValue()) - .set("entityId", entityId) - .set("name", name); + .set("projectId", projectId) + .set("fsId", featurestoreId) + .set("entityType", entityType.getValue()) + .set("entityId", entityId) + .set("name", name); if (value != null) { uriTemplate.set("value", value); @@ -75,27 +76,27 @@ private void add(Integer projectId, Integer featurestoreId, Integer entityId, St public void add(FeatureGroup featureGroup, String name, String value) throws FeatureStoreException, IOException { add(featureGroup.getFeatureStore().getProjectId(), featureGroup.getFeatureStore().getId(), featureGroup.getId(), - name, value); + name, value); } public void add(TrainingDataset trainingDataset, String name, String value) - throws FeatureStoreException, IOException { + throws FeatureStoreException, IOException { add(trainingDataset.getFeatureStore().getProjectId(), trainingDataset.getFeatureStore().getId(), - trainingDataset.getId(), name, value); + trainingDataset.getId(), name, value); } private Map get(Integer projectId, Integer featurestoreId, Integer entityId, String name) - throws FeatureStoreException, IOException { + throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = getInstance(); String pathTemplate = PROJECT_PATH - + FeatureStoreApi.FEATURE_STORE_PATH - + TAGS_PATH; + + FeatureStoreApi.FEATURE_STORE_PATH + + TAGS_PATH; UriTemplate uriTemplate = UriTemplate.fromTemplate(pathTemplate) - .set("projectId", projectId) - .set("fsId", featurestoreId) - .set("entityType", entityType.getValue()) - .set("entityId", entityId); + .set("projectId", projectId) + .set("fsId", featurestoreId) + .set("entityType", entityType.getValue()) + .set("entityId", entityId); if (!Strings.isNullOrEmpty(name)) { uriTemplate.set("name", name); @@ -108,33 +109,34 @@ private Map get(Integer projectId, Integer featurestoreId, Integ Tags tags = hopsworksClient.handleRequest(getRequest, Tags.class); return tags.getItems().stream() - .collect(Collectors.toMap(Tags::getName, Tags::getValue)); + .collect(Collectors.toMap(Tags::getName, Tags::getValue)); } public Map get(FeatureGroup featureGroup, String name) throws FeatureStoreException, IOException { return get(featureGroup.getFeatureStore().getProjectId(), featureGroup.getFeatureStore().getId(), - featureGroup.getId(), name); + featureGroup.getId(), name); } public Map get(TrainingDataset trainingDataset, String name) - throws FeatureStoreException, IOException { + throws FeatureStoreException, IOException { return get(trainingDataset.getFeatureStore().getProjectId(), trainingDataset.getFeatureStore().getId(), - trainingDataset.getId(), name); + trainingDataset.getId(), name); } - private void deleteTag(Integer projectId, Integer featurestoreId, Integer entityId, String name) throws FeatureStoreException, IOException { + private void deleteTag(Integer projectId, Integer featurestoreId, Integer entityId, String name) + throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = getInstance(); String pathTemplate = PROJECT_PATH - + FeatureStoreApi.FEATURE_STORE_PATH - + TAGS_PATH; + + FeatureStoreApi.FEATURE_STORE_PATH + + TAGS_PATH; String uri = UriTemplate.fromTemplate(pathTemplate) - .set("projectId", projectId) - .set("fsId", featurestoreId) - .set("entityType", entityType.getValue()) - .set("entityId", entityId) - .set("name", name) - .expand(); + .set("projectId", projectId) + .set("fsId", featurestoreId) + .set("entityType", entityType.getValue()) + .set("entityId", entityId) + .set("name", name) + .expand(); LOGGER.info("Sending metadata request: " + uri); HttpDelete httpDelete = new HttpDelete(uri); @@ -143,11 +145,11 @@ private void deleteTag(Integer projectId, Integer featurestoreId, Integer entity public void deleteTag(FeatureGroup featureGroup, String name) throws FeatureStoreException, IOException { deleteTag(featureGroup.getFeatureStore().getProjectId(), featureGroup.getFeatureStore().getId(), - featureGroup.getId(), name); + featureGroup.getId(), name); } public void deleteTag(TrainingDataset trainingDataset, String name) throws FeatureStoreException, IOException { deleteTag(trainingDataset.getFeatureStore().getProjectId(), trainingDataset.getFeatureStore().getId(), - trainingDataset.getId(), name); + trainingDataset.getId(), name); } } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java index 3d2bfe3254..23ffc8dad5 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java @@ -13,28 +13,21 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; -import com.logicalclocks.hsfs.FeatureGroup; import com.logicalclocks.hsfs.FeatureStore; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.TrainingDataset; import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH; -import static com.logicalclocks.hsfs.metadata.HopsworksClient.getInstance; public class TrainingDatasetApi { diff --git a/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java b/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java index dd7602e792..5b6a610913 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java +++ b/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.util; public class Constants { diff --git a/java/src/main/resources/checkstyle.xml b/java/src/main/resources/checkstyle.xml new file mode 100644 index 0000000000..dc6b3ee863 --- /dev/null +++ b/java/src/main/resources/checkstyle.xml @@ -0,0 +1,310 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From e755d289d4bb26bd7e08cf1347b0967aaaa23351 Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Wed, 29 Jul 2020 00:36:34 +0200 Subject: [PATCH 2/3] more checkstyle fixes --- .../com/logicalclocks/hsfs/DataFormat.java | 1 + .../hsfs/EntityEndpointType.java | 1 + .../java/com/logicalclocks/hsfs/Feature.java | 1 + .../com/logicalclocks/hsfs/FeatureStore.java | 26 +++++++------- .../hsfs/FeatureStoreException.java | 1 + .../hsfs/HopsworksConnection.java | 10 +++--- .../com/logicalclocks/hsfs/MainClass.java | 3 +- .../java/com/logicalclocks/hsfs/Project.java | 1 + .../com/logicalclocks/hsfs/SecretStore.java | 1 + .../java/com/logicalclocks/hsfs/Split.java | 1 + .../hsfs/StorageConnectorType.java | 1 + .../logicalclocks/hsfs/TrainingDataset.java | 1 + .../hsfs/TrainingDatasetType.java | 1 + .../hsfs/engine/FeatureGroupEngine.java | 15 +++++--- .../hsfs/engine/SparkEngine.java | 34 ++++++++++++------- .../hsfs/engine/TrainingDatasetEngine.java | 17 ++++++---- .../com/logicalclocks/hsfs/engine/Utils.java | 13 +++---- .../hsfs/metadata/Credentials.java | 1 + .../hsfs/metadata/FeatureGroupApi.java | 14 ++++---- .../hsfs/metadata/FeatureStoreApi.java | 1 + .../hsfs/metadata/HopsworksClient.java | 8 +++-- .../metadata/HopsworksExternalClient.java | 28 ++++++++------- .../hsfs/metadata/HopsworksHttpClient.java | 1 + .../hsfs/metadata/ProjectApi.java | 1 + .../hsfs/metadata/QueryConstructorApi.java | 9 ++--- .../hsfs/metadata/StorageConnectorApi.java | 4 ++- .../hsfs/metadata/UnauthorizedException.java | 1 + java/src/main/resources/checkstyle.xml | 1 + .../hsfs/TestHopsworksExternalClient.java | 2 +- 29 files changed, 123 insertions(+), 76 deletions(-) diff --git a/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java b/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java index 793b436884..89934ab206 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java +++ b/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java b/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java index 0134f95fee..c98fa1a60a 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java +++ b/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public enum EntityEndpointType { diff --git a/java/src/main/java/com/logicalclocks/hsfs/Feature.java b/java/src/main/java/com/logicalclocks/hsfs/Feature.java index 8655fc8503..945af856b8 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/Feature.java +++ b/java/src/main/java/com/logicalclocks/hsfs/Feature.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import lombok.AllArgsConstructor; diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java index 8b1969f9b6..f8a893ca49 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java @@ -50,7 +50,7 @@ public class FeatureStore { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureStore.class); - final Integer DEFAULT_VERSION = 1; + private static final Integer DEFAULT_VERSION = 1; public FeatureStore() throws FeatureStoreException { featureGroupApi = new FeatureGroupApi(); @@ -63,8 +63,9 @@ public FeatureStore() throws FeatureStoreException { * * @param name the name of the feature group * @param version the version of the feature group - * @return + * @return FeatureGroup * @throws FeatureStoreException + * @throws IOException */ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -75,12 +76,13 @@ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer versi * Get a feature group object with default version `1` from the feature store. * * @param name the name of the feature group - * @return + * @return FeatureGroup * @throws FeatureStoreException + * @throws IOException */ public FeatureGroup getFeatureGroup(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" + - DEFAULT_VERSION + "`."); + LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" + + DEFAULT_VERSION + "`."); return getFeatureGroup(name, DEFAULT_VERSION); } @@ -89,7 +91,7 @@ public Dataset sql(String query) { } public StorageConnector getStorageConnector(String name, StorageConnectorType type) - throws FeatureStoreException, IOException { + throws FeatureStoreException, IOException { return storageConnectorApi.getByNameAndType(this, name, type); } @@ -108,7 +110,7 @@ public TrainingDataset.TrainingDatasetBuilder createTrainingDataset() { * * @param name name of the training dataset * @param version version to get - * @return + * @return TrainingDataset * @throws FeatureStoreException * @throws IOException */ @@ -121,20 +123,20 @@ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer * Get a training dataset object with the default version `1` from the selected feature store. * * @param name name of the training dataset - * @return + * @return TrainingDataset * @throws FeatureStoreException * @throws IOException */ public TrainingDataset getTrainingDataset(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting training dataset `" + name + "`, defaulting to `" + - DEFAULT_VERSION + "`."); + LOGGER.info("VersionWarning: No version provided for getting training dataset `" + name + "`, defaulting to `" + + DEFAULT_VERSION + "`."); return getTrainingDataset(name, DEFAULT_VERSION); } @Override public String toString() { - return "FeatureStore{" + - "id=" + id + return "FeatureStore{" + + "id=" + id + ", name='" + name + '\'' + ", projectId=" + projectId + ", featureGroupApi=" + featureGroupApi diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java index 70ac339bbd..fb8cbe567a 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public class FeatureStoreException extends Exception { diff --git a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java index ce2859d6c5..9b7a176a77 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java +++ b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java @@ -59,7 +59,7 @@ public class HopsworksConnection implements Closeable { private String certPath; @Getter - private String ApiKeyFilePath; + private String apiKeyFilePath; private FeatureStoreApi featureStoreApi = new FeatureStoreApi(); @@ -70,7 +70,7 @@ public class HopsworksConnection implements Closeable { @Builder public HopsworksConnection(String host, int port, String project, Region region, SecretStore secretStore, boolean hostnameVerification, String trustStorePath, - String certPath, String ApiKeyFilePath) throws IOException, FeatureStoreException { + String certPath, String apiKeyFilePath) throws IOException, FeatureStoreException { this.host = host; this.port = port; this.project = project; @@ -79,10 +79,10 @@ public HopsworksConnection(String host, int port, String project, Region region, this.hostnameVerification = hostnameVerification; this.trustStorePath = trustStorePath; this.certPath = certPath; - this.ApiKeyFilePath = ApiKeyFilePath; + this.apiKeyFilePath = apiKeyFilePath; HopsworksClient hopsworksClient = HopsworksClient.setupHopsworksClient(host, port, region, secretStore, - hostnameVerification, trustStorePath, this.ApiKeyFilePath); + hostnameVerification, trustStorePath, this.apiKeyFilePath); projectObj = getProject(); hopsworksClient.downloadCredentials(projectObj, certPath); } @@ -90,7 +90,7 @@ public HopsworksConnection(String host, int port, String project, Region region, /** * Retrieve the project feature store. * - * @return + * @return FeatureStore * @throws IOException * @throws FeatureStoreException */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/MainClass.java b/java/src/main/java/com/logicalclocks/hsfs/MainClass.java index cebda56bcf..ca3f17b77d 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/MainClass.java +++ b/java/src/main/java/com/logicalclocks/hsfs/MainClass.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.logicalclocks.hsfs.engine.SparkEngine; @@ -26,7 +27,7 @@ public class MainClass { - private final static Logger LOGGER = LoggerFactory.getLogger(MainClass.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MainClass.class); public static void main(String[] args) throws Exception { diff --git a/java/src/main/java/com/logicalclocks/hsfs/Project.java b/java/src/main/java/com/logicalclocks/hsfs/Project.java index c40151f43b..860f374184 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/Project.java +++ b/java/src/main/java/com/logicalclocks/hsfs/Project.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import lombok.Getter; diff --git a/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java b/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java index 7862f8650b..b6750c9d02 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java +++ b/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public enum SecretStore { diff --git a/java/src/main/java/com/logicalclocks/hsfs/Split.java b/java/src/main/java/com/logicalclocks/hsfs/Split.java index 2df71738f2..f5c4cd5247 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/Split.java +++ b/java/src/main/java/com/logicalclocks/hsfs/Split.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import lombok.AllArgsConstructor; diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java b/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java index 79707ad38b..4414442792 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java +++ b/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public enum StorageConnectorType { diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java index 057c9d767f..d985d64713 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java +++ b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java b/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java index 50c9834bcb..ac017e8fe8 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java +++ b/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs; public enum TrainingDatasetType { diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java index 6fee7b3c90..8d7f5c2667 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java @@ -13,10 +13,12 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.engine; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.FeatureGroup; +import com.logicalclocks.hsfs.FeatureStore; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.Storage; import com.logicalclocks.hsfs.StorageConnector; @@ -48,7 +50,8 @@ public class FeatureGroupEngine { // Compute statistics /** - * Create the metadata and write the data to the online/offline feature store + * Create the metadata and write the data to the online/offline feature store. + * * @param featureGroup * @param dataset * @param primaryKeys @@ -93,8 +96,8 @@ public void saveFeatureGroup(FeatureGroup featureGroup, Dataset dataset, FeatureGroup apiFG = featureGroupApi.save(featureGroup); if (featureGroup.getVersion() == null) { - LOGGER.info("VersionWarning: No version provided for creating feature group `" + featureGroup.getName() + - "`, incremented version to `" + apiFG.getVersion() + "`."); + LOGGER.info("VersionWarning: No version provided for creating feature group `" + featureGroup.getName() + + "`, incremented version to `" + apiFG.getVersion() + "`."); } // Update the original object - Hopsworks returns the incremented version @@ -121,11 +124,15 @@ public void saveDataframe(FeatureGroup featureGroup, Dataset dataset, Stora case ALL: saveOfflineDataframe(featureGroup, dataset, saveMode, writeOptions); saveOnlineDataframe(featureGroup, dataset, saveMode, writeOptions); + break; + default: + throw new FeatureStoreException("Storage: " + storage + " not recognized"); } } /** - * Write dataframe to the offline feature store + * Write dataframe to the offline feature store. + * * @param featureGroup * @param dataset * @param saveMode diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java index 945a668ba8..cafca80e32 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.engine; import com.logicalclocks.hsfs.DataFormat; @@ -20,6 +21,7 @@ import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.Split; import com.logicalclocks.hsfs.StorageConnector; +import com.logicalclocks.hsfs.StorageConnectorType; import com.logicalclocks.hsfs.TrainingDataset; import com.logicalclocks.hsfs.util.Constants; import lombok.Getter; @@ -74,10 +76,8 @@ public Dataset jdbc(StorageConnector storageConnector, String query) throws } public void configureConnector(StorageConnector storageConnector) { - switch (storageConnector.getStorageConnectorType()) { - case S3: - configureS3Connector(storageConnector); - break; + if (storageConnector.getStorageConnectorType() == StorageConnectorType.S3) { + configureS3Connector(storageConnector); } } @@ -97,7 +97,8 @@ private void configureS3Connector(StorageConnector storageConnector) { } /** - * Setup Spark to write the data on the File System + * Setup Spark to write the data on the File System. + * * @param trainingDataset * @param dataset * @param writeOptions @@ -153,6 +154,9 @@ public Map getWriteOptions(Map providedOptions, break; case TFRECORDS: writeOptions.put(Constants.TF_CONNECTOR_RECORD_TYPE, "Example"); + break; + default: + break; } if (providedOptions != null && !providedOptions.isEmpty()) { @@ -177,6 +181,9 @@ public Map getReadOptions(Map providedOptions, D break; case TFRECORDS: readOptions.put(Constants.TF_CONNECTOR_RECORD_TYPE, "Example"); + break; + default: + break; } if (providedOptions != null && !providedOptions.isEmpty()) { @@ -197,19 +204,20 @@ public Map getReadOptions(Map providedOptions, D */ private void writeSplits(Dataset[] datasets, DataFormat dataFormat, Map writeOptions, SaveMode saveMode, String basePath, List splits) { - for (int i=0; i < datasets.length; i++) { + for (int i = 0; i < datasets.length; i++) { writeSingle(datasets[i], dataFormat, writeOptions, saveMode, new Path(basePath, splits.get(i).getName()).toString()); } } /** - * Write a single dataset split + * Write a single dataset split. + * * @param dataset * @param dataFormat * @param writeOptions * @param saveMode - * @param path: it should be the full path + * @param path it should be the full path */ private void writeSingle(Dataset dataset, DataFormat dataFormat, Map writeOptions, SaveMode saveMode, String path) { @@ -230,13 +238,14 @@ public Dataset read(DataFormat dataFormat, Map readOptions, } /** - * Build the option maps to write the dataset to the JDBC sink. URL, username and password are taken from the - * storage connector. + * Build the option maps to write the dataset to the JDBC sink. + * URL, username and password are taken from the storage connector. * They can however be overwritten by the user if they pass a option map. For instance if they want to change the + * * @param providedWriteOptions * @param featureGroup * @param storageConnector - * @return + * @return Map * @throws FeatureStoreException */ public Map getOnlineOptions(Map providedWriteOptions, @@ -254,7 +263,8 @@ public Map getOnlineOptions(Map providedWriteOpt } /** - * Write dataset on the JDBC sink + * Write dataset on the JDBC sink. + * * @param dataset * @param saveMode * @param writeOptions diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java index 2f980f07f0..9588178766 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.engine; import com.logicalclocks.hsfs.EntityEndpointType; @@ -42,7 +43,8 @@ public class TrainingDatasetEngine { // Compute statistics /** - * Make a REST call to Hopsworks to create the metadata and write the data on the File System + * Make a REST call to Hopsworks to create the metadata and write the data on the File System. + * * @param trainingDataset * @param dataset * @param userWriteOptions @@ -59,8 +61,8 @@ public void save(TrainingDataset trainingDataset, Dataset dataset, TrainingDataset apiTD = trainingDatasetApi.createTrainingDataset(trainingDataset); if (trainingDataset.getVersion() == null) { - LOGGER.info("VersionWarning: No version provided for creating training dataset `" + trainingDataset.getName() + - "`, incremented version to `" + apiTD.getVersion() + "`."); + LOGGER.info("VersionWarning: No version provided for creating training dataset `" + trainingDataset.getName() + + "`, incremented version to `" + apiTD.getVersion() + "`."); } // Update the original object - Hopsworks returns the full location and incremented version @@ -75,7 +77,8 @@ public void save(TrainingDataset trainingDataset, Dataset dataset, } /** - * Insert (append or overwrite) data on a training dataset + * Insert (append or overwrite) data on a training dataset. + * * @param trainingDataset * @param dataset * @param providedOptions @@ -112,11 +115,13 @@ public Dataset read(TrainingDataset trainingDataset, String split, Map getTag(TrainingDataset trainingDataset, String name) throws FeatureStoreException, IOException { + public Map getTag(TrainingDataset trainingDataset, String name) + throws FeatureStoreException, IOException { return tagsApi.get(trainingDataset, name); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/Utils.java b/java/src/main/java/com/logicalclocks/hsfs/engine/Utils.java index c5d792e4b5..a09ac7163f 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/Utils.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/Utils.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.engine; import com.logicalclocks.hsfs.Feature; @@ -56,24 +57,24 @@ public void schemaMatches(Dataset dataset, List features) throws F ).toArray(StructField[]::new)); if (!dataset.schema().equals(tdStructType)) { - throw new FeatureStoreException("The Dataframe schema: " + dataset.schema() + - " does not match the training dataset schema: " + tdStructType); + throw new FeatureStoreException("The Dataframe schema: " + dataset.schema() + + " does not match the training dataset schema: " + tdStructType); } } // TODO(Fabio): this should be moved in the backend public String getTableName(FeatureGroup offlineFeatureGroup) { - return offlineFeatureGroup.getFeatureStore().getName() + "." + - offlineFeatureGroup.getName() + "_" + offlineFeatureGroup.getVersion(); + return offlineFeatureGroup.getFeatureStore().getName() + "." + + offlineFeatureGroup.getName() + "_" + offlineFeatureGroup.getVersion(); } public Seq getPartitionColumns(FeatureGroup offlineFeatureGroup) { - List jPartitionCols = offlineFeatureGroup.getFeatures().stream() + List partitionCols = offlineFeatureGroup.getFeatures().stream() .filter(Feature::getPartition) .map(Feature::getName) .collect(Collectors.toList()); - return JavaConverters.asScalaIteratorConverter(jPartitionCols.iterator()).asScala().toSeq(); + return JavaConverters.asScalaIteratorConverter(partitionCols.iterator()).asScala().toSeq(); } public String getFgName(FeatureGroup featureGroup) { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java index 0a7bdf8a1e..dd5166f0e4 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java @@ -21,6 +21,7 @@ @NoArgsConstructor @AllArgsConstructor +@SuppressWarnings("checkstyle:MemberName") public class Credentials { private String kStore; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java index c1100bbe4e..6bab0571ec 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; @@ -23,16 +24,13 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; -import java.util.stream.Collectors; -import static com.logicalclocks.hsfs.metadata.HopsworksClient.*; +import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH; public class FeatureGroupApi { @@ -46,7 +44,7 @@ public class FeatureGroupApi { public FeatureGroup get(FeatureStore featureStore, String fgName, Integer fgVersion) throws IOException, FeatureStoreException { - HopsworksClient hopsworksClient = getInstance(); + HopsworksClient hopsworksClient = HopsworksClient.getInstance(); String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FEATURE_GROUP_PATH; @@ -69,7 +67,7 @@ public FeatureGroup get(FeatureStore featureStore, String fgName, Integer fgVers } public FeatureGroup save(FeatureGroup featureGroup) throws FeatureStoreException, IOException { - HopsworksClient hopsworksClient = getInstance(); + HopsworksClient hopsworksClient = HopsworksClient.getInstance(); String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FEATURE_GROUP_ROOT_PATH; @@ -91,7 +89,7 @@ public FeatureGroup save(FeatureGroup featureGroup) throws FeatureStoreException } public void delete(FeatureGroup featureGroup) throws FeatureStoreException, IOException { - HopsworksClient hopsworksClient = getInstance(); + HopsworksClient hopsworksClient = HopsworksClient.getInstance(); String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FEATURE_GROUP_ID_PATH; @@ -109,7 +107,7 @@ public void delete(FeatureGroup featureGroup) throws FeatureStoreException, IOEx } public void deleteContent(FeatureGroup featureGroup) throws FeatureStoreException, IOException { - HopsworksClient hopsworksClient = getInstance(); + HopsworksClient hopsworksClient = HopsworksClient.getInstance(); String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FEATURE_GROUP_CLEAR_PATH; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java index fa24dce2ca..089513e14c 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java index ed5327c923..6d4ff3847c 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.fasterxml.jackson.annotation.JsonInclude; @@ -62,7 +63,7 @@ public static void setInstance(HopsworksClient instance) { public static synchronized HopsworksClient setupHopsworksClient(String host, int port, Region region, SecretStore secretStore, boolean hostnameVerification, - String trustStorePath, String ApiKeyFilePath) + String trustStorePath, String apiKeyFilePath) throws FeatureStoreException { if (hopsworksClientInstance != null) { return hopsworksClientInstance; @@ -74,7 +75,7 @@ public static synchronized HopsworksClient setupHopsworksClient(String host, int hopsworksHttpClient = new HopsworksInternalClient(); } else { hopsworksHttpClient = new HopsworksExternalClient(host, port, region, - secretStore, hostnameVerification, trustStorePath, ApiKeyFilePath); + secretStore, hostnameVerification, trustStorePath, apiKeyFilePath); } } catch (Exception e) { throw new FeatureStoreException("Could not setup Hopsworks client", e); @@ -138,7 +139,8 @@ public T handleResponse(HttpResponse response) throws ClientProtocolException, I } } - public T handleRequest(HttpRequest request, ResponseHandler responseHandler) throws IOException, FeatureStoreException { + public T handleRequest(HttpRequest request, ResponseHandler responseHandler) + throws IOException, FeatureStoreException { return hopsworksHttpClient.handleRequest(request, responseHandler); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java index 04faccfcc3..51dca24daf 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.fasterxml.jackson.databind.ObjectMapper; @@ -73,7 +74,7 @@ public class HopsworksExternalClient implements HopsworksHttpClient { public HopsworksExternalClient(String host, int port, Region region, SecretStore secretStore, boolean hostnameVerification, - String trustStorePath, String ApiKeyFilepath) + String trustStorePath, String apiKeyFilepath) throws IOException, FeatureStoreException, KeyStoreException, CertificateException, NoSuchAlgorithmException, KeyManagementException { @@ -89,7 +90,7 @@ public HopsworksExternalClient(String host, int port, Region region, .setKeepAliveStrategy((httpResponse, httpContext) -> 30 * 1000) .build(); - apiKey = readAPIKey(secretStore, region, ApiKeyFilepath); + apiKey = readApiKey(secretStore, region, apiKeyFilepath); } public HopsworksExternalClient(CloseableHttpClient httpClient, HttpHost httpHost) { @@ -125,34 +126,35 @@ private Registry createConnectionFactory(HttpHost httpH } /** - * Read API key - We currently support 3 options: + * Read API key. + * We currently support 3 options: * - AWS Parameter store * - AWS Secrets manager * - FIle on the local file system * @param secretStore * @param region - * @param ApiKeyFilepath - * @return + * @param apiKeyFilepath + * @return String * @throws IOException * @throws FeatureStoreException */ - public String readAPIKey(SecretStore secretStore, Region region, String ApiKeyFilepath) + public String readApiKey(SecretStore secretStore, Region region, String apiKeyFilepath) throws IOException, FeatureStoreException { - if (!Strings.isNullOrEmpty(ApiKeyFilepath)) { - return FileUtils.readFileToString(Paths.get(ApiKeyFilepath).toFile()); + if (!Strings.isNullOrEmpty(apiKeyFilepath)) { + return FileUtils.readFileToString(Paths.get(apiKeyFilepath).toFile()); } switch (secretStore) { case PARAMETER_STORE: - return readAPIKeyParamStore(region, "api-key"); + return readApiKeyParamStore(region, "api-key"); case SECRET_MANAGER: - return readAPIKeySecretManager(region, "api-key"); + return readApiKeySecretManager(region, "api-key"); default: throw new FeatureStoreException("ApiKeyFilepath needs to be set for local mode"); } } - private String readAPIKeyParamStore(Region region, String secretKey) throws FeatureStoreException { + private String readApiKeyParamStore(Region region, String secretKey) throws FeatureStoreException { SsmClient ssmClient = SsmClient.builder() .region(region) .build(); @@ -160,7 +162,7 @@ private String readAPIKeyParamStore(Region region, String secretKey) throws Feat GetParameterRequest paramRequest = GetParameterRequest.builder() .name(paramName) .withDecryption(true) - .build(); + .build(); GetParameterResponse parameterResponse = ssmClient.getParameter(paramRequest); String apiKey = parameterResponse.parameter().value(); if (!Strings.isNullOrEmpty(apiKey)) { @@ -170,7 +172,7 @@ private String readAPIKeyParamStore(Region region, String secretKey) throws Feat } } - private String readAPIKeySecretManager(Region region, String secretKey) throws FeatureStoreException, IOException { + private String readApiKeySecretManager(Region region, String secretKey) throws FeatureStoreException, IOException { SecretsManagerClient secretsManagerClient = SecretsManagerClient.builder() .region(region) .build(); diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java index 4c81b37545..4d3f7760bb 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.logicalclocks.hsfs.FeatureStoreException; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java index bcffb284fd..f58cce1011 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java index 9b6da4d66a..cdd3af092e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; @@ -35,11 +36,11 @@ public class QueryConstructorApi { public FsQuery constructQuery(FeatureStore featureStore, Query query) throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = HopsworksClient.getInstance(); - String pathTemplate = HopsworksClient.PROJECT_PATH + - FeatureStoreApi.FEATURE_STORE_SERVICE_PATH + - QUERY_CONSTRUCTOR_PATH; + String pathTemplate = HopsworksClient.PROJECT_PATH + + FeatureStoreApi.FEATURE_STORE_SERVICE_PATH + + QUERY_CONSTRUCTOR_PATH; - String uri = UriTemplate .fromTemplate(pathTemplate) + String uri = UriTemplate.fromTemplate(pathTemplate) .set("projectId", featureStore.getProjectId()) .expand(); diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java index 5db851c642..ffd1dca45a 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; @@ -53,7 +54,8 @@ public StorageConnector getByNameAndType(FeatureStore featureStore, String name, return Arrays.stream(storageConnectors).filter(s -> s.getName().equals(name)) .findFirst() - .orElseThrow(() -> new FeatureStoreException("Could not find storage connector " + name + " with type " + type)); + .orElseThrow(() -> + new FeatureStoreException("Could not find storage connector " + name + " with type " + type)); } public StorageConnector getOnlineStorageConnector(FeatureStore featureStore) diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java index a129b60772..8376c54a5a 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java @@ -13,6 +13,7 @@ * * See the License for the specific language governing permissions and limitations under the License. */ + package com.logicalclocks.hsfs.metadata; import org.apache.http.client.ClientProtocolException; diff --git a/java/src/main/resources/checkstyle.xml b/java/src/main/resources/checkstyle.xml index dc6b3ee863..c11fc3d9ae 100644 --- a/java/src/main/resources/checkstyle.xml +++ b/java/src/main/resources/checkstyle.xml @@ -47,6 +47,7 @@ + diff --git a/java/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java b/java/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java index 5424442675..203c1ce253 100644 --- a/java/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java +++ b/java/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java @@ -61,7 +61,7 @@ public void testReadAPIKeyFromFile() throws IOException, FeatureStoreException { HttpHost httpHost = new HttpHost("test"); HopsworksExternalClient hopsworksExternalClient = new HopsworksExternalClient( httpClient, httpHost); - String apiKey = hopsworksExternalClient.readAPIKey(null, null, apiFilePath.toString()); + String apiKey = hopsworksExternalClient.readApiKey(null, null, apiFilePath.toString()); Assert.assertEquals("hello", apiKey); } From 887bf0a03085d945258947945d72cf72b9841e8b Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Wed, 29 Jul 2020 00:40:52 +0200 Subject: [PATCH 3/3] suppress error for Credentials.jks --- java/pom.xml | 1 + .../java/com/logicalclocks/hsfs/metadata/Credentials.java | 1 - java/src/main/resources/suppressions.xml | 8 ++++++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 java/src/main/resources/suppressions.xml diff --git a/java/pom.xml b/java/pom.xml index b42224b697..e4e5d4399a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -207,6 +207,7 @@ src/main/resources/checkstyle.xml + src/main/resources/suppressions.xml true true true diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java index dd5166f0e4..0a7bdf8a1e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java @@ -21,7 +21,6 @@ @NoArgsConstructor @AllArgsConstructor -@SuppressWarnings("checkstyle:MemberName") public class Credentials { private String kStore; diff --git a/java/src/main/resources/suppressions.xml b/java/src/main/resources/suppressions.xml new file mode 100644 index 0000000000..97e1489bf4 --- /dev/null +++ b/java/src/main/resources/suppressions.xml @@ -0,0 +1,8 @@ + + + + + + + \ No newline at end of file