From 90d477c981aed0b2d314207ff94b04eaa917ec20 Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Tue, 15 Dec 2020 15:38:31 +0100 Subject: [PATCH] [HOPSWORKS-2177] Refactor storage connectors so that they belong to the same entity (#382) --- .../spec/factories/feature_store_connector.rb | 22 ++ .../factories/feature_store_jdbc_connector.rb | 2 +- .../ruby/spec/helpers/featurestore_helper.rb | 27 +- .../spec/helpers/storage_connector_helper.rb | 180 +++------ .../spec/redshift_storage_connector_spec.rb | 92 ++--- .../test/ruby/spec/storage_connector_spec.rb | 254 ++++-------- .../test/ruby/spec/trainingdataset_spec.rb | 19 +- .../api/featurestore/FeaturestoreService.java | 14 +- .../FeaturestoreStorageConnectorService.java | 179 ++------- .../FeaturestoreRedshiftConnectorFacade.java | 71 ---- .../dao/user/activity/ActivityFacade.java | 2 + .../featurestore/FeaturestoreConstants.java | 11 +- .../featurestore/FeaturestoreController.java | 61 ++- .../app/FeaturestoreMetadataDTO.java | 16 +- .../featuregroup/FeaturegroupController.java | 2 +- .../OnDemandFeaturegroupController.java | 44 +-- .../online/OnlineFeaturestoreController.java | 63 ++- .../query/ConstructorController.java | 2 +- .../FeaturestoreConnectorFacade.java | 102 +++++ ...eaturestoreStorageConnectorController.java | 371 +++++++++--------- .../FeaturestoreStorageConnectorDTO.java | 49 +-- ...FeaturestoreHopsfsConnectorController.java | 271 ++----------- .../FeaturestoreHopsfsConnectorDTO.java | 16 +- .../FeaturestoreHopsfsConnectorFacade.java | 137 ------- .../FeaturestoreJdbcConnectorController.java | 274 ++----------- .../jdbc/FeaturestoreJdbcConnectorDTO.java | 14 +- .../jdbc/FeaturestoreJdbcConnectorFacade.java | 132 ------- ...aturestoreRedshiftConnectorController.java | 266 +++---------- .../FeaturestoreRedshiftConnectorDTO.java | 31 +- .../s3/FeaturestoreS3ConnectorController.java | 219 ++--------- .../s3/FeaturestoreS3ConnectorDTO.java | 21 +- .../s3/FeaturestoreS3ConnectorFacade.java | 132 ------- .../TrainingDatasetController.java | 63 +-- .../trainingdatasets/TrainingDatasetDTO.java | 35 +- .../ExternalTrainingDatasetController.java | 12 +- .../ExternalTrainingDatasetFacade.java | 6 +- .../HopsfsTrainingDatasetController.java | 9 +- .../hopsfs/HopsfsTrainingDatasetFacade.java | 6 +- .../common/project/ProjectController.java | 2 +- .../security/secrets/SecretsController.java | 3 +- .../entity/featurestore/Featurestore.java | 54 +-- .../ondemand/OnDemandFeaturegroup.java | 47 +-- .../FeaturestoreConnector.java | 188 +++++++++ .../FeaturestoreConnectorType.java | 21 +- .../hopsfs/FeaturestoreHopsfsConnector.java | 69 +--- .../jdbc/FeaturestoreJdbcConnector.java | 55 +-- .../FeatureStoreRedshiftConnector.java | 92 +---- .../s3/FeaturestoreS3Connector.java | 73 +--- .../external/ExternalTrainingDataset.java | 26 +- .../hopsfs/HopsfsTrainingDataset.java | 51 +-- .../main/resources/META-INF/persistence.xml | 1 + .../hops/hopsworks/restutils/RESTCodes.java | 5 +- .../scripts/controllers/featurestoreCtrl.js | 5 +- .../controllers/newStorageConnectorCtrl.js | 6 +- .../controllers/newTrainingDatasetCtrl.js | 5 +- .../scripts/services/FeaturestoreService.js | 18 +- .../yo/app/views/trainingDatasetViewInfo.html | 4 +- 57 files changed, 1242 insertions(+), 2710 deletions(-) create mode 100644 hopsworks-IT/src/test/ruby/spec/factories/feature_store_connector.rb delete mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorFacade.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java delete mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorFacade.java delete mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorFacade.java delete mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorFacade.java create mode 100644 hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java rename hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorType.java => hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnectorType.java (65%) diff --git a/hopsworks-IT/src/test/ruby/spec/factories/feature_store_connector.rb b/hopsworks-IT/src/test/ruby/spec/factories/feature_store_connector.rb new file mode 100644 index 0000000000..d710fb9d0a --- /dev/null +++ b/hopsworks-IT/src/test/ruby/spec/factories/feature_store_connector.rb @@ -0,0 +1,22 @@ +=begin + This file is part of Hopsworks + Copyright (C) 2020, Logical Clocks AB. All rights reserved + + Hopsworks is free software: you can redistribute it and/or modify it under the terms of + the GNU Affero General Public License as published by the Free Software Foundation, + either version 3 of the License, or (at your option) any later version. + + Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. + If not, see . +=end +class FeatureStoreConnector < ActiveRecord::Base + self.inheritance_column = 'xname' + + def self.table_name + "feature_store_connector" + end +end \ No newline at end of file diff --git a/hopsworks-IT/src/test/ruby/spec/factories/feature_store_jdbc_connector.rb b/hopsworks-IT/src/test/ruby/spec/factories/feature_store_jdbc_connector.rb index 6972cba849..8ccd353712 100644 --- a/hopsworks-IT/src/test/ruby/spec/factories/feature_store_jdbc_connector.rb +++ b/hopsworks-IT/src/test/ruby/spec/factories/feature_store_jdbc_connector.rb @@ -13,7 +13,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . =end -class FeatureStoreJDBCConnector < ActiveRecord::Base +class FeatureStoreJDBCConnector< ActiveRecord::Base def self.table_name "feature_store_jdbc_connector" end diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb index c25397f98b..fdf75efb4d 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb @@ -256,7 +256,9 @@ def update_hopsfs_training_dataset_metadata(project_id, featurestore_id, trainin version: 1, dataFormat: dataFormat, trainingDatasetType: trainingDatasetType, - storageConnectorId: hopsfs_connector.id + storageConnector: { + id: hopsfs_connector.id + } } json_data = json_data.to_json json_result = put update_training_dataset_metadata_endpoint, json_data @@ -274,7 +276,9 @@ def update_external_training_dataset_metadata(project_id, featurestore_id, train version: 1, dataFormat: "parquet", trainingDatasetType: trainingDatasetType, - storageConnectorId: s3_connector_id + storageConnector: { + id: s3_connector_id + } } json_data = json_data.to_json json_result = put update_training_dataset_metadata_endpoint, json_data @@ -296,8 +300,6 @@ def create_hopsfs_training_dataset(project_id, featurestore_id, hopsfs_connector create_training_dataset_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/trainingdatasets" name = name == nil ? "training_dataset_#{random_id}" : name data_format = data_format == nil ? "tfrecords" : data_format - connector_id = hopsfs_connector == nil ? nil : hopsfs_connector.id - connector_name = hopsfs_connector == nil ? nil : hopsfs_connector.name description = description == nil ? "testtrainingdatasetdescription" : description if features == nil && query == nil features = [ @@ -318,13 +320,18 @@ def create_hopsfs_training_dataset(project_id, featurestore_id, hopsfs_connector version: version, dataFormat: data_format, trainingDatasetType: trainingDatasetType, - storageConnectorId: connector_id, - storageConnectorName: connector_name, features: features, splits: splits, seed: 1234, queryDTO: query } + + unless hopsfs_connector.nil? + json_data["storageConnector"] = { + id: hopsfs_connector.id + } + end + json_result = post create_training_dataset_endpoint, json_data.to_json [json_result, name] end @@ -356,11 +363,17 @@ def create_external_training_dataset(project_id, featurestore_id, s3_connector_i dataFormat: "tfrecords", location: location, trainingDatasetType: trainingDatasetType, - storageConnectorId: s3_connector_id, features: features == nil ? default_features : features, splits: splits, seed: 1234 } + + unless s3_connector_id.nil? + json_data["storageConnector"] = { + id: s3_connector_id + } + end + json_data = json_data.to_json json_result = post create_training_dataset_endpoint, json_data return json_result, training_dataset_name diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb index f5ecddb67f..9450ead6de 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/storage_connector_helper.rb @@ -17,29 +17,19 @@ module StorageConnectorHelper def get_storage_connectors(project_id, featurestore_id, type) - get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{type}" + json_result = get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/" + connectors = JSON.parse(json_result) + connectors.select{|c| c['storageConnectorType'].eql?(type)}.map { |c| c.with_indifferent_access } end - def get_jdbc_storate_connector(project_id, featurestore_id, name) - get_storage_connector(project_id, featurestore_id, "JDBC", name) - end - - def get_storage_connector(project_id, featurestore_id, type, name) - connectors_json = get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{type}" - expect_status(200) - connectors = JSON.parse(connectors_json) - - connectors.select{|connector| connector['name'] == name}[0] - end - - def get_storage_connector_by_name(project_id, featurestore_id, type, name) - get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{type}/#{name}" + def get_storage_connector(project_id, featurestore_id, name) + get "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{name}" end def create_hopsfs_connector(project_id, featurestore_id, datasetName: "Resources") type = "featurestoreHopsfsConnectorDTO" - storageConnectorType = "HopsFS" - create_hopsfs_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/HOPSFS" + storageConnectorType = "HOPSFS" + create_hopsfs_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors" hopsfs_connector_name = "hopsfs_connector_#{random_id}" json_data = { name: hopsfs_connector_name, @@ -48,32 +38,29 @@ def create_hopsfs_connector(project_id, featurestore_id, datasetName: "Resources storageConnectorType: storageConnectorType, datasetName: datasetName } - json_data = json_data.to_json - json_result = post create_hopsfs_connector_endpoint, json_data - return json_result, hopsfs_connector_name + json_result = post create_hopsfs_connector_endpoint, json_data.to_json + [json_result, hopsfs_connector_name] end def update_hopsfs_connector(project_id, featurestore_id, connector_name, datasetName: "Resources") type = "featurestoreHopsfsConnectorDTO" - storageConnectorType = "HopsFS" - update_hopsfs_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/HOPSFS/" + connector_name - hopsfs_connector_name = "hopsfs_connector_#{random_id}" + storageConnectorType = "HOPSFS" + update_hopsfs_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" json_data = { - name: hopsfs_connector_name, + name: connector_name, description: "testhopsfsconnectordescription", type: type, storageConnectorType: storageConnectorType, datasetName: datasetName } - json_data = json_data.to_json - json_result = put update_hopsfs_connector_endpoint, json_data - return json_result, hopsfs_connector_name + json_result = put update_hopsfs_connector_endpoint, json_data.to_json + [json_result, connector_name] end def create_jdbc_connector(project_id, featurestore_id, connectionString: "jdbc://test") type = "featurestoreJdbcConnectorDTO" storageConnectorType = "JDBC" - create_jdbc_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/JDBC" + create_jdbc_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors" jdbc_connector_name = "jdbc_connector_#{random_id}" json_data = { name: jdbc_connector_name, @@ -91,70 +78,23 @@ def create_jdbc_connector(project_id, featurestore_id, connectionString: "jdbc:/ def update_jdbc_connector(project_id, featurestore_id, connector_name, connectionString: "jdbc://test") type = "featurestoreJdbcConnectorDTO" storageConnectorType = "JDBC" - update_jdbc_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/JDBC/" + connector_name - jdbc_connector_name = "jdbc_connector_#{random_id}" + update_jdbc_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" json_data = { - name: jdbc_connector_name, + name: connector_name, description: "testfeaturegroupdescription", type: type, storageConnectorType: storageConnectorType, connectionString: connectionString, arguments: "test1,test2" } - json_data = json_data.to_json - json_result = put update_jdbc_connector_endpoint, json_data - return json_result, jdbc_connector_name - end - - def create_s3_connector_with_or_without_access_key_and_secret_key(project_id, featurestore_id, with_access_and_secret_key, - access_key, secret_key, bucket: "test") - type = "featurestoreS3ConnectorDTO" - storageConnectorType = "S3" - create_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3" - s3_connector_name = "s3_connector_#{random_id}" - json_data = { - name: s3_connector_name, - description: "tests3connectordescription", - type: type, - storageConnectorType: storageConnectorType, - bucket: bucket - } - if with_access_and_secret_key - json_data["secretKey"] = access_key - json_data["accessKey"] = secret_key - else - - end - json_data = json_data.to_json - json_result = post create_s3_connector_endpoint, json_data - return json_result, s3_connector_name - end - - def create_s3_connector_without_encryption(project_id, featurestore_id, bucket: "test") - type = "featurestoreS3ConnectorDTO" - storageConnectorType = "S3" - create_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3" - s3_connector_name = "s3_connector_#{random_id}" - json_data = { - name: s3_connector_name, - description: "tests3connectordescription", - type: type, - storageConnectorType: storageConnectorType, - bucket: bucket, - secretKey: "test", - accessKey: "test" - } - json_data = json_data.to_json - json_result = post create_s3_connector_endpoint, json_data - return json_result, s3_connector_name + put update_jdbc_connector_endpoint, json_data.to_json end - def create_s3_connector_with_encryption(project_id, featurestore_id, with_encryption_key, encryption_algorithm, - encryption_key, access_key, secret_key, - bucket: "test") + def create_s3_connector(project_id, featurestore_id, encryption_algorithm: nil, encryption_key: nil, + access_key: nil, secret_key: nil, bucket: "test") type = "featurestoreS3ConnectorDTO" storageConnectorType = "S3" - create_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3" + create_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors" s3_connector_name = "s3_connector_#{random_id}" json_data = { name: s3_connector_name, @@ -162,49 +102,48 @@ def create_s3_connector_with_encryption(project_id, featurestore_id, with_encryp type: type, storageConnectorType: storageConnectorType, bucket: bucket, - secretKey: access_key, - accessKey: secret_key } - if with_encryption_key - json_data["serverEncryptionAlgorithm"] = encryption_algorithm + json_data["serverEncryptionAlgorithm"] = encryption_algorithm + unless encryption_key.nil? json_data["serverEncryptionKey"] = encryption_key - else - json_data["serverEncryptionAlgorithm"] = encryption_algorithm end - json_data = json_data.to_json - json_result = post create_s3_connector_endpoint, json_data - return json_result, s3_connector_name - end + unless access_key.nil? + json_data["secretKey"] = access_key + json_data["accessKey"] = secret_key + end + json_result = post create_s3_connector_endpoint, json_data.to_json + [json_result, s3_connector_name] + end - def update_s3_connector(project_id, featurestore_id, connector_name, s3_connector_name, with_access_keys, bucket: - "test") + def update_s3_connector(project_id, featurestore_id, connector_name, access_key: nil, secret_key: nil, bucket: "test") type = "featurestoreS3ConnectorDTO" storageConnectorType = "S3" - update_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3/" + connector_name + update_s3_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" json_data = { - name: s3_connector_name, + name: connector_name, description: "tests3connectordescription", type: type, storageConnectorType: storageConnectorType, bucket: bucket } - if with_access_keys - json_data["secretKey"] = "test2" - json_data["accessKey"] = "test2" + unless secret_key.nil? + json_data["secretKey"] = secret_key end - json_data = json_data.to_json - json_result = put update_s3_connector_endpoint, json_data - return json_result, s3_connector_name + unless access_key.nil? + json_data["accessKey"] = access_key + end + json_result = put update_s3_connector_endpoint, json_data.to_json + [json_result, connector_name] end def create_redshift_connector(project_id, featurestore_id, redshift_connector_name: nil, clusterIdentifier: "redshift-connector", databaseUserName: "awsUser", databasePassword: nil, iamRole: nil) type = "featurestoreRedshiftConnectorDTO" storageConnectorType = "REDSHIFT" - create_redshift_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{storageConnectorType}" + create_redshift_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/" redshift_connector_name ||= "redshift_connector_#{random_id}" json_data = { name: redshift_connector_name, @@ -222,20 +161,15 @@ def create_redshift_connector(project_id, featurestore_id, redshift_connector_na iamRole: iamRole, arguments: "test1,test2" } - json_data = json_data.to_json - json_result = post create_redshift_connector_endpoint, json_data - return json_result, redshift_connector_name + json_result = post create_redshift_connector_endpoint, json_data.to_json + [json_result, redshift_connector_name] end def update_redshift_connector(project_id, featurestore_id, connector_name, redshift_connector_json) - type = "featurestoreRedshiftConnectorDTO" - storageConnectorType = "REDSHIFT" - update_redshift_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{storageConnectorType}/#{connector_name}" - redshift_connector_json["type"] = type - redshift_connector_json["storageConnectorType"] = storageConnectorType - json_data = redshift_connector_json.to_json - json_result = put update_redshift_connector_endpoint, json_data - return json_result + update_redshift_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" + redshift_connector_json["type"] = "featurestoreRedshiftConnectorDTO" + redshift_connector_json["storageConnectorType"] = "REDSHIFT" + put update_redshift_connector_endpoint, redshift_connector_json.to_json end def with_jdbc_connector(project_id) @@ -253,7 +187,7 @@ def get_jdbc_connector_id def get_hopsfs_training_datasets_connector(project_name) connector_name = project_name + "_Training_Datasets" - return FeatureStoreHopsfsConnector.find_by(name: connector_name) + return FeatureStoreConnector.find_by(name: connector_name) end def get_s3_connector_id @@ -261,24 +195,20 @@ def get_s3_connector_id end def with_s3_connector(project_id) - encryption_algorithm = "AES256" - encryption_key = "Test" - secret_key = "test" - access_key = "test" - with_encryption_key = true featurestore_id = get_featurestore_id(project_id) - json_result, connector_name = create_s3_connector_with_encryption(project_id, featurestore_id, - with_encryption_key, encryption_algorithm, - encryption_key, access_key, secret_key, bucket: - "testbucket") + json_result, _ = create_s3_connector(project_id, featurestore_id, + encryption_algorithm: "AES256", + access_key: "test", secret_key: "test", + bucket: "testbucket") + parsed_json = JSON.parse(json_result) expect_status_details(201) connector_id = parsed_json["id"] @s3_connector_id = connector_id end - def delete_connector(project_id, featurestore_id, type, name) - delete "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{type}/#{name}" + def delete_connector(project_id, featurestore_id, name) + delete "#{ENV['HOPSWORKS_API']}/project/#{project_id}/featurestores/#{featurestore_id}/storageconnectors/#{name}" end def with_redshift_connectors diff --git a/hopsworks-IT/src/test/ruby/spec/redshift_storage_connector_spec.rb b/hopsworks-IT/src/test/ruby/spec/redshift_storage_connector_spec.rb index a1cbf38154..e95b58c1c7 100644 --- a/hopsworks-IT/src/test/ruby/spec/redshift_storage_connector_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/redshift_storage_connector_spec.rb @@ -20,8 +20,8 @@ @p1, @p2 = with_redshift_connectors @featurestore1 = get_featurestores_checked(@p1[:id])[0] @featurestore2 = get_featurestores_checked(@p2[:id])[0] - get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") - @connector = json_body[0] + connectors = get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") + @connector = connectors[0] end describe "get" do context 'without authentication' do @@ -43,15 +43,15 @@ create_session(@p1[:username], "Pass123") end it "should get" do - get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") + connectors = get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") expect_status_details(200) - expect(json_body.length).to eq 2 + expect(connectors.length).to eq 2 end it "should get by name" do - get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") + connectors = get_storage_connectors(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT") expect_status_details(200) - connector_name = json_body[0][:name] - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector_name) + connector_name = connectors[0][:name] + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], connector_name) expect_status_details(200) expect(json_body[:name]).to eq connector_name end @@ -139,7 +139,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -149,7 +149,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -159,7 +159,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -169,7 +169,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -179,7 +179,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -189,7 +189,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -199,7 +199,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -209,7 +209,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -219,7 +219,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -229,7 +229,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -239,36 +239,36 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end it "should update connector database password" do - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", "redshift_connector_pwd") + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], "redshift_connector_pwd") connector = json_body copyConnector = connector.clone copyConnector[:databasePassword] = "databasepwd" update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end it "should update connector database iam role" do - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", "redshift_connector_iam") + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], "redshift_connector_iam") connector = json_body copyConnector = connector.clone copyConnector[:iamRole] = "arn:aws:iam::123456789012:role/test-role-p1-owners" update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end it "should update connector from iam role to password" do - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", "redshift_connector_iam") + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], "redshift_connector_iam") connector = json_body copyConnector = connector.clone copyConnector[:iamRole] = nil @@ -276,14 +276,14 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) get_private_secret("redshift_redshift_connector_iam_#{@featurestore1["featurestoreId"]}") expect_status_details(200) end it "should update connector from password to iam role" do - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", "redshift_connector_pwd") + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], "redshift_connector_pwd") connector = json_body copyConnector = connector.clone copyConnector[:iamRole] = "arn:aws:iam::123456789012:role/test-role-p1-owners" @@ -291,7 +291,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) get_private_secret("redshift_redshift_connector_pwd_#{@featurestore1["featurestoreId"]}") @@ -311,7 +311,7 @@ update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, copyConnector) end @@ -320,36 +320,36 @@ copyConnector[:name] = "updated_connector_name" update_redshift_connector(@p1[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(400) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) expect(json_body[:name]).to eq @connector[:name] end it "should fail to update with iam role when password is set" do - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", "redshift_connector_pwd") + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], "redshift_connector_pwd") connector = json_body copyConnector = connector.clone copyConnector[:iamRole] = "arn:aws:iam::123456789012:role/test-role-p1" update_redshift_connector(@p2[:id], @featurestore2["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(400) - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, connector) end it "should fail to update with password when iam role set" do - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", "redshift_connector_iam") + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], "redshift_connector_iam") connector = json_body copyConnector = connector.clone copyConnector[:databasePassword] = "newPassword" update_redshift_connector(@p2[:id], @featurestore2["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(400) - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, connector) end it "should fail to update with no password and iam role" do - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", "redshift_connector_iam") + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], "redshift_connector_iam") connector = json_body copyConnector = connector.clone copyConnector[:databasePassword] = nil @@ -357,7 +357,7 @@ update_redshift_connector(@p2[:id], @featurestore2["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(400) - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, connector) end @@ -383,7 +383,7 @@ expect(json_body.length > 2).to eq true end it "should get connector in a shared fs with no password" do - get_storage_connector_by_name(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", "p3-connector-pwd") + get_storage_connector(@p3[:id], @featurestore1["featurestoreId"], "p3-connector-pwd") expect_status_details(200) expect(json_body.key?("databasePassword")).to eq false end @@ -398,7 +398,7 @@ expect_status_details(403) end it "should fail to edit connectors in a shared fs" do - get_storage_connector_by_name(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", "p3-connector-pwd") + get_storage_connector(@p3[:id], @featurestore1["featurestoreId"], "p3-connector-pwd") connector = json_body copyConnector = connector.clone copyConnector[:description] = "updated description p3" @@ -409,23 +409,23 @@ copyConnector[:arguments] = "test1,test2,test3,test4,p3" update_redshift_connector(@p3[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(403) - get_storage_connector_by_name(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p3[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, connector) end it "should fail to edit connector database password in a shared fs" do - get_storage_connector_by_name(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", "p3-connector-pwd") + get_storage_connector(@p3[:id], @featurestore1["featurestoreId"], "p3-connector-pwd") connector = json_body copyConnector = connector.clone copyConnector[:databasePassword] = "password-p3" update_redshift_connector(@p3[:id], @featurestore1["featurestoreId"], copyConnector[:name], copyConnector) expect_status_details(403) - get_storage_connector_by_name(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", connector[:name]) + get_storage_connector(@p3[:id], @featurestore1["featurestoreId"], connector[:name]) expect_status_details(200) check_redshift_connector_update(json_body, connector) end it "should fail to delete connector with password in a shared fs" do - delete_connector(@p3[:id], @featurestore1["featurestoreId"], "REDSHIFT", "p3-connector-pwd") + delete_connector(@p3[:id], @featurestore1["featurestoreId"], "p3-connector-pwd") expect_status_details(403) end end @@ -436,12 +436,12 @@ reset_session end it "should fail to delete connector" do - delete_connector(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + delete_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(401) end it "should fail to update a connector in a project with no role" do with_valid_session - delete_connector(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + delete_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(403) end end @@ -450,17 +450,17 @@ create_session(@p1[:username], "Pass123") end it "should delete connector by name" do - delete_connector(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + delete_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(200) - get_storage_connector_by_name(@p1[:id], @featurestore1["featurestoreId"], "REDSHIFT", @connector[:name]) + get_storage_connector(@p1[:id], @featurestore1["featurestoreId"], @connector[:name]) expect_status_details(400) end it "should delete secrete when deleting connector" do - get_storage_connector_by_name(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", "redshift_connector_pwd") + get_storage_connector(@p2[:id], @featurestore2["featurestoreId"], "redshift_connector_pwd") connector = json_body get_private_secret("redshift_redshift_connector_pwd_#{@featurestore2["featurestoreId"]}") expect_status_details(200) - delete_connector(@p2[:id], @featurestore2["featurestoreId"], "REDSHIFT", connector[:name]) + delete_connector(@p2[:id], @featurestore2["featurestoreId"], connector[:name]) expect_status_details(200) get_private_secret("redshift_redshift_connector_pwd_#{@featurestore2["featurestoreId"]}") expect_status_details(404) diff --git a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb index 9eaa9da270..78a4c0b976 100644 --- a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb @@ -22,6 +22,11 @@ with_valid_project end + after :each do + setVar("aws_instance_role", "false") + create_session(@project[:username], "Pass123") + end + it "should be able to add hopsfs connector to the featurestore" do project = get_project featurestore_id = get_featurestore_id(project.id) @@ -53,16 +58,15 @@ end it "should be able to add s3 connector to the featurestore without encryption information" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_without_encryption(project.id, featurestore_id, bucket: - "testbucket") + json_result, connector_name = create_s3_connector(project.id, featurestore_id, + bucket: "testbucket", + access_key: "access_key", + secret_key: "secret_key") parsed_json = JSON.parse(json_result) expect_status(201) - setVar("aws_instance_role", "false") expect(parsed_json.key?("id")).to be true expect(parsed_json.key?("name")).to be true expect(parsed_json.key?("description")).to be true @@ -77,20 +81,11 @@ end it "should not be able to add s3 connector without providing the access and secret key if the IAM Role is set to false" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - access_key = nil - secret_key = nil - with_access_and_secret_key = false - json_result, connector_name = create_s3_connector_with_or_without_access_key_and_secret_key(project.id, featurestore_id, - with_access_and_secret_key, - access_key, secret_key, - bucket: "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(400) - setVar("aws_instance_role", "false") expect(parsed_json.key?("errorCode")).to be true expect(parsed_json.key?("errorMsg")).to be true expect(parsed_json.key?("usrMsg")).to be true @@ -102,16 +97,9 @@ project = get_project create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - access_key = nil - secret_key = nil - with_access_and_secret_key = false - json_result, connector_name = create_s3_connector_with_or_without_access_key_and_secret_key(project.id, featurestore_id, - with_access_and_secret_key, - access_key, secret_key, - bucket: "testbucket") + json_result, connector_name = create_s3_connector(project.id, featurestore_id, bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(201) - setVar("aws_instance_role", "false") expect(parsed_json.key?("id")).to be true expect(parsed_json.key?("name")).to be true expect(parsed_json.key?("description")).to be true @@ -123,24 +111,19 @@ expect(parsed_json["name"] == connector_name).to be true expect(parsed_json["storageConnectorType"] == "S3").to be true expect(parsed_json["bucket"] == "testbucket").to be true + setVar("aws_instance_role", "false") + create_session(project[:username], "Pass123") end it "should be able to add s3 connector to the featurestore with encryption algorithm but no key" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") - encryption_algorithm = "AES256" - encryption_key = nil - access_key = "test" - secret_key = "test" - with_encryption_key = false; featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, secret_key, - access_key, bucket: "testbucket") + json_result, connector_name = create_s3_connector(project.id, featurestore_id, + encryption_algorithm: "AES256", + secret_key: "test", access_key: "test", + bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(201) - setVar("aws_instance_role", "false") expect(parsed_json.key?("id")).to be true expect(parsed_json.key?("name")).to be true expect(parsed_json.key?("description")).to be true @@ -152,26 +135,22 @@ expect(parsed_json["name"] == connector_name).to be true expect(parsed_json["storageConnectorType"] == "S3").to be true expect(parsed_json["bucket"] == "testbucket").to be true - expect(parsed_json["serverEncryptionAlgorithm"] == encryption_algorithm).to be true + expect(parsed_json["serverEncryptionAlgorithm"] == "AES256").to be true expect(parsed_json["serverEncryptionKey"] == nil).to be true end it "should be able to add s3 connector to the featurestore with encryption algorithm and with encryption key" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") encryption_algorithm = "SSE-KMS" encryption_key = "Test" - access_key = "test" - secret_key = "test" - with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key,access_key, - secret_key, bucket: "testbucket") + json_result, connector_name = create_s3_connector(project.id, featurestore_id, + encryption_algorithm: encryption_algorithm, + encryption_key: encryption_key, + access_key: "test", secret_key: "test", + bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(201) - setVar("aws_instance_role", "false") expect(parsed_json.key?("id")).to be true expect(parsed_json.key?("name")).to be true expect(parsed_json.key?("description")).to be true @@ -188,13 +167,9 @@ end it "should be able to share access keys for s3 connector : for members belonging to same project" do - setVar("aws_instance_role", "false") project = get_project - project_id = project.id - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_without_encryption(project.id, featurestore_id, bucket: - "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, access_key: "test", secret_key: "test", bucket: "testbucket") parsed_json = JSON.parse(json_result) member = create_user @@ -203,21 +178,19 @@ reset_session create_session(member[:email],"Pass123") - json_result1 = get "#{ENV['HOPSWORKS_API']}/project/" + project_id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3/" + connector_name + json_result1 = get "#{ENV['HOPSWORKS_API']}/project/#{project.id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" parsed_json1 = JSON.parse(json_result1) expect_status_details(200) expect(parsed_json1["name"] == connector_name).to be true - setVar("aws_instance_role", "false") + expect(parsed_json1["accessKey"]).to eql("test") + expect(parsed_json1["secretKey"]).to eql("test") end - it "should be able to share s3 connector between shared projects" do - setVar("aws_instance_role", "false") + it "should be able to share s3 connector between shared projects - without sharing the secret key" do project = get_project - project_id = project.id - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_without_encryption(project.id, featurestore_id, bucket: - "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, access_key: "test", + secret_key: "test", bucket: "testbucket") parsed_json = JSON.parse(json_result) connector_name = parsed_json["name"] @@ -234,94 +207,65 @@ #login with user for project 1 and accept dataset create_session(project1[:username],"Pass123") accept_dataset(project1, "#{@project[:projectname]}::#{featurestore}", datasetType: "&type=FEATURESTORE") - json_result1 = get "#{ENV['HOPSWORKS_API']}/project/" + project1.id.to_s + "/featurestores/" + - featurestore_id.to_s + "/storageconnectors/S3/" + connector_name - parsed_json1 = JSON.parse(json_result1) + + json_result1 = get "#{ENV['HOPSWORKS_API']}/project/#{project1.id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" expect_status_details(200) + parsed_json1 = JSON.parse(json_result1) expect(parsed_json1["name"] == connector_name).to be true - setVar("aws_instance_role", "false") + expect(parsed_json1["accessKey"]).to be nil end it "should not be able to add s3 connector to the featurestore with wrong encryption algorithm" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") - encryption_algorithm = "WRONG-ALGORITHM" - encryption_key = "Test" - access_key = "test" - secret_key = "test" with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, - access_key, secret_key, - bucket: "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, encryption_algorithm: "WRONG-ALGORITHM", + access_key: "test", secret_key: "test", bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(400) - setVar("aws_instance_role", "false") - setVar("aws_instance_role", "false") expect(parsed_json.key?("errorCode")).to be true expect(parsed_json.key?("errorMsg")).to be true expect(parsed_json.key?("usrMsg")).to be true expect(parsed_json["errorCode"] == 270104).to be true end - it "should not be able to add s3 connector to the featurestore with encryption key provided but no - encryption algorithm" do - setVar("aws_instance_role", "false") + it "should not be able to add s3 connector to the featurestore with encryption key provided but no encryption algorithm" do project = get_project - create_session(project[:username], "Pass123") encryption_algorithm = "" encryption_key = "Test" - access_key = "test" - secret_key = "test" - with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, - access_key, secret_key, - bucket: "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, + encryption_algorithm: encryption_algorithm, + encryption_key: encryption_key, + access_key: "test", secret_key: "test", + bucket: "testbucket") + parsed_json = JSON.parse(json_result) expect_status(400) - setVar("aws_instance_role", "false") - setVar("aws_instance_role", "false") expect(parsed_json.key?("errorCode")).to be true expect(parsed_json.key?("errorMsg")).to be true expect(parsed_json.key?("usrMsg")).to be true expect(parsed_json["errorCode"] == 270104).to be true end - it "should be not able to add s3 connector to the featurestore with wrong server key and access key pair" do - setVar("aws_instance_role", "false") + it "should be not able to add s3 connector to the featurestore without access key" do project = get_project - create_session(project[:username], "Pass123") - encryption_algorithm = "SSE-KMS" - encryption_key = "Test" - secret_key = "test" - access_key = nil - with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, - access_key, secret_key, - bucket: "testbucket") + json_result, _ = create_s3_connector(project.id, featurestore_id, secret_key: "test", bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(400) - setVar("aws_instance_role", "false") expect(parsed_json.key?("errorCode")).to be true expect(parsed_json.key?("errorMsg")).to be true expect(parsed_json.key?("usrMsg")).to be true end it "should not be able to add s3 connector to the featurestore without specifying a bucket" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_without_encryption(project.id, featurestore_id, bucket: nil) + json_result, _ = create_s3_connector(project.id, featurestore_id, + access_key: "test", secret_key: "test", bucket: nil) parsed_json = JSON.parse(json_result) expect_status(400) - setVar("aws_instance_role", "false") expect(parsed_json.key?("errorCode")).to be true expect(parsed_json.key?("errorMsg")).to be true expect(parsed_json.key?("usrMsg")).to be true @@ -330,7 +274,6 @@ it "should be able to add jdbc connector to the featurestore" do project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) json_result, connector_name = create_jdbc_connector(project.id, featurestore_id, connectionString: "jdbc://test2") parsed_json = JSON.parse(json_result) @@ -349,9 +292,8 @@ it "should not be able to add jdbc connector without a connection string to the featurestore" do project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_jdbc_connector(project.id, featurestore_id, connectionString: nil) + json_result, _ = create_jdbc_connector(project.id, featurestore_id, connectionString: nil) parsed_json = JSON.parse(json_result) expect_status(400) expect(parsed_json.key?("errorCode")).to be true @@ -362,13 +304,12 @@ it "should be able to delete a hopsfs connector from the featurestore" do project = get_project - create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") + json_result, _ = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") parsed_json = JSON.parse(json_result) expect_status(201) connector_name = parsed_json["name"] - delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/HOPSFS/" + connector_name + delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project.id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" delete delete_connector_endpoint expect_status(200) end @@ -377,12 +318,12 @@ project = get_project create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result, connector_name = create_s3_connector_without_encryption(project.id, featurestore_id, bucket: - "testbucket") + json_result, connector_name = create_s3_connector(project.id, featurestore_id, access_key: "test", + secret_key: "test", bucket: "testbucket") parsed_json = JSON.parse(json_result) expect_status(201) connector_name = parsed_json["name"] - delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/S3/" + connector_name + delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project.id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" delete delete_connector_endpoint expect_status(200) end @@ -395,7 +336,7 @@ parsed_json = JSON.parse(json_result) expect_status(201) connector_name = parsed_json["name"] - delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/storageconnectors/JDBC/" + connector_name + delete_connector_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{project.id}/featurestores/#{featurestore_id}/storageconnectors/#{connector_name}" delete delete_connector_endpoint expect_status(200) end @@ -404,11 +345,10 @@ project = get_project create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result1, connector_name1 = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") - parsed_json1 = JSON.parse(json_result1) + json_result1, connector_name = create_hopsfs_connector(project.id, featurestore_id, datasetName: "Resources") expect_status(201) - connector_name = parsed_json1["name"] - json_result2, connector_name2 = update_hopsfs_connector(project.id, featurestore_id, connector_name, datasetName: "Experiments") + + json_result2, _ = update_hopsfs_connector(project.id, featurestore_id, connector_name, datasetName: "Experiments") parsed_json2 = JSON.parse(json_result2) expect(parsed_json2.key?("id")).to be true expect(parsed_json2.key?("name")).to be true @@ -417,66 +357,45 @@ expect(parsed_json2.key?("featurestoreId")).to be true expect(parsed_json2.key?("datasetName")).to be true expect(parsed_json2.key?("hopsfsPath")).to be true - expect(parsed_json2["name"] == connector_name2).to be true + expect(parsed_json2["name"] == connector_name).to be true expect(parsed_json2["storageConnectorType"] == "HOPSFS").to be true expect(parsed_json2["datasetName"] == "Experiments").to be true end it "should fail to update connector name" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") - featurestore_id = get_featurestore_id(project.id) s3_connector_name = "s3_connector_#{random_id}" - with_access_keys = true - encryption_algorithm = "AES256" - encryption_key = "" - access_key = "test" - secret_key = "test" - with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result1, connector_name1 = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, - access_key, secret_key, - bucket: "testbucket") - - parsed_json1 = JSON.parse(json_result1) + _, connector_name = create_s3_connector(project.id, featurestore_id, + encryption_algorithm: "AES256", + access_key: "test", secret_key: "test", + bucket: "testbucket") expect_status(201) - connector_name = parsed_json1["name"] - update_s3_connector(project.id, featurestore_id, connector_name, s3_connector_name, with_access_keys, bucket: - "testbucket2") + update_s3_connector(project.id, featurestore_id, s3_connector_name, + access_key: "testdifferent", secret_key: "test", + bucket: "testbucket1") expect_status_details(400) - get_storage_connector_by_name(project.id, featurestore_id, "S3", connector_name) + json_result = get_storage_connector(project.id, featurestore_id, connector_name) expect_status_details(200) - expect(json_body[:name]).to eq connector_name - setVar("aws_instance_role", "false") + json_body = JSON.parse(json_result) + expect(json_body["name"]).to eql(connector_name) end it "should be able to update S3 connector in the featurestore: provide same connector name" do - setVar("aws_instance_role", "false") project = get_project - create_session(project[:username], "Pass123") - featurestore_id = get_featurestore_id(project.id) - with_access_keys = true - encryption_algorithm = "AES256" - encryption_key = "" - access_key = "test" - secret_key = "test" - with_encryption_key = true; featurestore_id = get_featurestore_id(project.id) - json_result1, connector_name1 = create_s3_connector_with_encryption(project.id, featurestore_id, with_encryption_key, - encryption_algorithm, encryption_key, - access_key, secret_key, - bucket: "testbucket") + json_result1, connector_name = create_s3_connector(project.id, featurestore_id, + encryption_algorithm: "AES256", + access_key: "test", secret_key: "test", + bucket: "testbucket") parsed_json1 = JSON.parse(json_result1) expect_status(201) - connector_name = parsed_json1["name"] - s3_connector_name = connector_name1 - json_result2, connector_name2 = update_s3_connector(project.id, featurestore_id, connector_name, - s3_connector_name, with_access_keys, bucket: - "testbucket2") + + json_result2, _ = update_s3_connector(project.id, featurestore_id, connector_name, + access_key: "testdifferent", secret_key: "test", + bucket: "testbucket2") parsed_json2 = JSON.parse(json_result2) expect(parsed_json2.key?("id")).to be true @@ -486,22 +405,19 @@ expect(parsed_json2.key?("featurestoreId")).to be true expect(parsed_json2.key?("bucket")).to be true expect(parsed_json2.key?("secretKey")).to be true - expect(parsed_json2.key?("accessKey")).to be true - expect(parsed_json2["name"] == connector_name2).to be true + expect(parsed_json2["name"] == connector_name).to be true expect(parsed_json2["storageConnectorType"] == "S3").to be true expect(parsed_json2["bucket"] == "testbucket2").to be true - setVar("aws_instance_role", "false") + expect(parsed_json2["accessKey"]).to eql("testdifferent") end it "should be able to update JDBC connector in the featurestore" do project = get_project create_session(project[:username], "Pass123") featurestore_id = get_featurestore_id(project.id) - json_result1, connector_name1 = create_jdbc_connector(project.id, featurestore_id, connectionString: "jdbc://test2") - parsed_json1 = JSON.parse(json_result1) + json_result1, connector_name = create_jdbc_connector(project.id, featurestore_id, connectionString: "jdbc://test2") expect_status(201) - connector_name = parsed_json1["name"] - json_result2, connector_name2 = update_jdbc_connector(project.id, featurestore_id, connector_name, connectionString: "jdbc://test3") + json_result2 = update_jdbc_connector(project.id, featurestore_id, connector_name, connectionString: "jdbc://test3") parsed_json2 = JSON.parse(json_result2) expect(parsed_json2.key?("id")).to be true expect(parsed_json2.key?("name")).to be true @@ -510,7 +426,7 @@ expect(parsed_json2.key?("featurestoreId")).to be true expect(parsed_json2.key?("connectionString")).to be true expect(parsed_json2.key?("arguments")).to be true - expect(parsed_json2["name"] == connector_name2).to be true + expect(parsed_json2["name"] == connector_name).to be true expect(parsed_json2["storageConnectorType"] == "JDBC").to be true expect(parsed_json2["connectionString"] == "jdbc://test3").to be true end @@ -519,7 +435,8 @@ # Storage connector looks like this: [project name]_[username]_onlinefeaturestore connector_name = "#{@project['projectname']}_#{@user['username']}_onlinefeaturestore" featurestore_id = get_featurestore_id(@project['id']) - connector = get_jdbc_storate_connector(@project['id'], featurestore_id, connector_name) + connector_json = get_storage_connector(@project['id'], featurestore_id, connector_name) + connector = JSON.parse(connector_json) expect(connector['connectionString']).to match(/jdbc:mysql:\/\/\d{1,3}\.\d{1,3}\.\d{1,3}.\d{1,3}/) end @@ -527,9 +444,10 @@ it "online storage connector connection string should be stored with consul name in the database" do # Storage connector looks like this: [project name]_[username]_onlinefeaturestore connector_name = "#{@project['projectname']}_#{@user['username']}_onlinefeaturestore" - connector_db = FeatureStoreJDBCConnector.find_by(name: connector_name) + connector_db = FeatureStoreConnector.find_by(name: connector_name) + jdbc_connector_db = FeatureStoreJDBCConnector.find_by(id: connector_db.jdbc_id) - expect(connector_db.connection_string).to start_with("jdbc:mysql://onlinefs.mysql.service.consul") + expect(jdbc_connector_db[:connection_string]).to start_with("jdbc:mysql://onlinefs.mysql.service.consul") end end end diff --git a/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb b/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb index 23ddaea5ae..c6504d6245 100644 --- a/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb @@ -46,15 +46,13 @@ expect(parsed_json.key?("dataFormat")).to be true expect(parsed_json.key?("trainingDatasetType")).to be true expect(parsed_json.key?("location")).to be true - expect(parsed_json.key?("storageConnectorId")).to be true - expect(parsed_json.key?("storageConnectorName")).to be true expect(parsed_json.key?("inodeId")).to be true expect(parsed_json.key?("features")).to be true expect(parsed_json.key?("seed")).to be true expect(parsed_json["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true expect(parsed_json["name"] == training_dataset_name).to be true expect(parsed_json["trainingDatasetType"] == "HOPSFS_TRAINING_DATASET").to be true - expect(parsed_json["storageConnectorId"] == connector.id).to be true + expect(parsed_json["storageConnector"]["id"] == connector.id).to be true expect(parsed_json["seed"] == 1234).to be true expect(parsed_json["fromQuery"]).to be false @@ -240,7 +238,7 @@ json_result, _ = create_hopsfs_training_dataset(project.id, featurestore_id, nil) parsed_json = JSON.parse(json_result) expect_status(201) - expect(parsed_json["storageConnectorName"] == "#{project['projectname']}_Training_Datasets") + expect(parsed_json["storageConnector"]["name"] == "#{project['projectname']}_Training_Datasets") end it "should be able to add a new hopsfs training dataset with a single feature label to the featurestore" do @@ -335,8 +333,6 @@ expect(parsed_json2.key?("version")).to be true expect(parsed_json2.key?("dataFormat")).to be true expect(parsed_json2.key?("trainingDatasetType")).to be true - expect(parsed_json2.key?("storageConnectorId")).to be true - expect(parsed_json2.key?("storageConnectorName")).to be true expect(parsed_json2.key?("inodeId")).to be true # make sure the dataformat didn't change @@ -1033,14 +1029,12 @@ expect(parsed_json.key?("dataFormat")).to be true expect(parsed_json.key?("trainingDatasetType")).to be true expect(parsed_json.key?("description")).to be true - expect(parsed_json.key?("storageConnectorId")).to be true - expect(parsed_json.key?("storageConnectorName")).to be true expect(parsed_json.key?("features")).to be true expect(parsed_json.key?("seed")).to be true expect(parsed_json["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true expect(parsed_json["name"] == training_dataset_name).to be true expect(parsed_json["trainingDatasetType"] == "EXTERNAL_TRAINING_DATASET").to be true - expect(parsed_json["storageConnectorId"] == connector_id).to be true + expect(parsed_json["storageConnector"]["id"] == connector_id).to be true expect(parsed_json["features"].length).to be 2 expect(parsed_json["seed"] == 1234).to be true end @@ -1064,7 +1058,6 @@ project = get_project featurestore_id = get_featurestore_id(project.id) json_result, training_dataset_name = create_external_training_dataset(project.id, featurestore_id, nil) - parsed_json = JSON.parse(json_result) expect_status(400) end @@ -1209,8 +1202,6 @@ expect(parsed_json2.key?("dataFormat")).to be true expect(parsed_json2.key?("trainingDatasetType")).to be true expect(parsed_json2.key?("description")).to be true - expect(parsed_json2.key?("storageConnectorId")).to be true - expect(parsed_json2.key?("storageConnectorName")).to be true expect(parsed_json2["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true expect(parsed_json2["description"] == "new description").to be true expect(parsed_json2["trainingDatasetType"] == "EXTERNAL_TRAINING_DATASET").to be true @@ -1225,7 +1216,7 @@ expect_status(201) training_dataset_id = parsed_json1["id"] - json_new_connector, _ = create_s3_connector_without_encryption(project.id, featurestore_id) + json_new_connector, _ = create_s3_connector(project.id, featurestore_id) new_connector = JSON.parse(json_new_connector) json_result2 = update_external_training_dataset_metadata(project.id, featurestore_id, @@ -1235,7 +1226,7 @@ expect_status(200) # make sure the name didn't change - expect(parsed_json2["storageConnectorId"]).to be connector_id + expect(parsed_json2["storageConnector"]["id"]).to be connector_id end it "should store and return the correct path within the bucket" do diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java index be61536e73..924a700792 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java @@ -37,11 +37,9 @@ import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO; import io.hops.hopsworks.common.featurestore.importjob.FeaturegroupImportJobDTO; -import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; import io.hops.hopsworks.common.featurestore.settings.FeaturestoreClientSettingsDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasetjob.TrainingDatasetJobControllerIface; import io.hops.hopsworks.common.featurestore.trainingdatasetjob.TrainingDatasetJobDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; @@ -124,8 +122,6 @@ public class FeaturestoreService { private JobsBuilder jobsBuilder; @Inject private FsQueryConstructorResource fsQueryConstructorResource; - @Inject - private OnlineFeaturestoreController onlineFeaturestoreController; private Project project; @@ -260,8 +256,7 @@ public Response getFeaturestoreByName( @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiOperation(value = "Get featurestore Metadata", - response = FeaturestoreClientSettingsDTO.class) + @ApiOperation(value = "Get featurestore Metadata", response = FeaturestoreClientSettingsDTO.class) public Response getFeaturestoreId(@Context SecurityContext sc, @PathParam("featurestoreName") String featurestoreName) throws FeaturestoreException, ServiceException { if (Strings.isNullOrEmpty(featurestoreName)) { @@ -276,12 +271,11 @@ public Response getFeaturestoreId(@Context SecurityContext sc, @PathParam("featu List trainingDatasets = trainingDatasetController.getTrainingDatasetsForFeaturestore(featurestore); List storageConnectors = - featurestoreStorageConnectorController.getAllStorageConnectorsForFeaturestore(user, featurestore); + featurestoreStorageConnectorController.getConnectorsForFeaturestore(user, project, featurestore); FeaturestoreClientSettingsDTO featurestoreClientSettingsDTO = new FeaturestoreClientSettingsDTO(); featurestoreClientSettingsDTO.setOnlineFeaturestoreEnabled(settings.isOnlineFeaturestore()); - String dbUsername = onlineFeaturestoreController.onlineDbUsername(project, user); - FeaturestoreJdbcConnectorDTO onlineFeaturestoreConnector = - featurestoreStorageConnectorController.getOnlineFeaturestoreConnector(user, dbUsername, featurestore); + FeaturestoreStorageConnectorDTO onlineFeaturestoreConnector = + featurestoreStorageConnectorController.getOnlineFeaturestoreConnector(user, project, featurestore); FeaturestoreMetadataDTO featurestoreMetadataDTO = new FeaturestoreMetadataDTO(featurestoreDTO, featuregroups, trainingDatasets, featurestoreClientSettingsDTO, storageConnectors, onlineFeaturestoreConnector); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java index 57ba665ae3..6a39d872b9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java @@ -21,14 +21,11 @@ import io.hops.hopsworks.api.filter.NoCacheResponse; import io.hops.hopsworks.api.filter.apiKey.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; -import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.FeaturestoreDTO; import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ProjectException; @@ -37,7 +34,6 @@ import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; -import io.hops.hopsworks.persistence.entity.user.activity.ActivityFlag; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; @@ -79,9 +75,7 @@ public class FeaturestoreStorageConnectorService { @EJB private FeaturestoreController featurestoreController; @EJB - private FeaturestoreStorageConnectorController featurestoreStorageConnectorController; - @EJB - private ActivityFacade activityFacade; + private FeaturestoreStorageConnectorController storageConnectorController; @EJB private JWTHelper jWTHelper; @EJB @@ -92,7 +86,6 @@ public class FeaturestoreStorageConnectorService { private Project project; private Featurestore featurestore; - /** * Set the project of the featurestore (provided by parent resource) * @@ -131,73 +124,30 @@ public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreExcepti public Response getStorageConnectors(@Context SecurityContext sc) throws FeaturestoreException { Users user = jWTHelper.getUserPrincipal(sc); List featurestoreStorageConnectorDTOS = - featurestoreStorageConnectorController.getAllStorageConnectorsForFeaturestore(user, featurestore); - GenericEntity> featurestoreStorageConnectorsGeneric = - new GenericEntity>(featurestoreStorageConnectorDTOS) { - }; - return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featurestoreStorageConnectorsGeneric) - .build(); - } - - /** - * Endpoint for getting all storage connectors of a specific type in a featurestore - * - * @param connectorType - * type of the storage connector, e.g S3, JDBC or HOPSFS - * @return a JSON representation of all the storage connectors with the provided type in the feature store - */ - @GET - @Path("/{connectorType : JDBC|REDSHIFT|S3|HOPSFS}") - @Produces(MediaType.APPLICATION_JSON) - @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) - @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiOperation(value = "Get all storage connectors of a specific type of a feature store", - response = FeaturestoreStorageConnectorDTO.class, responseContainer = "List") - public Response getStorageConnectorsOfType( - @ApiParam(value = "storage connector type", example = "JDBC") - @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, @Context SecurityContext sc) - throws FeaturestoreException { - verifyStorageConnectorType(connectorType); - Users user = jWTHelper.getUserPrincipal(sc); - List featurestoreStorageConnectorDTOS = - featurestoreStorageConnectorController.getAllStorageConnectorsForFeaturestoreWithType(user, featurestore, - connectorType); + storageConnectorController.getConnectorsForFeaturestore(user, project, featurestore); GenericEntity> featurestoreStorageConnectorsGeneric = - new GenericEntity>(featurestoreStorageConnectorDTOS) { - }; + new GenericEntity>(featurestoreStorageConnectorDTOS) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(featurestoreStorageConnectorsGeneric) .build(); } - /** - * Endpoint for getting a storage connector with a particular type and name in a feature store - * - * @param connectorType - * type of the storage connector, e.g S3, JDBC or HOPSFS - * @param connectorName - * the id of the storage connector - * @return a JSON representation of the connector - * @throws FeaturestoreException - */ @GET - @Path("{connectorType : JDBC|REDSHIFT|S3|HOPSFS}/{connectorName}") + @Path("{connectorName}") @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiOperation(value = "Get a storage connector with a specific name and type from a featurestore", + @ApiOperation(value = "Get a storage connector with a specific name and from a featurestore", response = FeaturestoreStorageConnectorDTO.class) - public Response getStorageConnectorWithId( - @ApiParam(value = "storage connector type", example = "JDBC", required = true) - @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, - @ApiParam(value = "Id of the storage connector", required = true) - @PathParam("connectorName") String connectorName, @Context SecurityContext sc) - throws FeaturestoreException { + public Response getStorageConnector(@Context SecurityContext sc, + @ApiParam(value = "Name of the storage connector", required = true) + @PathParam("connectorName") String connectorName) + throws FeaturestoreException { + Users user = jWTHelper.getUserPrincipal(sc); + verifyStorageConnectorName(connectorName); FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = - featurestoreStorageConnectorController.getStorageConnectorForFeaturestoreWithTypeAndName(user, featurestore, - connectorType, connectorName); + storageConnectorController.getConnectorWithName(user, project, featurestore, connectorName); GenericEntity featurestoreStorageConnectorDTOGenericEntity = new GenericEntity(featurestoreStorageConnectorDTO) { }; @@ -208,13 +158,10 @@ public Response getStorageConnectorWithId( /** * Endpoint for creating a storage connector with a particular type in a feature store * - * @param connectorType - * type of the storage connector, e.g S3, JDBC or HOPSFS * @return a JSON representation of the connector * @throws FeaturestoreException */ @POST - @Path("/{connectorType : JDBC|REDSHIFT|S3|HOPSFS}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @@ -222,25 +169,18 @@ public Response getStorageConnectorWithId( @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiOperation(value = "Create a new storage connector for the feature store", response = FeaturestoreStorageConnectorDTO.class) - public Response createNewStorageConnectorWithType(@Context SecurityContext sc, - @ApiParam(value = "storage connector type", example = "JDBC", required = true) - @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, - FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) - throws FeaturestoreException, UserException, ProjectException { + public Response createNewStorageConnector(@Context SecurityContext sc, + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) + throws FeaturestoreException, UserException, ProjectException { if (featurestoreStorageConnectorDTO == null) { throw new IllegalArgumentException("Input JSON for creating a new Feature Store Storage Connector Cannot be " + "null"); } - verifyStorageConnectorType(connectorType); Users user = jWTHelper.getUserPrincipal(sc); - FeaturestoreStorageConnectorDTO createdFeaturestoreStorageConnectorDTO = - featurestoreStorageConnectorController.createStorageConnectorWithType(user, featurestore, connectorType, - featurestoreStorageConnectorDTO); - activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + - featurestoreStorageConnectorDTO.getName(), project, user, ActivityFlag.SERVICE); + FeaturestoreStorageConnectorDTO createdFeaturestoreStorageConnectorDTO = storageConnectorController + .createStorageConnector(user, project, featurestore, featurestoreStorageConnectorDTO); GenericEntity featurestoreStorageConnectorDTOGenericEntity = - new GenericEntity(createdFeaturestoreStorageConnectorDTO) { - }; + new GenericEntity(createdFeaturestoreStorageConnectorDTO) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED) .entity(featurestoreStorageConnectorDTOGenericEntity).build(); } @@ -248,71 +188,61 @@ public Response createNewStorageConnectorWithType(@Context SecurityContext sc, /** * Endpoint for deleting a storage connector with a particular type and name in a feature store * - * @param connectorType - * type of the storage connector, e.g S3, JDBC or HOPSFS * @param connectorName * the id of the storage connector * @return a JSON representation of the deleted connector * @throws FeaturestoreException */ @DELETE - @Path("/{connectorType : JDBC|REDSHIFT|S3|HOPSFS}/{connectorName}") + @Path("{connectorName}") @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiOperation(value = "Delete storage connector with a specific id and type from a featurestore") - public Response deleteStorageConnectorWithTypeAndName(@Context SecurityContext sc, - @ApiParam(value = "storage connector type", example = "JDBC", required = true) - @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, - @ApiParam(value = "Id of the storage connector", required = true) - @PathParam("connectorName") String connectorName) throws FeaturestoreException, UserException, ProjectException { + @ApiOperation(value = "Delete storage connector with a specific name and type from a featurestore") + public Response deleteStorageConnector(@Context SecurityContext sc, + @ApiParam(value = "name of the storage connector", required = true) + @PathParam("connectorName") String connectorName) + throws FeaturestoreException, UserException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - featurestoreStorageConnectorController.deleteStorageConnectorWithTypeAndName(user, connectorType, connectorName, - featurestore); - activityFacade.persistActivity(ActivityFacade.REMOVED_FEATURESTORE_STORAGE_CONNECTOR + - connectorName, project, user, ActivityFlag.SERVICE); + storageConnectorController.deleteConnectorWithName(user, project, connectorName, featurestore); return Response.ok().build(); } /** * Endpoint for updating a storage connector with a particular type and id in a feature store * - * @param connectorType - * type of the storage connector, e.g S3, JDBC or HOPSFS * @param connectorName * the id of the storage connector * @return a JSON representation of the updated connector * @throws FeaturestoreException */ @PUT - @Path("/{connectorType : JDBC|REDSHIFT|S3|HOPSFS}/{connectorName}") + @Path("/{connectorName}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiOperation(value = "Get a storage connector with a specific id and type from a featurestore", + @ApiOperation(value = "Get a storage connector with a specific name and from a featurestore", response = FeaturestoreStorageConnectorDTO.class) - public Response updateStorageConnectorWithId( - @ApiParam(value = "storage connector type", example = "JDBC", required = true) - @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, - @ApiParam(value = "Name of the storage connector", required = true) - @PathParam("connectorName") String connectorName, - FeaturestoreStorageConnectorDTO featurestoreStorageConnectorInputDTO, @Context SecurityContext sc) - throws FeaturestoreException, UserException, ProjectException { + public Response updateStorageConnector(@ApiParam(value = "Name of the storage connector", required = true) + @PathParam("connectorName") String connectorName, + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorInputDTO, + @Context SecurityContext sc) + throws FeaturestoreException, UserException, ProjectException { if (featurestoreStorageConnectorInputDTO == null) { throw new IllegalArgumentException("Input JSON for updating a Feature Store Storage Connector Cannot be " + "null"); } Users user = jWTHelper.getUserPrincipal(sc); - verifyStorageConnectorTypeAndName(connectorType, connectorName); + verifyStorageConnectorName(connectorName); + storageConnectorController.updateStorageConnector(user, project, featurestore, + featurestoreStorageConnectorInputDTO, connectorName); FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = - featurestoreStorageConnectorController.updateStorageConnectorWithType(user, featurestore, - connectorType, featurestoreStorageConnectorInputDTO, connectorName); + storageConnectorController.getConnectorWithName(user, project, featurestore, connectorName); GenericEntity featurestoreStorageConnectorDTOGenericEntity = - new GenericEntity(featurestoreStorageConnectorDTO) { - }; + new GenericEntity(featurestoreStorageConnectorDTO) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK) .entity(featurestoreStorageConnectorDTOGenericEntity).build(); } @@ -346,51 +276,24 @@ public Response getOnlineFeaturestoreStorageConnector(@Context SecurityContext s " talk to an administrator."); } Users user = jWTHelper.getUserPrincipal(sc); - String dbUsername = onlineFeaturestoreController.onlineDbUsername(project, user); - FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = - featurestoreStorageConnectorController.getOnlineFeaturestoreConnector(user, dbUsername, featurestore); + FeaturestoreStorageConnectorDTO featurestoreJdbcConnectorDTO = + storageConnectorController.getOnlineFeaturestoreConnector(user, project, featurestore); GenericEntity featurestoreStorageConnectorDTOGenericEntity = new GenericEntity(featurestoreJdbcConnectorDTO) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK) .entity(featurestoreStorageConnectorDTOGenericEntity).build(); } - /** - * Verify path parameters (storage connector type and id) - * - * @param connectorType - * type provided as a path parameter to a request - * @param connectorName - * id provided as a path parameter to a request - */ - private void verifyStorageConnectorTypeAndName(FeaturestoreStorageConnectorType connectorType, - String connectorName) { - verifyStorageConnectorType(connectorType); - verifyStorageConnectorId(connectorName); - } - + /** * Verify path parameters (storage connector id) * * @param connectorName * id provided as a path parameter to a request */ - private void verifyStorageConnectorId(String connectorName) { + private void verifyStorageConnectorName(String connectorName) { if (Strings.isNullOrEmpty(connectorName)) { throw new IllegalArgumentException( RESTCodes.FeaturestoreErrorCode.STORAGE_CONNECTOR_ID_NOT_PROVIDED.getMessage()); } } - - /** - * Verify path parameters (storage connector type) - * - * @param connectorType - * type provided as a path parameter to a request - */ - private void verifyStorageConnectorType(FeaturestoreStorageConnectorType connectorType) { - if (connectorType == null) { - throw new IllegalArgumentException( - RESTCodes.FeaturestoreErrorCode.STORAGE_CONNECTOR_TYPE_NOT_PROVIDED.getMessage()); - } - } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorFacade.java deleted file mode 100644 index d1c3b555ab..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorFacade.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2020, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ -package io.hops.hopsworks.common.dao.featurestore.storageconnectors.redshift; - -import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector; - -import javax.ejb.Stateless; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; -import java.util.List; -import java.util.Optional; - -@Stateless -public class FeaturestoreRedshiftConnectorFacade extends AbstractFacade { - - @PersistenceContext(unitName = "kthfsPU") - private EntityManager em; - - public FeaturestoreRedshiftConnectorFacade() { - super(FeatureStoreRedshiftConnector.class); - } - - @Override - protected EntityManager getEntityManager() { - return em; - } - - public List findByFeaturestore(Featurestore featurestore) { - TypedQuery q = - em.createNamedQuery("FeatureStoreRedshiftConnector.findByFeaturestore", FeatureStoreRedshiftConnector.class) - .setParameter("featurestore", featurestore); - return q.getResultList(); - } - - public Optional findByIdAndFeaturestore(Integer id, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeatureStoreRedshiftConnector.findByFeaturestoreAndId", - FeatureStoreRedshiftConnector.class).setParameter("featurestore", featurestore).setParameter("id", id) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - public Optional findByNameAndFeaturestore(String name, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeatureStoreRedshiftConnector.findByNameAndFeaturestore", - FeatureStoreRedshiftConnector.class).setParameter("name", name).setParameter("featurestore", featurestore) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } -} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/user/activity/ActivityFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/user/activity/ActivityFacade.java index 9f95391ad5..a42fe5c66e 100755 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/user/activity/ActivityFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/user/activity/ActivityFacade.java @@ -90,6 +90,8 @@ public class ActivityFacade extends AbstractFacade { public static final String EDITED_TRAINING_DATASET = " edited training dataset named "; public static final String ADDED_FEATURESTORE_STORAGE_CONNECTOR = " added a storage connector for the featurestore " + "with name: "; + public static final String UPDATED_FEATURESTORE_STORAGE_CONNECTOR = + " updated a storage connector for the featurestore with name: "; public static final String REMOVED_FEATURESTORE_STORAGE_CONNECTOR = " added a storage connector for " + "the featurestore with name: "; public static final String REMOVED_FEATURESTORE_STORAGE_CONNECTOR_ID = " added a storage connector for " + diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreConstants.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreConstants.java index 384367b4c5..a1ab07fe2d 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreConstants.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreConstants.java @@ -15,7 +15,8 @@ */ package io.hops.hopsworks.common.featurestore; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; + +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetType; import java.util.Arrays; @@ -45,10 +46,10 @@ private FeaturestoreConstants() { public static final int ON_DEMAND_FEATUREGROUP_SQL_QUERY_MAX_LENGTH = 11000; public static final List TRAINING_DATASET_DATA_FORMATS = Arrays.asList(new String[]{"csv", "tfrecords", "tfrecord", "parquet", "tsv", "hdf5", "npy", "orc", "avro", "image", "petastorm"}); - public static final String JDBC_CONNECTOR_TYPE = FeaturestoreStorageConnectorType.JDBC.name(); - public static final String REDSHIFT_CONNECTOR_TYPE = FeaturestoreStorageConnectorType.REDSHIFT.name(); - public static final String HOPSFS_CONNECTOR_TYPE = FeaturestoreStorageConnectorType.HOPSFS.name(); - public static final String S3_CONNECTOR_TYPE = FeaturestoreStorageConnectorType.S3.name(); + public static final String JDBC_CONNECTOR_TYPE = FeaturestoreConnectorType.JDBC.name(); + public static final String HOPSFS_CONNECTOR_TYPE = FeaturestoreConnectorType.HOPSFS.name(); + public static final String S3_CONNECTOR_TYPE = FeaturestoreConnectorType.S3.name(); + public static final String REDSHIFT_CONNECTOR_TYPE = FeaturestoreConnectorType.REDSHIFT.name(); public static final String CACHED_FEATUREGROUP_DTO_TYPE = "cachedFeaturegroupDTO"; public static final String ON_DEMAND_FEATUREGROUP_DTO_TYPE = "onDemandFeaturegroupDTO"; public static final String HOPSFS_TRAINING_DATASET_TYPE = TrainingDatasetType.HOPSFS_TRAINING_DATASET.name(); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java index 5aa2390daa..4d657dd2c2 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java @@ -20,17 +20,22 @@ import io.hops.hopsworks.common.featurestore.app.FeaturestoreUtilJobDTO; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupFacade; import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; -import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorController; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorController; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; +import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; +import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetFacade; import io.hops.hopsworks.common.featurestore.utils.FeaturestoreUtils; import io.hops.hopsworks.common.hive.HiveController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.persistence.entity.dataset.Dataset; import io.hops.hopsworks.persistence.entity.dataset.DatasetSharedWith; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.activity.ActivityFlag; @@ -67,10 +72,6 @@ public class FeaturestoreController { @EJB private ActivityFacade activityFacade; @EJB - private FeaturestoreJdbcConnectorController featurestoreJdbcConnectorController; - @EJB - private FeaturestoreHopsfsConnectorController featurestoreHopsfsConnectorController; - @EJB private Settings settings; @EJB private OnlineFeaturestoreController onlineFeaturestoreController; @@ -82,6 +83,8 @@ public class FeaturestoreController { private FeaturegroupFacade featuregroupFacade; @EJB private TrainingDatasetFacade trainingDatasetFacade; + @EJB + private FeaturestoreStorageConnectorController featurestoreStorageConnectorController; private JAXBContext featurestoreUtilJobArgsJaxbContext = null; private Marshaller featurestoreUtilJobArgsMarshaller = null; @@ -235,7 +238,7 @@ public Featurestore getFeaturestoreWithId(Integer id) throws FeaturestoreExcepti * @throws FeaturestoreException */ public Featurestore createProjectFeatureStore(Project project, Users user, String featurestoreName, - Dataset trainingDatasetsFolder) throws FeaturestoreException { + Dataset trainingDatasetsFolder) throws FeaturestoreException, ProjectException, UserException { //Get HiveDbId for the newly created Hive featurestore DB Long hiveDbId = featurestoreFacade.getHiveDatabaseId(featurestoreName); @@ -247,16 +250,14 @@ public Featurestore createProjectFeatureStore(Project project, Users user, Strin featurestoreFacade.persist(featurestore); activityFacade.persistActivity(ActivityFacade.CREATED_FEATURESTORE + featurestoreName, project, project.getOwner(), ActivityFlag.SERVICE); - featurestoreJdbcConnectorController.createDefaultJdbcConnectorForOfflineFeaturestore(featurestore, - getOfflineFeaturestoreDbName(project), "JDBC connection to Hopsworks Project Feature Store Hive Database"); activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + getOfflineFeaturestoreDbName(project), project, project.getOwner(), ActivityFlag.SERVICE); - featurestoreJdbcConnectorController.createDefaultJdbcConnectorForOfflineFeaturestore(featurestore, - project.getName(), "JDBC connection to Hopsworks Project Hive Warehouse"); activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + project.getName(), project, project.getOwner(), ActivityFlag.SERVICE); - featurestoreHopsfsConnectorController.createHopsFsBackendForFeaturestoreConnector(featurestore, - trainingDatasetsFolder); + featurestoreStorageConnectorController + .createStorageConnector(user, project, featurestore, hopsfsTrainingDatasetConnector(trainingDatasetsFolder)); + featurestoreStorageConnectorController + .createStorageConnector(user, project, featurestore, createOfflineJdbcConnector(featurestoreName)); activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + trainingDatasetsFolder. getName(), project, project.getOwner(), ActivityFlag.SERVICE); if (settings.isOnlineFeaturestore()) { @@ -265,6 +266,40 @@ public Featurestore createProjectFeatureStore(Project project, Users user, Strin return featurestore; } + public FeaturestoreStorageConnectorDTO hopsfsTrainingDatasetConnector(Dataset hopsfsDataset) { + String name = hopsfsDataset.getName(); + String description = "HOPSFS backend for storing Training Datasets of the Hopsworks Feature Store"; + FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO = new FeaturestoreHopsfsConnectorDTO(); + featurestoreHopsfsConnectorDTO.setStorageConnectorType(FeaturestoreConnectorType.HOPSFS); + featurestoreHopsfsConnectorDTO.setName(name); + featurestoreHopsfsConnectorDTO.setDescription(description); + featurestoreHopsfsConnectorDTO.setDatasetName(hopsfsDataset.getName()); + + return featurestoreHopsfsConnectorDTO; + } + + + public FeaturestoreStorageConnectorDTO createOfflineJdbcConnector(String databaseName) throws FeaturestoreException { + String hiveEndpoint; + try { + hiveEndpoint = hiveController.getHiveServerInternalEndpoint(); + } catch (ServiceDiscoveryException ex) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, + Level.SEVERE, "Could not create Hive connection string", ex.getMessage(), ex); + } + + String connectionString = "jdbc:hive2://" + hiveEndpoint + "/" + databaseName + ";" + + "auth=noSasl;ssl=true;twoWay=true;"; + String arguments = "sslTrustStore,trustStorePassword,sslKeyStore,keyStorePassword"; + FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO(); + featurestoreJdbcConnectorDTO.setStorageConnectorType(FeaturestoreConnectorType.JDBC); + featurestoreJdbcConnectorDTO.setName(databaseName); + featurestoreJdbcConnectorDTO.setDescription("JDBC connector for the Offline Feature Store"); + featurestoreJdbcConnectorDTO.setConnectionString(connectionString); + featurestoreJdbcConnectorDTO.setArguments(arguments); + return featurestoreJdbcConnectorDTO; + } + /** * Converts a featurestore entity to a Featurestore DTO, supplements the featurestore entity * with Hive metadata and remove foreign keys that are less interesting for users. diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/app/FeaturestoreMetadataDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/app/FeaturestoreMetadataDTO.java index d1ec514eaa..d603686f21 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/app/FeaturestoreMetadataDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/app/FeaturestoreMetadataDTO.java @@ -20,7 +20,6 @@ import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO; import io.hops.hopsworks.common.featurestore.settings.FeaturestoreClientSettingsDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; import javax.xml.bind.annotation.XmlElement; @@ -42,15 +41,16 @@ public class FeaturestoreMetadataDTO { private List trainingDatasets; private FeaturestoreClientSettingsDTO settings; private List storageConnectors; - private FeaturestoreJdbcConnectorDTO onlineFeaturestoreConnector; + private FeaturestoreStorageConnectorDTO onlineFeaturestoreConnector; public FeaturestoreMetadataDTO() { } - + public FeaturestoreMetadataDTO(FeaturestoreDTO featurestore, - List featuregroups, List trainingDatasets, - FeaturestoreClientSettingsDTO featurestoreClientSettingsDTO, - List storageConnectors, FeaturestoreJdbcConnectorDTO onlineFeaturestoreConnector) { + List featuregroups, List trainingDatasets, + FeaturestoreClientSettingsDTO featurestoreClientSettingsDTO, + List storageConnectors, + FeaturestoreStorageConnectorDTO onlineFeaturestoreConnector) { this.featurestore = featurestore; this.featuregroups = featuregroups; this.trainingDatasets = trainingDatasets; @@ -85,7 +85,7 @@ public List getStorageConnectors() { } @XmlElement - public FeaturestoreJdbcConnectorDTO getOnlineFeaturestoreConnector() { + public FeaturestoreStorageConnectorDTO getOnlineFeaturestoreConnector() { return onlineFeaturestoreConnector; } @@ -111,7 +111,7 @@ public void setStorageConnectors( } public void setOnlineFeaturestoreConnector( - FeaturestoreJdbcConnectorDTO onlineFeaturestoreConnector) { + FeaturestoreStorageConnectorDTO onlineFeaturestoreConnector) { this.onlineFeaturestoreConnector = onlineFeaturestoreConnector; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java index 318cc872bd..a8708c31e4 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java @@ -270,7 +270,7 @@ private FeaturegroupDTO convertFeaturegrouptoDTO(Featuregroup featuregroup, Proj return cachedFeaturegroupDTO; case ON_DEMAND_FEATURE_GROUP: FeaturestoreJdbcConnectorDTO storageConnectorDTO = - new FeaturestoreJdbcConnectorDTO(featuregroup.getOnDemandFeaturegroup().getFeaturestoreJdbcConnector()); + new FeaturestoreJdbcConnectorDTO(featuregroup.getOnDemandFeaturegroup().getFeaturestoreConnector()); return new OnDemandFeaturegroupDTO(featurestoreName, featuregroup, storageConnectorDTO); default: throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATUREGROUP_TYPE.getMessage() diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/ondemand/OnDemandFeaturegroupController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/ondemand/OnDemandFeaturegroupController.java index fc6cba859a..0eee452c6d 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/ondemand/OnDemandFeaturegroupController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/ondemand/OnDemandFeaturegroupController.java @@ -18,6 +18,7 @@ import com.google.common.base.Strings; import io.hops.hopsworks.common.featurestore.FeaturestoreFacade; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreConnectorFacade; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; import io.hops.hopsworks.common.hdfs.HdfsUsersController; @@ -26,8 +27,7 @@ import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeature; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeaturegroup; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorFacade; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; import io.hops.hopsworks.persistence.entity.project.Project; @@ -55,7 +55,7 @@ public class OnDemandFeaturegroupController { @EJB private OnDemandFeaturegroupFacade onDemandFeaturegroupFacade; @EJB - private FeaturestoreJdbcConnectorFacade featurestoreJdbcConnectorFacade; + private FeaturestoreConnectorFacade featurestoreConnectorFacade; @EJB private FeaturestoreFacade featurestoreFacade; @EJB @@ -80,20 +80,17 @@ public OnDemandFeaturegroup createOnDemandFeaturegroup(Featurestore featurestore Project project, Users user) throws FeaturestoreException { //Verify User Input specific for on demand feature groups - FeaturestoreJdbcConnector featurestoreJdbcConnector = - getJdbcStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId()); + FeaturestoreConnector connector = getStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId()); if (Strings.isNullOrEmpty(onDemandFeaturegroupDTO.getQuery())){ throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.INVALID_SQL_QUERY, Level.FINE, "SQL Query cannot be empty"); } - // Create inode on the file system - //Persist on-demand featuregroup OnDemandFeaturegroup onDemandFeaturegroup = new OnDemandFeaturegroup(); onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription()); - onDemandFeaturegroup.setFeaturestoreJdbcConnector(featurestoreJdbcConnector); + onDemandFeaturegroup.setFeaturestoreConnector(connector); onDemandFeaturegroup.setQuery(onDemandFeaturegroupDTO.getQuery()); onDemandFeaturegroup.setFeatures(convertOnDemandFeatures(onDemandFeaturegroupDTO, onDemandFeaturegroup)); onDemandFeaturegroup.setInode(createFile(project, user, featurestore, onDemandFeaturegroupDTO)); @@ -113,14 +110,13 @@ public void updateOnDemandFeaturegroupMetadata(OnDemandFeaturegroup onDemandFeat OnDemandFeaturegroupDTO onDemandFeaturegroupDTO) throws FeaturestoreException { // Verify User Input specific for on demand feature groups - FeaturestoreJdbcConnector featurestoreJdbcConnector = - getJdbcStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId()); + FeaturestoreConnector connector = getStorageConnector(onDemandFeaturegroupDTO.getStorageConnector().getId()); // Update metadata in entity onDemandFeaturegroup.setDescription(onDemandFeaturegroupDTO.getDescription()); - onDemandFeaturegroup.setFeaturestoreJdbcConnector(featurestoreJdbcConnector); + onDemandFeaturegroup.setFeaturestoreConnector(connector); - // finally persist in database + // finally merge in database onDemandFeaturegroupFacade.updateMetadata(onDemandFeaturegroup); } @@ -185,23 +181,13 @@ private Path getFilePath(Featurestore featurestore, String name, Integer version name + "_" + version); } - /** - * Verifies the user input JDBC Connector Id for an on-demand feature group - * - * @param jdbcConnectorId the JDBC connector id to verify - * @returns the jdbc connector with the given id if it passed the validation - * @throws FeaturestoreException - */ - private FeaturestoreJdbcConnector getJdbcStorageConnector(Integer jdbcConnectorId) - throws FeaturestoreException { - if(jdbcConnectorId == null){ - throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.JDBC_CONNECTOR_ID_NOT_PROVIDED.getMessage()); - } - FeaturestoreJdbcConnector featurestoreJdbcConnector = featurestoreJdbcConnectorFacade.find(jdbcConnectorId); - if(featurestoreJdbcConnector == null) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.JDBC_CONNECTOR_NOT_FOUND, Level.FINE, - "JDBC connector with id: " + jdbcConnectorId + " was not found"); + private FeaturestoreConnector getStorageConnector(Integer connectorId) throws FeaturestoreException { + if(connectorId == null){ + throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_ID_NOT_PROVIDED.getMessage()); } - return featurestoreJdbcConnector; + + return featurestoreConnectorFacade.findById(connectorId) + .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, + "Connector with id: " + connectorId + " was not found")); } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java index 7fcf840b49..71ecba6c80 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java @@ -21,7 +21,7 @@ import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; import io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.cached.FeaturegroupPreview; -import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorController; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreConnectorFacade; import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; import io.hops.hopsworks.common.security.secrets.SecretsController; @@ -29,6 +29,9 @@ import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; import io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.project.team.ProjectRoleTypes; @@ -72,8 +75,6 @@ public class OnlineFeaturestoreController { @EJB private SecretsController secretsController; @EJB - private FeaturestoreJdbcConnectorController featurestoreJdbcConnectorController; - @EJB private OnlineFeaturestoreFacade onlineFeaturestoreFacade; @EJB private CachedFeaturegroupController cachedFeaturegroupController; @@ -83,6 +84,8 @@ public class OnlineFeaturestoreController { private UserFacade userFacade; @EJB private ServiceDiscoveryController serviceDiscoveryController; + @EJB + private FeaturestoreConnectorFacade featurestoreConnectorFacade; @PostConstruct public void init() { @@ -331,7 +334,7 @@ public void updateUserOnlineFeatureStoreDB(Users user, Featurestore featurestore } try { - featurestoreJdbcConnectorController.createJdbcConnectorForOnlineFeaturestore(dbuser, featurestore, db); + createJdbcConnectorForOnlineFeaturestore(dbuser, featurestore, db); } catch(Exception e) { //If the connector have already been created, skip this step } @@ -351,6 +354,43 @@ public void addOnlineFeatureStoreDB(String db) throws FeaturestoreException { onlineFeaturestoreFacade.createOnlineFeaturestoreDatabase(db); } + + /** + * Utility function for create a JDBC connection to the online featurestore for a particular user. + * + * @param onlineDbUsername the db-username of the connection + * @param featurestore the featurestore metadata + * @param dbName name of the MySQL database + * @return DTO of the newly created connector + * @throws FeaturestoreException + */ + public void createJdbcConnectorForOnlineFeaturestore(String onlineDbUsername, + Featurestore featurestore, String dbName) throws FeaturestoreException { + String connectorName = onlineDbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX; + if (featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName).isPresent()) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, + "a storage connector with that name already exists"); + } + + FeaturestoreConnector featurestoreConnector = new FeaturestoreConnector(); + featurestoreConnector.setName(connectorName); + featurestoreConnector.setDescription("JDBC connection to Hopsworks Project Online " + + "Feature Store NDB Database for user: " + onlineDbUsername); + featurestoreConnector.setFeaturestore(featurestore); + featurestoreConnector.setConnectorType(FeaturestoreConnectorType.JDBC); + + FeaturestoreJdbcConnector featurestoreJdbcConnector = new FeaturestoreJdbcConnector(); + featurestoreJdbcConnector.setConnectionString(settings.getFeaturestoreJdbcUrl() + dbName + + "?useSSL=false&allowPublicKeyRetrieval=true"); + featurestoreJdbcConnector.setArguments( + FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_PASSWORD_ARG + "=" + + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_PASSWORD_TEMPLATE + "," + + FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_USER_ARG + "=" + onlineDbUsername + + ",isolationLevel=NONE,batchsize=500"); + featurestoreConnector.setJdbcConnector(featurestoreJdbcConnector); + + featurestoreConnectorFacade.update(featurestoreConnector); + } /** * Drops an online feature store database and removes all associated users @@ -384,13 +424,6 @@ public void removeOnlineFeatureStore(Project project) throws FeaturestoreExcepti } } - /** - * Removes a user from a online feature store database in the project - * - * @param featurestore the project that owns the online feature store - * @param user the user to remove - * @throws FeaturestoreException - */ public void removeOnlineFeaturestoreUser(Featurestore featurestore, Users user) throws FeaturestoreException { String db = getOnlineFeaturestoreDbName(featurestore.getProject()); if (!checkIfDatabaseExists(db)) { @@ -399,15 +432,15 @@ public void removeOnlineFeaturestoreUser(Featurestore featurestore, Users user) } String dbUser = onlineDbUsername(featurestore.getProject().getName(), user.getUsername()); - String connectorName = dbUser + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX; SecretId id = new SecretId(user.getUid(), dbUser); secretsFacade.deleteSecret(id); onlineFeaturestoreFacade.removeOnlineFeaturestoreUser(dbUser); - featurestoreJdbcConnectorController.removeFeaturestoreJdbcConnector(connectorName, featurestore); + + featurestoreConnectorFacade.deleteByFeaturestoreName(featurestore, + dbUser + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX); } - - + /** * Gets the size of an online featurestore database. I.e the size of a MySQL-cluster database. * diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/query/ConstructorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/query/ConstructorController.java index 1be677e978..94524ab300 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/query/ConstructorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/query/ConstructorController.java @@ -642,7 +642,7 @@ private List getOnDemandAliases(Query query, if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) { FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO( - query.getFeaturegroup().getOnDemandFeaturegroup().getFeaturestoreJdbcConnector()); + query.getFeaturegroup().getOnDemandFeaturegroup().getFeaturestoreConnector()); OnDemandFeaturegroupDTO onDemandFeaturegroupDTO = new OnDemandFeaturegroupDTO(query.getFeaturegroup(), featurestoreJdbcConnectorDTO); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java new file mode 100644 index 0000000000..e7b42f6526 --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreConnectorFacade.java @@ -0,0 +1,102 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2020, Logical Clocks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ + +package io.hops.hopsworks.common.featurestore.storageconnectors; + +import io.hops.hopsworks.common.dao.AbstractFacade; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; + +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import javax.persistence.EntityManager; +import javax.persistence.NoResultException; +import javax.persistence.PersistenceContext; +import java.util.List; +import java.util.Optional; + +@Stateless +@TransactionAttribute(TransactionAttributeType.REQUIRED) +public class FeaturestoreConnectorFacade extends AbstractFacade { + + @PersistenceContext(unitName = "kthfsPU") + private EntityManager em; + + public FeaturestoreConnectorFacade() { + super(FeaturestoreConnector.class); + } + + @Override + protected EntityManager getEntityManager() { + return em; + } + + public Optional findById(Integer id) { + try { + return Optional.of(em.createNamedQuery("FeaturestoreConnector.findById", FeaturestoreConnector.class) + .setParameter("id", id) + .getSingleResult()); + } catch (NoResultException e) { + return Optional.empty(); + } + } + + public Optional findByIdType(Integer id, FeaturestoreConnectorType type) { + try { + return Optional.of(em.createNamedQuery("FeaturestoreConnector.findByIdType", FeaturestoreConnector.class) + .setParameter("id", id) + .setParameter("type", type) + .getSingleResult()); + } catch (NoResultException e) { + return Optional.empty(); + } + } + + public List findByFeaturestore(Featurestore featurestore) { + return em.createNamedQuery("FeaturestoreConnector.findByFeaturestore", FeaturestoreConnector.class) + .setParameter("featurestore", featurestore) + .getResultList(); + } + + public Optional findByFeaturestoreId(Featurestore featurestore, Integer id) { + try { + return Optional.of(em.createNamedQuery("FeaturestoreConnector.findByFeaturestoreId", FeaturestoreConnector.class) + .setParameter("featurestore", featurestore) + .setParameter("id", id) + .getSingleResult()); + } catch (NoResultException e) { + return Optional.empty(); + } + } + + public Optional findByFeaturestoreName(Featurestore featurestore, String name) { + try { + return Optional.of(em.createNamedQuery("FeaturestoreConnector.findByFeaturestoreName", + FeaturestoreConnector.class) + .setParameter("featurestore", featurestore) + .setParameter("name", name) + .getSingleResult()); + } catch (NoResultException e) { + return Optional.empty(); + } + } + + public void deleteByFeaturestoreName(Featurestore featurestore, String name) { + findByFeaturestoreName(featurestore, name).ifPresent(this::remove); + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java index fce4be2202..6771ec46d5 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java @@ -16,9 +16,12 @@ package io.hops.hopsworks.common.featurestore.storageconnectors; +import com.google.common.base.Strings; import io.hops.hopsworks.common.constants.auth.AllowedRoles; import io.hops.hopsworks.common.dao.project.team.ProjectTeamFacade; +import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; +import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorController; @@ -31,15 +34,21 @@ import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; +import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.user.activity.ActivityFlag; import io.hops.hopsworks.restutils.RESTCodes; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.transaction.Transactional; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.logging.Level; /** @@ -49,15 +58,21 @@ @TransactionAttribute(TransactionAttributeType.NEVER) public class FeaturestoreStorageConnectorController { @EJB - private FeaturestoreHopsfsConnectorController featurestoreHopsfsConnectorController; + private FeaturestoreHopsfsConnectorController hopsfsConnectorController; @EJB - private FeaturestoreJdbcConnectorController featurestoreJdbcConnectorController; + private FeaturestoreJdbcConnectorController jdbcConnectorController; @EJB - private FeaturestoreRedshiftConnectorController featurestoreRedshiftConnectorController; + private FeaturestoreRedshiftConnectorController redshiftConnectorController; @EJB - private FeaturestoreS3ConnectorController featurestoreS3ConnectorController; + private FeaturestoreS3ConnectorController s3ConnectorController; @EJB private ProjectTeamFacade projectTeamFacade; + @EJB + private FeaturestoreConnectorFacade featurestoreConnectorFacade; + @EJB + private OnlineFeaturestoreController onlineFeaturestoreController; + @EJB + private ActivityFacade activityFacade; /** * Returns a list with DTOs of all storage connectors for a featurestore @@ -67,80 +82,47 @@ public class FeaturestoreStorageConnectorController { * @param user the user making the request * @return List of JSON/XML DTOs of the storage connectors */ - public List getAllStorageConnectorsForFeaturestore(Users user, - Featurestore featurestore) + public List getConnectorsForFeaturestore(Users user, Project project, + Featurestore featurestore) throws FeaturestoreException { + List featurestoreConnectors = featurestoreConnectorFacade.findByFeaturestore(featurestore); List featurestoreStorageConnectorDTOS = new ArrayList<>(); - featurestoreStorageConnectorDTOS.addAll( - featurestoreJdbcConnectorController.getJdbcConnectorsForFeaturestore(user, featurestore)); - featurestoreStorageConnectorDTOS.addAll( - featurestoreRedshiftConnectorController.getConnectorsForFeaturestore(user, featurestore)); - featurestoreStorageConnectorDTOS.addAll( - featurestoreS3ConnectorController.getS3ConnectorsForFeaturestore(user, featurestore)); - featurestoreStorageConnectorDTOS.addAll(featurestoreHopsfsConnectorController.getHopsfsConnectors(featurestore)); + + for (FeaturestoreConnector featurestoreConnector : featurestoreConnectors) { + featurestoreStorageConnectorDTOS.add(convertToConnectorDTO(user, project, featurestoreConnector)); + } + return featurestoreStorageConnectorDTOS; } - /** - * Returns a list with DTOs of all storage connectors for a featurestore with a specific type - * - * @param user the user making the request - * @param featurestore the featurestore to query - * @param featurestoreStorageConnectorType the type of the storage connector - * @return List of JSON/XML DTOs of the storage connectors - */ - public List getAllStorageConnectorsForFeaturestoreWithType(Users user, - Featurestore featurestore, FeaturestoreStorageConnectorType featurestoreStorageConnectorType) - throws FeaturestoreException { - switch(featurestoreStorageConnectorType) { - case S3: - return featurestoreS3ConnectorController.getS3ConnectorsForFeaturestore(user, featurestore); - case JDBC: - return featurestoreJdbcConnectorController.getJdbcConnectorsForFeaturestore(user, featurestore); - case REDSHIFT: - return featurestoreRedshiftConnectorController.getConnectorsForFeaturestore(user, featurestore); - case HOPSFS: - return featurestoreHopsfsConnectorController.getHopsfsConnectors(featurestore); - default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + featurestoreStorageConnectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); - } + public FeaturestoreStorageConnectorDTO getConnectorWithName(Users user, Project project, + Featurestore featurestore, + String connectorName) + throws FeaturestoreException { + FeaturestoreConnector featurestoreConnector = + featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName) + .orElseThrow(() -> + new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, + "Cannot find storage connector with name: " + connectorName)); + + return convertToConnectorDTO(user, project, featurestoreConnector); } - /** - * - * @param user - * @param featurestore - * @param featurestoreStorageConnectorType - * @param storageConnectorName - * @return - * @throws FeaturestoreException - */ - public FeaturestoreStorageConnectorDTO getStorageConnectorForFeaturestoreWithTypeAndName(Users user, - Featurestore featurestore, FeaturestoreStorageConnectorType featurestoreStorageConnectorType, - String storageConnectorName) throws FeaturestoreException { - switch(featurestoreStorageConnectorType) { + private FeaturestoreStorageConnectorDTO convertToConnectorDTO(Users user, Project project, + FeaturestoreConnector featurestoreConnector) + throws FeaturestoreException { + switch (featurestoreConnector.getConnectorType()) { case S3: - return featurestoreS3ConnectorController.getS3ConnectorWithNameAndFeaturestore(user, featurestore, - storageConnectorName); + return s3ConnectorController.getS3ConnectorDTO(user, featurestoreConnector); case JDBC: - return featurestoreJdbcConnectorController.getJdbcConnectorWithNameAndFeaturestore(user, featurestore, - storageConnectorName); - case REDSHIFT: - return featurestoreRedshiftConnectorController.getConnectorsWithNameAndFeaturestore(user, featurestore, - storageConnectorName); + return jdbcConnectorController.getJdbcConnectorDTO(user, project, featurestoreConnector); case HOPSFS: - return featurestoreHopsfsConnectorController.getHopsFsConnectorWithNameAndFeaturestore(featurestore, - storageConnectorName); + return hopsfsConnectorController.getHopsfsConnectorDTO(featurestoreConnector); + case REDSHIFT: + return redshiftConnectorController.getRedshiftConnectorDTO(user, featurestoreConnector); default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + featurestoreStorageConnectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); + // We should not reach this point + throw new IllegalArgumentException("Feature Store connector type not recognized"); } } @@ -149,129 +131,147 @@ public FeaturestoreStorageConnectorDTO getStorageConnectorForFeaturestoreWithTyp * * @param user the user making the request * @param featurestore the featurestore to create the new connector - * @param featurestoreStorageConnectorType the type of the storage connector * @param featurestoreStorageConnectorDTO the data to use when creating the storage connector * @return A JSON/XML DTOs representation of the created storage connector * @throws FeaturestoreException */ - public FeaturestoreStorageConnectorDTO createStorageConnectorWithType(Users user, Featurestore featurestore, - FeaturestoreStorageConnectorType featurestoreStorageConnectorType, - FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) throws FeaturestoreException, UserException, - ProjectException { + public FeaturestoreStorageConnectorDTO createStorageConnector(Users user, Project project, Featurestore featurestore, + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) + throws FeaturestoreException, UserException, ProjectException { validateUser(user, featurestore); - switch(featurestoreStorageConnectorType) { - case S3: - return featurestoreS3ConnectorController.createFeaturestoreS3Connector(user, featurestore, - (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO); - case JDBC: - return featurestoreJdbcConnectorController.createFeaturestoreJdbcConnector(featurestore, - (FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO); - case REDSHIFT: - return featurestoreRedshiftConnectorController.createFeaturestoreRedshiftConnector(user, featurestore, - (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO); - case HOPSFS: - return featurestoreHopsfsConnectorController.createFeaturestoreHopsfsConnector(featurestore, - (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO); - default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + featurestoreStorageConnectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); + + if (featurestoreConnectorFacade.findByFeaturestoreName(featurestore, featurestoreStorageConnectorDTO.getName()) + .isPresent()) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, + "Redshift connector with the same name already exists. Name=" + featurestoreStorageConnectorDTO.getName()); } - } - /** - * Updates an existing Storage Connector of a specific type in a feature store - * - * @param user the user making the request - * @param featurestore the featurestore where the connector exists - * @param featurestoreStorageConnectorType the type of the storage connector - * @param featurestoreStorageConnectorDTO the data to use when updating the storage connector - * @param storageConnectorName name of the connector - * @return A JSON/XML DTOs representation of the updated storage connector - */ - public FeaturestoreStorageConnectorDTO updateStorageConnectorWithType(Users user, Featurestore featurestore, - FeaturestoreStorageConnectorType featurestoreStorageConnectorType, - FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO, String storageConnectorName) - throws FeaturestoreException, UserException, ProjectException { - validateUser(user, featurestore); - switch(featurestoreStorageConnectorType) { + FeaturestoreConnector featurestoreConnector = new FeaturestoreConnector(); + verifyName(featurestoreStorageConnectorDTO); + featurestoreConnector.setName(featurestoreStorageConnectorDTO.getName()); + + verifyDescription(featurestoreStorageConnectorDTO); + featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription()); + featurestoreConnector.setFeaturestore(featurestore); + + switch (featurestoreStorageConnectorDTO.getStorageConnectorType()) { + case HOPSFS: + featurestoreConnector.setConnectorType(FeaturestoreConnectorType.HOPSFS); + featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.createFeaturestoreHopsfsConnector( + featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO)); + break; case S3: - return featurestoreS3ConnectorController.updateFeaturestoreS3Connector(user, featurestore, - (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO, storageConnectorName); + featurestoreConnector.setConnectorType(FeaturestoreConnectorType.S3); + featurestoreConnector.setS3Connector(s3ConnectorController.createFeaturestoreS3Connector( + user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO)); + break; case JDBC: - return featurestoreJdbcConnectorController.updateFeaturestoreJdbcConnector(featurestore, - (FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO, storageConnectorName); + featurestoreConnector.setConnectorType(FeaturestoreConnectorType.JDBC); + featurestoreConnector.setJdbcConnector(jdbcConnectorController.createFeaturestoreJdbcConnector( + (FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO)); + break; case REDSHIFT: - return featurestoreRedshiftConnectorController.updateFeaturestoreRedshiftConnector(user, featurestore, - (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO, storageConnectorName); - case HOPSFS: - return featurestoreHopsfsConnectorController.updateFeaturestoreHopsfsConnector(featurestore, - (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO, storageConnectorName); + featurestoreConnector.setConnectorType(FeaturestoreConnectorType.REDSHIFT); + featurestoreConnector.setRedshiftConnector(redshiftConnectorController.createFeaturestoreRedshiftConnector( + user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO)); + break; default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + featurestoreStorageConnectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); + // We should not reach this point + throw new IllegalArgumentException("Feature Store connector type not recognized"); } + + // Update object to populate id (auto-increment) information + featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector); + + activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + + featurestoreConnector.getName(), project, user, ActivityFlag.SERVICE); + + return convertToConnectorDTO(user, project, featurestoreConnector); } - /** - * Deletes a storage connector with a specific type and id in a feature store - * - * @param user the user making the request - * @param featurestoreStorageConnectorType the type of the storage connector - * @param storageConnectorId id of the storage connector - * @param featurestore - * @return JSON/XML DTOs of the deleted storage connector - */ - public void deleteStorageConnectorWithTypeAndId(Users user, - FeaturestoreStorageConnectorType featurestoreStorageConnectorType, Integer storageConnectorId, - Featurestore featurestore) throws FeaturestoreException, UserException, ProjectException { + @TransactionAttribute(TransactionAttributeType.REQUIRED) + @Transactional(rollbackOn = FeaturestoreException.class) + public void updateStorageConnector(Users user, Project project, Featurestore featurestore, + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO, String connectorName) + throws FeaturestoreException, UserException, ProjectException { validateUser(user, featurestore); - switch (featurestoreStorageConnectorType) { + + if (!connectorName.equalsIgnoreCase(featurestoreStorageConnectorDTO.getName())) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, + Level.FINE, "Can not update connector name."); + } + + FeaturestoreConnector featurestoreConnector = + featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName) + .orElseThrow(() -> + new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CONNECTOR_NOT_FOUND, Level.FINE, + "Cannot find storage connector with name: " + connectorName)); + + verifyDescription(featurestoreStorageConnectorDTO); + featurestoreConnector.setDescription(featurestoreStorageConnectorDTO.getDescription()); + + switch (featurestoreConnector.getConnectorType()) { + case HOPSFS: + featurestoreConnector.setHopsfsConnector(hopsfsConnectorController.updateFeaturestoreHopsfsConnector( + featurestore, (FeaturestoreHopsfsConnectorDTO) featurestoreStorageConnectorDTO, + featurestoreConnector.getHopsfsConnector())); + break; case S3: - featurestoreS3ConnectorController.removeFeaturestoreS3Connector(user, storageConnectorId); + featurestoreConnector.setS3Connector(s3ConnectorController.updateFeaturestoreS3Connector( + user, featurestore, (FeaturestoreS3ConnectorDTO) featurestoreStorageConnectorDTO, + featurestoreConnector.getS3Connector())); break; case JDBC: - featurestoreJdbcConnectorController.removeFeaturestoreJdbcConnector(storageConnectorId); + featurestoreConnector.setJdbcConnector(jdbcConnectorController.updateFeaturestoreJdbcConnector( + (FeaturestoreJdbcConnectorDTO) featurestoreStorageConnectorDTO, featurestoreConnector.getJdbcConnector())); break; case REDSHIFT: - featurestoreRedshiftConnectorController.removeFeaturestoreRedshiftConnector(user, storageConnectorId); - break; - case HOPSFS: - featurestoreHopsfsConnectorController.removeFeaturestoreHopsfsConnector(storageConnectorId); + featurestoreConnector.setRedshiftConnector(redshiftConnectorController.updateFeaturestoreRedshiftConnector( + user, featurestore, (FeaturestoreRedshiftConnectorDTO) featurestoreStorageConnectorDTO, + featurestoreConnector.getRedshiftConnector())); break; default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + featurestoreStorageConnectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); + // We should not reach this point + throw new IllegalArgumentException("Feature Store connector type not recognized"); + } + + featurestoreConnector = featurestoreConnectorFacade.update(featurestoreConnector); + + activityFacade.persistActivity( + ActivityFacade.UPDATED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), + project, user, ActivityFlag.SERVICE); + } + + // The transaction here is required otherwise when calling the remove the entity is not going to be managed anymore + @TransactionAttribute(TransactionAttributeType.REQUIRED) + public void deleteConnectorWithName(Users user, Project project, String connectorName, Featurestore featurestore) + throws UserException{ + validateUser(user, featurestore); + Optional featurestoreConnectorOptional = + featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName); + if (!featurestoreConnectorOptional.isPresent()) { + return; } + FeaturestoreConnector featurestoreConnector = featurestoreConnectorOptional.get(); + featurestoreConnectorFacade.remove(featurestoreConnector); + activityFacade.persistActivity( + ActivityFacade.REMOVED_FEATURESTORE_STORAGE_CONNECTOR + featurestoreConnector.getName(), + project, user, ActivityFlag.SERVICE); } - /** - * Gets the JDBC connector of the online featurestore for a particular user and project. This connector is different - * from other connectors in that it includes a password reference to the secretsmanager that needs to be resolved. - * - * @param user the user making the request - * @param dbUsername the database username - * @param featurestore the featurestore metadata - * @return a JDBC DTO connector for the online featurestore. - * @throws FeaturestoreException - */ - @TransactionAttribute(TransactionAttributeType.NEVER) - public FeaturestoreJdbcConnectorDTO getOnlineFeaturestoreConnector(Users user, String dbUsername, - Featurestore featurestore) throws FeaturestoreException { - String onlineFeaturestoreConnectorName = dbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX; - return (FeaturestoreJdbcConnectorDTO) this.getAllStorageConnectorsForFeaturestoreWithType(user, featurestore, - FeaturestoreStorageConnectorType.JDBC).stream().filter(dto -> dto.getName() - .equalsIgnoreCase(onlineFeaturestoreConnectorName)).findFirst().orElseThrow(() -> - new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ONLINE_FEATURESTORE_JDBC_CONNECTOR_NOT_FOUND, - Level.SEVERE, "Cannot get online featurestore JDBC connector")); + public FeaturestoreStorageConnectorDTO getOnlineFeaturestoreConnector(Users user, Project project, + Featurestore featurestore) + throws FeaturestoreException { + String dbUsername = onlineFeaturestoreController.onlineDbUsername(project, user); + Optional featurestoreConnector = featurestoreConnectorFacade + .findByFeaturestoreName(featurestore, + dbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX); + + if (featurestoreConnector.isPresent()) { + return convertToConnectorDTO(user, project, featurestoreConnector.get()); + } else { + return null; + } } /** @@ -288,28 +288,25 @@ private void validateUser(Users user, Featurestore featurestore) throws UserExce } } - public void deleteStorageConnectorWithTypeAndName(Users user, FeaturestoreStorageConnectorType connectorType, - String connectorName, Featurestore featurestore) throws FeaturestoreException, UserException, ProjectException { - validateUser(user, featurestore); - switch (connectorType) { - case S3: - featurestoreS3ConnectorController.removeFeaturestoreS3Connector(user, featurestore, connectorName); - break; - case JDBC: - featurestoreJdbcConnectorController.removeFeaturestoreJdbcConnector(connectorName, featurestore); - break; - case REDSHIFT: - featurestoreRedshiftConnectorController.removeFeaturestoreRedshiftConnector(user, connectorName, featurestore); - break; - case HOPSFS: - featurestoreHopsfsConnectorController.removeFeaturestoreHopsfsConnector(connectorName, featurestore); - break; - default: - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_TYPE, Level.FINE, - "Unrecognized storage connector type " + connectorType + - ", Recognized storage connector types are: " + FeaturestoreStorageConnectorType.HOPSFS + ", " + - FeaturestoreStorageConnectorType.REDSHIFT + ", " + FeaturestoreStorageConnectorType.S3 + ", and " + - FeaturestoreStorageConnectorType.JDBC); + public void verifyName(FeaturestoreStorageConnectorDTO connectorDTO) throws FeaturestoreException { + if (Strings.isNullOrEmpty(connectorDTO.getName())) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, + RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_NAME + ", the storage connector name cannot be empty"); + } + if (connectorDTO.getName().length() > FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, + RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_NAME + ", the name should be less than " + + FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH + " characters, the provided name was: " + + connectorDTO.getName()); + } + } + + public void verifyDescription(FeaturestoreStorageConnectorDTO connectorDTO) throws FeaturestoreException { + if (connectorDTO.getDescription() != null && + connectorDTO.getDescription().length() > FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, + RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_DESCRIPTION + ", the description should be less than: " + + FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH); } } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorDTO.java index 2caa8ce77f..dc0a2fe7d6 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorDTO.java @@ -19,20 +19,16 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.base.Strings; -import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.redshift.FeaturestoreRedshiftConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorDTO; -import io.hops.hopsworks.exceptions.FeaturestoreException; -import io.hops.hopsworks.restutils.RESTCodes; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; -import java.util.logging.Level; - /** * Abstract storage connector in the featurestore. Contains the common fields and functionality between different * types of storage connectors @@ -43,7 +39,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) @JsonSubTypes({ - @JsonSubTypes.Type(value = FeaturestoreHopsfsConnectorDTO.class, name = "FeaturestoreJobDTO"), + @JsonSubTypes.Type(value = FeaturestoreHopsfsConnectorDTO.class, name = "FeaturestoreHopsfsConnectorDTO"), @JsonSubTypes.Type(value = FeaturestoreJdbcConnectorDTO.class, name = "FeaturestoreJdbcConnectorDTO"), @JsonSubTypes.Type(value = FeaturestoreRedshiftConnectorDTO.class, name = "FeaturestoreRedshiftConnectorDTO"), @JsonSubTypes.Type(value = FeaturestoreS3ConnectorDTO.class, name = "FeaturestoreS3ConnectorDTO")} @@ -53,18 +49,17 @@ public class FeaturestoreStorageConnectorDTO { private String description; private String name; private Integer featurestoreId; - private FeaturestoreStorageConnectorType storageConnectorType; + private FeaturestoreConnectorType storageConnectorType; public FeaturestoreStorageConnectorDTO() { } - public FeaturestoreStorageConnectorDTO(Integer id, String description, String name, Integer featurestoreId, - FeaturestoreStorageConnectorType featurestoreStorageConnectorType) { - this.id = id; - this.description = description; - this.name = name; - this.featurestoreId = featurestoreId; - this.storageConnectorType = featurestoreStorageConnectorType; + public FeaturestoreStorageConnectorDTO(FeaturestoreConnector featurestoreConnector) { + this.id = featurestoreConnector.getId(); + this.description = featurestoreConnector.getDescription(); + this.name = featurestoreConnector.getName(); + this.featurestoreId = featurestoreConnector.getFeaturestore().getId(); + this.storageConnectorType = featurestoreConnector.getConnectorType(); } @XmlElement @@ -104,34 +99,14 @@ public void setFeaturestoreId(Integer featurestoreId) { } @XmlElement - public FeaturestoreStorageConnectorType getStorageConnectorType() { + public FeaturestoreConnectorType getStorageConnectorType() { return storageConnectorType; } - public void setStorageConnectorType(FeaturestoreStorageConnectorType storageConnectorType) { + public void setStorageConnectorType(FeaturestoreConnectorType storageConnectorType) { this.storageConnectorType = storageConnectorType; } - public void verifyName() throws FeaturestoreException { - if (Strings.isNullOrEmpty(name)) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, - RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_NAME + ", the storage connector name cannot be empty"); - } - if (name.length() > FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, - RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_NAME + ", the name should be less than " + - FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH + " characters, the provided name was: " + name); - } - } - - public void verifyDescription() throws FeaturestoreException { - if (description != null && description.length() > FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, - RESTCodes.FeaturestoreErrorCode.ILLEGAL_FEATURE_DESCRIPTION + ", the description should be less than: " + - FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH); - } - } - @Override public String toString() { return "FeaturestoreStorageConnectorDTO{" + diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorController.java index 954d25e63b..5878cb0243 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorController.java @@ -18,13 +18,11 @@ import com.google.common.base.Strings; import io.hops.hopsworks.common.dataset.DatasetController; -import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.hdfs.inode.InodeController; -import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.persistence.entity.dataset.Dataset; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; import io.hops.hopsworks.restutils.RESTCodes; @@ -32,10 +30,8 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import java.util.List; -import java.util.Optional; +import javax.transaction.Transactional; import java.util.logging.Level; -import java.util.stream.Collectors; /** * Class controlling the interaction with the feature_store_hopsfs table and required business logic @@ -43,274 +39,63 @@ @Stateless @TransactionAttribute(TransactionAttributeType.NEVER) public class FeaturestoreHopsfsConnectorController { - @EJB - private FeaturestoreHopsfsConnectorFacade featurestoreHopsfsConnectorFacade; @EJB private InodeController inodeController; @EJB private DatasetController datasetController; - + /** * Creates a HOPSFS storage connector for a feature store * - * @param featurestore the featurestore * @param featurestoreHopsfsConnectorDTO the input data to use when creating the connector * @returns a DTO representing the created entity * @throws FeaturestoreException */ - public FeaturestoreHopsfsConnectorDTO createFeaturestoreHopsfsConnector( + public FeaturestoreHopsfsConnector createFeaturestoreHopsfsConnector( Featurestore featurestore, FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO) - throws FeaturestoreException { - verifyUserInput(featurestore, featurestoreHopsfsConnectorDTO); - Dataset dataset = datasetController.getByProjectAndDsName(featurestore.getProject(), - null, featurestoreHopsfsConnectorDTO.getDatasetName()); + throws FeaturestoreException { + Dataset dataset = + verifyHopsfsConnectorDatasetName(featurestoreHopsfsConnectorDTO.getDatasetName(), featurestore); + FeaturestoreHopsfsConnector featurestoreHopsfsConnector = new FeaturestoreHopsfsConnector(); - featurestoreHopsfsConnector.setName(featurestoreHopsfsConnectorDTO.getName()); - featurestoreHopsfsConnector.setDescription(featurestoreHopsfsConnectorDTO.getDescription()); featurestoreHopsfsConnector.setHopsfsDataset(dataset); - featurestoreHopsfsConnector.setFeaturestore(featurestore); - featurestoreHopsfsConnectorFacade.persist(featurestoreHopsfsConnector); - return convertHopsfsConnectorToDTO(featurestoreHopsfsConnector); + + return featurestoreHopsfsConnector; } - /** - * Updates a HOPSFS storage connector for a feature store - * - * @param featurestore the featurestore - * @param featurestoreHopsfsConnectorDTO the input data to use when updating the connector - * @param storageConnectorName name of the storage connector to update - * @returns a DTO representing the updated entity - * @throws FeaturestoreException FeaturestoreException - */ - public FeaturestoreHopsfsConnectorDTO updateFeaturestoreHopsfsConnector(Featurestore featurestore, - FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO, String storageConnectorName) - throws FeaturestoreException { - FeaturestoreHopsfsConnector featurestoreHopsfsConnector = verifyHopsfStorageConnectorName(featurestore, - storageConnectorName); + @TransactionAttribute(TransactionAttributeType.REQUIRED) + @Transactional(rollbackOn = FeaturestoreException.class) + public FeaturestoreHopsfsConnector updateFeaturestoreHopsfsConnector(Featurestore featurestore, + FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO, + FeaturestoreHopsfsConnector featurestoreHopsfsConnector) + throws FeaturestoreException { if(!Strings.isNullOrEmpty(featurestoreHopsfsConnectorDTO.getDatasetName())){ - verifyHopsfsConnectorDatasetName(featurestoreHopsfsConnectorDTO.getDatasetName(), featurestore); - Dataset dataset = datasetController.getByProjectAndDsName(featurestore.getProject(), - null, featurestoreHopsfsConnectorDTO.getDatasetName()); + Dataset dataset = + verifyHopsfsConnectorDatasetName(featurestoreHopsfsConnectorDTO.getDatasetName(), featurestore); featurestoreHopsfsConnector.setHopsfsDataset(dataset); } - if(!Strings.isNullOrEmpty(featurestoreHopsfsConnectorDTO.getName())){ - verifyHopsfsConnectorName(featurestoreHopsfsConnectorDTO.getName(), featurestore, true); - featurestoreHopsfsConnector.setName(featurestoreHopsfsConnectorDTO.getName()); - } - if(!Strings.isNullOrEmpty(featurestoreHopsfsConnectorDTO.getDescription())){ - verifyHopsfsConnectorDescription(featurestoreHopsfsConnectorDTO.getDescription()); - featurestoreHopsfsConnector.setDescription(featurestoreHopsfsConnectorDTO.getDescription()); - } - if(featurestore != null) { - featurestoreHopsfsConnector.setFeaturestore(featurestore); - } - FeaturestoreHopsfsConnector updatedFeaturestoreHopsfsConnector = - featurestoreHopsfsConnectorFacade.updateHopsfsConnector(featurestoreHopsfsConnector); - return convertHopsfsConnectorToDTO(updatedFeaturestoreHopsfsConnector); - } - - /** - * Creates a default HOPSFS storage backend for storing training datasets - * - * @param featurestore the featurestore - * @param hopsfsDataset the HOPSFS dataset - * @throws FeaturestoreException - */ - public void createHopsFsBackendForFeaturestoreConnector(Featurestore featurestore, Dataset hopsfsDataset) - throws FeaturestoreException { - String name = hopsfsDataset.getName(); - String description = "HOPSFS backend for storing Training Datasets of the Hopsworks Feature Store"; - FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO = new FeaturestoreHopsfsConnectorDTO(); - featurestoreHopsfsConnectorDTO.setName(name); - featurestoreHopsfsConnectorDTO.setDescription(description); - featurestoreHopsfsConnectorDTO.setDatasetName(hopsfsDataset.getName()); - createFeaturestoreHopsfsConnector(featurestore, featurestoreHopsfsConnectorDTO); - } - - /** - * Removes a HOPSFS storage backend with a particular Id - * - * @param featurestoreHopsfsId the id - * @returns DTO of the deleted entity - */ - public void removeFeaturestoreHopsfsConnector(Integer featurestoreHopsfsId){ - FeaturestoreHopsfsConnector featurestoreHopsfsConnector = - featurestoreHopsfsConnectorFacade.find(featurestoreHopsfsId); - if (featurestoreHopsfsConnector != null) { - featurestoreHopsfsConnectorFacade.remove(featurestoreHopsfsConnector); - } - } - - /** - * Removes a HOPSFS storage backend with a particular by name - * @param featurestoreHopsfsName - * @param featurestore - */ - public void removeFeaturestoreHopsfsConnector(String featurestoreHopsfsName, Featurestore featurestore){ - Optional featurestoreHopsfsConnector = - featurestoreHopsfsConnectorFacade.findByNameAndFeaturestore(featurestoreHopsfsName, featurestore); - if (featurestoreHopsfsConnector.isPresent()) { - featurestoreHopsfsConnectorFacade.remove(featurestoreHopsfsConnector.get()); - } - } - private FeaturestoreHopsfsConnector verifyHopsfStorageConnectorName(Featurestore featurestore, - String storageConnectorName) throws FeaturestoreException { - return featurestoreHopsfsConnectorFacade.findByNameAndFeaturestore(storageConnectorName, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, - Level.FINE, "HopsFs Connector not found HopsFsConnector name: " + storageConnectorName)); + return featurestoreHopsfsConnector; } - /** - * Verify user input name - * - * @param name the user input to verify - * @param featurestore the featurestore to query - * @param edit boolean flag whether the validation if for updating an existing connector or creating a new one - * @throws FeaturestoreException - */ - private void verifyHopsfsConnectorName(String name, Featurestore featurestore, Boolean edit) - throws FeaturestoreException { - if (Strings.isNullOrEmpty(name)) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, - Level.FINE, "Illegal storage connector name, the storage connector name cannot be empty"); - } - - if(name.length() > - FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - "Illegal storage connector name, the name should be less than " + - FeaturestoreConstants.STORAGE_CONNECTOR_NAME_MAX_LENGTH + " characters."); - } - - if(!edit){ - if(featurestore.getHopsfsConnections().stream() - .anyMatch(hopsfsCon -> hopsfsCon.getName().equalsIgnoreCase(name))) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - "Illegal storage connector name, the storage connector name should be unique, there already exists a " + - "HOPSFS connector with the same name "); - } - } - } - - /** - * Verify user featurestore - * - * @param featurestore the user input to verify - */ - private void verifyFeaturestoreInput(Featurestore featurestore){ - if (featurestore == null) { - throw new IllegalArgumentException("Featurestore was not found"); - } - } - - /** - * Verify user input description - * - * @param description the user input to verify - * @throws FeaturestoreException - */ - private void verifyHopsfsConnectorDescription(String description) throws FeaturestoreException { - if(description.length() > - FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_DESCRIPTION, Level.FINE, - "Illegal storage connector description, the description should be less than: " - + FeaturestoreConstants.STORAGE_CONNECTOR_DESCRIPTION_MAX_LENGTH); - } - } - - /** - * Verify user input dataset name - * - * @param datasetName the user input to verify - * @param featurestore the featurestore to query - * @throws FeaturestoreException - */ - private void verifyHopsfsConnectorDatasetName(String datasetName, Featurestore featurestore) - throws FeaturestoreException { + private Dataset verifyHopsfsConnectorDatasetName(String datasetName, Featurestore featurestore) + throws FeaturestoreException { Dataset dataset = datasetController.getByProjectAndDsName(featurestore.getProject(), null, datasetName); + if (dataset == null) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_HOPSFS_CONNECTOR_DATASET, Level.FINE, - "Illegal Hopsfs connector dataset, the dataset could not be found"); - } - } - - /** - * Validates user input for creating a new HOPSFS connector in a featurestore - * - * @param featurestore the featurestore - * @param featurestoreHopsfsConnectorDTO the input data to use when creating the connector - * @throws FeaturestoreException - */ - private void verifyUserInput(Featurestore featurestore, FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO) - throws FeaturestoreException { - if (featurestoreHopsfsConnectorDTO == null) { - throw new IllegalArgumentException("Input data is null"); + datasetName + " could not be found in project " + featurestore.getProject().getName()); } - verifyFeaturestoreInput(featurestore); - verifyHopsfsConnectorName(featurestoreHopsfsConnectorDTO.getName(), featurestore, false); - verifyHopsfsConnectorDescription(featurestoreHopsfsConnectorDTO.getDescription()); - verifyHopsfsConnectorDatasetName(featurestoreHopsfsConnectorDTO.getDatasetName(), featurestore); - } - /** - * Gets all HOPSFS connectors for a particular featurestore and project - * - * @param featurestore featurestore to query for hopsfs connectors - * @return list of XML/JSON DTOs of the hopsfs connectors - */ - public List getHopsfsConnectors(Featurestore featurestore) { - List hopsfsConnectors = - featurestoreHopsfsConnectorFacade.findByFeaturestore(featurestore); - return hopsfsConnectors.stream().map(hopsfsConnector -> (FeaturestoreStorageConnectorDTO) - convertHopsfsConnectorToDTO(hopsfsConnector)) - .collect(Collectors.toList()); - } - - /** - * - * @param featurestore - * @param storageConnectorName - * @return - * @throws FeaturestoreException - */ - public FeaturestoreStorageConnectorDTO getHopsFsConnectorWithNameAndFeaturestore(Featurestore featurestore, - String storageConnectorName) throws FeaturestoreException { - FeaturestoreHopsfsConnector featurestoreHopsfsConnector = - verifyHopsfStorageConnectorName(featurestore, storageConnectorName); - return convertHopsfsConnectorToDTO(featurestoreHopsfsConnector); + return dataset; } - /** - * Get the default storage connector for the feature store. The default storage connector is the HopsFS one that - * points to the TRAINING_DATASET dataset. - * @param featurestore - * @return - * @throws FeaturestoreException - */ - public FeaturestoreHopsfsConnector getDefaultStorageConnector(Featurestore featurestore) - throws FeaturestoreException { - String connectorName = featurestore.getProject().getName() + "_" + - Settings.ServiceDataset.TRAININGDATASETS.getName(); - return featurestoreHopsfsConnectorFacade.findByNameAndFeaturestore(connectorName, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, - Level.FINE, "Could not find default storage connector: " + connectorName)); - } - - /** - * Convert a FeaturestoreHopsfsConnector entity to a DTO - * - * @param featurestoreHopsfsConnector the entity to convert to DTO - * @return a DTO representation of the entity - */ - private FeaturestoreHopsfsConnectorDTO convertHopsfsConnectorToDTO( - FeaturestoreHopsfsConnector featurestoreHopsfsConnector) { + public FeaturestoreHopsfsConnectorDTO getHopsfsConnectorDTO(FeaturestoreConnector featurestoreConnector) { FeaturestoreHopsfsConnectorDTO featurestoreHopsfsConnectorDTO = new - FeaturestoreHopsfsConnectorDTO(featurestoreHopsfsConnector); - featurestoreHopsfsConnectorDTO.setHopsfsPath(inodeController.getPath( - featurestoreHopsfsConnector.getHopsfsDataset().getInode())); + FeaturestoreHopsfsConnectorDTO(featurestoreConnector); + featurestoreHopsfsConnectorDTO.setHopsfsPath( + inodeController.getPath(featurestoreConnector.getHopsfsConnector().getHopsfsDataset().getInode())); return featurestoreHopsfsConnectorDTO; } - } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorDTO.java index d179e6e79c..6e38633543 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorDTO.java @@ -16,9 +16,8 @@ package io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -34,22 +33,19 @@ public class FeaturestoreHopsfsConnectorDTO extends FeaturestoreStorageConnector private String datasetName; public FeaturestoreHopsfsConnectorDTO() { - super(null, null, null, null, null); } - - public FeaturestoreHopsfsConnectorDTO(FeaturestoreHopsfsConnector featurestoreHopsfsConnector) { - super(featurestoreHopsfsConnector.getId(), featurestoreHopsfsConnector.getDescription(), - featurestoreHopsfsConnector.getName(), featurestoreHopsfsConnector.getFeaturestore().getId(), - FeaturestoreStorageConnectorType.HOPSFS); + + public FeaturestoreHopsfsConnectorDTO(FeaturestoreConnector featurestoreConnector) { + super(featurestoreConnector); this.hopsfsPath = null; - this.datasetName = featurestoreHopsfsConnector.getHopsfsDataset().getName(); + this.datasetName = featurestoreConnector.getHopsfsConnector().getHopsfsDataset().getName(); } @XmlElement public String getHopsfsPath() { return hopsfsPath; } - + public void setHopsfsPath(String hopsfsPath) { this.hopsfsPath = hopsfsPath; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorFacade.java deleted file mode 100644 index b8276dd8f9..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/hopsfs/FeaturestoreHopsfsConnectorFacade.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2019, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ - -package io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs; - -import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; - -import javax.ejb.Stateless; -import javax.ejb.TransactionAttribute; -import javax.ejb.TransactionAttributeType; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; -import javax.validation.ConstraintViolationException; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A facade for the feature_store_hopsfs_connector table in the Hopsworks database, - * use this interface when performing database operations against the table. - */ -@Stateless -public class FeaturestoreHopsfsConnectorFacade extends AbstractFacade { - private static final Logger LOGGER = Logger.getLogger( - FeaturestoreHopsfsConnectorFacade.class.getName()); - @PersistenceContext(unitName = "kthfsPU") - private EntityManager em; - - public FeaturestoreHopsfsConnectorFacade() { - super(FeaturestoreHopsfsConnector.class); - } - - /** - * A transaction to persist a hopsfs connection for the featurestore in the database - * - * @param featurestoreHopsfsConnector the featurestore hopsfs connection to persist - */ - @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void persist(FeaturestoreHopsfsConnector featurestoreHopsfsConnector) { - try { - em.persist(featurestoreHopsfsConnector); - em.flush(); - } catch (ConstraintViolationException cve) { - LOGGER.log(Level.WARNING, "Could not persist the new HOPSFS connector", cve); - throw cve; - } - } - - /** - * Retrieves all hopsfs connectors for a particular featurestore - * - * @param featurestore featurestore get hopsfs connectors for - * @return list of hopsfs connectors - */ - public List findByFeaturestore(Featurestore featurestore) { - TypedQuery q = em.createNamedQuery("FeaturestoreHopsfsConnector.findByFeaturestore", - FeaturestoreHopsfsConnector.class).setParameter("featurestore", featurestore); - return q.getResultList(); - } - - /** - * Retrieves a particular hopsfs connector given its id - * - * @param id id of the hopsfs connector - * @return a single FeaturestoreHopsfsConnector entity - */ - public Optional findByIdAndFeaturestore(Integer id, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreHopsfsConnector.findByFeaturestoreAndId", - FeaturestoreHopsfsConnector.class) - .setParameter("id", id) - .setParameter("featurestore", featurestore) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - /** - * Retrive a hopsfs storage connector give its name and feature store - * @param name - * @param featurestore - * @return - */ - public Optional findByNameAndFeaturestore(String name, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreHopsfsConnector.findByNameAndFeaturestore", - FeaturestoreHopsfsConnector.class) - .setParameter("name", name) - .setParameter("featurestore", featurestore) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - /** - * Updates a hopsfs connector - * - * @param featurestoreHopsfsConnector the hopsfs connector to update - * @return the updated connector - */ - public FeaturestoreHopsfsConnector updateHopsfsConnector( - FeaturestoreHopsfsConnector featurestoreHopsfsConnector) { - em.merge(featurestoreHopsfsConnector); - return featurestoreHopsfsConnector; - } - - /** - * Gets the entity manager of the facade - * - * @return entity manager - */ - @Override - protected EntityManager getEntityManager() { - return em; - } - -} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorController.java index e1c3f1eb87..06cedf8a38 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorController.java @@ -21,16 +21,13 @@ import io.hops.hopsworks.common.dao.user.security.secrets.SecretPlaintext; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; -import io.hops.hopsworks.common.hive.HiveController; import io.hops.hopsworks.common.hosts.ServiceDiscoveryController; import io.hops.hopsworks.common.security.secrets.SecretsController; -import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.UserException; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; +import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.restutils.RESTCodes; @@ -38,9 +35,7 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import javax.transaction.Transactional; import java.util.logging.Level; import java.util.logging.Logger; @@ -51,67 +46,32 @@ @TransactionAttribute(TransactionAttributeType.NEVER) public class FeaturestoreJdbcConnectorController { - @EJB - private FeaturestoreJdbcConnectorFacade featurestoreJdbcConnectorFacade; @EJB private ServiceDiscoveryController serviceDiscoveryController; @EJB - private HiveController hiveController; - @EJB - private Settings settings; - @EJB private SecretsController secretsController; @EJB private OnlineFeaturestoreController onlineFeaturestoreController; private static final Logger LOGGER = Logger.getLogger(FeaturestoreJdbcConnectorController.class.getName()); - - /** - * Persists a JDBC connection for the feature store - * - * @param featurestore the feature store - * @param featurestoreJdbcConnectorDTO input to data to use when creating the storage connector - * @return a DTO representing the created entity - * @throws FeaturestoreException - */ - public FeaturestoreJdbcConnectorDTO createFeaturestoreJdbcConnector( - Featurestore featurestore, FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO) - throws FeaturestoreException { - verifyUserInput(featurestore, featurestoreJdbcConnectorDTO); + + public FeaturestoreJdbcConnector createFeaturestoreJdbcConnector( + FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO) + throws FeaturestoreException { + verifyJdbcConnectorConnectionString(featurestoreJdbcConnectorDTO.getConnectionString()); + verifyJdbcConnectorArguments(featurestoreJdbcConnectorDTO.getArguments()); + FeaturestoreJdbcConnector featurestoreJdbcConnector = new FeaturestoreJdbcConnector(); - featurestoreJdbcConnector.setName(featurestoreJdbcConnectorDTO.getName()); - featurestoreJdbcConnector.setDescription(featurestoreJdbcConnectorDTO.getDescription()); - featurestoreJdbcConnector.setFeaturestore(featurestore); featurestoreJdbcConnector.setArguments(featurestoreJdbcConnectorDTO.getArguments()); featurestoreJdbcConnector.setConnectionString(featurestoreJdbcConnectorDTO.getConnectionString()); - featurestoreJdbcConnectorFacade.persist(featurestoreJdbcConnector); - return new FeaturestoreJdbcConnectorDTO(featurestoreJdbcConnector); + return featurestoreJdbcConnector; } - /** - * Updates a JDBC connection for the feature store - * - * @param featurestore the feature store - * @param featurestoreJdbcConnectorDTO input to data to use when updating the storage connector - * @param storageConnectorName the name of the connector - * @return a DTO representing the updated entity - * @throws FeaturestoreException - */ - public FeaturestoreJdbcConnectorDTO updateFeaturestoreJdbcConnector( - Featurestore featurestore, FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO, - String storageConnectorName) throws FeaturestoreException { - - FeaturestoreJdbcConnector featurestoreJdbcConnector = verifyJdbcConnectorName(storageConnectorName, featurestore); - - if(!Strings.isNullOrEmpty(featurestoreJdbcConnectorDTO.getName())) { - verifyJdbcConnectorName(featurestoreJdbcConnectorDTO, featurestore, true); - featurestoreJdbcConnector.setName(featurestoreJdbcConnectorDTO.getName()); - } - - if(!Strings.isNullOrEmpty(featurestoreJdbcConnectorDTO.getDescription())) { - featurestoreJdbcConnectorDTO.verifyDescription(); - featurestoreJdbcConnector.setDescription(featurestoreJdbcConnectorDTO.getDescription()); - } + @TransactionAttribute(TransactionAttributeType.REQUIRED) + @Transactional(rollbackOn = FeaturestoreException.class) + public FeaturestoreJdbcConnector updateFeaturestoreJdbcConnector( + FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO, + FeaturestoreJdbcConnector featurestoreJdbcConnector) throws FeaturestoreException { if(!Strings.isNullOrEmpty(featurestoreJdbcConnectorDTO.getConnectionString())) { verifyJdbcConnectorConnectionString(featurestoreJdbcConnectorDTO.getConnectionString()); @@ -123,86 +83,7 @@ public FeaturestoreJdbcConnectorDTO updateFeaturestoreJdbcConnector( featurestoreJdbcConnector.setArguments(featurestoreJdbcConnectorDTO.getArguments()); } - if(featurestore != null) { - featurestoreJdbcConnector.setFeaturestore(featurestore); - } - - FeaturestoreJdbcConnector updatedConnector = - featurestoreJdbcConnectorFacade.updateJdbcConnector(featurestoreJdbcConnector); - return new FeaturestoreJdbcConnectorDTO(updatedConnector); - } - - /** - * Utility function for creating default JDBC connector for the offline Featurestore-project in Hopsworks - * - * @param featurestore the featurestore - * @param databaseName name of the Hive database - * @throws FeaturestoreException - */ - public void createDefaultJdbcConnectorForOfflineFeaturestore(Featurestore featurestore, String databaseName, - String description) throws FeaturestoreException { - try { - String hiveEndpoint = hiveController.getHiveServerInternalEndpoint(); - String connectionString = "jdbc:hive2://" + hiveEndpoint + "/" + databaseName + ";" + - "auth=noSasl;ssl=true;twoWay=true;"; - String arguments = "sslTrustStore,trustStorePassword,sslKeyStore,keyStorePassword"; - FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO(); - featurestoreJdbcConnectorDTO.setName(databaseName); - featurestoreJdbcConnectorDTO.setDescription(description); - featurestoreJdbcConnectorDTO.setConnectionString(connectionString); - featurestoreJdbcConnectorDTO.setArguments(arguments); - createFeaturestoreJdbcConnector(featurestore, featurestoreJdbcConnectorDTO); - } catch (ServiceDiscoveryException ex) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.JDBC_CONNECTOR_NOT_FOUND, - Level.SEVERE, "Could not create Hive connection string", - ex.getMessage(), ex); - } - } - - /** - * Removes a JDBC connection from the database with a particular id - * - * @param featurestoreJdbcId id of the JDBC connection - * @return DTO of the deleted entity - */ - public void removeFeaturestoreJdbcConnector(Integer featurestoreJdbcId){ - FeaturestoreJdbcConnector featurestoreJdbcConnector = featurestoreJdbcConnectorFacade.find(featurestoreJdbcId); - if (featurestoreJdbcConnector != null) { - featurestoreJdbcConnectorFacade.remove(featurestoreJdbcConnector); - } - } - - public void removeFeaturestoreJdbcConnector(String featurestoreJdbcName, Featurestore featurestore){ - Optional featurestoreJdbcConnector = - featurestoreJdbcConnectorFacade.findByNameAndFeaturestore(featurestoreJdbcName, featurestore); - featurestoreJdbcConnector.ifPresent(jdbcConnector -> featurestoreJdbcConnectorFacade.remove(jdbcConnector)); - } - - private FeaturestoreJdbcConnector verifyJdbcConnectorName(String jdbcConnectorName, Featurestore featurestore) - throws FeaturestoreException { - return featurestoreJdbcConnectorFacade.findByNameAndFeaturestore(jdbcConnectorName, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.JDBC_CONNECTOR_NOT_FOUND, - Level.FINE, "jdbcConnector name: " + jdbcConnectorName)); - } - - /** - * Verifies user input connector name string - * - * @param featurestoreJdbcConnectorDTO the user input to validate - * @param featurestore the featurestore to query - * @param edit boolean flag whether the validation if for updating an existing connector or creating a new one - * @throws FeaturestoreException - */ - private void verifyJdbcConnectorName(FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO, - Featurestore featurestore, Boolean edit) throws FeaturestoreException { - featurestoreJdbcConnectorDTO.verifyName(); - if (!edit) { - if (featurestoreJdbcConnectorFacade - .findByNameAndFeaturestore(featurestoreJdbcConnectorDTO.getName(), featurestore).isPresent()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - ", the storage connector name should be unique, there already exists a JDBC connector with the same name "); - } - } + return featurestoreJdbcConnector; } /** @@ -212,9 +93,8 @@ private void verifyJdbcConnectorName(FeaturestoreJdbcConnectorDTO featurestoreJd * @throws FeaturestoreException */ private void verifyJdbcConnectorConnectionString(String connectionString) throws FeaturestoreException { - if(Strings.isNullOrEmpty(connectionString) - || connectionString.length() - > FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_CONNECTIONSTRING_MAX_LENGTH) { + if(Strings.isNullOrEmpty(connectionString) || + connectionString.length() > FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_CONNECTIONSTRING_MAX_LENGTH) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_JDBC_CONNECTION_STRING, Level.FINE, ", the JDBC connection string should not be empty and not exceed: " + FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_CONNECTIONSTRING_MAX_LENGTH + " characters"); @@ -228,105 +108,27 @@ private void verifyJdbcConnectorConnectionString(String connectionString) throws * @throws FeaturestoreException */ private void verifyJdbcConnectorArguments(String arguments) throws FeaturestoreException { - if (!Strings.isNullOrEmpty(arguments) - && arguments.length() > FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_ARGUMENTS_MAX_LENGTH) { + if(!Strings.isNullOrEmpty(arguments) + && arguments.length() > FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_ARGUMENTS_MAX_LENGTH) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_JDBC_CONNECTION_ARGUMENTS, Level.FINE, - "Illegal JDBC Connection Arguments, the JDBC connection arguments should not exceed: " + - FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_ARGUMENTS_MAX_LENGTH + " characters"); + "JDBC connection arguments should not exceed: " + + FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_ARGUMENTS_MAX_LENGTH + " characters"); } } - - /** - * Validates user input for creating a new JDBC connector in a featurestore - * - * @param featurestore the featurestore - * @param featurestoreJdbcConnectorDTO input to data to use when creating the storage connector - * @throws FeaturestoreException - */ - private void verifyUserInput(Featurestore featurestore, FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO) - throws FeaturestoreException { - if(featurestoreJdbcConnectorDTO == null){ - throw new IllegalArgumentException("Input data is null"); - } - verifyJdbcConnectorName(featurestoreJdbcConnectorDTO, featurestore, false); - featurestoreJdbcConnectorDTO.verifyDescription(); - verifyJdbcConnectorConnectionString(featurestoreJdbcConnectorDTO.getConnectionString()); - verifyJdbcConnectorArguments(featurestoreJdbcConnectorDTO.getArguments()); - } - /** - * Gets all JDBC connectors for a particular featurestore and project - * - * @param user - * @param featurestore featurestore to query for jdbc connectors - * @return list of XML/JSON DTOs of the jdbc connectors - */ - public List getJdbcConnectorsForFeaturestore(Users user, Featurestore featurestore) + public FeaturestoreJdbcConnectorDTO getJdbcConnectorDTO(Users user, Project project, + FeaturestoreConnector featurestoreConnector) throws FeaturestoreException { - List jdbcConnectorDTOs = new ArrayList<>(); - List jdbcConnectors = featurestoreJdbcConnectorFacade.findByFeaturestore(featurestore); - - for (FeaturestoreJdbcConnector jdbcConnector : jdbcConnectors) { - FeaturestoreJdbcConnectorDTO jdbcConnectorDTO = - replaceOnlineFsConnectorUrl(new FeaturestoreJdbcConnectorDTO(jdbcConnector)); - - String userOnlineConnectorName = onlineFeaturestoreController.onlineDbUsername(featurestore.getProject(), user) - + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX; - if (jdbcConnectorDTO.getName().equals(userOnlineConnectorName)) { - setPasswordPlainTextForOnlineJdbcConnector(user, jdbcConnectorDTO, featurestore.getProject().getName()); - } - - jdbcConnectorDTOs.add(jdbcConnectorDTO); - } - - return jdbcConnectorDTOs; - } - - private FeaturestoreJdbcConnectorDTO getJdbcConnectorDTO(Users user, Featurestore featurestore, - FeaturestoreJdbcConnector featurestoreJdbcConnector) throws FeaturestoreException { FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = - new FeaturestoreJdbcConnectorDTO(featurestoreJdbcConnector); - if(featurestoreJdbcConnectorDTO.getName().equals(onlineFeaturestoreController.onlineDbUsername( - featurestore.getProject(), user) + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX)) { - setPasswordPlainTextForOnlineJdbcConnector(user, featurestoreJdbcConnectorDTO, - featurestore.getProject().getName()); + new FeaturestoreJdbcConnectorDTO(featurestoreConnector); + if(featurestoreJdbcConnectorDTO.getName() + .equals(onlineFeaturestoreController.onlineDbUsername(project, user) + + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX)) { + setPasswordPlainTextForOnlineJdbcConnector(user, featurestoreJdbcConnectorDTO, project.getName()); } - return replaceOnlineFsConnectorUrl(new FeaturestoreJdbcConnectorDTO(featurestoreJdbcConnector)); + return replaceOnlineFsConnectorUrl(featurestoreJdbcConnectorDTO); } - - /** - * Utility function for create a JDBC connection to the online featurestore for a particular user. - * - * @param onlineDbUsername the db-username of the connection - * @param featurestore the featurestore metadata - * @param dbName name of the MySQL database - * @return DTO of the newly created connector - * @throws FeaturestoreException - */ - public FeaturestoreJdbcConnectorDTO createJdbcConnectorForOnlineFeaturestore(String onlineDbUsername, - Featurestore featurestore, String dbName) throws FeaturestoreException { - String connectorName = onlineDbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX; - if (featurestoreJdbcConnectorFacade.findByNameAndFeaturestore(connectorName, featurestore).isPresent()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - "Illegal storage connector name, a storage connector with that name already exists"); - } - FeaturestoreJdbcConnectorDTO featurestoreJdbcConnectorDTO = new FeaturestoreJdbcConnectorDTO(); - featurestoreJdbcConnectorDTO - .setConnectionString(settings.getFeaturestoreJdbcUrl() + dbName + "?useSSL=false&allowPublicKeyRetrieval=true"); - featurestoreJdbcConnectorDTO.setDescription("JDBC connection to Hopsworks Project Online " + - "Feature Store NDB Database for user: " + onlineDbUsername); - featurestoreJdbcConnectorDTO.setArguments( - FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_PASSWORD_ARG + "=" + - FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_PASSWORD_TEMPLATE + "," + - FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_USER_ARG + "=" + onlineDbUsername + - ",isolationLevel=NONE,batchsize=500"); - featurestoreJdbcConnectorDTO.setStorageConnectorType(FeaturestoreStorageConnectorType.JDBC); - featurestoreJdbcConnectorDTO.setName(connectorName); - featurestoreJdbcConnectorDTO.setFeaturestoreId(featurestore.getId()); - return createFeaturestoreJdbcConnector(featurestore, featurestoreJdbcConnectorDTO); - } - /** * Gets the plain text password for the connector from the secret * @param user @@ -382,18 +184,4 @@ private FeaturestoreJdbcConnectorDTO replaceOnlineFsConnectorUrl(FeaturestoreJdb jdbcConnectorDTO.setConnectionString(connectionString); return jdbcConnectorDTO; } - - /** - * - * @param user - * @param featurestore - * @param storageConnectorName - * @return - * @throws FeaturestoreException - */ - public FeaturestoreStorageConnectorDTO getJdbcConnectorWithNameAndFeaturestore(Users user, Featurestore featurestore, - String storageConnectorName) throws FeaturestoreException { - FeaturestoreJdbcConnector featurestoreJdbcConnector = verifyJdbcConnectorName(storageConnectorName, featurestore); - return getJdbcConnectorDTO(user, featurestore, featurestoreJdbcConnector); - } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorDTO.java index af40258081..77d997b597 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorDTO.java @@ -16,9 +16,8 @@ package io.hops.hopsworks.common.featurestore.storageconnectors.jdbc; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -34,15 +33,12 @@ public class FeaturestoreJdbcConnectorDTO extends FeaturestoreStorageConnectorDT private String arguments; public FeaturestoreJdbcConnectorDTO() { - super(null, null, null, null, null); } - public FeaturestoreJdbcConnectorDTO(FeaturestoreJdbcConnector featurestoreJdbcConnector) { - super(featurestoreJdbcConnector.getId(), featurestoreJdbcConnector.getDescription(), - featurestoreJdbcConnector.getName(), featurestoreJdbcConnector.getFeaturestore().getId(), - FeaturestoreStorageConnectorType.JDBC); - this.connectionString = featurestoreJdbcConnector.getConnectionString(); - this.arguments = featurestoreJdbcConnector.getArguments(); + public FeaturestoreJdbcConnectorDTO(FeaturestoreConnector featurestoreConnector) { + super(featurestoreConnector); + this.connectionString = featurestoreConnector.getJdbcConnector().getConnectionString(); + this.arguments = featurestoreConnector.getJdbcConnector().getArguments(); } @XmlElement diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorFacade.java deleted file mode 100644 index 26bcfefa8c..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorFacade.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2019, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ - -package io.hops.hopsworks.common.featurestore.storageconnectors.jdbc; - -import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; - -import javax.ejb.Stateless; -import javax.ejb.TransactionAttribute; -import javax.ejb.TransactionAttributeType; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; -import javax.validation.ConstraintViolationException; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A facade for the feature_store_jdbc_connector table in the Hopsworks database, - * use this interface when performing database operations against the table. - */ -@Stateless -public class FeaturestoreJdbcConnectorFacade extends AbstractFacade { - private static final Logger LOGGER = Logger.getLogger( - FeaturestoreJdbcConnectorFacade.class.getName()); - @PersistenceContext(unitName = "kthfsPU") - private EntityManager em; - - public FeaturestoreJdbcConnectorFacade() { - super(FeaturestoreJdbcConnector.class); - } - - /** - * A transaction to persist a jdbc connection for the featurestore in the database - * - * @param featurestoreJdbcConnector the featurestore JDBC connection to persist - */ - @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void persist(FeaturestoreJdbcConnector featurestoreJdbcConnector) { - try { - em.persist(featurestoreJdbcConnector); - em.flush(); - } catch (ConstraintViolationException cve) { - LOGGER.log(Level.WARNING, "Could not persist the new JDBC connection", cve); - throw cve; - } - } - - /** - * Retrieves all jdbc connectors for a particular featurestore - * - * @param featurestore featurestore get jdbc connectors for - * @return list of JDBC connectors - */ - public List findByFeaturestore(Featurestore featurestore) { - TypedQuery q = em.createNamedQuery("FeaturestoreJdbcConnector.findByFeaturestore", - FeaturestoreJdbcConnector.class).setParameter("featurestore", featurestore); - return q.getResultList(); - } - - /** - * Retrieves a particular jdbc connector given its Id and featurestore from the database - * - * @param id id of the jdbc connector - * @param featurestore featurestore of the connector - * @return a single FeaturestoreJdbcConnector entity - */ - public Optional findByIdAndFeaturestore(Integer id, Featurestore featurestore) { - try { - return Optional.of( - em.createNamedQuery("FeaturestoreJdbcConnector.findByFeaturestoreAndId", FeaturestoreJdbcConnector.class) - .setParameter("featurestore", featurestore) - .setParameter("id", id) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - public Optional findByNameAndFeaturestore(String name, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreJdbcConnector.findByNameAndFeaturestore", - FeaturestoreJdbcConnector.class) - .setParameter("name", name) - .setParameter("featurestore", featurestore) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - /** - * Updates a jdbc connector - * - * @param featurestoreJdbcConnector the jdbc connector to update - * @return the updated connector - */ - public FeaturestoreJdbcConnector updateJdbcConnector( - FeaturestoreJdbcConnector featurestoreJdbcConnector) { - em.merge(featurestoreJdbcConnector); - return featurestoreJdbcConnector; - } - - /** - * Gets the entity manager of the facade - * - * @return entity manager - */ - @Override - protected EntityManager getEntityManager() { - return em; - } - -} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorController.java index da4a557322..42fcc72a69 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorController.java @@ -16,11 +16,9 @@ package io.hops.hopsworks.common.featurestore.storageconnectors.redshift; import com.google.common.base.Strings; -import io.hops.hopsworks.common.dao.featurestore.storageconnectors.redshift.FeaturestoreRedshiftConnectorFacade; import io.hops.hopsworks.common.dao.user.UserFacade; import io.hops.hopsworks.common.dao.user.security.secrets.SecretsFacade; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.security.secrets.SecretsController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; @@ -28,6 +26,7 @@ import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.secrets.Secret; @@ -37,21 +36,18 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.transaction.Transactional; import java.io.IOException; import java.security.GeneralSecurityException; -import java.util.List; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; @Stateless @TransactionAttribute(TransactionAttributeType.NEVER) public class FeaturestoreRedshiftConnectorController { + private static final Logger LOGGER = Logger.getLogger(FeaturestoreRedshiftConnectorController.class.getName()); - @EJB - private FeaturestoreRedshiftConnectorFacade featurestoreRedshiftConnectorFacade; @EJB private SecretsController secretsController; @EJB @@ -61,65 +57,12 @@ public class FeaturestoreRedshiftConnectorController { @EJB private UserFacade userFacade; - /** - * - * @param featurestore - * @return - */ - public List getConnectorsForFeaturestore(Users user, Featurestore featurestore) { - List featureStoreRedshiftConnectors = featurestoreRedshiftConnectorFacade - .findByFeaturestore(featurestore); - return featureStoreRedshiftConnectors.stream() - .map(redshiftConnector -> getFeaturestoreRedshiftConnectorDTO(user, redshiftConnector)) - .collect(Collectors.toList()); - } - - /** - * - * @param featurestore - * @param storageConnectorId - * @return - */ - public FeaturestoreRedshiftConnectorDTO getConnectorsWithIdAndFeaturestore(Users user, Featurestore featurestore, - Integer storageConnectorId) throws FeaturestoreException { - FeatureStoreRedshiftConnector featureStoreRedshiftConnector = - featurestoreRedshiftConnectorFacade.findByIdAndFeaturestore(storageConnectorId, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.REDSHIFT_CONNECTOR_NOT_FOUND, - Level.FINE, "Redshift Connector not found. ConnectorId: " + storageConnectorId)); - return getFeaturestoreRedshiftConnectorDTO(user, featureStoreRedshiftConnector); - } - - /** - * - * @param user - * @param featurestore - * @param storageConnectorName - * @return - * @throws FeaturestoreException - */ - public FeaturestoreStorageConnectorDTO getConnectorsWithNameAndFeaturestore(Users user, Featurestore featurestore, - String storageConnectorName) throws FeaturestoreException { - FeatureStoreRedshiftConnector featureStoreRedshiftConnector = - featurestoreRedshiftConnectorFacade.findByNameAndFeaturestore(storageConnectorName, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.REDSHIFT_CONNECTOR_NOT_FOUND, - Level.FINE, "Redshift Connector not found. Connector name: " + storageConnectorName)); - return getFeaturestoreRedshiftConnectorDTO(user, featureStoreRedshiftConnector); - } - - private FeaturestoreRedshiftConnectorDTO getFeaturestoreRedshiftConnectorDTO(Users user, - FeatureStoreRedshiftConnector featureStoreRedshiftConnector) { + public FeaturestoreRedshiftConnectorDTO getRedshiftConnectorDTO(Users user, + FeaturestoreConnector featurestoreConnector) { FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO = - new FeaturestoreRedshiftConnectorDTO(featureStoreRedshiftConnector); - if (featureStoreRedshiftConnector.getSecret() != null) { - Users owner = userFacade.find(featureStoreRedshiftConnector.getSecret().getId().getUid()); - try { - featurestoreRedshiftConnectorDTO.setDatabasePassword(secretsController - .getShared(user, owner.getUsername(), featureStoreRedshiftConnector.getSecret().getId().getName()) - .getPlaintext()); - } catch (UserException | ServiceException | ProjectException e) { - //user can't access secret - } - } + new FeaturestoreRedshiftConnectorDTO(featurestoreConnector); + featurestoreRedshiftConnectorDTO.setDatabasePassword( + getDatabasePassword(featurestoreConnector.getRedshiftConnector(), user)); return featurestoreRedshiftConnectorDTO; } @@ -129,13 +72,11 @@ private FeaturestoreRedshiftConnectorDTO getFeaturestoreRedshiftConnectorDTO(Use * @param featurestoreRedshiftConnectorDTO * @return */ - public FeaturestoreRedshiftConnectorDTO createFeaturestoreRedshiftConnector(Users user, Featurestore featurestore, - FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO) throws FeaturestoreException, UserException, - ProjectException { - verifyCreateDTO(featurestoreRedshiftConnectorDTO, featurestore); + public FeatureStoreRedshiftConnector createFeaturestoreRedshiftConnector(Users user, Featurestore featurestore, + FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO) + throws FeaturestoreException, UserException, ProjectException { + verifyCreateDTO(featurestoreRedshiftConnectorDTO); FeatureStoreRedshiftConnector featurestoreRedshiftConnector = new FeatureStoreRedshiftConnector(); - featurestoreRedshiftConnector.setName(featurestoreRedshiftConnectorDTO.getName()); - featurestoreRedshiftConnector.setDescription(featurestoreRedshiftConnectorDTO.getDescription()); featurestoreRedshiftConnector.setClusterIdentifier(featurestoreRedshiftConnectorDTO.getClusterIdentifier()); featurestoreRedshiftConnector.setDatabaseDriver(featurestoreRedshiftConnectorDTO.getDatabaseDriver()); featurestoreRedshiftConnector.setDatabaseEndpoint(featurestoreRedshiftConnectorDTO.getDatabaseEndpoint()); @@ -148,9 +89,7 @@ public FeaturestoreRedshiftConnectorDTO createFeaturestoreRedshiftConnector(User featurestoreRedshiftConnector.setAutoCreate(featurestoreRedshiftConnectorDTO.getAutoCreate()); featurestoreRedshiftConnector.setDatabaseGroup(featurestoreRedshiftConnectorDTO.getDatabaseGroup()); featurestoreRedshiftConnector.setArguments(featurestoreRedshiftConnectorDTO.getArguments()); - featurestoreRedshiftConnector.setFeatureStore(featurestore); - featurestoreRedshiftConnectorFacade.save(featurestoreRedshiftConnector); - return featurestoreRedshiftConnectorDTO; + return featurestoreRedshiftConnector; } private void setPassword(Users user, FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, @@ -168,14 +107,8 @@ private String createSecretName(Featurestore featurestore, String connectorName) return "redshift_" + connectorName.replaceAll(" ", "_").toLowerCase() + "_" + featurestore.getId(); } - private void verifyCreateDTO(FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, - Featurestore featurestore) throws FeaturestoreException { - verifyName(featurestoreRedshiftConnectorDTO, featurestore); - featurestoreRedshiftConnectorDTO.verifyDescription(); - if (Strings.isNullOrEmpty(featurestoreRedshiftConnectorDTO.getClusterIdentifier())) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, Level.FINE, - "Redshift connector cluster identifier cannot be empty"); - } + private void verifyCreateDTO(FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO) + throws FeaturestoreException { if(!Strings.isNullOrEmpty(featurestoreRedshiftConnectorDTO.getArguments()) && featurestoreRedshiftConnectorDTO.getArguments().length() > FeaturestoreConstants.JDBC_STORAGE_CONNECTOR_ARGUMENTS_MAX_LENGTH) { @@ -186,22 +119,10 @@ private void verifyCreateDTO(FeaturestoreRedshiftConnectorDTO featurestoreRedshi verifyPassword(featurestoreRedshiftConnectorDTO); } - private void verifyName(FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, Featurestore featurestore) - throws FeaturestoreException { - featurestoreRedshiftConnectorDTO.verifyName(); - Optional featureStoreRedshiftConnector = - featurestoreRedshiftConnectorFacade - .findByNameAndFeaturestore(featurestoreRedshiftConnectorDTO.getName(), featurestore); - if (featureStoreRedshiftConnector.isPresent()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - "Redshift connector with the same name already exists. Name=" + featurestoreRedshiftConnectorDTO.getName()); - } - } - private void verifyPassword(FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO) - throws FeaturestoreException { + throws FeaturestoreException { verifyPassword(featurestoreRedshiftConnectorDTO.getIamRole(), - featurestoreRedshiftConnectorDTO.getDatabasePassword()); + featurestoreRedshiftConnectorDTO.getDatabasePassword()); } private void verifyPassword(String iamRole, String password) throws FeaturestoreException { @@ -226,110 +147,71 @@ private void verifyPassword(String iamRole, Secret secret) throws FeaturestoreEx } } - /** - * - * @param featurestore - * @param featurestoreRedshiftConnectorDTO - * @param storageConnectorName - * @return - */ - public FeaturestoreRedshiftConnectorDTO updateFeaturestoreRedshiftConnector(Users user, Featurestore featurestore, - FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, String storageConnectorName) - throws FeaturestoreException, UserException, ProjectException { - Optional featureStoreRedshiftConnectorOptional = - featurestoreRedshiftConnectorFacade.findByNameAndFeaturestore(storageConnectorName, featurestore); - if (!featureStoreRedshiftConnectorOptional.isPresent()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.REDSHIFT_CONNECTOR_NOT_FOUND, - Level.FINE, "Connector name: " + storageConnectorName); - } - boolean updated = false; - FeatureStoreRedshiftConnector featureStoreRedshiftConnector = featureStoreRedshiftConnectorOptional.get(); - FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTOExisting = - getFeaturestoreRedshiftConnectorDTO(user, featureStoreRedshiftConnector); - if (shouldUpdate(featurestoreRedshiftConnectorDTO.getName(), featurestoreRedshiftConnectorDTOExisting.getName())) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, - Level.FINE, "Can not update connector name."); - } - //can set description to empty - if (shouldUpdate(featurestoreRedshiftConnectorDTO.getDescription(), - featurestoreRedshiftConnectorDTOExisting.getDescription())) { - featurestoreRedshiftConnectorDTO.verifyDescription(); - featureStoreRedshiftConnector.setDescription(featurestoreRedshiftConnectorDTO.getDescription()); - updated = true; - } + @TransactionAttribute(TransactionAttributeType.REQUIRED) + @Transactional(rollbackOn = FeaturestoreException.class) + public FeatureStoreRedshiftConnector updateFeaturestoreRedshiftConnector(Users user, Featurestore featurestore, + FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, + FeatureStoreRedshiftConnector featureStoreRedshiftConnector) + throws FeaturestoreException, UserException, ProjectException { + if (!Strings.isNullOrEmpty(featurestoreRedshiftConnectorDTO.getClusterIdentifier()) && - !featurestoreRedshiftConnectorDTO.getClusterIdentifier() - .equals(featurestoreRedshiftConnectorDTOExisting.getClusterIdentifier())) { + !featurestoreRedshiftConnectorDTO.getClusterIdentifier() + .equals(featureStoreRedshiftConnector.getClusterIdentifier())) { featureStoreRedshiftConnector.setClusterIdentifier(featurestoreRedshiftConnectorDTO.getClusterIdentifier()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getDatabaseDriver(), - featurestoreRedshiftConnectorDTO.getDatabaseDriver())) { + if (shouldUpdate(featureStoreRedshiftConnector.getDatabaseDriver(), + featurestoreRedshiftConnectorDTO.getDatabaseDriver())) { featureStoreRedshiftConnector.setDatabaseDriver(featurestoreRedshiftConnectorDTO.getDatabaseDriver()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getDatabaseEndpoint(), - featurestoreRedshiftConnectorDTO.getDatabaseEndpoint())) { + if (shouldUpdate(featureStoreRedshiftConnector.getDatabaseEndpoint(), + featurestoreRedshiftConnectorDTO.getDatabaseEndpoint())) { featureStoreRedshiftConnector.setDatabaseEndpoint(featurestoreRedshiftConnectorDTO.getDatabaseEndpoint()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getDatabaseName(), - featurestoreRedshiftConnectorDTO.getDatabaseName())) { + if (shouldUpdate(featureStoreRedshiftConnector.getDatabaseName(), + featurestoreRedshiftConnectorDTO.getDatabaseName())) { featureStoreRedshiftConnector.setDatabaseName(featurestoreRedshiftConnectorDTO.getDatabaseName()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTO.getDatabaseUserName(), - featurestoreRedshiftConnectorDTOExisting.getDatabaseUserName())) { + if (shouldUpdate(featureStoreRedshiftConnector.getDatabaseUserName(), + featurestoreRedshiftConnectorDTO.getDatabaseUserName())) { featureStoreRedshiftConnector.setDatabaseUserName(featurestoreRedshiftConnectorDTO.getDatabaseUserName()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getDatabaseGroup(), - featurestoreRedshiftConnectorDTO.getDatabaseGroup())) { + if (shouldUpdate(featureStoreRedshiftConnector.getDatabaseGroup(), + featurestoreRedshiftConnectorDTO.getDatabaseGroup())) { featureStoreRedshiftConnector.setDatabaseGroup(featurestoreRedshiftConnectorDTO.getDatabaseGroup()); - updated = true; } - if (featurestoreRedshiftConnectorDTO.getAutoCreate() != featurestoreRedshiftConnectorDTOExisting.getAutoCreate()) { + if (featurestoreRedshiftConnectorDTO.getAutoCreate() != featureStoreRedshiftConnector.getAutoCreate()) { featureStoreRedshiftConnector.setAutoCreate(featurestoreRedshiftConnectorDTO.getAutoCreate()); - updated = true; } if (shouldUpdate(featurestoreRedshiftConnectorDTO.getDatabasePort(), - featurestoreRedshiftConnectorDTOExisting.getDatabasePort())) { + featureStoreRedshiftConnector.getDatabasePort())) { featureStoreRedshiftConnector.setDatabasePort(featurestoreRedshiftConnectorDTO.getDatabasePort()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getTableName(), - featurestoreRedshiftConnectorDTO.getTableName())) { + if (shouldUpdate(featureStoreRedshiftConnector.getTableName(), + featurestoreRedshiftConnectorDTO.getTableName())) { featureStoreRedshiftConnector.setTableName(featurestoreRedshiftConnectorDTO.getTableName()); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getIamRole(), - featurestoreRedshiftConnectorDTO.getIamRole())) { + if (shouldUpdate(featureStoreRedshiftConnector.getIamRole(), + featurestoreRedshiftConnectorDTO.getIamRole())) { featureStoreRedshiftConnector.setIamRole(featurestoreRedshiftConnectorDTO.getIamRole()); - updated = true; } + Secret secret = null; - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getDatabasePassword(), - featurestoreRedshiftConnectorDTO.getDatabasePassword())) { + if (shouldUpdate(getDatabasePassword(featureStoreRedshiftConnector, user), + featurestoreRedshiftConnectorDTO.getDatabasePassword())) { secret = updatePassword(user, featurestoreRedshiftConnectorDTO, featurestore, featureStoreRedshiftConnector); - updated = true; } - if (shouldUpdate(featurestoreRedshiftConnectorDTOExisting.getArguments(), + if (shouldUpdate(featureStoreRedshiftConnector.getArguments(), featurestoreRedshiftConnectorDTO.getArguments())) { featureStoreRedshiftConnector.setArguments(featurestoreRedshiftConnectorDTO.getArguments()); - updated = true; } - if (updated) { - //verify if password or iam role is set - verifyPassword(featureStoreRedshiftConnector.getIamRole(), featureStoreRedshiftConnector.getSecret()); - featureStoreRedshiftConnector = featurestoreRedshiftConnectorFacade.update(featureStoreRedshiftConnector); - if (featureStoreRedshiftConnector.getSecret() == null && secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } - featurestoreRedshiftConnectorDTOExisting = getFeaturestoreRedshiftConnectorDTO(user, - featureStoreRedshiftConnector); + + verifyPassword(featureStoreRedshiftConnector.getIamRole(), featureStoreRedshiftConnector.getSecret()); + if (featureStoreRedshiftConnector.getSecret() == null && secret != null) { + secretsFacade.deleteSecret(secret.getId()); } - return featurestoreRedshiftConnectorDTOExisting; + + return featureStoreRedshiftConnector; } private Secret updatePassword(Users user, FeaturestoreRedshiftConnectorDTO featurestoreRedshiftConnectorDTO, @@ -363,40 +245,20 @@ private boolean shouldUpdate(Integer oldVal, Integer newVal) { return (oldVal == null && newVal != null) || (oldVal != null && !oldVal.equals(newVal)); } - /** - * - * @param storageConnectorId - * @return - */ - public void removeFeaturestoreRedshiftConnector(Users user, Integer storageConnectorId) throws ProjectException { - FeatureStoreRedshiftConnector featureStoreRedshiftConnector = - featurestoreRedshiftConnectorFacade.find(storageConnectorId); - if (featureStoreRedshiftConnector != null) { - Secret secret = featureStoreRedshiftConnector.getSecret(); - secretsController.checkCanAccessSecret(secret, user); - featurestoreRedshiftConnectorFacade.remove(featureStoreRedshiftConnector); - if (secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } + + private String getDatabasePassword(FeatureStoreRedshiftConnector featureStoreRedshiftConnector, Users user) { + if (featureStoreRedshiftConnector.getSecret() == null) { + return null; } - } - /** - * - * @param storageConnectorName - * @return - */ - public void removeFeaturestoreRedshiftConnector(Users user, String storageConnectorName, Featurestore featurestore) - throws ProjectException { - Optional featureStoreRedshiftConnector = - featurestoreRedshiftConnectorFacade.findByNameAndFeaturestore(storageConnectorName, featurestore); - if (featureStoreRedshiftConnector.isPresent()) { - Secret secret = featureStoreRedshiftConnector.get().getSecret(); - secretsController.checkCanAccessSecret(secret, user); - featurestoreRedshiftConnectorFacade.remove(featureStoreRedshiftConnector.get()); - if (secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } + Users owner = userFacade.find(featureStoreRedshiftConnector.getSecret().getId().getUid()); + try { + return secretsController + .getShared(user, owner.getUsername(), featureStoreRedshiftConnector.getSecret().getId().getName()) + .getPlaintext(); + } catch (UserException | ServiceException | ProjectException e) { + //user can't access secret } + return ""; } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorDTO.java index 0caa685701..e4942975e4 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/redshift/FeaturestoreRedshiftConnectorDTO.java @@ -16,8 +16,7 @@ package io.hops.hopsworks.common.featurestore.storageconnectors.redshift; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import org.apache.parquet.Strings; import javax.xml.bind.annotation.XmlElement; @@ -44,21 +43,19 @@ public class FeaturestoreRedshiftConnectorDTO extends FeaturestoreStorageConnect public FeaturestoreRedshiftConnectorDTO() { } - public FeaturestoreRedshiftConnectorDTO(FeatureStoreRedshiftConnector featureStoreRedshiftConnector) { - super(featureStoreRedshiftConnector.getId(), featureStoreRedshiftConnector.getDescription(), - featureStoreRedshiftConnector.getName(), featureStoreRedshiftConnector.getFeatureStore().getId(), - FeaturestoreStorageConnectorType.REDSHIFT); - this.clusterIdentifier = featureStoreRedshiftConnector.getClusterIdentifier(); - this.databaseDriver = featureStoreRedshiftConnector.getDatabaseDriver(); - this.databaseEndpoint = featureStoreRedshiftConnector.getDatabaseEndpoint(); - this.databaseName = featureStoreRedshiftConnector.getDatabaseName(); - this.databasePort = featureStoreRedshiftConnector.getDatabasePort(); - this.tableName = featureStoreRedshiftConnector.getTableName(); - this.databaseUserName = featureStoreRedshiftConnector.getDatabaseUserName(); - this.autoCreate = featureStoreRedshiftConnector.getAutoCreate(); - this.databaseGroup = featureStoreRedshiftConnector.getDatabaseGroup(); - this.iamRole = featureStoreRedshiftConnector.getIamRole(); - this.arguments = featureStoreRedshiftConnector.getArguments(); + public FeaturestoreRedshiftConnectorDTO(FeaturestoreConnector featurestoreConnector) { + super(featurestoreConnector); + this.clusterIdentifier = featurestoreConnector.getRedshiftConnector().getClusterIdentifier(); + this.databaseDriver = featurestoreConnector.getRedshiftConnector().getDatabaseDriver(); + this.databaseEndpoint = featurestoreConnector.getRedshiftConnector().getDatabaseEndpoint(); + this.databaseName = featurestoreConnector.getRedshiftConnector().getDatabaseName(); + this.databasePort = featurestoreConnector.getRedshiftConnector().getDatabasePort(); + this.tableName = featurestoreConnector.getRedshiftConnector().getTableName(); + this.databaseUserName = featurestoreConnector.getRedshiftConnector().getDatabaseUserName(); + this.autoCreate = featurestoreConnector.getRedshiftConnector().getAutoCreate(); + this.databaseGroup = featurestoreConnector.getRedshiftConnector().getDatabaseGroup(); + this.iamRole = featurestoreConnector.getRedshiftConnector().getIamRole(); + this.arguments = featurestoreConnector.getRedshiftConnector().getArguments(); } @XmlElement diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorController.java index 5031d95541..d20ab1c437 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorController.java @@ -22,7 +22,6 @@ import io.hops.hopsworks.common.dao.user.security.secrets.SecretPlaintext; import io.hops.hopsworks.common.dao.user.security.secrets.SecretsFacade; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.security.secrets.SecretsController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; @@ -30,6 +29,7 @@ import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3ConnectorAccessAndSecretKey; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3ConnectorEncryptionAlgorithm; @@ -42,11 +42,9 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.transaction.Transactional; import java.io.IOException; import java.security.GeneralSecurityException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -56,9 +54,9 @@ @Stateless @TransactionAttribute(TransactionAttributeType.NEVER) public class FeaturestoreS3ConnectorController { + private static final Logger LOGGER = Logger.getLogger(FeaturestoreS3ConnectorController.class.getName()); - @EJB - private FeaturestoreS3ConnectorFacade featurestoreS3ConnectorFacade; + @EJB private Settings settings; @EJB @@ -77,27 +75,20 @@ public class FeaturestoreS3ConnectorController { * @return DTO of the created entity * @throws FeaturestoreException */ - public FeaturestoreS3ConnectorDTO createFeaturestoreS3Connector(Users user, Featurestore featurestore, - FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO) throws FeaturestoreException, UserException, - ProjectException { + public FeaturestoreS3Connector createFeaturestoreS3Connector( + Users user, Featurestore featurestore, FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO) + throws FeaturestoreException, UserException, ProjectException { FeaturestoreS3ConnectorEncryptionAlgorithm encryptionAlgorithm = getEncryptionAlgorithm(featurestoreS3ConnectorDTO.getServerEncryptionAlgorithm()); - verifyUserInput(featurestore, featurestoreS3ConnectorDTO); + + verifyUserInput(featurestoreS3ConnectorDTO); FeaturestoreS3Connector featurestoreS3Connector = new FeaturestoreS3Connector(); - featurestoreS3Connector.setDescription(featurestoreS3ConnectorDTO.getDescription()); - featurestoreS3Connector.setName(featurestoreS3ConnectorDTO.getName()); featurestoreS3Connector.setBucket(featurestoreS3ConnectorDTO.getBucket()); - featurestoreS3Connector.setFeaturestore(featurestore); featurestoreS3Connector.setServerEncryptionAlgorithm(encryptionAlgorithm); featurestoreS3Connector.setServerEncryptionKey(featurestoreS3ConnectorDTO.getServerEncryptionKey()); featurestoreS3Connector.setIamRole(featurestoreS3ConnectorDTO.getIamRole()); setSecret(user, featurestoreS3ConnectorDTO, featurestoreS3Connector, featurestore); - featurestoreS3ConnectorFacade.persist(featurestoreS3Connector); - FeaturestoreS3ConnectorDTO createdFeaturestoreS3ConnectorDTO = - new FeaturestoreS3ConnectorDTO(featurestoreS3Connector); - createdFeaturestoreS3ConnectorDTO.setAccessKey(featurestoreS3ConnectorDTO.getAccessKey()); - createdFeaturestoreS3ConnectorDTO.setSecretKey(featurestoreS3ConnectorDTO.getSecretKey()); - return createdFeaturestoreS3ConnectorDTO; + return featurestoreS3Connector; } private void setSecret(Users user, FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO, @@ -121,52 +112,30 @@ private void setSecret(Users user, FeaturestoreS3ConnectorDTO featurestoreS3Conn private String createSecretName(Integer featurestoreId, String connectorName) { return "s3_" + connectorName.replaceAll(" ", "_").toLowerCase() + "_" + featurestoreId; } - - /** - * Updates a S3 connection as a backend for a feature store - * - * @param featurestore the featurestore - * @param featurestoreS3ConnectorDTO the data to use when creating the storage connector - * @param storageConnectorName the name of the connector - * @return DTO of the updated entity - * @throws FeaturestoreException - */ - public FeaturestoreS3ConnectorDTO updateFeaturestoreS3Connector(Users user, Featurestore featurestore, - FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO, String storageConnectorName) throws FeaturestoreException, - UserException, ProjectException { - FeaturestoreS3Connector featurestoreS3Connector = verifyS3ConnectorName(storageConnectorName, featurestore); - FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTOExisting = - getS3ConnectorWithIdAndFeaturestore(user, featurestoreS3Connector); - boolean updated = false; - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getName(), featurestoreS3ConnectorDTO.getName())) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_ARG, - Level.FINE, "Can not update connector name."); - } - //can set description to empty - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getDescription(), - featurestoreS3ConnectorDTO.getDescription())) { - featurestoreS3ConnectorDTO.verifyDescription(); - featurestoreS3Connector.setDescription(featurestoreS3ConnectorDTO.getDescription()); - updated = true; - } - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getBucket(), featurestoreS3ConnectorDTO.getBucket())) { + + @TransactionAttribute(TransactionAttributeType.REQUIRED) + @Transactional(rollbackOn = {FeaturestoreException.class, UserException.class, ProjectException.class}) + public FeaturestoreS3Connector updateFeaturestoreS3Connector(Users user, Featurestore featurestore, + FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO, FeaturestoreS3Connector featurestoreS3Connector) + throws FeaturestoreException, UserException, ProjectException { + + if (shouldUpdate(featurestoreS3Connector.getBucket(), featurestoreS3ConnectorDTO.getBucket())) { verifyS3ConnectorBucket(featurestoreS3ConnectorDTO.getBucket()); featurestoreS3Connector.setBucket(featurestoreS3ConnectorDTO.getBucket()); - updated = true; } - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getIamRole(), featurestoreS3ConnectorDTO.getIamRole())) { + if (shouldUpdate(featurestoreS3Connector.getIamRole(), featurestoreS3ConnectorDTO.getIamRole())) { featurestoreS3Connector.setIamRole(featurestoreS3ConnectorDTO.getIamRole()); - updated = true; } Secret secret = null; - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getAccessKey(), featurestoreS3ConnectorDTO.getAccessKey()) || - shouldUpdate(featurestoreS3ConnectorDTOExisting.getSecretKey(), featurestoreS3ConnectorDTO.getSecretKey())) { + FeaturestoreS3ConnectorAccessAndSecretKey keys = + getS3AccessAndSecretKeySecretForConnector(user, featurestoreS3Connector); + if (shouldUpdate(keys.getAccessKey(), featurestoreS3ConnectorDTO.getAccessKey()) || + shouldUpdate(keys.getSecretKey(), featurestoreS3ConnectorDTO.getSecretKey())) { secret = updateSecret(user, featurestoreS3ConnectorDTO, featurestore, featurestoreS3Connector); - updated = true; } - if (shouldUpdate(featurestoreS3ConnectorDTOExisting.getServerEncryptionAlgorithm(), + if (shouldUpdate(featurestoreS3Connector.getServerEncryptionAlgorithm().getAlgorithm(), featurestoreS3ConnectorDTO.getServerEncryptionAlgorithm())) { if (featurestoreS3ConnectorDTO.getServerEncryptionAlgorithm() != null) { FeaturestoreS3ConnectorEncryptionAlgorithm serverEncryptionAlgorithm = @@ -186,19 +155,15 @@ public FeaturestoreS3ConnectorDTO updateFeaturestoreS3Connector(Users user, Feat featurestoreS3Connector.setServerEncryptionAlgorithm(null); featurestoreS3Connector.setServerEncryptionKey(null); } - updated = true; } - if (updated) { - //verify if key or iam role is set - verifyKeyAndIAMRole(featurestoreS3Connector.getIamRole(), featurestoreS3Connector.getSecret()); - featurestoreS3Connector = featurestoreS3ConnectorFacade.updateS3Connector(featurestoreS3Connector); - if (featurestoreS3Connector.getSecret() == null && secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } - featurestoreS3ConnectorDTOExisting = getS3ConnectorWithIdAndFeaturestore(user, featurestoreS3Connector); + //verify if key or iam role is set + verifyKeyAndIAMRole(featurestoreS3Connector.getIamRole(), featurestoreS3Connector.getSecret()); + if (featurestoreS3Connector.getSecret() == null && secret != null) { + secretsFacade.deleteSecret(secret.getId()); } - return featurestoreS3ConnectorDTOExisting; + + return featurestoreS3Connector; } private void verifyKeyAndIAMRole(String iamRole, Secret secret) throws FeaturestoreException { @@ -251,78 +216,6 @@ private boolean keysNotNullOrEmpty(FeaturestoreS3ConnectorDTO featurestoreS3Conn !Strings.isNullOrEmpty(featurestoreS3ConnectorDTO.getAccessKey()); } - /** - * Removes an S3 connector from the feature store - * - * @param featurestoreS3Id id of the connection to remove - */ - public void removeFeaturestoreS3Connector(Users user, Integer featurestoreS3Id) throws ProjectException { - FeaturestoreS3Connector featurestoreS3Connector = featurestoreS3ConnectorFacade.find(featurestoreS3Id); - if (featurestoreS3Connector != null) { - Secret secret = featurestoreS3Connector.getSecret(); - secretsController.checkCanAccessSecret(secret, user); - featurestoreS3ConnectorFacade.remove(featurestoreS3Connector); - if (secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } - } - } - - /** - * Removes an S3 connector from the feature store - * - * @param featurestore - * @param storageConnectorName - */ - public void removeFeaturestoreS3Connector(Users user, Featurestore featurestore, String storageConnectorName) - throws ProjectException { - Optional featurestoreS3Connector = - featurestoreS3ConnectorFacade.findByNameAndFeaturestore(storageConnectorName, featurestore); - if (featurestoreS3Connector.isPresent()) { - Secret secret = featurestoreS3Connector.get().getSecret(); - secretsController.checkCanAccessSecret(secret, user); - featurestoreS3ConnectorFacade.remove(featurestoreS3Connector.get()); - if (secret != null) { - secretsFacade.deleteSecret(secret.getId()); - } - } - } - - private FeaturestoreS3Connector verifyS3ConnectorName(String name, Featurestore featurestore) - throws FeaturestoreException { - return featurestoreS3ConnectorFacade.findByNameAndFeaturestore(name, featurestore) - .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.S3_CONNECTOR_NOT_FOUND, - Level.FINE, "S3 Connector not found, S3 connector with name: " + name)); - } - - /** - * Validates the featurestore to query - * - * @param featurestore the featurestore to validate - */ - private void verifyFeaturestore(Featurestore featurestore){ - if (featurestore == null) { - throw new IllegalArgumentException("Featurestore was not found"); - } - } - - /** - * Validates user input name string - * @param featurestoreS3ConnectorDTO the input to validate - * @param featurestore the featurestore of the connector - * @throws FeaturestoreException - */ - private void verifyS3ConnectorName(FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO, Featurestore featurestore) - throws FeaturestoreException { - featurestoreS3ConnectorDTO.verifyName(); - if (featurestoreS3ConnectorFacade.findByNameAndFeaturestore(featurestoreS3ConnectorDTO.getName(), featurestore) - .isPresent()) { - throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE, - "Illegal storage connector name, the storage connector name should be unique, there already exists a S3 " + - "connector with the same name "); - } - } - /** * Validates user input bucket * @@ -396,18 +289,14 @@ private void verifyS3ConnectorServerEncryptionKey(String serverEncryptionKey) th /** * Validates user input for creating a new S3 connector in a featurestore * - * @param featurestore the featurestore * @param featurestoreS3ConnectorDTO the data to use when creating the storage connector * @throws FeaturestoreException */ - private void verifyUserInput(Featurestore featurestore, FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO) + private void verifyUserInput(FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO) throws FeaturestoreException { if (featurestoreS3ConnectorDTO == null) { throw new IllegalArgumentException("Null input data"); } - verifyFeaturestore(featurestore); - verifyS3ConnectorName(featurestoreS3ConnectorDTO,featurestore); - featurestoreS3ConnectorDTO.verifyDescription(); verifyS3ConnectorBucket(featurestoreS3ConnectorDTO.getBucket()); if (settings.isIAMRoleConfigured() || !Strings.isNullOrEmpty(featurestoreS3ConnectorDTO.getIamRole())) { @@ -449,12 +338,8 @@ private void verifySecretAndAccessKeysForIamRole(FeaturestoreS3ConnectorDTO feat "S3 IAM role not set."); } } - - /** - * @param s - * @return - * @throws FeaturestoreException - */ + + private FeaturestoreS3ConnectorEncryptionAlgorithm getEncryptionAlgorithm(String s) throws FeaturestoreException { if (Strings.isNullOrEmpty(s)) { return null; @@ -480,42 +365,12 @@ private FeaturestoreS3ConnectorEncryptionAlgorithm getEncryptionAlgorithm(String public String createS3AccessAndSecretKeysSecret(String accessKey, String secretKey) { return new JSONObject(new FeaturestoreS3ConnectorAccessAndSecretKey(accessKey, secretKey)).toString(); } - - /** - * Gets all S3 connectors for a particular featurestore and project - * - * @param featurestore featurestore to query for s3 connectors - * @return list of XML/JSON DTOs of the s3 connectors - */ - public List getS3ConnectorsForFeaturestore(Users user, Featurestore featurestore) { - List s3ConnectorDTOs = new ArrayList<>(); - List s3Connectors = featurestoreS3ConnectorFacade.findByFeaturestore(featurestore); - for(FeaturestoreS3Connector s3Connector : s3Connectors) { - s3ConnectorDTOs.add(getS3ConnectorWithIdAndFeaturestore(user, s3Connector)); - } - return s3ConnectorDTOs; - } - /** - * - * @param user - * @param featurestore - * @param name - * @return - * @throws FeaturestoreException - */ - public FeaturestoreS3ConnectorDTO getS3ConnectorWithNameAndFeaturestore(Users user, Featurestore featurestore, - String name) throws FeaturestoreException { - FeaturestoreS3Connector featurestoreS3Connector = verifyS3ConnectorName(name, featurestore); - return getS3ConnectorWithIdAndFeaturestore(user, featurestoreS3Connector); - } - - private FeaturestoreS3ConnectorDTO getS3ConnectorWithIdAndFeaturestore(Users user, - FeaturestoreS3Connector featurestoreS3Connector) { - FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO = new FeaturestoreS3ConnectorDTO(featurestoreS3Connector); - if (featurestoreS3Connector.getSecret() != null) { + public FeaturestoreS3ConnectorDTO getS3ConnectorDTO(Users user, FeaturestoreConnector featurestoreConnector) { + FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO = new FeaturestoreS3ConnectorDTO(featurestoreConnector); + if (featurestoreConnector.getS3Connector().getSecret() != null) { FeaturestoreS3ConnectorAccessAndSecretKey accessAndSecretKey = - getS3AccessAndSecretKeySecretForConnector(user, featurestoreS3Connector); + getS3AccessAndSecretKeySecretForConnector(user, featurestoreConnector.getS3Connector()); setAccessAndSecretKeysInDTO(featurestoreS3ConnectorDTO, accessAndSecretKey); } return featurestoreS3ConnectorDTO; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorDTO.java index 49e502c041..469ea00de3 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorDTO.java @@ -16,9 +16,8 @@ package io.hops.hopsworks.common.featurestore.storageconnectors.s3; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3ConnectorEncryptionAlgorithm; import javax.xml.bind.annotation.XmlElement; @@ -40,17 +39,15 @@ public class FeaturestoreS3ConnectorDTO extends FeaturestoreStorageConnectorDTO private String sessionToken; public FeaturestoreS3ConnectorDTO() { - super(null, null, null, null, null); } - - public FeaturestoreS3ConnectorDTO(FeaturestoreS3Connector featurestoreS3Connector) { - super(featurestoreS3Connector.getId(), featurestoreS3Connector.getDescription(), - featurestoreS3Connector.getName(), featurestoreS3Connector.getFeaturestore().getId(), - FeaturestoreStorageConnectorType.S3); - this.bucket = featurestoreS3Connector.getBucket(); - this.serverEncryptionAlgorithm = getEncryptionAlgorithmName(featurestoreS3Connector.getServerEncryptionAlgorithm()); - this.serverEncryptionKey = featurestoreS3Connector.getServerEncryptionKey(); - this.iamRole = featurestoreS3Connector.getIamRole(); + + public FeaturestoreS3ConnectorDTO(FeaturestoreConnector featurestoreConnector) { + super(featurestoreConnector); + this.bucket = featurestoreConnector.getS3Connector().getBucket(); + this.serverEncryptionAlgorithm = + getEncryptionAlgorithmName(featurestoreConnector.getS3Connector().getServerEncryptionAlgorithm()); + this.serverEncryptionKey = featurestoreConnector.getS3Connector().getServerEncryptionKey(); + this.iamRole = featurestoreConnector.getS3Connector().getIamRole(); } private String getEncryptionAlgorithmName(FeaturestoreS3ConnectorEncryptionAlgorithm serverEncryptionAlgorithm) { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorFacade.java deleted file mode 100644 index 81dfb99c82..0000000000 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/s3/FeaturestoreS3ConnectorFacade.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * This file is part of Hopsworks - * Copyright (C) 2019, Logical Clocks AB. All rights reserved - * - * Hopsworks is free software: you can redistribute it and/or modify it under the terms of - * the GNU Affero General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License along with this program. - * If not, see . - */ - -package io.hops.hopsworks.common.featurestore.storageconnectors.s3; - -import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; - -import javax.ejb.Stateless; -import javax.ejb.TransactionAttribute; -import javax.ejb.TransactionAttributeType; -import javax.persistence.EntityManager; -import javax.persistence.NoResultException; -import javax.persistence.PersistenceContext; -import javax.persistence.TypedQuery; -import javax.validation.ConstraintViolationException; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A facade for the feature_store_s3_connector table in the Hopsworks database, - * use this interface when performing database operations against the table. - */ -@Stateless -public class FeaturestoreS3ConnectorFacade extends AbstractFacade { - private static final Logger LOGGER = Logger.getLogger( - FeaturestoreS3ConnectorFacade.class.getName()); - @PersistenceContext(unitName = "kthfsPU") - private EntityManager em; - - public FeaturestoreS3ConnectorFacade() { - super(FeaturestoreS3Connector.class); - } - - /** - * A transaction to persist a s3 connection for the featurestore in the database - * - * @param featurestoreS3Connector the featurestore s3 connection to persist - */ - @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void persist(FeaturestoreS3Connector featurestoreS3Connector) { - try { - em.persist(featurestoreS3Connector); - em.flush(); - } catch (ConstraintViolationException cve) { - LOGGER.log(Level.WARNING, "Could not persist the new s3 connection", cve); - throw cve; - } - } - - /** - * Retrieves all s3 connectors for a particular featurestore - * - * @param featurestore featurestore get s3 connectors for - * @return list of s3 connectors - */ - public List findByFeaturestore(Featurestore featurestore) { - TypedQuery q = em.createNamedQuery("FeaturestoreS3Connector.findByFeaturestore", - FeaturestoreS3Connector.class).setParameter("featurestore", featurestore); - return q.getResultList(); - } - - /** - * Retrieves a particular s3 connector given its Id and featurestore from the database - * - * @param id id of the s3 connector - * @param featurestore featurestore of the connector - * @return a single FeaturestoreS3Connector entity - */ - public Optional findByIdAndFeaturestore(Integer id, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreS3Connector.findByFeaturestoreAndId", - FeaturestoreS3Connector.class) - .setParameter("featurestore", featurestore) - .setParameter("id", id) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - public Optional findByNameAndFeaturestore(String name, Featurestore featurestore) { - try { - return Optional.of(em.createNamedQuery("FeaturestoreS3Connector.findByFeaturestoreAndName", - FeaturestoreS3Connector.class) - .setParameter("featurestore", featurestore) - .setParameter("name", name) - .getSingleResult()); - } catch (NoResultException e) { - return Optional.empty(); - } - } - - /** - * Updates a s3 connector - * - * @param featurestoreS3Connector the s3 connector to update - * @return the updated connector - */ - public FeaturestoreS3Connector updateS3Connector( - FeaturestoreS3Connector featurestoreS3Connector) { - em.merge(featurestoreS3Connector); - return featurestoreS3Connector; - } - - /** - * Gets the entity manager of the facade - * - * @return entity manager - */ - @Override - protected EntityManager getEntityManager() { - return em; - } - -} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java index ac652d7755..b0590e1180 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java @@ -33,9 +33,7 @@ import io.hops.hopsworks.common.featurestore.query.Join; import io.hops.hopsworks.common.featurestore.query.Query; import io.hops.hopsworks.common.featurestore.query.QueryDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorController; -import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorFacade; -import io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorFacade; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreConnectorFacade; import io.hops.hopsworks.common.featurestore.trainingdatasets.external.ExternalTrainingDatasetController; import io.hops.hopsworks.common.featurestore.trainingdatasets.external.ExternalTrainingDatasetFacade; import io.hops.hopsworks.common.featurestore.trainingdatasets.hopsfs.HopsfsTrainingDatasetController; @@ -55,8 +53,8 @@ import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.FeaturegroupType; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.TimeTravelFormat; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnectorType; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetFeature; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetJoin; @@ -117,16 +115,10 @@ public class TrainingDatasetController { @EJB private FeaturestoreInputValidation featurestoreInputValidation; @EJB - private FeaturestoreHopsfsConnectorFacade hopsfsConnectorFacade; - @EJB - private FeaturestoreS3ConnectorFacade S3ConnectorFacade; - @EJB private InodeController inodeController; @EJB private HopsFSProvenanceController fsProvenanceController; @EJB - private FeaturestoreHopsfsConnectorController hopsfsConnectorController; - @EJB private DistributedFsService dfs; @EJB private HdfsUsersController hdfsUsersBean; @@ -140,6 +132,8 @@ public class TrainingDatasetController { private OnlineFeaturestoreController onlineFeaturestoreController; @EJB private FeaturegroupController featuregroupController; + @EJB + private FeaturestoreConnectorFacade featurestoreConnectorFacade; /** * Gets all trainingDatasets for a particular featurestore and project @@ -228,20 +222,23 @@ public TrainingDatasetDTO createTrainingDataset(Users user, Project project, Fea verifyTrainingDatasetInput(trainingDatasetDTO); Inode inode = null; - FeaturestoreHopsfsConnector hopsfsConnector = null; - FeaturestoreS3Connector s3Connector = null; + FeaturestoreConnector featurestoreConnector; if(trainingDatasetDTO.getTrainingDatasetType() == TrainingDatasetType.HOPSFS_TRAINING_DATASET) { - if (trainingDatasetDTO.getStorageConnectorId() == null) { - hopsfsConnector = hopsfsConnectorController.getDefaultStorageConnector(featurestore); + if (trainingDatasetDTO.getStorageConnector() == null) { + String connectorName = + featurestore.getProject().getName() + "_" + Settings.ServiceDataset.TRAININGDATASETS.getName(); + featurestoreConnector = featurestoreConnectorFacade.findByFeaturestoreName(featurestore, connectorName) + .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, + Level.FINE, "HOPSFS Connector: " + connectorName)); } else { - hopsfsConnector = hopsfsConnectorFacade. - findByIdAndFeaturestore(trainingDatasetDTO.getStorageConnectorId(), featurestore) + featurestoreConnector = featurestoreConnectorFacade + .findByIdType(trainingDatasetDTO.getStorageConnector().getId(), FeaturestoreConnectorType.HOPSFS) .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.HOPSFS_CONNECTOR_NOT_FOUND, - Level.FINE, "HOPSFS Connector: " + trainingDatasetDTO.getStorageConnectorId())); + Level.FINE, "HOPSFS Connector: " + trainingDatasetDTO.getStorageConnector().getId())); } - Dataset trainingDatasetsFolder = hopsfsConnector.getHopsfsDataset(); + Dataset trainingDatasetsFolder = featurestoreConnector.getHopsfsConnector().getHopsfsDataset(); // TODO(Fabio) account for path String trainingDatasetPath = getTrainingDatasetPath( @@ -255,8 +252,9 @@ public TrainingDatasetDTO createTrainingDataset(Users user, Project project, Fea udfso.mkdir(trainingDatasetPath); inode = inodeController.getInodeAtPath(trainingDatasetPath); - TrainingDatasetDTO completeTrainingDatasetDTO = createTrainingDatasetMetadata(project, user, featurestore, - trainingDatasetDTO, hopsfsConnector, inode, s3Connector); + TrainingDatasetDTO completeTrainingDatasetDTO = + createTrainingDatasetMetadata(project, user, featurestore, + trainingDatasetDTO, featurestoreConnector, inode); fsProvenanceController.trainingDatasetAttachXAttr(trainingDatasetPath, completeTrainingDatasetDTO, udfso); return completeTrainingDatasetDTO; } finally { @@ -265,11 +263,17 @@ public TrainingDatasetDTO createTrainingDataset(Users user, Project project, Fea } } } else { - s3Connector = S3ConnectorFacade.findByIdAndFeaturestore(trainingDatasetDTO.getStorageConnectorId(), featurestore) + if (trainingDatasetDTO.getStorageConnector() == null) { + throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.S3_CONNECTOR_NOT_FOUND, + Level.FINE, "Storage connector is empty"); + } + + featurestoreConnector = featurestoreConnectorFacade + .findByIdType(trainingDatasetDTO.getStorageConnector().getId(), FeaturestoreConnectorType.S3) .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.S3_CONNECTOR_NOT_FOUND, - Level.FINE, "S3 connector: " + trainingDatasetDTO.getStorageConnectorId())); - return createTrainingDatasetMetadata(project, user, featurestore, trainingDatasetDTO, hopsfsConnector, inode, - s3Connector); + Level.FINE, "S3 connector: " + trainingDatasetDTO.getStorageConnector().getId())); + return createTrainingDatasetMetadata(project, user, featurestore, + trainingDatasetDTO, featurestoreConnector, inode); } } @@ -285,9 +289,8 @@ public TrainingDatasetDTO createTrainingDataset(Users user, Project project, Fea @TransactionAttribute(TransactionAttributeType.REQUIRED) private TrainingDatasetDTO createTrainingDatasetMetadata(Project project, Users user, Featurestore featurestore, TrainingDatasetDTO trainingDatasetDTO, - FeaturestoreHopsfsConnector hopsfsConnector, - Inode inode, - FeaturestoreS3Connector S3Connector) + FeaturestoreConnector featurestoreConnector, + Inode inode) throws FeaturestoreException, ServiceException { //Create specific dataset type HopsfsTrainingDataset hopsfsTrainingDataset = null; @@ -295,10 +298,10 @@ private TrainingDatasetDTO createTrainingDatasetMetadata(Project project, Users switch (trainingDatasetDTO.getTrainingDatasetType()) { case HOPSFS_TRAINING_DATASET: hopsfsTrainingDataset = - hopsfsTrainingDatasetFacade.createHopsfsTrainingDataset(hopsfsConnector, inode); + hopsfsTrainingDatasetFacade.createHopsfsTrainingDataset(featurestoreConnector, inode); break; case EXTERNAL_TRAINING_DATASET: - externalTrainingDataset = externalTrainingDatasetFacade.createExternalTrainingDataset(S3Connector, + externalTrainingDataset = externalTrainingDatasetFacade.createExternalTrainingDataset(featurestoreConnector, trainingDatasetDTO.getLocation()); break; default: diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetDTO.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetDTO.java index a2233a6d1f..37de604c49 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetDTO.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetDTO.java @@ -18,11 +18,11 @@ import io.hops.hopsworks.common.featurestore.feature.TrainingDatasetFeatureDTO; import io.hops.hopsworks.common.featurestore.query.QueryDTO; +import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.split.TrainingDatasetSplitDTO; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetType; import io.hops.hopsworks.common.featurestore.FeaturestoreEntityDTO; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -43,11 +43,7 @@ public class TrainingDatasetDTO extends FeaturestoreEntityDTO { private List splits = new ArrayList<>(); private Long seed = null; - private Integer storageConnectorId; // Need to know which storage connector to use it - // ID + type would be unique. However momentarily keep also the name, until we switch to - // rest api v2 with expansion on the storage controller. - private String storageConnectorName; - private FeaturestoreStorageConnectorType storageConnectorType; + private FeaturestoreStorageConnectorDTO storageConnector; // This is here for the frontend. The frontend uses a rest call to get the total size of // a subdirectory - the rest call requires the inode id. @@ -86,28 +82,12 @@ public void setDataFormat(String dataFormat) { this.dataFormat = dataFormat; } - public Integer getStorageConnectorId() { - return storageConnectorId; + public FeaturestoreStorageConnectorDTO getStorageConnector() { + return storageConnector; } - public void setStorageConnectorId(Integer storageConnectorId) { - this.storageConnectorId = storageConnectorId; - } - - public String getStorageConnectorName() { - return storageConnectorName; - } - - public void setStorageConnectorName(String storageConnectorName) { - this.storageConnectorName = storageConnectorName; - } - - public FeaturestoreStorageConnectorType getStorageConnectorType() { - return storageConnectorType; - } - - public void setStorageConnectorType(FeaturestoreStorageConnectorType storageConnectorType) { - this.storageConnectorType = storageConnectorType; + public void setStorageConnector(FeaturestoreStorageConnectorDTO storageConnector) { + this.storageConnector = storageConnector; } public Long getInodeId() { @@ -178,9 +158,6 @@ public String toString() { ", trainingDatasetType=" + trainingDatasetType + ", splits=" + splits + ", seed=" + seed + - ", storageConnectorId=" + storageConnectorId + - ", storageConnectorName='" + storageConnectorName + '\'' + - ", storageConnectorType=" + storageConnectorType + ", inodeId=" + inodeId + '}'; } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetController.java index 7f030b8f6a..ffdee65aab 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetController.java @@ -16,11 +16,11 @@ package io.hops.hopsworks.common.featurestore.trainingdatasets.external; +import io.hops.hopsworks.common.featurestore.storageconnectors.s3.FeaturestoreS3ConnectorDTO; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.external.ExternalTrainingDataset; import com.google.common.base.Strings; import io.hops.hopsworks.common.featurestore.FeaturestoreConstants; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; import javax.ejb.EJB; @@ -58,10 +58,10 @@ public void removeExternalTrainingDataset(ExternalTrainingDataset externalTraini public TrainingDatasetDTO convertExternalTrainingDatasetToDTO(TrainingDatasetDTO trainingDatasetDTO, TrainingDataset trainingDataset) { ExternalTrainingDataset externalTrainingDataset = trainingDataset.getExternalTrainingDataset(); + FeaturestoreS3ConnectorDTO featurestoreS3ConnectorDTO = + new FeaturestoreS3ConnectorDTO(externalTrainingDataset.getFeaturestoreConnector()); - trainingDatasetDTO.setStorageConnectorId(externalTrainingDataset.getFeaturestoreS3Connector().getId()); - trainingDatasetDTO.setStorageConnectorName(externalTrainingDataset.getFeaturestoreS3Connector().getName()); - trainingDatasetDTO.setStorageConnectorType(FeaturestoreStorageConnectorType.S3); + trainingDatasetDTO.setStorageConnector(featurestoreS3ConnectorDTO); trainingDatasetDTO.setLocation(buildDatasetPath(trainingDataset)); return trainingDatasetDTO; @@ -78,8 +78,8 @@ private String buildDatasetPath(TrainingDataset trainingDataset) { bucketFolder = trainingDataset.getExternalTrainingDataset().getPath(); } - return "s3://" + Paths.get(trainingDataset.getExternalTrainingDataset().getFeaturestoreS3Connector().getBucket(), - bucketFolder, + return "s3://" + Paths.get(trainingDataset.getExternalTrainingDataset() + .getFeaturestoreConnector().getS3Connector().getBucket(), bucketFolder, trainingDataset.getName() + "_" + trainingDataset.getVersion()).toString(); } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetFacade.java index d73155bcc6..d976e7557e 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/external/ExternalTrainingDatasetFacade.java @@ -17,7 +17,7 @@ package io.hops.hopsworks.common.featurestore.trainingdatasets.external; import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.external.ExternalTrainingDataset; import javax.ejb.Stateless; @@ -44,9 +44,9 @@ public ExternalTrainingDatasetFacade() { * @param path * @return */ - public ExternalTrainingDataset createExternalTrainingDataset(FeaturestoreS3Connector connector, String path) { + public ExternalTrainingDataset createExternalTrainingDataset(FeaturestoreConnector connector, String path) { ExternalTrainingDataset externalTrainingDataset = new ExternalTrainingDataset(); - externalTrainingDataset.setFeaturestoreS3Connector(connector); + externalTrainingDataset.setFeaturestoreConnector(connector); externalTrainingDataset.setPath(path); em.persist(externalTrainingDataset); em.flush(); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetController.java index be0ae59bf5..fe3d3554cd 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetController.java @@ -18,7 +18,7 @@ import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; import com.logicalclocks.servicediscoverclient.service.Service; -import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType; +import io.hops.hopsworks.common.featurestore.storageconnectors.hopsfs.FeaturestoreHopsfsConnectorDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.inode.InodeController; @@ -72,9 +72,10 @@ public TrainingDatasetDTO convertHopsfsTrainingDatasetToDTO(TrainingDatasetDTO t namenodeService.getAddress() + ":" + namenodeService.getPort(), inodeController.getPath(hopsfsTrainingDataset.getInode())).toString()); trainingDatasetDTO.setInodeId(hopsfsTrainingDataset.getInode().getId()); - trainingDatasetDTO.setStorageConnectorId(hopsfsTrainingDataset.getFeaturestoreHopsfsConnector().getId()); - trainingDatasetDTO.setStorageConnectorName(hopsfsTrainingDataset.getFeaturestoreHopsfsConnector().getName()); - trainingDatasetDTO.setStorageConnectorType(FeaturestoreStorageConnectorType.HOPSFS); + + FeaturestoreHopsfsConnectorDTO hopsfsConnectorDTO = + new FeaturestoreHopsfsConnectorDTO(hopsfsTrainingDataset.getFeaturestoreConnector()); + trainingDatasetDTO.setStorageConnector(hopsfsConnectorDTO); return trainingDatasetDTO; } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetFacade.java index 7dc753d148..9cb49dda1d 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetFacade.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/trainingdatasets/hopsfs/HopsfsTrainingDatasetFacade.java @@ -17,7 +17,7 @@ package io.hops.hopsworks.common.featurestore.trainingdatasets.hopsfs; import io.hops.hopsworks.common.dao.AbstractFacade; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.hopsfs.HopsfsTrainingDataset; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; @@ -45,10 +45,10 @@ public HopsfsTrainingDatasetFacade() { * @param inode * @return */ - public HopsfsTrainingDataset createHopsfsTrainingDataset(FeaturestoreHopsfsConnector connector, Inode inode) { + public HopsfsTrainingDataset createHopsfsTrainingDataset(FeaturestoreConnector connector, Inode inode) { HopsfsTrainingDataset hopsfsTrainingDataset = new HopsfsTrainingDataset(); hopsfsTrainingDataset.setInode(inode); - hopsfsTrainingDataset.setFeaturestoreHopsfsConnector(connector); + hopsfsTrainingDataset.setFeaturestoreConnector(connector); em.persist(hopsfsTrainingDataset); em.flush(); return hopsfsTrainingDataset; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java index fcb023f7b2..a5399568c3 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java @@ -899,7 +899,7 @@ private Future addServingManager(Project pro */ private void addServiceFeaturestore(Project project, Users user, DistributedFileSystemOps dfso, ProvTypeDTO datasetProvCore) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException, UserException { String featurestoreName = featurestoreController.getOfflineFeaturestoreDbName(project); try { //Create HiveDB for the featurestore diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/secrets/SecretsController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/secrets/SecretsController.java index f19f7a95c7..b7bf4ea588 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/secrets/SecretsController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/security/secrets/SecretsController.java @@ -52,7 +52,7 @@ import java.util.stream.Collectors; @Stateless -@TransactionAttribute(TransactionAttributeType.NEVER) +@TransactionAttribute(TransactionAttributeType.SUPPORTS) /** * Stateless bean for managing Secrets of users * Secrets are encrypted with Hopsworks master encryption password @@ -441,6 +441,7 @@ private String bytes2string(byte[] bytes) { return new String(bytes, Charset.defaultCharset()); } + @TransactionAttribute(TransactionAttributeType.SUPPORTS) public void checkCanAccessSecret(Secret secret, Users user) throws ProjectException { if (secret != null && !projectTeamFacade.isUserMemberOfProject(projectFacade.find(secret.getProjectIdScope()), user)) { diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/Featurestore.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/Featurestore.java index 82a029ceb1..840627644e 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/Featurestore.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/Featurestore.java @@ -16,14 +16,9 @@ package io.hops.hopsworks.persistence.entity.featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; import io.hops.hopsworks.persistence.entity.project.Project; import javax.persistence.Basic; -import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -33,14 +28,12 @@ import javax.persistence.ManyToOne; import javax.persistence.NamedQueries; import javax.persistence.NamedQuery; -import javax.persistence.OneToMany; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; -import java.util.Collection; import java.util.Date; /** @@ -75,18 +68,14 @@ public class Featurestore implements Serializable { @NotNull @Column(name = "hive_db_id") private Long hiveDbId; - @OneToMany(cascade = CascadeType.ALL, mappedBy = "featurestore") - private Collection featurestoreJdbcConnectorConnections; - @OneToMany(cascade = CascadeType.ALL, mappedBy = "featurestore") - private Collection featurestoreS3ConnectorConnections; - @OneToMany(cascade = CascadeType.ALL, mappedBy = "featurestore") - private Collection hopsfsConnections; - @OneToMany(cascade = CascadeType.ALL, mappedBy = "featureStore") - private Collection featureStoreRedshiftConnectorCollection; public Featurestore() { } + public static long getSerialVersionUID() { + return serialVersionUID; + } + public Integer getId() { return id; } @@ -118,41 +107,6 @@ public Long getHiveDbId() { public void setHiveDbId(Long hiveDbId) { this.hiveDbId = hiveDbId; } - - public Collection getFeaturestoreJdbcConnectorConnections() { - return featurestoreJdbcConnectorConnections; - } - - public void setFeaturestoreJdbcConnectorConnections( - Collection featurestoreJdbcConnectorConnections) { - this.featurestoreJdbcConnectorConnections = featurestoreJdbcConnectorConnections; - } - - public Collection getFeaturestoreS3ConnectorConnections() { - return featurestoreS3ConnectorConnections; - } - - public void setFeaturestoreS3ConnectorConnections( - Collection featurestoreS3ConnectorConnections) { - this.featurestoreS3ConnectorConnections = featurestoreS3ConnectorConnections; - } - - public Collection getHopsfsConnections() { - return hopsfsConnections; - } - - public void setHopsfsConnections(Collection hopsfsConnections) { - this.hopsfsConnections = hopsfsConnections; - } - - public Collection getFeatureStoreRedshiftConnectorCollection() { - return featureStoreRedshiftConnectorCollection; - } - - public void setFeatureStoreRedshiftConnectorCollection( - Collection featureStoreRedshiftConnectorCollection) { - this.featureStoreRedshiftConnectorCollection = featureStoreRedshiftConnectorCollection; - } @Override public boolean equals(Object o) { diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/ondemand/OnDemandFeaturegroup.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/ondemand/OnDemandFeaturegroup.java index 92b73e8027..ecf5301fc8 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/ondemand/OnDemandFeaturegroup.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/featuregroup/ondemand/OnDemandFeaturegroup.java @@ -16,7 +16,7 @@ package io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; import javax.persistence.Basic; @@ -36,6 +36,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; import java.util.Collection; +import java.util.Objects; /** * Entity class representing the on_demand_feature_group table in Hopsworks database. @@ -61,8 +62,8 @@ public class OnDemandFeaturegroup implements Serializable { private String query; @Column(name = "description") private String description; - @JoinColumn(name = "jdbc_connector_id", referencedColumnName = "id") - private FeaturestoreJdbcConnector featurestoreJdbcConnector; + @JoinColumn(name = "connector_id", referencedColumnName = "id") + private FeaturestoreConnector featurestoreConnector; @OneToMany(cascade = CascadeType.ALL, mappedBy = "onDemandFeaturegroup") private Collection features; @@ -92,16 +93,15 @@ public String getDescription() { public void setDescription(String description) { this.description = description; } - - public FeaturestoreJdbcConnector getFeaturestoreJdbcConnector() { - return featurestoreJdbcConnector; + + public FeaturestoreConnector getFeaturestoreConnector() { + return featurestoreConnector; } - - public void setFeaturestoreJdbcConnector( - FeaturestoreJdbcConnector featurestoreJdbcConnector) { - this.featurestoreJdbcConnector = featurestoreJdbcConnector; + + public void setFeaturestoreConnector(FeaturestoreConnector featurestoreConnector) { + this.featurestoreConnector = featurestoreConnector; } - + public Collection getFeatures() { return features; } @@ -124,29 +124,14 @@ public void setInode(Inode inode) { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof OnDemandFeaturegroup)) { - return false; - } + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; OnDemandFeaturegroup that = (OnDemandFeaturegroup) o; - - if (!id.equals(that.id)) { - return false; - } - - if (!query.equals(that.query)) { - return false; - } - if (!description.equals(that.description)) { - return false; - } - if (features != null && !features.equals(that.features)) return false; - return featurestoreJdbcConnector.equals(that.featurestoreJdbcConnector); + + return Objects.equals(id, that.id); } - + @Override public int hashCode() { return id != null ? id.hashCode() : 0; diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java new file mode 100644 index 0000000000..732e9a7580 --- /dev/null +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnector.java @@ -0,0 +1,188 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2020, Logical Clocks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ + +package io.hops.hopsworks.persistence.entity.featurestore.storageconnector; + +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; + +import javax.persistence.Basic; +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.Table; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + +/** + * Entity class representing a feature_store_connector table in Hopsworks database. + * An instance of this class represents a row in the database. + */ +@Entity +@Table(name = "feature_store_connector", catalog = "hopsworks") +@XmlRootElement +@NamedQueries({ + @NamedQuery(name = "FeaturestoreConnector.findAll", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn"), + @NamedQuery(name = "FeaturestoreConnector.findById", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn WHERE fsConn.id = :id"), + @NamedQuery(name = "FeaturestoreConnector.findByIdType", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + + "WHERE fsConn.id = :id AND fsConn.connectorType= :type"), + @NamedQuery(name = "FeaturestoreConnector.findByFeaturestore", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn WHERE fsConn.featurestore = :featurestore"), + @NamedQuery(name = "FeaturestoreConnector.findByFeaturestoreId", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + + "WHERE fsConn.featurestore = :featurestore AND fsConn.id = :id"), + @NamedQuery(name = "FeaturestoreConnector.findByFeaturestoreName", + query = "SELECT fsConn FROM FeaturestoreConnector fsConn " + + "WHERE fsConn.featurestore = :featurestore AND fsConn.name = :name")}) +public class FeaturestoreConnector implements Serializable { + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Basic(optional = false) + @Column(name = "id") + private Integer id; + @JoinColumn(name = "feature_store_id", referencedColumnName = "id") + private Featurestore featurestore; + @Column(name = "description") + private String description; + @Basic(optional = false) + @Column(name = "name") + private String name; + + @Basic(optional = false) + @Column(name = "type") + @Enumerated(EnumType.ORDINAL) + private FeaturestoreConnectorType connectorType; + + @JoinColumn(name = "jdbc_id", referencedColumnName = "id") + @ManyToOne(cascade = CascadeType.ALL) + private FeaturestoreJdbcConnector jdbcConnector; + @JoinColumn(name = "s3_id", referencedColumnName = "id") + @ManyToOne(cascade = CascadeType.ALL) + private FeaturestoreS3Connector s3Connector; + @JoinColumn(name = "hopsfs_id", referencedColumnName = "id") + @ManyToOne(cascade = CascadeType.ALL) + private FeaturestoreHopsfsConnector hopsfsConnector; + @JoinColumn(name = "redshift_id", referencedColumnName = "id") + @ManyToOne(cascade = CascadeType.ALL) + private FeatureStoreRedshiftConnector redshiftConnector; + + public static long getSerialVersionUID() { + return serialVersionUID; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public Featurestore getFeaturestore() { + return featurestore; + } + + public void setFeaturestore(Featurestore featurestore) { + this.featurestore = featurestore; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public FeaturestoreConnectorType getConnectorType() { + return connectorType; + } + + public void setConnectorType(FeaturestoreConnectorType connectorType) { + this.connectorType = connectorType; + } + + public FeaturestoreJdbcConnector getJdbcConnector() { + return jdbcConnector; + } + + public void setJdbcConnector(FeaturestoreJdbcConnector jdbcConnector) { + this.jdbcConnector = jdbcConnector; + } + + public FeaturestoreS3Connector getS3Connector() { + return s3Connector; + } + + public void setS3Connector(FeaturestoreS3Connector s3Connector) { + this.s3Connector = s3Connector; + } + + public FeaturestoreHopsfsConnector getHopsfsConnector() { + return hopsfsConnector; + } + + public void setHopsfsConnector(FeaturestoreHopsfsConnector hopsfsConnector) { + this.hopsfsConnector = hopsfsConnector; + } + + public FeatureStoreRedshiftConnector getRedshiftConnector() { + return redshiftConnector; + } + + public void setRedshiftConnector(FeatureStoreRedshiftConnector redshiftConnector) { + this.redshiftConnector = redshiftConnector; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FeaturestoreConnector that = (FeaturestoreConnector) o; + + return id.equals(that.id); + } + + @Override + public int hashCode() { + return id != null ? id.hashCode() : 0; + } +} diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorType.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnectorType.java similarity index 65% rename from hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorType.java rename to hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnectorType.java index c8f9ac4a9d..ddd380ed93 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorType.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/FeaturestoreConnectorType.java @@ -1,6 +1,6 @@ /* * This file is part of Hopsworks - * Copyright (C) 2019, Logical Clocks AB. All rights reserved + * Copyright (C) 2020, Logical Clocks AB. All rights reserved * * Hopsworks is free software: you can redistribute it and/or modify it under the terms of * the GNU Affero General Public License as published by the Free Software Foundation, @@ -14,20 +14,11 @@ * If not, see . */ -package io.hops.hopsworks.common.featurestore.storageconnectors; +package io.hops.hopsworks.persistence.entity.featurestore.storageconnector; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Type of Storage Connector - */ -public enum FeaturestoreStorageConnectorType { - @JsonProperty("HopsFS") - HOPSFS, - @JsonProperty("JDBC") +public enum FeaturestoreConnectorType { JDBC, - @JsonProperty("REDSHIFT") - REDSHIFT, - @JsonProperty("S3") - S3; + HOPSFS, + S3, + REDSHIFT; } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/hopsfs/FeaturestoreHopsfsConnector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/hopsfs/FeaturestoreHopsfsConnector.java index 660275eb06..e45112ba6b 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/hopsfs/FeaturestoreHopsfsConnector.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/hopsfs/FeaturestoreHopsfsConnector.java @@ -17,7 +17,6 @@ package io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs; import io.hops.hopsworks.persistence.entity.dataset.Dataset; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import javax.persistence.Basic; import javax.persistence.Column; @@ -27,8 +26,6 @@ import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; import javax.persistence.Table; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; @@ -40,19 +37,6 @@ @Entity @Table(name = "feature_store_hopsfs_connector", catalog = "hopsworks") @XmlRootElement -@NamedQueries({ - @NamedQuery(name = "FeaturestoreHopsfsConnector.findAll", query = "SELECT fshopsfs FROM " + - "FeaturestoreHopsfsConnector " + - "fshopsfs"), - @NamedQuery(name = "FeaturestoreHopsfsConnector.findById", - query = "SELECT fshopsfs FROM FeaturestoreHopsfsConnector fshopsfs WHERE fshopsfs.id = :id"), - @NamedQuery(name = "FeaturestoreHopsfsConnector.findByFeaturestore", query = "SELECT fshopsfs " + - "FROM FeaturestoreHopsfsConnector fshopsfs WHERE fshopsfs.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreHopsfsConnector.findByNameAndFeaturestore", query = "SELECT fshopsfs " + - "FROM FeaturestoreHopsfsConnector fshopsfs " + - "WHERE fshopsfs.featurestore = :featurestore AND fshopsfs.name = :name"), - @NamedQuery(name = "FeaturestoreHopsfsConnector.findByFeaturestoreAndId", query = "SELECT fshopsfs " + - "FROM FeaturestoreHopsfsConnector fshopsfs WHERE fshopsfs.featurestore = :featurestore AND fshopsfs.id = :id")}) public class FeaturestoreHopsfsConnector implements Serializable { private static final long serialVersionUID = 1L; @Id @@ -60,16 +44,9 @@ public class FeaturestoreHopsfsConnector implements Serializable { @Basic(optional = false) @Column(name = "id") private Integer id; - @JoinColumn(name = "feature_store_id", referencedColumnName = "id") - private Featurestore featurestore; @JoinColumn(name = "hopsfs_dataset", referencedColumnName = "id") @ManyToOne(optional = false) private Dataset hopsfsDataset; - @Column(name = "description") - private String description; - @Basic(optional = false) - @Column(name = "name") - private String name; public static long getSerialVersionUID() { return serialVersionUID; @@ -83,14 +60,6 @@ public void setId(Integer id) { this.id = id; } - public Featurestore getFeaturestore() { - return featurestore; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } - public Dataset getHopsfsDataset() { return hopsfsDataset; } @@ -98,41 +67,19 @@ public Dataset getHopsfsDataset() { public void setHopsfsDataset(Dataset hopsfsDataset) { this.hopsfsDataset = hopsfsDataset; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - + @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof FeaturestoreHopsfsConnector)) return false; - - FeaturestoreHopsfsConnector - that = (FeaturestoreHopsfsConnector) o; - - if (!id.equals(that.id)) return false; - if (!featurestore.equals(that.featurestore)) return false; - return hopsfsDataset.equals(that.hopsfsDataset); + if (o == null || getClass() != o.getClass()) return false; + + FeaturestoreHopsfsConnector that = (FeaturestoreHopsfsConnector) o; + + return id.equals(that.id); } - + @Override public int hashCode() { - int result = id.hashCode(); - result = 31 * result + featurestore.hashCode(); - result = 31 * result + hopsfsDataset.hashCode(); - return result; + return id.hashCode(); } } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/jdbc/FeaturestoreJdbcConnector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/jdbc/FeaturestoreJdbcConnector.java index 5a412dc17d..9fe2791681 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/jdbc/FeaturestoreJdbcConnector.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/jdbc/FeaturestoreJdbcConnector.java @@ -16,17 +16,12 @@ package io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; - import javax.persistence.Basic; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; import javax.persistence.Table; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; @@ -38,20 +33,6 @@ @Entity @Table(name = "feature_store_jdbc_connector", catalog = "hopsworks") @XmlRootElement -@NamedQueries({ - @NamedQuery(name = "FeaturestoreJdbcConnector.findAll", query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector " + - "fsjdbc"), - @NamedQuery(name = "FeaturestoreJdbcConnector.findById", - query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.id = :id"), - @NamedQuery(name = "FeaturestoreJdbcConnector.findByFeaturestore", query = "SELECT fsjdbc " + - "FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreJdbcConnector.findByNameAndFeaturestore", - query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector fsjdbc WHERE lower(fsjdbc.name) = lower(:name) AND " + - "fsjdbc.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreJdbcConnector.findByFeaturestoreAndId", query = "SELECT fsjdbc " + - "FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.featurestore = :featurestore AND fsjdbc.id = :id"), - @NamedQuery(name = "FeaturestoreJdbcConnector.findByName", query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector " + - "fsjdbc WHERE lower(fsjdbc.name) = lower(:name)")}) public class FeaturestoreJdbcConnector implements Serializable { private static final long serialVersionUID = 1L; @Id @@ -59,18 +40,11 @@ public class FeaturestoreJdbcConnector implements Serializable { @Basic(optional = false) @Column(name = "id") private Integer id; - @JoinColumn(name = "feature_store_id", referencedColumnName = "id") - private Featurestore featurestore; @Basic(optional = false) @Column(name = "connection_string") private String connectionString; @Column(name = "arguments") private String arguments; - @Column(name = "description") - private String description; - @Basic(optional = false) - @Column(name = "name") - private String name; public static long getSerialVersionUID() { return serialVersionUID; @@ -83,15 +57,7 @@ public Integer getId() { public void setId(Integer id) { this.id = id; } - - public Featurestore getFeaturestore() { - return featurestore; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } - + public String getConnectionString() { return connectionString; } @@ -107,23 +73,7 @@ public String getArguments() { public void setArguments(String arguments) { this.arguments = arguments; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - + @Override public boolean equals(Object o) { if (this == o) return true; @@ -132,7 +82,6 @@ public boolean equals(Object o) { FeaturestoreJdbcConnector that = (FeaturestoreJdbcConnector) o; if (!id.equals(that.id)) return false; - if (!featurestore.equals(that.featurestore)) return false; if (!connectionString.equals(that.connectionString)) return false; return connectionString.equals(that.connectionString); } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/redshift/FeatureStoreRedshiftConnector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/redshift/FeatureStoreRedshiftConnector.java index 65950e8827..3a763b63c3 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/redshift/FeatureStoreRedshiftConnector.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/redshift/FeatureStoreRedshiftConnector.java @@ -15,7 +15,6 @@ */ package io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.user.security.secrets.Secret; import javax.persistence.Basic; @@ -28,8 +27,6 @@ import javax.persistence.JoinColumn; import javax.persistence.JoinColumns; import javax.persistence.ManyToOne; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; import javax.persistence.Table; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; @@ -39,50 +36,6 @@ @Entity @Table(name = "feature_store_redshift_connector", catalog = "hopsworks") @XmlRootElement -@NamedQueries({ - @NamedQuery(name = "FeatureStoreRedshiftConnector.findAll", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findById", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.id = :id") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByNameAndFeaturestore", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.name = :name AND f.featureStore = :featurestore") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByFeaturestore", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.featureStore = :featurestore") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByFeaturestoreAndId", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.id = :id AND f.featureStore = :featurestore") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByClusterIdentifier", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.clusterIdentifier = :clusterIdentifier") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByDatabaseName", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.databaseName = :databaseName") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByDatabasePort", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.databasePort = :databasePort") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByDatabaseUserName", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.databaseUserName = :databaseUserName") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByRedshiftIamRole", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.iamRole = :redshiftIamRole") - , - @NamedQuery(name = "FeatureStoreRedshiftConnector.findByArguments", - query - = "SELECT f FROM FeatureStoreRedshiftConnector f WHERE f.arguments = :arguments")}) public class FeatureStoreRedshiftConnector implements Serializable { private static final long serialVersionUID = 1L; @@ -93,8 +46,7 @@ public class FeatureStoreRedshiftConnector implements Serializable { private Integer id; @Basic(optional = false) @NotNull - @Size(min = 1, - max = 64) + @Size(min = 1, max = 64) @Column(name = "cluster_identifier") private String clusterIdentifier; @Basic(optional = false) @@ -127,15 +79,6 @@ public class FeatureStoreRedshiftConnector implements Serializable { @Size(max = 2000) @Column(name = "arguments") private String arguments; - @JoinColumn(name = "feature_store_id", - referencedColumnName = "id") - @ManyToOne(optional = false) - private Featurestore featureStore; - @Column(name = "description") - private String description; - @Basic(optional = false) - @Column(name = "name") - private String name; @JoinColumns({@JoinColumn(name = "database_pwd_secret_uid", referencedColumnName = "uid"), @JoinColumn(name = "database_pwd_secret_name", referencedColumnName = "secret_name")}) @ManyToOne(cascade = CascadeType.ALL) @@ -261,30 +204,6 @@ public void setArguments(String arguments) { this.arguments = arguments; } - public Featurestore getFeatureStore() { - return featureStore; - } - - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - @Override public int hashCode() { int hash = 0; @@ -304,13 +223,4 @@ public boolean equals(Object object) { } return true; } - - @Override - public String toString() { - return "FeatureStoreRedshiftConnector{" + - "id=" + id + - ", clusterIdentifier='" + clusterIdentifier + '\'' + - ", description='" + description + '\'' + - '}'; - } } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/s3/FeaturestoreS3Connector.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/s3/FeaturestoreS3Connector.java index e84a6a352f..9192fb3be8 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/s3/FeaturestoreS3Connector.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/s3/FeaturestoreS3Connector.java @@ -16,7 +16,6 @@ package io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.user.security.secrets.Secret; import javax.persistence.Basic; @@ -31,8 +30,6 @@ import javax.persistence.JoinColumn; import javax.persistence.JoinColumns; import javax.persistence.ManyToOne; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; import javax.persistence.Table; import javax.validation.constraints.Size; import javax.xml.bind.annotation.XmlRootElement; @@ -45,16 +42,6 @@ @Entity @Table(name = "feature_store_s3_connector", catalog = "hopsworks") @XmlRootElement -@NamedQueries({ - @NamedQuery(name = "FeaturestoreS3Connector.findAll", query = "SELECT fss FROM FeaturestoreS3Connector fss"), - @NamedQuery(name = "FeaturestoreS3Connector.findById", - query = "SELECT fss FROM FeaturestoreS3Connector fss WHERE fss.id = :id"), - @NamedQuery(name = "FeaturestoreS3Connector.findByFeaturestore", query = "SELECT fss " + - "FROM FeaturestoreS3Connector fss WHERE fss.featurestore = :featurestore"), - @NamedQuery(name = "FeaturestoreS3Connector.findByFeaturestoreAndName", query = "SELECT fss " + - "FROM FeaturestoreS3Connector fss WHERE fss.featurestore = :featurestore AND fss.name = :name"), - @NamedQuery(name = "FeaturestoreS3Connector.findByFeaturestoreAndId", query = "SELECT fss " + - "FROM FeaturestoreS3Connector fss WHERE fss.featurestore = :featurestore AND fss.id = :id")}) public class FeaturestoreS3Connector implements Serializable { private static final long serialVersionUID = 1L; @Id @@ -62,15 +49,8 @@ public class FeaturestoreS3Connector implements Serializable { @Basic(optional = false) @Column(name = "id") private Integer id; - @JoinColumn(name = "feature_store_id", referencedColumnName = "id") - private Featurestore featurestore; @Column(name = "bucket") private String bucket; - @Column(name = "description") - private String description; - @Basic(optional = false) - @Column(name = "name") - private String name; @Size(max = 2048) @Column(name = "iam_role") private String iamRole; @@ -96,14 +76,6 @@ public void setId(Integer id) { this.id = id; } - public Featurestore getFeaturestore() { - return featurestore; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } - public String getBucket() { return bucket; } @@ -112,22 +84,6 @@ public void setBucket(String bucket) { this.bucket = bucket; } - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - public String getIamRole() { return iamRole; } @@ -145,7 +101,7 @@ public void setServerEncryptionAlgorithm(FeaturestoreS3ConnectorEncryptionAlgori public String getServerEncryptionKey() { return serverEncryptionKey; } public void setServerEncryptionKey(String serverEncryptionKey) { this.serverEncryptionKey = serverEncryptionKey; } - + public Secret getSecret() { return secret; } @@ -156,29 +112,16 @@ public void setSecret(Secret secret) { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof FeaturestoreS3Connector)) { - return false; - } - + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FeaturestoreS3Connector that = (FeaturestoreS3Connector) o; - - if (!id.equals(that.id)) { - return false; - } - if (!featurestore.equals(that.featurestore)) { - return false; - } - return bucket.equals(that.bucket); + + return id.equals(that.id); } - + @Override public int hashCode() { - int result = id.hashCode(); - result = 31 * result + featurestore.hashCode(); - result = 31 * result + bucket.hashCode(); - return result; + return id.hashCode(); } } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/external/ExternalTrainingDataset.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/external/ExternalTrainingDataset.java index 516a5ccb45..4431caad22 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/external/ExternalTrainingDataset.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/external/ExternalTrainingDataset.java @@ -16,7 +16,7 @@ package io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.external; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import javax.persistence.Basic; import javax.persistence.Column; @@ -53,8 +53,8 @@ public class ExternalTrainingDataset implements Serializable { private Integer id; @Column(name = "path") private String path; - @JoinColumn(name = "s3_connector_id", referencedColumnName = "id") - private FeaturestoreS3Connector featurestoreS3Connector; + @JoinColumn(name = "connector_id", referencedColumnName = "id") + private FeaturestoreConnector featurestoreConnector; public Integer getId() { return id; @@ -69,14 +69,13 @@ public void setId(Integer id) { public void setPath(String path) { this.path = path; } - - public FeaturestoreS3Connector getFeaturestoreS3Connector() { - return featurestoreS3Connector; + + public FeaturestoreConnector getFeaturestoreConnector() { + return featurestoreConnector; } - - public void setFeaturestoreS3Connector( - FeaturestoreS3Connector featurestoreS3Connector) { - this.featurestoreS3Connector = featurestoreS3Connector; + + public void setFeaturestoreConnector(FeaturestoreConnector featurestoreConnector) { + this.featurestoreConnector = featurestoreConnector; } @Override @@ -86,14 +85,11 @@ public boolean equals(Object o) { ExternalTrainingDataset that = (ExternalTrainingDataset) o; - if (!Objects.equals(id, that.id)) return false; - return Objects.equals(featurestoreS3Connector, that.featurestoreS3Connector); + return Objects.equals(id, that.id); } @Override public int hashCode() { - int result = id != null ? id.hashCode() : 0; - result = 31 * result + (featurestoreS3Connector != null ? featurestoreS3Connector.hashCode() : 0); - return result; + return id != null ? id.hashCode() : 0; } } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/hopsfs/HopsfsTrainingDataset.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/hopsfs/HopsfsTrainingDataset.java index fd19342090..fd280aeed2 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/hopsfs/HopsfsTrainingDataset.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/trainingdataset/hopsfs/HopsfsTrainingDataset.java @@ -16,7 +16,7 @@ package io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.hopsfs; -import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.hopsfs.FeaturestoreHopsfsConnector; +import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; import javax.persistence.Basic; @@ -33,6 +33,7 @@ import javax.persistence.Table; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; +import java.util.Objects; /** * Entity class representing the hopsfs_training_dataset table in Hopsworks database. @@ -62,9 +63,9 @@ public class HopsfsTrainingDataset implements Serializable { referencedColumnName = "partition_id")}) @ManyToOne(optional = false) private Inode inode; - @JoinColumn(name = "hopsfs_connector_id", referencedColumnName = "id") - private FeaturestoreHopsfsConnector featurestoreHopsfsConnector; - + @JoinColumn(name = "connector_id", referencedColumnName = "id") + private FeaturestoreConnector featurestoreConnector; + public Integer getId() { return id; } @@ -80,41 +81,27 @@ public Inode getInode() { public void setInode(Inode inode) { this.inode = inode; } - - public FeaturestoreHopsfsConnector getFeaturestoreHopsfsConnector() { - return featurestoreHopsfsConnector; + + public FeaturestoreConnector getFeaturestoreConnector() { + return featurestoreConnector; } - - public void setFeaturestoreHopsfsConnector( - FeaturestoreHopsfsConnector featurestoreHopsfsConnector) { - this.featurestoreHopsfsConnector = featurestoreHopsfsConnector; + + public void setFeaturestoreConnector(FeaturestoreConnector featurestoreConnector) { + this.featurestoreConnector = featurestoreConnector; } - + @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof HopsfsTrainingDataset)) { - return false; - } - + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HopsfsTrainingDataset that = (HopsfsTrainingDataset) o; - - if (!id.equals(that.id)) { - return false; - } - if (!inode.equals(that.inode)) { - return false; - } - return featurestoreHopsfsConnector.equals(that.featurestoreHopsfsConnector); + + return Objects.equals(id, that.id); } - + @Override public int hashCode() { - int result = id.hashCode(); - result = 31 * result + inode.hashCode(); - result = 31 * result + featurestoreHopsfsConnector.hashCode(); - return result; + return id != null ? id.hashCode() : 0; } } diff --git a/hopsworks-persistence/src/main/resources/META-INF/persistence.xml b/hopsworks-persistence/src/main/resources/META-INF/persistence.xml index 717406e49a..58bef8a906 100644 --- a/hopsworks-persistence/src/main/resources/META-INF/persistence.xml +++ b/hopsworks-persistence/src/main/resources/META-INF/persistence.xml @@ -52,6 +52,7 @@ io.hops.hopsworks.persistence.entity.dela.certs.ClusterCertificate io.hops.hopsworks.persistence.entity.featurestore.Featurestore io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeature + io.hops.hopsworks.persistence.entity.featurestore.storageconnector.FeaturestoreConnector io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector io.hops.hopsworks.persistence.entity.featurestore.storageconnector.redshift.FeatureStoreRedshiftConnector io.hops.hopsworks.persistence.entity.featurestore.storageconnector.s3.FeaturestoreS3Connector diff --git a/hopsworks-rest-utils/src/main/java/io/hops/hopsworks/restutils/RESTCodes.java b/hopsworks-rest-utils/src/main/java/io/hops/hopsworks/restutils/RESTCodes.java index 612f858a6f..dd15f35806 100644 --- a/hopsworks-rest-utils/src/main/java/io/hops/hopsworks/restutils/RESTCodes.java +++ b/hopsworks-rest-utils/src/main/java/io/hops/hopsworks/restutils/RESTCodes.java @@ -1366,9 +1366,8 @@ public enum FeaturestoreErrorCode implements RESTErrorCode { Response.Status.BAD_REQUEST), ILLEGAL_FEATURE_DESCRIPTION(41, "Illegal feature description", Response.Status.BAD_REQUEST), - JDBC_CONNECTOR_NOT_FOUND(42, "JDBC Connector not found", - Response.Status.BAD_REQUEST), - JDBC_CONNECTOR_ID_NOT_PROVIDED(43, "JDBC Connector Id was not provided", Response.Status.BAD_REQUEST), + CONNECTOR_NOT_FOUND(42, "Connector not found", Response.Status.BAD_REQUEST), + CONNECTOR_ID_NOT_PROVIDED(43, "Connector Id was not provided", Response.Status.BAD_REQUEST), INVALID_SQL_QUERY(44, "Invalid SQL query", Response.Status.BAD_REQUEST), S3_CONNECTOR_NOT_FOUND(45, "S3 Connector not found", Response.Status.BAD_REQUEST), HOPSFS_CONNECTOR_NOT_FOUND(46, "HopsFs Connector not found", Response.Status.BAD_REQUEST), diff --git a/hopsworks-web/yo/app/scripts/controllers/featurestoreCtrl.js b/hopsworks-web/yo/app/scripts/controllers/featurestoreCtrl.js index b6b7b1964b..eae7005e12 100644 --- a/hopsworks-web/yo/app/scripts/controllers/featurestoreCtrl.js +++ b/hopsworks-web/yo/app/scripts/controllers/featurestoreCtrl.js @@ -293,16 +293,15 @@ angular.module('hopsWorksApp') + "" } } - } + } /** * Delete a storage connector * * @param connector the connector to delete */ self.deleteStorageConnector = function (connector) { - FeaturestoreService.deleteStorageConnector(self.projectId, self.featurestore, connector.name, - connector.storageConnectorType).then( + FeaturestoreService.deleteStorageConnector(self.projectId, self.featurestore, connector.name).then( function (success) { self.getStorageConnectors(self.featurestore); growl.success("Storage connector deleted", {title: 'Success', ttl: 1000}); diff --git a/hopsworks-web/yo/app/scripts/controllers/newStorageConnectorCtrl.js b/hopsworks-web/yo/app/scripts/controllers/newStorageConnectorCtrl.js index 74295b7037..afede08cd4 100644 --- a/hopsworks-web/yo/app/scripts/controllers/newStorageConnectorCtrl.js +++ b/hopsworks-web/yo/app/scripts/controllers/newStorageConnectorCtrl.js @@ -432,8 +432,8 @@ angular.module('hopsWorksApp') }; var createStorageConnector = function (type, storageConnectorJson) { - FeaturestoreService.createStorageConnector(self.projectId, storageConnectorJson, self.featurestore, - type).then( + FeaturestoreService.createStorageConnector(self.projectId, storageConnectorJson, self.featurestore) + .then( function (success) { self.working = false; growl.success(type + " Storage Connector created", { title: 'Success', ttl: 1000 }); @@ -446,7 +446,7 @@ angular.module('hopsWorksApp') var updateStorageConnector = function (type, storageConnectorJson) { FeaturestoreService.updateStorageConnector(self.projectId, storageConnectorJson, self.featurestore, - type, self.storageConnector.name).then( + self.storageConnector.name).then( function (success) { self.working = false; growl.success(type + " Storage Connector updated", { title: 'Success', ttl: 1000 }); diff --git a/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js b/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js index 3ea38362fb..381717b850 100644 --- a/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js +++ b/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js @@ -516,8 +516,9 @@ angular.module('hopsWorksApp') "dataFormat": self.trainingDatasetFormat, "features": self.featureBasket, "trainingDatasetType": self.sinkType === 0 ? self.hopsfsTrainingDatasetType : self.externalTrainingDatasetType, - "storageConnectorId": self.sinkType === 0 ? self.selectedHopsfsConnector.id : self.selectedS3Connector.id, - "storageConnectorName": self.sinkType === 0 ? self.selectedHopsfsConnector.name : self.selectedS3Connector.name, + "storageConnector": { + "id": self.sinkType === 0 ? self.selectedHopsfsConnector.id : self.selectedS3Connector.id + }, "location": self.path, } if(self.configureJob){ diff --git a/hopsworks-web/yo/app/scripts/services/FeaturestoreService.js b/hopsworks-web/yo/app/scripts/services/FeaturestoreService.js index f452e81a84..6fc0ac578d 100644 --- a/hopsworks-web/yo/app/scripts/services/FeaturestoreService.js +++ b/hopsworks-web/yo/app/scripts/services/FeaturestoreService.js @@ -316,13 +316,12 @@ angular.module('hopsWorksApp') * @param projectId project where the featuregroup will be created * @param storageConnectorJson the JSON payload * @param featurestore featurestore where the connector will be created - * @param storageConnectorType the type of the storage connector * * @returns {HttpPromise} */ - createStorageConnector: function(projectId, storageConnectorJson, featurestore, storageConnectorType) { + createStorageConnector: function(projectId, storageConnectorJson, featurestore) { return $http.post('/api/project/' + projectId + '/featurestores/' + - featurestore.featurestoreId + "/storageconnectors/" + storageConnectorType, + featurestore.featurestoreId + "/storageconnectors/", JSON.stringify(storageConnectorJson), {headers: {'Content-Type': 'application/json'}}); }, @@ -333,15 +332,12 @@ angular.module('hopsWorksApp') * @param storageConnectorJson the JSON payload * @param featurestore featurestore where the connector will be created * @param storageConnectorName the name of the connector - * @param storageConnectorType the type of the storage connector * * @returns {HttpPromise} */ - updateStorageConnector: function(projectId, storageConnectorJson, featurestore, storageConnectorType, - storageConnectorName) { + updateStorageConnector: function(projectId, storageConnectorJson, featurestore, storageConnectorName) { return $http.put('/api/project/' + projectId + '/featurestores/' + - featurestore.featurestoreId + "/storageconnectors/" + storageConnectorType + "/" - + storageConnectorName, + featurestore.featurestoreId + "/storageconnectors/" + storageConnectorName, JSON.stringify(storageConnectorJson), {headers: {'Content-Type': 'application/json'}}); }, @@ -351,13 +347,11 @@ angular.module('hopsWorksApp') * @param projectId the project of the featurestore * @param featurestore the featurestore * @param storageConnectorName the name of the connector - * @param storageConnectorType the type of the storage connector * @returns {HttpPromise} */ - deleteStorageConnector: function(projectId, featurestore, storageConnectorName, storageConnectorType) { + deleteStorageConnector: function(projectId, featurestore, storageConnectorName) { return $http.delete('/api/project/' + projectId + '/featurestores/' + - featurestore.featurestoreId + "/storageconnectors/" + storageConnectorType + "/" + - storageConnectorName); + featurestore.featurestoreId + "/storageconnectors/" + storageConnectorName); }, /** * Sends a POST request to the backend for writing args for featurestore util job to HDFS diff --git a/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html b/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html index f115abafc5..9f2657af64 100644 --- a/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html +++ b/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html @@ -168,7 +168,7 @@ S3 Storage Connector Name - {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.storageConnectorName}} + {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.storageConnector.name}} @@ -178,7 +178,7 @@ S3 Storage Connector Id - {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.storageConnectorId}} + {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.storageConnector.name}}