diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index e643ad90312ab..a7b4dbaa7fbdf 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -10,35 +10,28 @@ include::install_remove.asciidoc[] [[repository-azure-usage]] ==== Azure Repository -To enable Azure repositories, you have first to set your azure storage settings in `elasticsearch.yml` file: +To enable Azure repositories, you have first to define your azure storage settings as +{ref}/secure-settings.html[secured settings]: -[source,yaml] ----- -cloud: - azure: - storage: - my_account: - account: your_azure_storage_account - key: your_azure_storage_key ----- +[source,sh] +---------------------------------------------------------------- +bin/elasticsearch-keystore add azure.client.default.account +bin/elasticsearch-keystore add azure.client.default.key +---------------------------------------------------------------- + +Where `account` is the azure account name and `key` the azure secret key. Note that you can also define more than one account: -[source,yaml] ----- -cloud: - azure: - storage: - my_account1: - account: your_azure_storage_account1 - key: your_azure_storage_key1 - default: true - my_account2: - account: your_azure_storage_account2 - key: your_azure_storage_key2 ----- +[source,sh] +---------------------------------------------------------------- +bin/elasticsearch-keystore add azure.client.default.account +bin/elasticsearch-keystore add azure.client.default.key +bin/elasticsearch-keystore add azure.client.secondary.account +bin/elasticsearch-keystore add azure.client.secondary.key +---------------------------------------------------------------- -`my_account1` is the default account which will be used by a repository unless you set an explicit one. +`default` is the default account name which will be used by a repository unless you set an explicit one. You can set the client side timeout to use when making any single request. It can be defined globally, per account or both. It's not set by default which means that elasticsearch is using the @@ -53,23 +46,13 @@ before retrying after a first timeout or failure. The maximum backoff period is [source,yaml] ---- -cloud: - azure: - storage: - timeout: 10s - my_account1: - account: your_azure_storage_account1 - key: your_azure_storage_key1 - default: true - max_retries: 7 - my_account2: - account: your_azure_storage_account2 - key: your_azure_storage_key2 - timeout: 30s +cloud.azure.storage.timeout: 10s +azure.client.default.max_retries: 7 +azure.client.secondary.timeout: 30s ---- -In this example, timeout will be `10s` per try for `my_account1` with `7` retries before failing -and `30s` per try for `my_account2` with `3` retries. +In this example, timeout will be `10s` per try for `default` with `7` retries before failing +and `30s` per try for `secondary` with `3` retries. [IMPORTANT] .Supported Azure Storage Account types @@ -89,10 +72,9 @@ https://azure.microsoft.com/en-gb/documentation/articles/storage-premium-storage The Azure repository supports following settings: -`account`:: +`client`:: - Azure account settings to use. Defaults to the only one if you set a single - account or to the one marked as `default` if you have more than one. + Azure named client to use. Defaults to `default`. `container`:: @@ -153,14 +135,14 @@ PUT _snapshot/my_backup3 { "type": "azure", "settings": { - "account": "my_account1" + "client": "secondary" } } PUT _snapshot/my_backup4 { "type": "azure", "settings": { - "account": "my_account2", + "client": "secondary", "location_mode": "primary_only" } } diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 60f5bed842e3b..3264b512b2de9 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -45,4 +45,8 @@ thirdPartyAudit.excludes = [ integTestCluster { setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource' setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh' + keystoreSetting 'azure.client.default.account', 'cloudazureresource' + keystoreSetting 'azure.client.default.key', 'abcdefgh' + keystoreSetting 'azure.client.secondary.account', 'cloudazureresource' + keystoreSetting 'azure.client.secondary.key', 'abcdefgh' } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java index 0f32180860c37..868b661b50899 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java @@ -22,7 +22,6 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.storage.AzureStorageService; -import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; @@ -45,7 +44,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { private final AzureStorageService client; - private final String accountName; + private final String clientName; private final LocationMode locMode; private final String container; private final String repositoryName; @@ -56,7 +55,7 @@ public AzureBlobStore(RepositoryMetaData metadata, Settings settings, this.client = client; this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); this.repositoryName = metadata.name(); - this.accountName = Repository.ACCOUNT_SETTING.get(metadata.settings()); + this.clientName = Repository.CLIENT_NAME.get(metadata.settings()); String modeStr = Repository.LOCATION_MODE_SETTING.get(metadata.settings()); if (Strings.hasLength(modeStr)) { @@ -91,7 +90,7 @@ public BlobContainer blobContainer(BlobPath path) { public void delete(BlobPath path) { String keyPath = path.buildAsString(); try { - this.client.deleteFiles(this.accountName, this.locMode, container, keyPath); + this.client.deleteFiles(this.clientName, this.locMode, container, keyPath); } catch (URISyntaxException | StorageException e) { logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage()); } @@ -103,41 +102,41 @@ public void close() { public boolean doesContainerExist(String container) { - return this.client.doesContainerExist(this.accountName, this.locMode, container); + return this.client.doesContainerExist(this.clientName, this.locMode, container); } public void deleteFiles(String container, String path) throws URISyntaxException, StorageException { - this.client.deleteFiles(this.accountName, this.locMode, container, path); + this.client.deleteFiles(this.clientName, this.locMode, container, path); } public boolean blobExists(String container, String blob) throws URISyntaxException, StorageException { - return this.client.blobExists(this.accountName, this.locMode, container, blob); + return this.client.blobExists(this.clientName, this.locMode, container, blob); } public void deleteBlob(String container, String blob) throws URISyntaxException, StorageException { - this.client.deleteBlob(this.accountName, this.locMode, container, blob); + this.client.deleteBlob(this.clientName, this.locMode, container, blob); } public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException, IOException { - return this.client.getInputStream(this.accountName, this.locMode, container, blob); + return this.client.getInputStream(this.clientName, this.locMode, container, blob); } public OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException { - return this.client.getOutputStream(this.accountName, this.locMode, container, blob); + return this.client.getOutputStream(this.clientName, this.locMode, container, blob); } public Map listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException { - return this.client.listBlobsByPrefix(this.accountName, this.locMode, container, keyPath, prefix); + return this.client.listBlobsByPrefix(this.clientName, this.locMode, container, keyPath, prefix); } public void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException { - this.client.moveBlob(this.accountName, this.locMode, container, sourceBlob, targetBlob); + this.client.moveBlob(this.clientName, this.locMode, container, sourceBlob, targetBlob); } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java index 5b6575a4d1471..79455a78c005c 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java @@ -45,12 +45,19 @@ public interface AzureStorageService { ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB); final class Storage { + @Deprecated public static final String PREFIX = "cloud.azure.storage."; + @Deprecated public static final Setting STORAGE_ACCOUNTS = Setting.groupSetting(Storage.PREFIX, Setting.Property.NodeScope); - public static final Setting TIMEOUT_SETTING = - Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope); + /** + * Azure timeout (defaults to -1 minute) + * @deprecated We don't want to support global timeout settings anymore + */ + @Deprecated + static final Setting TIMEOUT_SETTING = + Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope, Property.Deprecated); } boolean doesContainerExist(String account, LocationMode mode, String container); diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index be4b7f9f437e7..c928d79c0c242 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -45,39 +45,57 @@ import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; -import java.security.AccessController; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.Map; public class AzureStorageServiceImpl extends AbstractComponent implements AzureStorageService { - final AzureStorageSettings primaryStorageSettings; - final Map secondariesStorageSettings; + final Map storageSettings; + final Map deprecatedStorageSettings; final Map clients; - public AzureStorageServiceImpl(Settings settings) { + public AzureStorageServiceImpl(Settings settings, Map regularStorageSettings) { super(settings); - Tuple> storageSettings = AzureStorageSettings.parse(settings); - this.primaryStorageSettings = storageSettings.v1(); - this.secondariesStorageSettings = storageSettings.v2(); + if (regularStorageSettings.isEmpty()) { + this.storageSettings = new HashMap<>(); + // We have deprecated settings so we need to migrate them to the new implementation + Tuple> storageSettingsMapTuple = AzureStorageSettings.loadLegacy(settings); + deprecatedStorageSettings = storageSettingsMapTuple.v2(); + if (storageSettingsMapTuple.v1() != null) { + if (storageSettingsMapTuple.v1().getName().equals("default") == false) { + // We add the primary configuration to the list of all settings with its deprecated name in case someone is + // forcing a specific configuration name when creating the repository instance + deprecatedStorageSettings.put(storageSettingsMapTuple.v1().getName(), storageSettingsMapTuple.v1()); + } + // We add the primary configuration to the list of all settings as the "default" one + deprecatedStorageSettings.put("default", storageSettingsMapTuple.v1()); + } else { + // If someone did not register any settings or deprecated settings, they + // basically can't use the plugin + throw new IllegalArgumentException("If you want to use an azure repository, you need to define a client configuration."); + } + + + } else { + this.storageSettings = regularStorageSettings; + this.deprecatedStorageSettings = new HashMap<>(); + } this.clients = new HashMap<>(); logger.debug("starting azure storage client instance"); - // We register the primary client if any - if (primaryStorageSettings != null) { - logger.debug("registering primary client for account [{}]", primaryStorageSettings.getAccount()); - createClient(primaryStorageSettings); + // We register all regular azure clients + for (Map.Entry azureStorageSettingsEntry : this.storageSettings.entrySet()) { + logger.debug("registering regular client for account [{}]", azureStorageSettingsEntry.getKey()); + createClient(azureStorageSettingsEntry.getValue()); } - // We register all secondary clients - for (Map.Entry azureStorageSettingsEntry : secondariesStorageSettings.entrySet()) { - logger.debug("registering secondary client for account [{}]", azureStorageSettingsEntry.getKey()); + // We register all deprecated azure clients + for (Map.Entry azureStorageSettingsEntry : this.deprecatedStorageSettings.entrySet()) { + logger.debug("registering deprecated client for account [{}]", azureStorageSettingsEntry.getKey()); createClient(azureStorageSettingsEntry.getValue()); } } @@ -107,32 +125,25 @@ void createClient(AzureStorageSettings azureStorageSettings) { CloudBlobClient getSelectedClient(String account, LocationMode mode) { logger.trace("selecting a client for account [{}], mode [{}]", account, mode.name()); - AzureStorageSettings azureStorageSettings = null; - - if (this.primaryStorageSettings == null) { - throw new IllegalArgumentException("No primary azure storage can be found. Check your elasticsearch.yml."); - } - - if (Strings.hasLength(account)) { - azureStorageSettings = this.secondariesStorageSettings.get(account); - } - - // if account is not secondary, it's the primary + AzureStorageSettings azureStorageSettings = this.storageSettings.get(account); if (azureStorageSettings == null) { - if (Strings.hasLength(account) == false || primaryStorageSettings.getName() == null || account.equals(primaryStorageSettings.getName())) { - azureStorageSettings = primaryStorageSettings; + // We can't find a client that has been registered using regular settings so we try deprecated client + azureStorageSettings = this.deprecatedStorageSettings.get(account); + if (azureStorageSettings == null) { + // We did not get an account. That's bad. + if (Strings.hasLength(account)) { + throw new IllegalArgumentException("Can not find named azure client [" + account + + "]. Check your elasticsearch.yml."); + } + throw new IllegalArgumentException("Can not find primary/secondary client using deprecated settings. " + + "Check your elasticsearch.yml."); } } - if (azureStorageSettings == null) { - // We did not get an account. That's bad. - throw new IllegalArgumentException("Can not find azure account [" + account + "]. Check your elasticsearch.yml."); - } - CloudBlobClient client = this.clients.get(azureStorageSettings.getAccount()); if (client == null) { - throw new IllegalArgumentException("Can not find an azure client for account [" + account + "]"); + throw new IllegalArgumentException("Can not find an azure client for account [" + azureStorageSettings.getAccount() + "]"); } // NOTE: for now, just set the location mode in case it is different; diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java index 600d5fe97f879..5478ba60e0ea5 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java @@ -22,7 +22,11 @@ import com.microsoft.azure.storage.RetryPolicy; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.AffixSetting; +import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.TimeValue; @@ -32,30 +36,70 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage.STORAGE_ACCOUNTS; public final class AzureStorageSettings { - private static final Setting TIMEOUT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "timeout", - (key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Setting.Property.NodeScope)); - private static final Setting ACCOUNT_SETTING = - Setting.affixKeySetting(Storage.PREFIX, "account", (key) -> Setting.simpleString(key, Setting.Property.NodeScope)); - private static final Setting KEY_SETTING = - Setting.affixKeySetting(Storage.PREFIX, "key", (key) -> Setting.simpleString(key, Setting.Property.NodeScope)); - private static final Setting DEFAULT_SETTING = - Setting.affixKeySetting(Storage.PREFIX, "default", (key) -> Setting.boolSetting(key, false, Setting.Property.NodeScope)); + // prefix for azure client settings + private static final String PREFIX = "azure.client."; + + /** + * Azure account name + */ + public static final AffixSetting ACCOUNT_SETTING = Setting.affixKeySetting(PREFIX, "account", + key -> SecureSetting.secureString(key, null)); + /** * max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */ private static final Setting MAX_RETRIES_SETTING = - Setting.affixKeySetting(Storage.PREFIX, "max_retries", + Setting.affixKeySetting(PREFIX, "max_retries", (key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope)); + /** + * Azure key + */ + public static final AffixSetting KEY_SETTING = Setting.affixKeySetting(PREFIX, "key", + key -> SecureSetting.secureString(key, null)); + + public static final AffixSetting TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "timeout", + (key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Property.NodeScope)); + + + @Deprecated + public static final Setting DEPRECATED_TIMEOUT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "timeout", + (key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Property.NodeScope, Property.Deprecated)); + @Deprecated + public static final Setting DEPRECATED_ACCOUNT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "account", + (key) -> Setting.simpleString(key, Property.NodeScope, Property.Deprecated)); + @Deprecated + public static final Setting DEPRECATED_KEY_SETTING = Setting.affixKeySetting(Storage.PREFIX, "key", + (key) -> Setting.simpleString(key, Property.NodeScope, Property.Deprecated)); + @Deprecated + public static final Setting DEPRECATED_DEFAULT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "default", + (key) -> Setting.boolSetting(key, false, Property.NodeScope, Property.Deprecated)); + + + @Deprecated private final String name; private final String account; private final String key; private final TimeValue timeout; + @Deprecated private final boolean activeByDefault; private final int maxRetries; + public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries) { + this.name = null; + this.account = account; + this.key = key; + this.timeout = timeout; + this.activeByDefault = false; + this.maxRetries = maxRetries; + } + + @Deprecated public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) { this.name = name; this.account = account; @@ -65,6 +109,7 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t this.maxRetries = maxRetries; } + @Deprecated public String getName() { return name; } @@ -81,7 +126,8 @@ public TimeValue getTimeout() { return timeout; } - public boolean isActiveByDefault() { + @Deprecated + public Boolean isActiveByDefault() { return activeByDefault; } @@ -103,39 +149,81 @@ public String toString() { } /** - * Parses settings and read all settings available under cloud.azure.storage.* + * Parses settings and read all legacy settings available under cloud.azure.storage.* * @param settings settings to parse * @return A tuple with v1 = primary storage and v2 = secondary storage */ - public static Tuple> parse(Settings settings) { - List storageSettings = createStorageSettings(settings); + @Deprecated + public static Tuple> loadLegacy(Settings settings) { + List storageSettings = createStorageSettingsDeprecated(settings); return Tuple.tuple(getPrimary(storageSettings), getSecondaries(storageSettings)); } - private static List createStorageSettings(Settings settings) { + /** + * Parses settings and read all settings available under azure.client.* + * @param settings settings to parse + * @return All the named configurations + */ + public static Map load(Settings settings) { + // Get the list of existing named configurations + Set clientNames = settings.getGroups(PREFIX).keySet(); + Map storageSettings = new HashMap<>(); + for (String clientName : clientNames) { + storageSettings.put(clientName, getClientSettings(settings, clientName)); + } + + if (storageSettings.containsKey("default") == false && storageSettings.isEmpty() == false) { + // in case no setting named "default" has been set, let's define our "default" + // as the first named config we get + AzureStorageSettings defaultSettings = storageSettings.values().iterator().next(); + storageSettings.put("default", defaultSettings); + } + return Collections.unmodifiableMap(storageSettings); + } + + // pkg private for tests + /** Parse settings for a single client. */ + static AzureStorageSettings getClientSettings(Settings settings, String clientName) { + try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); + SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) { + return new AzureStorageSettings(account.toString(), key.toString(), + getValue(settings, clientName, TIMEOUT_SETTING), + getValue(settings, clientName, MAX_RETRIES_SETTING)); + } + } + + @Deprecated + private static List createStorageSettingsDeprecated(Settings settings) { // ignore global timeout which has the same prefix but does not belong to any group - Settings groups = Storage.STORAGE_ACCOUNTS.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false)); + Settings groups = STORAGE_ACCOUNTS.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false)); List storageSettings = new ArrayList<>(); for (String groupName : groups.getAsGroups().keySet()) { storageSettings.add( new AzureStorageSettings( groupName, - getValue(settings, groupName, ACCOUNT_SETTING), - getValue(settings, groupName, KEY_SETTING), - getValue(settings, groupName, TIMEOUT_SETTING), - getValue(settings, groupName, DEFAULT_SETTING), + getValue(settings, groupName, DEPRECATED_ACCOUNT_SETTING), + getValue(settings, groupName, DEPRECATED_KEY_SETTING), + getValue(settings, groupName, DEPRECATED_TIMEOUT_SETTING), + getValue(settings, groupName, DEPRECATED_DEFAULT_SETTING), getValue(settings, groupName, MAX_RETRIES_SETTING)) ); } return storageSettings; } - private static T getValue(Settings settings, String groupName, Setting setting) { + private static T getConfigValue(Settings settings, String clientName, + Setting.AffixSetting clientSetting) { + Setting concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName); + return concreteSetting.get(settings); + } + + public static T getValue(Settings settings, String groupName, Setting setting) { Setting.AffixKey k = (Setting.AffixKey) setting.getRawKey(); String fullKey = k.toConcreteKey(groupName).toString(); return setting.getConcreteSetting(fullKey).get(settings); } + @Deprecated private static AzureStorageSettings getPrimary(List settings) { if (settings.isEmpty()) { return null; @@ -162,6 +250,7 @@ private static AzureStorageSettings getPrimary(List settin } } + @Deprecated private static Map getSecondaries(List settings) { Map secondaries = new HashMap<>(); // when only one setting is defined, we don't have secondaries @@ -172,6 +261,6 @@ private static Map getSecondaries(List clientsSettings; + // overridable for tests protected AzureStorageService createStorageService(Settings settings) { - return new AzureStorageServiceImpl(settings); + return new AzureStorageServiceImpl(settings, clientsSettings); + } + + public AzureRepositoryPlugin(Settings settings) { + // eagerly load client settings so that secure settings are read + clientsSettings = AzureStorageSettings.load(settings); } @Override @@ -53,7 +61,12 @@ public Map getRepositories(Environment env, NamedXCo @Override public List> getSettings() { - return Collections.singletonList(AzureStorageService.Storage.STORAGE_ACCOUNTS); + return Arrays.asList( + AzureStorageService.Storage.STORAGE_ACCOUNTS, + AzureStorageSettings.ACCOUNT_SETTING, + AzureStorageSettings.KEY_SETTING, + AzureStorageSettings.TIMEOUT_SETTING + ); } @Override diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 2b99e6a6f8e68..2a02ec5f8d8c8 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -62,7 +62,12 @@ public class AzureRepository extends BlobStoreRepository { public static final String TYPE = "azure"; public static final class Repository { - public static final Setting ACCOUNT_SETTING = Setting.simpleString("account", Property.NodeScope); + + @Deprecated // Replaced by client + public static final Setting ACCOUNT_SETTING = new Setting<>("account", "default", Function.identity(), + Property.NodeScope, Property.Deprecated); + public static final Setting CLIENT_NAME = new Setting<>("client", ACCOUNT_SETTING, Function.identity()); + public static final Setting CONTAINER_SETTING = new Setting<>("container", "elasticsearch-snapshots", Function.identity(), Property.NodeScope); public static final Setting BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java index 35afc77f75242..8f6cdce113e7a 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java @@ -24,7 +24,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase.ThirdParty; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java index 0452380ceb98f..b232ee12e05c4 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java @@ -22,20 +22,31 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.RetryExponentialRetry; import com.microsoft.azure.storage.blob.CloudBlobClient; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import java.net.URI; import java.net.URISyntaxException; +import java.util.Map; import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_ACCOUNT_SETTING; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_DEFAULT_SETTING; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_KEY_SETTING; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_TIMEOUT_SETTING; +import static org.elasticsearch.repositories.azure.AzureSettingsParserTests.getConcreteSetting; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class AzureStorageServiceTests extends ESTestCase { - static final Settings settings = Settings.builder() + + @Deprecated + static final Settings deprecatedSettings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") .put("cloud.azure.storage.azure1.key", "mykey1") .put("cloud.azure.storage.azure1.default", true) @@ -46,78 +57,59 @@ public class AzureStorageServiceTests extends ESTestCase { .put("cloud.azure.storage.azure3.timeout", "30s") .build(); - public void testGetSelectedClientWithNoPrimaryAndSecondary() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.EMPTY); - try { - azureStorageService.getSelectedClient("whatever", LocationMode.PRIMARY_ONLY); - fail("we should have raised an IllegalArgumentException"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("No primary azure storage can be found. Check your elasticsearch.yml.")); - } - } - - public void testGetSelectedClientWithNoSecondary() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder() - .put("cloud.azure.storage.azure1.account", "myaccount1") - .put("cloud.azure.storage.azure1.key", "mykey1") - .build()); - CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); - } - - public void testGetDefaultClientWithNoSecondary() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder() - .put("cloud.azure.storage.azure1.account", "myaccount1") - .put("cloud.azure.storage.azure1.key", "mykey1") - .build()); - CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); - } - - public void testGetSelectedClientPrimary() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); - CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + private MockSecureSettings buildSecureSettings() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure1.key", "mykey1"); + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.key", "mykey2"); + secureSettings.setString("azure.client.azure3.account", "myaccount3"); + secureSettings.setString("azure.client.azure3.key", "mykey3"); + return secureSettings; } - - public void testGetSelectedClientSecondary1() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); - CloudBlobClient client = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure2"))); + private Settings buildSettings() { + Settings settings = Settings.builder() + .setSecureSettings(buildSecureSettings()) + .build(); + return settings; } - public void testGetSelectedClientSecondary2() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); - CloudBlobClient client = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure3"))); - } + public void testReadSecuredSettings() { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("azure.client.azure1.account", "myaccount1"); + secureSettings.setString("azure.client.azure1.key", "mykey1"); + secureSettings.setString("azure.client.azure2.account", "myaccount2"); + secureSettings.setString("azure.client.azure2.key", "mykey2"); + secureSettings.setString("azure.client.azure3.account", "myaccount3"); + secureSettings.setString("azure.client.azure3.key", "mykey3"); + Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - public void testGetDefaultClientWithPrimaryAndSecondaries() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); - CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + Map loadedSettings = AzureStorageSettings.load(settings); + assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default")); } - public void testGetSelectedClientNonExisting() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); + public void testGetSelectedClientWithNoPrimaryAndSecondary() { try { - azureStorageService.getSelectedClient("azure4", LocationMode.PRIMARY_ONLY); + new AzureStorageServiceMock(Settings.EMPTY); fail("we should have raised an IllegalArgumentException"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Can not find azure account [azure4]. Check your elasticsearch.yml.")); + assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration.")); } } - public void testGetSelectedClientDefault() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); - CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY); - assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + public void testGetSelectedClientNonExisting() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(buildSettings()); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + azureStorageService.getSelectedClient("azure4", LocationMode.PRIMARY_ONLY); + }); + assertThat(e.getMessage(), is("Can not find named azure client [azure4]. Check your elasticsearch.yml.")); } public void testGetSelectedClientGlobalTimeout() { Settings timeoutSettings = Settings.builder() - .put(settings) + .setSecureSettings(buildSecureSettings()) .put(AzureStorageService.Storage.TIMEOUT_SETTING.getKey(), "10s") + .put("azure.client.azure3.timeout", "30s") .build(); AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); @@ -125,10 +117,16 @@ public void testGetSelectedClientGlobalTimeout() { assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(10 * 1000)); CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); assertThat(client3.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(30 * 1000)); + + assertSettingDeprecationsAndWarnings(new Setting[]{AzureStorageService.Storage.TIMEOUT_SETTING}); } public void testGetSelectedClientDefaultTimeout() { - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings); + Settings timeoutSettings = Settings.builder() + .setSecureSettings(buildSecureSettings()) + .put("azure.client.azure3.timeout", "30s") + .build(); + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), nullValue()); CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); @@ -136,37 +134,26 @@ public void testGetSelectedClientDefaultTimeout() { } public void testGetSelectedClientNoTimeout() { - Settings timeoutSettings = Settings.builder() - .put("cloud.azure.storage.azure.account", "myaccount") - .put("cloud.azure.storage.azure.key", "mykey") - .build(); - - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); - CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY); + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(buildSettings()); + CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(nullValue())); } public void testGetSelectedClientBackoffPolicy() { - Settings timeoutSettings = Settings.builder() - .put("cloud.azure.storage.azure.account", "myaccount") - .put("cloud.azure.storage.azure.key", "mykey") - .build(); - - AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); - CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY); + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(buildSettings()); + CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue())); assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class)); } public void testGetSelectedClientBackoffPolicyNbRetries() { Settings timeoutSettings = Settings.builder() - .put("cloud.azure.storage.azure.account", "myaccount") - .put("cloud.azure.storage.azure.key", "mykey") + .setSecureSettings(buildSecureSettings()) .put("cloud.azure.storage.azure.max_retries", 7) .build(); AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings); - CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY); + CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue())); assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class)); } @@ -176,7 +163,7 @@ public void testGetSelectedClientBackoffPolicyNbRetries() { */ class AzureStorageServiceMock extends AzureStorageServiceImpl { AzureStorageServiceMock(Settings settings) { - super(settings); + super(settings, AzureStorageSettings.load(settings)); } // We fake the client here @@ -197,4 +184,87 @@ public void testBlobNameFromUri() throws URISyntaxException { name = blobNameFromUri(new URI("https://127.0.0.1/container/path/to/myfile")); assertThat(name, is("path/to/myfile")); } + + // Deprecated settings. We still test them until we remove definitely the deprecated settings + + @Deprecated + public void testGetSelectedClientWithNoSecondary() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .build()); + CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1") + }); + } + + @Deprecated + public void testGetDefaultClientWithNoSecondary() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.builder() + .put("cloud.azure.storage.azure1.account", "myaccount1") + .put("cloud.azure.storage.azure1.key", "mykey1") + .build()); + CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1") + }); + } + + @Deprecated + public void testGetSelectedClientPrimary() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings); + CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + assertDeprecatedWarnings(); + } + + @Deprecated + public void testGetSelectedClientSecondary1() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings); + CloudBlobClient client = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure2"))); + assertDeprecatedWarnings(); + } + + @Deprecated + public void testGetSelectedClientSecondary2() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings); + CloudBlobClient client = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure3"))); + assertDeprecatedWarnings(); + } + + @Deprecated + public void testGetDefaultClientWithPrimaryAndSecondaries() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings); + CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + assertDeprecatedWarnings(); + } + + @Deprecated + public void testGetSelectedClientDefault() { + AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(deprecatedSettings); + CloudBlobClient client = azureStorageService.getSelectedClient("default", LocationMode.PRIMARY_ONLY); + assertThat(client.getEndpoint(), is(URI.create("https://azure1"))); + assertDeprecatedWarnings(); + } + + private void assertDeprecatedWarnings() { + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure3"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure3"), + getConcreteSetting(DEPRECATED_TIMEOUT_SETTING, "azure3") + }); + } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java index b67073d7d389f..17b43715253c8 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java @@ -34,6 +34,9 @@ import static org.hamcrest.Matchers.contains; +/** + * TODO as we moved credentials to secure settings, we don't need anymore to keep this test in 7.x + */ public class AzureStorageSettingsFilterTests extends ESTestCase { static final Settings settings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") @@ -46,7 +49,7 @@ public class AzureStorageSettingsFilterTests extends ESTestCase { .build(); public void testSettingsFiltering() throws IOException { - AzureRepositoryPlugin p = new AzureRepositoryPlugin(); + AzureRepositoryPlugin p = new AzureRepositoryPlugin(settings); SettingsModule module = new SettingsModule(Settings.EMPTY, p.getSettings(), p.getSettingsFilter()); SettingsFilter settingsFilter = ModuleTestCase.bindAndGetInstance(module, SettingsFilter.class); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 0765ea919adf2..75ef13d7d8745 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -23,7 +23,6 @@ import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cluster.metadata.RepositoryMetaData; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java index ac2c51441f9a4..d0fbdb98e0315 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java @@ -19,20 +19,24 @@ package org.elasticsearch.repositories.azure; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.cloud.azure.storage.AzureStorageSettings; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; +import org.elasticsearch.test.ESTestCase; import java.util.Map; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_ACCOUNT_SETTING; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_DEFAULT_SETTING; +import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_KEY_SETTING; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class AzureSettingsParserTests extends LuceneTestCase { +public class AzureSettingsParserTests extends ESTestCase { public void testParseTwoSettingsExplicitDefault() { Settings settings = Settings.builder() @@ -43,7 +47,7 @@ public void testParseTwoSettingsExplicitDefault() { .put("cloud.azure.storage.azure2.key", "mykey2") .build(); - Tuple> tuple = AzureStorageSettings.parse(settings); + Tuple> tuple = AzureStorageSettings.loadLegacy(settings); assertThat(tuple.v1(), notNullValue()); assertThat(tuple.v1().getAccount(), is("myaccount1")); assertThat(tuple.v1().getKey(), is("mykey1")); @@ -51,6 +55,13 @@ public void testParseTwoSettingsExplicitDefault() { assertThat(tuple.v2().get("azure2"), notNullValue()); assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2")); assertThat(tuple.v2().get("azure2").getKey(), is("mykey2")); + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2") + }); } public void testParseUniqueSettings() { @@ -59,11 +70,15 @@ public void testParseUniqueSettings() { .put("cloud.azure.storage.azure1.key", "mykey1") .build(); - Tuple> tuple = AzureStorageSettings.parse(settings); + Tuple> tuple = AzureStorageSettings.loadLegacy(settings); assertThat(tuple.v1(), notNullValue()); assertThat(tuple.v1().getAccount(), is("myaccount1")); assertThat(tuple.v1().getKey(), is("mykey1")); assertThat(tuple.v2().keySet(), hasSize(0)); + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1") + }); } public void testParseTwoSettingsNoDefault() { @@ -75,11 +90,17 @@ public void testParseTwoSettingsNoDefault() { .build(); try { - AzureStorageSettings.parse(settings); + AzureStorageSettings.loadLegacy(settings); fail("Should have failed with a SettingsException (no default data store)"); } catch (SettingsException ex) { assertEquals(ex.getMessage(), "No default Azure data store configured"); } + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"), + }); } public void testParseTwoSettingsTooManyDefaultSet() { @@ -93,17 +114,30 @@ public void testParseTwoSettingsTooManyDefaultSet() { .build(); try { - AzureStorageSettings.parse(settings); + AzureStorageSettings.loadLegacy(settings); fail("Should have failed with a SettingsException (multiple default data stores)"); } catch (SettingsException ex) { assertEquals(ex.getMessage(), "Multiple default Azure data stores configured: [azure1] and [azure2]"); } - + assertSettingDeprecationsAndWarnings(new Setting[]{ + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure1"), + getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_KEY_SETTING, "azure2"), + getConcreteSetting(DEPRECATED_DEFAULT_SETTING, "azure2") + }); } public void testParseEmptySettings() { - Tuple> tuple = AzureStorageSettings.parse(Settings.EMPTY); + Tuple> tuple = AzureStorageSettings.loadLegacy(Settings.EMPTY); assertThat(tuple.v1(), nullValue()); assertThat(tuple.v2().keySet(), hasSize(0)); } + + public static Setting getConcreteSetting(Setting setting, String groupName) { + Setting.AffixKey k = (Setting.AffixKey) setting.getRawKey(); + String concreteKey = k.toConcreteKey(groupName).toString(); + return setting.getConcreteSetting(concreteKey); + } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreListSnapshotsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreListSnapshotsTests.java index e05911cd1e939..6760b418ed50e 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreListSnapshotsTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreListSnapshotsTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyIntegTestCase; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl; +import org.elasticsearch.cloud.azure.storage.AzureStorageSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.azure.AzureRepository.Repository; import org.elasticsearch.test.ESIntegTestCase; @@ -58,7 +59,8 @@ transportClientRatio = 0.0) public class AzureSnapshotRestoreListSnapshotsTests extends AbstractAzureWithThirdPartyIntegTestCase { - private final AzureStorageService azureStorageService = new AzureStorageServiceImpl(readSettingsFromFile()); + private final AzureStorageService azureStorageService = new AzureStorageServiceImpl(readSettingsFromFile(), + AzureStorageSettings.load(readSettingsFromFile())); private final String containerName = getContainerName(); public void testList() throws Exception { diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java index fe4d458cb651a..aea47f38ef3ef 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyIntegTestCase; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl; +import org.elasticsearch.cloud.azure.storage.AzureStorageSettings; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -474,7 +475,8 @@ public void testNonExistingRepo_23() { */ public void testRemoveAndCreateContainer() throws Exception { final String container = getContainerName().concat("-testremove"); - final AzureStorageService storageService = new AzureStorageServiceImpl(internalCluster().getDefaultSettings()); + final AzureStorageService storageService = new AzureStorageServiceImpl(internalCluster().getDefaultSettings(), + AzureStorageSettings.load(internalCluster().getDefaultSettings())); // It could happen that we run this test really close to a previous one // so we might need some time to be able to create the container @@ -528,7 +530,7 @@ public static void wipeRepositories(String... repositories) { */ public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException { Settings settings = readSettingsFromFile(); - AzureStorageService client = new AzureStorageServiceImpl(settings); + AzureStorageService client = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings)); for (String container : containers) { client.removeContainer(null, LocationMode.PRIMARY_ONLY, container); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java index 551599b0918e0..79a9d997cac8a 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java @@ -21,7 +21,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml index e3979cd32c357..2bd98c0a363ed 100644 --- a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml +++ b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml @@ -1,5 +1,9 @@ -"Repository can be registered": +"Deprecated Repository can be registered": + - skip: + features: warnings - do: + warnings: + - "[account] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." snapshot.create_repository: repository: test_repo_azure verify: false @@ -24,3 +28,55 @@ - match : { test_repo_azure.settings.base_path : "backups" } - match : { test_repo_azure.settings.chunk_size: "32m" } - match : { test_repo_azure.settings.compress : "true" } +--- +"Default repository can be registered": + - do: + snapshot.create_repository: + repository: test_repo_azure + verify: false + body: + type: azure + settings: + container : "backup-container" + base_path : "backups" + chunk_size: "32m" + compress : true + + - is_true: acknowledged + + - do: + snapshot.get_repository: + repository: test_repo_azure + + - is_true : test_repo_azure + - match : { test_repo_azure.settings.container : "backup-container" } + - match : { test_repo_azure.settings.base_path : "backups" } + - match : { test_repo_azure.settings.chunk_size: "32m" } + - match : { test_repo_azure.settings.compress : "true" } +--- +"Named client repository can be registered": + - do: + snapshot.create_repository: + repository: test_repo_azure + verify: false + body: + type: azure + settings: + client : "secondary" + container : "backup-container" + base_path : "backups" + chunk_size: "32m" + compress : true + + - is_true: acknowledged + + - do: + snapshot.get_repository: + repository: test_repo_azure + + - is_true : test_repo_azure + - match : { test_repo_azure.settings.client : "secondary" } + - match : { test_repo_azure.settings.container : "backup-container" } + - match : { test_repo_azure.settings.base_path : "backups" } + - match : { test_repo_azure.settings.chunk_size: "32m" } + - match : { test_repo_azure.settings.compress : "true" }