Skip to content

Commit

Permalink
Decrease polling frequency for Key Vault tests (#20772)
Browse files Browse the repository at this point in the history 
Resolves #20735. Also fixes some slow Certificates mock tests.
  • Loading branch information
@heaths
heaths authored May 3, 2021
1 parent bfb5054 commit fe0bf1a
Show file tree
Hide file tree
Showing 10 changed files with 46 additions and 38 deletions.
6 changes: 3 additions & 3 deletions sdk/core/Azure.Core.TestFramework/src/OperationInterceptor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,11 @@ private void CheckArguments(object[] invocationArguments)
var interval = (TimeSpan) invocationArguments[0];
if (interval < TimeSpan.FromSeconds(1))
{
throw new InvalidOperationException($"Fast polling interval of {interval} detected in playback mode." +
$"Please use the default WaitForCompletion()." +
throw new InvalidOperationException($"Fast polling interval of {interval} detected in playback mode. " +
$"Please use the default WaitForCompletion(). " +
$"The test framework would automatically reduce the interval in playback.");
}
}
}
}
}
}
4 changes: 2 additions & 2 deletions sdk/keyvault/Azure.Security.KeyVault.Administration/tests/AdministrationTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ protected AdministrationTestBase(bool isAsync, RecordedTestMode? mode)
/// </summary>
protected TimeSpan PollingInterval => Recording.Mode == RecordedTestMode.Playback
? TimeSpan.Zero
: TimeSpan.FromSeconds(2);
: KeyVaultTestEnvironment.DefaultPollingInterval;

[TearDown]
public virtual async Task Cleanup()
Expand Down Expand Up @@ -85,7 +85,7 @@ public async Task DelayAsync(TimeSpan? delay = null, TimeSpan? playbackDelay = n

if (Mode != RecordedTestMode.Playback)
{
await Task.Delay(delay ?? TimeSpan.FromSeconds(1));
await Task.Delay(delay ?? KeyVaultTestEnvironment.DefaultPollingInterval);
}
else if (playbackDelay != null)
{
Expand Down
3 changes: 2 additions & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Administration/tests/BackupRestoreTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using Azure.Security.KeyVault.Tests;
using Azure.Storage;
using Azure.Storage.Sas;
using NUnit.Framework;
Expand Down Expand Up @@ -53,7 +54,7 @@ protected override void Start()

// The service polls every second, so wait a bit to make sure the operation appears completed.
protected async Task WaitForOperationAsync() =>
await DelayAsync(TimeSpan.FromSeconds(2));
await DelayAsync(KeyVaultTestEnvironment.DefaultPollingInterval);

private string GenerateSasToken()
{
Expand Down
44 changes: 26 additions & 18 deletions sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificateClientLiveTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using Azure.Core.TestFramework;
using Azure.Security.KeyVault.Keys.Cryptography;
using Azure.Security.KeyVault.Tests;
using NUnit.Framework;
using Org.BouncyCastle.Asn1;
using Org.BouncyCastle.Asn1.Pkcs;
Expand Down Expand Up @@ -31,6 +32,9 @@ namespace Azure.Security.KeyVault.Certificates.Tests
{
public partial class CertificateClientLiveTests : CertificatesTestBase
{
// The service sends back a Retry-After header of 10s anyway.
private static readonly TimeSpan DefaultCertificateOperationPollingInterval = TimeSpan.FromSeconds(10);

private static MethodInfo s_clearCacheMethod;

public CertificateClientLiveTests(bool isAsync, CertificateClientOptions.ServiceVersion serviceVersion)
Expand Down Expand Up @@ -93,6 +97,7 @@ public async Task VerifyGetCertificateOperation()
RegisterForCleanup(certName);

CertificateOperation getOperation = await Client.GetCertificateOperationAsync(certName);
getOperation = InstrumentOperation(getOperation);

Assert.IsNotNull(getOperation);
}
Expand Down Expand Up @@ -123,7 +128,7 @@ public async Task VerifyCancelCertificateOperation()
}

OperationCanceledException ex = Assert.ThrowsAsync<OperationCanceledException>(
async () => await WaitForCompletion(operation),
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default),
$"Expected exception {nameof(OperationCanceledException)} not thrown. Operation status: {operation?.Properties?.Status}, error: {operation?.Properties?.Error?.Message}");

Assert.AreEqual("The operation was canceled so no value is available.", ex.Message);
Expand Down Expand Up @@ -155,7 +160,7 @@ public async Task VerifyUnexpectedCancelCertificateOperation()
}

OperationCanceledException ex = Assert.ThrowsAsync<OperationCanceledException>(
async () => await WaitForCompletion(operation),
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default),
$"Expected exception {nameof(OperationCanceledException)} not thrown. Operation status: {operation?.Properties?.Status}, error: {operation?.Properties?.Error?.Message}");
Assert.AreEqual("The operation was canceled so no value is available.", ex.Message);

Expand All @@ -178,7 +183,8 @@ public async Task VerifyDeleteCertificateOperation()

await operation.DeleteAsync();

InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(async () => await WaitForCompletion(operation));
InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default));
Assert.AreEqual("The operation was deleted so no value is available.", ex.Message);

Assert.IsTrue(operation.HasCompleted);
Expand Down Expand Up @@ -208,7 +214,8 @@ public async Task VerifyUnexpectedDeleteCertificateOperation()
Assert.Inconclusive("The create operation completed before it could be canceled.");
}

InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(async () => await WaitForCompletion(operation));
InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default));
Assert.AreEqual("The operation was deleted so no value is available.", ex.Message);

Assert.IsTrue(operation.HasCompleted);
Expand Down Expand Up @@ -238,11 +245,12 @@ public async Task VerifyCertificateOperationError()
certificatePolicy.IssuerName = issuerName;

operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);
operation = InstrumentOperation(operation);

RegisterForCleanup(certName);

using CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
TimeSpan pollingInterval = TimeSpan.FromSeconds((Mode == RecordedTestMode.Playback) ? 0 : 2);
TimeSpan pollingInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : KeyVaultTestEnvironment.DefaultPollingInterval;

while (!operation.HasCompleted)
{
Expand Down Expand Up @@ -304,7 +312,7 @@ public async Task VerifyGetCertificateCompleted()

RegisterForCleanup(certName);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificateWithPolicy certificateWithPolicy = await Client.GetCertificateAsync(certName);

Expand Down Expand Up @@ -335,7 +343,7 @@ public async Task VerifyGetCertificateCompletedSubsequently()

// Need to call the real async wait method or the sync version of this test fails because it's using the instrumented Client directly.
using CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
TimeSpan pollingInterval = TimeSpan.FromSeconds((Mode == RecordedTestMode.Playback) ? 0 : 2);
TimeSpan pollingInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : KeyVaultTestEnvironment.DefaultPollingInterval;

await operation.WaitForCompletionAsync(pollingInterval, cts.Token);

Expand All @@ -358,7 +366,7 @@ public async Task VerifyUpdateCertificate()

RegisterForCleanup(certName);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);
CertificateProperties originalProperties = original.Properties;
Assert.IsTrue(originalProperties.Enabled);
Assert.IsEmpty(originalProperties.Tags);
Expand All @@ -384,7 +392,7 @@ public async Task VerifyDeleteRecoverPurge()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, DefaultPolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand Down Expand Up @@ -551,7 +559,7 @@ public async Task ValidateMergeCertificate()
Assert.AreEqual(csrInfo.Subject.ToString(), serverCertificate.Subject);
Assert.AreEqual(serverCertificateName, mergedServerCertificate.Name);

KeyVaultCertificateWithPolicy completedServerCertificate = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy completedServerCertificate = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.AreEqual(mergedServerCertificate.Name, completedServerCertificate.Name);
CollectionAssert.AreEqual(mergedServerCertificate.Cer, completedServerCertificate.Cer);
Expand Down Expand Up @@ -693,7 +701,7 @@ public async Task VerifyGetCertificatePolicy()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand All @@ -716,7 +724,7 @@ public async Task VerifyUpdateCertificatePolicy()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand Down Expand Up @@ -766,7 +774,7 @@ public async Task DownloadLatestCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificate certificate = await Client.GetCertificateAsync(name);

Expand Down Expand Up @@ -808,7 +816,7 @@ public async Task DownloadVersionedCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificate certificate = await Client.GetCertificateAsync(name);
string version = certificate.Properties.Version;
Expand All @@ -823,7 +831,7 @@ public async Task DownloadVersionedCertificate(string contentType)
policy.Exportable = false;
operation = await Client.StartCreateCertificateAsync(name, policy);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

certificate = await Client.GetCertificateAsync(name);
Assert.AreNotEqual(version, certificate.Properties.Version);
Expand Down Expand Up @@ -861,7 +869,7 @@ public async Task DownloadNonExportableCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

using X509Certificate2 x509certificate = await Client.DownloadCertificateAsync(name);
Assert.IsFalse(x509certificate.HasPrivateKey);
Expand Down Expand Up @@ -892,7 +900,7 @@ public async Task DownloadECDsaCertificateSignRemoteVerifyLocal([EnumValues] Cer
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation, TimeSpan.FromSeconds(5));
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

// Sign data remotely.
byte[] plaintext = Encoding.UTF8.GetBytes(nameof(DownloadECDsaCertificateSignRemoteVerifyLocal));
Expand Down Expand Up @@ -944,7 +952,7 @@ public async Task DownloadECDsaCertificateSignLocalVerifyRemote([EnumValues] Cer
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation, TimeSpan.FromSeconds(5));
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

// Download the certificate and sign data locally.
byte[] plaintext = Encoding.UTF8.GetBytes(nameof(DownloadECDsaCertificateSignRemoteVerifyLocal));
Expand Down
2 changes: 1 addition & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificateOperationTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ private CertificateClient CreateClient(HttpPipelineTransport transport)

private async ValueTask<KeyVaultCertificateWithPolicy> WaitForOperationAsync(CertificateOperation operation)
{
return await operation.WaitForCompletionAsync();
return await operation.WaitForCompletionAsync(TimeSpan.Zero, default);
}

public class MockCredential : TokenCredential
Expand Down
8 changes: 1 addition & 7 deletions sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificatesTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ namespace Azure.Security.KeyVault.Certificates.Tests
[NonParallelizable]
public abstract class CertificatesTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(5);
protected readonly TimeSpan PollingInterval = KeyVaultTestEnvironment.DefaultPollingInterval;
private readonly CertificateClientOptions.ServiceVersion _serviceVersion;

public CertificateClient Client { get; set; }
Expand Down Expand Up @@ -202,12 +202,6 @@ protected async Task PurgeCertificate(string name)
}
}

protected async Task<KeyVaultCertificateWithPolicy> WaitForCompletion(CertificateOperation operation, TimeSpan? pollingInterval = null)
{
pollingInterval ??= TimeSpan.FromSeconds(1);
return await operation.WaitForCompletionAsync(pollingInterval.Value, default).TimeoutAfter(TimeSpan.FromMinutes(1));
}

protected Task WaitForDeletedCertificate(string name)
{
if (Mode == RecordedTestMode.Playback)
Expand Down
4 changes: 2 additions & 2 deletions sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeyClientLiveTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,7 +649,7 @@ public async Task GetDeletedKey()
DeletedKey deletedKey = operation.Value;

// Wait a little longer since live tests are failing with only a 2s delay.
await WaitForDeletedKey(keyName, TimeSpan.FromSeconds(5));
await WaitForDeletedKey(keyName, KeyVaultTestEnvironment.DefaultPollingInterval);

DeletedKey polledSecret = await Client.GetDeletedKeyAsync(keyName);

Expand Down Expand Up @@ -910,7 +910,7 @@ public async Task GetDeletedKeys()
{
// WaitForDeletedKey disables recording, so we can wait concurrently.
// Wait a little longer for deleting keys since tests occasionally fail after max attempts.
deletingKeys.Add(WaitForDeletedKey(deletedKey.Name, delay: TimeSpan.FromSeconds(5)));
deletingKeys.Add(WaitForDeletedKey(deletedKey.Name, delay: KeyVaultTestEnvironment.DefaultPollingInterval));
}

await Task.WhenAll(deletingKeys);
Expand Down
6 changes: 3 additions & 3 deletions sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeysTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public abstract class KeysTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected TimeSpan PollingInterval => Recording.Mode == RecordedTestMode.Playback
? TimeSpan.Zero
: TimeSpan.FromSeconds(2);
: KeyVaultTestEnvironment.DefaultPollingInterval;

public KeyClient Client { get; private set; }

Expand Down Expand Up @@ -260,7 +260,7 @@ protected Task WaitForDeletedKey(string name, TimeSpan? delay = null)

using (Recording.DisableRecording())
{
delay ??= TimeSpan.FromSeconds(5);
delay ??= KeyVaultTestEnvironment.DefaultPollingInterval;
return TestRetryHelper.RetryAsync(async () => await Client.GetDeletedKeyAsync(name), delay: delay.Value);
}
}
Expand All @@ -274,7 +274,7 @@ protected Task WaitForPurgedKey(string name, TimeSpan? delay = null)

using (Recording.DisableRecording())
{
delay ??= TimeSpan.FromSeconds(5);
delay ??= KeyVaultTestEnvironment.DefaultPollingInterval;
return TestRetryHelper.RetryAsync(async () => {
try
{
Expand Down
2 changes: 1 addition & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Secrets/tests/SecretsTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace Azure.Security.KeyVault.Secrets.Tests
[NonParallelizable]
public abstract class SecretsTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(10);
protected readonly TimeSpan PollingInterval = KeyVaultTestEnvironment.DefaultPollingInterval;
private readonly SecretClientOptions.ServiceVersion _serviceVersion;

public SecretClient Client { get; set; }
Expand Down
5 changes: 5 additions & 0 deletions sdk/keyvault/Azure.Security.KeyVault.Shared/tests/KeyVaultTestEnvironment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ public class KeyVaultTestEnvironment : TestEnvironment

private const string StorageUriFormat = "https://{0}.blob.core.windows.net";

/// <summary>
/// Gets the default polling interval to use in tests.
/// </summary>
public static TimeSpan DefaultPollingInterval { get; } = TimeSpan.FromSeconds(5);

/// <summary>
/// Gets the URI to Key Vault.
/// </summary>
Expand Down

0 comments on commit fe0bf1a

Please sign in to comment.