Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Decrease polling frequency for Key Vault tests #20772

Merged
merged 2 commits into from
May 3, 2021
Merged

Decrease polling frequency for Key Vault tests #20772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ecdccd5
Decrease polling frequency for Key Vault tests
heaths Apr 30, 2021
b3923d0
Resolve PR feedback
heaths May 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions sdk/core/Azure.Core.TestFramework/src/OperationInterceptor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,11 @@ private void CheckArguments(object[] invocationArguments)
var interval = (TimeSpan) invocationArguments[0];
if (interval < TimeSpan.FromSeconds(1))
{
throw new InvalidOperationException($"Fast polling interval of {interval} detected in playback mode." +
$"Please use the default WaitForCompletion()." +
throw new InvalidOperationException($"Fast polling interval of {interval} detected in playback mode. " +
$"Please use the default WaitForCompletion(). " +
$"The test framework would automatically reduce the interval in playback.");
}
}
}
}
}
}
4 changes: 2 additions & 2 deletions sdk/keyvault/Azure.Security.KeyVault.Administration/tests/AdministrationTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ protected AdministrationTestBase(bool isAsync, RecordedTestMode? mode)
/// </summary>
protected TimeSpan PollingInterval => Recording.Mode == RecordedTestMode.Playback
? TimeSpan.Zero
: TimeSpan.FromSeconds(2);
: KeyVaultTestEnvironment.DefaultPollingInterval;

[TearDown]
public virtual async Task Cleanup()
Expand Down Expand Up @@ -85,7 +85,7 @@ public async Task DelayAsync(TimeSpan? delay = null, TimeSpan? playbackDelay = n

if (Mode != RecordedTestMode.Playback)
{
await Task.Delay(delay ?? TimeSpan.FromSeconds(1));
await Task.Delay(delay ?? KeyVaultTestEnvironment.DefaultPollingInterval);
}
else if (playbackDelay != null)
{
Expand Down
3 changes: 2 additions & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Administration/tests/BackupRestoreTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using Azure.Security.KeyVault.Tests;
using Azure.Storage;
using Azure.Storage.Sas;
using NUnit.Framework;
Expand Down Expand Up @@ -53,7 +54,7 @@ protected override void Start()

// The service polls every second, so wait a bit to make sure the operation appears completed.
protected async Task WaitForOperationAsync() =>
await DelayAsync(TimeSpan.FromSeconds(2));
await DelayAsync(KeyVaultTestEnvironment.DefaultPollingInterval);

private string GenerateSasToken()
{
Expand Down
44 changes: 26 additions & 18 deletions sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificateClientLiveTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using Azure.Core.TestFramework;
using Azure.Security.KeyVault.Keys.Cryptography;
using Azure.Security.KeyVault.Tests;
using NUnit.Framework;
using Org.BouncyCastle.Asn1;
using Org.BouncyCastle.Asn1.Pkcs;
Expand Down Expand Up @@ -31,6 +32,9 @@ namespace Azure.Security.KeyVault.Certificates.Tests
{
public partial class CertificateClientLiveTests : CertificatesTestBase
{
// The service sends back a Retry-After header of 10s anyway.
private static readonly TimeSpan DefaultCertificateOperationPollingInterval = TimeSpan.FromSeconds(10);

private static MethodInfo s_clearCacheMethod;

public CertificateClientLiveTests(bool isAsync, CertificateClientOptions.ServiceVersion serviceVersion)
Expand Down Expand Up @@ -93,6 +97,7 @@ public async Task VerifyGetCertificateOperation()
RegisterForCleanup(certName);

CertificateOperation getOperation = await Client.GetCertificateOperationAsync(certName);
getOperation = InstrumentOperation(getOperation);

Assert.IsNotNull(getOperation);
}
Expand Down Expand Up @@ -123,7 +128,7 @@ public async Task VerifyCancelCertificateOperation()
}

OperationCanceledException ex = Assert.ThrowsAsync<OperationCanceledException>(
async () => await WaitForCompletion(operation),
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default),
$"Expected exception {nameof(OperationCanceledException)} not thrown. Operation status: {operation?.Properties?.Status}, error: {operation?.Properties?.Error?.Message}");

Assert.AreEqual("The operation was canceled so no value is available.", ex.Message);
Expand Down Expand Up @@ -155,7 +160,7 @@ public async Task VerifyUnexpectedCancelCertificateOperation()
}

OperationCanceledException ex = Assert.ThrowsAsync<OperationCanceledException>(
async () => await WaitForCompletion(operation),
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default),
$"Expected exception {nameof(OperationCanceledException)} not thrown. Operation status: {operation?.Properties?.Status}, error: {operation?.Properties?.Error?.Message}");
Assert.AreEqual("The operation was canceled so no value is available.", ex.Message);

Expand All @@ -178,7 +183,8 @@ public async Task VerifyDeleteCertificateOperation()

await operation.DeleteAsync();

InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(async () => await WaitForCompletion(operation));
InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default));
Assert.AreEqual("The operation was deleted so no value is available.", ex.Message);

Assert.IsTrue(operation.HasCompleted);
Expand Down Expand Up @@ -208,7 +214,8 @@ public async Task VerifyUnexpectedDeleteCertificateOperation()
Assert.Inconclusive("The create operation completed before it could be canceled.");
}

InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(async () => await WaitForCompletion(operation));
InvalidOperationException ex = Assert.ThrowsAsync<InvalidOperationException>(
async () => await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default));
Assert.AreEqual("The operation was deleted so no value is available.", ex.Message);

Assert.IsTrue(operation.HasCompleted);
Expand Down Expand Up @@ -238,11 +245,12 @@ public async Task VerifyCertificateOperationError()
certificatePolicy.IssuerName = issuerName;

operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);
operation = InstrumentOperation(operation);

RegisterForCleanup(certName);

using CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
TimeSpan pollingInterval = TimeSpan.FromSeconds((Mode == RecordedTestMode.Playback) ? 0 : 2);
TimeSpan pollingInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : KeyVaultTestEnvironment.DefaultPollingInterval;

while (!operation.HasCompleted)
{
Expand Down Expand Up @@ -304,7 +312,7 @@ public async Task VerifyGetCertificateCompleted()

RegisterForCleanup(certName);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificateWithPolicy certificateWithPolicy = await Client.GetCertificateAsync(certName);

Expand Down Expand Up @@ -335,7 +343,7 @@ public async Task VerifyGetCertificateCompletedSubsequently()

// Need to call the real async wait method or the sync version of this test fails because it's using the instrumented Client directly.
using CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromMinutes(2));
TimeSpan pollingInterval = TimeSpan.FromSeconds((Mode == RecordedTestMode.Playback) ? 0 : 2);
TimeSpan pollingInterval = Mode == RecordedTestMode.Playback ? TimeSpan.Zero : KeyVaultTestEnvironment.DefaultPollingInterval;

await operation.WaitForCompletionAsync(pollingInterval, cts.Token);
heaths marked this conversation as resolved.
Show resolved Hide resolved

Expand All @@ -358,7 +366,7 @@ public async Task VerifyUpdateCertificate()

RegisterForCleanup(certName);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);
CertificateProperties originalProperties = original.Properties;
Assert.IsTrue(originalProperties.Enabled);
Assert.IsEmpty(originalProperties.Tags);
Expand All @@ -384,7 +392,7 @@ public async Task VerifyDeleteRecoverPurge()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, DefaultPolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand Down Expand Up @@ -551,7 +559,7 @@ public async Task ValidateMergeCertificate()
Assert.AreEqual(csrInfo.Subject.ToString(), serverCertificate.Subject);
Assert.AreEqual(serverCertificateName, mergedServerCertificate.Name);

KeyVaultCertificateWithPolicy completedServerCertificate = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy completedServerCertificate = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.AreEqual(mergedServerCertificate.Name, completedServerCertificate.Name);
CollectionAssert.AreEqual(mergedServerCertificate.Cer, completedServerCertificate.Cer);
Expand Down Expand Up @@ -693,7 +701,7 @@ public async Task VerifyGetCertificatePolicy()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand All @@ -716,7 +724,7 @@ public async Task VerifyUpdateCertificatePolicy()

CertificateOperation operation = await Client.StartCreateCertificateAsync(certName, certificatePolicy);

KeyVaultCertificateWithPolicy original = await WaitForCompletion(operation);
KeyVaultCertificateWithPolicy original = await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

Assert.NotNull(original);

Expand Down Expand Up @@ -766,7 +774,7 @@ public async Task DownloadLatestCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificate certificate = await Client.GetCertificateAsync(name);

Expand Down Expand Up @@ -808,7 +816,7 @@ public async Task DownloadVersionedCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

KeyVaultCertificate certificate = await Client.GetCertificateAsync(name);
string version = certificate.Properties.Version;
Expand All @@ -823,7 +831,7 @@ public async Task DownloadVersionedCertificate(string contentType)
policy.Exportable = false;
operation = await Client.StartCreateCertificateAsync(name, policy);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

certificate = await Client.GetCertificateAsync(name);
Assert.AreNotEqual(version, certificate.Properties.Version);
Expand Down Expand Up @@ -861,7 +869,7 @@ public async Task DownloadNonExportableCertificate(string contentType)
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation);
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

using X509Certificate2 x509certificate = await Client.DownloadCertificateAsync(name);
Assert.IsFalse(x509certificate.HasPrivateKey);
Expand Down Expand Up @@ -892,7 +900,7 @@ public async Task DownloadECDsaCertificateSignRemoteVerifyLocal([EnumValues] Cer
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation, TimeSpan.FromSeconds(5));
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

// Sign data remotely.
byte[] plaintext = Encoding.UTF8.GetBytes(nameof(DownloadECDsaCertificateSignRemoteVerifyLocal));
Expand Down Expand Up @@ -944,7 +952,7 @@ public async Task DownloadECDsaCertificateSignLocalVerifyRemote([EnumValues] Cer
CertificateOperation operation = await Client.StartCreateCertificateAsync(name, policy);
RegisterForCleanup(name);

await WaitForCompletion(operation, TimeSpan.FromSeconds(5));
await operation.WaitForCompletionAsync(DefaultCertificateOperationPollingInterval, default);

// Download the certificate and sign data locally.
byte[] plaintext = Encoding.UTF8.GetBytes(nameof(DownloadECDsaCertificateSignRemoteVerifyLocal));
Expand Down
2 changes: 1 addition & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificateOperationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ private CertificateClient CreateClient(HttpPipelineTransport transport)

private async ValueTask<KeyVaultCertificateWithPolicy> WaitForOperationAsync(CertificateOperation operation)
{
return await operation.WaitForCompletionAsync();
return await operation.WaitForCompletionAsync(TimeSpan.Zero, default);
}

public class MockCredential : TokenCredential
Expand Down
8 changes: 1 addition & 7 deletions sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificatesTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ namespace Azure.Security.KeyVault.Certificates.Tests
[NonParallelizable]
public abstract class CertificatesTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(5);
protected readonly TimeSpan PollingInterval = KeyVaultTestEnvironment.DefaultPollingInterval;
private readonly CertificateClientOptions.ServiceVersion _serviceVersion;

public CertificateClient Client { get; set; }
Expand Down Expand Up @@ -202,12 +202,6 @@ protected async Task PurgeCertificate(string name)
}
}

protected async Task<KeyVaultCertificateWithPolicy> WaitForCompletion(CertificateOperation operation, TimeSpan? pollingInterval = null)
{
pollingInterval ??= TimeSpan.FromSeconds(1);
return await operation.WaitForCompletionAsync(pollingInterval.Value, default).TimeoutAfter(TimeSpan.FromMinutes(1));
}

protected Task WaitForDeletedCertificate(string name)
{
if (Mode == RecordedTestMode.Playback)
Expand Down
4 changes: 2 additions & 2 deletions sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeyClientLiveTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,7 +649,7 @@ public async Task GetDeletedKey()
DeletedKey deletedKey = operation.Value;

// Wait a little longer since live tests are failing with only a 2s delay.
await WaitForDeletedKey(keyName, TimeSpan.FromSeconds(5));
await WaitForDeletedKey(keyName, KeyVaultTestEnvironment.DefaultPollingInterval);

DeletedKey polledSecret = await Client.GetDeletedKeyAsync(keyName);

Expand Down Expand Up @@ -910,7 +910,7 @@ public async Task GetDeletedKeys()
{
// WaitForDeletedKey disables recording, so we can wait concurrently.
// Wait a little longer for deleting keys since tests occasionally fail after max attempts.
deletingKeys.Add(WaitForDeletedKey(deletedKey.Name, delay: TimeSpan.FromSeconds(5)));
deletingKeys.Add(WaitForDeletedKey(deletedKey.Name, delay: KeyVaultTestEnvironment.DefaultPollingInterval));
}

await Task.WhenAll(deletingKeys);
Expand Down
6 changes: 3 additions & 3 deletions sdk/keyvault/Azure.Security.KeyVault.Keys/tests/KeysTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public abstract class KeysTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected TimeSpan PollingInterval => Recording.Mode == RecordedTestMode.Playback
? TimeSpan.Zero
: TimeSpan.FromSeconds(2);
: KeyVaultTestEnvironment.DefaultPollingInterval;

public KeyClient Client { get; private set; }

Expand Down Expand Up @@ -260,7 +260,7 @@ protected Task WaitForDeletedKey(string name, TimeSpan? delay = null)

using (Recording.DisableRecording())
{
delay ??= TimeSpan.FromSeconds(5);
delay ??= KeyVaultTestEnvironment.DefaultPollingInterval;
return TestRetryHelper.RetryAsync(async () => await Client.GetDeletedKeyAsync(name), delay: delay.Value);
}
}
Expand All @@ -274,7 +274,7 @@ protected Task WaitForPurgedKey(string name, TimeSpan? delay = null)

using (Recording.DisableRecording())
{
delay ??= TimeSpan.FromSeconds(5);
delay ??= KeyVaultTestEnvironment.DefaultPollingInterval;
return TestRetryHelper.RetryAsync(async () => {
try
{
Expand Down
2 changes: 1 addition & 1 deletion sdk/keyvault/Azure.Security.KeyVault.Secrets/tests/SecretsTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace Azure.Security.KeyVault.Secrets.Tests
[NonParallelizable]
public abstract class SecretsTestBase : RecordedTestBase<KeyVaultTestEnvironment>
{
protected readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(10);
protected readonly TimeSpan PollingInterval = KeyVaultTestEnvironment.DefaultPollingInterval;
private readonly SecretClientOptions.ServiceVersion _serviceVersion;

public SecretClient Client { get; set; }
Expand Down
5 changes: 5 additions & 0 deletions sdk/keyvault/Azure.Security.KeyVault.Shared/tests/KeyVaultTestEnvironment.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ public class KeyVaultTestEnvironment : TestEnvironment

private const string StorageUriFormat = "https://{0}.blob.core.windows.net";

/// <summary>
/// Gets the default polling interval to use in tests.
/// </summary>
public static TimeSpan DefaultPollingInterval { get; } = TimeSpan.FromSeconds(5);

/// <summary>
/// Gets the URI to Key Vault.
/// </summary>
Expand Down