From b1de661f1e1752309fb419887dee51c1ffa1c48f Mon Sep 17 00:00:00 2001 From: Azad Abbasi Date: Tue, 16 Feb 2021 17:55:34 -0800 Subject: [PATCH] Add async methods to the ResolverClient (#18814) --- .../src/Fetchers/LocalModelFetcher.cs | 2 +- .../src/ModelQuery.cs | 147 +++++++++--------- .../src/ModelRepositoryConstants.cs | 2 + .../src/RepositoryHandler.cs | 73 +++++++-- .../src/ResolverClient.cs | 48 +++++- .../tests/ClientTests.cs | 6 +- .../tests/ModelQueryTests.cs | 4 +- 7 files changed, 177 insertions(+), 105 deletions(-) diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/Fetchers/LocalModelFetcher.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/Fetchers/LocalModelFetcher.cs index 496062d3660ad..404df3984e317 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/Fetchers/LocalModelFetcher.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/Fetchers/LocalModelFetcher.cs @@ -65,7 +65,7 @@ public FetchResult Fetch(string dtmi, Uri repositoryUri, CancellationToken cance fnfError = string.Format(CultureInfo.CurrentCulture, ServiceStrings.ErrorFetchingModelContent, tryContentPath); } - throw new RequestFailedException(fnfError, new FileNotFoundException(fnfError)); + throw new FileNotFoundException(fnfError); } catch (Exception ex) { diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelQuery.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelQuery.cs index ddbea223d2cb5..c1b0c33615d42 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelQuery.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelQuery.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Text.Json; -using System.Threading.Tasks; namespace Azure.Iot.ModelsRepository { @@ -29,16 +28,14 @@ public ModelMetadata GetMetadata() public string GetId() { - using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions)) - { - JsonElement _root = document.RootElement; + using JsonDocument document = JsonDocument.Parse(_content, _parseOptions); + JsonElement _root = document.RootElement; - if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("@id", out JsonElement id)) + if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("@id", out JsonElement id)) + { + if (id.ValueKind == JsonValueKind.String) { - if (id.ValueKind == JsonValueKind.String) - { - return id.GetString(); - } + return id.GetString(); } } @@ -49,34 +46,32 @@ public IList GetExtends() { List dependencies = new List(); - using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions)) - { - JsonElement _root = document.RootElement; + using JsonDocument document = JsonDocument.Parse(_content, _parseOptions); + JsonElement _root = document.RootElement; - if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("extends", out JsonElement extends)) + if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("extends", out JsonElement extends)) + { + if (extends.ValueKind == JsonValueKind.Array) { - if (extends.ValueKind == JsonValueKind.Array) + foreach (JsonElement extendElement in extends.EnumerateArray()) { - foreach (JsonElement extendElement in extends.EnumerateArray()) + if (extendElement.ValueKind == JsonValueKind.String) { - if (extendElement.ValueKind == JsonValueKind.String) - { - dependencies.Add(extendElement.GetString()); - } - else if (extendElement.ValueKind == JsonValueKind.Object) - { - // extends can have multiple levels and can contain components. - // TODO: Support object ctor - inefficient serialize. - ModelMetadata nested_interface = new ModelQuery(JsonSerializer.Serialize(extendElement)).GetMetadata(); - dependencies.AddRange(nested_interface.Dependencies); - } + dependencies.Add(extendElement.GetString()); + } + else if (extendElement.ValueKind == JsonValueKind.Object) + { + // extends can have multiple levels and can contain components. + // TODO: Support object ctor - inefficient serialize. + ModelMetadata nested_interface = new ModelQuery(JsonSerializer.Serialize(extendElement)).GetMetadata(); + dependencies.AddRange(nested_interface.Dependencies); } - } - else if (extends.ValueKind == JsonValueKind.String) - { - dependencies.Add(extends.GetString()); } } + else if (extends.ValueKind == JsonValueKind.String) + { + dependencies.Add(extends.GetString()); + } } return dependencies; @@ -87,44 +82,42 @@ public IList GetComponentSchemas() { List componentSchemas = new List(); - using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions)) - { - JsonElement _root = document.RootElement; + using JsonDocument document = JsonDocument.Parse(_content, _parseOptions); + JsonElement _root = document.RootElement; - if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("contents", out JsonElement contents)) + if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("contents", out JsonElement contents)) + { + if (contents.ValueKind == JsonValueKind.Array) { - if (contents.ValueKind == JsonValueKind.Array) + foreach (JsonElement element in contents.EnumerateArray()) { - foreach (JsonElement element in contents.EnumerateArray()) + if (element.TryGetProperty("@type", out JsonElement type)) { - if (element.TryGetProperty("@type", out JsonElement type)) + if (type.ValueKind == JsonValueKind.String && type.GetString() == "Component") { - if (type.ValueKind == JsonValueKind.String && type.GetString() == "Component") + if (element.TryGetProperty("schema", out JsonElement schema)) { - if (element.TryGetProperty("schema", out JsonElement schema)) + if (schema.ValueKind == JsonValueKind.String) { - if (schema.ValueKind == JsonValueKind.String) - { - componentSchemas.Add(schema.GetString()); - } - else if (schema.ValueKind == JsonValueKind.Array) + componentSchemas.Add(schema.GetString()); + } + else if (schema.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement schemaElement in schema.EnumerateArray()) { - foreach (JsonElement schemaElement in schema.EnumerateArray()) + if (schemaElement.ValueKind == JsonValueKind.String) { - if (schemaElement.ValueKind == JsonValueKind.String) - { - componentSchemas.Add(schemaElement.GetString()); - } + componentSchemas.Add(schemaElement.GetString()); } } - else if (schema.ValueKind == JsonValueKind.Object) + } + else if (schema.ValueKind == JsonValueKind.Object) + { + if (schema.TryGetProperty("extends", out JsonElement schemaObjExtends)) { - if (schema.TryGetProperty("extends", out JsonElement schemaObjExtends)) + if (schemaObjExtends.ValueKind == JsonValueKind.String) { - if (schemaObjExtends.ValueKind == JsonValueKind.String) - { - componentSchemas.Add(schemaObjExtends.GetString()); - } + componentSchemas.Add(schemaObjExtends.GetString()); } } } @@ -138,39 +131,43 @@ public IList GetComponentSchemas() return componentSchemas; } - public async Task> ListToDictAsync() + public Dictionary ListToDict() { Dictionary result = new Dictionary(); - using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions)) - { - JsonElement _root = document.RootElement; + using JsonDocument document = JsonDocument.Parse(_content, _parseOptions); + JsonElement _root = document.RootElement; - if (_root.ValueKind == JsonValueKind.Array) + if (_root.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement element in _root.EnumerateArray()) { - foreach (JsonElement element in _root.EnumerateArray()) + if (element.ValueKind == JsonValueKind.Object) { - if (element.ValueKind == JsonValueKind.Object) - { - using (MemoryStream stream = new MemoryStream()) - { - await JsonSerializer.SerializeAsync(stream, element).ConfigureAwait(false); - stream.Position = 0; + using MemoryStream stream = WriteJsonElementToStream(element); - using (StreamReader streamReader = new StreamReader(stream)) - { - string serialized = await streamReader.ReadToEndAsync().ConfigureAwait(false); + using StreamReader streamReader = new StreamReader(stream); + string serialized = streamReader.ReadToEnd(); - string id = new ModelQuery(serialized).GetId(); - result.Add(id, serialized); - } - } - } + string id = new ModelQuery(serialized).GetId(); + result.Add(id, serialized); } } } return result; } + + private static MemoryStream WriteJsonElementToStream(JsonElement item) + { + var memoryStream = new MemoryStream(); + using var writer = new Utf8JsonWriter(memoryStream); + + item.WriteTo(writer); + writer.Flush(); + memoryStream.Seek(0, SeekOrigin.Begin); + + return memoryStream; + } } } diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelRepositoryConstants.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelRepositoryConstants.cs index c4a160fa3d71d..def1f4040019f 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelRepositoryConstants.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelRepositoryConstants.cs @@ -8,6 +8,8 @@ internal static class ModelRepositoryConstants // Set EventSource name to package name replacing '.' with '-' public const string ModelRepositoryEventSourceName = "Azure-Iot-ModelsRepository"; + public const string DefaultModelsRepository = "https://devicemodels.azure.com"; + // File Extensions public const string JsonFileExtension = ".json"; public const string ExpandedJsonFileExtension = ".expanded.json"; diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/RepositoryHandler.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/RepositoryHandler.cs index 7782dcaa76111..c2ae5f54a6e25 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/RepositoryHandler.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/RepositoryHandler.cs @@ -34,40 +34,49 @@ public RepositoryHandler(Uri repositoryUri, ClientDiagnostics clientdiagnostics, public async Task> ProcessAsync(string dtmi, CancellationToken cancellationToken) { - return await ProcessAsync(new List() { dtmi }, cancellationToken).ConfigureAwait(false); + return await ProcessAsync(new List { dtmi }, true, cancellationToken).ConfigureAwait(false); } - public async Task> ProcessAsync(IEnumerable dtmis, CancellationToken cancellationToken) + public IDictionary Process(string dtmi, CancellationToken cancellationToken) { - Dictionary processedModels = new Dictionary(); - Queue toProcessModels = new Queue(); + return ProcessAsync(new List { dtmi }, false, cancellationToken).EnsureCompleted(); + } - foreach (string dtmi in dtmis) - { - if (!DtmiConventions.IsDtmi(dtmi)) - { - ResolverEventSource.Instance.InvalidDtmiInput(dtmi); - string invalidArgMsg = string.Format(CultureInfo.CurrentCulture, ServiceStrings.InvalidDtmiFormat, dtmi); - throw new ResolverException(dtmi, invalidArgMsg, new ArgumentException(invalidArgMsg)); - } + public Task> ProcessAsync(IEnumerable dtmis, CancellationToken cancellationToken) + { + return ProcessAsync(dtmis, true, cancellationToken); + } - toProcessModels.Enqueue(dtmi); - } + public IDictionary Process(IEnumerable dtmis, CancellationToken cancellationToken) + { + return ProcessAsync(dtmis, false, cancellationToken).EnsureCompleted(); + } - while (toProcessModels.Count != 0 && !cancellationToken.IsCancellationRequested) + private async Task> ProcessAsync(IEnumerable dtmis, bool async, CancellationToken cancellationToken) + { + var processedModels = new Dictionary(); + Queue toProcessModels = PrepareWork(dtmis); + + while (toProcessModels.Count != 0) { + cancellationToken.ThrowIfCancellationRequested(); + string targetDtmi = toProcessModels.Dequeue(); if (processedModels.ContainsKey(targetDtmi)) { ResolverEventSource.Instance.SkippingPreprocessedDtmi(targetDtmi); continue; } + ResolverEventSource.Instance.ProcessingDtmi(targetDtmi); - FetchResult result = await FetchAsync(targetDtmi, cancellationToken).ConfigureAwait(false); + FetchResult result = async + ? await FetchAsync(targetDtmi, cancellationToken).ConfigureAwait(false) + : Fetch(targetDtmi, cancellationToken); + if (result.FromExpanded) { - Dictionary expanded = await new ModelQuery(result.Definition).ListToDictAsync().ConfigureAwait(false); + Dictionary expanded = new ModelQuery(result.Definition).ListToDict(); foreach (KeyValuePair kvp in expanded) { @@ -122,5 +131,35 @@ private async Task FetchAsync(string dtmi, CancellationToken cancel throw new ResolverException(dtmi, ex.Message, ex); } } + + private FetchResult Fetch(string dtmi, CancellationToken cancellationToken) + { + try + { + return _modelFetcher.Fetch(dtmi, RepositoryUri, cancellationToken); + } + catch (Exception ex) + { + throw new ResolverException(dtmi, ex.Message, ex); + } + } + + private static Queue PrepareWork(IEnumerable dtmis) + { + var toProcessModels = new Queue(); + foreach (string dtmi in dtmis) + { + if (!DtmiConventions.IsDtmi(dtmi)) + { + ResolverEventSource.Instance.InvalidDtmiInput(dtmi); + string invalidArgMsg = string.Format(CultureInfo.CurrentCulture, ServiceStrings.InvalidDtmiFormat, dtmi); + throw new ResolverException(dtmi, invalidArgMsg, new ArgumentException(invalidArgMsg)); + } + + toProcessModels.Enqueue(dtmi); + } + + return toProcessModels; + } } } diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ResolverClient.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ResolverClient.cs index 5fbc02ff2b528..5ac245960703c 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ResolverClient.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ResolverClient.cs @@ -17,7 +17,6 @@ namespace Azure.Iot.ModelsRepository /// public class ResolverClient { - internal const string DefaultRepository = "https://devicemodels.azure.com"; private readonly RepositoryHandler _repositoryHandler; private readonly ClientDiagnostics _clientDiagnostics; @@ -25,7 +24,7 @@ public class ResolverClient /// Initializes the ResolverClient with default client options while pointing to /// the Azure IoT Plug and Play Model repository https://devicemodels.azure.com for resolution. /// - public ResolverClient() : this(new Uri(DefaultRepository), new ResolverClientOptions()) { } + public ResolverClient() : this(new Uri(DefaultModelsRepository), new ResolverClientOptions()) { } /// /// Initializes the ResolverClient with default client options while pointing to @@ -43,7 +42,7 @@ public class ResolverClient /// /// ResolverClientOptions to configure resolution and client behavior. /// - public ResolverClient(ResolverClientOptions options) : this(new Uri(DefaultRepository), options) { } + public ResolverClient(ResolverClientOptions options) : this(new Uri(DefaultModelsRepository), options) { } /// /// Initializes the ResolverClient with default client options while pointing to @@ -97,15 +96,30 @@ public ResolverClient(Uri repositoryUri, ResolverClientOptions options) /// The cancellationToken. [SuppressMessage( "Usage", - "AZC0004:DO provide both asynchronous and synchronous variants for all service methods.", + "AZC0015:Unexpected client method return type.", Justification = "")] + public virtual async Task> ResolveAsync(string dtmi, CancellationToken cancellationToken = default) + { + return await _repositoryHandler.ProcessAsync(dtmi, cancellationToken).ConfigureAwait(false); + } + + /// + /// Resolves a model definition identified by and optionally its dependencies. + /// + /// + /// An IDictionary containing the model definition(s) where the key is the dtmi + /// and the value is the raw model definition string. + /// + /// Thrown when a resolution failure occurs. + /// A well-formed DTDL model Id. For example 'dtmi:com:example:Thermostat;1'. + /// The cancellationToken. [SuppressMessage( "Usage", "AZC0015:Unexpected client method return type.", Justification = "")] - public virtual async Task> ResolveAsync(string dtmi, CancellationToken cancellationToken = default) + public virtual IDictionary Resolve(string dtmi, CancellationToken cancellationToken = default) { - return await _repositoryHandler.ProcessAsync(dtmi, cancellationToken).ConfigureAwait(false); + return _repositoryHandler.Process(dtmi, cancellationToken); } /// @@ -118,13 +132,28 @@ public virtual async Task> ResolveAsync(string dtmi, /// Thrown when a resolution failure occurs. /// A collection of well-formed DTDL model Ids. /// The cancellationToken. - [SuppressMessage("Usage", "AZC0004:DO provide both asynchronous and synchronous variants for all service methods.", Justification = "")] [SuppressMessage("Usage", "AZC0015:Unexpected client method return type.", Justification = "")] public virtual async Task> ResolveAsync(IEnumerable dtmis, CancellationToken cancellationToken = default) { return await _repositoryHandler.ProcessAsync(dtmis, cancellationToken).ConfigureAwait(false); } + /// + /// Resolves a collection of model definitions identified by and optionally their dependencies. + /// + /// + /// An IDictionary containing the model definition(s) where the key is the dtmi + /// and the value is the raw model definition string. + /// + /// Thrown when a resolution failure occurs. + /// A collection of well-formed DTDL model Ids. + /// The cancellationToken. + [SuppressMessage("Usage", "AZC0015:Unexpected client method return type.", Justification = "")] + public virtual IDictionary Resolve(IEnumerable dtmis, CancellationToken cancellationToken = default) + { + return _repositoryHandler.Process(dtmis, cancellationToken); + } + /// /// Evaluates whether an input is valid. /// @@ -139,5 +168,10 @@ public virtual async Task> ResolveAsync(IEnumerable< /// Gets the ResolverClientOptions associated with the ResolverClient instance. /// public ResolverClientOptions ClientOptions => _repositoryHandler.ClientOptions; + + /// + /// Azure Device Models Repository used by default. + /// + public static string DefaultModelsRepository => ModelRepositoryConstants.DefaultModelsRepository; } } diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ClientTests.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ClientTests.cs index aff871d3c72c4..e0096a6ee5f71 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ClientTests.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ClientTests.cs @@ -18,9 +18,9 @@ public void CtorOverloads() ResolverClientOptions options = new ResolverClientOptions(); - Assert.AreEqual(new Uri(ResolverClient.DefaultRepository), new ResolverClient().RepositoryUri); - Assert.AreEqual($"{ResolverClient.DefaultRepository}/", new ResolverClient().RepositoryUri.AbsoluteUri); - Assert.AreEqual(new Uri(ResolverClient.DefaultRepository), new ResolverClient(options).RepositoryUri); + Assert.AreEqual(new Uri(ResolverClient.DefaultModelsRepository), new ResolverClient().RepositoryUri); + Assert.AreEqual($"{ResolverClient.DefaultModelsRepository}/", new ResolverClient().RepositoryUri.AbsoluteUri); + Assert.AreEqual(new Uri(ResolverClient.DefaultModelsRepository), new ResolverClient(options).RepositoryUri); Assert.AreEqual(remoteUri, new ResolverClient(remoteUri).RepositoryUri); Assert.AreEqual(remoteUri, new ResolverClient(remoteUri, options).RepositoryUri); diff --git a/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ModelQueryTests.cs b/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ModelQueryTests.cs index 674ee96b31c6b..1f828351a1740 100644 --- a/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ModelQueryTests.cs +++ b/sdk/modelsrepository/Azure.Iot.ModelsRepository/tests/ModelQueryTests.cs @@ -163,13 +163,13 @@ public void GetModelDependencies(string id, string extends, string contents, str } [Test] - public async Task ListToDictAsync() + public void ListToDict() { string testRepoPath = TestHelpers.TestLocalModelRepository; string expandedContent = File.ReadAllText( $"{testRepoPath}/dtmi/com/example/temperaturecontroller-1.expanded.json", Encoding.UTF8); ModelQuery query = new ModelQuery(expandedContent); - Dictionary transformResult = await query.ListToDictAsync(); + Dictionary transformResult = query.ListToDict(); // Assert KPI's for TemperatureController;1. // Ensure transform of expanded content to dictionary is what we'd expect.