Skip to content

Commit

Permalink
Add async methods to the ResolverClient (#18814)
Browse files Browse the repository at this point in the history
  • Loading branch information
@azabbasi
azabbasi authored Feb 17, 2021
1 parent 0f86217 commit b1de661
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 105 deletions.
2 changes: 1 addition & 1 deletion sdk/modelsrepository/Azure.Iot.ModelsRepository/src/Fetchers/LocalModelFetcher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public FetchResult Fetch(string dtmi, Uri repositoryUri, CancellationToken cance
fnfError = string.Format(CultureInfo.CurrentCulture, ServiceStrings.ErrorFetchingModelContent, tryContentPath);
}

throw new RequestFailedException(fnfError, new FileNotFoundException(fnfError));
throw new FileNotFoundException(fnfError);
}
catch (Exception ex)
{
Expand Down
147 changes: 72 additions & 75 deletions sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using System.Threading.Tasks;

namespace Azure.Iot.ModelsRepository
{
Expand All @@ -29,16 +28,14 @@ public ModelMetadata GetMetadata()

public string GetId()
{
using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions))
{
JsonElement _root = document.RootElement;
using JsonDocument document = JsonDocument.Parse(_content, _parseOptions);
JsonElement _root = document.RootElement;

if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("@id", out JsonElement id))
if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("@id", out JsonElement id))
{
if (id.ValueKind == JsonValueKind.String)
{
if (id.ValueKind == JsonValueKind.String)
{
return id.GetString();
}
return id.GetString();
}
}

Expand All @@ -49,34 +46,32 @@ public IList<string> GetExtends()
{
List<string> dependencies = new List<string>();

using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions))
{
JsonElement _root = document.RootElement;
using JsonDocument document = JsonDocument.Parse(_content, _parseOptions);
JsonElement _root = document.RootElement;

if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("extends", out JsonElement extends))
if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("extends", out JsonElement extends))
{
if (extends.ValueKind == JsonValueKind.Array)
{
if (extends.ValueKind == JsonValueKind.Array)
foreach (JsonElement extendElement in extends.EnumerateArray())
{
foreach (JsonElement extendElement in extends.EnumerateArray())
if (extendElement.ValueKind == JsonValueKind.String)
{
if (extendElement.ValueKind == JsonValueKind.String)
{
dependencies.Add(extendElement.GetString());
}
else if (extendElement.ValueKind == JsonValueKind.Object)
{
// extends can have multiple levels and can contain components.
// TODO: Support object ctor - inefficient serialize.
ModelMetadata nested_interface = new ModelQuery(JsonSerializer.Serialize(extendElement)).GetMetadata();
dependencies.AddRange(nested_interface.Dependencies);
}
dependencies.Add(extendElement.GetString());
}
else if (extendElement.ValueKind == JsonValueKind.Object)
{
// extends can have multiple levels and can contain components.
// TODO: Support object ctor - inefficient serialize.
ModelMetadata nested_interface = new ModelQuery(JsonSerializer.Serialize(extendElement)).GetMetadata();
dependencies.AddRange(nested_interface.Dependencies);
}
}
else if (extends.ValueKind == JsonValueKind.String)
{
dependencies.Add(extends.GetString());
}
}
else if (extends.ValueKind == JsonValueKind.String)
{
dependencies.Add(extends.GetString());
}
}

return dependencies;
Expand All @@ -87,44 +82,42 @@ public IList<string> GetComponentSchemas()
{
List<string> componentSchemas = new List<string>();

using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions))
{
JsonElement _root = document.RootElement;
using JsonDocument document = JsonDocument.Parse(_content, _parseOptions);
JsonElement _root = document.RootElement;

if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("contents", out JsonElement contents))
if (_root.ValueKind == JsonValueKind.Object && _root.TryGetProperty("contents", out JsonElement contents))
{
if (contents.ValueKind == JsonValueKind.Array)
{
if (contents.ValueKind == JsonValueKind.Array)
foreach (JsonElement element in contents.EnumerateArray())
{
foreach (JsonElement element in contents.EnumerateArray())
if (element.TryGetProperty("@type", out JsonElement type))
{
if (element.TryGetProperty("@type", out JsonElement type))
if (type.ValueKind == JsonValueKind.String && type.GetString() == "Component")
{
if (type.ValueKind == JsonValueKind.String && type.GetString() == "Component")
if (element.TryGetProperty("schema", out JsonElement schema))
{
if (element.TryGetProperty("schema", out JsonElement schema))
if (schema.ValueKind == JsonValueKind.String)
{
if (schema.ValueKind == JsonValueKind.String)
{
componentSchemas.Add(schema.GetString());
}
else if (schema.ValueKind == JsonValueKind.Array)
componentSchemas.Add(schema.GetString());
}
else if (schema.ValueKind == JsonValueKind.Array)
{
foreach (JsonElement schemaElement in schema.EnumerateArray())
{
foreach (JsonElement schemaElement in schema.EnumerateArray())
if (schemaElement.ValueKind == JsonValueKind.String)
{
if (schemaElement.ValueKind == JsonValueKind.String)
{
componentSchemas.Add(schemaElement.GetString());
}
componentSchemas.Add(schemaElement.GetString());
}
}
else if (schema.ValueKind == JsonValueKind.Object)
}
else if (schema.ValueKind == JsonValueKind.Object)
{
if (schema.TryGetProperty("extends", out JsonElement schemaObjExtends))
{
if (schema.TryGetProperty("extends", out JsonElement schemaObjExtends))
if (schemaObjExtends.ValueKind == JsonValueKind.String)
{
if (schemaObjExtends.ValueKind == JsonValueKind.String)
{
componentSchemas.Add(schemaObjExtends.GetString());
}
componentSchemas.Add(schemaObjExtends.GetString());
}
}
}
Expand All @@ -138,39 +131,43 @@ public IList<string> GetComponentSchemas()
return componentSchemas;
}

public async Task<Dictionary<string, string>> ListToDictAsync()
public Dictionary<string, string> ListToDict()
{
Dictionary<string, string> result = new Dictionary<string, string>();

using (JsonDocument document = JsonDocument.Parse(_content, _parseOptions))
{
JsonElement _root = document.RootElement;
using JsonDocument document = JsonDocument.Parse(_content, _parseOptions);
JsonElement _root = document.RootElement;

if (_root.ValueKind == JsonValueKind.Array)
if (_root.ValueKind == JsonValueKind.Array)
{
foreach (JsonElement element in _root.EnumerateArray())
{
foreach (JsonElement element in _root.EnumerateArray())
if (element.ValueKind == JsonValueKind.Object)
{
if (element.ValueKind == JsonValueKind.Object)
{
using (MemoryStream stream = new MemoryStream())
{
await JsonSerializer.SerializeAsync(stream, element).ConfigureAwait(false);
stream.Position = 0;
using MemoryStream stream = WriteJsonElementToStream(element);

using (StreamReader streamReader = new StreamReader(stream))
{
string serialized = await streamReader.ReadToEndAsync().ConfigureAwait(false);
using StreamReader streamReader = new StreamReader(stream);
string serialized = streamReader.ReadToEnd();

string id = new ModelQuery(serialized).GetId();
result.Add(id, serialized);
}
}
}
string id = new ModelQuery(serialized).GetId();
result.Add(id, serialized);
}
}
}

return result;
}

private static MemoryStream WriteJsonElementToStream(JsonElement item)
{
var memoryStream = new MemoryStream();
using var writer = new Utf8JsonWriter(memoryStream);

item.WriteTo(writer);
writer.Flush();
memoryStream.Seek(0, SeekOrigin.Begin);

return memoryStream;
}
}
}
2 changes: 2 additions & 0 deletions sdk/modelsrepository/Azure.Iot.ModelsRepository/src/ModelRepositoryConstants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ internal static class ModelRepositoryConstants
// Set EventSource name to package name replacing '.' with '-'
public const string ModelRepositoryEventSourceName = "Azure-Iot-ModelsRepository";

public const string DefaultModelsRepository = "https://devicemodels.azure.com";

// File Extensions
public const string JsonFileExtension = ".json";
public const string ExpandedJsonFileExtension = ".expanded.json";
Expand Down
73 changes: 56 additions & 17 deletions sdk/modelsrepository/Azure.Iot.ModelsRepository/src/RepositoryHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,40 +34,49 @@ public RepositoryHandler(Uri repositoryUri, ClientDiagnostics clientdiagnostics,

public async Task<IDictionary<string, string>> ProcessAsync(string dtmi, CancellationToken cancellationToken)
{
return await ProcessAsync(new List<string>() { dtmi }, cancellationToken).ConfigureAwait(false);
return await ProcessAsync(new List<string> { dtmi }, true, cancellationToken).ConfigureAwait(false);
}

public async Task<IDictionary<string, string>> ProcessAsync(IEnumerable<string> dtmis, CancellationToken cancellationToken)
public IDictionary<string, string> Process(string dtmi, CancellationToken cancellationToken)
{
Dictionary<string, string> processedModels = new Dictionary<string, string>();
Queue<string> toProcessModels = new Queue<string>();
return ProcessAsync(new List<string> { dtmi }, false, cancellationToken).EnsureCompleted();
}

foreach (string dtmi in dtmis)
{
if (!DtmiConventions.IsDtmi(dtmi))
{
ResolverEventSource.Instance.InvalidDtmiInput(dtmi);
string invalidArgMsg = string.Format(CultureInfo.CurrentCulture, ServiceStrings.InvalidDtmiFormat, dtmi);
throw new ResolverException(dtmi, invalidArgMsg, new ArgumentException(invalidArgMsg));
}
public Task<IDictionary<string, string>> ProcessAsync(IEnumerable<string> dtmis, CancellationToken cancellationToken)
{
return ProcessAsync(dtmis, true, cancellationToken);
}

toProcessModels.Enqueue(dtmi);
}
public IDictionary<string, string> Process(IEnumerable<string> dtmis, CancellationToken cancellationToken)
{
return ProcessAsync(dtmis, false, cancellationToken).EnsureCompleted();
}

while (toProcessModels.Count != 0 && !cancellationToken.IsCancellationRequested)
private async Task<IDictionary<string, string>> ProcessAsync(IEnumerable<string> dtmis, bool async, CancellationToken cancellationToken)
{
var processedModels = new Dictionary<string, string>();
Queue<string> toProcessModels = PrepareWork(dtmis);

while (toProcessModels.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();

string targetDtmi = toProcessModels.Dequeue();
if (processedModels.ContainsKey(targetDtmi))
{
ResolverEventSource.Instance.SkippingPreprocessedDtmi(targetDtmi);
continue;
}

ResolverEventSource.Instance.ProcessingDtmi(targetDtmi);

FetchResult result = await FetchAsync(targetDtmi, cancellationToken).ConfigureAwait(false);
FetchResult result = async
? await FetchAsync(targetDtmi, cancellationToken).ConfigureAwait(false)
: Fetch(targetDtmi, cancellationToken);

if (result.FromExpanded)
{
Dictionary<string, string> expanded = await new ModelQuery(result.Definition).ListToDictAsync().ConfigureAwait(false);
Dictionary<string, string> expanded = new ModelQuery(result.Definition).ListToDict();

foreach (KeyValuePair<string, string> kvp in expanded)
{
Expand Down Expand Up @@ -122,5 +131,35 @@ private async Task<FetchResult> FetchAsync(string dtmi, CancellationToken cancel
throw new ResolverException(dtmi, ex.Message, ex);
}
}

private FetchResult Fetch(string dtmi, CancellationToken cancellationToken)
{
try
{
return _modelFetcher.Fetch(dtmi, RepositoryUri, cancellationToken);
}
catch (Exception ex)
{
throw new ResolverException(dtmi, ex.Message, ex);
}
}

private static Queue<string> PrepareWork(IEnumerable<string> dtmis)
{
var toProcessModels = new Queue<string>();
foreach (string dtmi in dtmis)
{
if (!DtmiConventions.IsDtmi(dtmi))
{
ResolverEventSource.Instance.InvalidDtmiInput(dtmi);
string invalidArgMsg = string.Format(CultureInfo.CurrentCulture, ServiceStrings.InvalidDtmiFormat, dtmi);
throw new ResolverException(dtmi, invalidArgMsg, new ArgumentException(invalidArgMsg));
}

toProcessModels.Enqueue(dtmi);
}

return toProcessModels;
}
}
}
Loading

0 comments on commit b1de661

Please sign in to comment.