Add support for instructions in markdown files with YAML front-matter

.netconfig

@@ -164,3 +164,6 @@
 	sha = c0a15b3c5e42a6f5e73b8e43ad3a335d7d6f3787
 	etag = fe8de7929a8ecdb631911233ae3c6bad034b26b9802e62c3521918207f6d4068
 	weak
+[file "src/Agents/Extensions/YamlConfigurationStreamParser.cs"]
+    url = https://github.com/andrewlock/NetEscapades.Configuration/blob/master/src/NetEscapades.Configuration.Yaml/YamlConfigurationStreamParser.cs
+    weak

readme.md

@@ -133,6 +133,34 @@ This can be used by leveraging [Tomlyn.Extensions.Configuration](https://www.nug
 > avoiding unnecessary tokens being used for indentation while allowing flexible 
 > formatting in the config file.
 
+For longer instructions, markdown format plus YAML front-matter can be used for better readability:
+
+```yaml
+---
+id: ai.agents.notes
+description: Provides free-form memory
+client: grok
+options: 
+  modelid: grok-4-fast
+---
+You organize and keep notes for the user.
+# Some header
+More content
+
+## Another header
+...
+```
+
+Use the provided `AddInstructionsFile` extension method to load instructions from files as follows:
+
+```csharp
+var host = new HostApplicationBuilder(args);
+host.Configuration.AddInstructionsFile("notes.md", optional: false, reloadOnChange: true);
+```
+
+The `id` field in the front-matter is required and specifies the configuration section name, and 
+all other fields are added as if they were specified under it in the configuration.
+
 ### Extensible AI Contexts
 
 The Microsoft [agent framework](https://learn.microsoft.com/en-us/agent-framework/overview/agent-framework-overview) allows extending 

sample/Client/Client.csproj

@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.10.0" />
     <PackageReference Include="Smith" Version="0.2.5" />
     <PackageReference Include="Spectre.Console" Version="0.52.0" />
     <PackageReference Include="Spectre.Console.Json" Version="0.52.0" />
@@ -22,6 +23,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\src\Agents\Agents.csproj" />
     <ProjectReference Include="..\..\src\Extensions\Extensions.csproj" />
     <ProjectReference Include="..\..\src\Extensions.CodeAnalysis\Extensions.CodeAnalysis.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
   </ItemGroup>

sample/Client/Program.cs

@@ -1,17 +1,16 @@
 using System.Net.Http.Json;
-using System.Text.Json.Serialization;
 using Devlooped.Extensions.AI.OpenAI;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Trace;
-using Spectre.Console;
 
 var builder = App.CreateBuilder(args);
 #if DEBUG
 builder.Environment.EnvironmentName = Environments.Development;
 #endif
 
 builder.AddServiceDefaults();
-builder.Services.AddHttpClient();
+builder.Services.AddHttpClient()
+    .ConfigureHttpClientDefaults(b => b.AddStandardResilienceHandler());
 
 var app = builder.Build(async (IServiceProvider services, CancellationToken cancellation) =>
 {

sample/Client/appsettings.json

@@ -7,5 +7,12 @@
         "Endpoint": "http://localhost:5117/notes/v1"
       }
     }
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "System.Net.Http.HttpClient": "Error",
+      "Microsoft.Hosting": "Error"
+    }
   }
 }

sample/Directory.Build.targets

@@ -12,7 +12,7 @@
 
   <ItemGroup>
     <None Update=".env" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="Never" />
-    <Content Include="*.json;*.ini;*.toml" Exclude="@(Content)" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="*.json;*.ini;*.toml;*.md" Exclude="@(Content)" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(IsAspireHost)' != 'true'">

sample/Server/appsettings.Development.json

@@ -2,7 +2,9 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Warning",
+      "Microsoft.Hosting": "Warning",
+      "System.Net.Http.HttpClient": "Warning"
     }
   }
 }

sample/Server/appsettings.json

@@ -7,22 +7,12 @@
   },
   "AllowedHosts": "*",
   "AI": {
-    "Agents": {
-      "Notes": {
-        "Name": "notes",
-        "Description": "Provides free-form memory",
-        "Instructions": "You organize and keep notes for the user, using JSON-LD",
-        "Client": "Grok",
-        "Options": {
-          "ModelId": "grok-4"
-        }
-      }
-    },
     "Clients": {
       "Grok": {
-        "Endpoint": "https://api.grok.ai/v1",
+        "Endpoint": "https://api.x.ai/v1",
         "ModelId": "grok-4-fast-non-reasoning"
       }
     }
-  }
+  },
+  "OpenTelemetry:ConsoleExporter": true
 }

sample/Server/notes.md

@@ -0,0 +1,8 @@
+---
+id: ai.agents.notes
+description: Provides free-form memory
+client: Grok
+options: 
+  modelid: grok-4-fast
+---
+You organize and keep notes for the user, using JSON-LD

sample/ServiceDefaults.cs

@@ -1,11 +1,13 @@
-using DotNetEnv.Configuration;
+using Devlooped.Agents.AI;
+using DotNetEnv.Configuration;
 using OpenTelemetry;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
 using Tomlyn.Extensions.Configuration;
 
 
+
 #if WEB
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.AspNetCore.Diagnostics.HealthChecks;
@@ -54,7 +56,10 @@ public static TBuilder ConfigureOpenTelemetry<TBuilder>(this TBuilder builder)
                           || httpContext.Request.Path.StartsWithSegments(AlivenessEndpointPath)));
 #endif
                 tracing.AddHttpClientInstrumentation();
-                tracing.AddConsoleExporter();
+
+                // Only add console exporter if explicitly enabled in configuration
+                if (builder.Configuration.GetValue<bool>("OpenTelemetry:ConsoleExporter"))
+                    tracing.AddConsoleExporter();
             })
             .WithMetrics(metrics =>
             {
@@ -83,8 +88,12 @@ public static TBuilder ConfigureReload<TBuilder>(this TBuilder builder)
         {
             foreach (var toml in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.toml", SearchOption.AllDirectories))
                 builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+
             foreach (var json in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.json", SearchOption.AllDirectories))
                 builder.Configuration.AddJsonFile(json, optional: false, reloadOnChange: true);
+
+            foreach (var md in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.md", SearchOption.AllDirectories))
+                builder.Configuration.AddInstructionsFile(md, optional: false, reloadOnChange: true);
         }
         else
         {
@@ -95,11 +104,14 @@ public static TBuilder ConfigureReload<TBuilder>(this TBuilder builder)
             // Only use configs outside of bin/ and obj/ directories since we want reload to happen from source files not output files
             bool IsSource(string path) => !path.StartsWith(outDir) && !path.StartsWith(objDir);
 
+            foreach (var toml in Directory.EnumerateFiles(baseDir, "*.toml", SearchOption.AllDirectories).Where(IsSource))
+                builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+
             foreach (var json in Directory.EnumerateFiles(baseDir, "*.json", SearchOption.AllDirectories).Where(IsSource))
                 builder.Configuration.AddJsonFile(json, optional: false, reloadOnChange: true);
 
-            foreach (var toml in Directory.EnumerateFiles(baseDir, "*.toml", SearchOption.AllDirectories).Where(IsSource))
-                builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+            foreach (var md in Directory.EnumerateFiles(baseDir, "*.md", SearchOption.AllDirectories).Where(IsSource))
+                builder.Configuration.AddInstructionsFile(md, optional: false, reloadOnChange: true);
         }
 
         return builder;

src/Agents/Agents.csproj

@@ -28,14 +28,20 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.10" />
     <PackageReference Include="PolySharp" Version="1.15.0" PrivateAssets="all" />
+    <PackageReference Include="YamlDotNet" Version="16.3.0" />
   </ItemGroup>
 
   <ItemGroup>
     <None Include="..\..\osmfeula.txt" Link="osmfeula.txt" PackagePath="OSMFEULA.txt" />
+    <None Include="C:\Code\AI\src\Agents\Extensions\.editorconfig" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Extensions\Extensions.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <EditorConfigFiles Remove="C:\Code\AI\src\Agents\Extensions\.editorconfig" />
+  </ItemGroup>
+
 </Project>

src/Agents/ConfigurableAIAgent.cs

@@ -1,6 +1,5 @@
 using System.ComponentModel;
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
 using System.Text.Json;
 using Devlooped.Extensions.AI;
 using Devlooped.Extensions.AI.Grok;

src/Agents/ConfigurableInstructionsExtensions.cs

@@ -0,0 +1,86 @@
+using System.ComponentModel;
+using System.Text;
+using Microsoft.Extensions.Configuration;
+using NetEscapades.Configuration.Yaml;
+
+namespace Devlooped.Agents.AI;
+
+[EditorBrowsable(EditorBrowsableState.Never)]
+public static class ConfigurableInstructionsExtensions
+{
+    /// <summary>
+    /// Adds an instructions markdown file with optional YAML front-matter to the configuration sources.
+    /// </summary>
+    public static IConfigurationBuilder AddInstructionsFile(this IConfigurationBuilder builder, string path, bool optional = false, bool reloadOnChange = false)
+        => builder.Add<InstructionsConfigurationSource>(source =>
+        {
+            source.Path = path;
+            source.Optional = optional;
+            source.ReloadOnChange = reloadOnChange;
+            source.ResolveFileProvider();
+        });
+
+    /// <summary>
+    /// Adds an instructions markdown stream with optional YAML front-matter to the configuration sources.
+    /// </summary>
+    public static IConfigurationBuilder AddInstructionsStream(this IConfigurationBuilder builder, Stream stream)
+        => Throw.IfNull(builder).Add((InstructionsStreamConfigurationSource source) => source.Stream = stream);
+
+    static class InstructionsParser
+    {
+        public static Dictionary<string, string?> Parse(Stream stream)
+        {
+            using var reader = new StreamReader(stream);
+            var frontMatter = new StringBuilder();
+            var line = reader.ReadLine();
+            // First line must be the front-matter according to spec.
+            if (line == "---")
+            {
+                while ((line = reader.ReadLine()) != "---" && !reader.EndOfStream)
+                    frontMatter.AppendLine(line);
+            }
+
+            if (frontMatter.Length > 0 && line != "---")
+                throw new FormatException("Instructions markdown front-matter is not properly closed with '---'.");
+
+            var instructions = reader.ReadToEnd().Trim();
+            var data = new YamlConfigurationStreamParser().Parse(new MemoryStream(Encoding.UTF8.GetBytes(frontMatter.ToString())));
+            if (!data.TryGetValue("id", out var value) || Convert.ToString(value) is not { Length: > 1 } id)
+                throw new FormatException("Instructions markdown file must contain YAML front-matter with an 'id' key that specifies the section identifier.");
+
+            data.Remove("id");
+            // id should use the config delimiter rather than dot (which is a typical mistake when coming from TOML)
+            id = id.Replace(".", ConfigurationPath.KeyDelimiter);
+            var result = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
+            foreach (var entry in data)
+                result[$"{id}{ConfigurationPath.KeyDelimiter}{entry.Key}"] = entry.Value;
+
+            result[$"{id}{ConfigurationPath.KeyDelimiter}instructions"] = instructions;
+            return result;
+        }
+    }
+
+    class InstructionsStreamConfigurationSource : StreamConfigurationSource
+    {
+        public override IConfigurationProvider Build(IConfigurationBuilder builder) => new InstructionsStreamConfigurationProvider(this);
+    }
+
+    class InstructionsStreamConfigurationProvider(InstructionsStreamConfigurationSource source) : StreamConfigurationProvider(source)
+    {
+        public override void Load(Stream stream) => Data = InstructionsParser.Parse(stream);
+    }
+
+    class InstructionsConfigurationSource : FileConfigurationSource
+    {
+        public override IConfigurationProvider Build(IConfigurationBuilder builder)
+        {
+            EnsureDefaults(builder);
+            return new InstructionsConfigurationProvider(this);
+        }
+    }
+
+    class InstructionsConfigurationProvider(FileConfigurationSource source) : FileConfigurationProvider(source)
+    {
+        public override void Load(Stream stream) => Data = InstructionsParser.Parse(stream);
+    }
+}

src/Agents/Extensions/.editorconfig

@@ -0,0 +1,2 @@
+[*.cs]
+generated_code = true

src/Agents/Extensions/Resources.cs

@@ -0,0 +1,16 @@
+// <auto-generated />
+using System.Globalization;
+
+namespace NetEscapades.Configuration.Yaml
+{
+    internal static class Resources
+    {
+        /// <summary>
+        /// A duplicate key '{0}' was found.
+        /// </summary>
+        internal static string FormatError_KeyIsDuplicated(object p0)
+        {
+            return string.Format(CultureInfo.CurrentCulture, "A duplicate key '{0}' was found.", p0);
+        }
+    }
+}