Introduce Microsoft.Extensions.DataIngestion.Abstractions (#6949)

Author: adamsitnik

src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/IngestionChunk.cs

@@ -0,0 +1,71 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.DataIngestion;
+
+/// <summary>
+/// Represents a chunk of content extracted from an <see cref="IngestionDocument"/>.
+/// </summary>
+/// <typeparam name="T">The type of the content.</typeparam>
+[DebuggerDisplay("Content = {Content}")]
+public sealed class IngestionChunk<T>
+{
+    private Dictionary<string, object>? _metadata;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="IngestionChunk{T}"/> class.
+    /// </summary>
+    /// <param name="content">The content of the chunk.</param>
+    /// <param name="document">The document from which this chunk was extracted.</param>
+    /// <param name="context">Additional context for the chunk.</param>
+    /// <exception cref="ArgumentNullException">
+    /// <paramref name="content"/> or <paramref name="document"/> is <see langword="null"/>.
+    /// </exception>
+    /// <exception cref="ArgumentException">
+    /// <paramref name="content"/> is a string that is empty or contains only white-space characters.
+    /// </exception>
+    public IngestionChunk(T content, IngestionDocument document, string? context = null)
+    {
+        if (typeof(T) == typeof(string))
+        {
+            Content = (T)(object)Throw.IfNullOrEmpty((string)(object)content!);
+        }
+        else
+        {
+            Content = Throw.IfNull(content);
+        }
+
+        Document = Throw.IfNull(document);
+        Context = context;
+    }
+
+    /// <summary>
+    /// Gets the content of the chunk.
+    /// </summary>
+    public T Content { get; }
+
+    /// <summary>
+    /// Gets the document from which this chunk was extracted.
+    /// </summary>
+    public IngestionDocument Document { get; }
+
+    /// <summary>
+    /// Gets additional context for the chunk.
+    /// </summary>
+    public string? Context { get; }
+
+    /// <summary>
+    /// Gets a value indicating whether this chunk has metadata.
+    /// </summary>
+    public bool HasMetadata => _metadata?.Count > 0;
+
+    /// <summary>
+    /// Gets the metadata associated with this chunk.
+    /// </summary>
+    public IDictionary<string, object> Metadata => _metadata ??= [];
+}

src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/IngestionChunkProcessor.cs

@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Microsoft.Extensions.DataIngestion;
+
+/// <summary>
+/// Processes chunks in a pipeline.
+/// </summary>
+/// <typeparam name="T">The type of the chunk content.</typeparam>
+public abstract class IngestionChunkProcessor<T>
+{
+    /// <summary>
+    /// Processes chunks asynchronously.
+    /// </summary>
+    /// <param name="chunks">The chunks to process.</param>
+    /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+    /// <returns>The processed chunks.</returns>
+    public abstract IAsyncEnumerable<IngestionChunk<T>> ProcessAsync(IAsyncEnumerable<IngestionChunk<T>> chunks, CancellationToken cancellationToken = default);
+}

src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/IngestionChunkWriter.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Extensions.DataIngestion;
+
+/// <summary>
+/// Writes chunks to a destination.
+/// </summary>
+/// <typeparam name="T">The type of the chunk content.</typeparam>
+public abstract class IngestionChunkWriter<T> : IDisposable
+{
+    /// <summary>
+    /// Writes chunks asynchronously.
+    /// </summary>
+    /// <param name="chunks">The chunks to write.</param>
+    /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+    /// <returns>A task representing the asynchronous write operation.</returns>
+    public abstract Task WriteAsync(IAsyncEnumerable<IngestionChunk<T>> chunks, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Disposes the writer and releases all associated resources.
+    /// </summary>
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>
+    /// Disposes the writer.
+    /// </summary>
+    /// <param name="disposing">true if called from dispose, false if called from finalizer.</param>
+    protected virtual void Dispose(bool disposing)
+    {
+    }
+}

src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/IngestionChunker.cs

@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Microsoft.Extensions.DataIngestion;
+
+/// <summary>
+/// Splits an <see cref="IngestionDocument"/> into chunks.
+/// </summary>
+/// <typeparam name="T">The type of the chunk content.</typeparam>
+public abstract class IngestionChunker<T>
+{
+    /// <summary>
+    /// Splits a document into chunks asynchronously.
+    /// </summary>
+    /// <param name="document">The document to split.</param>
+    /// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
+    /// <returns>The chunks created from the document.</returns>
+    public abstract IAsyncEnumerable<IngestionChunk<T>> ProcessAsync(IngestionDocument document, CancellationToken cancellationToken = default);
+}

src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/IngestionDocument.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.DataIngestion;
+
+/// <summary>
+/// A format-agnostic container that normalizes diverse input formats into a structured hierarchy.
+/// </summary>
+public sealed class IngestionDocument
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="IngestionDocument"/> class.
+    /// </summary>
+    /// <param name="identifier">The unique identifier for the document.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="identifier"/> is <see langword="null"/>.</exception>
+    public IngestionDocument(string identifier)
+    {
+        Identifier = Throw.IfNullOrEmpty(identifier);
+    }
+
+    /// <summary>
+    /// Gets the unique identifier for the document.
+    /// </summary>
+    public string Identifier { get; }
+
+    /// <summary>
+    /// Gets the sections of the document.
+    /// </summary>
+    public IList<IngestionDocumentSection> Sections { get; } = [];
+
+    /// <summary>
+    /// Iterate over all elements in the document, including those in nested sections.
+    /// </summary>
+    /// <returns>An enumerable collection of elements.</returns>
+    /// <remarks>
+    /// Sections themselves are not included.
+    /// </remarks>
+    public IEnumerable<IngestionDocumentElement> EnumerateContent()
+    {
+        Stack<IngestionDocumentElement> elementsToProcess = new();
+
+        for (int sectionIndex = Sections.Count - 1; sectionIndex >= 0; sectionIndex--)
+        {
+            elementsToProcess.Push(Sections[sectionIndex]);
+        }
+
+        while (elementsToProcess.Count > 0)
+        {
+            IngestionDocumentElement currentElement = elementsToProcess.Pop();
+
+            if (currentElement is not IngestionDocumentSection nestedSection)
+            {
+                yield return currentElement;
+            }
+            else
+            {
+                for (int i = nestedSection.Elements.Count - 1; i >= 0; i--)
+                {
+                    elementsToProcess.Push(nestedSection.Elements[i]);
+                }
+            }
+        }
+    }
+}