From 3ac65175bf9a487d15dc808d32423b4ee7fddd69 Mon Sep 17 00:00:00 2001
From: Maksym Kucherov <maximus.the0@gmail.com>
Date: Mon, 10 Jun 2024 10:46:52 -0400
Subject: [PATCH] Remove null values and skip empty documents.

---
 Examples/ExampleBase/ExampleBase.cs | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/Examples/ExampleBase/ExampleBase.cs b/Examples/ExampleBase/ExampleBase.cs
index 33f54330..4d0c28b1 100644
--- a/Examples/ExampleBase/ExampleBase.cs
+++ b/Examples/ExampleBase/ExampleBase.cs
@@ -112,6 +112,7 @@ protected static IEnumerable<Dictionary<string, object>> GetEvidence(
             // Consume the stream start event.
             yamlReader.Consume<StreamStart>();
             int records = 0;
+            int skipped = 0;
             // Keep going as long as we have more document records.
             while (yamlReader.TryConsume<DocumentStart>(out _))
             {
@@ -119,12 +120,28 @@ protected static IEnumerable<Dictionary<string, object>> GetEvidence(
                 records++;
                 if (logger != null && records % 1000 == 0)
                 {
-                    logger.LogInformation($"Processed {records} records");
+                    logger.LogInformation($"Processed {records} records ({skipped} skipped)");
                 }
 
                 // Deserialize the record
                 var data = deserializer.Deserialize<Dictionary<string, object>>(yamlReader);
-                yield return data;
+
+                // Remove null values
+                foreach(var keyWithNullValue in data.Where(kvp => kvp.Value is null).Select(kvp => kvp.Key).ToList())
+                {
+                    logger.LogWarning($"Document at offset {records-1} contains null value for key: '{keyWithNullValue}'!");
+                    data.Remove(keyWithNullValue);
+                }
+
+                if (data.Count > 0)
+                {
+                    yield return data;
+                }
+                else
+                {
+                    logger.LogWarning($"Document at offset {records - 1} contains no usable evidence!");
+                    ++skipped;
+                }
 
                 // Required to move to the start of the next record.
                 yamlReader.TryConsume<DocumentEnd>(out _);