Reindex attachment honours ingest-attachment field names (#3600)

This commit updates the Attachment Json converter to use the
ingest-geoip attachment field names when serializing an attachment.

The current serialization implementation is based on mapper-attachments plugin which
was deprecated in 5.x and removed in 6.x.

Fixes #3502

Author: russcam

src/Nest/Mapping/Types/Specialized/Attachment/Attachment.cs

@@ -18,11 +18,17 @@ public class Attachment
 		/// attachment.
 		/// </summary>
 		[JsonIgnore]
-		public bool ContainsMetadata => !Name.IsNullOrEmpty() ||
+		public bool ContainsMetadata =>
+			!Author.IsNullOrEmpty() ||
+			ContentLength.HasValue ||
 			!ContentType.IsNullOrEmpty() ||
-			!Language.IsNullOrEmpty() ||
+			Date.HasValue ||
 			DetectLanguage.HasValue ||
-			IndexedCharacters.HasValue;
+			IndexedCharacters.HasValue ||
+			!Keywords.IsNullOrEmpty() ||
+			!Language.IsNullOrEmpty() ||
+			!Name.IsNullOrEmpty() ||
+			!Title.IsNullOrEmpty();
 
 		/// <summary>
 		/// The base64 encoded content. Can be explicitly set
@@ -94,51 +100,86 @@ internal class AttachmentConverter : JsonConverter
 		public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
 		{
 			var attachment = (Attachment)value;
-			if (!attachment.ContainsMetadata)
-				writer.WriteValue(attachment.Content);
-			else
+			if (attachment.ContainsMetadata)
 			{
 				writer.WriteStartObject();
-				writer.WritePropertyName("_content");
-				writer.WriteValue(attachment.Content);
 
-				if (!string.IsNullOrEmpty(attachment.Name))
+				if (!string.IsNullOrEmpty(attachment.Content))
 				{
-					writer.WritePropertyName("_name");
-					writer.WriteValue(attachment.Name);
+					writer.WritePropertyName("content");
+					writer.WriteValue(attachment.Content);
+				}
+
+				if (!string.IsNullOrEmpty(attachment.Author))
+				{
+					writer.WritePropertyName("author");
+					writer.WriteValue(attachment.Author);
+				}
+
+				if (!string.IsNullOrEmpty(attachment.ContentType))
+				{
+					writer.WritePropertyName("content_length");
+					writer.WriteValue(attachment.ContentLength);
 				}
 
 				if (!string.IsNullOrEmpty(attachment.ContentType))
 				{
-					writer.WritePropertyName("_content_type");
+					writer.WritePropertyName("content_type");
 					writer.WriteValue(attachment.ContentType);
 				}
 
-				if (!string.IsNullOrEmpty(attachment.Language))
+				if (attachment.Date.HasValue)
 				{
-					writer.WritePropertyName("_language");
-					writer.WriteValue(attachment.Language);
+					writer.WritePropertyName("date");
+					writer.WriteValue(attachment.Date.Value);
 				}
 
 				if (attachment.DetectLanguage.HasValue)
 				{
-					writer.WritePropertyName("_detect_language");
+					writer.WritePropertyName("detect_language");
 					writer.WriteValue(attachment.DetectLanguage.Value);
 				}
 
 				if (attachment.IndexedCharacters.HasValue)
 				{
-					writer.WritePropertyName("_indexed_chars");
+					writer.WritePropertyName("indexed_chars");
 					writer.WriteValue(attachment.IndexedCharacters.Value);
 				}
 
+				if (!string.IsNullOrEmpty(attachment.Keywords))
+				{
+					writer.WritePropertyName("keywords");
+					writer.WriteValue(attachment.Keywords);
+				}
+
+				if (!string.IsNullOrEmpty(attachment.Language))
+				{
+					writer.WritePropertyName("language");
+					writer.WriteValue(attachment.Language);
+				}
+
+				if (!string.IsNullOrEmpty(attachment.Name))
+				{
+					writer.WritePropertyName("name");
+					writer.WriteValue(attachment.Name);
+				}
+
+				if (!string.IsNullOrEmpty(attachment.Title))
+				{
+					writer.WritePropertyName("title");
+					writer.WriteValue(attachment.Title);
+				}
+
 				writer.WriteEndObject();
 			}
+			else
+				writer.WriteValue(attachment.Content);
 		}
 
 		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
 		{
-			if (reader.TokenType == JsonToken.String) return new Attachment { Content = (string)reader.Value };
+			if (reader.TokenType == JsonToken.String)
+				return new Attachment { Content = (string)reader.Value };
 
 			if (reader.TokenType == JsonToken.StartObject)
 			{
@@ -181,13 +222,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 							case "_content_type":
 							case "content_type":
 							case "contenttype":
-							case "contentType":
 								attachment.ContentType = reader.ReadAsString();
 								break;
 							case "_content_length":
 							case "content_length":
 							case "contentlength":
-							case "contentLength":
 								reader.Read();
 								switch (reader.TokenType)
 								{
@@ -207,6 +246,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 								attachment.Language = reader.ReadAsString();
 								break;
 							case "_detect_language":
+							case "detect_language":
 								attachment.DetectLanguage = reader.ReadAsBoolean();
 								break;
 							case "_indexed_chars":

src/Tests/Tests/Document/Single/Index/IndexIngestAttachmentApiTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using Elastic.Xunit.XunitPlumbing;
 using Elasticsearch.Net;
 using FluentAssertions;
 using Nest;
@@ -29,16 +30,22 @@ static TestDocument()
 		public static string TestPdfDocument { get; }
 	}
 
+	public class IngestedAttachment
+	{
+		public Attachment Attachment { get; set; }
+		public string Content { get; set; }
+		public int Id { get; set; }
+	}
+
 	public class IndexIngestAttachmentApiTests
 		: ApiIntegrationTestBase<IntrusiveOperationCluster, IIndexResponse,
-			IIndexRequest<IndexIngestAttachmentApiTests.IngestedAttachment>,
-			IndexDescriptor<IndexIngestAttachmentApiTests.IngestedAttachment>,
-			IndexRequest<IndexIngestAttachmentApiTests.IngestedAttachment>>
+			IIndexRequest<IngestedAttachment>,
+			IndexDescriptor<IngestedAttachment>,
+			IndexRequest<IngestedAttachment>>
 	{
 		public IndexIngestAttachmentApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
 		protected override bool ExpectIsValid => true;
-
 		protected override object ExpectJson => new { id = 1, content = Content };
 		protected override int ExpectStatusCode => 201;
 
@@ -69,7 +76,7 @@ protected override string UrlPath
 			Content = Content
 		};
 
-		private static string PipelineId { get; } = "pipeline-" + Guid.NewGuid().ToString("N").Substring(0, 8);
+		private static string PipelineId { get; } = "pipeline-" + RandomString();
 
 		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
 		{
@@ -138,12 +145,96 @@ protected override void ExpectResponse(IIndexResponse response)
 			attachment.ContentLength.Should().Be(96);
 			attachment.Content.Should().Contain("mapper-attachment support");
 		}
+	}
+
+	public class ReindexAttachmentApiTests : IClusterFixture<IntrusiveOperationCluster>
+	{
+		private readonly IElasticClient _client;
+
+		private static string RandomString { get; } = Guid.NewGuid().ToString("N").Substring(0, 8);
+		private static string Index { get; } = "project" + RandomString;
+		private static string PipelineId { get; } = "pipeline-" + RandomString;
+		private static string Content => TestDocument.TestPdfDocument;
+
+		private IngestedAttachment Document => new IngestedAttachment
+		{
+			Id = 1,
+			Content = Content
+		};
+
+		public ReindexAttachmentApiTests(IntrusiveOperationCluster cluster)
+		{
+			_client = cluster.Client;
+
+			_client.PutPipeline(new PutPipelineRequest(PipelineId)
+			{
+				Description = "Attachment pipeline test",
+				Processors = new List<IProcessor>
+				{
+					new AttachmentProcessor
+					{
+						Field = "content",
+						TargetField = "attachment"
+					}
+				}
+			});
+
+			var createIndexResponse = _client.CreateIndex(Index, c => c
+				.Mappings(m => m
+					.Map<IngestedAttachment>(mm => mm
+						.Properties(p => p
+							.Text(s => s
+								.Name(f => f.Content)
+							)
+							.Object<Attachment>(o => o
+								.Name(f => f.Attachment)
+							)
+						)
+					)
+				)
+			);
+
+			createIndexResponse.ShouldBeValid();
+			var indexResponse = _client.Index<IngestedAttachment>(Document, i => i
+				.Index(Index)
+				.Refresh(Refresh.True)
+				.Pipeline(PipelineId)
+			);
+
+			indexResponse.ShouldBeValid();
+		}
 
-		public class IngestedAttachment
+		[I]
+		public void ReindexIntoAnotherIndexCorrectlySerializesAttachment()
 		{
-			public Attachment Attachment { get; set; }
-			public string Content { get; set; }
-			public int Id { get; set; }
+			var getResponse = _client.Get<IngestedAttachment>(1, g => g
+				.Index(Index)
+			);
+
+			getResponse.ShouldBeValid();
+			var ingestedAttachment = getResponse.Source;
+			ingestedAttachment.Should().NotBeNull();
+
+			var indexResponse = _client.Index<IngestedAttachment>(ingestedAttachment, i => i
+				.Index(Index + "2")
+				.Refresh(Refresh.True)
+			);
+
+			indexResponse.ShouldBeValid();
+
+			getResponse = _client.Get<IngestedAttachment>(1, g => g
+				.Index(Index + "2")
+			);
+
+			getResponse.ShouldBeValid();
+			ingestedAttachment.Should().NotBeNull();
+			ingestedAttachment.Attachment.Title.Should().Be("Attachment Test Document");
+			ingestedAttachment.Attachment.Keywords.Should().Be("nest,test,document");
+			ingestedAttachment.Attachment.Date.Should().Be(new DateTime(2016, 12, 08, 3, 5, 13, DateTimeKind.Utc));
+			ingestedAttachment.Attachment.ContentType.Should().Be("application/pdf");
+			ingestedAttachment.Attachment.Author.Should().Be("Russ Cam");
+			ingestedAttachment.Attachment.Language.Should().Be("fr");
+			ingestedAttachment.Attachment.ContentLength.Should().Be(96);
 		}
 	}
 }