Skip to content

Commit c94b49f

Browse files
russcamrusscam
committed
Reindex attachment honours ingest-attachment field names (#3600)
This commit updates the Attachment Json converter to use the ingest-geoip attachment field names when serializing an attachment. The current serialization implementation is based on mapper-attachments plugin which was deprecated in 5.x and removed in 6.x. Fixes #3502 (cherry picked from commit 5a78a43)
1 parent 39ffdb6 commit c94b49f

File tree

2 files changed

+157
-27
lines changed

2 files changed

+157
-27
lines changed

‎src/Nest/Mapping/Types/Specialized/Attachment/Attachment.cs‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,17 @@ public class Attachment
2020
/// attachment.
2121
/// </summary>
2222
[IgnoreDataMember]
23-
public bool ContainsMetadata => !Name.IsNullOrEmpty() ||
23+
public bool ContainsMetadata =>
24+
!Author.IsNullOrEmpty() ||
25+
ContentLength.HasValue ||
2426
!ContentType.IsNullOrEmpty() ||
25-
!Language.IsNullOrEmpty() ||
27+
Date.HasValue ||
2628
DetectLanguage.HasValue ||
27-
IndexedCharacters.HasValue;
29+
IndexedCharacters.HasValue ||
30+
!Keywords.IsNullOrEmpty() ||
31+
!Language.IsNullOrEmpty() ||
32+
!Name.IsNullOrEmpty() ||
33+
!Title.IsNullOrEmpty();
2834

2935
/// <summary>
3036
/// The base64 encoded content. Can be explicitly set
@@ -93,7 +99,7 @@ public class Attachment
9399

94100
internal class AttachmentFormatter : IJsonFormatter<Attachment>
95101
{
96-
private static readonly AutomataDictionary _automataDictionary = new AutomataDictionary
102+
private static readonly AutomataDictionary AutomataDictionary = new AutomataDictionary
97103
{
98104
{ "_content", 0 },
99105
{ "content", 0 },
@@ -104,15 +110,13 @@ internal class AttachmentFormatter : IJsonFormatter<Attachment>
104110
{ "date", 4 },
105111
{ "_content_type", 5 },
106112
{ "content_type", 5 },
107-
{ "contenttype", 5 },
108-
{ "contentType", 5 },
109113
{ "_content_length", 6 },
110114
{ "content_length", 6 },
111115
{ "contentlength", 6 },
112-
{ "contentLength", 6 },
113116
{ "_language", 7 },
114117
{ "language", 7 },
115118
{ "_detect_language", 8 },
119+
{ "detect_language", 8 },
116120
{ "_indexed_chars", 9 },
117121
{ "indexed_chars", 9 },
118122
{ "title", 10 },
@@ -132,7 +136,7 @@ public Attachment Deserialize(ref JsonReader reader, IJsonFormatterResolver form
132136
while (reader.ReadIsInObject(ref count))
133137
{
134138
var propertyName = reader.ReadPropertyNameSegmentRaw();
135-
if (_automataDictionary.TryGetValue(propertyName, out var value))
139+
if (AutomataDictionary.TryGetValue(propertyName, out var value))
136140
{
137141
switch (value)
138142
{
@@ -193,44 +197,79 @@ public void Serialize(ref JsonWriter writer, Attachment value, IJsonFormatterRes
193197
else
194198
{
195199
writer.WriteBeginObject();
196-
writer.WritePropertyName("_content");
200+
writer.WritePropertyName("content");
197201
writer.WriteString(value.Content);
198202

199-
if (!string.IsNullOrEmpty(value.Name))
203+
if (!string.IsNullOrEmpty(value.Author))
200204
{
201205
writer.WriteValueSeparator();
202-
writer.WritePropertyName("_name");
203-
writer.WriteString(value.Name);
206+
writer.WritePropertyName("author");
207+
writer.WriteString(value.Author);
208+
}
209+
210+
if (value.ContentLength.HasValue)
211+
{
212+
writer.WriteValueSeparator();
213+
writer.WritePropertyName("content_length");
214+
writer.WriteInt64(value.ContentLength.Value);
204215
}
205216

206217
if (!string.IsNullOrEmpty(value.ContentType))
207218
{
208219
writer.WriteValueSeparator();
209-
writer.WritePropertyName("_content_type");
220+
writer.WritePropertyName("content_type");
210221
writer.WriteString(value.ContentType);
211222
}
212223

213-
if (!string.IsNullOrEmpty(value.Language))
224+
if (value.Date.HasValue)
214225
{
215226
writer.WriteValueSeparator();
216-
writer.WritePropertyName("_language");
217-
writer.WriteString(value.Language);
227+
writer.WritePropertyName("date");
228+
formatterResolver.GetFormatter<DateTime?>().Serialize(ref writer, value.Date, formatterResolver);
218229
}
219230

220231
if (value.DetectLanguage.HasValue)
221232
{
222233
writer.WriteValueSeparator();
223-
writer.WritePropertyName("_detect_language");
234+
writer.WritePropertyName("detect_language");
224235
writer.WriteBoolean(value.DetectLanguage.Value);
225236
}
226237

227238
if (value.IndexedCharacters.HasValue)
228239
{
229240
writer.WriteValueSeparator();
230-
writer.WritePropertyName("_indexed_chars");
241+
writer.WritePropertyName("indexed_chars");
231242
writer.WriteInt64(value.IndexedCharacters.Value);
232243
}
233244

245+
if (!string.IsNullOrEmpty(value.Keywords))
246+
{
247+
writer.WriteValueSeparator();
248+
writer.WritePropertyName("keywords");
249+
writer.WriteString(value.Keywords);
250+
}
251+
252+
if (!string.IsNullOrEmpty(value.Language))
253+
{
254+
writer.WriteValueSeparator();
255+
writer.WritePropertyName("language");
256+
writer.WriteString(value.Language);
257+
}
258+
259+
if (!string.IsNullOrEmpty(value.Name))
260+
{
261+
writer.WriteValueSeparator();
262+
writer.WritePropertyName("name");
263+
writer.WriteString(value.Name);
264+
}
265+
266+
if (!string.IsNullOrEmpty(value.Title))
267+
{
268+
writer.WriteValueSeparator();
269+
writer.WritePropertyName("title");
270+
writer.WriteString(value.Title);
271+
}
272+
234273
writer.WriteEndObject();
235274
}
236275
}

‎src/Tests/Tests/Document/Single/Index/IndexIngestAttachmentApiTests.cs‎

Lines changed: 100 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.IO;
4+
using Elastic.Xunit.XunitPlumbing;
45
using Elasticsearch.Net;
56
using FluentAssertions;
67
using Nest;
@@ -29,16 +30,22 @@ static TestDocument()
2930
public static string TestPdfDocument { get; }
3031
}
3132

33+
public class IngestedAttachment
34+
{
35+
public Attachment Attachment { get; set; }
36+
public string Content { get; set; }
37+
public int Id { get; set; }
38+
}
39+
3240
public class IndexIngestAttachmentApiTests
3341
: ApiIntegrationTestBase<IntrusiveOperationCluster, IIndexResponse,
34-
IIndexRequest<IndexIngestAttachmentApiTests.IngestedAttachment>,
35-
IndexDescriptor<IndexIngestAttachmentApiTests.IngestedAttachment>,
36-
IndexRequest<IndexIngestAttachmentApiTests.IngestedAttachment>>
42+
IIndexRequest<IngestedAttachment>,
43+
IndexDescriptor<IngestedAttachment>,
44+
IndexRequest<IngestedAttachment>>
3745
{
3846
public IndexIngestAttachmentApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
3947

4048
protected override bool ExpectIsValid => true;
41-
4249
protected override object ExpectJson => new { id = 1, content = Content };
4350
protected override int ExpectStatusCode => 201;
4451

@@ -70,7 +77,7 @@ protected override string UrlPath
7077
Content = Content
7178
};
7279

73-
private static string PipelineId { get; } = "pipeline-" + Guid.NewGuid().ToString("N").Substring(0, 8);
80+
private static string PipelineId { get; } = "pipeline-" + RandomString();
7481

7582
protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
7683
{
@@ -137,12 +144,96 @@ protected override void ExpectResponse(IIndexResponse response)
137144
attachment.ContentLength.Should().Be(96);
138145
attachment.Content.Should().Contain("mapper-attachment support");
139146
}
147+
}
148+
149+
public class ReindexAttachmentApiTests : IClusterFixture<IntrusiveOperationCluster>
150+
{
151+
private readonly IElasticClient _client;
152+
153+
private static string RandomString { get; } = Guid.NewGuid().ToString("N").Substring(0, 8);
154+
private static string Index { get; } = "project" + RandomString;
155+
private static string PipelineId { get; } = "pipeline-" + RandomString;
156+
private static string Content => TestDocument.TestPdfDocument;
157+
158+
private IngestedAttachment Document => new IngestedAttachment
159+
{
160+
Id = 1,
161+
Content = Content
162+
};
163+
164+
public ReindexAttachmentApiTests(IntrusiveOperationCluster cluster)
165+
{
166+
_client = cluster.Client;
167+
168+
_client.PutPipeline(new PutPipelineRequest(PipelineId)
169+
{
170+
Description = "Attachment pipeline test",
171+
Processors = new List<IProcessor>
172+
{
173+
new AttachmentProcessor
174+
{
175+
Field = "content",
176+
TargetField = "attachment"
177+
}
178+
}
179+
});
180+
181+
var createIndexResponse = _client.CreateIndex(Index, c => c
182+
.Mappings(m => m
183+
.Map<IngestedAttachment>(mm => mm
184+
.Properties(p => p
185+
.Text(s => s
186+
.Name(f => f.Content)
187+
)
188+
.Object<Attachment>(o => o
189+
.Name(f => f.Attachment)
190+
)
191+
)
192+
)
193+
)
194+
);
195+
196+
createIndexResponse.ShouldBeValid();
197+
var indexResponse = _client.Index<IngestedAttachment>(Document, i => i
198+
.Index(Index)
199+
.Refresh(Refresh.True)
200+
.Pipeline(PipelineId)
201+
);
202+
203+
indexResponse.ShouldBeValid();
204+
}
140205

141-
public class IngestedAttachment
206+
[I]
207+
public void ReindexIntoAnotherIndexCorrectlySerializesAttachment()
142208
{
143-
public Attachment Attachment { get; set; }
144-
public string Content { get; set; }
145-
public int Id { get; set; }
209+
var getResponse = _client.Get<IngestedAttachment>(1, g => g
210+
.Index(Index)
211+
);
212+
213+
getResponse.ShouldBeValid();
214+
var ingestedAttachment = getResponse.Source;
215+
ingestedAttachment.Should().NotBeNull();
216+
217+
var indexResponse = _client.Index<IngestedAttachment>(ingestedAttachment, i => i
218+
.Index(Index + "2")
219+
.Refresh(Refresh.True)
220+
);
221+
222+
indexResponse.ShouldBeValid();
223+
224+
getResponse = _client.Get<IngestedAttachment>(1, g => g
225+
.Index(Index + "2")
226+
);
227+
228+
getResponse.ShouldBeValid();
229+
ingestedAttachment.Should().NotBeNull();
230+
ingestedAttachment.Attachment.Title.Should().Be("Attachment Test Document");
231+
ingestedAttachment.Attachment.Keywords.Should().Be("nest,test,document");
232+
ingestedAttachment.Attachment.Date.Should().Be(new DateTime(2016, 12, 08, 3, 5, 13, DateTimeKind.Utc));
233+
ingestedAttachment.Attachment.ContentType.Should().Be("application/pdf");
234+
ingestedAttachment.Attachment.Author.Should().Be("Russ Cam");
235+
ingestedAttachment.Attachment.Language.Should().Be("fr");
236+
ingestedAttachment.Attachment.ContentLength.Should().Be(96);
146237
}
147238
}
148239
}

0 commit comments

Comments
 (0)