Feature/nori plugin

src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs

@@ -29,6 +29,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 				case "simple": return o.ToObject<SimpleAnalyzer>(ElasticContractResolver.Empty);
 				case "fingerprint": return o.ToObject<FingerprintAnalyzer>(ElasticContractResolver.Empty);
 				case "kuromoji": return o.ToObject<KuromojiAnalyzer>(ElasticContractResolver.Empty);
+				case "nori": return o.ToObject<NoriAnalyzer>(ElasticContractResolver.Empty);
 				default:
 					if (o.Property("tokenizer") != null)
 						return o.ToObject<CustomAnalyzer>(ElasticContractResolver.Empty);

src/Nest/Analysis/Analyzers/Analyzers.cs

@@ -97,5 +97,9 @@ public AnalyzersDescriptor Fingerprint(string name, Func<FingerprintAnalyzerDesc
 		/// </summary>
 		public AnalyzersDescriptor Kuromoji(string name, Func<KuromojiAnalyzerDescriptor, IKuromojiAnalyzer> selector = null) =>
 			Assign(name, selector.InvokeOrDefault(new KuromojiAnalyzerDescriptor()));
+
+		/// <inheritdoc cref="INoriAnalyzer"/>
+		public AnalyzersDescriptor Nori(string name, Func<NoriAnalyzerDescriptor, INoriAnalyzer> selector) =>
+			Assign(name, selector?.Invoke(new NoriAnalyzerDescriptor()));
 	}
 }

src/Nest/Analysis/Analyzers/NoriAnalyzer.cs

@@ -0,0 +1,65 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	///The nori analyzer consists of the following tokenizer and token filters:
+	///<para> - nori_tokenizer</para>
+	///<para> - nori_part_of_speech token filter</para>
+	///<para> - nori_readingform token filter</para>
+	///<para> - lowercase token filter</para>
+	/// </summary>
+	public interface INoriAnalyzer : IAnalyzer
+	{
+		/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
+		[JsonProperty("decompound_mode")]
+		NoriDecompoundMode? DecompoundMode { get; set; }
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
+		[JsonProperty("user_dictionary")]
+		string UserDictionary { get; set; }
+
+		/// <inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags"/>
+		[JsonProperty("stoptags")]
+		IEnumerable<string> StopTags { get; set; }
+	}
+
+	/// <inheritdoc cref="INoriAnalyzer"/>
+	public class NoriAnalyzer : AnalyzerBase, INoriAnalyzer
+	{
+		public NoriAnalyzer() : base("nori") {}
+
+		/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
+	    public NoriDecompoundMode? DecompoundMode { get; set; }
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
+	    public string UserDictionary { get; set; }
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public IEnumerable<string> StopTags { get; set; }
+	}
+
+	/// <inheritdoc cref="INoriAnalyzer"/>
+	public class NoriAnalyzerDescriptor : AnalyzerDescriptorBase<NoriAnalyzerDescriptor, INoriAnalyzer>, INoriAnalyzer
+	{
+		protected override string Type => "nori";
+
+	    NoriDecompoundMode? INoriAnalyzer.DecompoundMode { get; set; }
+	    string INoriAnalyzer.UserDictionary { get; set; }
+		IEnumerable<string> INoriAnalyzer.StopTags { get; set; }
+
+		/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
+		public NoriAnalyzerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode);
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
+		public NoriAnalyzerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path);
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public NoriAnalyzerDescriptor StopTags(IEnumerable<string> stopTags) => Assign(a => a.StopTags = stopTags);
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public NoriAnalyzerDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags);
+
+	}
+}

src/Nest/Analysis/TokenFilters/NoriPartOfSpeechTokenFilter.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary> The nori_part_of_speech token filter removes tokens that match a set of part-of-speech tags. </summary>
+	public interface INoriPartOfSpeechTokenFilter : ITokenFilter
+	{
+		/// <summary> An array of part-of-speech tags that should be removed. </summary>
+		[JsonProperty("stoptags")]
+		IEnumerable<string> StopTags { get; set; }
+	}
+
+	/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
+	public class NoriPartOfSpeechTokenFilter : TokenFilterBase, INoriPartOfSpeechTokenFilter
+	{
+		public NoriPartOfSpeechTokenFilter() : base("nori_part_of_speech") { }
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public IEnumerable<string> StopTags { get; set; }
+	}
+	/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
+	public class NoriPartOfSpeechTokenFilterDescriptor
+		: TokenFilterDescriptorBase<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter>, INoriPartOfSpeechTokenFilter
+	{
+		protected override string Type => "nori_part_of_speech";
+
+		IEnumerable<string> INoriPartOfSpeechTokenFilter.StopTags { get; set; }
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public NoriPartOfSpeechTokenFilterDescriptor StopTags(IEnumerable<string> stopTags) => Assign(a => a.StopTags = stopTags);
+
+		///<inheritdoc cref="INoriPartOfSpeechTokenFilter.StopTags" />
+		public NoriPartOfSpeechTokenFilterDescriptor StopTags(params string[] stopTags) => Assign(a => a.StopTags = stopTags);
+
+	}
+
+}

src/Nest/Analysis/TokenFilters/TokenFilters.cs

@@ -317,5 +317,10 @@ public TokenFiltersDescriptor IcuNormalization(string name, Func<IcuNormalizatio
 		/// </summary>
 		public TokenFiltersDescriptor IcuTransform(string name, Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			Assign(name, selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
+		public TokenFiltersDescriptor NoriPartOfSpeech(string name, Func<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter> selector) =>
+			Assign(name, selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor()));
+
 	}
 }

src/Nest/Analysis/Tokenizers/NoriTokenizer.cs

@@ -0,0 +1,63 @@
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	/// <summary> The decompound mode determines how the tokenizer handles compound tokens. </summary>
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum NoriDecompoundMode
+	{
+		/// <summary> Decomposes compounds and discards the original form (default). </summary>
+		[EnumMember(Value="discard")]
+		Discard,
+		/// <summary> No decomposition for compounds </summary>
+		[EnumMember(Value="none")]
+		None,
+		/// <summary> Decomposes compounds and keeps the original form </summary>
+		[EnumMember(Value="mixed")]
+		Mixed
+	}
+
+	/// <summary> Tokenizer that ships with the analysis-nori plugin</summary>
+	public interface INoriTokenizer : ITokenizer
+	{
+		/// <summary>
+		/// The regular expression pattern, defaults to \W+.
+		/// </summary>
+		[JsonProperty("decompound_mode")]
+		NoriDecompoundMode? DecompoundMode { get; set; }
+
+		/// <summary>
+		/// The Nori tokenizer uses the mecab-ko-dic dictionary by default. A user_dictionary with custom nouns (NNG) may be appended to
+		/// the default dictionary. This property allows you to specify this file on disk
+		/// </summary>
+		[JsonProperty("user_dictionary")]
+		string UserDictionary { get; set; }
+	}
+
+	/// <inheritdoc cref="INoriTokenizer"/>
+	public class NoriTokenizer : TokenizerBase, INoriTokenizer
+    {
+		public NoriTokenizer() => this.Type = "nori_tokenizer";
+		/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
+	    public NoriDecompoundMode? DecompoundMode { get; set; }
+		/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
+	    public string UserDictionary { get; set; }
+    }
+	/// <inheritdoc cref="INoriTokenizer"/>
+	public class NoriTokenizerDescriptor
+		: TokenizerDescriptorBase<NoriTokenizerDescriptor, INoriTokenizer>, INoriTokenizer
+	{
+		protected override string Type => "nori_tokenizer";
+
+	    NoriDecompoundMode? INoriTokenizer.DecompoundMode { get; set; }
+	    string INoriTokenizer.UserDictionary { get; set; }
+
+		/// <inheritdoc cref="INoriTokenizer.DecompoundMode"/>
+		public NoriTokenizerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(a => a.DecompoundMode = mode);
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionary"/>
+		public NoriTokenizerDescriptor UserDictionary(string path) => Assign(a => a.UserDictionary = path);
+	}
+}

src/Nest/Analysis/Tokenizers/Tokenizers.cs

@@ -19,7 +19,7 @@ public Tokenizers(Dictionary<string, ITokenizer> container)
 		public void Add(string name, ITokenizer analyzer) => BackingDictionary.Add(name, analyzer);
 	}
 
-	public class TokenizersDescriptor :IsADictionaryDescriptorBase<TokenizersDescriptor, ITokenizers, string, ITokenizer>
+	public class TokenizersDescriptor : IsADictionaryDescriptorBase<TokenizersDescriptor, ITokenizers, string, ITokenizer>
 	{
 		public TokenizersDescriptor() : base(new Tokenizers()) { }
 
@@ -112,5 +112,10 @@ public TokenizersDescriptor Kuromoji(string name, Func<KuromojiTokenizerDescript
 		/// </summary>
 		public TokenizersDescriptor Icu(string name, Func<IcuTokenizerDescriptor, IIcuTokenizer> selector) =>
 			Assign(name, selector?.Invoke(new IcuTokenizerDescriptor()));
+
+		/// <inheritdoc cref="INoriTokenizer"/>
+		public TokenizersDescriptor Nori(string name, Func<NoriTokenizerDescriptor, INoriTokenizer> selector) =>
+			Assign(name, selector?.Invoke(new NoriTokenizerDescriptor()));
+
 	}
 }

src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs

@@ -323,6 +323,10 @@ public AnalyzeTokenFiltersDescriptor IcuNormalization(Func<IcuNormalizationToken
 		/// </summary>
 		public AnalyzeTokenFiltersDescriptor IcuTransform(Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			AssignIfNotNull(selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		/// <inheritdoc cref="INoriPartOfSpeechTokenFilter"/>
+		public AnalyzeTokenFiltersDescriptor NoriPartOfSpeech(Func<NoriPartOfSpeechTokenFilterDescriptor, INoriPartOfSpeechTokenFilter> selector) =>
+			AssignIfNotNull(selector.Invoke(new NoriPartOfSpeechTokenFilterDescriptor()));
 	}
 
 }

src/Nest/Indices/Analyze/AnalyzeTokenizersDescriptor.cs

@@ -91,5 +91,9 @@ public ITokenizer Kuromoji(Func<KuromojiTokenizerDescriptor, IKuromojiTokenizer>
 		/// </summary>
 		public ITokenizer Icu(Func<IcuTokenizerDescriptor, IIcuTokenizer> selector) =>
 			(selector?.Invoke(new IcuTokenizerDescriptor()));
+
+		/// <inheritdoc cref="INoriTokenizer"/>
+		public ITokenizer Nori(Func<NoriTokenizerDescriptor, INoriTokenizer> selector) =>
+			selector.Invoke(new NoriTokenizerDescriptor());
 	}
 }

src/Tests/Tests.Core/ManagedElasticsearch/Clusters/ReadOnlyCluster.cs

@@ -1,11 +1,11 @@
-using Elastic.Managed.Ephemeral.Plugins;
-using Tests.Core.ManagedElasticsearch.NodeSeeders;
+using Tests.Core.ManagedElasticsearch.NodeSeeders;
+using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin;
 
 namespace Tests.Core.ManagedElasticsearch.Clusters
 {
 	public class ReadOnlyCluster : ClientTestClusterBase
 	{
-		public ReadOnlyCluster() : base(ElasticsearchPlugin.MapperMurmur3) { }
+		public ReadOnlyCluster() : base(MapperMurmur3) { }
 
 		protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNode();
 	}

src/Tests/Tests.Core/ManagedElasticsearch/Clusters/WritableCluster.cs

@@ -1,13 +1,21 @@
 using Elastic.Managed.Ephemeral.Plugins;
 using Tests.Core.ManagedElasticsearch.NodeSeeders;
+using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin;
 
 namespace Tests.Core.ManagedElasticsearch.Clusters
 {
 	/// <summary> Use this cluster for api's that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster instead. </summary>
 	public class WritableCluster : ClientTestClusterBase
 	{
 		public WritableCluster() : base(new ClientTestClusterConfiguration(
-			ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.IngestAttachment, ElasticsearchPlugin.AnalysisKuromoji, ElasticsearchPlugin.AnalysisIcu, ElasticsearchPlugin.AnalysisPhonetic, ElasticsearchPlugin.MapperMurmur3
+			IngestGeoIp,
+			IngestAttachment,
+			AnalysisKuromoji,
+			AnalysisIcu,
+			AnalysisPhonetic,
+			MapperMurmur3,
+			//TODO move this to elasticsearch-net abstractions
+			new ElasticsearchPlugin("analysis-nori", v => v >= "6.4.0")
 		)
 		{
 			MaxConcurrency = 4

src/Tests/Tests.Core/Tests.Core.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Tests.Domain\Tests.Domain.csproj" />
-    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20180902T153954" />
+    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20180925T171717" />
     <PackageReference Include="xunit" Version="2.3.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
     <PackageReference Include="FluentAssertions" Version="4.19.2" />

src/Tests/Tests.Domain/Tests.Domain.csproj

@@ -12,7 +12,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Bogus" Version="22.1.2" />
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20180902T153954" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20180925T171717" />
     <ProjectReference Include="..\Tests.Configuration\Tests.Configuration.csproj" />
   </ItemGroup>
 </Project>

src/Tests/Tests/Analysis/AnalysisComponentTestBase.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Elastic.Xunit;
+using Elastic.Xunit.XunitPlumbing;
+using FluentAssertions;
+using Nest;
+using Tests.Core.Client;
+using Tests.Core.ManagedElasticsearch.Clusters;
+using Tests.Core.Serialization;
+using Tests.Framework.Integration;
+
+namespace Tests.Analysis
+{
+	public interface IAnalysisAssertion
+	{
+		string Name { get; }
+		object Json { get; }
+	}
+	public interface IAnalysisAssertion<out TComponent, out TContainer, in TDescriptor> : IAnalysisAssertion
+		where TContainer : class
+	{
+		TComponent Initializer { get; }
+		Func<string, TDescriptor, IPromise<TContainer>> Fluent { get; }
+	}
+
+	[IntegrationTestCluster(typeof(WritableCluster))]
+	public abstract class AnalysisComponentTestBase<TAssertion, TComponent, TContainer, TDescriptor>
+		: IAnalysisAssertion<TComponent, TContainer, TDescriptor>
+		where TAssertion : AnalysisComponentTestBase<TAssertion, TComponent, TContainer, TDescriptor>, new()
+		where TContainer : class
+	{
+		private static readonly SingleEndpointUsage<ICreateIndexResponse> Usage = new SingleEndpointUsage<ICreateIndexResponse>
+		(
+			fluent: (s, c) => c.CreateIndex(s, AssertionSetup.FluentCall),
+			fluentAsync: (s, c) => c.CreateIndexAsync(s, AssertionSetup.FluentCall),
+			request: (s, c) => c.CreateIndex(AssertionSetup.InitializerCall(s)),
+			requestAsync: (s, c) => c.CreateIndexAsync(AssertionSetup.InitializerCall(s)),
+			valuePrefix: $"test-{typeof(TAssertion).Name.ToLowerInvariant()}"
+		)
+		{
+			OnAfterCall = c=> c.DeleteIndex(Usage.CallUniqueValues.Value)
+		};
+		protected static TAssertion AssertionSetup { get; } = new TAssertion();
+
+		protected AnalysisComponentTestBase()
+		{
+			this.Client = (ElasticXunitRunner.CurrentCluster as ReadOnlyCluster)?.Client ?? TestClient.DefaultInMemoryClient;
+			Usage.KickOffOnce(this.Client, oneRandomCall: true);
+		}
+
+		private IElasticClient Client { get; }
+
+		public abstract string Name { get; }
+		public abstract TComponent Initializer { get; }
+		public abstract Func<string, TDescriptor, IPromise<TContainer>> Fluent { get; }
+		public abstract object Json { get; }
+
+		private Func<CreateIndexDescriptor, ICreateIndexRequest> FluentCall => i =>i.Settings(s => s.Analysis(this.FluentAnalysis));
+		protected abstract IAnalysis FluentAnalysis(AnalysisDescriptor an);
+
+		private CreateIndexRequest InitializerCall(string index) => new CreateIndexRequest(index)
+		{
+			Settings = new IndexSettings { Analysis = this.InitializerAnalysis() }
+		};
+		protected abstract Nest.Analysis InitializerAnalysis();
+
+		[U] public virtual async Task TestPutSettingsRequest() => await Usage.AssertOnAllResponses(r =>
+		{
+			var json = new { settings = new { analysis = this.AnalysisJson } };
+			SerializationTestHelper.Expect(json).FromRequest(r);
+		});
+
+		protected abstract object AnalysisJson { get; }
+
+		[I] public virtual async Task TestPutSettingsResponse() => await Usage.AssertOnAllResponses(r =>
+		{
+			r.ApiCall.HttpStatusCode.Should().Be(200);
+		});
+
+	}
+}