Fix/7.0 ga code generation

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -133,13 +133,27 @@ private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFi
 			var endpoint = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
 			endpoint.Value.RestSpecName = endpoint.Key;
 			endpoint.Value.CsharpMethodName = CreateMethodName(endpoint.Key);
+
+			PatchUrlParts(jsonFile, endpoint.Value.Url);
 			return endpoint;
 		}
 
+		private static void PatchUrlParts(string jsonFile, ApiUrl url)
+		{
+			if (url.IsPartless) return;
+			foreach (var kv in url.Parts)
+			{
+				var required = url.ExposedApiPaths.All(p => p.Path.Contains($"{{{kv.Key}}}"));
+				if (kv.Value.Required != required)
+					Warnings.Add($"{jsonFile} has part: {kv.Key} listed as {kv.Value.Required} but should be {required}");
+				kv.Value.Required = required;
+			}
+		}
+
 		private static void PatchOfficialSpec(JObject original, string jsonFile)
 		{
 			var directory = Path.GetDirectoryName(jsonFile);
-			var patchFile = Path.Combine(directory, Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
+			var patchFile = Path.Combine(directory,"..", "_Patches", Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
 			if (!File.Exists(patchFile)) return;
 
 			var patchedJson = JObject.Parse(File.ReadAllText(patchFile));

src/CodeGeneration/ApiGenerator/ApiGenerator.csproj

@@ -12,7 +12,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
-    <PackageReference Include="ShellProgressBar" Version="4.1.1" />
+    <PackageReference Include="ShellProgressBar" Version="4.2.0" />
     <PackageReference Include="CsQuery.Core" Version="2.0.1" />
     <!-- https://github.com/toddams/RazorLight/issues/172 -->
     <PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.1" />

src/CodeGeneration/ApiGenerator/CodeConfiguration.cs

@@ -56,6 +56,7 @@ public static class CodeConfiguration
 			"indices.unfreeze.json",
 
 			"ccr.follow_info.json",
+			"ccr.forget_follower.json"
 		};
 
 

src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs

@@ -20,7 +20,7 @@ public IEnumerable<string> Paths
 			set => _paths = (value ?? Enumerable.Empty<string>()).ToList();
 		}
 
-		public IDictionary<string, ApiUrlPart> Parts { private get; set; }
+		public IDictionary<string, ApiUrlPart> Parts { get; set; }
 
 		public IEnumerable<ApiPath> ExposedApiPaths
 		{

src/CodeGeneration/ApiGenerator/Program.cs

@@ -10,14 +10,14 @@ public static class Program
 		private static void Main(string[] args)
 		{
 			var redownloadCoreSpecification = false;
+			var generateCode = false;
 			var downloadBranch = DownloadBranch;
 
 			var answer = "invalid";
 			while (answer != "y" && answer != "n" && answer != "")
 			{
-				Console.Write("Download online rest specifications? [Y/N] (default N): ");
-				//answer = Console.ReadLine()?.Trim().ToLowerInvariant();
-				answer = "n";
+				Console.Write("Download online rest specifications? [y/N] (default N): ");
+				answer = Console.ReadLine()?.Trim().ToLowerInvariant();
 				redownloadCoreSpecification = answer == "y";
 			}
 
@@ -40,7 +40,15 @@ private static void Main(string[] args)
 			if (redownloadCoreSpecification)
 				RestSpecDownloader.Download(downloadBranch);
 
-			ApiGenerator.Generate(downloadBranch, "Core", "XPack");
+			answer = "invalid";
+			while (answer != "y" && answer != "n" && answer != "")
+			{
+				Console.Write("Generate code from the specification files on disk? [Y/n] (default Y): ");
+				answer = Console.ReadLine()?.Trim().ToLowerInvariant();
+				generateCode = answer == "y" || answer == "";
+			}
+			if (generateCode)
+				ApiGenerator.Generate(downloadBranch, "Core", "XPack");
 		}
 	}
 }

src/CodeGeneration/ApiGenerator/RestSpecDownloader.cs

@@ -15,6 +15,7 @@ public class RestSpecDownloader
 		private static readonly Dictionary<string, string> OnlineSpecifications = new Dictionary<string, string>
 		{
 			{ "Core", "https://github.com/elastic/elasticsearch/tree/{version}/rest-api-spec/src/main/resources/rest-api-spec/api" },
+			{ "XPack", "https://github.com/elastic/elasticsearch/tree/{version}/x-pack/plugin/src/test/resources/rest-api-spec/api"}
 		};
 
 		private static readonly ProgressBarOptions SubProgressBarOptions = new ProgressBarOptions
@@ -32,7 +33,6 @@ private RestSpecDownloader(string branch)
 					let url = kv.Value.Replace("{version}", branch)
 					select new Specification { FolderOnDisk = kv.Key, Branch = branch, GithubListingUrl = url }).ToList();
 
-
 			using (var pbar = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions))
 			{
 				foreach (var spec in specifications)

src/CodeGeneration/ApiGenerator/RestSpecification/Core/bulk.json

@@ -4,14 +4,7 @@
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/_bulk",
-      "paths": ["/_bulk", "/{index}/_bulk"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/_bulk",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_bulk", "/{index}/_bulk", "/{index}/{type}/_bulk"],
       "parts": {
         "index": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/clear_scroll.json

@@ -3,15 +3,8 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_search/scroll",
-      "paths": [ "/_search/scroll"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/_search/scroll/{scroll_id}",
-          "description" : "A scroll id can be quite large and should be specified as part of the body"
-        }
-      ],
+      "path": "/_search/scroll/{scroll_id}",
+      "paths": ["/_search/scroll/{scroll_id}", "/_search/scroll"],
       "parts": {
         "scroll_id": {
           "type" : "list",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/count.json

@@ -4,14 +4,7 @@
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_count",
-      "paths": ["/_count", "/{index}/_count"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/_count",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_count", "/{index}/_count", "/{index}/{type}/_count"],
       "parts": {
         "index": {
           "type" : "list",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/create.json

@@ -4,14 +4,7 @@
     "methods": ["PUT","POST"],
     "url": {
       "path": "/{index}/_create/{id}",
-      "paths": ["/{index}/_create/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}/_create",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_create/{id}", "/{index}/{type}/{id}/_create"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/delete.json

@@ -4,14 +4,7 @@
     "methods": ["DELETE"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/_doc/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/{type}/{id}", "/{index}/_doc/{id}"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/delete_by_query.json

@@ -4,14 +4,7 @@
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_delete_by_query",
-      "paths": ["/{index}/_delete_by_query"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/_delete_by_query",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_delete_by_query", "/{index}/{type}/_delete_by_query"],
       "comment": "most things below this are just copied from search.json",
       "parts": {
         "index": {

src/CodeGeneration/ApiGenerator/RestSpecification/Core/exists.json

@@ -4,14 +4,7 @@
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/_doc/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_doc/{id}", "/{index}/{type}/{id}"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/exists_source.json

@@ -4,14 +4,7 @@
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/_source/{id}",
-      "paths": ["/{index}/_source/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}/_source",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_source/{id}", "/{index}/{type}/{id}/_source"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/explain.json

@@ -4,14 +4,7 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/_explain/{id}",
-      "paths": ["/{index}/_explain/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}/_explain",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_explain/{id}", "/{index}/{type}/{id}/_explain"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/get.json

@@ -4,14 +4,7 @@
     "methods": ["GET"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/_doc/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_doc/{id}", "/{index}/{type}/{id}"],
       "parts": {
         "id": {
           "type" : "string",
@@ -65,6 +58,14 @@
           "type" : "list",
           "description" : "A list of fields to extract and return from the _source field"
         },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "A list of fields to exclude from the returned _source field"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "A list of fields to extract and return from the _source field"
+        },
         "version" : {
           "type" : "number",
           "description" : "Explicit version number for concurrency control"

src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_source.json

@@ -4,14 +4,7 @@
     "methods": ["GET"],
     "url": {
       "path": "/{index}/_source/{id}",
-      "paths": ["/{index}/_source/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}/_source",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_source/{id}", "/{index}/{type}/{id}/_source"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/index.json

@@ -4,19 +4,7 @@
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/{index}/_doc",
-      "paths": ["/{index}/_doc/{id}", "/{index}/_doc"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}",
-          "description" : "Specifying types in urls has been deprecated"
-        },
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/{type}", "/{index}/{type}/{id}", "/{index}/_doc/{id}", "/{index}/_doc"],
       "parts": {
         "id": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.exists_type.json

@@ -2,10 +2,6 @@
   "indices.exists_type": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-types-exists.html",
     "methods": ["HEAD"],
-    "deprecated" : {
-      "version" : "7.0",
-      "description" : "Types are being removed from elasticsearch and therefor this API is on the way out. Read more here: https://www.elastic.co/guide/en/elasticsearch/reference/master/removal-of-types.html"
-    },
     "url": {
       "path": "/{index}/_mapping/{type}",
       "paths": ["/{index}/_mapping/{type}"],

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_field_mapping.json

@@ -4,19 +4,7 @@
     "methods": ["GET"],
     "url": {
       "path": "/_mapping/field/{fields}",
-      "paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/_mapping/{type}/field/{fields}",
-          "description" : "Specifying types in urls has been deprecated"
-        },
-        {
-          "version" : "7.0",
-          "path" : "/{index}/_mapping/{type}/field/{fields}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}", "/_mapping/{type}/field/{fields}", "/{index}/_mapping/{type}/field/{fields}"],
       "parts": {
         "index": {
           "type" : "list",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get_mapping.json

@@ -4,19 +4,7 @@
     "methods": ["GET"],
     "url": {
       "path": "/_mapping",
-      "paths": ["/_mapping", "/{index}/_mapping"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/_mapping/{type}",
-          "description" : "Specifying types in urls has been deprecated"
-        },
-        {
-          "version" : "7.0",
-          "path" : "/{index}/_mapping/{type}",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_mapping", "/{index}/_mapping", "/_mapping/{type}", "/{index}/_mapping/{type}"],
       "parts": {
         "index": {
           "type" : "list",