[ML] Fix apiDocs extractor script (elastic#82582) (elastic#83170)

* [ML] fix serializer script

* [ML] fix extractor

* [ML] bump apiDoc version

* [ML] update yarn.lock

package.json

@@ -574,8 +574,8 @@
     "angular-recursion": "^1.0.5",
     "angular-route": "^1.8.0",
     "angular-sortable-view": "^0.0.17",
-    "apidoc": "^0.20.1",
-    "apidoc-markdown": "^5.0.0",
+    "apidoc": "^0.25.0",
+    "apidoc-markdown": "^5.1.8",
     "apollo-link": "^1.2.3",
     "apollo-link-error": "^1.1.7",
     "apollo-link-state": "^0.4.1",

x-pack/plugins/ml/server/routes/apidoc.json

@@ -1,6 +1,6 @@
 {
   "name": "ml_kibana_api",
-  "version": "7.8.0",
+  "version": "7.11.0",
   "description": "This is the documentation of the REST API provided by the Machine Learning Kibana plugin. Each API is experimental and can include breaking changes in any version.",
   "title": "ML Kibana API",
   "order": [

x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts

@@ -0,0 +1,209 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { extractDocumentation } from './schema_extractor';
+import * as path from 'path';
+
+describe('schema_extractor', () => {
+  it('should serialize schema definition', () => {
+    const result = extractDocumentation([
+      path.resolve(__dirname, '..', 'schemas', 'datafeeds_schema.ts'),
+    ]);
+
+    expect(result.get('startDatafeedSchema')).toEqual([
+      {
+        name: 'start',
+        documentation: '',
+        type: 'string | number',
+      },
+      {
+        name: 'end',
+        documentation: '',
+        type: 'string | number',
+      },
+      {
+        name: 'timeout',
+        documentation: '',
+        type: 'any',
+      },
+    ]);
+
+    expect(result.get('datafeedConfigSchema')).toEqual([
+      {
+        name: 'datafeed_id',
+        documentation: '',
+        type: 'string',
+      },
+      {
+        name: 'feed_id',
+        documentation: '',
+        type: 'string',
+      },
+      {
+        name: 'aggregations',
+        documentation: '',
+        type: 'any',
+      },
+      {
+        name: 'aggs',
+        documentation: '',
+        type: 'any',
+      },
+      {
+        name: 'chunking_config',
+        documentation: '',
+        type: 'chunking_config',
+        nested: [
+          {
+            name: 'mode',
+            documentation: '',
+            type: 'string',
+          },
+          {
+            name: 'time_span',
+            documentation: '',
+            type: 'string',
+          },
+        ],
+      },
+      {
+        name: 'frequency',
+        documentation: '',
+        type: 'string',
+      },
+      {
+        name: 'indices',
+        documentation: '',
+        type: 'string[]',
+      },
+      {
+        name: 'indexes',
+        documentation: '',
+        type: 'string[]',
+      },
+      {
+        name: 'job_id',
+        documentation: '',
+        type: 'string',
+      },
+      {
+        name: 'query',
+        documentation: '',
+        type: 'any',
+      },
+      {
+        name: 'max_empty_searches',
+        documentation: '',
+        type: 'number',
+      },
+      {
+        name: 'query_delay',
+        documentation: '',
+        type: 'string',
+      },
+      {
+        name: 'script_fields',
+        documentation: '',
+        type: 'any',
+      },
+      {
+        name: 'scroll_size',
+        documentation: '',
+        type: 'number',
+      },
+      {
+        name: 'delayed_data_check_config',
+        documentation: '',
+        type: 'any',
+      },
+      {
+        name: 'indices_options',
+        documentation: '',
+        type: 'indices_options',
+        nested: [
+          {
+            name: 'expand_wildcards',
+            documentation: '',
+            type: 'string[]',
+          },
+          {
+            name: 'ignore_unavailable',
+            documentation: '',
+            type: 'boolean',
+          },
+          {
+            name: 'allow_no_indices',
+            documentation: '',
+            type: 'boolean',
+          },
+          {
+            name: 'ignore_throttled',
+            documentation: '',
+            type: 'boolean',
+          },
+        ],
+      },
+    ]);
+
+    expect(result.get('deleteDatafeedQuerySchema')).toEqual([
+      {
+        name: 'force',
+        documentation: '',
+        type: 'any', // string
+      },
+    ]);
+  });
+
+  it('serializes schema with nested objects and nullable', () => {
+    const result = extractDocumentation([
+      path.resolve(__dirname, '..', 'schemas', 'results_service_schema.ts'),
+    ]);
+    expect(result.get('getCategorizerStatsSchema')).toEqual([
+      {
+        name: 'partitionByValue',
+        documentation:
+          'Optional value to fetch the categorizer stats where results are filtered by partition_by_value = value',
+        type: 'any', // FIXME string
+      },
+    ]);
+
+    // @ts-ignore
+    expect(result.get('partitionFieldValuesSchema')![5].nested[0]).toEqual({
+      name: 'partition_field',
+      documentation: '',
+      type: 'partition_field',
+      nested: [
+        {
+          name: 'applyTimeRange',
+          documentation: '',
+          type: 'boolean',
+        },
+        {
+          name: 'anomalousOnly',
+          documentation: '',
+          type: 'boolean',
+        },
+        {
+          name: 'sort',
+          documentation: '',
+          type: 'sort',
+          nested: [
+            {
+              name: 'by',
+              documentation: '',
+              type: 'string',
+            },
+            {
+              name: 'order',
+              documentation: '',
+              type: 'string',
+            },
+          ],
+        },
+      ],
+    });
+  });
+});

x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.ts

@@ -90,65 +90,77 @@ export function extractDocumentation(
     return members;
   }
 
-  function resolveTypeArgument(type: ts.Type): ts.SymbolTable | string {
-    // required to extract members
-    type.getProperty('type');
-
-    // @ts-ignores
-    let members = type.members;
+  /**
+   * Extracts properties of the type.
+   * @param type
+   */
+  function resolveTypeProperties(type: ts.Type): ts.Symbol[] {
+    let props = type.getProperties();
 
     const typeArguments = checker.getTypeArguments((type as unknown) as ts.TypeReference);
 
     if (type.aliasTypeArguments) {
       // @ts-ignores
-      members = type.aliasTypeArguments[0].members;
+      props = type.aliasTypeArguments[0].getProperties();
     }
 
     if (typeArguments.length > 0) {
-      members = resolveTypeArgument(typeArguments[0]);
-    }
-
-    if (members === undefined) {
-      members = checker.typeToString(type);
+      props = resolveTypeProperties(typeArguments[0]);
     }
 
-    return members;
+    return props;
   }
 
   function serializeProperty(symbol: ts.Symbol): DocEntry {
     // @ts-ignore
     const typeOfSymbol = symbol.type;
-    const typeArguments = checker.getTypeArguments((typeOfSymbol as unknown) as ts.TypeReference);
+    if (typeOfSymbol === undefined) {
+      return {
+        name: symbol.getName(),
+        documentation: getCommentString(symbol),
+        type: 'any',
+      };
+    }
 
-    let resultType: ts.Type = typeOfSymbol;
+    let targetType: ts.TypeReference | ts.Type =
+      typeOfSymbol.getProperty('type')?.type ?? typeOfSymbol;
 
-    let members;
-    if (typeArguments.length > 0) {
-      members = resolveTypeArgument(typeArguments[0]);
-      resultType = typeArguments[0];
+    const isArrayOf = targetType.symbol?.name === 'Array';
+    if (isArrayOf) {
+      targetType = checker.getTypeArguments(targetType as ts.TypeReference)[0];
     }
 
-    let typeAsString = checker.typeToString(resultType);
-
+    let typeAsString = checker.typeToString(targetType);
     const nestedEntries: DocEntry[] = [];
-    if (members && typeof members !== 'string' && members.size > 0) {
-      // we hit an object or collection
-      typeAsString =
-        resultType.symbol.name === 'Array' || typeOfSymbol.symbol.name === 'Array'
-          ? `${symbol.getName()}[]`
-          : symbol.getName();
-
-      members.forEach((member) => {
-        nestedEntries.push(serializeProperty(member));
-      });
+
+    if (
+      targetType.aliasTypeArguments ||
+      checker.getTypeArguments(targetType as ts.TypeReference).length > 0
+    ) {
+      // Resolve complex types, objects and arrays, that contain nested properties
+      const typeProperties = resolveTypeProperties(targetType);
+
+      if (Array.isArray(typeProperties) && typeProperties.length > 0) {
+        // we hit an object or collection
+        typeAsString =
+          targetType.symbol?.name === 'Array' || typeOfSymbol.symbol?.name === 'Array'
+            ? `${symbol.getName()}[]`
+            : symbol.getName();
+
+        typeProperties.forEach((member) => {
+          nestedEntries.push(serializeProperty(member));
+        });
+      }
     }
 
-    return {
+    const res = {
       name: symbol.getName(),
       documentation: getCommentString(symbol),
-      type: typeAsString,
+      type: isArrayOf ? `${typeAsString}[]` : typeAsString,
       ...(nestedEntries.length > 0 ? { nested: nestedEntries } : {}),
     };
+
+    return res;
   }
 
   function getCommentString(symbol: ts.Symbol): string {

x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts

@@ -10,10 +10,10 @@ import { DocEntry, extractDocumentation } from './schema_extractor';
 import { ApiParameter, Block } from './types';
 
 export function postProcess(parsedFiles: any[]): void {
-  const schemasDirPath = `${__dirname}${path.sep}..${path.sep}..${path.sep}schemas${path.sep}`;
+  const schemasDirPath = path.resolve(__dirname, '..', '..', 'schemas');
   const schemaFiles = fs
     .readdirSync(schemasDirPath)
-    .map((filename) => path.resolve(schemasDirPath + filename));
+    .map((filename) => path.resolve(schemasDirPath, filename));
 
   const schemaDocs = extractDocumentation(schemaFiles);