Implement recursive plugin discovery (elastic#68811)

* implements recursive scanning in plugin discovery system

* update optimizer to find plugins in sub-directories

* update renovate

* update optimizer IT snapshot

* refactor processPluginSearchPaths$ and add test for inaccessible manifest

* add symlink test

* add maxDepth to the optimizer

* adapt mockFs definitions

* remove `flat` usage
# Conflicts:
#	renovate.json5

package.json

@@ -356,6 +356,7 @@
  "@types/markdown-it": "^0.0.7",
  "@types/minimatch": "^2.0.29",
  "@types/mocha": "^7.0.2",
+ "@types/mock-fs": "^4.10.0",
  "@types/moment-timezone": "^0.5.12",
  "@types/mustache": "^0.8.31",
  "@types/node": ">=10.17.17 <10.20.0",
@@ -468,6 +469,7 @@
  "listr": "^0.14.1",
  "load-grunt-config": "^3.0.1",
  "mocha": "^7.1.1",
+ "mock-fs": "^4.12.0",
  "mock-http-server": "1.3.0",
  "ms-chromium-edge-driver": "^0.2.3",
  "multistream": "^2.1.1",

packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts

@@ -41,18 +41,18 @@ it('parses kibana.json files of plugins found in pluginDirs', () => {
  "id": "bar",
  "isUiPlugin": true,
  },
- Object {
- "directory": <absolute path>/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/baz,
- "extraPublicDirs": Array [],
- "id": "baz",
- "isUiPlugin": false,
- },
  Object {
  "directory": <absolute path>/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo,
  "extraPublicDirs": Array [],
  "id": "foo",
  "isUiPlugin": true,
  },
+ Object {
+ "directory": <absolute path>/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/nested/baz,
+ "extraPublicDirs": Array [],
+ "id": "baz",
+ "isUiPlugin": false,
+ },
  Object {
  "directory": <absolute path>/packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz,
  "extraPublicDirs": Array [],

packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts

@@ -37,7 +37,7 @@ export function findKibanaPlatformPlugins(scanDirs: string[], paths: string[]) {
  .sync(
  Array.from(
  new Set([
- ...scanDirs.map((dir) => `${dir}/*/kibana.json`),
+ ...scanDirs.map(nestedScanDirPaths).reduce((dirs, current) => [...dirs, ...current], []),
  ...paths.map((path) => `${path}/kibana.json`),
  ])
  ),
@@ -51,6 +51,17 @@ export function findKibanaPlatformPlugins(scanDirs: string[], paths: string[]) {
  );
 }
 
+function nestedScanDirPaths(dir: string): string[] {
+ // down to 5 level max
+ return [
+ `${dir}/*/kibana.json`,
+ `${dir}/*/*/kibana.json`,
+ `${dir}/*/*/*/kibana.json`,
+ `${dir}/*/*/*/*/kibana.json`,
+ `${dir}/*/*/*/*/*/kibana.json`,
+ ];
+}
+
 function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin {
  if (!Path.isAbsolute(manifestPath)) {
  throw new TypeError('expected new platform manifest path to be absolute');

src/core/server/plugins/discovery/plugins_discovery.test.mocks.ts

@@ -17,14 +17,5 @@
  * under the License.
  */
 
-export const mockReaddir = jest.fn();
-export const mockReadFile = jest.fn();
-export const mockStat = jest.fn();
-jest.mock('fs', () => ({
- readdir: mockReaddir,
- readFile: mockReadFile,
- stat: mockStat,
-}));
-
 export const mockPackage = new Proxy({ raw: {} as any }, { get: (obj, prop) => obj.raw[prop] });
 jest.mock('../../../../../package.json', () => mockPackage);