Merge branch 'master' into alerting/instance-pagination

* master: (22 commits)
  skip flaky suite (elastic#50018)
  skip settings tests (elastic#57608)
  skip failing suite (elastic#44631)
  [SIEM] [Case] Initial UI (elastic#57283)
  handle viewing sample dashboards on default dist (elastic#57510)
  Fix detection of "system requests" in plugins (elastic#57149)
  [ML] New Platform server shim: update job service schema (elastic#57614)
  skip flaky suite (elastic#44631)
  [APM] Update monospace font family variable (elastic#57555)
  skip flaky test (elastic#57377)
  Skip save query tests (elastic#57589)
  [Maps] allow simultaneous opening of multiple tooltips (elastic#57226)
  [Uptime] Fix/host connected components (elastic#56969)
  [logs][metrics][docs] Update screenshots for 7.6 (elastic#57254)
  [ML] New Platform server shim: update job service routes to use new platform router (elastic#57403)
  [Maps] Fix document source top hits split by scripted field (elastic#57481)
  Use log4j pattern syntax (elastic#57433)
  [ML] Categorization field example endpoint tests (elastic#57471)
  [Lens] Filter out pinned filters from saved object of Lens (elastic#57197)
  Lens client side shim cleanup (elastic#56976)
  ...

docs/logs/using.asciidoc

@@ -8,7 +8,6 @@ You can also view related application traces or uptime information where availab
 
 [role="screenshot"]
 image::logs/images/logs-console.png[Logs Console in Kibana]
-// ++ Update this
 
 [float]
 [[logs-search]]

src/core/MIGRATION.md

@@ -56,6 +56,7 @@
       - [On the server side](#on-the-server-side)
       - [On the client side](#on-the-client-side)
     - [Updates an application navlink at runtime](#updates-an-app-navlink-at-runtime)
+    - [Logging config migration](#logging-config-migration)
 
 Make no mistake, it is going to take a lot of work to move certain plugins to the new platform. Our target is to migrate the entire repo over to the new platform throughout 7.x and to remove the legacy plugin system no later than 8.0, and this is only possible if teams start on the effort now.
 
@@ -1655,4 +1656,7 @@ export class MyPlugin implements Plugin {
        tooltip: 'Application disabled',
      })
   }
-```
+```
+
+### Logging config migration
+[Read](./server/logging/README.md#logging-config-migration)

src/core/server/logging/README.md

@@ -1,4 +1,12 @@
 # Logging
+- [Loggers, Appenders and Layouts](#loggers-appenders-and-layouts)
+- [Logger hierarchy](#logger-hierarchy)
+- [Log level](#log-level)
+- [Layouts](#layouts)
+  - [Pattern layout](#pattern-layout)
+  - [JSON layout](#json-layout)
+- [Configuration](#configuration)
+- [Usage](#usage)
 
 The way logging works in Kibana is inspired by `log4j 2` logging framework used by [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html#logging).
 The main idea is to have consistent logging behaviour (configuration, log format etc.) across the entire Elastic Stack 
@@ -52,12 +60,68 @@ custom appenders, so one should always make the choice explicitly.
 
 There are two types of layout supported at the moment: `pattern` and `json`. 
 
-With `pattern` layout it's possible to define a string pattern with special placeholders wrapped into curly braces that
+### Pattern layout
+With `pattern` layout it's possible to define a string pattern with special placeholders `%conversion_pattern` (see the table below) that
 will be replaced with data from the actual log message. By default the following pattern is used: 
-`[{timestamp}][{level}][{context}] {message}`. Also `highlight` option can be enabled for `pattern` layout so that
+`[%date][%level][%logger]%meta %message`. Also `highlight` option can be enabled for `pattern` layout so that
 some parts of the log message are highlighted with different colors that may be quite handy if log messages are forwarded
 to the terminal with color support.
+`pattern` layout uses a sub-set of [log4j2 pattern syntax](https://logging.apache.org/log4j/2.x/manual/layouts.html#PatternLayout)
+and **doesn't implement** all `log4j2` capabilities. The conversions that are provided out of the box are:
 
+#### level
+Outputs the [level](#log-level) of the logging event.
+Example of `%level` output:
+```bash
+TRACE
+DEBUG
+INFO
+```
+
+##### logger
+Outputs the name of the logger that published the logging event.
+Example of `%logger` output:
+```bash
+server
+server.http
+server.http.Kibana
+```
+
+#### message
+Outputs the application supplied message associated with the logging event.
+
+#### meta
+Outputs the entries of `meta` object data in **json** format, if one is present in the event.
+Example of `%meta` output:
+```bash
+// Meta{from: 'v7', to: 'v8'}
+'{"from":"v7","to":"v8"}'
+// Meta empty object
+'{}'
+// no Meta provided
+''
+```
+
+##### date
+Outputs the date of the logging event. The date conversion specifier may be followed by a set of braces containing a name of predefined date format and canonical timezone name.
+Timezone name is expected to be one from [TZ database name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
+Example of `%date` output:
+
+| Conversion pattern                       | Example                                                          |
+| ---------------------------------------- | ---------------------------------------------------------------- |
+| `%date`                                  | `2012-02-01T14:30:22.011Z` uses `ISO8601` format by default      |
+| `%date{ISO8601}`                         | `2012-02-01T14:30:22.011Z`                                       |
+| `%date{ISO8601_TZ}`                      | `2012-02-01T09:30:22.011-05:00`   `ISO8601` with timezone        |
+| `%date{ISO8601_TZ}{America/Los_Angeles}` | `2012-02-01T06:30:22.011-08:00`                                  |
+| `%date{ABSOLUTE}`                        | `09:30:22.011`                                                   |
+| `%date{ABSOLUTE}{America/Los_Angeles}`   | `06:30:22.011`                                                   |
+| `%date{UNIX}`                            | `1328106622`                                                     |
+| `%date{UNIX_MILLIS}`                     | `1328106622011`                                                  |
+
+#### pid
+Outputs the process ID.
+
+### JSON layout
 With `json` layout log messages will be formatted as JSON strings that include timestamp, log level, context, message 
 text and any other metadata that may be associated with the log message itself.
 
@@ -88,7 +152,7 @@ logging:
       kind: console
       layout:
         kind: pattern
-        pattern: [{timestamp}][{level}] {message}
+        pattern: "[%date][%level] %message"
     json-file-appender:
       kind: file
       path: /var/log/kibana-json.log
@@ -179,3 +243,81 @@ The log will be less verbose with `warn` level for the `server` context:
 [2017-07-25T18:54:41.639Z][ERROR][server] Message with `error` log level.
 [2017-07-25T18:54:41.639Z][FATAL][server] Message with `fatal` log level.
 ```
+
+### Logging config migration
+Compatibility with the legacy logging system is assured until the end of the `v7` version.
+All log messages handled by `root` context are forwarded to the legacy logging service. If you re-write
+root appenders, make sure that it contains `default` appender to provide backward compatibility.
+**Note**: If you define an appender for a context, the log messages aren't handled by the
+`root` context anymore and not forwarded to the legacy logging service.
+
+#### logging.dest
+By default logs in *stdout*. With new Kibana logging you can use pre-existing `console` appender or
+define a custom one.
+```yaml
+logging:
+  loggers:
+    - context: your-plugin
+      appenders: [console]
+```
+Logs in a *file* if given file path. You should define a custom appender with `kind: file` 
+```yaml
+
+logging:
+  appenders:
+    file:
+      kind: file
+      path: /var/log/kibana.log
+      layout:
+        kind: pattern
+  loggers:
+    - context: your-plugin
+      appenders: [file]
+``` 
+#### logging.json
+Defines the format of log output. Logs in JSON if `true`. With new logging config you can adjust
+the output format with [layouts](#layouts).
+
+#### logging.quiet
+Suppresses all logging output other than error messages. With new logging, config can be achieved 
+with adjusting minimum required [logging level](#log-level)
+```yaml
+  loggers:
+    - context: my-plugin
+      appenders: [console]
+      level: error
+# or for all output
+logging.root.level: error
+```
+
+#### logging.silent:
+Suppresses all logging output.
+```yaml
+logging.root.level: off
+```
+
+#### logging.verbose:
+Logs all events
+```yaml
+logging.root.level: all
+```
+
+#### logging.timezone
+Set to the canonical timezone id to log events using that timezone. New logging config allows
+to [specify timezone](#date) for `layout: pattern`.
+```yaml
+logging:
+  appenders:
+    custom-console:
+      kind: console
+      layout:
+        kind: pattern
+        highlight: true
+        pattern: "[%level] [%date{ISO8601_TZ}{America/Los_Angeles}][%logger] %message"
+```
+
+#### logging.events
+Define a custom logger for a specific context.
+
+#### logging.filter
+TBD

src/core/server/logging/integration_tests/logging.test.ts

@@ -29,7 +29,7 @@ function createRoot() {
           layout: {
             highlight: false,
             kind: 'pattern',
-            pattern: '{level}|{context}|{message}',
+            pattern: '%level|%logger|%message',
           },
         },
       },

src/core/server/logging/layouts/conversions/timestamp.ts → src/core/server/logging/layouts/conversions/date.ts

@@ -22,7 +22,7 @@ import { last } from 'lodash';
 import { Conversion } from './type';
 import { LogRecord } from '../../log_record';
 
-const timestampRegExp = /{timestamp({(?<format>[^}]+)})?({(?<timezone>[^}]+)})?}/gi;
+const dateRegExp = /%date({(?<format>[^}]+)})?({(?<timezone>[^}]+)})?/g;
 
 const formats = {
   ISO8601: 'ISO8601',
@@ -54,10 +54,11 @@ function formatDate(date: Date, dateFormat: string = formats.ISO8601, timezone?:
 }
 
 function validateDateFormat(input: string) {
-  if (Reflect.has(formats, input)) return;
-  throw new Error(
-    `Date format expected one of ${Reflect.ownKeys(formats).join(', ')}, but given: ${input}`
-  );
+  if (!Reflect.has(formats, input)) {
+    throw new Error(
+      `Date format expected one of ${Reflect.ownKeys(formats).join(', ')}, but given: ${input}`
+    );
+  }
 }
 
 function validateTimezone(timezone: string) {
@@ -66,7 +67,7 @@ function validateTimezone(timezone: string) {
 }
 
 function validate(rawString: string) {
-  for (const matched of rawString.matchAll(timestampRegExp)) {
+  for (const matched of rawString.matchAll(dateRegExp)) {
     const { format, timezone } = matched.groups!;
 
     if (format) {
@@ -78,9 +79,9 @@ function validate(rawString: string) {
   }
 }
 
-export const TimestampConversion: Conversion = {
-  pattern: timestampRegExp,
-  formatter(record: LogRecord, highlight: boolean, ...matched: any[]) {
+export const DateConversion: Conversion = {
+  pattern: dateRegExp,
+  convert(record: LogRecord, highlight: boolean, ...matched: any[]) {
     const groups: Record<string, string | undefined> = last(matched);
     const { format, timezone } = groups;
 

src/legacy/core_plugins/kibana/server/lib/system_api.js → src/core/server/logging/layouts/conversions/index.ts

@@ -16,16 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+export { Conversion } from './type';
 
-const SYSTEM_API_HEADER_NAME = 'kbn-system-api';
-
-/**
- * Checks on the *server-side*, if an HTTP request is a system API request
- *
- * @param request HAPI request object
- * @return        true if request is a system API request; false, otherwise
- * @deprecated Use KibanaRequest#isSystemApi
- */
-export function isSystemApiRequest(request) {
-  return !!request.headers[SYSTEM_API_HEADER_NAME];
-}
+export { LoggerConversion } from './logger';
+export { LevelConversion } from './level';
+export { MessageConversion } from './message';
+export { MetaConversion } from './meta';
+export { PidConversion } from './pid';
+export { DateConversion } from './date';

src/core/server/logging/layouts/conversions/level.ts

@@ -32,8 +32,8 @@ const LEVEL_COLORS = new Map([
 ]);
 
 export const LevelConversion: Conversion = {
-  pattern: /{level}/gi,
-  formatter(record: LogRecord, highlight: boolean) {
+  pattern: /%level/g,
+  convert(record: LogRecord, highlight: boolean) {
     let message = record.level.id.toUpperCase().padEnd(5);
     if (highlight && LEVEL_COLORS.has(record.level)) {
       const color = LEVEL_COLORS.get(record.level)!;

src/core/server/logging/layouts/conversions/context.ts → src/core/server/logging/layouts/conversions/logger.ts

@@ -22,9 +22,9 @@ import chalk from 'chalk';
 import { Conversion } from './type';
 import { LogRecord } from '../../log_record';
 
-export const ContextConversion: Conversion = {
-  pattern: /{context}/gi,
-  formatter(record: LogRecord, highlight: boolean) {
+export const LoggerConversion: Conversion = {
+  pattern: /%logger/g,
+  convert(record: LogRecord, highlight: boolean) {
     let message = record.context;
     if (highlight) {
       message = chalk.magenta(message);

src/core/server/logging/layouts/conversions/message.ts

@@ -21,8 +21,8 @@ import { Conversion } from './type';
 import { LogRecord } from '../../log_record';
 
 export const MessageConversion: Conversion = {
-  pattern: /{message}/gi,
-  formatter(record: LogRecord) {
+  pattern: /%message/g,
+  convert(record: LogRecord) {
     // Error stack is much more useful than just the message.
     return (record.error && record.error.stack) || record.message;
   },

src/core/server/logging/layouts/conversions/meta.ts

@@ -20,8 +20,8 @@ import { Conversion } from './type';
 import { LogRecord } from '../../log_record';
 
 export const MetaConversion: Conversion = {
-  pattern: /{meta}/gi,
-  formatter(record: LogRecord) {
-    return record.meta ? `[${JSON.stringify(record.meta)}]` : '';
+  pattern: /%meta/g,
+  convert(record: LogRecord) {
+    return record.meta ? `${JSON.stringify(record.meta)}` : '';
   },
 };

src/core/server/logging/layouts/conversions/pid.ts

@@ -21,8 +21,8 @@ import { Conversion } from './type';
 import { LogRecord } from '../../log_record';
 
 export const PidConversion: Conversion = {
-  pattern: /{pid}/gi,
-  formatter(record: LogRecord) {
+  pattern: /%pid/g,
+  convert(record: LogRecord) {
     return String(record.pid);
   },
 };

src/core/server/logging/layouts/conversions/type.ts

@@ -20,6 +20,6 @@ import { LogRecord } from 'kibana/server';
 
 export interface Conversion {
   pattern: RegExp;
-  formatter: (record: LogRecord, highlight: boolean) => string;
+  convert: (record: LogRecord, highlight: boolean) => string;
   validate?: (input: string) => void;
 }