fields.spec.yml

##
## Describes the specification for a data stream's various field definition files
##
spec:
  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
  definitions:
    metric_aggregation:
      type: string
      enum:
        - min
        - max
        - sum
        - value_count
        - avg
    runtime:
      description: >
        Specifies if this field is evaluated at query time. If it is set to true,
        it creates a runtime script with `params._source['<field>']`. If this field
        is a string, it creates the runtime script using this string set in the
        definition. More info at https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-mapping-fields.html
      anyOf:
        - type: string
        - type: boolean
      default: false
      examples:
        - true
        - "doc['message'].value().doSomething()"

  type: array
  items:
    type: object
    additionalProperties: false
    patternProperties:
      # Soft validation on some properties that can be present but not validated or used yet.
      "^(level|group)$":
        anyOf:
          - type: string
          - type: integer
      "^(footnote|format|title)$":
        type: string
      "^(default_field|norms)$":
        type: boolean

    properties:
      name:
        description: >
          Name of field. Names containing dots are automatically split into
          sub-fields.
          Names with wildcards generate dynamic mappings.
        type: string
        pattern: '^[\-*_\/@A-Za-z0-9]+(\.[\-*_\/@A-Za-z0-9]+)*$'

      type:
        description: >
          Datatype of field. If the type is set to object, a dynamic mapping is created. In this case, if the name doesn't
          contain any wildcard, the wildcard is added as the last segment of the path.
        type: string
        enum:
        - aggregate_metric_double
        - alias
        - histogram
        - constant_keyword
        - text
        - match_only_text
        - keyword
        - long
        - integer # Fleet always maps this as 'long'.
        - short
        - byte
        - double
        - float
        - half_float
        - scaled_float
        - date
        - date_nanos
        - boolean
        - binary
        - integer_range
        - float_range
        - long_range
        - double_range
        - date_range
        - ip_range
        - group
        - geo_point
        - object
        - ip
        - nested
        - flattened
        - wildcard
        - version
        - unsigned_long
        - counted_keyword
        - semantic_text

      description:
        description: Short description of field
        type: string

      expected_values:
        description: An array of expected values for the field. When defined, these are the only expected values.
        type: array
        items:
          type: string

      value:
        description: The value to associate with a constant_keyword field.
        type: string

      example:
        description: Example values for this field.
        anyOf:
          - type: string
          - type: number
          - type: boolean
          - type: "null"
          - type: array
            items:
              anyOf:
                - type: string
                - type: number
                - type: boolean
          - type: object
            additionalProperties: true

      metric_type:
        description: >
          The metric type of a numeric field. This is attached to the
          field as a `time_series_metric` mapping parameter. A gauge is a
          single-value measurement that can go up or down over time, such as a
          temperature. A counter is a single-value cumulative counter that only
          goes up, such as the number of requests processed by a web server. By
          default, no metric type is associated with a field.
        type: string
        enum:
        - counter
        - gauge

      unit:
        description: >
          Unit type to associate with a numeric field. This is attached to the
          field as metadata (via `meta`). By default, a field does not have a
          unit. The convention for percents is to use value 1 to mean 100%.
        type: string
        enum:
        - byte
        - percent
        - d
        - h
        - m
        - s
        - ms
        - micros
        - nanos

      date_format:
        description: >
          The date format(s) that can be parsed.
          Type date format default to `strict_date_optional_time||epoch_millis`, see the [doc](https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html#date-params).

          In JSON documents, dates are represented as strings.
          Elasticsearch uses a set of preconfigured formats to recognize
          and parse these strings into a long value representing
          _milliseconds-since-the-epoch_ in UTC.

          Besides the [built-in formats](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats), your own [custom
          formats](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#custom-date-formats) can be specified using the familiar `yyyy/MM/dd` syntax.
        type: string

      dimension:
        description: >
          Declare a field as dimension of time series. This is
          attached to the field as a `time_series_dimension` mapping parameter.
        type: boolean
        default: false

      pattern:
        description: >
          Regular expression pattern matching the allowed values for the field.
          This is used for development-time data validation.
        type: string
        examples:
          - '^[a-zA-Z]$'

      external:
        description: External source reference
        type: string
        enum:
          - ecs

      fields:
        description: Sub-fields, when type is group
        $ref: "#"     # JSON-schema syntax for pointing to the root of the schema

      doc_values:
        description: >
          Controls whether doc values are enabled for a field. All fields which
          support doc values have them enabled by default. If you are sure that
          you don’t need to sort or aggregate on a field, or access the field
          value from a script, you can disable doc values in order to save disk
          space. You cannot disable doc values for wildcard fields.
        type: boolean

      index:
        description: >
          The index option controls whether field values are indexed. Fields
          that are not indexed are typically not queryable.
        type: boolean
        default: true

      store:
        description: >
          By default, field values are indexed, but not stored. This means that
          the field can be queried, but the original field cannot be retrieved.
          Setting this value to true ensures that the field is also stored, what
          is useful in situations where the field would not be available even in
          the _source, like when synthetic source mode is enabled.
        type: boolean
        default: false

      copy_to:
        description: >
          The copy_to parameter allows you to copy the values of multiple fields
          into a group field, which can then be queried as a single field.
        type: string

      enabled:
        description: >
          The enabled setting, which can be applied only to the top-level
          mapping definition and to object fields, causes Elasticsearch to skip
          parsing of the contents of the field entirely. The JSON can still be
          retrieved from the _source field, but it is not searchable or stored
          in any other way.
        type: boolean

      dynamic:
        description: >
          The dynamic parameter controls whether new fields are added
          dynamically. It accepts the following values:

          true -  New fields are added to the mapping (default).

          runtime - New fields are added to the mapping as runtime fields. These
          fields are not indexed, and are loaded from _source at query time.

          false - New fields are ignored. These fields will not be indexed or
          searchable, but will still appear in the _source field of returned
          hits. These fields will not be added to the mapping, and new fields
          must be added explicitly.

          strict -  If new fields are detected, an exception is thrown and the
          document is rejected. New fields must be explicitly added to the
          mapping.
        default: true
        enum:
          - true
          - false
          - strict
          - runtime  # Not supported at this time by Fleet.

      scaling_factor:
        description: >
          The scaling factor to use when encoding values. Values will be
          multiplied by this factor at index time and rounded to the closest
          long value. For instance, a scaled_float with a scaling_factor of 10
          would internally store 2.34 as 23 and all search-time operations
          (queries, aggregations, sorting) will behave as if the document had a
          value of 2.3. High values of scaling_factor improve accuracy but also
          increase space requirements. Only valid for 'type: scaled_float'.
        type: integer
        default: 1000

      analyzer:
        description: >
          Name of the analyzer to use for indexing. Unless search_analyzer is
          specified this analyzer is used for both indexing and searching.
          Only valid for 'type: text'.
        type: string

      search_analyzer:
        description: >
          Name of the analyzer to use for searching. Only valid for 'type: text'.
        type: string

      multi_fields:
        description: >
          It is often useful to index the same field in different ways for
          different purposes. This is the purpose of multi-fields. For instance,
          a string field could be mapped as a text field for full-text search,
          and as a keyword field for sorting or aggregations.

          Fleet honors this for `keyword`, `text`, and `wildcard` types.
        $ref: "#" # JSON-schema syntax for pointing to the root of the schema

      null_value:
        description: >
          The null_value parameter allows you to replace explicit null values
          with the specified value so that it can be indexed and searched.

          A null value cannot be indexed or searched. When a field is set to
          null, (or an empty array or an array of null values) it is treated as
          though that field has no values.

          The null_value needs to be the same data type as the field. For
          instance, a long field cannot have a string null_value.

          The null_value only influences how data is indexed, it doesn’t modify
          the _source document.
        examples:
          - "NULL"

      ignore_malformed:
        description: >
          Trying to index the wrong data type into a field throws an exception 
          by default, and rejects the whole document. The ignore_malformed 
          parameter, if set to true, allows the exception to be ignored. The 
          malformed field is not indexed, but other fields in the document are 
          processed normally. Defaults to false.
        type: boolean
        default: false

      ignore_above:
        description: >
          Strings longer than the ignore_above setting will not be indexed or
          stored. For arrays of strings, ignore_above will be applied for each
          array element separately and string elements longer than ignore_above
          will not be indexed or stored. Fleet honors this for `keyword` and
          `wildcard` types. Defaults to 1024.
        type: integer
        default: 1024

      object_type:
        description: >
          Type of the members of the object when `type: object` is used. In
          these cases a dynamic template is created so direct subobjects of
          this field have the type indicated.
          When `object_type_mapping_type` is also used, the dynamic mapping is
          only applied to values that have the given type, as detected by the
          JSON parser.
        type: string
        enum:
          - boolean
          - byte
          - double
          - float
          - histogram
          - keyword
          - long
          - object
          - short
          - text
          - integer
          - half_float
          - scaled_float
          - unsigned_long

      object_type_mapping_type:
        description: >
          Type that members of a field of with `type: object` must have in the
          source document.
          This type corresponds to the data type detected by the JSON parser,
          and is translated to the `match_mapping_type` parameter of
          Elasticsearch dynamic templates.
        type: string
        enum:
          - '*'
          - array
          - double
          - 'false'
          - long
          - 'null'
          - object
          - string
          - 'true'

      path:
        description: >
          For alias type fields this is the path to the target field. Note that
          this must be the full path, including any parent objects
          (e.g. object1.object2.field).
        type: string

      normalize:
        description: >
          Specifies the expected normalizations for a field.
          `array` normalization implies that the values in the field should always be
          an array, even if they are single values.
        type: array
        items:
          type: string
          enum:
            - array

      normalizer:
        description: >
          Specifies the name of a normalizer to apply to keyword fields. A
          simple normalizer called lowercase ships with elasticsearch and can be
          used. Custom normalizers can be defined as part of analysis index
          settings.
        type: string

      include_in_parent:
        description: >
          For nested field types, this specifies if all fields in the nested
          object are also added to the parent document as standard (flat)
          fields.
        type: boolean
        default: false

      include_in_root:
        description: >
          For nested field types, this specifies if all fields in the nested
          object are also added to the root document as standard (flat)
          fields.
        type: boolean
        default: false

      subobjects:
        description: >
          Specifies if field names containing dots should be expanded into subobjects.
          For example, if this is set to `true`, a field named `foo.bar` will be
          expanded into an object with a field named `bar` inside an object
          named `foo`.
        type: boolean
        default: true

      inference_id:
        description: >
          For semantic_text fields, this specifies the id of the inference
          endpoint associated with the field
        type: string

      # Conditional properties.
      default_metric: true
      metrics: true
      runtime: true

    allOf:
      - if:
          properties:
            type:
              const: aggregate_metric_double
          required:
            - type
        then:
          required:
            - default_metric
          properties:
            default_metric:
              $ref: "#/definitions/metric_aggregation"
        else:
          not:
            required:
              - default_metric
      - if:
          properties:
            type:
              const: aggregate_metric_double
          required:
            - type
        then:
          required:
            - metrics
          properties:
            metrics:
              type: array
              items:
                $ref: "#/definitions/metric_aggregation"
        else:
          not:
            required:
              - metrics
      - if:
          properties:
            type:
              enum:
                - boolean
                - date
                - double
                - geo_point
                - ip
                - keyword
                - long
          required:
            - type
        then:
          properties:
            runtime:
              $ref: "#/definitions/runtime"
        else:
          not:
            required:
              - runtime
      - if:
          required:
            - metric_type
          not:
            required:
              - object_type
        then:
          oneOf:
            - properties:
                type:
                  enum:
                    - histogram
                    - aggregate_metric_double
                    - long
                    - integer
                    - short
                    - byte
                    - double
                    - float
                    - half_float
                    - scaled_float
                    - unsigned_long
              required:
                - type
            - required:
                - external
      - if:
          required:
            - object_type
        then:
          properties:
            type:
              enum:
                - object
          required:
            - type
      - if:
          properties:
            type:
              const: object
          required:
            - metric_type
            - object_type
        then:
          properties:
            type:
              enum:
                - object
            object_type:
              enum:
                - histogram
                - long
                - integer
                - short
                - byte
                - double
                - float
                - half_float
                - scaled_float
                - unsigned_long
          required:
            - type
            - object_type
      - if:
          properties:
            type:
              const: object
          required:
            - type
        then:
          oneOf:
            - required:
              - object_type
              properties:
                enabled:
                  const: true
            - properties:
                enabled:
                  const: false
              required:
                - enabled
              not:
                required:
                  - object_type
      - if:
          properties:
            fields:
              minContent: 1
          required:
            - fields
        then:
          properties:
            type:
              enum:
                - group
                - nested
      - if:
          required:
            - subobjects
        then:
          properties:
            type:
              enum:
                - object
          required:
            - type
      - if:
          required:
            - inference_id
        then:
          properties:
            type:
              enum:
                - semantic_text
          required:
            - type

    required:
    - name

# JSON patches for newer versions should be placed on top
versions:
  - before: 3.4.0
    patch:
      - op: remove
        path: "/items/properties/type/enum/35" #remove semantic_text type
      - op: remove
        path: "/items/allOf/9" # removing inference_id when type is semantic_text
      - op: remove
        path: "/items/properties/inference_id" # removing inference_id field
  - before: 3.2.0
    patch:
      - op: remove
        path: "/items/properties/store"
  - before: 3.1.0
    patch:
      - op: remove
        path: "/items/properties/type/enum/34" #remove counted_keyword type
      - op: remove
        path: "/items/allOf/8" # removing subobjects when type is object
      - op: remove
        path: "/items/properties/subobjects"
  - before: 3.0.3
    patch:
      - op: remove
        path: "/items/allOf/6/then/oneOf"
      - op: add
        path: "/items/allOf/6/then/required"
        value:
          - object_type
  - before: 3.0.0
    patch:
      - op: remove
        path: "/items/allOf/7" # removing requirement of object_type and minimum fields when type is object
      - op: remove
        path: "/items/allOf/6" # removing requirement of type group when element has subfields
  - before: 2.10.0
    patch:
      - op: remove
        path: "/items/allOf/5" # removing if related to object_type and metric_type
  - before: 2.8.0
    patch:
      - op: remove
        path: "/items/properties/runtime"
      - op: remove
        path: "/items/allOf/2" # remove if related to runtime fields allowed types
  - before: 2.7.0
    patch:
      - op: remove
        path: "/items/properties/type/enum/0"
  - before: 2.0.0
    patch:
      - op: add
        path: "/items/properties/type/enum/29"
        value: "array"
      - op: replace
        path: "/items/additionalProperties"
        value: true
      - op: remove
        path: "/items/patternProperties"
        value: true
      - op: remove
        path: "/items/properties/example"
      - op: remove
        # removing if related to ensure type is object if object_type is defined
        # it should be /items/allOf/4 , but there is a JSON Patch removing second element for runtime fields in 2.8.0
        path: "/items/allOf/3"