default.yml

---
description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."
processors:
  - pipeline:
      if: ctx.message.startsWith('{')
      name: '{{ IngestPipeline "third-party" }}'
  - set:
      field: event.ingested
      value: '{{_ingest.timestamp}}'
  - set:
      field: ecs.version
      value: '8.11.0'
  - rename:
      field: message
      target_field: event.original
      ignore_missing: true
      if: 'ctx.event?.original == null'
  - remove:
      field: message
      ignore_missing: true
      if: 'ctx.event?.original != null'
      description: 'The `message` field is no longer required if the document has an `event.original` field.'
  - grok:
      field: event.original
      patterns:
      - '(%{IPORHOST:destination.domain}:?%{POSINT:destination.port}? )?(%{IPORHOST:source.address}:?%{POSINT:source.port}? )%{DATA:apache.access.identity} %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(%{DATA:apache.access.tls_handshake.error})?((%{WORD:http.request.method}?) %{DATA:_tmp.url_orig}? HTTP/%{NUMBER:http.version})?(%{WORD:http.request.method}? ?%{DATA:apache.access.http.request_headers}?(-)? %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version})?(-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( %{NUMBER:apache.access.response_time})?( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?( X-Forwarded-For="%{ADDRESS_LIST:apache.access.remote_addresses}")?'
      - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
        "-" %{NUMBER:http.response.status_code:long} -'
      - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
        %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig}
        HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
      ignore_missing: true
      pattern_definitions:
        ADDRESS_LIST: (%{IP})("?,?\s*(%{IP}))*
  - split:
      field: apache.access.remote_addresses
      separator: '"?,\s*'
      ignore_missing: true
  - set:
      field: network.forwarded_ip
      value: "{{{apache.access.remote_addresses.0}}}"
      if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0
  - script:
      if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0
      lang: painless
      tag: Get source address
      description: Extract from remote_addresses, the first non-private IP to ctx.client.ip
      source: >-
        boolean isPrivateCIDR(def ip) {
          CIDR class_a_network = new CIDR('10.0.0.0/8');
          CIDR class_b_network = new CIDR('172.16.0.0/12');
          CIDR class_c_network = new CIDR('192.168.0.0/16');

          try {
            return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);
          } catch (IllegalArgumentException e) {
            return false;
          }
        }

        try {
          if (ctx.client == null) {
            Map map = new HashMap();
            ctx.put("client", map);
          }

          def found = false;
          for (def item : ctx.apache.access.remote_addresses) {
            if (!isPrivateCIDR(item)) {
              ctx.client.ip = item;
              found = true;
              break;
            }
          }
          if (!found) {
            ctx.client.ip = ctx.apache.access.remote_addresses[0];
          }
        }
        catch (Exception e) {
          ctx.client.ip = null;
        }
  - append:
      field: apache.access.remote_addresses
      value: ["{{source.address}}"]
      if: ctx.source?.address != null
  - uri_parts:
      field: _tmp.url_orig
      ignore_failure: true
  - remove:
      field:
        - _tmp
      ignore_missing: true
  - set:
      field: url.domain
      value: "{{destination.domain}}"
      if: ctx.url?.domain == null && ctx.destination?.domain != null
  - set:
      field: event.kind
      value: event
  - append:
      field: event.category
      value: web
  - set:
      field: event.outcome
      value: success
      if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400"
  - set:
      field: event.outcome
      value: failure
      if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399"
  - grok:
      field: source.address
      ignore_missing: true
      patterns:
      - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
  - remove:
      field: event.created
      ignore_missing: true
      ignore_failure: true
  - rename:
      field: '@timestamp'
      target_field: event.created
  - date:
      field: apache.access.time
      target_field: '@timestamp'
      formats:
      - dd/MMM/yyyy:H:m:s Z
      ignore_failure: true
  - remove:
      field: apache.access.time
      ignore_failure: true
  - user_agent:
      field: user_agent.original
      ignore_failure: true
  - geoip:
      field: source.ip
      target_field: source.geo
      ignore_missing: true
  - geoip:
      database_file: GeoLite2-ASN.mmdb
      field: source.ip
      target_field: source.as
      properties:
      - asn
      - organization_name
      ignore_missing: true
  - rename:
      field: source.as.asn
      target_field: source.as.number
      ignore_missing: true
  - rename:
      field: source.as.organization_name
      target_field: source.as.organization.name
      ignore_missing: true
  - set:
      field: tls.cipher
      value: '{{apache.access.ssl.cipher}}'
      if: ctx.apache?.access?.ssl?.cipher != null
  - script:
      lang: painless
      if: ctx.apache?.access?.ssl?.protocol != null
      source: >-
        def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
        if (parts.length != 2) {
          return;
        }
        if (parts[1].contains(".")) {
          ctx.tls.version = parts[1];
        } else {
          ctx.tls.version = parts[1] + ".0";
        }
        ctx.tls.version_protocol = parts[0];
  - convert:
      field: source.address
      type: ip
      ignore_missing: true
      on_failure:
          - set:
              field: tmp_host
              copy_from: source.address
          - append:
              field: related.hosts
              value: '{{{tmp_host}}}'
              allow_duplicates: false
              if: "ctx.tmp_host != null"
          - set:
              field: tmp_host
              value: ""
              if: "ctx.tmp_host != null"
  - append:
      field: related.ip
      value: '{{{source.address}}}'
      allow_duplicates: false
      if: "ctx.tmp_host == null"
  - convert:
      field: destination.domain
      type: ip
      ignore_missing: true
      on_failure:
        - set:
            field: tmp_host
            copy_from: destination.domain
        - append:
            field: related.hosts
            value: '{{{tmp_host}}}'
            allow_duplicates: false
            if: "ctx.tmp_host != null"
        - set:
            field: tmp_host
            value: ""
            if: "ctx.tmp_host != null"
  - convert:
      field: destination.port
      type: long
      if: "ctx.destination?.port != null"
  - convert:
      field: source.port
      type: long
      if: "ctx.source?.port != null"
  - append:
      field: related.ip
      value: '{{{destination.domain}}}'
      allow_duplicates: false
      if: "ctx.tmp_host == null"
  - convert:
      field: apache.access.response_time
      tag: 'convert_response_time_to_long'
      type: long
      ignore_missing: true
      on_failure:
        - append:
            field: error.message
            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
  - script:
      lang: painless
      source: |-
        boolean drop(Object o) {
          if (o == null || o == '' || o == '-') {
            return true;
          } else if (o instanceof Map) {
            ((Map) o).values().removeIf(v -> drop(v));
            return (((Map) o).size() == 0);
          } else if (o instanceof List) {
            ((List) o).removeIf(v -> drop(v));
            return (((List) o).size() == 0);
          }
          return false;
        }
        drop(ctx);
      description: Drops null and empty values recursively from the Elasticsearch document context.
on_failure:
  - set:
      field: error.message
      value: '{{ _ingest.on_failure_message }}'