elastic main

.github/workflows/build_elastic_common_schema_toolchain.yml

@@ -0,0 +1,59 @@
+name: Build ECS Toolchain Image
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::068738303278:role/GithubECSRepoPolicy
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      # Setting up Docker Buildx with docker-container driver is required
+      # Setting up Docker Buildx with docker-container driver is required
+      # at the moment to be able to use a subdirectory with Git context
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Debug
+        run: 'echo Branch name: "${{ github.head_ref || github.ref_name }}"'
+
+      # Always push with the branch name, this allows for external testing
+      - name: Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:docker"
+          tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:${{ github.head_ref || github.ref_name }}
+          cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache
+          cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache
+          push: true
+          build-args: |
+            BRANCH=${{ github.head_ref || github.ref_name }}
+
+      # Once it's in main, we want to update to the latest stable version
+      - name: Push Latest Tag
+        if: github.ref_name == 'main'
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:docker"
+          tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest
+          cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache
+          cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache
+          push: true
+          build-args: |
+            BRANCH=${{ github.ref_name }}

.github/workflows/docs-preview-comment.yml

@@ -10,7 +10,7 @@ jobs:
   doc-preview:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
         name: Add doc preview links
         with:
           script: |

.github/workflows/stale.yml

@@ -15,10 +15,21 @@ jobs:
 
     steps:
     - name: "Check PRs"
-      uses: actions/stale@v4
+      uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: 'This PR is stale because it has been open for 60 days with no activity.'
+        stale-pr-message: |
+          Hi!
+
+          We just realized that we haven't looked into this PR in a while. We're
+          sorry!
+
+          We're labeling this PR as `Stale` to make it hit our filters and
+          make sure we get back to it as soon as possible. In the meantime, it'd
+          be extremely helpful if you could take a look at it as well and confirm its
+          relevance. A simple comment with a nice emoji will be enough `:+1`.
+
+          Thank you for your contribution!
         stale-pr-label: 'stale'
         ascending: true
         days-before-pr-stale: 60

.github/workflows/test.yml

@@ -1,15 +1,19 @@
 name: Tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     name: Unit Tests
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
         with:
           python-version: '3.x'
       - run: git fetch --prune --unshallow --tags
-      - run: make check
+      - run: make check yamllint

Makefile

@@ -86,7 +86,7 @@ misspell:
 	fi
 	./build/misspell/bin/misspell -error README.md CONTRIBUTING.md schemas/* docs/* experimental/schemas/*
 
-# Warn re misspell removal     
+# Warn re misspell removal
 .PHONY: misspell_warn
 misspell_warn:
 	@echo "Warning: due to lack of cross-platform support, misspell is no longer included in this task and may be deprecated in future\n"
@@ -110,4 +110,4 @@ build/ve/bin/activate: scripts/requirements.txt scripts/requirements-dev.txt
 # Check YAML syntax (currently not enforced).
 .PHONY: yamllint
 yamllint: ve
-	build/ve/bin/yamllint schemas/*.yml
+	build/ve/bin/yamllint -d '{extends: default, rules: {line-length: disable}}' schemas/*.yml

docker/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:latest
+ARG BRANCH=main
+RUN mkdir /ecs && apt update && apt install -y git python3-pip && apt clean
+WORKDIR /ecs
+RUN git clone https://github.com/huntresslabs/ecs . && git checkout ${BRANCH} && python3 -m pip install --break-system-packages -r scripts/requirements.txt
+COPY scripts/entry_point.sh entry_point.sh
+ENTRYPOINT ["/bin/bash", "/ecs/entry_point.sh"]

docker/scripts/entry_point.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+python3 scripts/generator.py --subset /data_stream/subset --out /data_stream --include /include /data_stream/include
+code=$?
+if [ $code -ne 0 ]; then
+    exit $code
+fi
+
+# Moving this functionality into the ECS tool
+# for yaml_file in $(find {"/include","/data_stream/include"} -name '*.yml' -type f); do
+#     file_name="${yaml_file##*/}"
+#     if [ "$(yq '.0 | has("settings")' $yaml_file)" == "true" ]; then
+#         out_file="/data_stream/generated/elasticsearch/composable/component/${file_name%.yml}.json"
+#         echo "Adding settings from ${file_name} to ${out_file##*/}"
+#         yq '.0.settings' -o json $yaml_file | jq '.tmp.template.settings = .' | jq '.tmp' > /tmp/settings.json
+#         jq -s '.[0] * .[1]' $out_file /tmp/settings.json > /tmp/combined.json
+#         mv /tmp/combined.json $out_file
+#     else
+#         echo "$file_name does NOT have settings"
+#     fi
+# done
+echo "Opening permissions"
+chmod -R 'u=rwX,g=rwX,o=rwX' "/data_stream/generated"

docs/fields/field-details.asciidoc

@@ -865,6 +865,24 @@ example: `true`
 
 // ===============================================================
 
+|
+[[field-code-signature-flags]]
+<<field-code-signature-flags, code_signature.flags>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The flags used to sign the process.
+
+type: keyword
+
+
+
+example: `570522385`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-code-signature-signing-id]]
 <<field-code-signature-signing-id, code_signature.signing_id>>
@@ -1610,7 +1628,7 @@ example: `co.uk`
 [[ecs-device]]
 === Device Fields
 
-Fields that describe a device instance and its characteristics.  Data collected for applications and processes running on a (mobile) device can be enriched with these fields to describe the identity, type and other characteristics of the device.
+Fields that describe a device instance and its characteristics. Data collected for applications and processes running on a (mobile) device can be enriched with these fields to describe the identity, type and other characteristics of the device.
 
 This field group definition is based on the Device namespace of the OpenTelemetry Semantic Conventions (https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/).
 
@@ -1629,7 +1647,7 @@ beta::[ These fields are in beta and are subject to change.]
 [[field-device-id]]
 <<field-device-id, device.id>>
 
-a| The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device. 
+a| The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device.
 
 On iOS, this value must be equal to the vendor identifier (https://developer.apple.com/documentation/uikit/uidevice/1620059-identifierforvendor). On Android, this value must be equal to the Firebase Installation ID or a globally unique UUID which is persisted across sessions in your application.
 
@@ -1693,6 +1711,24 @@ example: `Samsung Galaxy S6`
 
 // ===============================================================
 
+|
+[[field-device-serial-number]]
+<<field-device-serial-number, device.serial_number>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication.
+
+type: keyword
+
+
+
+example: `DJGAQS4CW5`
+
+| core
+
+// ===============================================================
+
 |=====
 
 
@@ -4811,6 +4847,24 @@ Note that this fieldset is used for common hashes that may be computed over a ra
 
 // ===============================================================
 
+|
+[[field-hash-cdhash]]
+<<field-hash-cdhash, hash.cdhash>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+Code directory hash, utilized to uniquely identify and authenticate the integrity of the executable code.
+
+type: keyword
+
+
+
+example: `3783b4052fd474dbe30676b45c329e7a6d44acd9`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-hash-md5]]
 <<field-hash-md5, hash.md5>>
@@ -8685,6 +8739,8 @@ The `process` fields are expected to be nested at:
 
 * `process.previous`
 
+* `process.responsible`
+
 * `process.session_leader`
 
 * `process.session_leader.parent`
@@ -8839,6 +8895,14 @@ Note: this reuse should contain an array of process field set objects.
 // ===============================================================
 
 
+| `process.responsible.*`
+| <<ecs-process,process>>| beta:[ This field is beta and subject to change.]
+
+Responsible process in macOS tracks the originating process of an app, key for understanding permissions and hierarchy.
+
+// ===============================================================
+
+
 | `process.saved_group.*`
 | <<ecs-group,group>>
 | The saved group (sgid).
@@ -9142,7 +9206,7 @@ Note: this field should contain an array of values.
 [[ecs-risk]]
 === Risk information Fields
 
-Fields for describing risk score and risk level of entities such as hosts and users.  These fields are not allowed to be nested under `event.*`. Please continue to use  `event.risk_score` and `event.risk_score_norm` for event risk.
+Fields for describing risk score and risk level of entities such as hosts and users. These fields are not allowed to be nested under `event.*`. Please continue to use `event.risk_score` and `event.risk_score_norm` for event risk.
 
 beta::[ These fields are in beta and are subject to change.]