Skip to content

Commit

Permalink
[Bug] Update Prebuilt Detection Rules Release Process (#3403)
Browse files Browse the repository at this point in the history 
* release fleet workflow updates; build package integration reference changes

* updated commit hash extraction to output to env

* adjusted bump-pkg-versions to only include release if necessary

* fixed flake errors

* add historical argument for build-release set to yes by default

* Update detection_rules/devtools.py

* fixed fleet workflow; updated registry data references

* updated job names

* removed extract commit hash job and consolidated into fleet pr job

* added echo statement for current branch before checkout

* removed id from extract commit hash

(cherry picked from commit 7df7ab5)
  • Loading branch information
@terrancedejesus @github-actions
terrancedejesus authored and github-actions[bot] committed Feb 6, 2024
1 parent 8f024f1 commit 47e31e3
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 161 deletions.
274 changes: 120 additions & 154 deletions .github/workflows/release-fleet.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,162 +28,128 @@ on:
type: choice
description: 'New Package'
required: true
default: "true"
options:
- "true"
- "false"
add_historical:
type: choice
description: 'Add Historical Rules'
required: true
options:
- "yes"
- "no"
commit_hash:
description: 'Commit hash'
required: true

jobs:
check-commit:
name: Check Commit Hash
runs-on: ubuntu-latest
outputs:
is_locked_commit: ${{ steps.check_commit.outputs.check_message }}
steps:
- name: Checkout detection-rules
uses: actions/checkout@v3
with:
path: detection-rules
fetch-depth: 0

- name: Check commit message
id: check_commit
env:
COMMIT_HASH: "${{github.event.inputs.commit_hash}}"
run: |
cd detection-rules
COMMIT_MESSAGE=$(git show -s --format=%B $COMMIT_HASH | grep "Lock versions for releases" || true)
if [ -z "$COMMIT_MESSAGE" ]; then
echo "::set-output name=check_message::false"
else
echo "::set-output name=check_message::true"
fi
shell: bash

fleet-pr:
name: Fleet PR
needs: check-commit
if: needs.check-commit.outputs.is_locked_commit == 'true'
runs-on: ubuntu-latest
steps:
- name: Validate the source branch
uses: actions/github-script@v3
with:
script: |
if ('refs/heads/main' === '${{github.ref}}') {
core.setFailed('Forbidden branch')
}
- name: Checkout detection-rules
uses: actions/checkout@v3
with:
path: detection-rules
fetch-depth: 0

- name: Checkout elastic/integrations
uses: actions/checkout@v3
with:
token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
repository: ${{github.event.inputs.target_repo}}
path: integrations

- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8

- name: Install Python dependencies
run: |
cd detection-rules
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
- name: Checkout commit hash
env:
COMMIT_HASH: ${{github.event.inputs.commit_hash}}
run: |
cd detection-rules
git checkout $COMMIT_HASH
- name: Bump prebuilt rules package version
env:
PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
NEW_PACKAGE: "${{github.event.inputs.new_package}}"
run: |
cd detection-rules
python -m detection_rules dev bump-pkg-versions \
--patch-release \
--new-package $NEW_PACKAGE \
--maturity $PACKAGE_MATURITY
- name: Store release tag
if: ${{github.event.inputs.package_maturity}} == "ga"
id: packages-version
run: |
cd detection-rules
output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
echo "::set-output name=pkg_version::$output"
- name: Create release tag
if: ${{github.event.inputs.package_maturity}} == "ga"
env:
RELEASE_TAG: "integration-v${{ steps.packages-version.outputs.pkg_version }}"
run: |
cd detection-rules
git tag $RELEASE_TAG
git push origin $RELEASE_TAG
- name: Build release package
env:
HISTORICAL: "${{github.event.inputs.add_historical}}"
run: |
cd detection-rules
python -m detection_rules dev build-release --add-historical $HISTORICAL
- name: Set github config
run: |
git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
git config --global user.name "protectionsmachine"
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: '^1.20.1'
check-latest: true

- name: Build elastic-package
run: |
go install github.com/elastic/elastic-package@latest
- name: Create the PR to Integrations
env:
DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
TARGET_REPO: "${{github.event.inputs.target_repo}}"
TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
LOCAL_REPO: "../integrations"
GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
run: |
cd detection-rules
python -m detection_rules dev integrations-pr \
$LOCAL_REPO \
--github-repo $TARGET_REPO \
--base-branch $TARGET_BRANCH \
--assign ${{github.actor}} \
$DRAFT_ARGS
- name: Archive production artifacts
uses: actions/upload-artifact@v3
with:
name: release-files
path: |
detection-rules/releases
fleet-pr:
name: Build package and create PR to integrations
runs-on: ubuntu-latest
steps:
- name: Validate the source branch
uses: actions/github-script@v3
with:
script: |
if ('refs/heads/main' === '${{github.ref}}') {
core.setFailed('Forbidden branch')
}
- name: Checkout detection-rules
uses: actions/checkout@v3
with:
path: detection-rules
fetch-depth: 0

- name: Extract version lock commit hash
run: |
cd detection-rules
COMMIT_HASH=$(git log --grep='Lock versions for releases' -1 --format='%H')
echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
echo "Extracted commit hash: $COMMIT_HASH"
- name: Checkout commit hash
run: |
cd detection-rules
echo "Current branch is $GITHUB_REF"
echo "Checking out commit hash $COMMIT_HASH"
git checkout $COMMIT_HASH
- name: Checkout elastic/integrations
uses: actions/checkout@v3
with:
token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
repository: ${{github.event.inputs.target_repo}}
path: integrations

- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8

- name: Install Python dependencies
run: |
cd detection-rules
python -m pip install --upgrade pip
pip cache purge
pip install .[dev]
- name: Bump prebuilt rules package version
env:
PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
NEW_PACKAGE: "${{github.event.inputs.new_package}}"
run: |
cd detection-rules
python -m detection_rules dev bump-pkg-versions \
--patch-release \
--new-package $NEW_PACKAGE \
--maturity $PACKAGE_MATURITY
- name: Store release tag
if: github.event.inputs.package_maturity == 'ga'
run: |
cd detection-rules
output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
echo "pkg_version=$output" >> $GITHUB_ENV
- name: Create release tag
if: github.event.inputs.package_maturity == 'ga'
run: |
cd detection-rules
RELEASE_TAG="integration-v${{ env.pkg_version }}"
echo "Creating release tag: $RELEASE_TAG"
git tag $RELEASE_TAG
git push origin $RELEASE_TAG
- name: Build release package
run: |
cd detection-rules
python -m detection_rules dev build-release
- name: Set github config
run: |
git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
git config --global user.name "protectionsmachine"
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: '^1.20.1'
check-latest: true

- name: Build elastic-package
run: |
go install github.com/elastic/elastic-package@latest
- name: Create the PR to Integrations
env:
DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
TARGET_REPO: "${{github.event.inputs.target_repo}}"
TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
LOCAL_REPO: "../integrations"
GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
run: |
cd detection-rules
python -m detection_rules dev integrations-pr \
$LOCAL_REPO \
--github-repo $TARGET_REPO \
--base-branch $TARGET_BRANCH \
--assign ${{github.actor}} \
$DRAFT_ARGS
- name: Archive production artifacts
uses: actions/upload-artifact@v3
with:
name: release-files
path: |
detection-rules/releases
17 changes: 10 additions & 7 deletions detection_rules/devtools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,14 @@ def dev_group():
@click.option('--update-version-lock', '-u', is_flag=True,
help='Save version.lock.json file with updated rule versions in the package')
@click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
@click.option('--add-historical', type=str, required=True, default="no",
@click.option('--add-historical', type=str, required=True, default="yes",
help='Generate historical package-registry files')
@click.option('--update-message', type=str, help='Update message for new package')
def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str,
update_message: str, release=None, verbose=True):
"""Assemble all the rules into Kibana-ready release files."""
config = load_dump(config_file)['package']
registry_data = config['registry_data']
add_historical = True if add_historical == "yes" else False

if generate_navigator:
Expand All @@ -111,17 +112,18 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
package.save(verbose=verbose)

if add_historical:
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", config['name'])
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
registry_data['conditions']['kibana.version'].strip("^"))
sde = SecurityDetectionEngine()
historical_rules = sde.load_integration_assets(previous_pkg_version)
historical_rules = sde.transform_legacy_assets(historical_rules)

docs = IntegrationSecurityDocsMDX(config['registry_data']['version'], Path(f'releases/{config["name"]}-docs'),
docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'),
True, historical_rules, package, note=update_message)
docs.generate()

click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
package.add_historical_rules(historical_rules, config['registry_data']['version'])
package.add_historical_rules(historical_rules, registry_data['version'])

if verbose:
package.get_package_hash(verbose=verbose)
Expand Down Expand Up @@ -207,7 +209,7 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
if patch_release:
latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
maturity, pkg_data["name"])
maturity, pkg_kibana_ver)

# if an existing minor or major does not have a package, bump from the last
# example is 8.10.0-beta.1 is last, but on 9.0.0 major
Expand All @@ -219,13 +221,14 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,

if maturity == "ga":
pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_patch())
pkg_data["registry_data"]["release"] = maturity
else:
# passing in true or false from GH actions; not using eval() for security purposes
if new_package == "true":
latest_patch_release_ver = latest_patch_release_ver.bump_patch()
pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_prerelease("beta"))
pkg_data["registry_data"]["release"] = maturity

if 'release' in pkg_data['registry_data']:
pkg_data['registry_data']['release'] = maturity

click.echo(f"Kibana version: {pkg_data['name']}")
click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}")
Expand Down

0 comments on commit 47e31e3

Please sign in to comment.