diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index 53bb7f483e3..d187fc907b9 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -28,162 +28,128 @@ on: type: choice description: 'New Package' required: true + default: "true" options: - "true" - "false" - add_historical: - type: choice - description: 'Add Historical Rules' - required: true - options: - - "yes" - - "no" - commit_hash: - description: 'Commit hash' - required: true jobs: - check-commit: - name: Check Commit Hash - runs-on: ubuntu-latest - outputs: - is_locked_commit: ${{ steps.check_commit.outputs.check_message }} - steps: - - name: Checkout detection-rules - uses: actions/checkout@v3 - with: - path: detection-rules - fetch-depth: 0 - - - name: Check commit message - id: check_commit - env: - COMMIT_HASH: "${{github.event.inputs.commit_hash}}" - run: | - cd detection-rules - COMMIT_MESSAGE=$(git show -s --format=%B $COMMIT_HASH | grep "Lock versions for releases" || true) - if [ -z "$COMMIT_MESSAGE" ]; then - echo "::set-output name=check_message::false" - else - echo "::set-output name=check_message::true" - fi - shell: bash - - fleet-pr: - name: Fleet PR - needs: check-commit - if: needs.check-commit.outputs.is_locked_commit == 'true' - runs-on: ubuntu-latest - steps: - - name: Validate the source branch - uses: actions/github-script@v3 - with: - script: | - if ('refs/heads/main' === '${{github.ref}}') { - core.setFailed('Forbidden branch') - } - - - name: Checkout detection-rules - uses: actions/checkout@v3 - with: - path: detection-rules - fetch-depth: 0 - - - name: Checkout elastic/integrations - uses: actions/checkout@v3 - with: - token: ${{ secrets.READ_WRITE_RELEASE_FLEET }} - repository: ${{github.event.inputs.target_repo}} - path: integrations - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install Python dependencies - run: | - cd detection-rules - python -m pip install --upgrade pip - pip cache purge - pip install .[dev] - - - name: Checkout commit hash - env: - COMMIT_HASH: ${{github.event.inputs.commit_hash}} - run: | - cd detection-rules - git checkout $COMMIT_HASH - - - name: Bump prebuilt rules package version - env: - PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}" - NEW_PACKAGE: "${{github.event.inputs.new_package}}" - run: | - cd detection-rules - python -m detection_rules dev bump-pkg-versions \ - --patch-release \ - --new-package $NEW_PACKAGE \ - --maturity $PACKAGE_MATURITY - - - name: Store release tag - if: ${{github.event.inputs.package_maturity}} == "ga" - id: packages-version - run: | - cd detection-rules - output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+') - echo "::set-output name=pkg_version::$output" - - - name: Create release tag - if: ${{github.event.inputs.package_maturity}} == "ga" - env: - RELEASE_TAG: "integration-v${{ steps.packages-version.outputs.pkg_version }}" - run: | - cd detection-rules - git tag $RELEASE_TAG - git push origin $RELEASE_TAG - - - name: Build release package - env: - HISTORICAL: "${{github.event.inputs.add_historical}}" - run: | - cd detection-rules - python -m detection_rules dev build-release --add-historical $HISTORICAL - - - name: Set github config - run: | - git config --global user.email "72879786+protectionsmachine@users.noreply.github.com" - git config --global user.name "protectionsmachine" - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version: '^1.20.1' - check-latest: true - - - name: Build elastic-package - run: | - go install github.com/elastic/elastic-package@latest - - - name: Create the PR to Integrations - env: - DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}" - TARGET_REPO: "${{github.event.inputs.target_repo}}" - TARGET_BRANCH: "${{github.event.inputs.target_branch}}" - LOCAL_REPO: "../integrations" - GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}" - run: | - cd detection-rules - python -m detection_rules dev integrations-pr \ - $LOCAL_REPO \ - --github-repo $TARGET_REPO \ - --base-branch $TARGET_BRANCH \ - --assign ${{github.actor}} \ - $DRAFT_ARGS - - - name: Archive production artifacts - uses: actions/upload-artifact@v3 - with: - name: release-files - path: | - detection-rules/releases + fleet-pr: + name: Build package and create PR to integrations + runs-on: ubuntu-latest + steps: + - name: Validate the source branch + uses: actions/github-script@v3 + with: + script: | + if ('refs/heads/main' === '${{github.ref}}') { + core.setFailed('Forbidden branch') + } + - name: Checkout detection-rules + uses: actions/checkout@v3 + with: + path: detection-rules + fetch-depth: 0 + + - name: Extract version lock commit hash + run: | + cd detection-rules + COMMIT_HASH=$(git log --grep='Lock versions for releases' -1 --format='%H') + echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV + echo "Extracted commit hash: $COMMIT_HASH" + + - name: Checkout commit hash + run: | + cd detection-rules + echo "Current branch is $GITHUB_REF" + echo "Checking out commit hash $COMMIT_HASH" + git checkout $COMMIT_HASH + + - name: Checkout elastic/integrations + uses: actions/checkout@v3 + with: + token: ${{ secrets.READ_WRITE_RELEASE_FLEET }} + repository: ${{github.event.inputs.target_repo}} + path: integrations + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install Python dependencies + run: | + cd detection-rules + python -m pip install --upgrade pip + pip cache purge + pip install .[dev] + + - name: Bump prebuilt rules package version + env: + PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}" + NEW_PACKAGE: "${{github.event.inputs.new_package}}" + run: | + cd detection-rules + python -m detection_rules dev bump-pkg-versions \ + --patch-release \ + --new-package $NEW_PACKAGE \ + --maturity $PACKAGE_MATURITY + + - name: Store release tag + if: github.event.inputs.package_maturity == 'ga' + run: | + cd detection-rules + output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+') + echo "pkg_version=$output" >> $GITHUB_ENV + + - name: Create release tag + if: github.event.inputs.package_maturity == 'ga' + run: | + cd detection-rules + RELEASE_TAG="integration-v${{ env.pkg_version }}" + echo "Creating release tag: $RELEASE_TAG" + git tag $RELEASE_TAG + git push origin $RELEASE_TAG + + - name: Build release package + run: | + cd detection-rules + python -m detection_rules dev build-release + + - name: Set github config + run: | + git config --global user.email "72879786+protectionsmachine@users.noreply.github.com" + git config --global user.name "protectionsmachine" + + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: '^1.20.1' + check-latest: true + + - name: Build elastic-package + run: | + go install github.com/elastic/elastic-package@latest + + - name: Create the PR to Integrations + env: + DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}" + TARGET_REPO: "${{github.event.inputs.target_repo}}" + TARGET_BRANCH: "${{github.event.inputs.target_branch}}" + LOCAL_REPO: "../integrations" + GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}" + run: | + cd detection-rules + python -m detection_rules dev integrations-pr \ + $LOCAL_REPO \ + --github-repo $TARGET_REPO \ + --base-branch $TARGET_BRANCH \ + --assign ${{github.actor}} \ + $DRAFT_ARGS + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: release-files + path: | + detection-rules/releases diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index ba3f2c63cdc..05fe05b8dde 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -86,13 +86,14 @@ def dev_group(): @click.option('--update-version-lock', '-u', is_flag=True, help='Save version.lock.json file with updated rule versions in the package') @click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files') -@click.option('--add-historical', type=str, required=True, default="no", +@click.option('--add-historical', type=str, required=True, default="yes", help='Generate historical package-registry files') @click.option('--update-message', type=str, help='Update message for new package') def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str, update_message: str, release=None, verbose=True): """Assemble all the rules into Kibana-ready release files.""" config = load_dump(config_file)['package'] + registry_data = config['registry_data'] add_historical = True if add_historical == "yes" else False if generate_navigator: @@ -111,17 +112,18 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo package.save(verbose=verbose) if add_historical: - previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", config['name']) + previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", + registry_data['conditions']['kibana.version'].strip("^")) sde = SecurityDetectionEngine() historical_rules = sde.load_integration_assets(previous_pkg_version) historical_rules = sde.transform_legacy_assets(historical_rules) - docs = IntegrationSecurityDocsMDX(config['registry_data']['version'], Path(f'releases/{config["name"]}-docs'), + docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'), True, historical_rules, package, note=update_message) docs.generate() click.echo(f'[+] Adding historical rules from {previous_pkg_version} package') - package.add_historical_rules(historical_rules, config['registry_data']['version']) + package.add_historical_rules(historical_rules, registry_data['version']) if verbose: package.get_package_hash(verbose=verbose) @@ -207,7 +209,7 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta")) if patch_release: latest_patch_release_ver = find_latest_integration_version("security_detection_engine", - maturity, pkg_data["name"]) + maturity, pkg_kibana_ver) # if an existing minor or major does not have a package, bump from the last # example is 8.10.0-beta.1 is last, but on 9.0.0 major @@ -219,13 +221,14 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, if maturity == "ga": pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_patch()) - pkg_data["registry_data"]["release"] = maturity else: # passing in true or false from GH actions; not using eval() for security purposes if new_package == "true": latest_patch_release_ver = latest_patch_release_ver.bump_patch() pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_prerelease("beta")) - pkg_data["registry_data"]["release"] = maturity + + if 'release' in pkg_data['registry_data']: + pkg_data['registry_data']['release'] = maturity click.echo(f"Kibana version: {pkg_data['name']}") click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}")