diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml index 444d275c250..9af8574bbfb 100644 --- a/.github/workflows/release-docs.yml +++ b/.github/workflows/release-docs.yml @@ -53,8 +53,8 @@ jobs: env: UPDATE_MESSAGE: ${{ github.event.inputs.update_message }} REGISTRY_VERSION: ${{ github.event.inputs.post_version }} - PRE_VERSION: "integration-v${{ github.event.inputs.pre_version }}" - POST_VERSION: "integration-v${{ github.event.inputs.post_version }}" + PRE_VERSION: ${{ github.event.inputs.pre_version }} + POST_VERSION: ${{ github.event.inputs.post_version }} run: | cd detection-rules python -m detection_rules dev build-integration-docs $REGISTRY_VERSION \ diff --git a/.github/workflows/release-fleet.yml b/.github/workflows/release-fleet.yml index d187fc907b9..bcf26ab1d59 100644 --- a/.github/workflows/release-fleet.yml +++ b/.github/workflows/release-fleet.yml @@ -1,4 +1,4 @@ -name: release-fleet +name: Release Fleet on: workflow_dispatch: inputs: @@ -71,6 +71,7 @@ jobs: token: ${{ secrets.READ_WRITE_RELEASE_FLEET }} repository: ${{github.event.inputs.target_repo}} path: integrations + fetch-depth: 0 - name: Set up Python 3.8 uses: actions/setup-python@v2 diff --git a/detection_rules/devtools.py b/detection_rules/devtools.py index 05fe05b8dde..1c9a5c5bb13 100644 --- a/detection_rules/devtools.py +++ b/detection_rules/devtools.py @@ -86,15 +86,13 @@ def dev_group(): @click.option('--update-version-lock', '-u', is_flag=True, help='Save version.lock.json file with updated rule versions in the package') @click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files') -@click.option('--add-historical', type=str, required=True, default="yes", - help='Generate historical package-registry files') +@click.option('--generate-docs', is_flag=True, default=False, help='Generate markdown documentation') @click.option('--update-message', type=str, help='Update message for new package') -def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str, +def build_release(config_file, update_version_lock: bool, generate_navigator: bool, generate_docs: str, update_message: str, release=None, verbose=True): """Assemble all the rules into Kibana-ready release files.""" config = load_dump(config_file)['package'] registry_data = config['registry_data'] - add_historical = True if add_historical == "yes" else False if generate_navigator: config['generate_navigator'] = True @@ -105,26 +103,27 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo if verbose: click.echo(f'[+] Building package {config.get("name")}') - package = Package.from_config(config, verbose=verbose, historical=add_historical) + package = Package.from_config(config, verbose=verbose) if update_version_lock: default_version_lock.manage_versions(package.rules, save_changes=True, verbose=verbose) package.save(verbose=verbose) - if add_historical: - previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", - registry_data['conditions']['kibana.version'].strip("^")) - sde = SecurityDetectionEngine() - historical_rules = sde.load_integration_assets(previous_pkg_version) - historical_rules = sde.transform_legacy_assets(historical_rules) - + previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", + registry_data['conditions']['kibana.version'].strip("^")) + sde = SecurityDetectionEngine() + historical_rules = sde.load_integration_assets(previous_pkg_version) + historical_rules = sde.transform_legacy_assets(historical_rules) + package.add_historical_rules(historical_rules, registry_data['version']) + click.echo(f'[+] Adding historical rules from {previous_pkg_version} package') + + # NOTE: stopgap solution until security doc migration + if generate_docs: + click.echo(f'[+] Generating security docs for {registry_data["version"]} package') docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'), True, historical_rules, package, note=update_message) docs.generate() - click.echo(f'[+] Adding historical rules from {previous_pkg_version} package') - package.add_historical_rules(historical_rules, registry_data['version']) - if verbose: package.get_package_hash(verbose=verbose) click.echo(f'- {len(package.rules)} rules included') @@ -136,14 +135,14 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin' ) -> (Dict[str, TOMLRule], Dict[str, TOMLRule], Dict[str, DeprecatedRule]): """Build documents from two git tags for an integration package.""" pre_rules = RuleCollection() - pre_rules.load_git_tag(pre, remote, skip_query_validation=True) + pre_rules.load_git_tag(f'integration-v{pre}', remote, skip_query_validation=True) if pre_rules.errors: click.echo(f'error loading {len(pre_rules.errors)} rule(s) from: {pre}, skipping:') click.echo(' - ' + '\n - '.join([str(p) for p in pre_rules.errors])) post_rules = RuleCollection() - post_rules.load_git_tag(post, remote, skip_query_validation=True) + post_rules.load_git_tag(f'integration-v{post}', remote, skip_query_validation=True) if post_rules.errors: click.echo(f'error loading {len(post_rules.errors)} rule(s) from: {post}, skipping:') @@ -155,12 +154,12 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin' @dev_group.command('build-integration-docs') @click.argument('registry-version') -@click.option('--pre', required=True, help='Tag for pre-existing rules') -@click.option('--post', required=True, help='Tag for rules post updates') +@click.option('--pre', required=True, type=str, help='Tag for pre-existing rules') +@click.option('--post', required=True, type=str, help='Tag for rules post updates') @click.option('--directory', '-d', type=Path, required=True, help='Output directory to save docs to') @click.option('--force', '-f', is_flag=True, help='Bypass the confirmation prompt') @click.option('--remote', '-r', default='origin', help='Override the remote from "origin"') -@click.option('--update-message', default='Rule Updates.', help='Update message for new package') +@click.option('--update-message', default='Rule Updates.', type=str, help='Update message for new package') @click.pass_context def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, post: str, directory: Path, force: bool, update_message: str, @@ -170,6 +169,10 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, if not click.confirm(f'This will refresh tags and may overwrite local tags for: {pre} and {post}. Continue?'): ctx.exit(1) + assert Version.parse(pre) < Version.parse(post), f'pre: {pre} is not less than post: {post}' + assert Version.parse(pre), f'pre: {pre} is not a valid semver' + assert Version.parse(post), f'post: {post} is not a valid semver' + rules_changes = get_release_diff(pre, post, remote) docs = IntegrationSecurityDocs(registry_version, directory, True, *rules_changes, update_message=update_message) package_dir = docs.generate() diff --git a/detection_rules/docs.py b/detection_rules/docs.py index 7de6fbcc2c8..47575a2e5ef 100644 --- a/detection_rules/docs.py +++ b/detection_rules/docs.py @@ -531,6 +531,11 @@ def __init__(self, rule_id: str, rule: dict, changelog: Dict[str, dict], package self.package = package_str self.rule_title = f'prebuilt-rule-{self.package}-{name_to_title(self.rule["name"])}' + # NOTE: This pattern is used to replace markdown links with asciidoc compatible links + # upstream in security-docs repo where CI checks fail if markdown links are used + self.elastic_hyperlink_pattern = \ + r'\[.*?\]\(((?:https://(?:www\.)?elastic\.co|https://docs\.elastic\.co)/.*?)\)' + # set some defaults self.rule.setdefault('max_signals', 100) self.rule.setdefault('interval', '5m') @@ -549,6 +554,8 @@ def generate(self, title: str = None) -> str: ] if 'note' in self.rule: page.extend([self.guide_str(), '']) + if 'setup' in self.rule: + page.extend([self.setup_str(), '']) if 'query' in self.rule: page.extend([self.query_str(), '']) if 'threat' in self.rule: @@ -557,6 +564,7 @@ def generate(self, title: str = None) -> str: return '\n'.join(page) def metadata_str(self) -> str: + """Add the metadata section to the rule detail page.""" fields = { 'type': 'Rule type', 'index': 'Rule indices', @@ -589,13 +597,21 @@ def metadata_str(self) -> str: return '\n'.join(values) def guide_str(self) -> str: - return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(self.rule["note"], code="markdown")}' + """Add the guide section to the rule detail page.""" + guide = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['note']) + return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(guide, code="markdown")}' + + def setup_str(self) -> str: + """Add the setup section to the rule detail page.""" + setup = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['setup']) + return f'{AsciiDoc.title(4, "Setup")}\n\n\n{AsciiDoc.code(setup, code="markdown")}' def query_str(self) -> str: - # TODO: code=sql - would require updating existing + """Add the query section to the rule detail page.""" return f'{AsciiDoc.title(4, "Rule query")}\n\n\n{AsciiDoc.code(self.rule["query"])}' def threat_mapping_str(self) -> str: + """Add the threat mapping section to the rule detail page.""" values = [AsciiDoc.bold_kv('Framework', 'MITRE ATT&CK^TM^'), ''] for entry in self.rule['threat']: diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index d09bf16fe77..63114ec38f0 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -223,7 +223,7 @@ def get_package_hash(self, as_api=True, verbose=True): return sha256 @classmethod - def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = False) -> 'Package': + def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = True) -> 'Package': """Load a rules package given a config.""" all_rules = RuleCollection.default() config = config or {}