[Bug] Adjust build-release CLI and fix links when generating securi…

…ty docs (#3434)

* removed historical argument; added setup string; fixed links

* fixing flake errors

* added types for command arguments

* adjusted get_release_diff to append strings for release tags

* set fetch-depth to 0 for integrations checkout in workflow

* changed the name of the workflow

* removed TODOs

* adjusted release docs workflow to remove prefix for release tags

* adjusted URL replacement only if pointed to docs site

* added elastic website to regex pattern

* add docstrings; adjusted regex; add note for stopgap

* added a note about the regex pattern for elastic URLs

(cherry picked from commit 06b97ec)

.github/workflows/release-docs.yml

@@ -53,8 +53,8 @@ jobs:
         env:
           UPDATE_MESSAGE: ${{ github.event.inputs.update_message }}
           REGISTRY_VERSION: ${{ github.event.inputs.post_version }}
-          PRE_VERSION: "integration-v${{ github.event.inputs.pre_version }}"
-          POST_VERSION: "integration-v${{ github.event.inputs.post_version }}"
+          PRE_VERSION: ${{ github.event.inputs.pre_version }}
+          POST_VERSION: ${{ github.event.inputs.post_version }}
         run: |
           cd detection-rules
           python -m detection_rules dev build-integration-docs $REGISTRY_VERSION \

.github/workflows/release-fleet.yml

@@ -1,4 +1,4 @@
-name: release-fleet
+name: Release Fleet
 on:
   workflow_dispatch:
     inputs:
@@ -71,6 +71,7 @@ jobs:
             token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
             repository: ${{github.event.inputs.target_repo}}
             path: integrations
+            fetch-depth: 0
 
         - name: Set up Python 3.8
           uses: actions/setup-python@v2

detection_rules/devtools.py

@@ -86,15 +86,13 @@ def dev_group():
 @click.option('--update-version-lock', '-u', is_flag=True,
               help='Save version.lock.json file with updated rule versions in the package')
 @click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
-@click.option('--add-historical', type=str, required=True, default="yes",
-              help='Generate historical package-registry files')
+@click.option('--generate-docs', is_flag=True, default=False, help='Generate markdown documentation')
 @click.option('--update-message', type=str, help='Update message for new package')
-def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str,
+def build_release(config_file, update_version_lock: bool, generate_navigator: bool, generate_docs: str,
                   update_message: str, release=None, verbose=True):
     """Assemble all the rules into Kibana-ready release files."""
     config = load_dump(config_file)['package']
     registry_data = config['registry_data']
-    add_historical = True if add_historical == "yes" else False
 
     if generate_navigator:
         config['generate_navigator'] = True
@@ -105,26 +103,27 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
     if verbose:
         click.echo(f'[+] Building package {config.get("name")}')
 
-    package = Package.from_config(config, verbose=verbose, historical=add_historical)
+    package = Package.from_config(config, verbose=verbose)
 
     if update_version_lock:
         default_version_lock.manage_versions(package.rules, save_changes=True, verbose=verbose)
     package.save(verbose=verbose)
 
-    if add_historical:
-        previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
-                                                               registry_data['conditions']['kibana.version'].strip("^"))
-        sde = SecurityDetectionEngine()
-        historical_rules = sde.load_integration_assets(previous_pkg_version)
-        historical_rules = sde.transform_legacy_assets(historical_rules)
-
+    previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
+                                                           registry_data['conditions']['kibana.version'].strip("^"))
+    sde = SecurityDetectionEngine()
+    historical_rules = sde.load_integration_assets(previous_pkg_version)
+    historical_rules = sde.transform_legacy_assets(historical_rules)
+    package.add_historical_rules(historical_rules, registry_data['version'])
+    click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
+
+    # NOTE: stopgap solution until security doc migration
+    if generate_docs:
+        click.echo(f'[+] Generating security docs for {registry_data["version"]} package')
         docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'),
                                           True, historical_rules, package, note=update_message)
         docs.generate()
 
-        click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
-        package.add_historical_rules(historical_rules, registry_data['version'])
-
     if verbose:
         package.get_package_hash(verbose=verbose)
         click.echo(f'- {len(package.rules)} rules included')
@@ -136,14 +135,14 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin'
                      ) -> (Dict[str, TOMLRule], Dict[str, TOMLRule], Dict[str, DeprecatedRule]):
     """Build documents from two git tags for an integration package."""
     pre_rules = RuleCollection()
-    pre_rules.load_git_tag(pre, remote, skip_query_validation=True)
+    pre_rules.load_git_tag(f'integration-v{pre}', remote, skip_query_validation=True)
 
     if pre_rules.errors:
         click.echo(f'error loading {len(pre_rules.errors)} rule(s) from: {pre}, skipping:')
         click.echo(' - ' + '\n - '.join([str(p) for p in pre_rules.errors]))
 
     post_rules = RuleCollection()
-    post_rules.load_git_tag(post, remote, skip_query_validation=True)
+    post_rules.load_git_tag(f'integration-v{post}', remote, skip_query_validation=True)
 
     if post_rules.errors:
         click.echo(f'error loading {len(post_rules.errors)} rule(s) from: {post}, skipping:')
@@ -155,12 +154,12 @@ def get_release_diff(pre: str, post: str, remote: Optional[str] = 'origin'
 
 @dev_group.command('build-integration-docs')
 @click.argument('registry-version')
-@click.option('--pre', required=True, help='Tag for pre-existing rules')
-@click.option('--post', required=True, help='Tag for rules post updates')
+@click.option('--pre', required=True, type=str, help='Tag for pre-existing rules')
+@click.option('--post', required=True, type=str, help='Tag for rules post updates')
 @click.option('--directory', '-d', type=Path, required=True, help='Output directory to save docs to')
 @click.option('--force', '-f', is_flag=True, help='Bypass the confirmation prompt')
 @click.option('--remote', '-r', default='origin', help='Override the remote from "origin"')
-@click.option('--update-message', default='Rule Updates.', help='Update message for new package')
+@click.option('--update-message', default='Rule Updates.', type=str, help='Update message for new package')
 @click.pass_context
 def build_integration_docs(ctx: click.Context, registry_version: str, pre: str, post: str,
                            directory: Path, force: bool, update_message: str,
@@ -170,6 +169,10 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str,
         if not click.confirm(f'This will refresh tags and may overwrite local tags for: {pre} and {post}. Continue?'):
             ctx.exit(1)
 
+    assert Version.parse(pre) < Version.parse(post), f'pre: {pre} is not less than post: {post}'
+    assert Version.parse(pre), f'pre: {pre} is not a valid semver'
+    assert Version.parse(post), f'post: {post} is not a valid semver'
+
     rules_changes = get_release_diff(pre, post, remote)
     docs = IntegrationSecurityDocs(registry_version, directory, True, *rules_changes, update_message=update_message)
     package_dir = docs.generate()

detection_rules/docs.py

@@ -531,6 +531,11 @@ def __init__(self, rule_id: str, rule: dict, changelog: Dict[str, dict], package
         self.package = package_str
         self.rule_title = f'prebuilt-rule-{self.package}-{name_to_title(self.rule["name"])}'
 
+        # NOTE: This pattern is used to replace markdown links with asciidoc compatible links
+        # upstream in security-docs repo where CI checks fail if markdown links are used
+        self.elastic_hyperlink_pattern = \
+            r'\[.*?\]\(((?:https://(?:www\.)?elastic\.co|https://docs\.elastic\.co)/.*?)\)'
+
         # set some defaults
         self.rule.setdefault('max_signals', 100)
         self.rule.setdefault('interval', '5m')
@@ -549,6 +554,8 @@ def generate(self, title: str = None) -> str:
         ]
         if 'note' in self.rule:
             page.extend([self.guide_str(), ''])
+        if 'setup' in self.rule:
+            page.extend([self.setup_str(), ''])
         if 'query' in self.rule:
             page.extend([self.query_str(), ''])
         if 'threat' in self.rule:
@@ -557,6 +564,7 @@ def generate(self, title: str = None) -> str:
         return '\n'.join(page)
 
     def metadata_str(self) -> str:
+        """Add the metadata section to the rule detail page."""
         fields = {
             'type': 'Rule type',
             'index': 'Rule indices',
@@ -589,13 +597,21 @@ def metadata_str(self) -> str:
         return '\n'.join(values)
 
     def guide_str(self) -> str:
-        return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(self.rule["note"], code="markdown")}'
+        """Add the guide section to the rule detail page."""
+        guide = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['note'])
+        return f'{AsciiDoc.title(4, "Investigation guide")}\n\n\n{AsciiDoc.code(guide, code="markdown")}'
+
+    def setup_str(self) -> str:
+        """Add the setup section to the rule detail page."""
+        setup = re.sub(self.elastic_hyperlink_pattern, r'\1', self.rule['setup'])
+        return f'{AsciiDoc.title(4, "Setup")}\n\n\n{AsciiDoc.code(setup, code="markdown")}'
 
     def query_str(self) -> str:
-        # TODO: code=sql - would require updating existing
+        """Add the query section to the rule detail page."""
         return f'{AsciiDoc.title(4, "Rule query")}\n\n\n{AsciiDoc.code(self.rule["query"])}'
 
     def threat_mapping_str(self) -> str:
+        """Add the threat mapping section to the rule detail page."""
         values = [AsciiDoc.bold_kv('Framework', 'MITRE ATT&CK^TM^'), '']
 
         for entry in self.rule['threat']:

detection_rules/packaging.py

@@ -223,7 +223,7 @@ def get_package_hash(self, as_api=True, verbose=True):
         return sha256
 
     @classmethod
-    def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = False) -> 'Package':
+    def from_config(cls, config: dict = None, verbose: bool = False, historical: bool = True) -> 'Package':
         """Load a rules package given a config."""
         all_rules = RuleCollection.default()
         config = config or {}