diff --git a/.editorconfig b/.editorconfig index d076e156cfc..c104d5e006e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,6 +14,7 @@ indentation_guess = true indent_style = space indent_size = unset tab_width = 4 +max_line_length = 99 [{**.yml, **.yaml, **.jinja}] indent_style = space diff --git a/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml b/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml index 5c10b6af9aa..bbb5132b5f0 100644 --- a/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml +++ b/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml @@ -35,7 +35,6 @@ references: nist: AU-3,AU-3(1),AU-8(a),AU-8(b),AU-12(3),AU-14(1) srg: SRG-OS-000037-GPOS-00015,SRG-OS-000038-GPOS-00016,SRG-OS-000039-GPOS-00017,SRG-OS-000040-GPOS-00018,SRG-OS-000041-GPOS-00019,SRG-OS-000042-GPOS-00020,SRG-OS-000042-GPOS-00021,SRG-OS-000055-GPOS-00026,SRG-OS-000254-GPOS-00095,SRG-OS-000255-GPOS-00096,SRG-OS-000303-GPOS-00120,SRG-OS-000337-GPOS-00129,SRG-OS-000358-GPOS-00145,SRG-OS-000359-GPOS-00146 stigid: AOSX-14-001013 - stigid@ubuntu2004: UBTU-20-010182 ocil_clause: 'auditing is not enabled or running' diff --git a/ssg/build_cpe.py b/ssg/build_cpe.py index fc19bcd9f76..0a694490ffe 100644 --- a/ssg/build_cpe.py +++ b/ssg/build_cpe.py @@ -140,7 +140,6 @@ class CPEItem(XCCDFEntity): KEYS = dict( name=lambda: "", - title=lambda: "", check_id=lambda: "", bash_conditional=lambda: "", ansible_conditional=lambda: "", diff --git a/ssg/build_sce.py b/ssg/build_sce.py index 633eb579ff3..4b21da03dd6 100644 --- a/ssg/build_sce.py +++ b/ssg/build_sce.py @@ -166,8 +166,9 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output): # While we don't _write_ it, we still need to parse SCE # metadata from the templated content. Render it internally. - raw_sce_content = template_builder.get_lang_for_rule( - rule_id, rule.title, rule.template, 'sce-bash') + raw_sce_content = template_builder.get_lang_contents_for_templatable( + rule, langs['sce-bash'] + ) ext = '.sh' filename = rule_id + ext diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 2dc17ac067b..c87ccf8444c 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -12,7 +12,7 @@ import ssg.build_remediations -from .build_cpe import CPEDoesNotExist, CPEALLogicalTest, CPEALFactRef, ProductCPEs +from .build_cpe import CPEALLogicalTest, CPEALFactRef, ProductCPEs from .constants import (XCCDF12_NS, OSCAP_BENCHMARK, OSCAP_GROUP, @@ -40,55 +40,14 @@ from .yaml import DocumentationNotComplete, open_and_macro_expand from .utils import required_key, mkdir_p -from .xml import ElementTree as ET, add_xhtml_namespace, register_namespaces, parse_file -from .shims import unicode_func +from .xml import ElementTree as ET, register_namespaces, parse_file import ssg.build_stig -from .entities.common import ( - XCCDFEntity, - add_sub_element, -) +from .entities.common import add_sub_element, make_items_product_specific, \ + XCCDFEntity, Templatable from .entities.profile import Profile, ProfileWithInlinePolicies -def add_sub_element(parent, tag, ns, data): - """ - Creates a new child element under parent with tag tag, and sets - data as the content under the tag. In particular, data is a string - to be parsed as an XML tree, allowing sub-elements of children to be - added. - - If data should not be parsed as an XML tree, either escape the contents - before passing into this function, or use ElementTree.SubElement(). - - Returns the newly created subelement of type tag. - """ - namespaced_data = add_xhtml_namespace(data) - # This is used because our YAML data contain XML and XHTML elements - # ET.SubElement() escapes the < > characters by < and > - # and therefore it does not add child elements - # we need to do a hack instead - # TODO: Remove this function after we move to Markdown everywhere in SSG - ustr = unicode_func('<{0} xmlns="{3}" xmlns:xhtml="{2}">{1}').format( - tag, namespaced_data, xhtml_namespace, ns) - - try: - element = ET.fromstring(ustr.encode("utf-8")) - except Exception: - msg = ("Error adding subelement to an element '{0}' from string: '{1}'" - .format(parent.tag, ustr)) - raise RuntimeError(msg) - - # Apart from HTML and XML elements the rule descriptions and similar - # also contain elements, where we need to add the prefix - # to create a full reference. - for x in element.findall(".//{%s}sub" % XCCDF12_NS): - x.set("idref", OSCAP_VALUE + x.get("idref")) - x.set("use", "legacy") - parent.append(element) - return element - - def reorder_according_to_ordering(unordered, ordering, regex=None): ordered = [] if regex is None: @@ -187,7 +146,6 @@ class Value(XCCDFEntity): """Represents XCCDF Value """ KEYS = dict( - title=lambda: "", description=lambda: "", type=lambda: "", operator=lambda: "equals", @@ -260,7 +218,6 @@ class Benchmark(XCCDFEntity): """Represents XCCDF Benchmark """ KEYS = dict( - title=lambda: "", status=lambda: "", description=lambda: "", notice_id=lambda: "", @@ -322,7 +279,7 @@ def process_input_dict(cls, input_contents, env_yaml, product_cpes): return data def represent_as_dict(self): - data = super(Benchmark, cls).represent_as_dict() + data = super(Benchmark, self).represent_as_dict() data["rear-matter"] = data["rear_matter"] del data["rear_matter"] @@ -480,7 +437,6 @@ class Group(XCCDFEntity): KEYS = dict( prodtype=lambda: "all", - title=lambda: "", description=lambda: "", warnings=lambda: list(), requires=lambda: list(), @@ -692,12 +648,11 @@ def filterfunc(rule): return filterfunc -class Rule(XCCDFEntity): +class Rule(XCCDFEntity, Templatable): """Represents XCCDF Rule """ KEYS = dict( prodtype=lambda: "all", - title=lambda: "", description=lambda: "", rationale=lambda: "", severity=lambda: "", @@ -718,13 +673,13 @@ class Rule(XCCDFEntity): platforms=lambda: set(), sce_metadata=lambda: dict(), inherited_platforms=lambda: set(), - template=lambda: None, cpe_platform_names=lambda: set(), inherited_cpe_platform_names=lambda: set(), bash_conditional=lambda: None, fixes=lambda: dict(), - ** XCCDFEntity.KEYS + **XCCDFEntity.KEYS ) + KEYS.update(**Templatable.KEYS) MANDATORY_KEYS = { "title", @@ -737,7 +692,6 @@ class Rule(XCCDFEntity): ID_LABEL = "rule_id" PRODUCT_REFERENCES = ("stigid", "cis",) - GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",) def __init__(self, id_): super(Rule, self).__init__(id_) @@ -895,21 +849,11 @@ def load_policy_specific_content(self, rule_filename, env_yaml): env_yaml, policy_specific_content_files) self.policy_specific_content = policy_specific_content - def make_template_product_specific(self, product): - product_suffix = "@{0}".format(product) - - if not self.template: - return - - not_specific_vars = self.template.get("vars", dict()) - specific_vars = self._make_items_product_specific( - not_specific_vars, product_suffix, True) - self.template["vars"] = specific_vars - - not_specific_backends = self.template.get("backends", dict()) - specific_backends = self._make_items_product_specific( - not_specific_backends, product_suffix, True) - self.template["backends"] = specific_backends + def get_template_context(self, env_yaml): + ctx = super(Rule, self).get_template_context(env_yaml) + if self.identifiers: + ctx["cce_identifiers"] = self.identifiers + return ctx def make_refs_and_identifiers_product_specific(self, product): product_suffix = "@{0}".format(product) @@ -933,7 +877,7 @@ def make_refs_and_identifiers_product_specific(self, product): ) for name, (dic, allow_overwrites) in to_set.items(): try: - new_items = self._make_items_product_specific( + new_items = make_items_product_specific( dic, product_suffix, allow_overwrites) except ValueError as exc: msg = ( @@ -950,43 +894,6 @@ def make_refs_and_identifiers_product_specific(self, product): self._verify_stigid_format(product) - def _make_items_product_specific(self, items_dict, product_suffix, allow_overwrites=False): - new_items = dict() - for full_label, value in items_dict.items(): - if "@" not in full_label and full_label not in new_items: - new_items[full_label] = value - continue - - label = full_label.split("@")[0] - - # this test should occur before matching product_suffix with the product qualifier - # present in the reference, so it catches problems even for products that are not - # being built at the moment - if label in Rule.GLOBAL_REFERENCES: - msg = ( - "You cannot use product-qualified for the '{item_u}' reference. " - "Please remove the product-qualifier and merge values with the " - "existing reference if there is any. Original line: {item_q}: {value_q}" - .format(item_u=label, item_q=full_label, value_q=value) - ) - raise ValueError(msg) - - if not full_label.endswith(product_suffix): - continue - - if label in items_dict and not allow_overwrites and value != items_dict[label]: - msg = ( - "There is a product-qualified '{item_q}' item, " - "but also an unqualified '{item_u}' item " - "and those two differ in value - " - "'{value_q}' vs '{value_u}' respectively." - .format(item_q=full_label, item_u=label, - value_q=value, value_u=items_dict[label]) - ) - raise ValueError(msg) - new_items[label] = value - return new_items - def validate_identifiers(self, yaml_file): if self.identifiers is None: raise ValueError("Empty identifier section in file %s" % yaml_file) diff --git a/ssg/constants.py b/ssg/constants.py index ed68c053325..7f88a087487 100644 --- a/ssg/constants.py +++ b/ssg/constants.py @@ -454,6 +454,8 @@ 'eks': 'Amazon Elastic Kubernetes Service', } +# References that can not be used with product-qualifiers +GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",) # Application constants DEFAULT_GID_MIN = 1000 diff --git a/ssg/entities/common.py b/ssg/entities/common.py index ccb3786067c..164051b46d5 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -7,17 +7,64 @@ from copy import deepcopy from ..xml import ElementTree as ET, add_xhtml_namespace -from ..constants import xhtml_namespace from ..yaml import DocumentationNotComplete, open_and_macro_expand from ..shims import unicode_func from ..constants import ( + xhtml_namespace, XCCDF_REFINABLE_PROPERTIES, XCCDF12_NS, OSCAP_VALUE, + GLOBAL_REFERENCES ) +def extract_reference_from_product_specific_label(items_dict, full_label, value, allow_overwrites): + label = full_label.split("@")[0] + + if label in GLOBAL_REFERENCES: + msg = ( + "You cannot use product-qualified for the '{item_u}' reference. " + "Please remove the product-qualifier and merge values with the " + "existing reference if there is any. Original line: {item_q}: {value_q}" + .format(item_u=label, item_q=full_label, value_q=value) + ) + raise ValueError(msg) + + if not allow_overwrites and label in items_dict and value != items_dict[label]: + msg = ( + "There is a product-qualified '{item_q}' item, " + "but also an unqualified '{item_u}' item " + "and those two differ in value - " + "'{value_q}' vs '{value_u}' respectively." + .format(item_q=full_label, item_u=label, + value_q=value, value_u=items_dict[label]) + ) + raise ValueError(msg) + + return label + + +def make_items_product_specific(items_dict, product_suffix, allow_overwrites=False): + new_items = dict() + for full_label, value in items_dict.items(): + if "@" not in full_label: + new_items[full_label] = value + continue + + # This procedure should occur before matching product_suffix with the product qualifier + # present in the reference, so it catches problems even for products that are not + # being built at the moment + label = extract_reference_from_product_specific_label(items_dict, full_label, value, + allow_overwrites) + + if not full_label.endswith(product_suffix): + continue + + new_items[label] = value + return new_items + + def add_sub_element(parent, tag, ns, data): """ Creates a new child element under parent with tag tag, and sets @@ -81,6 +128,7 @@ class XCCDFEntity(object): """ KEYS = dict( id_=lambda: "", + title=lambda: "", definition_location=lambda: "", ) @@ -309,3 +357,95 @@ def update_with(self, rhs): updated_refinements = self._subtract_refinements(extended_refinements) updated_refinements.update(self.refine_rules) self.refine_rules = updated_refinements + + +class Templatable(object): + """ + The Templatable is a mix-in sidekick for XCCDFEntity-based classes + that have templates. It contains methods used by the template Builder + class. + + Methods `get_template_context` and `get_template_vars` are subject for + overloading by XCCDFEntity subclasses that want to customize template + input. + """ + + KEYS = dict( + template=lambda: None, + ) + + def __init__(self): + pass + + def is_templated(self): + return isinstance(self.template, dict) + + def get_template_name(self): + if not self.is_templated(): + return None + try: + return self.template["name"] + except KeyError: + raise ValueError( + "Templatable {0} is missing template name under template key".format(self)) + + def get_template_context(self, env_yaml): + # TODO: The first two variables, 'rule_id' and 'rule_title' are expected by some + # templates and macros even if they are not rendered in a rule context. + # Better name for these variables are 'entity_id' and 'entity_title'. + return { + "rule_id": self.id_, + "rule_title": self.title, + "products": env_yaml["product"], + } + + def get_template_vars(self, env_yaml): + if "vars" not in self.template: + raise ValueError( + "Templatable {0} does not contain mandatory 'vars:' key under " + "'template:' key.".format(self)) + template_vars = self.template["vars"] + + # Add the rule ID which will be used in template preprocessors (template.py) + # as a unique sub-element for a variety of composite IDs. + # TODO: The name _rule_id is a legacy from the era when rule was the only + # context for a template. Preprocessors implicitly depend on this name. + # A better name is '_entity_id' (as in XCCDF Entity). + template_vars["_rule_id"] = self.id_ + + return make_items_product_specific(template_vars, env_yaml["product"], + allow_overwrites=True) + + def extract_configured_backend_lang(self, avail_langs): + """ + Returns list of languages that should be generated + based on the Templatable's template option `template.backends`. + """ + if not self.is_templated(): + return [] + + if "backends" in self.template: + backends = self.template["backends"] + for lang in backends: + if lang not in avail_langs: + raise RuntimeError("Templatable {0} wants to generate unknown language '{1}" + .format(self, lang)) + return [lang for name, lang in avail_langs.items() if backends.get(name, "on") == "on"] + + return avail_langs.values() + + def make_template_product_specific(self, product): + if not self.is_templated(): + return + + product_suffix = "@{0}".format(product) + + not_specific_vars = self.template.get("vars", dict()) + specific_vars = make_items_product_specific( + not_specific_vars, product_suffix, True) + self.template["vars"] = specific_vars + + not_specific_backends = self.template.get("backends", dict()) + specific_backends = make_items_product_specific( + not_specific_backends, product_suffix, True) + self.template["backends"] = specific_backends diff --git a/ssg/entities/profile_base.py b/ssg/entities/profile_base.py index da0a5319f89..f9233414864 100644 --- a/ssg/entities/profile_base.py +++ b/ssg/entities/profile_base.py @@ -33,7 +33,6 @@ class Profile(XCCDFEntity, SelectionHandler): """Represents XCCDF profile """ KEYS = dict( - title=lambda: "", description=lambda: "", extends=lambda: "", metadata=lambda: None, diff --git a/ssg/templates.py b/ssg/templates.py index 05384a6664f..268c30cabc9 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -5,53 +5,66 @@ import imp import glob -import ssg.build_yaml +from collections import namedtuple + import ssg.utils import ssg.yaml +import ssg.jinja +import ssg.build_yaml + from ssg.build_cpe import ProductCPEs -from collections import namedtuple -templating_lang = namedtuple( +TemplatingLang = namedtuple( "templating_language_attributes", ["name", "file_extension", "template_type", "lang_specific_dir"]) -template_type = ssg.utils.enum("remediation", "check") - -languages = { - "anaconda": templating_lang("anaconda", ".anaconda", template_type.remediation, "anaconda"), - "ansible": templating_lang("ansible", ".yml", template_type.remediation, "ansible"), - "bash": templating_lang("bash", ".sh", template_type.remediation, "bash"), - "blueprint": templating_lang("blueprint", ".toml", template_type.remediation, "blueprint"), - "ignition": templating_lang("ignition", ".yml", template_type.remediation, "ignition"), - "kubernetes": templating_lang("kubernetes", ".yml", template_type.remediation, "kubernetes"), - "oval": templating_lang("oval", ".xml", template_type.check, "oval"), - "puppet": templating_lang("puppet", ".pp", template_type.remediation, "puppet"), - "sce-bash": templating_lang("sce-bash", ".sh", template_type.remediation, "sce") + +TemplateType = ssg.utils.enum("REMEDIATION", "CHECK") + +LANGUAGES = { + "anaconda": TemplatingLang("anaconda", ".anaconda", TemplateType.REMEDIATION, "anaconda"), + "ansible": TemplatingLang("ansible", ".yml", TemplateType.REMEDIATION, "ansible"), + "bash": TemplatingLang("bash", ".sh", TemplateType.REMEDIATION, "bash"), + "blueprint": TemplatingLang("blueprint", ".toml", TemplateType.REMEDIATION, "blueprint"), + "ignition": TemplatingLang("ignition", ".yml", TemplateType.REMEDIATION, "ignition"), + "kubernetes": TemplatingLang("kubernetes", ".yml", TemplateType.REMEDIATION, "kubernetes"), + "oval": TemplatingLang("oval", ".xml", TemplateType.CHECK, "oval"), + "puppet": TemplatingLang("puppet", ".pp", TemplateType.REMEDIATION, "puppet"), + "sce-bash": TemplatingLang("sce-bash", ".sh", TemplateType.REMEDIATION, "sce") } -preprocessing_file_name = "template.py" -templates = dict() +PREPROCESSING_FILE_NAME = "template.py" +TEMPLATE_YAML_FILE_NAME = "template.yml" -class Template(): - def __init__(self, template_root_directory, name): - self.template_root_directory = template_root_directory +class Template: + def __init__(self, templates_root_directory, name): + self.langs = [] + self.templates_root_directory = templates_root_directory self.name = name - self.template_path = os.path.join(self.template_root_directory, self.name) - self.template_yaml_path = os.path.join(self.template_path, "template.yml") - self.preprocessing_file_path = os.path.join(self.template_path, preprocessing_file_name) - - def load(self): + self.template_path = os.path.join(self.templates_root_directory, self.name) + self.template_yaml_path = os.path.join(self.template_path, TEMPLATE_YAML_FILE_NAME) + self.preprocessing_file_path = os.path.join(self.template_path, PREPROCESSING_FILE_NAME) + + @classmethod + def load_template(cls, templates_root_directory, name): + maybe_template = cls(templates_root_directory, name) + if maybe_template._looks_like_template(): + maybe_template._load() + return maybe_template + return None + + def _load(self): if not os.path.exists(self.preprocessing_file_path): self.preprocessing_file_path = None - self.langs = [] + template_yaml = ssg.yaml.open_raw(self.template_yaml_path) for supported_lang in template_yaml["supported_languages"]: - if supported_lang not in languages.keys(): + if supported_lang not in LANGUAGES.keys(): raise ValueError( "The template {0} declares to support the {1} language," "but this language is not supported by the content.".format( self.name, supported_lang)) - lang = languages[supported_lang] + lang = LANGUAGES[supported_lang] langfilename = lang.name + ".template" if not os.path.exists(os.path.join(self.template_path, langfilename)): raise ValueError( @@ -60,11 +73,16 @@ def load(self): self.langs.append(lang) def preprocess(self, parameters, lang): - # if no template.py file exists, skip this preprocessing part + parameters = self._preprocess_with_template_module(parameters, lang) + # TODO: Remove this right after the variables in templates are renamed to lowercase + parameters = {k.upper(): v for k, v in parameters.items()} + return parameters + + def _preprocess_with_template_module(self, parameters, lang): if self.preprocessing_file_path is not None: unique_dummy_module_name = "template_" + self.name - preprocess_mod = imp.load_source( - unique_dummy_module_name, self.preprocessing_file_path) + preprocess_mod = imp.load_source(unique_dummy_module_name, + self.preprocessing_file_path) if not hasattr(preprocess_mod, "preprocess"): msg = ( "The '{name}' template's preprocessing file {preprocessing_file} " @@ -73,17 +91,12 @@ def preprocess(self, parameters, lang): ) raise ValueError(msg) parameters = preprocess_mod.preprocess(parameters.copy(), lang) - # TODO: Remove this right after the variables in templates are renamed - # to lowercase - uppercases = dict() - for k, v in parameters.items(): - uppercases[k.upper()] = v - return uppercases - - def looks_like_template(self): - if not os.path.isdir(self.template_root_directory): + return parameters + + def _looks_like_template(self): + if not os.path.isdir(self.template_path): return False - if os.path.islink(self.template_root_directory): + if os.path.islink(self.template_path): return False template_sources = sorted(glob.glob(os.path.join(self.template_path, "*.template"))) if not os.path.isfile(self.template_yaml_path) and not template_sources: @@ -101,9 +114,8 @@ class Builder(object): output directory for remediations into the constructor. Then, call the method build() to perform a build. """ - def __init__( - self, env_yaml, resolved_rules_dir, templates_dir, - remediations_dir, checks_dir, platforms_dir, cpe_items_dir): + def __init__(self, env_yaml, resolved_rules_dir, templates_dir, + remediations_dir, checks_dir, platforms_dir, cpe_items_dir): self.env_yaml = env_yaml self.resolved_rules_dir = resolved_rules_dir self.templates_dir = templates_dir @@ -112,198 +124,111 @@ def __init__( self.platforms_dir = platforms_dir self.cpe_items_dir = cpe_items_dir self.output_dirs = dict() - for lang_name, lang in languages.items(): + self.templates = dict() + self._init_lang_output_dirs() + self._init_and_load_templates() + self.product_cpes = ProductCPEs() + self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml) + + def _init_and_load_templates(self): + for item in sorted(os.listdir(self.templates_dir)): + maybe_template = Template.load_template(self.templates_dir, item) + if maybe_template is not None: + self.templates[item] = maybe_template + + def _init_lang_output_dirs(self): + for lang_name, lang in LANGUAGES.items(): lang_dir = lang.lang_specific_dir - if lang.template_type == template_type.check: + if lang.template_type == TemplateType.CHECK: output_dir = self.checks_dir else: output_dir = self.remediations_dir dir_ = os.path.join(output_dir, lang_dir) self.output_dirs[lang_name] = dir_ - # scan directory structure and dynamically create list of templates - for item in sorted(os.listdir(self.templates_dir)): - maybe_template = Template(templates_dir, item) - if maybe_template.looks_like_template(): - maybe_template.load() - templates[item] = maybe_template - self.product_cpes = ProductCPEs() - self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml) - - def build_lang_file( - self, rule_id, template_name, template_vars, lang, local_env_yaml): - """ - Builds and returns templated content for a given rule for a given - language; does not write the output to disk. - """ - if lang not in templates[template_name].langs: - return None - - template_file_name = lang.name + ".template" - template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) - template_parameters = templates[template_name].preprocess(template_vars, lang.name) - jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters) - filled_template = ssg.jinja.process_file_with_macros( - template_file_path, jinja_dict) - - return filled_template - def build_lang( - self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None): + def get_resolved_langs_to_generate(self, templatable): """ - Builds templated content for a given rule for a given language. - Writes the output to the correct build directories. + Given a specific Templatable instance, determine which languages are + generated by the combination of the template supported_languages AND + the Templatable's template configuration 'backends'. """ - if lang not in templates[template_name].langs or lang.name == "sce-bash": - return - - filled_template = self.build_lang_file(rule_id, template_name, - template_vars, lang, - local_env_yaml) - - ext = lang.file_extension - output_file_name = rule_id + ext - output_filepath = os.path.join( - self.output_dirs[lang.name], output_file_name) - - with open(output_filepath, "w") as f: - f.write(filled_template) - - def get_langs_to_generate(self, rule): - """ - For a given rule returns list of languages that should be generated - from templates. This is controlled by "template_backends" in rule.yml. - """ - if "backends" in rule.template: - backends = rule.template["backends"] - for lang in backends: - if lang not in languages.keys(): - raise RuntimeError( - "Rule {0} wants to generate unknown language '{1}" - "from a template.".format(rule.id_, lang) - ) - langs_to_generate = [] - for lang_name, lang in languages.items(): - backend = backends.get(lang_name, "on") - if backend == "on": - langs_to_generate.append(lang) - return langs_to_generate - else: - return languages.values() - - def get_template_name(self, template, rule_id): - """ - Given a template dictionary from a Rule instance, determine the name - of the template (from templates) this rule uses. - """ - try: - template_name = template["name"] - except KeyError: + template_name = templatable.get_template_name() + if template_name not in self.templates.keys(): raise ValueError( - "Rule {0} is missing template name under template key".format( - rule_id)) - if template_name not in templates.keys(): - raise ValueError( - "Rule {0} uses template {1} which does not exist.".format( - rule_id, template_name)) - return template_name + "Templatable {0} uses template {1} which does not exist." + .format(templatable, template_name)) + template_langs = set(self.templates[template_name].langs) + rule_langs = set(templatable.extract_configured_backend_lang(LANGUAGES)) + return rule_langs.intersection(template_langs) - def get_resolved_langs_to_generate(self, rule): + def process_template_lang_file(self, template_name, template_vars, lang, local_env_yaml): """ - Given a specific Rule instance, determine which languages are - generated by the combination of the rule's template_backends AND - the rule's template keys. + Processes template for a given template name and language and returns rendered content. """ - if rule.template is None: - return None + if lang not in self.templates[template_name].langs: + raise ValueError("Language {0} is not available for template {1}." + .format(lang.name, template_name)) - rule_langs = set(self.get_langs_to_generate(rule)) - template_name = self.get_template_name(rule.template, rule.id_) - template_langs = set(templates[template_name].langs) - return rule_langs.intersection(template_langs) + template_file_name = lang.name + ".template" + template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) + template_parameters = self.templates[template_name].preprocess(template_vars, lang.name) + env_yaml = self.env_yaml.copy() + env_yaml.update(local_env_yaml) + jinja_dict = ssg.utils.merge_dicts(env_yaml, template_parameters) + return ssg.jinja.process_file_with_macros(template_file_path, jinja_dict) - def process_product_vars(self, all_variables): + def get_lang_contents_for_templatable(self, templatable, language): """ - Given a dictionary with the format key[@]=value, filter out - and only take keys that apply to this product (unqualified or qualified - to exactly this product). Returns a new dict. + For the specified Templatable, build and return only the specified language content. """ - processed = dict(filter(lambda item: '@' not in item[0], all_variables.items())) - suffix = '@' + self.env_yaml['product'] - for variable in filter(lambda key: key.endswith(suffix), all_variables): - new_variable = variable[:-len(suffix)] - value = all_variables[variable] - processed[new_variable] = value + template_name = templatable.get_template_name() + template_vars = templatable.get_template_vars(self.env_yaml) - return processed + # Checks and remediations are processed with a custom YAML dict + local_env_yaml = templatable.get_template_context(self.env_yaml) + try: + return self.process_template_lang_file(template_name, template_vars, + language, local_env_yaml) + except Exception as e: + raise RuntimeError("Unable to generate {0} template language for Templatable {1}: {2}" + .format(language.name, templatable, e)) + + def write_lang_contents_for_templatable(self, filled_template, lang, templatable): + output_file_name = templatable.id_ + lang.file_extension + output_filepath = os.path.join(self.output_dirs[lang.name], output_file_name) + with open(output_filepath, "w") as f: + f.write(filled_template) - def build_rule(self, rule_id, rule_title, template, langs_to_generate, identifiers, - platforms=None): + def build_lang_for_templatable(self, templatable, lang): """ - Builds templated content for a given rule for selected languages, + Builds templated content of a given Templatable for a selected language, writing the output to the correct build directories. """ - template_name = self.get_template_name(template, rule_id) - try: - template_vars = self.process_product_vars(template["vars"]) - except KeyError: - raise ValueError( - "Rule {0} does not contain mandatory 'vars:' key under " - "'template:' key.".format(rule_id)) - # Add the rule ID which will be reused in OVAL templates as OVAL - # definition ID so that the build system matches the generated - # check with the rule. - template_vars["_rule_id"] = rule_id - # checks and remediations are processed with a custom YAML dict - local_env_yaml = self.env_yaml.copy() - local_env_yaml["rule_id"] = rule_id - local_env_yaml["rule_title"] = rule_title - local_env_yaml["products"] = self.env_yaml["product"] - if identifiers is not None: - local_env_yaml["cce_identifiers"] = identifiers - - for lang in langs_to_generate: - try: - self.build_lang( - rule_id, template_name, template_vars, lang, local_env_yaml, platforms) - except Exception as e: - raise e( - "Error building templated {0} content for rule {1}".format(lang, rule_id)) + filled_template = self.get_lang_contents_for_templatable(templatable, lang) + self.write_lang_contents_for_templatable(filled_template, lang, templatable) - def get_lang_for_rule(self, rule_id, rule_title, template, language): + def build_rule(self, rule): """ - For the specified rule, build and return only the specified language - content. + Builds templated content of a given Rule for all available languages, + writing the output to the correct build directories. """ - template_name = self.get_template_name(template, rule_id) - try: - template_vars = self.process_product_vars(template["vars"]) - except KeyError: - raise ValueError( - "Rule {0} does not contain mandatory 'vars:' key under " - "'template:' key.".format(rule_id)) - # Add the rule ID which will be reused in OVAL templates as OVAL - # definition ID so that the build system matches the generated - # check with the rule. - template_vars["_rule_id"] = rule_id - # checks and remediations are processed with a custom YAML dict - local_env_yaml = self.env_yaml.copy() - local_env_yaml["rule_id"] = rule_id - local_env_yaml["rule_title"] = rule_title - local_env_yaml["products"] = self.env_yaml["product"] - - return self.build_lang_file(rule_id, template_name, template_vars, - language, local_env_yaml) + for lang in self.get_resolved_langs_to_generate(rule): + if lang.name != "sce-bash": + self.build_lang_for_templatable(rule, lang) def build_extra_ovals(self): declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") declaration = ssg.yaml.open_raw(declaration_path) for oval_def_id, template in declaration.items(): - langs_to_generate = [languages["oval"]] # Since OVAL definition ID in shorthand format is always the same # as rule ID, we can use it instead of the rule ID even if no rule # with that ID exists - self.build_rule( - oval_def_id, oval_def_id, template, langs_to_generate, None) + rule = ssg.build_yaml.Rule.get_instance_from_full_dict({ + "id_": oval_def_id, + "title": oval_def_id, + "template": template, + }) + self.build_lang_for_templatable(rule, LANGUAGES["oval"]) def build_all_rules(self): for rule_file in sorted(os.listdir(self.resolved_rules_dir)): @@ -313,19 +238,14 @@ def build_all_rules(self): except ssg.build_yaml.DocumentationNotComplete: # Happens on non-debug build when a rule is "documentation-incomplete" continue - if rule.template is None: - # rule is not templated, skipping - continue - langs_to_generate = self.get_langs_to_generate(rule) - self.build_rule(rule.id_, rule.title, rule.template, langs_to_generate, - rule.identifiers, platforms=rule.platforms) + if rule.is_templated(): + self.build_rule(rule) def build(self): """ - Builds all templated content for all languages, writing - the output to the correct build directories. + Builds all templated content for all languages, + writing the output to the correct build directories. """ - for dir_ in self.output_dirs.values(): if not os.path.exists(dir_): os.makedirs(dir_) diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py index 30a4113da04..c055a4f3bc6 100644 --- a/tests/ssg_test_suite/common.py +++ b/tests/ssg_test_suite/common.py @@ -505,9 +505,8 @@ def load_test(absolute_path, rule_template, local_env_yaml): template_name = rule_template['name'] template_vars = rule_template['vars'] # Load template parameters and apply it to the test case. - maybe_template = ssg.templates.Template(_SHARED_TEMPLATES, template_name) - if maybe_template.looks_like_template(): - maybe_template.load() + maybe_template = ssg.templates.Template.load_template(_SHARED_TEMPLATES, template_name) + if maybe_template is not None: template_parameters = maybe_template.preprocess(template_vars, "tests") else: raise ValueError("Rule uses template '{}' " diff --git a/tests/unit/ssg-module/data/templates/extra_ovals.yml b/tests/unit/ssg-module/data/templates/extra_ovals.yml new file mode 100644 index 00000000000..c9b7e00725b --- /dev/null +++ b/tests/unit/ssg-module/data/templates/extra_ovals.yml @@ -0,0 +1,7 @@ +package_avahi_installed: + name: package_installed + vars: + pkgname: avahi + pkgname@ubuntu2004: avahi-daemon + pkgname@rhel8: avahi8 + diff --git a/tests/unit/ssg-module/data/templates/package_installed/oval.template b/tests/unit/ssg-module/data/templates/package_installed/oval.template new file mode 100644 index 00000000000..51a5cb4a08e --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/oval.template @@ -0,0 +1,11 @@ + + + {{{ oval_metadata("The " + pkg_system|upper + " package " + PKGNAME + " should be installed.", affected_platforms=["multi_platform_all"]) }}} + + + + +{{{ oval_test_package_installed(package=PKGNAME, evr=EVR, test_id="test_package_"+PKGNAME+"_installed") }}} + diff --git a/tests/unit/ssg-module/data/templates/package_installed/template.py b/tests/unit/ssg-module/data/templates/package_installed/template.py new file mode 100644 index 00000000000..cfb47b7af5d --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/template.py @@ -0,0 +1,12 @@ +import re + + +def preprocess(data, lang): + if "evr" in data: + evr = data["evr"] + if evr and not re.match(r'\d:\d[\d\w+.]*-\d[\d\w+.]*', evr, 0): + raise RuntimeError( + "ERROR: input violation: evr key should be in " + "epoch:version-release format, but package {0} has set " + "evr to {1}".format(data["pkgname"], evr)) + return data diff --git a/tests/unit/ssg-module/data/templates/package_installed/template.yml b/tests/unit/ssg-module/data/templates/package_installed/template.yml new file mode 100644 index 00000000000..2f6f2d2c7cb --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/template.yml @@ -0,0 +1,2 @@ +supported_languages: + - oval diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py new file mode 100644 index 00000000000..723e749df1a --- /dev/null +++ b/tests/unit/ssg-module/test_templates.py @@ -0,0 +1,38 @@ +import os + +import ssg.utils +import ssg.products +import ssg.build_yaml +import ssg.build_cpe +import ssg.templates as tpl + +from ssg.environment import open_environment +from ssg.yaml import ordered_load + + +ssg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +DATADIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) +templates_dir = os.path.join(DATADIR, "templates") +cpe_items_dir = os.path.join(DATADIR, "applicability") + +build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") +product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") +env_yaml = open_environment(build_config_yaml, product_yaml) + + +def test_render_extra_ovals(): + builder = ssg.templates.Builder( + env_yaml, '', templates_dir, + '', '', '', cpe_items_dir) + + declaration_path = os.path.join(builder.templates_dir, "extra_ovals.yml") + declaration = ssg.yaml.open_raw(declaration_path) + for oval_def_id, template in declaration.items(): + rule = ssg.build_yaml.Rule.get_instance_from_full_dict({ + "id_": oval_def_id, + "title": oval_def_id, + "template": template, + }) + oval_content = builder.get_lang_contents_for_templatable(rule, + ssg.templates.LANGUAGES["oval"]) + assert "%s" % (oval_def_id,) in oval_content