Refactor input_type code into classes

Makefile

@@ -24,5 +24,5 @@ clean:
 codestyle:
 	which flake8 || echo "Install flake8 with pip3 install --user flake8"
 	# ignores line length and reclass related errors
-	flake8 --ignore E501 . | grep -v "reclass"
+	flake8 --ignore E501 . --exclude=reclass
 	@echo

kapitan/cli.py

@@ -203,7 +203,10 @@ def main():
         if args.vars:
             ext_vars = dict(var.split('=') for var in args.vars)
         json_output = None
-        _search_imports = lambda cwd, imp: search_imports(cwd, imp, search_paths)
+
+        def _search_imports(cwd, imp):
+            return search_imports(cwd, imp, search_paths)
+
         json_output = jsonnet_file(file_path, import_callback=_search_imports,
                                    native_callbacks=resource_callbacks(search_paths),
                                    ext_vars=ext_vars)

kapitan/inputs/base.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Kapitan Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import logging
+import os
+import yaml
+import ujson as json
+
+from kapitan.errors import CompileError, KapitanError
+from kapitan.refs.base import Revealer
+from kapitan.utils import PrettyDumper
+
+logger = logging.getLogger(__name__)
+
+
+class InputType(object):
+    def __init__(self, type_name, compile_path, search_paths, ref_controller):
+        self.type_name = type_name
+        self.compile_path = compile_path
+        self.search_paths = search_paths
+        self.ref_controller = ref_controller
+
+    def compile_obj(self, comp_obj, ext_vars, **kwargs):
+        """
+        run compile_input_path() for each input_path in comp_obj
+        kwargss are passed into compile_input_path()
+        """
+        input_type = comp_obj["input_type"]
+        assert input_type == self.type_name
+        input_paths = comp_obj["input_paths"]
+
+        for input_path in input_paths:
+            self.compile_input_path(input_path, comp_obj, ext_vars, **kwargs)
+
+    def compile_input_path(self, input_path, comp_obj, ext_vars, **kwargs):
+        """
+        compile and validate input_path in comp_obj
+        kwargs are passed into compile_file()
+        """
+        target_name = ext_vars["target"]
+        output_path = comp_obj["output_path"]
+        output_type = comp_obj.get("output_type", self.default_output_type())
+        file_found = False
+
+        for path in self.search_paths:
+            compile_file_sp = os.path.join(path, input_path)
+            if os.path.exists(compile_file_sp):
+                file_found = True
+                logger.debug("Compiling %s", compile_file_sp)
+                try:
+                    _compile_path = os.path.join(self.compile_path, target_name, output_path)
+                    self.compile_file(compile_file_sp, _compile_path, ext_vars, output=output_type,
+                                      target_name=target_name, **kwargs)
+                except KapitanError as e:
+                    raise CompileError("{}\nCompile error: failed to compile target: {}".format(e, target_name))
+
+        if not file_found:
+            raise CompileError("Compile error: {} for target: {} not found in "
+                               "search_paths: {}".format(input_path, target_name, self.search_paths))
+
+    def make_compile_dirs(self, target_name, output_path):
+        """make compile dirs, skips if dirs exist"""
+        _compile_path = os.path.join(self.compile_path, target_name, output_path)
+        # support writing to an already existent dir
+        os.makedirs(_compile_path, exist_ok=True)
+
+    def compile_file(self, file_path, compile_path, ext_vars, **kwargs):
+        """implements compilation for file_path to compile_path with ext_vars"""
+        return NotImplementedError
+
+    def default_output_type(self):
+        "returns default output_type value"
+        return NotImplementedError
+
+
+class CompilingFile(object):
+    def __init__(self, context, fp, ref_controller, **kwargs):
+        self.fp = fp
+        self.ref_controller = ref_controller
+        self.kwargs = kwargs
+        self.revealer = Revealer(ref_controller)
+
+    def write(self, data):
+        """write data into file"""
+        reveal = self.kwargs.get('reveal', False)
+        target_name = self.kwargs.get('target_name', None)
+        if reveal:
+            self.fp.write(self.revealer.reveal_raw(data))
+        else:
+            self.fp.write(self.revealer.compile_raw(data, target_name=target_name))
+
+    def write_yaml(self, obj):
+        """recursively compile or reveal refs and convert obj to yaml and write to file"""
+        indent = self.kwargs.get('indent', 2)
+        reveal = self.kwargs.get('reveal', False)
+        target_name = self.kwargs.get('target_name', None)
+        if reveal:
+            self.revealer.reveal_obj(obj)
+        else:
+            self.revealer.compile_obj(obj, target_name=target_name)
+        yaml.dump(obj, stream=self.fp, indent=indent, Dumper=PrettyDumper, default_flow_style=False)
+        logger.debug("Wrote %s", self.fp.name)
+
+    def write_json(self, obj):
+        """recursively hash or reveal refs and convert obj to json and write to file"""
+        indent = self.kwargs.get('indent', 2)
+        reveal = self.kwargs.get('reveal', False)
+        target_name = self.kwargs.get('target_name', None)
+        if reveal:
+            self.revealer.reveal_obj(obj)
+        else:
+            self.revealer.compile_obj(obj, target_name=target_name)
+        json.dump(obj, self.fp, indent=indent, escape_forward_slashes=False)
+        logger.debug("Wrote %s", self.fp.name)
+
+
+class CompiledFile(object):
+    def __init__(self, name, ref_controller, **kwargs):
+        self.name = name
+        self.fp = None
+        self.ref_controller = ref_controller
+        self.kwargs = kwargs
+
+    def __enter__(self):
+        mode = self.kwargs.get("mode", "r")
+        self.fp = open(self.name, mode)
+        return CompilingFile(self, self.fp, self.ref_controller, **self.kwargs)
+
+    def __exit__(self, *args):
+        self.fp.close()

kapitan/inputs/jinja2.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Kapitan Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import logging
+import os
+
+from kapitan.inputs.base import InputType, CompiledFile
+from kapitan.resources import inventory
+from kapitan.utils import render_jinja2
+
+logger = logging.getLogger(__name__)
+
+
+class Jinja2(InputType):
+    def __init__(self, compile_path, search_paths, ref_controller):
+        super().__init__("jinja2", compile_path, search_paths, ref_controller)
+
+    def compile_file(self, file_path, compile_path, ext_vars, **kwargs):
+        """
+        Write items in path as jinja2 rendered files to compile_path.
+        path can be either a file or directory.
+        kwargs:
+            reveal: default False, set to reveal refs on compile
+            target_name: default None, set to current target being compiled
+        """
+        reveal = kwargs.get('reveal', False)
+        target_name = kwargs.get('target_name', None)
+
+        # set ext_vars and inventory for jinja2 context
+        context = ext_vars.copy()
+        context["inventory"] = inventory(self.search_paths, target_name)
+        context["inventory_global"] = inventory(self.search_paths, None)
+
+        for item_key, item_value in render_jinja2(file_path, context).items():
+            full_item_path = os.path.join(compile_path, item_key)
+            os.makedirs(os.path.dirname(full_item_path), exist_ok=True)
+
+            with CompiledFile(full_item_path, self.ref_controller, mode="w", reveal=reveal,
+                              target_name=target_name) as fp:
+                fp.write(item_value["content"])
+                mode = item_value["mode"]
+                os.chmod(full_item_path, mode)
+                logger.debug("Wrote %s with mode %.4o", full_item_path, mode)
+
+    def default_output_type(self):
+        # no output_type options for jinja2
+        return None

kapitan/inputs/jsonnet.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+#
+# Copyright 2018 The Kapitan Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import ujson as json
+
+from kapitan.inputs.base import InputType, CompiledFile
+from kapitan.resources import search_imports, resource_callbacks
+from kapitan.utils import jsonnet_file, prune_empty
+
+logger = logging.getLogger(__name__)
+
+
+class Jsonnet(InputType):
+    def __init__(self, compile_path, search_paths, ref_controller):
+        super().__init__("jsonnet", compile_path, search_paths, ref_controller)
+
+    def compile_file(self, file_path, compile_path, ext_vars, **kwargs):
+        """
+        Write file_path (jsonnet evaluated) items as files to compile_path.
+        ext_vars will be passed as parameters to jsonnet_file()
+        kwargs:
+            output: default 'yaml', accepts 'json'
+            prune: default False, accepts True
+            reveal: default False, set to reveal refs on compile
+            target_name: default None, set to current target being compiled
+            indent: default 2
+        """
+        def _search_imports(cwd, imp):
+            return search_imports(cwd, imp, self.search_paths)
+
+        json_output = jsonnet_file(file_path, import_callback=_search_imports,
+                                   native_callbacks=resource_callbacks(self.search_paths),
+                                   ext_vars=ext_vars)
+        json_output = json.loads(json_output)
+
+        output = kwargs.get('output', 'yaml')
+        prune = kwargs.get('prune', False)
+        reveal = kwargs.get('reveal', False)
+        target_name = kwargs.get('target_name', None)
+        indent = kwargs.get('indent', 2)
+
+        if prune:
+            json_output = prune_empty(json_output)
+            logger.debug("Pruned output for: %s", file_path)
+
+        for item_key, item_value in json_output.items():
+            # write each item to disk
+            if output == 'json':
+                file_path = os.path.join(compile_path, '%s.%s' % (item_key, output))
+                with CompiledFile(file_path, self.ref_controller, mode="w", reveal=reveal, target_name=target_name,
+                                  indent=indent) as fp:
+                    fp.write_json(item_value)
+            elif output == 'yaml':
+                file_path = os.path.join(compile_path, '%s.%s' % (item_key, "yml"))
+                with CompiledFile(file_path, self.ref_controller, mode="w", reveal=reveal, target_name=target_name,
+                                  indent=indent) as fp:
+                    fp.write_yaml(item_value)
+            else:
+                raise ValueError('output is neither "json" or "yaml"')
+
+    def default_output_type(self):
+        return "yaml"

kapitan/refs/secrets/awskms.py

@@ -114,10 +114,7 @@ def _encrypt(self, data, key, encode_base64):
             if isinstance(data, str):
                 _data = data.encode()
         try:
-            response = awskms_obj().encrypt(
-                KeyId=key,
-                Plaintext=_data
-                )
+            response = awskms_obj().encrypt(KeyId=key, Plaintext=_data)
             ciphertext = base64.b64encode(response['CiphertextBlob'])
             self.data = ciphertext
             self.key = key