Gojsonnet

examples/kubernetes/compiled/all-glob/manifests/01_yaml_load.yml

@@ -0,0 +1,3 @@
+json_str: '{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "busybox", "namespace":
+  "default"}, "spec": {"containers": [{"image": "busybox", "command": ["sleep", "3600"],
+  "imagePullPolicy": "IfNotPresent", "name": "busybox"}], "restartPolicy": "Always"}}'

examples/kubernetes/components/jsonnet_callbacks_test/main.jsonnet

@@ -0,0 +1,8 @@
+local kube = import "lib/kube.libjsonnet";
+local kap = import "lib/kapitan.libjsonnet";
+local inventory = kap.inventory();
+local p = inventory.parameters;
+
+{
+    "01_yaml_load": {json_str: kap.yaml_load("components/busybox/pod.yml")},
+}

kapitan/cached.py

@@ -35,6 +35,38 @@ def reset_cache():
     revealer_obj = None
 
 
+def from_dict(cache_dict):
+    global inv, inv_cache, gpg_obj, gkms_obj, awskms_obj, azkms_obj, dot_kapitan, ref_controller_obj, revealer_obj, inv_sources, args
+
+    inv = cache_dict["inv"]
+    inv_cache = cache_dict["inv_cache"]
+    inv_sources = cache_dict["inv_sources"]
+    gpg_obj = cache_dict["gpg_obj"]
+    gkms_obj = cache_dict["gkms_obj"]
+    awskms_obj = cache_dict["awskms_obj"]
+    azkms_obj = cache_dict["azkms_obj"]
+    dot_kapitan = cache_dict["dot_kapitan"]
+    ref_controller_obj = cache_dict["ref_controller_obj"]
+    revealer_obj = cache_dict["revealer_obj"]
+    args = cache_dict["args"]
+
+
+def as_dict():
+    return {
+        "inv": inv,
+        "inv_cache": inv_cache,
+        "inv_sources": inv_sources,
+        "gpg_obj": gpg_obj,
+        "gkms_obj": gkms_obj,
+        "awskms_obj": awskms_obj,
+        "azkms_obj": azkms_obj,
+        "dot_kapitan": dot_kapitan,
+        "ref_controller_obj": ref_controller_obj,
+        "revealer_obj": revealer_obj,
+        "args": args,
+    }
+
+
 def reset_inv():
     """clears the inv while fetching remote inventories"""
     global inv

kapitan/cli.py

@@ -15,7 +15,6 @@
 import multiprocessing
 import os
 import sys
-from functools import partial
 
 import yaml
 from kapitan import cached, defaults, setup_logging
@@ -100,7 +99,7 @@ def main():
         # set 'fork' method as a more deterministic/conservative
         # and compatible multiprocessing method for Linux and MacOS
         # see https://github.com/kapicorp/kapitan/issues/641
-        multiprocessing.set_start_method("fork")
+        multiprocessing.set_start_method("spawn")
     # main() is explicitly multiple times in tests
     # and will raise RuntimeError
     except RuntimeError:
@@ -292,7 +291,7 @@ def main():
     )
 
     inventory_parser = subparser.add_parser("inventory", aliases=["i"], help="show inventory")
-    inventory_parser.set_defaults(func=partial(generate_inventory, inventory_parser))
+    inventory_parser.set_defaults(func=generate_inventory)
 
     inventory_parser.add_argument(
         "--target-name",
@@ -543,6 +542,9 @@ def main():
     )
     args = parser.parse_args()
 
+    if getattr(args, 'func', None) == generate_inventory and args.pattern and args.target_name == "":
+        parser.error("--pattern requires --target_name")
+
     logger.debug("Running with args: %s", args)
 
     try:

kapitan/resources.py

@@ -21,7 +21,6 @@
 import kapitan.cached as cached
 import yaml
 from kapitan import __file__ as kapitan_install_path
-from kapitan import cached as cached
 from kapitan.errors import CompileError, InventoryError, KapitanError
 from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string
 
@@ -36,6 +35,8 @@
 except ImportError:
     from yaml import SafeLoader as YamlLoader
 
+JSONNET_CACHE = {}
+
 
 def resource_callbacks(search_paths):
     """
@@ -212,7 +213,10 @@ def search_imports(cwd, import_str, search_paths):
     needs to be closured.
     """
     basename = os.path.basename(import_str)
-    full_import_path = os.path.join(cwd, import_str)
+    full_import_path = os.path.normpath(os.path.join(cwd, import_str))
+
+    if full_import_path in JSONNET_CACHE:
+        return full_import_path, JSONNET_CACHE[full_import_path]
 
     if not os.path.exists(full_import_path):
         # if import_str not found, search in install_path
@@ -241,6 +245,7 @@ def search_imports(cwd, import_str, search_paths):
     normalised_path_content = ""
     with open(normalised_path) as f:
         normalised_path_content = f.read()
+        JSONNET_CACHE[normalised_path] = normalised_path_content
 
     return normalised_path, normalised_path_content
 
@@ -253,7 +258,6 @@ def inventory(search_paths, target, inventory_path=None):
     set inventory_path to read custom path. None defaults to value set via cli
     Returns a dictionary with the inventory for target
     """
-
     if inventory_path is None:
         # grab inventory_path value from cli subcommand
         inventory_path_arg = cached.args.get("compile") or cached.args.get("inventory")
@@ -282,9 +286,7 @@ def inventory(search_paths, target, inventory_path=None):
     return inventory_reclass(full_inv_path)["nodes"][target]
 
 
-def generate_inventory(parser, args):
-    if args.pattern and args.target_name == "":
-        parser.error("--pattern requires --target_name")
+def generate_inventory(args):
     try:
         inv = inventory_reclass(args.inventory_path)
         if args.target_name != "":

kapitan/targets.py

@@ -31,7 +31,7 @@
 from kapitan.inputs.external import External
 from kapitan.remoteinventory.fetch import fetch_inventories, list_sources
 from kapitan.resources import inventory_reclass
-from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache
+from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache, JSONNET_AVAILABLE
 from kapitan.validator.kubernetes_validator import KubernetesManifestValidator
 
 from reclass.errors import NotFoundError, ReclassException
@@ -40,9 +40,7 @@
 
 
 def check_jsonnet_import():
-    try:
-        import _jsonnet
-    except ImportError:
+    if not JSONNET_AVAILABLE:
         logger.info(
             "Note: jsonnet is not yet supported on ARM/M1. You can still use kadet and jinja2 for templating"
         )
@@ -98,14 +96,6 @@ def compile_targets(
 
         # append "compiled" to output_path so we can safely overwrite it
         compile_path = os.path.join(output_path, "compiled")
-        worker = partial(
-            compile_target,
-            search_paths=search_paths,
-            compile_path=temp_compile_path,
-            ref_controller=ref_controller,
-            inventory_path=inventory_path,
-            **kwargs,
-        )
 
         if not target_objs:
             raise CompileError("Error: no targets found")
@@ -131,6 +121,16 @@ def compile_targets(
                 output_path, target_objs, dep_cache_dir, kwargs.get("force_fetch", False), pool
             )
 
+        worker = partial(
+            compile_target,
+            search_paths=search_paths,
+            compile_path=temp_compile_path,
+            ref_controller=ref_controller,
+            inventory_path=inventory_path,
+            globals_cached=cached.as_dict(),
+            **kwargs,
+        )
+
         # compile_target() returns None on success
         # so p is only not None when raising an exception
         [p.get() for p in pool.imap_unordered(worker, target_objs) if p]
@@ -418,13 +418,16 @@ def search_targets(inventory_path, targets, labels):
     return targets_found
 
 
-def compile_target(target_obj, search_paths, compile_path, ref_controller, **kwargs):
+def compile_target(target_obj, search_paths, compile_path, ref_controller, globals_cached=None, **kwargs):
     """Compiles target_obj and writes to compile_path"""
     start = time.time()
     compile_objs = target_obj["compile"]
     ext_vars = target_obj["vars"]
     target_name = ext_vars["target"]
 
+    if globals_cached:
+        cached.from_dict(globals_cached)
+
     for comp_obj in compile_objs:
         input_type = comp_obj["input_type"]
         output_path = comp_obj["output_path"]
@@ -462,6 +465,7 @@ def compile_target(target_obj, search_paths, compile_path, ref_controller, **kwa
             err_msg = 'Invalid input_type: "{}". Supported input_types: jsonnet, jinja2, kadet, helm, copy, remove, external'
             raise CompileError(err_msg.format(input_type))
 
+        # logger.info("about to compile %s ", target_obj["target_full_path"])
         input_compiler.make_compile_dirs(target_name, output_path)
         input_compiler.compile_obj(comp_obj, ext_vars, **kwargs)
 

kapitan/utils.py

@@ -21,9 +21,21 @@
 from functools import lru_cache, wraps
 from hashlib import sha256
 
+JSONNET_AVAILABLE = True
+try:
+    import _gojsonnet as jsonnet
+
+    logging.debug("Using GO jsonnet over C jsonnet")
+except ImportError:
+    try:
+        import _jsonnet as jsonnet
+    except ImportError:
+        JSONNET_AVAILABLE = False
+
 import jinja2
 import requests
 import yaml
+
 from kapitan import cached, defaults
 from kapitan.errors import CompileError
 from kapitan.inputs.jinja2_filters import load_jinja2_filters, load_jinja2_filters_from_file
@@ -37,11 +49,6 @@
 
 logger = logging.getLogger(__name__)
 
-try:
-    import _jsonnet as jsonnet
-except ImportError as e:
-    logger.debug("Could not import jsonnet: %s", e)
-
 
 try:
     from yaml import CSafeLoader as YamlLoader

setup.py

@@ -75,4 +75,5 @@ def install_deps():
             "kapitan=kapitan.cli:main",
         ],
     },
+    extras_require={"gojsonnet": ["gojsonnet==0.17.0"]},
 )

tests/test_kubernetes_compiled/all-glob/manifests/01_yaml_load.yml

@@ -0,0 +1,3 @@
+json_str: '{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "busybox", "namespace":
+  "default"}, "spec": {"containers": [{"image": "busybox", "command": ["sleep", "3600"],
+  "imagePullPolicy": "IfNotPresent", "name": "busybox"}], "restartPolicy": "Always"}}'