CFE-3803 Modified cfbs add folder behavior

.gitignore

@@ -14,3 +14,7 @@ node_modules/
 build/
 dist/
 .cache
+
+# Test Outputs
+test/add_folder_sample/cfbs.json
+test/add_folder_sample/out

cfbs/commands.py

@@ -31,6 +31,12 @@
 definition = None
 
 
+# This function is for clearing the global for pytest cases when it should be changing
+def clear_definition():
+    global definition
+    definition = None
+
+
 def get_definition() -> dict:
     global definition
     if not definition:
@@ -82,7 +88,7 @@ def init_command(index=None) -> int:
         "build": [],
     }
     if index:
-        definition['index'] = index
+        definition["index"] = index
 
     write_json(cfbs_filename(), definition)
     assert is_cfbs_repo()
@@ -202,9 +208,10 @@ def local_module_copy(module, counter, max_length):
         f"{counter:03d} {pad_right(name, max_length)} @ local                                    (Copied)"
     )
 
+
 def _get_path_from_url(url):
     if not url.startswith(("https://", "ssh://", "git://")):
-        if ("://" in url):
+        if "://" in url:
             return user_error("Unsupported URL protocol in '%s'" % url)
         else:
             # It's a path already, just remove trailing slashes (if any).
@@ -215,12 +222,13 @@ def _get_path_from_url(url):
         match = re.match(r"ssh://(\w+)@(.+)", url)
         if match is not None:
             path = match[2]
-    path = path or url[url.index("://") + 3:]
+    path = path or url[url.index("://") + 3 :]
     path = strip_right(path, ".git")
     path = path.strip("/")
 
     return path
 
+
 def _get_git_repo_commit_sha(repo_path):
     assert os.path.isdir(os.path.join(repo_path, ".git"))
 
@@ -233,6 +241,7 @@ def _get_git_repo_commit_sha(repo_path):
     with open(os.path.join(repo_path, ".git", head_ref)) as f:
         return f.read().strip()
 
+
 def _clone_index_repo(repo_url):
     assert repo_url.startswith(("https://", "ssh://", "git://"))
 
@@ -267,7 +276,10 @@ def _clone_index_repo(repo_url):
     if os.path.exists(index_path):
         return (index_path, commit)
     else:
-        user_error("Repository '%s' doesn't contain a valid cfbs.json index file" % repo_url)
+        user_error(
+            "Repository '%s' doesn't contain a valid cfbs.json index file" % repo_url
+        )
+
 
 def add_command(to_add: list, added_by="cfbs add", index_path=None) -> int:
     if not to_add:
@@ -288,7 +300,10 @@ def add_command(to_add: list, added_by="cfbs add", index_path=None) -> int:
     # URL specified in to_add, but no specific modules => let's add all (with a prompt)
     if len(to_add) == 0:
         modules = index.get_modules()
-        answer = input("Do you want to add all %d modules from the repository? [y/N] " % len(modules))
+        answer = input(
+            "Do you want to add all %d modules from the repository? [y/N] "
+            % len(modules)
+        )
         if answer.lower() not in ("y", "yes"):
             return 0
         to_add = modules.keys()
@@ -513,6 +528,38 @@ def build_step(module, step, max_length):
             touch(dst)
         assert os.path.isfile(dst)
         sh(f"cat '{src}' >> '{dst}'")
+    elif operation == "directory":
+        src, dst = args
+        if dst in [".", "./"]:
+            dst = ""
+        print("{} directory '{}' 'masterfiles/{}'".format(prefix, src, dst))
+        dstarg = dst  # save this for adding .cf files to inputs
+        src, dst = os.path.join(source, src), os.path.join(destination, dst)
+        defjson = os.path.join(destination, "def.json")
+        merged = read_json(defjson)
+        if not merged:
+            merged = {}
+        if "classes" not in merged:
+            merged["classes"] = {}
+        if "services_autorun_bundles" not in merged["classes"]:
+            merged["classes"]["services_autorun_bundles"] = ["any"]
+        inputs = []
+        for root, dirs, files in os.walk(src):
+            for f in files:
+                if f.endswith(".cf"):
+                    inputs.append(os.path.join(dstarg, f))
+                    cp(os.path.join(root, f), os.path.join(destination, dstarg, f))
+                elif f == "def.json":
+                    extra = read_json(os.path.join(root, f))
+                    if extra:
+                        merged = merge_json(merged, extra)
+                else:
+                    cp(os.path.join(root, f), os.path.join(destination, dstarg, f))
+        if "inputs" in merged:
+            merged["inputs"].extend(inputs)
+        else:
+            merged["inputs"] = inputs
+        write_json(defjson, merged)
     else:
         user_error(f"Unknown build step operation: {operation}")
 

cfbs/index.py

@@ -21,7 +21,6 @@
     sh,
 )
 
-from cfbs.pretty import pretty_file, pretty
 
 def _local_module_data_cf_file(module):
     target = os.path.basename(module)
@@ -44,11 +43,12 @@ def _local_module_data_json_file(module):
 
 
 def _local_module_data_subdir(module):
+    assert module.startswith("./")
+    dst = os.path.join("services", "cfbs", module[2:])
     return {
         "description": "Local subdirectory added using cfbs command line",
         "tags": ["local"],
-        "dependencies": ["autorun"],
-        "steps": [f"copy {module} services/autorun/"],
+        "steps": ["directory {} {}".format(module, dst)],
         "added_by": "cfbs add",
     }
 
@@ -65,6 +65,7 @@ def generate_index_for_local_module(module):
     if module.endswith(".json"):
         return _local_module_data_json_file(module)
 
+
 class Index:
     def __init__(self, path):
         self.path = path

test/add_folder_sample/baz/biz/bazbiz.cf

@@ -0,0 +1,6 @@
+bundle agent bazbiz
+{
+  meta:
+    "tags" slist => { "autorun" };
+  reports: "I am bazbiz.";
+}

test/add_folder_sample/baz/biz/def.json

@@ -0,0 +1,5 @@
+{
+  "vars": {
+    "bazbizthing": "superlative"
+  }
+}

test/add_folder_sample/foo/def.json

@@ -0,0 +1,5 @@
+{
+  "vars": {
+    "foo_thing": "awesome"
+  }
+}

test/add_folder_sample/foo/foo.cf

@@ -0,0 +1,6 @@
+bundle agent foo
+{
+  meta:
+    "tags" slist => { "autorun" };
+  reports: "I am foo.";
+}

test/conftest.py

@@ -0,0 +1,9 @@
+import os
+import pytest
+
+
+@pytest.fixture(scope="function")
+def chdir(request):
+    os.chdir(os.path.join(os.path.dirname(__file__), request.param))
+    yield
+    os.chdir(request.config.invocation_dir)

test/test_add_folder.py

@@ -0,0 +1,51 @@
+import json
+import os
+import pytest
+from cfbs.commands import (
+    build_command,
+    init_command,
+    add_command,
+)
+from cfbs.utils import read_json
+
+
+@pytest.mark.parametrize("chdir", ["add_folder_sample"], indirect=True)
+def test_add_folders_and_build(chdir):
+    os.system("rm -rf out")
+    os.system("rm -f cfbs.json")
+    init_command()
+    add_command(["masterfiles"])
+    add_command(["./foo/"])
+    add_command(["./bar/"])
+    add_command(["./baz/"])
+    build_command()
+
+    expected = json.loads(
+        """{
+  "classes": { "services_autorun_bundles": ["any"] },
+  "vars": { "foo_thing": "awesome", "bazbizthing": "superlative" },
+  "inputs": ["services/cfbs/foo/foo.cf", "services/cfbs/baz/bazbiz.cf"]
+}"""
+    )
+    actual = read_json("out/masterfiles/def.json")
+    assert actual == expected
+
+    actual = []
+    for root, dirs, files in os.walk("out/masterfiles/services/cfbs"):
+        for dir in dirs:
+            actual.append(os.path.join(root, dir))
+        for file in files:
+            actual.append(os.path.join(root, file))
+    # sort these, different orders on different systems
+    actual = sorted(actual)
+    expected = [
+        "out/masterfiles/services/cfbs/bar",
+        "out/masterfiles/services/cfbs/bar/bar.json",
+        "out/masterfiles/services/cfbs/baz",
+        "out/masterfiles/services/cfbs/baz/bazbiz.cf",
+        "out/masterfiles/services/cfbs/baz/bazbiz.json",
+        "out/masterfiles/services/cfbs/foo",
+        "out/masterfiles/services/cfbs/foo/foo.cf",
+        "out/masterfiles/services/cfbs/foo/oddfile.txt",
+    ]
+    assert actual == expected

test/test_showinfo.py

@@ -1,19 +1,30 @@
 import os
 import re
-from cfbs.commands import info_command
+import pytest
+from cfbs.commands import info_command, clear_definition
 
-os.chdir(os.path.join(os.path.dirname(__file__),"sample"))
 
-def test_noargs(capfd):
+@pytest.mark.parametrize("chdir", ["sample"], indirect=True)
+def test_noargs(capfd, chdir):
+    clear_definition()
     info_command([])
-    out, err = capfd.readouterr()
+    out, _ = capfd.readouterr()
     assert out == "\n"
 
-def test_showinfo(capfd):
-    info_command("autorun masterfiles foo/main.cf bar/my.json bar/baz/main.cf nosuchfile".split(" "))
-    out, err = capfd.readouterr()
 
-    assert re.search(r"""Module: autorun
+@pytest.mark.parametrize("chdir", ["sample"], indirect=True)
+def test_showinfo(capfd, chdir):
+    clear_definition()
+    assert os.path.exists("cfbs.json")
+    info_command(
+        "autorun masterfiles foo/main.cf bar/my.json bar/baz/main.cf nosuchfile".split(
+            " "
+        )
+    )
+    out, _ = capfd.readouterr()
+
+    assert re.search(
+        r"""Module: autorun
 Version: \d+\.\d+\.\d+
 Status: Added
 By: https:\/\/github.com\/cfengine
@@ -22,20 +33,27 @@ def test_showinfo(capfd):
 Commit: [a-zA-Z0-9]+
 Added By: ./foo/main.cf
 Description: Enable autorun functionality
-""", out, re.M)
-
+""",
+        out,
+        re.M,
+    ), out
 
-    assert re.search(r"""Module: masterfiles
+    assert re.search(
+        r"""Module: masterfiles
 Version: \d+\.\d+\.\d+
 Status: Not Added
 By: https:\/\/github.com\/cfengine
-Tags: official, base
+Tags: official, base, supported
 Repo: https:\/\/github.com\/cfengine\/masterfiles
 Commit: [a-zA-Z0-9]+
 Description: Official CFEngine Masterfiles Policy Framework \(MPF\)
-""", out, re.M)
+""",
+        out,
+        re.M,
+    ), out
 
-    assert """Module: ./foo/main.cf
+    assert (
+        """Module: ./foo/main.cf
 Status: Added
 Tags: local
 Dependencies: autorun
@@ -57,7 +75,6 @@ def test_showinfo(capfd):
 
 Module 'nosuchfile' does not exist
 
-""" in out
-
-def __main__():
-  test_showinfo()
+"""
+        in out
+    )