From 33aca59c74f7df5224a8468c229ed114fdd3e2d4 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Wed, 13 Oct 2021 16:55:06 -0500 Subject: [PATCH] Modified cfbs add folder behavior Folder is searched recursively. All .cf files are added to inputs. Any def.json files are merged into top-level def.json. All files are copied to services/cfbs/. Ticket: CFE-3803 --- .gitignore | 4 ++ cfbs/commands.py | 57 ++++++++++++++++++++-- cfbs/index.py | 7 +-- test/add_folder_sample/bar/bar.json | 0 test/add_folder_sample/baz/biz/bazbiz.cf | 6 +++ test/add_folder_sample/baz/biz/bazbiz.json | 0 test/add_folder_sample/baz/biz/def.json | 5 ++ test/add_folder_sample/foo/def.json | 5 ++ test/add_folder_sample/foo/foo.cf | 6 +++ test/add_folder_sample/foo/oddfile.txt | 0 test/conftest.py | 9 ++++ test/test_add_folder.py | 51 +++++++++++++++++++ test/test_showinfo.py | 53 +++++++++++++------- 13 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 test/add_folder_sample/bar/bar.json create mode 100644 test/add_folder_sample/baz/biz/bazbiz.cf create mode 100644 test/add_folder_sample/baz/biz/bazbiz.json create mode 100644 test/add_folder_sample/baz/biz/def.json create mode 100644 test/add_folder_sample/foo/def.json create mode 100644 test/add_folder_sample/foo/foo.cf create mode 100644 test/add_folder_sample/foo/oddfile.txt create mode 100644 test/conftest.py create mode 100644 test/test_add_folder.py diff --git a/.gitignore b/.gitignore index e5e89d59..9fb1dba7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ node_modules/ build/ dist/ .cache + +# Test Outputs +test/add_folder_sample/cfbs.json +test/add_folder_sample/out diff --git a/cfbs/commands.py b/cfbs/commands.py index a6d8d190..88d51969 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -31,6 +31,12 @@ definition = None +# This function is for clearing the global for pytest cases when it should be changing +def clear_definition(): + global definition + definition = None + + def get_definition() -> dict: global definition if not definition: @@ -82,7 +88,7 @@ def init_command(index=None) -> int: "build": [], } if index: - definition['index'] = index + definition["index"] = index write_json(cfbs_filename(), definition) assert is_cfbs_repo() @@ -202,9 +208,10 @@ def local_module_copy(module, counter, max_length): f"{counter:03d} {pad_right(name, max_length)} @ local (Copied)" ) + def _get_path_from_url(url): if not url.startswith(("https://", "ssh://", "git://")): - if ("://" in url): + if "://" in url: return user_error("Unsupported URL protocol in '%s'" % url) else: # It's a path already, just remove trailing slashes (if any). @@ -215,12 +222,13 @@ def _get_path_from_url(url): match = re.match(r"ssh://(\w+)@(.+)", url) if match is not None: path = match[2] - path = path or url[url.index("://") + 3:] + path = path or url[url.index("://") + 3 :] path = strip_right(path, ".git") path = path.strip("/") return path + def _get_git_repo_commit_sha(repo_path): assert os.path.isdir(os.path.join(repo_path, ".git")) @@ -233,6 +241,7 @@ def _get_git_repo_commit_sha(repo_path): with open(os.path.join(repo_path, ".git", head_ref)) as f: return f.read().strip() + def _clone_index_repo(repo_url): assert repo_url.startswith(("https://", "ssh://", "git://")) @@ -267,7 +276,10 @@ def _clone_index_repo(repo_url): if os.path.exists(index_path): return (index_path, commit) else: - user_error("Repository '%s' doesn't contain a valid cfbs.json index file" % repo_url) + user_error( + "Repository '%s' doesn't contain a valid cfbs.json index file" % repo_url + ) + def add_command(to_add: list, added_by="cfbs add", index_path=None) -> int: if not to_add: @@ -288,7 +300,10 @@ def add_command(to_add: list, added_by="cfbs add", index_path=None) -> int: # URL specified in to_add, but no specific modules => let's add all (with a prompt) if len(to_add) == 0: modules = index.get_modules() - answer = input("Do you want to add all %d modules from the repository? [y/N] " % len(modules)) + answer = input( + "Do you want to add all %d modules from the repository? [y/N] " + % len(modules) + ) if answer.lower() not in ("y", "yes"): return 0 to_add = modules.keys() @@ -513,6 +528,38 @@ def build_step(module, step, max_length): touch(dst) assert os.path.isfile(dst) sh(f"cat '{src}' >> '{dst}'") + elif operation == "directory": + src, dst = args + if dst in [".", "./"]: + dst = "" + print("{} directory '{}' 'masterfiles/{}'".format(prefix, src, dst)) + dstarg = dst # save this for adding .cf files to inputs + src, dst = os.path.join(source, src), os.path.join(destination, dst) + defjson = os.path.join(destination, "def.json") + merged = read_json(defjson) + if not merged: + merged = {} + if "classes" not in merged: + merged["classes"] = {} + if "services_autorun_bundles" not in merged["classes"]: + merged["classes"]["services_autorun_bundles"] = ["any"] + inputs = [] + for root, dirs, files in os.walk(src): + for f in files: + if f.endswith(".cf"): + inputs.append(os.path.join(dstarg, f)) + cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) + elif f == "def.json": + extra = read_json(os.path.join(root, f)) + if extra: + merged = merge_json(merged, extra) + else: + cp(os.path.join(root, f), os.path.join(destination, dstarg, f)) + if "inputs" in merged: + merged["inputs"].extend(inputs) + else: + merged["inputs"] = inputs + write_json(defjson, merged) else: user_error(f"Unknown build step operation: {operation}") diff --git a/cfbs/index.py b/cfbs/index.py index f8f1b026..a7ec5a9c 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -21,7 +21,6 @@ sh, ) -from cfbs.pretty import pretty_file, pretty def _local_module_data_cf_file(module): target = os.path.basename(module) @@ -44,11 +43,12 @@ def _local_module_data_json_file(module): def _local_module_data_subdir(module): + assert module.startswith("./") + dst = os.path.join("services", "cfbs", module[2:]) return { "description": "Local subdirectory added using cfbs command line", "tags": ["local"], - "dependencies": ["autorun"], - "steps": [f"copy {module} services/autorun/"], + "steps": ["directory {} {}".format(module, dst)], "added_by": "cfbs add", } @@ -65,6 +65,7 @@ def generate_index_for_local_module(module): if module.endswith(".json"): return _local_module_data_json_file(module) + class Index: def __init__(self, path): self.path = path diff --git a/test/add_folder_sample/bar/bar.json b/test/add_folder_sample/bar/bar.json new file mode 100644 index 00000000..e69de29b diff --git a/test/add_folder_sample/baz/biz/bazbiz.cf b/test/add_folder_sample/baz/biz/bazbiz.cf new file mode 100644 index 00000000..13ad8cb9 --- /dev/null +++ b/test/add_folder_sample/baz/biz/bazbiz.cf @@ -0,0 +1,6 @@ +bundle agent bazbiz +{ + meta: + "tags" slist => { "autorun" }; + reports: "I am bazbiz."; +} diff --git a/test/add_folder_sample/baz/biz/bazbiz.json b/test/add_folder_sample/baz/biz/bazbiz.json new file mode 100644 index 00000000..e69de29b diff --git a/test/add_folder_sample/baz/biz/def.json b/test/add_folder_sample/baz/biz/def.json new file mode 100644 index 00000000..46aa5ec3 --- /dev/null +++ b/test/add_folder_sample/baz/biz/def.json @@ -0,0 +1,5 @@ +{ + "vars": { + "bazbizthing": "superlative" + } +} diff --git a/test/add_folder_sample/foo/def.json b/test/add_folder_sample/foo/def.json new file mode 100644 index 00000000..08d32064 --- /dev/null +++ b/test/add_folder_sample/foo/def.json @@ -0,0 +1,5 @@ +{ + "vars": { + "foo_thing": "awesome" + } +} diff --git a/test/add_folder_sample/foo/foo.cf b/test/add_folder_sample/foo/foo.cf new file mode 100644 index 00000000..bff62b4c --- /dev/null +++ b/test/add_folder_sample/foo/foo.cf @@ -0,0 +1,6 @@ +bundle agent foo +{ + meta: + "tags" slist => { "autorun" }; + reports: "I am foo."; +} diff --git a/test/add_folder_sample/foo/oddfile.txt b/test/add_folder_sample/foo/oddfile.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..fa6da5b5 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,9 @@ +import os +import pytest + + +@pytest.fixture(scope="function") +def chdir(request): + os.chdir(os.path.join(os.path.dirname(__file__), request.param)) + yield + os.chdir(request.config.invocation_dir) diff --git a/test/test_add_folder.py b/test/test_add_folder.py new file mode 100644 index 00000000..826ad955 --- /dev/null +++ b/test/test_add_folder.py @@ -0,0 +1,51 @@ +import json +import os +import pytest +from cfbs.commands import ( + build_command, + init_command, + add_command, +) +from cfbs.utils import read_json + + +@pytest.mark.parametrize("chdir", ["add_folder_sample"], indirect=True) +def test_add_folders_and_build(chdir): + os.system("rm -rf out") + os.system("rm -f cfbs.json") + init_command() + add_command(["masterfiles"]) + add_command(["./foo/"]) + add_command(["./bar/"]) + add_command(["./baz/"]) + build_command() + + expected = json.loads( + """{ + "classes": { "services_autorun_bundles": ["any"] }, + "vars": { "foo_thing": "awesome", "bazbizthing": "superlative" }, + "inputs": ["services/cfbs/foo/foo.cf", "services/cfbs/baz/bazbiz.cf"] +}""" + ) + actual = read_json("out/masterfiles/def.json") + assert actual == expected + + actual = [] + for root, dirs, files in os.walk("out/masterfiles/services/cfbs"): + for dir in dirs: + actual.append(os.path.join(root, dir)) + for file in files: + actual.append(os.path.join(root, file)) + # sort these, different orders on different systems + actual = sorted(actual) + expected = [ + "out/masterfiles/services/cfbs/bar", + "out/masterfiles/services/cfbs/bar/bar.json", + "out/masterfiles/services/cfbs/baz", + "out/masterfiles/services/cfbs/baz/bazbiz.cf", + "out/masterfiles/services/cfbs/baz/bazbiz.json", + "out/masterfiles/services/cfbs/foo", + "out/masterfiles/services/cfbs/foo/foo.cf", + "out/masterfiles/services/cfbs/foo/oddfile.txt", + ] + assert actual == expected diff --git a/test/test_showinfo.py b/test/test_showinfo.py index 42d338e1..cf1fcbfd 100644 --- a/test/test_showinfo.py +++ b/test/test_showinfo.py @@ -1,19 +1,30 @@ import os import re -from cfbs.commands import info_command +import pytest +from cfbs.commands import info_command, clear_definition -os.chdir(os.path.join(os.path.dirname(__file__),"sample")) -def test_noargs(capfd): +@pytest.mark.parametrize("chdir", ["sample"], indirect=True) +def test_noargs(capfd, chdir): + clear_definition() info_command([]) - out, err = capfd.readouterr() + out, _ = capfd.readouterr() assert out == "\n" -def test_showinfo(capfd): - info_command("autorun masterfiles foo/main.cf bar/my.json bar/baz/main.cf nosuchfile".split(" ")) - out, err = capfd.readouterr() - assert re.search(r"""Module: autorun +@pytest.mark.parametrize("chdir", ["sample"], indirect=True) +def test_showinfo(capfd, chdir): + clear_definition() + assert os.path.exists("cfbs.json") + info_command( + "autorun masterfiles foo/main.cf bar/my.json bar/baz/main.cf nosuchfile".split( + " " + ) + ) + out, _ = capfd.readouterr() + + assert re.search( + r"""Module: autorun Version: \d+\.\d+\.\d+ Status: Added By: https:\/\/github.com\/cfengine @@ -22,20 +33,27 @@ def test_showinfo(capfd): Commit: [a-zA-Z0-9]+ Added By: ./foo/main.cf Description: Enable autorun functionality -""", out, re.M) - +""", + out, + re.M, + ), out - assert re.search(r"""Module: masterfiles + assert re.search( + r"""Module: masterfiles Version: \d+\.\d+\.\d+ Status: Not Added By: https:\/\/github.com\/cfengine -Tags: official, base +Tags: official, base, supported Repo: https:\/\/github.com\/cfengine\/masterfiles Commit: [a-zA-Z0-9]+ Description: Official CFEngine Masterfiles Policy Framework \(MPF\) -""", out, re.M) +""", + out, + re.M, + ), out - assert """Module: ./foo/main.cf + assert ( + """Module: ./foo/main.cf Status: Added Tags: local Dependencies: autorun @@ -57,7 +75,6 @@ def test_showinfo(capfd): Module 'nosuchfile' does not exist -""" in out - -def __main__(): - test_showinfo() +""" + in out + )