Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix handling of labeled hosts by pbench-postprocess-tools (fwd port of #3456) #3472

Merged
merged 7 commits into from
Jul 3, 2023
Merged

Fix handling of labeled hosts by pbench-postprocess-tools (fwd port of #3456) #3472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e7dec0a
Fix handling of labeled hosts by pbench-postprocess-tools (forward po…
ndokos Jun 9, 2023
5f02aae
Address review comments
ndokos Jun 27, 2023
af0547f
Address review comments
ndokos Jun 29, 2023
8870ca9
Address review comments
ndokos Jun 29, 2023
8bb4e4f
More review comments addressed
ndokos Jun 30, 2023
b649481
... more review comments
ndokos Jun 30, 2023
7d88a0c
Merge branch 'main' into fwd3456
ndokos Jul 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent/VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.72.0
dbutenhof marked this conversation as resolved.
Show resolved Hide resolved
1.0
25 changes: 25 additions & 0 deletions agent/util-scripts/gold/pbench-postprocess-tools/test-07.1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
+++ Running test-07.1 pbench-postprocess-tools --group=foobar --dir=/var/tmp/pbench-test-utils/pbench/42-iter/sample42
--- Finished test-07.1 pbench-postprocess-tools (status=0)
iostat: post-processing data
+++ pbench tree state
/var/tmp/pbench-test-utils/pbench
/var/tmp/pbench-test-utils/pbench/42-iter
/var/tmp/pbench-test-utils/pbench/42-iter/sample42
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/csv
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-postprocess.err
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-postprocess.out
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-stdout.txt
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log
/var/tmp/pbench-test-utils/pbench/tmp
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com/foo
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com/iostat
=== /var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com/foo:
--interval=10
=== /var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com/iostat:
--interval=10
--- pbench tree state
14 changes: 7 additions & 7 deletions agent/util-scripts/gold/pbench-postprocess-tools/test-07.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ iostat: post-processing data
/var/tmp/pbench-test-utils/pbench/42-iter
/var/tmp/pbench-test-utils/pbench/42-iter/sample42
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/csv
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-postprocess.err
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-postprocess.out
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-stdout.txt
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat/csv
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat/iostat-postprocess.err
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat/iostat-postprocess.out
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat/iostat-stdout.txt
/var/tmp/pbench-test-utils/pbench/42-iter/sample42/tools-foobar/bar:testhost.example.com/postprocess.log
/var/tmp/pbench-test-utils/pbench/tmp
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar
/var/tmp/pbench-test-utils/pbench/tools-v1-foobar/testhost.example.com
Expand Down
14 changes: 10 additions & 4 deletions agent/util-scripts/pbench-postprocess-tools
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,14 @@ fi
let failures=0
pbench-list-tools --group ${group} --with-option 2>/dev/null | while read hostentry ;do
# Parse an entry from the output of `pbench-list-tools` above.
# The format is: "group: <group>; host: <host>; tools: <tool> <options, <tool> <options>, ...
# The format is: "group: <group>; host: <host> [, label: <label>]; tools: <tool> <options>, <tool> <options>, ...
IFS=';' read group_spec host_spec tools_spec <<<"${hostentry}"
IFS=':' read dummy host junk <<<"${host_spec}"
IFS=',' read hostpart labelpart <<<"${host_spec}"
IFS=':' read dummy host junk <<<"${hostpart}"
IFS=':' read dummy label junk <<<"${labelpart}"
IFS=':' read dummy tools_entry junk <<<"${tools_spec}"
host=${host# *}
label=${label# *}
# Associative array: the keys are the tool names, and the values are the options
declare -A tools=()
IFS=',' read -a otools <<<"${tools_entry# *}"
Expand All @@ -100,8 +103,11 @@ pbench-list-tools --group ${group} --with-option 2>/dev/null | while read hosten
tools[${atool[0]}]=${atool[@]:1}
done

# FIXME: add support for label applied to the hostname directory.
host_tool_output_dir="${tool_output_dir}/${host}"
if [[ -z "${label}" ]] ;then
host_tool_output_dir="${tool_output_dir}/${host}"
else
host_tool_output_dir="${tool_output_dir}/${label}:${host}"
fi
if [[ -d "${host_tool_output_dir}" ]]; then
for tool_name in ${!tools[@]} ;do
if [[ ! -x "${pbench_bin}/tool-scripts/${tool_name}" ]]; then
Expand Down
1 change: 1 addition & 0 deletions ...amples/pbench-postprocess-tools/test-07.1/pbench/tools-v1-foobar/testhost.example.com/foo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--interval=10
1 change: 1 addition & 0 deletions ...les/pbench-postprocess-tools/test-07.1/pbench/tools-v1-foobar/testhost.example.com/iostat
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--interval=10
12 changes: 10 additions & 2 deletions agent/util-scripts/unittests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@ declare -A tools=(
[test-05]="pbench-start-tools"
[test-06]="pbench-stop-tools"
[test-07]="pbench-postprocess-tools"
[test-07.1]="pbench-postprocess-tools"
[test-08]="pbench-kill-tools"
[test-09]="pbench-kill-tools"
[test-10]="pbench-kill-tools"
Expand Down Expand Up @@ -451,6 +452,7 @@ declare -A options=(
[test-05]="--group=default --dir=42-iter/sample42"
[test-06]="--group=default --dir=42-iter/sample42"
[test-07]="--group=foobar --dir=${_testdir}/42-iter/sample42"
[test-07.1]="--group=foobar --dir=${_testdir}/42-iter/sample42"
[test-08]="--group=barfoo --dir=42-iter/sample42"
[test-09]="--group=barfoo"
[test-10]="--dir=barfoo"
Expand Down Expand Up @@ -566,7 +568,10 @@ declare -A expected_status=(
declare -A pre_hooks=(
[test-05]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client'
[test-06]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client; mkdir -p ${_testdir}/42-iter/sample42/tools-default/testhost.example.com/iostat; touch ${_testdir}/42-iter/sample42/tools-default/testhost.example.com/iostat/iostat-stdout.txt'
[test-07]='mkdir -p ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/iostat; touch ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-stdout.txt'
# with labeled host
[test-07]='mkdir -p ${_testdir}/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat; touch ${_testdir}/42-iter/sample42/tools-foobar/bar:testhost.example.com/iostat/iostat-stdout.txt'
# with unlabeled host
[test-07.1]='mkdir -p ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/iostat; touch ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/iostat/iostat-stdout.txt'
[test-11.16]='mkdir ${_testdir}/tmp; (echo foo; echo bar) > ${_testdir}/tmp/good-remote-file'
[test-11.17]='mkdir ${_testdir}/tmp; touch ${_testdir}/tmp/empty-remote-file'
[test-19]='ln -s mock-pbench-tool-meister-client ${_testopt}/unittest-scripts/pbench-tool-meister-client'
Expand Down Expand Up @@ -623,7 +628,10 @@ function sort_tmlogs {
declare -A post_hooks=(
[test-05]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client'
[test-06]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client'
[test-07]='cat ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log >> ${_testout} 2>&1'
# with labeled host
[test-07]='cat ${_testdir}/42-iter/sample42/tools-foobar/bar:testhost.example.com/postprocess.log >> ${_testout} 2>&1'
# with unlabeled host
[test-07.1]='cat ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log >> ${_testout} 2>&1'
[test-19]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client'
[test-53]='sort_testlog; sort_tmlogs; filter_tmerrs'
[test-54]='sed -Ei "s/^optional arguments:/options:/" ${_testout}'
Expand Down
116 changes: 54 additions & 62 deletions lib/pbench/cli/agent/commands/tools/list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,19 @@ def print_results(toolinfo: dict, with_option: bool) -> bool:
"""
printed = False
for group, gval in sorted(toolinfo.items()):
for host, tools in sorted(gval.items()):
for host, hostitems in sorted(gval.items()):
label = hostitems["label"]
host_string = f"host: {host}" + (f", label: {label}" if label else "")
tools = hostitems["tools"]
if tools:
if not with_option:
s = ", ".join(sorted(tools.keys()))
tool_string = ", ".join(sorted(tools.keys()))
else:
tools_with_options = [
tools_with_options = (
dbutenhof marked this conversation as resolved.
Show resolved Hide resolved
f"{t} {' '.join(v)}" for t, v in sorted(tools.items())
]
s = ", ".join(tools_with_options)
print(f"group: {group}; host: {host}; tools: {s}")
)
tool_string = ", ".join(tools_with_options)
print(f"group: {group}; {host_string}; tools: {tool_string}")
printed = True
return printed

Expand All @@ -54,76 +57,65 @@ def execute(self) -> int:
groups = self.groups

opts = self.context.with_option
toolname = self.context.name
found = False
tool_info = {}

if not self.context.name:
for group in groups:
tool_info[group] = {}
try:
tg_dir = self.gen_tools_group_dir(group).glob("*/**")
except BadToolGroup:
self.logger.error("Bad tool group: %s", group)
return 1

for path in tg_dir:
host = path.name
tool_info[group][host] = {}
for tool in sorted(self.tools(path)):
tool_info[group][host][tool] = (
sorted((path / tool).read_text().rstrip("\n").split("\n"))
if opts
else ""
)
for group in groups:
tool_info[group] = {}
try:
tg_dir = self.gen_tools_group_dir(group)
except BadToolGroup:
self.logger.error("Bad tool group: %s", group)
return 1

if tool_info:
found = self.print_results(tool_info, self.context.with_option)
if not found:
msg = "No tools found"
if self.context.group:
msg += f' in group "{self.context.group[0]}"'
self.logger.warn(msg)
else:
self.logger.warn("No tool groups found")
for path in tg_dir.iterdir():
# skip __trigger__ if present
if not path.is_dir():
continue

return 0
host = path.name
tool_info[group][host] = {"label": None, "tools": {}}

else:
# List the groups which include this tool
tool = self.context.name
found = False
for group in groups:
try:
tg_dir = self.gen_tools_group_dir(group)
except BadToolGroup:
self.logger.error("Bad tool group: %s", group)
return 1

tool_info[group] = {}
for path in tg_dir.iterdir():
# skip files like __label__ and __trigger__
if not path.is_dir():
continue

host = path.name
tool_info[group][host] = {}
# Check to see if the tool is in any of the hosts.
if tool in self.tools(path):
tool_info[group][host][tool] = (
sorted((path / tool).read_text().rstrip("\n").split("\n"))
if opts
else ""
)
found = True
toolsdict = tool_info[group][host]["tools"]
if toolname:
# Check if the tool is in any of the hosts.
found = toolname in self.tools(path)
toolslist = [toolname] if found else []
else:
# no tool name was specified
toolslist = sorted(self.tools(path))

label = path / "__label__"
v = label.read_text().rstrip("\n") if label.exists() else None
tool_info[group][host]["label"] = v

for tool in toolslist:
v = (path / tool).read_text().rstrip("\n").split("\n")
toolsdict[tool] = sorted(v) if opts else ""

if toolname:
if found:
self.print_results(tool_info, self.context.with_option)
return 0
else:
msg = f'Tool "{self.context.name}" not found in '
msg = f'Tool "{toolname}" not found in '
msg += self.context.group[0] if self.context.group else "any group"
self.logger.error(msg)
return 1

elif tool_info:
found = self.print_results(tool_info, self.context.with_option)
if not found:
msg = "No tools found"
if self.context.group:
msg += f' in group "{self.context.group[0]}"'
self.logger.warn(msg)
else:
self.logger.warn("No tool groups found")

return 0


def _group_option(f):
"""Group name option"""
Expand Down
44 changes: 44 additions & 0 deletions lib/pbench/test/functional/agent/cli/commands/tools/test_tools_list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,41 @@ def tools_on_multiple_hosts(self, pbench_run):
tool = p / "perf"
tool.write_text("--record-opts='-a --freq=100'")

# Issue #3454
@pytest.fixture
def labels_on_multiple_hosts(self, pbench_run, tools_on_multiple_hosts):
# Note that this fixture depends on `tools_on_multiple_hosts'.
# It embelishes the structure produced by
# `tools_on_multiple_hosts' by adding labels to some host
# entries in the tool-like directory structure that
# `tools_on_multiple_hosts' establishes. Think of it as a
# double for-loop, like the one in `tools_on_multiple_hosts',
# only unrolled.

# row 1
group = "default"

# column 1
host = "th1.example.com"
label = pbench_run / f"tools-v1-{group}" / host / "__label__"
label.write_text("foo")

# column 2
host = "th2.example.com"
label = pbench_run / f"tools-v1-{group}" / host / "__label__"
label.write_text("bar")

# row 2
group = "test"

# column 1
host = "th1.example.com"
label = pbench_run / f"tools-v1-{group}" / host / "__label__"
label.write_text("bar")

# column 2
# th2 has no label

# Issue #2346
def test_existing_group_options(self, single_group_tools, agent_config):
command = ["pbench-list-tools", "--with-option"]
Expand Down Expand Up @@ -309,3 +344,12 @@ def test_multiple_hosts_with_options(self, tools_on_multiple_hosts, agent_config
b"group: default; host: th1.example.com; tools: iostat --interval=30, mpstat --interval=300, sar --interval=10\ngroup: default; host: th2.example.com; tools: iostat --interval=30, mpstat --interval=300, perf --record-opts='-a --freq=100'\ngroup: test; host: th1.example.com; tools: iostat --interval=30, mpstat --interval=300, sar --interval=10\ngroup: test; host: th2.example.com; tools: iostat --interval=30, mpstat --interval=300, perf --record-opts='-a --freq=100'"
in out
)

def test_multiple_hosts_with_labels(self, labels_on_multiple_hosts, agent_config):
command = ["pbench-list-tools", "--with-option"]
out, err, exitcode = pytest.helpers.capture(command)
assert EMPTY == err and exitcode == 0
assert (
b"group: default; host: th1.example.com, label: foo; tools: iostat --interval=30, mpstat --interval=300, sar --interval=10\ngroup: default; host: th2.example.com, label: bar; tools: iostat --interval=30, mpstat --interval=300, perf --record-opts='-a --freq=100'\ngroup: test; host: th1.example.com, label: bar; tools: iostat --interval=30, mpstat --interval=300, sar --interval=10\ngroup: test; host: th2.example.com; tools: iostat --interval=30, mpstat --interval=300, perf --record-opts='-a --freq=100'"
in out
)