From 60ee88a132d68540195a34301c23c3e7cf01067a Mon Sep 17 00:00:00 2001 From: Ranlvor Date: Fri, 23 Oct 2015 17:21:51 +0200 Subject: [PATCH 1/5] check: run all hosts in parallel This change chops the runtime from 6m23.125s to 12.210s realtime --- scripts/check | 100 +++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/scripts/check b/scripts/check index 988b7bb..6e08828 100755 --- a/scripts/check +++ b/scripts/check @@ -5,10 +5,9 @@ import os import socket import subprocess import nmap +import concurrent.futures from optparse import OptionParser - -ip4, ip6 = 0, 0 - +from functools import reduce ANSI_COLOR_ERR = "\x1b[31m" ANSI_COLOR_WARN = "\x1b[33m" @@ -16,19 +15,22 @@ ANSI_COLOR_OK = "\x1b[32m" ANSI_COLOR_RESET = "\x1b[0m" -def error(*arg): - print(ANSI_COLOR_ERR, *arg, file=sys.stderr, - end='%s\n' % ANSI_COLOR_RESET) +def error(arg): + printToResult(ANSI_COLOR_ERR + " " + arg) + +def warn(arg): + printToResult(ANSI_COLOR_WARN + " " + arg) -def warn(*arg): - print(ANSI_COLOR_WARN, *arg, file=sys.stderr, - end='%s\n' % ANSI_COLOR_RESET) +def ok(arg): + printToResult(ANSI_COLOR_OK + " " + arg) -def ok(*arg): - print(ANSI_COLOR_OK, *arg, file=sys.stderr, - end='%s\n' % ANSI_COLOR_RESET) + +def printToResult(arg): + global output + output += str(arg) + output += '%s\n' % ANSI_COLOR_RESET def check_host_lookup(hostname, port): @@ -143,44 +145,58 @@ def get_hosts_data(srcdir): yield(dict(community=fname, addresses=addresses)) -def do_checks(srcdir): - global ip4, ip6 +def do_checks_host(host): + global ip4, ip6, output errcnt = 0 warncnt = 0 + ip4 = 0 + ip6 = 0 + output = "" + + printToResult("Checking {community}".format(community=host['community'])) + if not host['addresses']: + warn("no addresses specified") + warncnt += 1 + for address in host['addresses']: + host, port = address + + # dns lookup + records = check_host_lookup(host, port) + if not records: + errcnt += 1 + else: + for record in records: + if record[1] is not socket.SOCK_DGRAM: + # vpn connections are udp based, so skip + # everything else + continue + + if not check_icmp_reachability(record): + errcnt += 1 + else: + port_state = check_udp_reachability(record) + if not port_state: + errcnt += 1 - for host in get_hosts_data(srcdir): - print("Checking {community}".format(community=host['community'])) - if not host['addresses']: - warn("no addresses specified") - warncnt += 1 - for address in host['addresses']: - host, port = address - - # dns lookup - records = check_host_lookup(host, port) - if not records: - errcnt += 1 - else: - for record in records: - if record[1] is not socket.SOCK_DGRAM: - # vpn connections are udp based, so skip - # everything else - continue + return (errcnt, warncnt, ip4, ip6, output) - if not check_icmp_reachability(record): - errcnt += 1 - else: - port_state = check_udp_reachability(record) - if not port_state: - errcnt += 1 - print("\nfound {}/{} working ipv4/ipv6 peers".format(ip4, ip6)) +def do_reduce_results(a, b): + print (a[4], end="") + print (b[4], end="") + return (a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3], "") - error("{} errors".format(errcnt)) - warn("{} warnings".format(warncnt)) - return 0 if errcnt == 0 else 1 +def do_checks(srcdir): + with concurrent.futures.ProcessPoolExecutor(max_workers=200) as executor: + results = executor.map(do_checks_host, get_hosts_data(srcdir)) + result = reduce(do_reduce_results, results) + + print("\nfound {}/{} working ipv4/ipv6 peers".format(result[2], result[3])) + print("{}{} errors{}".format(ANSI_COLOR_ERR, result[0], ANSI_COLOR_RESET)) + print("{}{} warnings{}".format(ANSI_COLOR_WARN, result[1], ANSI_COLOR_RESET)) + return 0 if result[0] == 0 else 1 if __name__ == "__main__": From 37f4f14e38e627c2d01e337e1572b7ba7addb9db Mon Sep 17 00:00:00 2001 From: Ranlvor Date: Sun, 24 Jan 2016 22:31:10 +0100 Subject: [PATCH 2/5] check: fixed some minor style-issues --- scripts/check | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/check b/scripts/check index 6e08828..eac1197 100755 --- a/scripts/check +++ b/scripts/check @@ -16,21 +16,21 @@ ANSI_COLOR_RESET = "\x1b[0m" def error(arg): - printToResult(ANSI_COLOR_ERR + " " + arg) + print_to_result(ANSI_COLOR_ERR + " " + arg) def warn(arg): - printToResult(ANSI_COLOR_WARN + " " + arg) + print_to_result(ANSI_COLOR_WARN + " " + arg) def ok(arg): - printToResult(ANSI_COLOR_OK + " " + arg) + print_to_result(ANSI_COLOR_OK + " " + arg) -def printToResult(arg): +def print_to_result(arg): global output output += str(arg) - output += '%s\n' % ANSI_COLOR_RESET + output += '{}\n'.format(ANSI_COLOR_RESET) def check_host_lookup(hostname, port): @@ -154,7 +154,7 @@ def do_checks_host(host): ip6 = 0 output = "" - printToResult("Checking {community}".format(community=host['community'])) + print_to_result("Checking {community}".format(community=host['community'])) if not host['addresses']: warn("no addresses specified") warncnt += 1 @@ -182,14 +182,13 @@ def do_checks_host(host): return (errcnt, warncnt, ip4, ip6, output) -def do_reduce_results(a, b): - print (a[4], end="") - print (b[4], end="") - return (a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3], "") - - def do_checks(srcdir): with concurrent.futures.ProcessPoolExecutor(max_workers=200) as executor: + def do_reduce_results(first, second): + print(first[4], end="") + print(second[4], end="") + return (first[0] + second[0], first[1] + second[1], first[2] + second[2], first[3] + second[3], "") + results = executor.map(do_checks_host, get_hosts_data(srcdir)) result = reduce(do_reduce_results, results) From b026ac2332198a5a324f5cda2e4b9059bfafeb6d Mon Sep 17 00:00:00 2001 From: Ranlvor Date: Sun, 24 Jan 2016 22:47:52 +0100 Subject: [PATCH 3/5] check: Made output a non-global list --- scripts/check | 65 ++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/scripts/check b/scripts/check index eac1197..ae40680 100755 --- a/scripts/check +++ b/scripts/check @@ -15,33 +15,31 @@ ANSI_COLOR_OK = "\x1b[32m" ANSI_COLOR_RESET = "\x1b[0m" -def error(arg): - print_to_result(ANSI_COLOR_ERR + " " + arg) +def error(arg, output): + print_to_result(ANSI_COLOR_ERR + " " + arg, output) -def warn(arg): - print_to_result(ANSI_COLOR_WARN + " " + arg) +def warn(arg, output): + print_to_result(ANSI_COLOR_WARN + " " + arg, output) -def ok(arg): - print_to_result(ANSI_COLOR_OK + " " + arg) +def ok(arg, output): + print_to_result(ANSI_COLOR_OK + " " + arg, output) -def print_to_result(arg): - global output - output += str(arg) - output += '{}\n'.format(ANSI_COLOR_RESET) +def print_to_result(arg, output): + output.append(str(arg)) -def check_host_lookup(hostname, port): +def check_host_lookup(hostname, port, output): try: return socket.getaddrinfo(hostname, port) except Exception: - error("DNS Lookup for {hostname} failed".format(hostname=hostname)) + error("DNS Lookup for {hostname} failed".format(hostname=hostname), output) return [] -def check_icmp_reachability(gai_record): +def check_icmp_reachability(gai_record, output): host = gai_record[4][0] family = gai_record[0] @@ -53,11 +51,11 @@ def check_icmp_reachability(gai_record): stdout=subprocess.PIPE) child.communicate() if child.returncode: - error("{host} is icmp unreachable".format(host=host)) + error("{host} is icmp unreachable".format(host=host), output) return True if child.returncode == 0 else False -def check_udp_reachability(gai_record): +def check_udp_reachability(gai_record, output): global ip4, ip6 host, port = gai_record[4][:2] @@ -74,10 +72,10 @@ def check_udp_reachability(gai_record): if state == 'closed': error("{host} port {port}/udp is {state}" - .format(host=host, port=port, state=state)) + .format(host=host, port=port, state=state), output) else: ok("{host} port {port}/udp is {state}" - .format(host=host, port=port, state=state)) + .format(host=host, port=port, state=state), output) if family is socket.AddressFamily.AF_INET: ip4 += 1 @@ -98,6 +96,7 @@ def get_hosts_data(srcdir): ignore_key = False addresses = [] port = 655 # tinc default port + output = list() for line in f.readlines(): @@ -117,24 +116,24 @@ def get_hosts_data(srcdir): try: port = int(v) except ValueError: - error("non-integer default port given") + error("non-integer default port given", output) elif k == "address": if " " in v: parts = v.split(' ') if len(parts) != 2: - error("unknown address format") + error("unknown address format", output) try: int(parts[1]) addresses.append(parts) except ValueError: - error("non-integer port given") + error("non-integer port given", output) else: addresses.append((v, None)) elif k in ('ecdsapublickey', 'ed25519publickey'): continue else: error("unknown key {key} with value {val}" - .format(key=k, val=v)) + .format(key=k, val=v), output) # set explicit port for address/port pairs for i, addr in enumerate(addresses): @@ -142,27 +141,27 @@ def get_hosts_data(srcdir): item = (addr[0], port) addresses[i] = item - yield(dict(community=fname, addresses=addresses)) + yield(dict(community=fname, addresses=addresses, output=output)) def do_checks_host(host): - global ip4, ip6, output + global ip4, ip6 errcnt = 0 warncnt = 0 ip4 = 0 ip6 = 0 - output = "" + output = host['output'] - print_to_result("Checking {community}".format(community=host['community'])) + print_to_result("Checking {community}".format(community=host['community']), output) if not host['addresses']: - warn("no addresses specified") + warn("no addresses specified", output) warncnt += 1 for address in host['addresses']: host, port = address # dns lookup - records = check_host_lookup(host, port) + records = check_host_lookup(host, port, output) if not records: errcnt += 1 else: @@ -172,22 +171,24 @@ def do_checks_host(host): # everything else continue - if not check_icmp_reachability(record): + if not check_icmp_reachability(record, output): errcnt += 1 else: - port_state = check_udp_reachability(record) + port_state = check_udp_reachability(record, output) if not port_state: errcnt += 1 + output.append("") #we want a linebreak after each host return (errcnt, warncnt, ip4, ip6, output) def do_checks(srcdir): with concurrent.futures.ProcessPoolExecutor(max_workers=200) as executor: def do_reduce_results(first, second): - print(first[4], end="") - print(second[4], end="") - return (first[0] + second[0], first[1] + second[1], first[2] + second[2], first[3] + second[3], "") + linebreak = '{}\n'.format(ANSI_COLOR_RESET) + print(linebreak.join(first[4]), end="") + print(linebreak.join(second[4]), end="") + return (first[0] + second[0], first[1] + second[1], first[2] + second[2], first[3] + second[3], list()) results = executor.map(do_checks_host, get_hosts_data(srcdir)) result = reduce(do_reduce_results, results) From 243f8a8e0baff87f444899be5c0abb641e6b5e69 Mon Sep 17 00:00:00 2001 From: Ranlvor Date: Sun, 24 Jan 2016 22:53:58 +0100 Subject: [PATCH 4/5] check: made ip4 and ip6 non-global --- scripts/check | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/check b/scripts/check index ae40680..ba8a6fd 100755 --- a/scripts/check +++ b/scripts/check @@ -55,9 +55,7 @@ def check_icmp_reachability(gai_record, output): return True if child.returncode == 0 else False -def check_udp_reachability(gai_record, output): - global ip4, ip6 - +def check_udp_reachability(gai_record, output, ip4, ip6): host, port = gai_record[4][:2] family = gai_record[0] @@ -82,7 +80,7 @@ def check_udp_reachability(gai_record, output): else: ip6 += 1 - return False if state == 'closed' else True + return (False, ip4, ip6) if state == 'closed' else (True, ip4, ip6) def get_hosts_data(srcdir): @@ -145,8 +143,6 @@ def get_hosts_data(srcdir): def do_checks_host(host): - global ip4, ip6 - errcnt = 0 warncnt = 0 ip4 = 0 @@ -174,7 +170,7 @@ def do_checks_host(host): if not check_icmp_reachability(record, output): errcnt += 1 else: - port_state = check_udp_reachability(record, output) + port_state, ip4, ip6 = check_udp_reachability(record, output, ip4, ip6) if not port_state: errcnt += 1 From e9d73d78da0dcec2734132fc2bc7b6d7bb0d6fb6 Mon Sep 17 00:00:00 2001 From: Ranlvor Date: Sun, 24 Jan 2016 22:59:01 +0100 Subject: [PATCH 5/5] check: made result-tuples less confusing by using dicts --- scripts/check | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/check b/scripts/check index ba8a6fd..3d17d14 100755 --- a/scripts/check +++ b/scripts/check @@ -175,24 +175,25 @@ def do_checks_host(host): errcnt += 1 output.append("") #we want a linebreak after each host - return (errcnt, warncnt, ip4, ip6, output) + return dict(errcnt=errcnt, warncnt=warncnt, ip4=ip4, ip6=ip6, output=output) def do_checks(srcdir): with concurrent.futures.ProcessPoolExecutor(max_workers=200) as executor: def do_reduce_results(first, second): linebreak = '{}\n'.format(ANSI_COLOR_RESET) - print(linebreak.join(first[4]), end="") - print(linebreak.join(second[4]), end="") - return (first[0] + second[0], first[1] + second[1], first[2] + second[2], first[3] + second[3], list()) + print(linebreak.join(first['output']), end="") + print(linebreak.join(second['output']), end="") + return dict(errcnt=first['errcnt'] + second['errcnt'], warncnt=first['warncnt'] + second['warncnt'], + ip4=first['ip4'] + second['ip4'], ip6=first['ip6'] + second['ip6'], output=list()) results = executor.map(do_checks_host, get_hosts_data(srcdir)) result = reduce(do_reduce_results, results) - print("\nfound {}/{} working ipv4/ipv6 peers".format(result[2], result[3])) - print("{}{} errors{}".format(ANSI_COLOR_ERR, result[0], ANSI_COLOR_RESET)) - print("{}{} warnings{}".format(ANSI_COLOR_WARN, result[1], ANSI_COLOR_RESET)) - return 0 if result[0] == 0 else 1 + print("\nfound {}/{} working ipv4/ipv6 peers".format(result['ip4'], result['ip6'])) + print("{}{} errors{}".format(ANSI_COLOR_ERR, result['errcnt'], ANSI_COLOR_RESET)) + print("{}{} warnings{}".format(ANSI_COLOR_WARN, result['warncnt'], ANSI_COLOR_RESET)) + return 0 if result['errcnt'] == 0 else 1 if __name__ == "__main__":