From fa36c8c38794660d828964b980885a99db36a87c Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 17:56:38 +0000 Subject: [PATCH 01/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 198 6380 --- scripts/sendGraph.py | 114 +++++++++++++ scripts/updateGraph.py | 376 +++++++++++++++++++++++++++++++++++++++++ web/graphData.py | 19 +++ web/graphPlotter.py | 35 ++++ web/updateGraph.py | 16 ++ web/web.py | 24 +++ 6 files changed, 584 insertions(+) create mode 100644 scripts/updateGraph.py diff --git a/scripts/sendGraph.py b/scripts/sendGraph.py index e7ae3bc..4b8c3a0 100755 --- a/scripts/sendGraph.py +++ b/scripts/sendGraph.py @@ -1,97 +1,157 @@ #!/usr/bin/env python3 + # Based on Kyrias' sendGraph script. Requires Python 3, requests and cjdns. + # You can install them using pip: pip3 install cjdns requests + ############################################################################### + # CONFIG + + # URL where data is sent + # www.fc00.org for clearnet access + # h.fc00.org for hyperboria + # [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access + url = 'http://www.fc00.org/sendGraph' + + # update your email address, so I can contact you in case something goes wrong + your_mail = 'your@email.here' + + + # ---------------------- + # RPC connection details + # ---------------------- + + # If this is set to True connection details will be loaded from ~/.cjdnsadmin + cjdns_use_default = True + + # otherwise these are used. + cjdns_ip = '127.0.0.1' + cjdns_port = 11234 + cjdns_password = 'NONE' + + ############################################################################### + + import sys + import traceback + import json + import argparse + + import requests + + import cjdns + from cjdns import key_utils + from cjdns import admin_tools + + import queue + import threading + + def main(): + parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') parser.add_argument('-v', '--verbose', help='increase output verbosity', dest='verbose', action='store_true') parser.set_defaults(verbose=False) args = parser.parse_args() + con = connect() + nodes = dump_node_store(con) edges = {} + get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) + for k in nodes: get_peer_queue.put(k) + for i in range(8): t = threading.Thread(target=worker, args=[nodes, get_peer_queue, result_queue, args.verbose]) t.daemon = True t.start() + for i in range(len(nodes)): peers, node_ip = result_queue.get() get_edges_for_peers(edges, peers, node_ip) + send_graph(nodes, edges) sys.exit(0) + def worker(nodes, get_peer_queue, result, verbose=False): + con = connect() + while True: try: k = get_peer_queue.get_nowait() except queue.Empty: return + node = nodes[k] if verbose: print('fetch', node) node_ip = node['ip'] + peers = get_all_peers(con, node['path']) + result.put((peers, node_ip)) + def connect(): + try: if cjdns_use_default: print('Connecting using default or ~/.cjdnsadmin credentials...') @@ -100,28 +160,37 @@ def connect(): print('Connecting to port {:d}...'.format(cjdns_port)) con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) + return con + except: print('Connection failed!') print(traceback.format_exc()) sys.exit(1) + + def dump_node_store(con): + nodes = dict() + i = 0 while True: res = con.NodeStore_dumpTable(i) + if not 'routingTable' in res: break + for n in res['routingTable']: if not all(key in n for key in ('addr', 'path', 'ip')): continue + ip = n['ip'] path = n['path'] addr = n['addr'] @@ -129,21 +198,29 @@ def dump_node_store(con): if 'version' in n: version = n['version'] + nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} + if not 'more' in res or res['more'] != 1: break + i += 1 + return nodes + + def get_peers(con, path, nearbyPath=''): + formatted_path = path if nearbyPath: formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) + i = 1 retry = 2 while i < retry + 1: @@ -153,11 +230,14 @@ def get_peers(con, path, nearbyPath=''): res = con.RouterModule_getPeers(path) + + if res['error'] == 'not_found': print('get_peers: node with path {:s} not found, skipping.' .format(formatted_path)) return [] + elif res['error'] != 'none': print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' .format(res['error'], formatted_path, retry-i)) @@ -167,54 +247,72 @@ def get_peers(con, path, nearbyPath=''): else: return res['peers'] + i += 1 + print('get_peers: failed on final try, skipping {:s}' .format(formatted_path)) return [] + + def get_all_peers(con, path): + peers = set() keys = set() + res = get_peers(con, path) peers.update(res) + if not res: return keys + last_peer = res[-1] checked_paths = set() + while len(res) > 1: last_path = (last_peer.split('.', 1)[1] .rsplit('.', 2)[0]) + if last_path in checked_paths: break else: checked_paths.add(last_path) + res = get_peers(con, path, last_path) if res: last_peer = res[-1] else: break + peers.update(res) + for peer in peers: key = peer.split('.', 5)[-1] keys |= {key} + return keys + + def get_edges_for_peers(edges, peers, node_ip): + for peer_key in peers: peer_ip = key_utils.to_ipv6(peer_key) + if node_ip > peer_ip: A = node_ip B = peer_ip @@ -222,41 +320,57 @@ def get_edges_for_peers(edges, peers, node_ip): A = peer_ip B = node_ip + edge = { 'a': A, 'b': B } + if A not in edges: edges[A] = [] + if not([True for edge in edges[A] if edge['b'] == B]): edges[A] += [edge] + + def send_graph(nodes, edges): + graph = { 'nodes': [], 'edges': [edge for sublist in edges.values() for edge in sublist], } + for node in nodes.values(): graph['nodes'].append({ 'ip': node['ip'], 'version': node['version'], }) + print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) + + from pprint import pprint + pprint(graph) + return json_graph = json.dumps(graph) print('Sending data to {:s}...'.format(url)) + payload = {'data': json_graph, 'mail': your_mail, 'version': 2} r = requests.post(url, data=payload) + if r.text == 'OK': print('Done!') else: print('{:s}'.format(r.text)) + if __name__ == '__main__': + main() diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py new file mode 100644 index 0000000..4b8c3a0 --- /dev/null +++ b/scripts/updateGraph.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 + +# Based on Kyrias' sendGraph script. Requires Python 3, requests and cjdns. + +# You can install them using pip: pip3 install cjdns requests + +############################################################################### + +# CONFIG + + + +# URL where data is sent + +# www.fc00.org for clearnet access + +# h.fc00.org for hyperboria + +# [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access + +url = 'http://www.fc00.org/sendGraph' + + + +# update your email address, so I can contact you in case something goes wrong + +your_mail = 'your@email.here' + + + + + +# ---------------------- + +# RPC connection details + +# ---------------------- + + + +# If this is set to True connection details will be loaded from ~/.cjdnsadmin + +cjdns_use_default = True + + + +# otherwise these are used. + +cjdns_ip = '127.0.0.1' + +cjdns_port = 11234 + +cjdns_password = 'NONE' + + + +############################################################################### + + + +import sys + +import traceback + +import json + +import argparse + + + +import requests + + + +import cjdns + +from cjdns import key_utils + +from cjdns import admin_tools + + + +import queue + +import threading + + + +def main(): + + parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') + parser.add_argument('-v', '--verbose', help='increase output verbosity', + dest='verbose', action='store_true') + parser.set_defaults(verbose=False) + args = parser.parse_args() + + + con = connect() + + + nodes = dump_node_store(con) + edges = {} + + + get_peer_queue = queue.Queue(0) + result_queue = queue.Queue(0) + + + for k in nodes: + get_peer_queue.put(k) + + + for i in range(8): + t = threading.Thread(target=worker, args=[nodes, get_peer_queue, result_queue, + args.verbose]) + t.daemon = True + t.start() + + + for i in range(len(nodes)): + peers, node_ip = result_queue.get() + get_edges_for_peers(edges, peers, node_ip) + + + send_graph(nodes, edges) + sys.exit(0) + + +def worker(nodes, get_peer_queue, result, verbose=False): + + con = connect() + + + while True: + try: + k = get_peer_queue.get_nowait() + except queue.Empty: + return + + + node = nodes[k] + if verbose: + print('fetch', node) + node_ip = node['ip'] + + + peers = get_all_peers(con, node['path']) + + + result.put((peers, node_ip)) + + +def connect(): + + try: + if cjdns_use_default: + print('Connecting using default or ~/.cjdnsadmin credentials...') + con = cjdns.connectWithAdminInfo() + else: + print('Connecting to port {:d}...'.format(cjdns_port)) + con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) + + + return con + + + except: + print('Connection failed!') + print(traceback.format_exc()) + sys.exit(1) + + + + +def dump_node_store(con): + + nodes = dict() + + + i = 0 + while True: + res = con.NodeStore_dumpTable(i) + + + if not 'routingTable' in res: + break + + + for n in res['routingTable']: + if not all(key in n for key in ('addr', 'path', 'ip')): + continue + + + ip = n['ip'] + path = n['path'] + addr = n['addr'] + version = None + if 'version' in n: + version = n['version'] + + + nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} + + + if not 'more' in res or res['more'] != 1: + break + + + i += 1 + + + return nodes + + + + +def get_peers(con, path, nearbyPath=''): + + formatted_path = path + if nearbyPath: + formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) + + + i = 1 + retry = 2 + while i < retry + 1: + if nearbyPath: + res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) + else: + res = con.RouterModule_getPeers(path) + + + + + if res['error'] == 'not_found': + print('get_peers: node with path {:s} not found, skipping.' + .format(formatted_path)) + return [] + + + elif res['error'] != 'none': + print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' + .format(res['error'], formatted_path, retry-i)) + elif res['result'] == 'timeout': + print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' + .format(formatted_path, retry-i)) + else: + return res['peers'] + + + i += 1 + + + print('get_peers: failed on final try, skipping {:s}' + .format(formatted_path)) + return [] + + + + +def get_all_peers(con, path): + + peers = set() + keys = set() + + + res = get_peers(con, path) + peers.update(res) + + + if not res: + return keys + + + last_peer = res[-1] + checked_paths = set() + + + while len(res) > 1: + last_path = (last_peer.split('.', 1)[1] + .rsplit('.', 2)[0]) + + + if last_path in checked_paths: + break + else: + checked_paths.add(last_path) + + + res = get_peers(con, path, last_path) + if res: + last_peer = res[-1] + else: + break + + + peers.update(res) + + + for peer in peers: + key = peer.split('.', 5)[-1] + keys |= {key} + + + return keys + + + + +def get_edges_for_peers(edges, peers, node_ip): + + for peer_key in peers: + peer_ip = key_utils.to_ipv6(peer_key) + + + if node_ip > peer_ip: + A = node_ip + B = peer_ip + else: + A = peer_ip + B = node_ip + + + edge = { 'a': A, + 'b': B } + + + if A not in edges: + edges[A] = [] + + + if not([True for edge in edges[A] if edge['b'] == B]): + edges[A] += [edge] + + + + +def send_graph(nodes, edges): + + graph = { + 'nodes': [], + 'edges': [edge for sublist in edges.values() + for edge in sublist], + } + + + for node in nodes.values(): + graph['nodes'].append({ + 'ip': node['ip'], + 'version': node['version'], + }) + + + print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) + + + from pprint import pprint + pprint(graph) + return + json_graph = json.dumps(graph) + print('Sending data to {:s}...'.format(url)) + + + payload = {'data': json_graph, 'mail': your_mail, 'version': 2} + r = requests.post(url, data=payload) + + + if r.text == 'OK': + print('Done!') + else: + print('{:s}'.format(r.text)) + + +if __name__ == '__main__': + + main() diff --git a/web/graphData.py b/web/graphData.py index aa9a2dd..e529767 100644 --- a/web/graphData.py +++ b/web/graphData.py @@ -1,31 +1,44 @@ import json + from database import NodeDB + from graph import Node, Edge + import traceback + import time + + def insert_graph_data(config, data, mail, ip, version): + try: graph_data = json.loads(data) except ValueError: return 'Invalid JSON' + log = '[%s] ip: %s, version: %d, mail: %r, nodes: %d, edges: %d' % ( time.strftime('%Y-%m-%d %H:%M:%S'), ip, version, mail, len(graph_data['nodes']), len(graph_data['edges'])) + with open(config['LOG'], 'a') as f: f.write(log + '\n') + if mail == 'your@email.here': return 'Please change email address in config.' + if version != 2: return 'You are using outdated version of sendGraph script. Get new version from https://github.com/zielmicha/fc00.org/blob/master/scripts/sendGraph.py' + nodes = dict() edges = [] + try: for n in graph_data['nodes']: try: @@ -34,6 +47,7 @@ def insert_graph_data(config, data, mail, ip, version): except Exception: pass + for e in graph_data['edges']: try: edge = Edge(nodes[e['a']], nodes[e['b']]) @@ -43,13 +57,17 @@ def insert_graph_data(config, data, mail, ip, version): except Exception: return 'Invalid JSON nodes' + print "Accepted %d nodes and %d links." % (len(nodes), len(edges)) + if len(nodes) == 0 or len(edges) == 0: return 'No valid nodes or edges' + uploaded_by = ip + try: with NodeDB(config) as db: db.insert_graph(nodes, edges, uploaded_by) @@ -57,4 +75,5 @@ def insert_graph_data(config, data, mail, ip, version): traceback.print_exc() return 'Database failure' + return None diff --git a/web/graphPlotter.py b/web/graphPlotter.py index ff48d44..d438202 100644 --- a/web/graphPlotter.py +++ b/web/graphPlotter.py @@ -1,44 +1,66 @@ import pygraphviz as pgv + import time + import json + import networkx as nx + from networkx.algorithms import centrality + + def position_nodes(nodes, edges): + G = pgv.AGraph(strict=True, directed=False, size='10!') + for n in nodes.values(): G.add_node(n.ip, label=n.label, version=n.version) + for e in edges: G.add_edge(e.a.ip, e.b.ip, len=1.0) + G.layout(prog='neato', args='-Gepsilon=0.0001 -Gmaxiter=100000') + return G + def compute_betweenness(G): + ng = nx.Graph() for start in G.iternodes(): others = G.neighbors(start) for other in others: ng.add_edge(start, other) + c = centrality.betweenness_centrality(ng) + for k, v in c.items(): c[k] = v + return c + def canonalize_ip(ip): + return ':'.join( i.rjust(4, '0') for i in ip.split(':') ) + def load_db(): + with open('nodedb/nodes') as f: return dict([ (canonalize_ip(v[0]), v[1]) for v in [ l.split(None)[:2] for l in f.readlines() ] if len(v) > 1 ]) + def get_graph_json(G): + max_neighbors = 1 for n in G.iternodes(): neighbors = len(G.neighbors(n)) @@ -46,15 +68,18 @@ def get_graph_json(G): max_neighbors = neighbors print 'Max neighbors: %d' % max_neighbors + out_data = { 'created': int(time.time()), 'nodes': [], 'edges': [] } + centralities = compute_betweenness(G) db = load_db() + for n in G.iternodes(): neighbor_ratio = len(G.neighbors(n)) / float(max_neighbors) pos = n.attr['pos'].split(',', 1) @@ -63,6 +88,7 @@ def get_graph_json(G): size = (pcentrality ** 0.3 / 500) * 1000 + 1 name = db.get(n.name) + out_data['nodes'].append({ 'id': n.name, 'label': name if name else n.attr['label'], @@ -75,26 +101,35 @@ def get_graph_json(G): 'centrality': '%.4f' % centrality }) + for e in G.iteredges(): out_data['edges'].append({ 'sourceID': e[0], 'targetID': e[1] }) + return json.dumps(out_data) + + def _gradient_color(ratio, colors): + jump = 1.0 / (len(colors) - 1) gap_num = int(ratio / (jump + 0.0000001)) + a = colors[gap_num] b = colors[gap_num + 1] + ratio = (ratio - gap_num * jump) * (len(colors) - 1) + r = a[0] + (b[0] - a[0]) * ratio g = a[1] + (b[1] - a[1]) * ratio b = a[2] + (b[2] - a[2]) * ratio + return '#%02x%02x%02x' % (r, g, b) diff --git a/web/updateGraph.py b/web/updateGraph.py index 1777863..218ad95 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -1,29 +1,45 @@ #!/usr/bin/env python + from flask import Config + from database import NodeDB + import graphPlotter + + + def generate_graph(time_limit=60*60*3): + nodes, edges = load_graph_from_db(time_limit) print '%d nodes, %d edges' % (len(nodes), len(edges)) + graph = graphPlotter.position_nodes(nodes, edges) json = graphPlotter.get_graph_json(graph) + with open('static/graph.json', 'w') as f: f.write(json) + + def load_graph_from_db(time_limit): + config = Config('./') config.from_pyfile('web_config.cfg') + with NodeDB(config) as db: nodes = db.get_nodes(time_limit) edges = db.get_edges(nodes, 60*60*24*7) return (nodes, edges) + + if __name__ == '__main__': + generate_graph() diff --git a/web/web.py b/web/web.py index 3eea61b..114d3e7 100644 --- a/web/web.py +++ b/web/web.py @@ -1,10 +1,17 @@ from flask import Flask, render_template, request + from graphData import insert_graph_data + + app = Flask(__name__) + app.config.from_pyfile('web_config.cfg') + + def get_ip(): + try: ip = request.headers['x-real-ip'] except KeyError: @@ -13,22 +20,36 @@ def get_ip(): ip = request.headers['x-atomshare-real-ip'] return ip + @app.context_processor + def add_ip(): + return dict(ip=get_ip()) + + @app.route('/') + @app.route('/network') + def page_network(): + return render_template('network.html', page='network') + @app.route('/about') + def page_about(): + return render_template('about.html', page='about') + @app.route('/sendGraph', methods=['POST']) + def page_sendGraph(): + print "Receiving graph from %s" % (request.remote_addr) data = request.form['data'] @@ -36,10 +57,13 @@ def page_sendGraph(): version = int(request.form.get('version', '1')) ret = insert_graph_data(ip=get_ip(), config=app.config, data=data, mail=mail, version=version) + if ret == None: return 'OK' else: return 'Error: %s' % ret + if __name__ == '__main__': + app.run(host='localhost', port=3000) From 5d772e18b8d81ce275c89eb2a9d0ee448b9ec5a4 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 18:34:22 +0000 Subject: [PATCH 02/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 50 1097 --- scripts/updateGraph.py | 73 +++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index 4b8c3a0..6c07bce 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -12,11 +12,11 @@ # URL where data is sent -# www.fc00.org for clearnet access +# www.fc00.org for clearnet access -# h.fc00.org for hyperboria +# h.fc00.org for hyperboria -# [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access +# [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access url = 'http://www.fc00.org/sendGraph' @@ -46,9 +46,9 @@ # otherwise these are used. -cjdns_ip = '127.0.0.1' +cjdns_ip = '127.0.0.1' -cjdns_port = 11234 +cjdns_port = 11234 cjdns_password = 'NONE' @@ -57,6 +57,7 @@ ############################################################################### +from pprint import pprint import sys @@ -82,6 +83,7 @@ import queue +from concurrent.futures import ThreadPoolExecutor import threading @@ -105,50 +107,28 @@ def main(): get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) + e = ThreadPoolExecutor(max_workers=1) - for k in nodes: - get_peer_queue.put(k) - - - for i in range(8): - t = threading.Thread(target=worker, args=[nodes, get_peer_queue, result_queue, - args.verbose]) - t.daemon = True - t.start() - - - for i in range(len(nodes)): - peers, node_ip = result_queue.get() + args = zip((node['ip'],node['path']) for node in nodes.values()) + args = list(args) + print(args) + + for peers, node_ip in e.map(get_peers_derp, *args): get_edges_for_peers(edges, peers, node_ip) - send_graph(nodes, edges) sys.exit(0) -def worker(nodes, get_peer_queue, result, verbose=False): - - con = connect() - - - while True: - try: - k = get_peer_queue.get_nowait() - except queue.Empty: - return - - - node = nodes[k] - if verbose: - print('fetch', node) - node_ip = node['ip'] - - - peers = get_all_peers(con, node['path']) - - - result.put((peers, node_ip)) - +def con(): + con = threading.local.get('con') + if con is None: + con = connect() + threading.local['con'] = con + return con + +def get_peers_derp(ip,path): + return get_all_peers(con, path), ip def connect(): @@ -229,7 +209,8 @@ def get_peers(con, path, nearbyPath=''): else: res = con.RouterModule_getPeers(path) - + pprint((path,nearbyPath,res)) + raise RuntimeError('boop') if res['error'] == 'not_found': @@ -275,7 +256,6 @@ def get_all_peers(con, path): last_peer = res[-1] checked_paths = set() - while len(res) > 1: last_path = (last_peer.split('.', 1)[1] .rsplit('.', 2)[0]) @@ -340,13 +320,13 @@ def send_graph(nodes, edges): graph = { 'nodes': [], 'edges': [edge for sublist in edges.values() - for edge in sublist], + for edge in sublist], } for node in nodes.values(): graph['nodes'].append({ - 'ip': node['ip'], + 'ip': node['ip'], 'version': node['version'], }) @@ -354,7 +334,6 @@ def send_graph(nodes, edges): print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) - from pprint import pprint pprint(graph) return json_graph = json.dumps(graph) From facbb8fab5249f37819ec0a88dfe0e8c00d7336f Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 19:03:21 +0000 Subject: [PATCH 03/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 260 6611 --- scripts/sendGraph.py | 114 ------------- scripts/updateGraph.py | 366 ++++++++++++++++++++++------------------- 2 files changed, 197 insertions(+), 283 deletions(-) diff --git a/scripts/sendGraph.py b/scripts/sendGraph.py index 4b8c3a0..e7ae3bc 100755 --- a/scripts/sendGraph.py +++ b/scripts/sendGraph.py @@ -1,157 +1,97 @@ #!/usr/bin/env python3 - # Based on Kyrias' sendGraph script. Requires Python 3, requests and cjdns. - # You can install them using pip: pip3 install cjdns requests - ############################################################################### - # CONFIG - - # URL where data is sent - # www.fc00.org for clearnet access - # h.fc00.org for hyperboria - # [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access - url = 'http://www.fc00.org/sendGraph' - - # update your email address, so I can contact you in case something goes wrong - your_mail = 'your@email.here' - - - # ---------------------- - # RPC connection details - # ---------------------- - - # If this is set to True connection details will be loaded from ~/.cjdnsadmin - cjdns_use_default = True - - # otherwise these are used. - cjdns_ip = '127.0.0.1' - cjdns_port = 11234 - cjdns_password = 'NONE' - - ############################################################################### - - import sys - import traceback - import json - import argparse - - import requests - - import cjdns - from cjdns import key_utils - from cjdns import admin_tools - - import queue - import threading - - def main(): - parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') parser.add_argument('-v', '--verbose', help='increase output verbosity', dest='verbose', action='store_true') parser.set_defaults(verbose=False) args = parser.parse_args() - con = connect() - nodes = dump_node_store(con) edges = {} - get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) - for k in nodes: get_peer_queue.put(k) - for i in range(8): t = threading.Thread(target=worker, args=[nodes, get_peer_queue, result_queue, args.verbose]) t.daemon = True t.start() - for i in range(len(nodes)): peers, node_ip = result_queue.get() get_edges_for_peers(edges, peers, node_ip) - send_graph(nodes, edges) sys.exit(0) - def worker(nodes, get_peer_queue, result, verbose=False): - con = connect() - while True: try: k = get_peer_queue.get_nowait() except queue.Empty: return - node = nodes[k] if verbose: print('fetch', node) node_ip = node['ip'] - peers = get_all_peers(con, node['path']) - result.put((peers, node_ip)) - def connect(): - try: if cjdns_use_default: print('Connecting using default or ~/.cjdnsadmin credentials...') @@ -160,37 +100,28 @@ def connect(): print('Connecting to port {:d}...'.format(cjdns_port)) con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) - return con - except: print('Connection failed!') print(traceback.format_exc()) sys.exit(1) - - def dump_node_store(con): - nodes = dict() - i = 0 while True: res = con.NodeStore_dumpTable(i) - if not 'routingTable' in res: break - for n in res['routingTable']: if not all(key in n for key in ('addr', 'path', 'ip')): continue - ip = n['ip'] path = n['path'] addr = n['addr'] @@ -198,29 +129,21 @@ def dump_node_store(con): if 'version' in n: version = n['version'] - nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} - if not 'more' in res or res['more'] != 1: break - i += 1 - return nodes - - def get_peers(con, path, nearbyPath=''): - formatted_path = path if nearbyPath: formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) - i = 1 retry = 2 while i < retry + 1: @@ -230,14 +153,11 @@ def get_peers(con, path, nearbyPath=''): res = con.RouterModule_getPeers(path) - - if res['error'] == 'not_found': print('get_peers: node with path {:s} not found, skipping.' .format(formatted_path)) return [] - elif res['error'] != 'none': print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' .format(res['error'], formatted_path, retry-i)) @@ -247,72 +167,54 @@ def get_peers(con, path, nearbyPath=''): else: return res['peers'] - i += 1 - print('get_peers: failed on final try, skipping {:s}' .format(formatted_path)) return [] - - def get_all_peers(con, path): - peers = set() keys = set() - res = get_peers(con, path) peers.update(res) - if not res: return keys - last_peer = res[-1] checked_paths = set() - while len(res) > 1: last_path = (last_peer.split('.', 1)[1] .rsplit('.', 2)[0]) - if last_path in checked_paths: break else: checked_paths.add(last_path) - res = get_peers(con, path, last_path) if res: last_peer = res[-1] else: break - peers.update(res) - for peer in peers: key = peer.split('.', 5)[-1] keys |= {key} - return keys - - def get_edges_for_peers(edges, peers, node_ip): - for peer_key in peers: peer_ip = key_utils.to_ipv6(peer_key) - if node_ip > peer_ip: A = node_ip B = peer_ip @@ -320,57 +222,41 @@ def get_edges_for_peers(edges, peers, node_ip): A = peer_ip B = node_ip - edge = { 'a': A, 'b': B } - if A not in edges: edges[A] = [] - if not([True for edge in edges[A] if edge['b'] == B]): edges[A] += [edge] - - def send_graph(nodes, edges): - graph = { 'nodes': [], 'edges': [edge for sublist in edges.values() for edge in sublist], } - for node in nodes.values(): graph['nodes'].append({ 'ip': node['ip'], 'version': node['version'], }) - print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) - - from pprint import pprint - pprint(graph) - return json_graph = json.dumps(graph) print('Sending data to {:s}...'.format(url)) - payload = {'data': json_graph, 'mail': your_mail, 'version': 2} r = requests.post(url, data=payload) - if r.text == 'OK': print('Done!') else: print('{:s}'.format(r.text)) - if __name__ == '__main__': - main() diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index 6c07bce..e0d5092 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -8,8 +8,6 @@ # CONFIG - - # URL where data is sent # www.fc00.org for clearnet access @@ -20,30 +18,20 @@ url = 'http://www.fc00.org/sendGraph' - - # update your email address, so I can contact you in case something goes wrong your_mail = 'your@email.here' - - - - # ---------------------- # RPC connection details # ---------------------- - - # If this is set to True connection details will be loaded from ~/.cjdnsadmin cjdns_use_default = True - - # otherwise these are used. cjdns_ip = '127.0.0.1' @@ -52,11 +40,9 @@ cjdns_password = 'NONE' - - ############################################################################### - +import db from pprint import pprint import sys @@ -67,289 +53,331 @@ import argparse - - import requests - - import cjdns from cjdns import key_utils from cjdns import admin_tools - - import queue from concurrent.futures import ThreadPoolExecutor import threading +def main(): +parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') -def main(): +parser.add_argument('-v', '--verbose', help='increase output verbosity', + +dest='verbose', action='store_true') + +parser.set_defaults(verbose=False) + +args = parser.parse_args() + +con = connect() - parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') - parser.add_argument('-v', '--verbose', help='increase output verbosity', - dest='verbose', action='store_true') - parser.set_defaults(verbose=False) - args = parser.parse_args() +nodes = dump_node_store(con) +edges = {} - con = connect() +get_peer_queue = queue.Queue(0) +result_queue = queue.Queue(0) - nodes = dump_node_store(con) - edges = {} +e = ThreadPoolExecutor(max_workers=1) +args = zip(*((node['ip'],node['path']) for node in nodes.values())) - get_peer_queue = queue.Queue(0) - result_queue = queue.Queue(0) +for peers, node_ip in e.map(get_peers_derp, *args): - e = ThreadPoolExecutor(max_workers=1) +get_edges_for_peers(edges, peers, node_ip) - args = zip((node['ip'],node['path']) for node in nodes.values()) - args = list(args) - print(args) - - for peers, node_ip in e.map(get_peers_derp, *args): - get_edges_for_peers(edges, peers, node_ip) +send_graph(nodes, edges) - send_graph(nodes, edges) - sys.exit(0) +sys.exit(0) +local = threading.local() def con(): - con = threading.local.get('con') - if con is None: - con = connect() - threading.local['con'] = con - return con - + +try: return local.con + +except AttributeError: pass + +con = connect() + +local.con = con + +return con + def get_peers_derp(ip,path): - return get_all_peers(con, path), ip + +peers = db.get_peers(ip) + +if not peers: + +peers = get_all_peers(con(), path) + +db.set_peers(ip,peers) + +return peers,ip def connect(): - try: - if cjdns_use_default: - print('Connecting using default or ~/.cjdnsadmin credentials...') - con = cjdns.connectWithAdminInfo() - else: - print('Connecting to port {:d}...'.format(cjdns_port)) - con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) +try: + +if cjdns_use_default: + +print('Connecting using default or ~/.cjdnsadmin credentials...') +con = cjdns.connectWithAdminInfo() - return con +else: +print('Connecting to port {:d}...'.format(cjdns_port)) - except: - print('Connection failed!') - print(traceback.format_exc()) - sys.exit(1) +con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) +return con +except: +print('Connection failed!') + +print(traceback.format_exc()) + +sys.exit(1) def dump_node_store(con): - nodes = dict() +nodes = dict() +i = 0 - i = 0 - while True: - res = con.NodeStore_dumpTable(i) +while True: +res = con.NodeStore_dumpTable(i) - if not 'routingTable' in res: - break +if not 'routingTable' in res: +break - for n in res['routingTable']: - if not all(key in n for key in ('addr', 'path', 'ip')): - continue +for n in res['routingTable']: +if not all(key in n for key in ('addr', 'path', 'ip')): - ip = n['ip'] - path = n['path'] - addr = n['addr'] - version = None - if 'version' in n: - version = n['version'] +continue +ip = n['ip'] - nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} +path = n['path'] +addr = n['addr'] - if not 'more' in res or res['more'] != 1: - break +version = None +if 'version' in n: - i += 1 +version = n['version'] +nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} - return nodes +if not 'more' in res or res['more'] != 1: +break +i += 1 +return nodes def get_peers(con, path, nearbyPath=''): - formatted_path = path - if nearbyPath: - formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) +formatted_path = path + +if nearbyPath: + +formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) + +i = 1 + +retry = 2 + +while i < retry + 1: + +if nearbyPath: + +res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) + +else: + +res = con.RouterModule_getPeers(path) + +pprint((path,nearbyPath,res)) + +raise RuntimeError('boop') +if res['error'] == 'not_found': - i = 1 - retry = 2 - while i < retry + 1: - if nearbyPath: - res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) - else: - res = con.RouterModule_getPeers(path) +print('get_peers: node with path {:s} not found, skipping.' - pprint((path,nearbyPath,res)) - raise RuntimeError('boop') +.format(formatted_path)) +return [] - if res['error'] == 'not_found': - print('get_peers: node with path {:s} not found, skipping.' - .format(formatted_path)) - return [] +elif res['error'] != 'none': +print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' - elif res['error'] != 'none': - print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' - .format(res['error'], formatted_path, retry-i)) - elif res['result'] == 'timeout': - print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' - .format(formatted_path, retry-i)) - else: - return res['peers'] +.format(res['error'], formatted_path, retry-i)) +elif res['result'] == 'timeout': - i += 1 +print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' +.format(formatted_path, retry-i)) - print('get_peers: failed on final try, skipping {:s}' - .format(formatted_path)) - return [] +else: +return res['peers'] +i += 1 +print('get_peers: failed on final try, skipping {:s}' + +.format(formatted_path)) + +return [] def get_all_peers(con, path): - peers = set() - keys = set() +peers = set() + +keys = set() + +res = get_peers(con, path) + +peers.update(res) +if not res: - res = get_peers(con, path) - peers.update(res) +return keys +last_peer = res[-1] - if not res: - return keys +checked_paths = set() +while len(res) > 1: - last_peer = res[-1] - checked_paths = set() +last_path = (last_peer.split('.', 1)[1] - while len(res) > 1: - last_path = (last_peer.split('.', 1)[1] - .rsplit('.', 2)[0]) +.rsplit('.', 2)[0]) +if last_path in checked_paths: - if last_path in checked_paths: - break - else: - checked_paths.add(last_path) +break +else: - res = get_peers(con, path, last_path) - if res: - last_peer = res[-1] - else: - break +checked_paths.add(last_path) +res = get_peers(con, path, last_path) - peers.update(res) +if res: +last_peer = res[-1] - for peer in peers: - key = peer.split('.', 5)[-1] - keys |= {key} +else: +break - return keys +peers.update(res) +for peer in peers: +key = peer.split('.', 5)[-1] +keys |= {key} + +return keys def get_edges_for_peers(edges, peers, node_ip): - for peer_key in peers: - peer_ip = key_utils.to_ipv6(peer_key) +for peer_key in peers: + +peer_ip = key_utils.to_ipv6(peer_key) +if node_ip > peer_ip: - if node_ip > peer_ip: - A = node_ip - B = peer_ip - else: - A = peer_ip - B = node_ip +A = node_ip +B = peer_ip - edge = { 'a': A, - 'b': B } +else: +A = peer_ip - if A not in edges: - edges[A] = [] +B = node_ip +edge = { 'a': A, - if not([True for edge in edges[A] if edge['b'] == B]): - edges[A] += [edge] +'b': B } +if A not in edges: +edges[A] = [] +if not([True for edge in edges[A] if edge['b'] == B]): + +edges[A] += [edge] def send_graph(nodes, edges): - graph = { - 'nodes': [], - 'edges': [edge for sublist in edges.values() - for edge in sublist], - } +graph = { + +'nodes': [], + +'edges': [edge for sublist in edges.values() + +for edge in sublist], + +} + +for node in nodes.values(): + +graph['nodes'].append({ + +'ip': node['ip'], + +'version': node['version'], + +}) +print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) - for node in nodes.values(): - graph['nodes'].append({ - 'ip': node['ip'], - 'version': node['version'], - }) +pprint(graph) +return - print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) +json_graph = json.dumps(graph) +print('Sending data to {:s}...'.format(url)) - pprint(graph) - return - json_graph = json.dumps(graph) - print('Sending data to {:s}...'.format(url)) +payload = {'data': json_graph, 'mail': your_mail, 'version': 2} +r = requests.post(url, data=payload) - payload = {'data': json_graph, 'mail': your_mail, 'version': 2} - r = requests.post(url, data=payload) +if r.text == 'OK': +print('Done!') - if r.text == 'OK': - print('Done!') - else: - print('{:s}'.format(r.text)) +else: +print('{:s}'.format(r.text)) if __name__ == '__main__': - main() +main() From eea20cb7aad98e936124ad5291b145795e7f53d4 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 19:05:17 +0000 Subject: [PATCH 04/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 206 5550 --- scripts/updateGraph.py | 445 +++++++++++++++-------------------------- 1 file changed, 158 insertions(+), 287 deletions(-) diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index e0d5092..afd60a5 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -1,383 +1,254 @@ #!/usr/bin/env python3 # Based on Kyrias' sendGraph script. Requires Python 3, requests and cjdns. - # You can install them using pip: pip3 install cjdns requests - ############################################################################### - # CONFIG # URL where data is sent - # www.fc00.org for clearnet access - # h.fc00.org for hyperboria - # [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access - url = 'http://www.fc00.org/sendGraph' # update your email address, so I can contact you in case something goes wrong - your_mail = 'your@email.here' -# ---------------------- +# ---------------------- # RPC connection details - # ---------------------- # If this is set to True connection details will be loaded from ~/.cjdnsadmin - cjdns_use_default = True # otherwise these are used. - cjdns_ip = '127.0.0.1' - cjdns_port = 11234 - cjdns_password = 'NONE' ############################################################################### - import db from pprint import pprint - import sys - import traceback - import json - import argparse import requests import cjdns - from cjdns import key_utils - from cjdns import admin_tools import queue - from concurrent.futures import ThreadPoolExecutor import threading def main(): - -parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') - -parser.add_argument('-v', '--verbose', help='increase output verbosity', - -dest='verbose', action='store_true') - -parser.set_defaults(verbose=False) - -args = parser.parse_args() - -con = connect() - -nodes = dump_node_store(con) - -edges = {} - -get_peer_queue = queue.Queue(0) - -result_queue = queue.Queue(0) - -e = ThreadPoolExecutor(max_workers=1) - -args = zip(*((node['ip'],node['path']) for node in nodes.values())) - -for peers, node_ip in e.map(get_peers_derp, *args): - -get_edges_for_peers(edges, peers, node_ip) - -send_graph(nodes, edges) - -sys.exit(0) + parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') + parser.add_argument('-v', '--verbose', help='increase output verbosity', + dest='verbose', action='store_true') + parser.set_defaults(verbose=False) + args = parser.parse_args() + + con = connect() + + nodes = dump_node_store(con) + edges = {} + + get_peer_queue = queue.Queue(0) + result_queue = queue.Queue(0) + e = ThreadPoolExecutor(max_workers=1) + args = zip(*((node['ip'],node['path']) for node in nodes.values())) + for peers, node_ip in e.map(get_peers_derp, *args): + get_edges_for_peers(edges, peers, node_ip) + send_graph(nodes, edges) + sys.exit(0) local = threading.local() - + def con(): - -try: return local.con - -except AttributeError: pass - -con = connect() - -local.con = con - -return con - + try: return local.con + except AttributeError: pass + con = connect() + local.con = con + return con + def get_peers_derp(ip,path): - -peers = db.get_peers(ip) - -if not peers: - -peers = get_all_peers(con(), path) - -db.set_peers(ip,peers) - -return peers,ip - + peers = db.get_peers(ip) + if not peers: + peers = get_all_peers(con(), path) + db.set_peers(ip,peers) + return peers,ip def connect(): + try: + if cjdns_use_default: + print('Connecting using default or ~/.cjdnsadmin credentials...') + con = cjdns.connectWithAdminInfo() + else: + print('Connecting to port {:d}...'.format(cjdns_port)) + con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) -try: - -if cjdns_use_default: - -print('Connecting using default or ~/.cjdnsadmin credentials...') + return con -con = cjdns.connectWithAdminInfo() + except: + print('Connection failed!') + print(traceback.format_exc()) + sys.exit(1) -else: - -print('Connecting to port {:d}...'.format(cjdns_port)) - -con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) - -return con - -except: - -print('Connection failed!') - -print(traceback.format_exc()) - -sys.exit(1) def dump_node_store(con): + nodes = dict() -nodes = dict() - -i = 0 - -while True: - -res = con.NodeStore_dumpTable(i) - -if not 'routingTable' in res: - -break - -for n in res['routingTable']: - -if not all(key in n for key in ('addr', 'path', 'ip')): - -continue - -ip = n['ip'] - -path = n['path'] + i = 0 + while True: + res = con.NodeStore_dumpTable(i) -addr = n['addr'] + if not 'routingTable' in res: + break -version = None + for n in res['routingTable']: + if not all(key in n for key in ('addr', 'path', 'ip')): + continue -if 'version' in n: + ip = n['ip'] + path = n['path'] + addr = n['addr'] + version = None + if 'version' in n: + version = n['version'] -version = n['version'] + nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} -nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} + if not 'more' in res or res['more'] != 1: + break -if not 'more' in res or res['more'] != 1: + i += 1 -break + return nodes -i += 1 - -return nodes def get_peers(con, path, nearbyPath=''): + formatted_path = path + if nearbyPath: + formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) + + i = 1 + retry = 2 + while i < retry + 1: + if nearbyPath: + res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) + else: + res = con.RouterModule_getPeers(path) + pprint((path,nearbyPath,res)) + raise RuntimeError('boop') + + if res['error'] == 'not_found': + print('get_peers: node with path {:s} not found, skipping.' + .format(formatted_path)) + return [] + + elif res['error'] != 'none': + print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' + .format(res['error'], formatted_path, retry-i)) + elif res['result'] == 'timeout': + print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' + .format(formatted_path, retry-i)) + else: + return res['peers'] + + i += 1 + + print('get_peers: failed on final try, skipping {:s}' + .format(formatted_path)) + return [] -formatted_path = path - -if nearbyPath: - -formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) - -i = 1 - -retry = 2 - -while i < retry + 1: - -if nearbyPath: - -res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) - -else: - -res = con.RouterModule_getPeers(path) - -pprint((path,nearbyPath,res)) - -raise RuntimeError('boop') - -if res['error'] == 'not_found': - -print('get_peers: node with path {:s} not found, skipping.' - -.format(formatted_path)) - -return [] - -elif res['error'] != 'none': - -print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' - -.format(res['error'], formatted_path, retry-i)) - -elif res['result'] == 'timeout': - -print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' - -.format(formatted_path, retry-i)) - -else: - -return res['peers'] - -i += 1 - -print('get_peers: failed on final try, skipping {:s}' - -.format(formatted_path)) - -return [] def get_all_peers(con, path): + peers = set() + keys = set() -peers = set() - -keys = set() - -res = get_peers(con, path) - -peers.update(res) - -if not res: - -return keys - -last_peer = res[-1] - -checked_paths = set() + res = get_peers(con, path) + peers.update(res) -while len(res) > 1: + if not res: + return keys -last_path = (last_peer.split('.', 1)[1] + last_peer = res[-1] + checked_paths = set() + while len(res) > 1: + last_path = (last_peer.split('.', 1)[1] + .rsplit('.', 2)[0]) -.rsplit('.', 2)[0]) + if last_path in checked_paths: + break + else: + checked_paths.add(last_path) -if last_path in checked_paths: + res = get_peers(con, path, last_path) + if res: + last_peer = res[-1] + else: + break -break + peers.update(res) -else: + for peer in peers: + key = peer.split('.', 5)[-1] + keys |= {key} -checked_paths.add(last_path) + return keys -res = get_peers(con, path, last_path) - -if res: - -last_peer = res[-1] - -else: - -break - -peers.update(res) - -for peer in peers: - -key = peer.split('.', 5)[-1] - -keys |= {key} - -return keys def get_edges_for_peers(edges, peers, node_ip): + for peer_key in peers: + peer_ip = key_utils.to_ipv6(peer_key) -for peer_key in peers: - -peer_ip = key_utils.to_ipv6(peer_key) - -if node_ip > peer_ip: - -A = node_ip - -B = peer_ip - -else: - -A = peer_ip - -B = node_ip - -edge = { 'a': A, - -'b': B } + if node_ip > peer_ip: + A = node_ip + B = peer_ip + else: + A = peer_ip + B = node_ip -if A not in edges: + edge = { 'a': A, + 'b': B } -edges[A] = [] + if A not in edges: + edges[A] = [] -if not([True for edge in edges[A] if edge['b'] == B]): + if not([True for edge in edges[A] if edge['b'] == B]): + edges[A] += [edge] -edges[A] += [edge] def send_graph(nodes, edges): + graph = { + 'nodes': [], + 'edges': [edge for sublist in edges.values() + for edge in sublist], + } -graph = { + for node in nodes.values(): + graph['nodes'].append({ + 'ip': node['ip'], + 'version': node['version'], + }) -'nodes': [], + print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) -'edges': [edge for sublist in edges.values() + pprint(graph) + return + json_graph = json.dumps(graph) + print('Sending data to {:s}...'.format(url)) -for edge in sublist], + payload = {'data': json_graph, 'mail': your_mail, 'version': 2} + r = requests.post(url, data=payload) -} - -for node in nodes.values(): - -graph['nodes'].append({ - -'ip': node['ip'], - -'version': node['version'], - -}) - -print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) - -pprint(graph) - -return - -json_graph = json.dumps(graph) - -print('Sending data to {:s}...'.format(url)) - -payload = {'data': json_graph, 'mail': your_mail, 'version': 2} - -r = requests.post(url, data=payload) - -if r.text == 'OK': - -print('Done!') - -else: - -print('{:s}'.format(r.text)) + if r.text == 'OK': + print('Done!') + else: + print('{:s}'.format(r.text)) if __name__ == '__main__': - -main() + main() From 60c8f74dbb14d6dff03c7d6baf68978290e8c6a3 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 19:14:06 +0000 Subject: [PATCH 05/11] auto (/extra/user/packages/git/fc00.org/scripts/db.py) 53 1998 --- scripts/db.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 scripts/db.py diff --git a/scripts/db.py b/scripts/db.py new file mode 100644 index 0000000..70307b0 --- /dev/null +++ b/scripts/db.py @@ -0,0 +1,61 @@ +import sqlite3 +import threading +l = threading.local() + +cache = os.path.expanduser("~/.cache") + +l.conn = sqlite3.Connection(os.path.join(cache,"fc00.sqlite")) + +l.conn.execute('CREATE TABLE IF NOT EXISTS versions (latest INTEGER PRIMARY KEY)') +with l.conn,closing(conn.cursor()) as c: + c.execute('SELECT latest FROM versions') + latest = c.fetchone() + if latest: + latest = latest[0] + else: + latest = 0 + c.execute('INSERT INTO versions (latest) VALUES (0)') + +def version(n): + def deco(f): + if n > latest: + f() + with closing(l.conn.cursor()) as c: + c.execute('UPDATE versions SET latest = ?',(n,)) + return deco + +@version(1) +def _(): + with closing(l.conn.cursor()) as c: + c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, ip TEXT UNIQUE, checked TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL') + c.execute('CREATE INDEX byChecked ON nodes(checked)') + c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id), blue INTEGER REFERENCES nodes(id), UNIQUE(red,blue))') + +def get_peers(ip): + with closing(l.conn.cursor()) as c: + ident = peer2node(ip,c) + c.execute("SELECT checked < datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) + ok = c.fetchone() + if not ok or not ok[0]: + return () + c.execute("""SELECT ip FROM nodes +WHERE id IN ( + SELECT blue FROM links WHERE red = ( + SELECT id FROM nodes WHERE id = ?)) +""",(ident,)) + return [row[0] for row in c.fetchall()] + +def set_peers(ip,peers): + with l.conn, closing(l.conn.cursor()) as c: + peers = [peer2node(peer,c) for peer in peers] + ident = peer2node(ip,c) + for p in peers: + c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', + (ident,p)) + +def peer2node(ip,c): + c.execute('SELECT id FROM nodes WHERE ip = ?',(ip,)) + ident = c.fetchone() + if ident: + return ident[0] + c.execute('INSERT INTO nodes (ip)',(ip,)) From 78ece30ed37b712ccb496d49ca258763026ccc07 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 19:31:24 +0000 Subject: [PATCH 06/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 60 1127 --- scripts/db.py | 44 ++++++++++++++++++++++++------------------ scripts/updateGraph.py | 20 +++++++++++++------ 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index 70307b0..60330ce 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -1,13 +1,19 @@ -import sqlite3 +import sqlite3,os + +from contextlib import closing import threading l = threading.local() cache = os.path.expanduser("~/.cache") -l.conn = sqlite3.Connection(os.path.join(cache,"fc00.sqlite")) - +def conn(): + try: return l.conn + except AttributeError: pass + l.conn = sqlite3.Connection(os.path.join(cache,"fc00.sqlite")) + return l.conn +conn() l.conn.execute('CREATE TABLE IF NOT EXISTS versions (latest INTEGER PRIMARY KEY)') -with l.conn,closing(conn.cursor()) as c: +with l.conn,closing(l.conn.cursor()) as c: c.execute('SELECT latest FROM versions') latest = c.fetchone() if latest: @@ -20,42 +26,42 @@ def version(n): def deco(f): if n > latest: f() - with closing(l.conn.cursor()) as c: + with l.conn,closing(l.conn.cursor()) as c: c.execute('UPDATE versions SET latest = ?',(n,)) return deco @version(1) def _(): with closing(l.conn.cursor()) as c: - c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, ip TEXT UNIQUE, checked TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL') + c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, key TEXT UNIQUE, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)') c.execute('CREATE INDEX byChecked ON nodes(checked)') c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id), blue INTEGER REFERENCES nodes(id), UNIQUE(red,blue))') -def get_peers(ip): - with closing(l.conn.cursor()) as c: - ident = peer2node(ip,c) +def get_peers(key): + with closing(conn().cursor()) as c: + ident = peer2node(key,c) c.execute("SELECT checked < datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) ok = c.fetchone() if not ok or not ok[0]: return () - c.execute("""SELECT ip FROM nodes + c.execute("""SELECT key FROM nodes WHERE id IN ( SELECT blue FROM links WHERE red = ( SELECT id FROM nodes WHERE id = ?)) """,(ident,)) return [row[0] for row in c.fetchall()] -def set_peers(ip,peers): - with l.conn, closing(l.conn.cursor()) as c: +def set_peers(key,peers): + with conn(), closing(l.conn.cursor()) as c: peers = [peer2node(peer,c) for peer in peers] - ident = peer2node(ip,c) - for p in peers: - c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', - (ident,p)) + ident = peer2node(key,c) + for p in peers: + c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', + (ident,p)) -def peer2node(ip,c): - c.execute('SELECT id FROM nodes WHERE ip = ?',(ip,)) +def peer2node(key,c): + c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) ident = c.fetchone() if ident: return ident[0] - c.execute('INSERT INTO nodes (ip)',(ip,)) + c.execute('INSERT INTO nodes (key) VALUES (?)',(key,)) diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index afd60a5..8a4d063 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -60,7 +60,12 @@ def main(): get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) e = ThreadPoolExecutor(max_workers=1) - args = zip(*((node['ip'],node['path']) for node in nodes.values())) + pprint(nodes) + sys.exit(0) + def args(): + for node in nodes.values(): + yield node['ip'],peerFromAddr(node['addr']),node['path'] + args = zip(*args()) for peers, node_ip in e.map(get_peers_derp, *args): get_edges_for_peers(edges, peers, node_ip) send_graph(nodes, edges) @@ -75,11 +80,13 @@ def con(): local.con = con return con -def get_peers_derp(ip,path): - peers = db.get_peers(ip) +def get_peers_derp(ip,key,path): + peers = db.get_peers(key) if not peers: peers = get_all_peers(con(), path) - db.set_peers(ip,peers) + pprint(('no peers in db',key,peers)) + db.set_peers(key,peers) + sys.exit(0) return peers,ip def connect(): try: @@ -142,7 +149,6 @@ def get_peers(con, path, nearbyPath=''): else: res = con.RouterModule_getPeers(path) pprint((path,nearbyPath,res)) - raise RuntimeError('boop') if res['error'] == 'not_found': print('get_peers: node with path {:s} not found, skipping.' @@ -195,11 +201,13 @@ def get_all_peers(con, path): peers.update(res) for peer in peers: - key = peer.split('.', 5)[-1] + key = keyFromAddr(peer) keys |= {key} return keys +def keyFromAddr(addr): + return addr.split('.', 5)[-1] def get_edges_for_peers(edges, peers, node_ip): for peer_key in peers: From 2860d5c72aa6ff7bd6a075745ecc9ab7cac0f885 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 20:22:48 +0000 Subject: [PATCH 07/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 97 2106 --- scripts/db.py | 30 +++++++++++++++---- scripts/updateGraph.py | 68 ++++++++++++++++++++---------------------- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index 60330ce..f6eab3c 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -1,4 +1,4 @@ -import sqlite3,os +import sqlite3,os,time from contextlib import closing import threading @@ -35,12 +35,27 @@ def _(): with closing(l.conn.cursor()) as c: c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, key TEXT UNIQUE, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)') c.execute('CREATE INDEX byChecked ON nodes(checked)') - c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id), blue INTEGER REFERENCES nodes(id), UNIQUE(red,blue))') - + c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id) NOT NULL, blue INTEGER REFERENCES nodes(id) NOT NULL, UNIQUE(red,blue))') + +def retry_on_locked(s): + def deco(f): + def wrapper(*a,**kw): + while True: + try: + return f(*a,**kw) + except sqlite3.OperationalError as e: + if e.error_code != 5: + raise + print(e.args) + time.sleep(s) + return wrapper + return deco + +@retry_on_locked(1) def get_peers(key): - with closing(conn().cursor()) as c: + with conn(),closing(l.conn.cursor()) as c: ident = peer2node(key,c) - c.execute("SELECT checked < datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) + c.execute("SELECT checked > datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) ok = c.fetchone() if not ok or not ok[0]: return () @@ -50,7 +65,8 @@ def get_peers(key): SELECT id FROM nodes WHERE id = ?)) """,(ident,)) return [row[0] for row in c.fetchall()] - + +@retry_on_locked(1) def set_peers(key,peers): with conn(), closing(l.conn.cursor()) as c: peers = [peer2node(peer,c) for peer in peers] @@ -58,6 +74,7 @@ def set_peers(key,peers): for p in peers: c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', (ident,p)) + c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) def peer2node(key,c): c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) @@ -65,3 +82,4 @@ def peer2node(key,c): if ident: return ident[0] c.execute('INSERT INTO nodes (key) VALUES (?)',(key,)) + return c.lastrowid diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index 8a4d063..bba4204 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -59,16 +59,20 @@ def main(): get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) - e = ThreadPoolExecutor(max_workers=1) - pprint(nodes) - sys.exit(0) + e = ThreadPoolExecutor(max_workers=8) def args(): - for node in nodes.values(): - yield node['ip'],peerFromAddr(node['addr']),node['path'] + for ip,node in nodes.items(): + yield ip,keyFromAddr(node['addr']),node['path'] args = zip(*args()) - for peers, node_ip in e.map(get_peers_derp, *args): - get_edges_for_peers(edges, peers, node_ip) - send_graph(nodes, edges) + dbnodes = {} + for peers, node_id, ip in e.map(get_peers_derp, *args): + get_edges_for_peers(edges, peers, node_id) + dbnodes[node_id] = { + 'ip': ip, + 'peers': peers, + 'id': id, + } + send_graph(dbnodes, edges) sys.exit(0) local = threading.local() @@ -81,13 +85,15 @@ def con(): return con def get_peers_derp(ip,key,path): - peers = db.get_peers(key) + print('check',ip) + ident,peers = db.get_peers(key) if not peers: peers = get_all_peers(con(), path) - pprint(('no peers in db',key,peers)) - db.set_peers(key,peers) - sys.exit(0) - return peers,ip + pprint(('adding peers to db',len(peers))) + peers = db.set_peers(key,peers) + else: + pprint(('got db peers!',len(peers))) + return peers,ident,ip def connect(): try: if cjdns_use_default: @@ -148,7 +154,6 @@ def get_peers(con, path, nearbyPath=''): res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) else: res = con.RouterModule_getPeers(path) - pprint((path,nearbyPath,res)) if res['error'] == 'not_found': print('get_peers: node with path {:s} not found, skipping.' @@ -209,16 +214,14 @@ def get_all_peers(con, path): def keyFromAddr(addr): return addr.split('.', 5)[-1] -def get_edges_for_peers(edges, peers, node_ip): +def get_edges_for_peers(edges, peers, node_key): for peer_key in peers: - peer_ip = key_utils.to_ipv6(peer_key) - - if node_ip > peer_ip: - A = node_ip - B = peer_ip + if node_key > peer_key: + A = node_key + B = peer_key else: - A = peer_ip - B = node_ip + A = peer_key + B = node_key edge = { 'a': A, 'b': B } @@ -231,21 +234,16 @@ def get_edges_for_peers(edges, peers, node_ip): def send_graph(nodes, edges): - graph = { - 'nodes': [], - 'edges': [edge for sublist in edges.values() - for edge in sublist], - } - - for node in nodes.values(): - graph['nodes'].append({ - 'ip': node['ip'], - 'version': node['version'], - }) - print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) - pprint(graph) + with open('out.dot','wt') as out: + out.write('graph cjdns {\n') + for ident,node in nodes.items(): + out.write(' n{} [label="{}"];\n',ident,node['ip'].rsplit(':',1)[-1]) + for node,peers in edges.items(): + for p in peers: + out.write(' n{} -> n{}\n',node,p); + out.write('}\n') return json_graph = json.dumps(graph) print('Sending data to {:s}...'.format(url)) From 2f5db7fc695662edc7ecc492ff9569d3cf117575 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 20:58:56 +0000 Subject: [PATCH 08/11] auto (/extra/user/packages/git/fc00.org/scripts/db.py) 66 1811 --- scripts/db.py | 44 ++++++++++++++++++++++++++++++++++-------- scripts/updateGraph.py | 24 ++++++++++++----------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index f6eab3c..69e096d 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -36,7 +36,37 @@ def _(): c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, key TEXT UNIQUE, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)') c.execute('CREATE INDEX byChecked ON nodes(checked)') c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id) NOT NULL, blue INTEGER REFERENCES nodes(id) NOT NULL, UNIQUE(red,blue))') - + +@version(2) +def _(): + with closing(l.conn.cursor()) as c: + c.execute("ALTER TABLE nodes ADD COLUMN ip TEXT"); + +def fixkeys(key2ip): + @version(3) + def _(): + conn() + l.conn.create_function("key2ip", 1, key2ip) + with closing(l.conn.cursor()) as c: + c.execute('ALTER TABLE nodes RENAME TO oldnodes') + c.execute('''CREATE TABLE nodes ( +id INTEGER PRIMARY KEY, +key TEXT NOT NULL UNIQUE, +ip TEXT NOT NULL, +checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') + c.execute('INSERT INTO nodes SELECT id,key,key2ip(key) as ip,checked FROM oldnodes') + c.execute('ALTER TABLE links RENAME TO oldlinks') + c.execute('''CREATE TABLE links ( +id INTEGER PRIMARY KEY, +red INTEGER REFERENCES nodes(id) NOT NULL, +blue INTEGER REFERENCES nodes(id) NOT NULL, +UNIQUE(red,blue)) + ''') + c.execute('INSERT INTO links SELECT id,red,blue FROM oldlinks') + c.execute('DROP TABLE oldlinks') + c.execute('DROP TABLE oldnodes') + c.execute('VACUUM ANALYZE') + def retry_on_locked(s): def deco(f): def wrapper(*a,**kw): @@ -58,23 +88,21 @@ def get_peers(key): c.execute("SELECT checked > datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) ok = c.fetchone() if not ok or not ok[0]: - return () - c.execute("""SELECT key FROM nodes -WHERE id IN ( - SELECT blue FROM links WHERE red = ( - SELECT id FROM nodes WHERE id = ?)) -""",(ident,)) - return [row[0] for row in c.fetchall()] + return ident,() + c.execute("SELECT (SELECT ip FROM nodes WHERE id = blue),blue FROM links WHERE red = ?",(ident,)) + return ident,[row[0] for row in c.fetchall()] @retry_on_locked(1) def set_peers(key,peers): with conn(), closing(l.conn.cursor()) as c: peers = [peer2node(peer,c) for peer in peers] ident = peer2node(key,c) + peers = [peer for peer in peers if peer != ident] for p in peers: c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', (ident,p)) c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) + return peers def peer2node(key,c): c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index bba4204..5706f6c 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -46,6 +46,7 @@ import threading def main(): + db.fixkeys(key_utils.to_ipv6) parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') parser.add_argument('-v', '--verbose', help='increase output verbosity', dest='verbose', action='store_true') @@ -59,19 +60,20 @@ def main(): get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) - e = ThreadPoolExecutor(max_workers=8) + e = ThreadPoolExecutor(max_workers=1) def args(): for ip,node in nodes.items(): yield ip,keyFromAddr(node['addr']),node['path'] args = zip(*args()) dbnodes = {} - for peers, node_id, ip in e.map(get_peers_derp, *args): + for peers, node_id, ip in map(get_peers_derp, *args): get_edges_for_peers(edges, peers, node_id) dbnodes[node_id] = { 'ip': ip, 'peers': peers, 'id': id, } + print('otay!') send_graph(dbnodes, edges) sys.exit(0) @@ -228,21 +230,21 @@ def get_edges_for_peers(edges, peers, node_key): if A not in edges: edges[A] = [] - - if not([True for edge in edges[A] if edge['b'] == B]): - edges[A] += [edge] - + edges[A] = B def send_graph(nodes, edges): print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) with open('out.dot','wt') as out: - out.write('graph cjdns {\n') + out.write('digraph cjdns {\n') for ident,node in nodes.items(): - out.write(' n{} [label="{}"];\n',ident,node['ip'].rsplit(':',1)[-1]) - for node,peers in edges.items(): - for p in peers: - out.write(' n{} -> n{}\n',node,p); + out.write(' n{} [label="{}"];\n'.format( + ident, + node['ip'].rsplit(':',1)[-1])) + for node,peer in edges.items(): + out.write(' n{} -> n{}\n'.format( + node, + peer)); out.write('}\n') return json_graph = json.dumps(graph) From e82daccd4d8f9131bfcce939b5c0668ed85ba5a7 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 22:11:47 +0000 Subject: [PATCH 09/11] auto (/extra/user/packages/git/fc00.org/scripts/updateGraph.py) 51 988 --- scripts/db.py | 18 +++++++++++++----- scripts/updateGraph.py | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index 69e096d..f8dd99d 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -42,7 +42,11 @@ def _(): with closing(l.conn.cursor()) as c: c.execute("ALTER TABLE nodes ADD COLUMN ip TEXT"); -def fixkeys(key2ip): +key2ip = None + +def fixkeys(derp): + global key2ip + key2ip = derp @version(3) def _(): conn() @@ -90,24 +94,28 @@ def get_peers(key): if not ok or not ok[0]: return ident,() c.execute("SELECT (SELECT ip FROM nodes WHERE id = blue),blue FROM links WHERE red = ?",(ident,)) - return ident,[row[0] for row in c.fetchall()] + return ident,c.fetchall() @retry_on_locked(1) def set_peers(key,peers): with conn(), closing(l.conn.cursor()) as c: - peers = [peer2node(peer,c) for peer in peers] + try: + peers = [peer2node(peer,c) for peer in peers] + except: + print(peers) + raise ident = peer2node(key,c) peers = [peer for peer in peers if peer != ident] for p in peers: c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', (ident,p)) c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) - return peers + return get_peers(key) def peer2node(key,c): c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) ident = c.fetchone() if ident: return ident[0] - c.execute('INSERT INTO nodes (key) VALUES (?)',(key,)) + c.execute('INSERT INTO nodes (key,ip) VALUES (?,?)',(key,key2ip(key))) return c.lastrowid diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index 5706f6c..dd8bf69 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -45,6 +45,15 @@ from concurrent.futures import ThreadPoolExecutor import threading +def addpeersto(d,n,ip,peers=set()): + if n in d: + d[n]['peers'].update(peers) + else: + d[n] = { + 'ip': ip, + 'peers': set(peers) + } + def main(): db.fixkeys(key_utils.to_ipv6) parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') @@ -68,11 +77,10 @@ def args(): dbnodes = {} for peers, node_id, ip in map(get_peers_derp, *args): get_edges_for_peers(edges, peers, node_id) - dbnodes[node_id] = { - 'ip': ip, - 'peers': peers, - 'id': id, - } + addpeersto(dbnodes,node_id,ip,peers) + + for ip, id in peers: + addpeersto(dbnodes,id,ip) print('otay!') send_graph(dbnodes, edges) sys.exit(0) @@ -92,7 +100,7 @@ def get_peers_derp(ip,key,path): if not peers: peers = get_all_peers(con(), path) pprint(('adding peers to db',len(peers))) - peers = db.set_peers(key,peers) + ident,peers = db.set_peers(key,peers) else: pprint(('got db peers!',len(peers))) return peers,ident,ip @@ -217,7 +225,11 @@ def keyFromAddr(addr): return addr.split('.', 5)[-1] def get_edges_for_peers(edges, peers, node_key): - for peer_key in peers: + for derp in peers: + try: ip,peer_key = derp + except: + pprint(peers) + raise if node_key > peer_key: A = node_key B = peer_key @@ -237,6 +249,7 @@ def send_graph(nodes, edges): with open('out.dot','wt') as out: out.write('digraph cjdns {\n') + out.write(" overlap=false;\n"); for ident,node in nodes.items(): out.write(' n{} [label="{}"];\n'.format( ident, @@ -246,10 +259,17 @@ def send_graph(nodes, edges): node, peer)); out.write('}\n') - return + + graph = { + 'nodes': + ({'ip': node['ip'], 'version': node['version']} for node in nodes), + 'edges': [{'a': A, 'b': B} for a,b in edges.items()] + } json_graph = json.dumps(graph) + print(json_graph) + return print('Sending data to {:s}...'.format(url)) - + payload = {'data': json_graph, 'mail': your_mail, 'version': 2} r = requests.post(url, data=payload) From be4e1d2e82ef0d72c05222fb31622293f73b55b9 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 22:26:50 +0000 Subject: [PATCH 10/11] auto (/extra/user/packages/git/fc00.org/scripts/db.py) 77 2188 --- scripts/db.py | 63 ++++++++++++++++++++++++++++-------------- scripts/updateGraph.py | 17 +++++++----- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index f8dd99d..a4c474e 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -43,7 +43,18 @@ def _(): c.execute("ALTER TABLE nodes ADD COLUMN ip TEXT"); key2ip = None - + +def redoLinks(c): + c.execute('ALTER TABLE links RENAME TO oldlinks') + c.execute('''CREATE TABLE links ( +id INTEGER PRIMARY KEY, +red INTEGER REFERENCES nodes(id) NOT NULL, +blue INTEGER REFERENCES nodes(id) NOT NULL, +UNIQUE(red,blue)) + ''') + c.execute('INSERT INTO links SELECT id,red,blue FROM oldlinks') + c.execute('DROP TABLE oldlinks') + def fixkeys(derp): global key2ip key2ip = derp @@ -59,18 +70,28 @@ def _(): ip TEXT NOT NULL, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') c.execute('INSERT INTO nodes SELECT id,key,key2ip(key) as ip,checked FROM oldnodes') - c.execute('ALTER TABLE links RENAME TO oldlinks') - c.execute('''CREATE TABLE links ( -id INTEGER PRIMARY KEY, -red INTEGER REFERENCES nodes(id) NOT NULL, -blue INTEGER REFERENCES nodes(id) NOT NULL, -UNIQUE(red,blue)) - ''') - c.execute('INSERT INTO links SELECT id,red,blue FROM oldlinks') - c.execute('DROP TABLE oldlinks') + redoLinks(c) c.execute('DROP TABLE oldnodes') c.execute('VACUUM ANALYZE') +@version(4) +def _(): + l.conn.execute("ALTER TABLE nodes ADD COLUMN lastVersion INT") + +@version(5) +def _(): + with closing(l.conn.cursor()) as c: + c.execute('ALTER TABLE nodes RENAME TO oldnodes') + c.execute('''CREATE TABLE nodes ( + id INTEGER PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + ip TEXT NOT NULL, + lastVersion INTEGER, +checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') + c.execute('DROP INDEX byChecked;') + c.execute('CREATE INDEX byChecked ON nodes(checked)') + redoLinks(c) + def retry_on_locked(s): def deco(f): def wrapper(*a,**kw): @@ -86,9 +107,9 @@ def wrapper(*a,**kw): return deco @retry_on_locked(1) -def get_peers(key): +def get_peers(key,lastVersion): with conn(),closing(l.conn.cursor()) as c: - ident = peer2node(key,c) + ident = peer2node(key,c,lastVersion) c.execute("SELECT checked > datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) ok = c.fetchone() if not ok or not ok[0]: @@ -97,14 +118,11 @@ def get_peers(key): return ident,c.fetchall() @retry_on_locked(1) -def set_peers(key,peers): +def set_peers(key,peers,lastVersion): with conn(), closing(l.conn.cursor()) as c: - try: - peers = [peer2node(peer,c) for peer in peers] - except: - print(peers) - raise - ident = peer2node(key,c) + # getPeers doesn't return the peer's versions, only their key. + peers = [peer2node(peer,c) for peer in peers] + ident = peer2node(key,c,lastVersion) peers = [peer for peer in peers if peer != ident] for p in peers: c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', @@ -112,10 +130,13 @@ def set_peers(key,peers): c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) return get_peers(key) -def peer2node(key,c): +def peer2node(key,c,lastVersion=None): c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) ident = c.fetchone() if ident: + c.execute('UPDATE nodes SET lastVersion = ? WHERE id = ?', + (lastVersion,ident)) return ident[0] - c.execute('INSERT INTO nodes (key,ip) VALUES (?,?)',(key,key2ip(key))) + c.execute('INSERT INTO nodes (key,ip,lastVersion) VALUES (?,?,?)', + (key,key2ip(key),lastVersion)) return c.lastrowid diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index dd8bf69..8445802 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -72,7 +72,7 @@ def main(): e = ThreadPoolExecutor(max_workers=1) def args(): for ip,node in nodes.items(): - yield ip,keyFromAddr(node['addr']),node['path'] + yield ip,keyFromAddr(node['addr']),node['path'],node['version'] args = zip(*args()) dbnodes = {} for peers, node_id, ip in map(get_peers_derp, *args): @@ -94,13 +94,13 @@ def con(): local.con = con return con -def get_peers_derp(ip,key,path): - print('check',ip) - ident,peers = db.get_peers(key) +def get_peers_derp(ip,key,path,version): + print('check',ip,version) + ident,peers = db.get_peers(key,version) if not peers: peers = get_all_peers(con(), path) pprint(('adding peers to db',len(peers))) - ident,peers = db.set_peers(key,peers) + ident,peers = db.set_peers(key,peers,version) else: pprint(('got db peers!',len(peers))) return peers,ident,ip @@ -262,8 +262,11 @@ def send_graph(nodes, edges): graph = { 'nodes': - ({'ip': node['ip'], 'version': node['version']} for node in nodes), - 'edges': [{'a': A, 'b': B} for a,b in edges.items()] + [{'ip': node['ip'], + 'version': + db.getVersion(node['id']) + } for node in nodes], + 'edges': [{'a': A, 'b': B} for A,B in edges.items()] } json_graph = json.dumps(graph) print(json_graph) From d5c1373e3a811621a3c1dd9bd20aa8a89f503487 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 1 Jan 2016 23:19:12 +0000 Subject: [PATCH 11/11] working database saves right, seems roughly the same format, only queries peers once an hour. --- scripts/db.py | 30 ++++++++++++++++++++++-------- scripts/updateGraph.py | 24 +++++++++++++++--------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index a4c474e..76ada7c 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -88,10 +88,12 @@ def _(): ip TEXT NOT NULL, lastVersion INTEGER, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') - c.execute('DROP INDEX byChecked;') + #c.execute('DROP INDEX byChecked;') c.execute('CREATE INDEX byChecked ON nodes(checked)') + c.execute('INSERT INTO nodes (id,key,checked) SELECT id,key,checked FROM oldnodes') redoLinks(c) - + c.execute('DROP TABLE oldnodes') + def retry_on_locked(s): def deco(f): def wrapper(*a,**kw): @@ -105,12 +107,18 @@ def wrapper(*a,**kw): time.sleep(s) return wrapper return deco - + +def get_version(ident): + with conn(),closing(l.conn.cursor()) as c: + c.execute('SELECT lastVersion FROM nodes WHERE id = ?',(ident,)) + row = c.fetchone() + if row: return row[0] + @retry_on_locked(1) def get_peers(key,lastVersion): with conn(),closing(l.conn.cursor()) as c: ident = peer2node(key,c,lastVersion) - c.execute("SELECT checked > datetime('now','-1 day') FROM nodes WHERE id = ?",(ident,)) + c.execute("SELECT checked > datetime('now','-1 hour') FROM nodes WHERE id = ?",(ident,)) ok = c.fetchone() if not ok or not ok[0]: return ident,() @@ -128,15 +136,21 @@ def set_peers(key,peers,lastVersion): c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', (ident,p)) c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) - return get_peers(key) + return get_peers(key,lastVersion) def peer2node(key,c,lastVersion=None): c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) ident = c.fetchone() if ident: - c.execute('UPDATE nodes SET lastVersion = ? WHERE id = ?', - (lastVersion,ident)) - return ident[0] + ident = ident[0] + if lastVersion: + c.execute('SELECT lastVersion FROM nodes WHERE id = ?', + (ident,)) + test = c.fetchone() + if test and test[0] is not None and test[0] != lastVersion: + c.execute('UPDATE nodes SET lastVersion = ? WHERE id = ?', + (lastVersion,ident)) + return ident c.execute('INSERT INTO nodes (key,ip,lastVersion) VALUES (?,?,?)', (key,key2ip(key),lastVersion)) return c.lastrowid diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py index 8445802..1fad607 100644 --- a/scripts/updateGraph.py +++ b/scripts/updateGraph.py @@ -69,13 +69,13 @@ def main(): get_peer_queue = queue.Queue(0) result_queue = queue.Queue(0) - e = ThreadPoolExecutor(max_workers=1) + e = ThreadPoolExecutor(max_workers=4) def args(): for ip,node in nodes.items(): yield ip,keyFromAddr(node['addr']),node['path'],node['version'] args = zip(*args()) dbnodes = {} - for peers, node_id, ip in map(get_peers_derp, *args): + for peers, node_id, ip in e.map(get_peers_derp, *args): get_edges_for_peers(edges, peers, node_id) addpeersto(dbnodes,node_id,ip,peers) @@ -99,10 +99,10 @@ def get_peers_derp(ip,key,path,version): ident,peers = db.get_peers(key,version) if not peers: peers = get_all_peers(con(), path) - pprint(('adding peers to db',len(peers))) + print(('adding peers to db',len(peers))) ident,peers = db.set_peers(key,peers,version) else: - pprint(('got db peers!',len(peers))) + print(('got db peers!',len(peers))) return peers,ident,ip def connect(): try: @@ -262,12 +262,18 @@ def send_graph(nodes, edges): graph = { 'nodes': - [{'ip': node['ip'], - 'version': - db.getVersion(node['id']) - } for node in nodes], - 'edges': [{'a': A, 'b': B} for A,B in edges.items()] + [], + 'edges': [{'a': nodes[A]['ip'], + 'b': nodes[B]['ip']} for A,B in edges.items()] } + for ident,node in nodes.items(): + version = db.get_version(ident) + if version is None: + continue + graph['nodes'].append({ + 'ip': node['ip'], + 'version': version + } ) json_graph = json.dumps(graph) print(json_graph) return