diff --git a/scripts/db.py b/scripts/db.py new file mode 100644 index 0000000..76ada7c --- /dev/null +++ b/scripts/db.py @@ -0,0 +1,156 @@ +import sqlite3,os,time + +from contextlib import closing +import threading +l = threading.local() + +cache = os.path.expanduser("~/.cache") + +def conn(): + try: return l.conn + except AttributeError: pass + l.conn = sqlite3.Connection(os.path.join(cache,"fc00.sqlite")) + return l.conn +conn() +l.conn.execute('CREATE TABLE IF NOT EXISTS versions (latest INTEGER PRIMARY KEY)') +with l.conn,closing(l.conn.cursor()) as c: + c.execute('SELECT latest FROM versions') + latest = c.fetchone() + if latest: + latest = latest[0] + else: + latest = 0 + c.execute('INSERT INTO versions (latest) VALUES (0)') + +def version(n): + def deco(f): + if n > latest: + f() + with l.conn,closing(l.conn.cursor()) as c: + c.execute('UPDATE versions SET latest = ?',(n,)) + return deco + +@version(1) +def _(): + with closing(l.conn.cursor()) as c: + c.execute('CREATE TABLE nodes (id INTEGER PRIMARY KEY, key TEXT UNIQUE, checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)') + c.execute('CREATE INDEX byChecked ON nodes(checked)') + c.execute('CREATE TABLE links (id INTEGER PRIMARY KEY, red INTEGER REFERENCES nodes(id) NOT NULL, blue INTEGER REFERENCES nodes(id) NOT NULL, UNIQUE(red,blue))') + +@version(2) +def _(): + with closing(l.conn.cursor()) as c: + c.execute("ALTER TABLE nodes ADD COLUMN ip TEXT"); + +key2ip = None + +def redoLinks(c): + c.execute('ALTER TABLE links RENAME TO oldlinks') + c.execute('''CREATE TABLE links ( +id INTEGER PRIMARY KEY, +red INTEGER REFERENCES nodes(id) NOT NULL, +blue INTEGER REFERENCES nodes(id) NOT NULL, +UNIQUE(red,blue)) + ''') + c.execute('INSERT INTO links SELECT id,red,blue FROM oldlinks') + c.execute('DROP TABLE oldlinks') + +def fixkeys(derp): + global key2ip + key2ip = derp + @version(3) + def _(): + conn() + l.conn.create_function("key2ip", 1, key2ip) + with closing(l.conn.cursor()) as c: + c.execute('ALTER TABLE nodes RENAME TO oldnodes') + c.execute('''CREATE TABLE nodes ( +id INTEGER PRIMARY KEY, +key TEXT NOT NULL UNIQUE, +ip TEXT NOT NULL, +checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') + c.execute('INSERT INTO nodes SELECT id,key,key2ip(key) as ip,checked FROM oldnodes') + redoLinks(c) + c.execute('DROP TABLE oldnodes') + c.execute('VACUUM ANALYZE') + +@version(4) +def _(): + l.conn.execute("ALTER TABLE nodes ADD COLUMN lastVersion INT") + +@version(5) +def _(): + with closing(l.conn.cursor()) as c: + c.execute('ALTER TABLE nodes RENAME TO oldnodes') + c.execute('''CREATE TABLE nodes ( + id INTEGER PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + ip TEXT NOT NULL, + lastVersion INTEGER, +checked TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)''') + #c.execute('DROP INDEX byChecked;') + c.execute('CREATE INDEX byChecked ON nodes(checked)') + c.execute('INSERT INTO nodes (id,key,checked) SELECT id,key,checked FROM oldnodes') + redoLinks(c) + c.execute('DROP TABLE oldnodes') + +def retry_on_locked(s): + def deco(f): + def wrapper(*a,**kw): + while True: + try: + return f(*a,**kw) + except sqlite3.OperationalError as e: + if e.error_code != 5: + raise + print(e.args) + time.sleep(s) + return wrapper + return deco + +def get_version(ident): + with conn(),closing(l.conn.cursor()) as c: + c.execute('SELECT lastVersion FROM nodes WHERE id = ?',(ident,)) + row = c.fetchone() + if row: return row[0] + +@retry_on_locked(1) +def get_peers(key,lastVersion): + with conn(),closing(l.conn.cursor()) as c: + ident = peer2node(key,c,lastVersion) + c.execute("SELECT checked > datetime('now','-1 hour') FROM nodes WHERE id = ?",(ident,)) + ok = c.fetchone() + if not ok or not ok[0]: + return ident,() + c.execute("SELECT (SELECT ip FROM nodes WHERE id = blue),blue FROM links WHERE red = ?",(ident,)) + return ident,c.fetchall() + +@retry_on_locked(1) +def set_peers(key,peers,lastVersion): + with conn(), closing(l.conn.cursor()) as c: + # getPeers doesn't return the peer's versions, only their key. + peers = [peer2node(peer,c) for peer in peers] + ident = peer2node(key,c,lastVersion) + peers = [peer for peer in peers if peer != ident] + for p in peers: + c.execute('INSERT OR REPLACE INTO links (red,blue) VALUES (?,?)', + (ident,p)) + c.execute("UPDATE nodes SET checked = datetime('now') WHERE id = ?",(ident,)) + return get_peers(key,lastVersion) + +def peer2node(key,c,lastVersion=None): + c.execute('SELECT id FROM nodes WHERE key = ?',(key,)) + ident = c.fetchone() + if ident: + ident = ident[0] + if lastVersion: + c.execute('SELECT lastVersion FROM nodes WHERE id = ?', + (ident,)) + test = c.fetchone() + if test and test[0] is not None and test[0] != lastVersion: + c.execute('UPDATE nodes SET lastVersion = ? WHERE id = ?', + (lastVersion,ident)) + return ident + c.execute('INSERT INTO nodes (key,ip,lastVersion) VALUES (?,?,?)', + (key,key2ip(key),lastVersion)) + return c.lastrowid diff --git a/scripts/updateGraph.py b/scripts/updateGraph.py new file mode 100644 index 0000000..1fad607 --- /dev/null +++ b/scripts/updateGraph.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 + +# Based on Kyrias' sendGraph script. Requires Python 3, requests and cjdns. +# You can install them using pip: pip3 install cjdns requests +############################################################################### +# CONFIG + +# URL where data is sent +# www.fc00.org for clearnet access +# h.fc00.org for hyperboria +# [fc53:dcc5:e89d:9082:4097:6622:5e82:c654] for DNS-less access +url = 'http://www.fc00.org/sendGraph' + +# update your email address, so I can contact you in case something goes wrong +your_mail = 'your@email.here' + + +# ---------------------- +# RPC connection details +# ---------------------- + +# If this is set to True connection details will be loaded from ~/.cjdnsadmin +cjdns_use_default = True + +# otherwise these are used. +cjdns_ip = '127.0.0.1' +cjdns_port = 11234 +cjdns_password = 'NONE' + +############################################################################### +import db +from pprint import pprint +import sys +import traceback +import json +import argparse + +import requests + +import cjdns +from cjdns import key_utils +from cjdns import admin_tools + +import queue +from concurrent.futures import ThreadPoolExecutor +import threading + +def addpeersto(d,n,ip,peers=set()): + if n in d: + d[n]['peers'].update(peers) + else: + d[n] = { + 'ip': ip, + 'peers': set(peers) + } + +def main(): + db.fixkeys(key_utils.to_ipv6) + parser = argparse.ArgumentParser(description='Submit nodes and links to fc00') + parser.add_argument('-v', '--verbose', help='increase output verbosity', + dest='verbose', action='store_true') + parser.set_defaults(verbose=False) + args = parser.parse_args() + + con = connect() + + nodes = dump_node_store(con) + edges = {} + + get_peer_queue = queue.Queue(0) + result_queue = queue.Queue(0) + e = ThreadPoolExecutor(max_workers=4) + def args(): + for ip,node in nodes.items(): + yield ip,keyFromAddr(node['addr']),node['path'],node['version'] + args = zip(*args()) + dbnodes = {} + for peers, node_id, ip in e.map(get_peers_derp, *args): + get_edges_for_peers(edges, peers, node_id) + addpeersto(dbnodes,node_id,ip,peers) + + for ip, id in peers: + addpeersto(dbnodes,id,ip) + print('otay!') + send_graph(dbnodes, edges) + sys.exit(0) + +local = threading.local() + +def con(): + try: return local.con + except AttributeError: pass + con = connect() + local.con = con + return con + +def get_peers_derp(ip,key,path,version): + print('check',ip,version) + ident,peers = db.get_peers(key,version) + if not peers: + peers = get_all_peers(con(), path) + print(('adding peers to db',len(peers))) + ident,peers = db.set_peers(key,peers,version) + else: + print(('got db peers!',len(peers))) + return peers,ident,ip +def connect(): + try: + if cjdns_use_default: + print('Connecting using default or ~/.cjdnsadmin credentials...') + con = cjdns.connectWithAdminInfo() + else: + print('Connecting to port {:d}...'.format(cjdns_port)) + con = cjdns.connect(cjdns_ip, cjdns_port, cjdns_password) + + return con + + except: + print('Connection failed!') + print(traceback.format_exc()) + sys.exit(1) + + +def dump_node_store(con): + nodes = dict() + + i = 0 + while True: + res = con.NodeStore_dumpTable(i) + + if not 'routingTable' in res: + break + + for n in res['routingTable']: + if not all(key in n for key in ('addr', 'path', 'ip')): + continue + + ip = n['ip'] + path = n['path'] + addr = n['addr'] + version = None + if 'version' in n: + version = n['version'] + + nodes[ip] = {'ip': ip, 'path': path, 'addr': addr, 'version': version} + + if not 'more' in res or res['more'] != 1: + break + + i += 1 + + return nodes + + +def get_peers(con, path, nearbyPath=''): + formatted_path = path + if nearbyPath: + formatted_path = '{:s} (nearby {:s})'.format(path, nearbyPath) + + i = 1 + retry = 2 + while i < retry + 1: + if nearbyPath: + res = con.RouterModule_getPeers(path, nearbyPath=nearbyPath) + else: + res = con.RouterModule_getPeers(path) + + if res['error'] == 'not_found': + print('get_peers: node with path {:s} not found, skipping.' + .format(formatted_path)) + return [] + + elif res['error'] != 'none': + print('get_peers: failed with error `{:s}` on {:s}, trying again. {:d} tries remaining.' + .format(res['error'], formatted_path, retry-i)) + elif res['result'] == 'timeout': + print('get_peers: timed out on {:s}, trying again. {:d} tries remaining.' + .format(formatted_path, retry-i)) + else: + return res['peers'] + + i += 1 + + print('get_peers: failed on final try, skipping {:s}' + .format(formatted_path)) + return [] + + +def get_all_peers(con, path): + peers = set() + keys = set() + + res = get_peers(con, path) + peers.update(res) + + if not res: + return keys + + last_peer = res[-1] + checked_paths = set() + while len(res) > 1: + last_path = (last_peer.split('.', 1)[1] + .rsplit('.', 2)[0]) + + if last_path in checked_paths: + break + else: + checked_paths.add(last_path) + + res = get_peers(con, path, last_path) + if res: + last_peer = res[-1] + else: + break + + peers.update(res) + + for peer in peers: + key = keyFromAddr(peer) + keys |= {key} + + return keys + +def keyFromAddr(addr): + return addr.split('.', 5)[-1] + +def get_edges_for_peers(edges, peers, node_key): + for derp in peers: + try: ip,peer_key = derp + except: + pprint(peers) + raise + if node_key > peer_key: + A = node_key + B = peer_key + else: + A = peer_key + B = node_key + + edge = { 'a': A, + 'b': B } + + if A not in edges: + edges[A] = [] + edges[A] = B + +def send_graph(nodes, edges): + print('Nodes: {:d}\nEdges: {:d}\n'.format(len(nodes), len(edges))) + + with open('out.dot','wt') as out: + out.write('digraph cjdns {\n') + out.write(" overlap=false;\n"); + for ident,node in nodes.items(): + out.write(' n{} [label="{}"];\n'.format( + ident, + node['ip'].rsplit(':',1)[-1])) + for node,peer in edges.items(): + out.write(' n{} -> n{}\n'.format( + node, + peer)); + out.write('}\n') + + graph = { + 'nodes': + [], + 'edges': [{'a': nodes[A]['ip'], + 'b': nodes[B]['ip']} for A,B in edges.items()] + } + for ident,node in nodes.items(): + version = db.get_version(ident) + if version is None: + continue + graph['nodes'].append({ + 'ip': node['ip'], + 'version': version + } ) + json_graph = json.dumps(graph) + print(json_graph) + return + print('Sending data to {:s}...'.format(url)) + + payload = {'data': json_graph, 'mail': your_mail, 'version': 2} + r = requests.post(url, data=payload) + + if r.text == 'OK': + print('Done!') + else: + print('{:s}'.format(r.text)) + +if __name__ == '__main__': + main() diff --git a/web/graphData.py b/web/graphData.py index aa9a2dd..e529767 100644 --- a/web/graphData.py +++ b/web/graphData.py @@ -1,31 +1,44 @@ import json + from database import NodeDB + from graph import Node, Edge + import traceback + import time + + def insert_graph_data(config, data, mail, ip, version): + try: graph_data = json.loads(data) except ValueError: return 'Invalid JSON' + log = '[%s] ip: %s, version: %d, mail: %r, nodes: %d, edges: %d' % ( time.strftime('%Y-%m-%d %H:%M:%S'), ip, version, mail, len(graph_data['nodes']), len(graph_data['edges'])) + with open(config['LOG'], 'a') as f: f.write(log + '\n') + if mail == 'your@email.here': return 'Please change email address in config.' + if version != 2: return 'You are using outdated version of sendGraph script. Get new version from https://github.com/zielmicha/fc00.org/blob/master/scripts/sendGraph.py' + nodes = dict() edges = [] + try: for n in graph_data['nodes']: try: @@ -34,6 +47,7 @@ def insert_graph_data(config, data, mail, ip, version): except Exception: pass + for e in graph_data['edges']: try: edge = Edge(nodes[e['a']], nodes[e['b']]) @@ -43,13 +57,17 @@ def insert_graph_data(config, data, mail, ip, version): except Exception: return 'Invalid JSON nodes' + print "Accepted %d nodes and %d links." % (len(nodes), len(edges)) + if len(nodes) == 0 or len(edges) == 0: return 'No valid nodes or edges' + uploaded_by = ip + try: with NodeDB(config) as db: db.insert_graph(nodes, edges, uploaded_by) @@ -57,4 +75,5 @@ def insert_graph_data(config, data, mail, ip, version): traceback.print_exc() return 'Database failure' + return None diff --git a/web/graphPlotter.py b/web/graphPlotter.py index ff48d44..d438202 100644 --- a/web/graphPlotter.py +++ b/web/graphPlotter.py @@ -1,44 +1,66 @@ import pygraphviz as pgv + import time + import json + import networkx as nx + from networkx.algorithms import centrality + + def position_nodes(nodes, edges): + G = pgv.AGraph(strict=True, directed=False, size='10!') + for n in nodes.values(): G.add_node(n.ip, label=n.label, version=n.version) + for e in edges: G.add_edge(e.a.ip, e.b.ip, len=1.0) + G.layout(prog='neato', args='-Gepsilon=0.0001 -Gmaxiter=100000') + return G + def compute_betweenness(G): + ng = nx.Graph() for start in G.iternodes(): others = G.neighbors(start) for other in others: ng.add_edge(start, other) + c = centrality.betweenness_centrality(ng) + for k, v in c.items(): c[k] = v + return c + def canonalize_ip(ip): + return ':'.join( i.rjust(4, '0') for i in ip.split(':') ) + def load_db(): + with open('nodedb/nodes') as f: return dict([ (canonalize_ip(v[0]), v[1]) for v in [ l.split(None)[:2] for l in f.readlines() ] if len(v) > 1 ]) + def get_graph_json(G): + max_neighbors = 1 for n in G.iternodes(): neighbors = len(G.neighbors(n)) @@ -46,15 +68,18 @@ def get_graph_json(G): max_neighbors = neighbors print 'Max neighbors: %d' % max_neighbors + out_data = { 'created': int(time.time()), 'nodes': [], 'edges': [] } + centralities = compute_betweenness(G) db = load_db() + for n in G.iternodes(): neighbor_ratio = len(G.neighbors(n)) / float(max_neighbors) pos = n.attr['pos'].split(',', 1) @@ -63,6 +88,7 @@ def get_graph_json(G): size = (pcentrality ** 0.3 / 500) * 1000 + 1 name = db.get(n.name) + out_data['nodes'].append({ 'id': n.name, 'label': name if name else n.attr['label'], @@ -75,26 +101,35 @@ def get_graph_json(G): 'centrality': '%.4f' % centrality }) + for e in G.iteredges(): out_data['edges'].append({ 'sourceID': e[0], 'targetID': e[1] }) + return json.dumps(out_data) + + def _gradient_color(ratio, colors): + jump = 1.0 / (len(colors) - 1) gap_num = int(ratio / (jump + 0.0000001)) + a = colors[gap_num] b = colors[gap_num + 1] + ratio = (ratio - gap_num * jump) * (len(colors) - 1) + r = a[0] + (b[0] - a[0]) * ratio g = a[1] + (b[1] - a[1]) * ratio b = a[2] + (b[2] - a[2]) * ratio + return '#%02x%02x%02x' % (r, g, b) diff --git a/web/updateGraph.py b/web/updateGraph.py index 1777863..218ad95 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -1,29 +1,45 @@ #!/usr/bin/env python + from flask import Config + from database import NodeDB + import graphPlotter + + + def generate_graph(time_limit=60*60*3): + nodes, edges = load_graph_from_db(time_limit) print '%d nodes, %d edges' % (len(nodes), len(edges)) + graph = graphPlotter.position_nodes(nodes, edges) json = graphPlotter.get_graph_json(graph) + with open('static/graph.json', 'w') as f: f.write(json) + + def load_graph_from_db(time_limit): + config = Config('./') config.from_pyfile('web_config.cfg') + with NodeDB(config) as db: nodes = db.get_nodes(time_limit) edges = db.get_edges(nodes, 60*60*24*7) return (nodes, edges) + + if __name__ == '__main__': + generate_graph() diff --git a/web/web.py b/web/web.py index 3eea61b..114d3e7 100644 --- a/web/web.py +++ b/web/web.py @@ -1,10 +1,17 @@ from flask import Flask, render_template, request + from graphData import insert_graph_data + + app = Flask(__name__) + app.config.from_pyfile('web_config.cfg') + + def get_ip(): + try: ip = request.headers['x-real-ip'] except KeyError: @@ -13,22 +20,36 @@ def get_ip(): ip = request.headers['x-atomshare-real-ip'] return ip + @app.context_processor + def add_ip(): + return dict(ip=get_ip()) + + @app.route('/') + @app.route('/network') + def page_network(): + return render_template('network.html', page='network') + @app.route('/about') + def page_about(): + return render_template('about.html', page='about') + @app.route('/sendGraph', methods=['POST']) + def page_sendGraph(): + print "Receiving graph from %s" % (request.remote_addr) data = request.form['data'] @@ -36,10 +57,13 @@ def page_sendGraph(): version = int(request.form.get('version', '1')) ret = insert_graph_data(ip=get_ip(), config=app.config, data=data, mail=mail, version=version) + if ret == None: return 'OK' else: return 'Error: %s' % ret + if __name__ == '__main__': + app.run(host='localhost', port=3000)