diff --git a/main.py b/main.py index 4e06260..eba4a64 100644 --- a/main.py +++ b/main.py @@ -5,4 +5,8 @@ request.download_reflections_pages() master_list = reflection_scraper.get_peoples_words() +f = open('master_list.js', 'w') +s = json.dumps(master_list, f) +f.write('var master_list='+s) +f.close() db.save_words_in_reflections(master_list) \ No newline at end of file diff --git a/reflection_scraper.py b/reflection_scraper.py index e137e8a..f39cedc 100644 --- a/reflection_scraper.py +++ b/reflection_scraper.py @@ -1,6 +1,7 @@ from bs4 import BeautifulSoup import os import re +import string #create_frequency_dict can't handle keywords with characters like #+, *, or ? that have special meanings in regex @@ -14,6 +15,30 @@ def get_keywords(filename): keywords.append(line.strip()) return keywords +extra_stops = ['code', 'planned', 'learn', 'learned', 'learning', 'will', 'continue', 'time', "i'm", 'today', 'tomorrow', 'yesterday'] + +def generate_popular(reflections_mapping, extra_stops): + '''input: mapping of names to reflections text + output: list of top 50 popular 'key' words''' + word_bucket = {} + file = open('stopword.txt', 'r') + dump = file.read() + file.close() + stopwords = dump.split() + for person in reflections_mapping: + words = reflections_mapping[person].strip().lower().split() + for word in words: + wordy = word.strip(string.punctuation+string.whitespace) + if stopwords.count(wordy) == 0: + if wordy in word_bucket: + word_bucket[wordy] += 1 + else: + word_bucket[wordy] = 1 + sorted = word_bucket.items() + sorted.sort(key=lambda x: x[1]) + sorted.reverse() + return sorted[0:50] + def get_file_names(dir_path): '''input: path to directory output: list of file names in directory''' @@ -77,4 +102,4 @@ def get_peoples_words(): reflections = get_reflections(file_names) reflections_dict = scrape_reflections(reflections) db_entries = create_db_entries(keywords, reflections_dict) - return db_entries + return db_entries \ No newline at end of file diff --git a/server.py b/server.py index dd89316..2f87a9d 100644 --- a/server.py +++ b/server.py @@ -1,13 +1,18 @@ -from flask import Flask, render_template, request, send_from_directory +from flask import Flask, render_template, request from pymongo import Connection import json import os +import re + +import reflection_scraper app = Flask(__name__) connection = Connection() db = connection.reflections collection = db.words +reflections_dict = {} + @app.route('/') def start(): return render_template('index.html') @@ -15,7 +20,7 @@ def start(): #How do we return an icon??? @app.route('/favicon.ico') def get_icon(): - return send_from_directory(os.path.join(app.root_path, 'static'), 'pw.ico') + return @app.route('/', methods=["GET"]) #create match_data to return as json object for display @@ -27,20 +32,14 @@ def get_icon(): #described above. def get_JSON(name): - - print "got request for " + name - match_data = {} match_data['name'] = name match_data['children'] = [] doc = collection.find_one({'name':name}) - print "name: " + name my_keywords = doc['keywords'].keys() - kw_matches_by_name = {} - for keyword in my_keywords: word_data = {} word_data['name'] = keyword @@ -50,10 +49,8 @@ def get_JSON(name): match_docs = collection.find({'keywords.'+keyword:{'$exists':True}}).sort('keywords.'+keyword, -1) for match_doc in match_docs[:10]: - match_name = match_doc['name'] - if match_name != name: - match_names.append(match_name) - kw_matches_by_name[match_name] = kw_matches_by_name.get(match_name, 0) + 1 + if match_doc['name'] != name: + match_names.append(match_doc['name']) for person in match_names: # INCLUDE TO CONCEAL LAST NAMES @@ -70,12 +67,25 @@ def get_JSON(name): match_data['children'].append(word_data) - match_data['top'] = max(kw_matches_by_name, key=kw_matches_by_name.get) - return json.dumps(match_data) + +@app.route('//') +def get_mentions(name, term): + global reflections_dict + if len(reflections_dict) == 0: + reflections_dict = get_ref_dict(name, term) + print reflections_dict + places = [m.start() for m in re.finditer(term, reflections_dict[name])] + snippets = [] + for loc in places: + snippets.append(['...' + reflections_dict[name][loc-100:loc+100] + '...']) + return json.dumps(snippets) + +def get_ref_dict(name, term): + return reflection_scraper.scrape_reflections(reflection_scraper.get_reflections(reflection_scraper.get_file_names('html/'))) if __name__ == '__main__': - port = int(os.environ.get('PORT', 5000)) - app.run(host='0.0.0.0', port=port) + port = int(os.environ.get('PORT', 5757)) + app.run(debug=True) diff --git a/static/style.css b/static/style.css index 5307706..51fbaf5 100644 --- a/static/style.css +++ b/static/style.css @@ -12,14 +12,6 @@ h1 { letter-spacing: -1px; } -h2 { - font-family: 'Arbutus Slab', serif; - font-size: 24px; - text-align: center; - color: #574D37; - letter-spacing: -1px; -} - form { text-align: center; } @@ -27,7 +19,7 @@ form { label { font-family: 'Arbutus Slab', serif; font-size: 18px; - color: #F82CBF; + color: #F93; } circle { @@ -43,11 +35,10 @@ circle { } #submit { - background-color: #F82CBF; + background-color: #F93; padding: 5px; color: white; font-size: 14px; - border: none; } .gallery { @@ -65,4 +56,42 @@ circle { font-size: 14px; letter-spacing: 1px; color: #574D37; +} + +.tooltip { + position: absolute; + background-color: white; + padding: 5px; + border-radius: 10px; + box-shadow: 0 0 15px 5px #E3E3E3; + width: 300px; + +} + +.close{ + width:15px; + height:15px; + border-radius:50px; + font-size:.75em; + font-family: sans-serif; + color: white; + text-align:center; + text-decoration:none; + background:red; + display: inline-block; + margin: 5px; + float: left; +} + +.close:hover { + background-color: rgb(175,10,10); + cursor: pointer; +} + +.hidden { + display: none; +} + +.not-option { + color: #999; } \ No newline at end of file diff --git a/static/tree.js b/static/tree.js index 76087e9..5b07316 100644 --- a/static/tree.js +++ b/static/tree.js @@ -2,7 +2,6 @@ //see original at http://mbostock.github.com/d3/ex/tree.html var generate_tree = function(){ - console.log("entering generate tree") var radius = 960 / 2; var tree = d3.layout.tree() @@ -16,10 +15,10 @@ var generate_tree = function(){ document.getElementById("chart").innerHTML = ""; var vis = d3.select("#chart").append("svg") - .attr("width", radius * 2 + 400) + .attr("width", radius * 2 + 150) .attr("height", radius * 2 + 200) .append("g") - .attr("transform", "translate(" + (radius + 200) + "," + (radius + 100) + ")"); + .attr("transform", "translate(" + (radius + 50) + "," + (radius + 100) + ")"); //grab input name var name = document.getElementById('name').value @@ -37,8 +36,18 @@ var generate_tree = function(){ .data(nodes) .enter().append("g") .attr("class", "node") - .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; }); - + .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; }) + .attr('id', function(d) { + if (d.parent){ + return d.name + '-'+ d.parent.name + } + else { + return d.name + } + + }) + .attr('title', function(d){return d.depth}) + node.append("circle") .attr("r", 4.5); @@ -49,20 +58,55 @@ var generate_tree = function(){ .attr("transform", function(d) { return d.x < 180 ? "translate(8)" : "rotate(180)translate(-8)"; }) .text(function(d) { return d.name; }); - var top_match = (nodes[0].top) - $("#top").html("Top Match: " + top_match) + $('.node').click(function(e) { + var name = $.trim($(this).attr('id').split('-')[0].split('(')[0]) + var parent = $(this).attr('id').split('-')[1] + var closed = null; + //if something's already open + if ($('.tooltip').length > 0) { + closed = $('.tooltip').remove() + } + //if it doesn't belong to just clicked thing + if ((!closed) || (closed.attr('id') !== name + '-' + parent)){ + var depth = $(this).attr('title') + var x = e.pageX + var y = e.pageY-100 + if (depth == 2){ + $.get('/' + name + '/' + parent, function(data){ + var lines = $.parseJSON(data) + var currDisp = 0 + d3.select('#chart').append('div') + .style('top', y + 'px') + .style('left', x + 'px') + .attr('class', 'tooltip') + .attr('id', name + '-' + parent) + .html("x

" + lines[currDisp]+ "

Previous Next
") + + $('.close').click(function(){ + $(this).parent().remove() + }) + + $('#next').click(function() { + currDisp = Math.min(currDisp+1, lines.length-1) + $('#snip').text(lines[currDisp][0]) + }) + + $('#prev').click(function() { + currDisp = Math.max(currDisp-1, 0) + $('#snip').text(lines[currDisp][0]) + }) + }); + } + } + }) }); }; + $(document).ready(function(){ $('#form').submit(function(){ generate_tree(); return false; }); - $('#name').keypress(function(e){ - if (e.keyCode===13) { - generate_tree(); - } - }); $('#name').focus() }); \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index d305d4e..04b8de6 100644 --- a/templates/index.html +++ b/templates/index.html @@ -14,7 +14,6 @@

PairWise


-

\ No newline at end of file