better

main.py

@@ -5,4 +5,8 @@
 
 request.download_reflections_pages()
 master_list = reflection_scraper.get_peoples_words()
+f = open('master_list.js', 'w')
+s = json.dumps(master_list, f)
+f.write('var master_list='+s)
+f.close()
 db.save_words_in_reflections(master_list)

reflection_scraper.py

@@ -1,6 +1,7 @@
 from bs4 import BeautifulSoup
 import os
 import re
+import string
 
 #create_frequency_dict can't handle keywords with characters like
 #+, *, or ? that have special meanings in regex
@@ -14,6 +15,30 @@ def get_keywords(filename):
     keywords.append(line.strip())
   return keywords
 
+extra_stops = ['code', 'planned', 'learn', 'learned', 'learning', 'will', 'continue', 'time', "i'm", 'today', 'tomorrow', 'yesterday']
+
+def generate_popular(reflections_mapping, extra_stops):
+  '''input: mapping of names to reflections text
+  output: list of top 50 popular 'key' words'''
+  word_bucket = {}
+  file = open('stopword.txt', 'r')
+  dump = file.read()
+  file.close()
+  stopwords = dump.split()
+  for person in reflections_mapping:
+    words = reflections_mapping[person].strip().lower().split()
+    for word in words:
+      wordy = word.strip(string.punctuation+string.whitespace)
+      if stopwords.count(wordy) == 0:
+        if wordy in word_bucket:
+          word_bucket[wordy] += 1
+        else:
+          word_bucket[wordy] = 1
+  sorted = word_bucket.items()
+  sorted.sort(key=lambda x: x[1])
+  sorted.reverse()
+  return sorted[0:50]
+
 def get_file_names(dir_path):
   '''input: path to directory
   output: list of file names in directory'''
@@ -77,4 +102,4 @@ def get_peoples_words():
   reflections = get_reflections(file_names)
   reflections_dict = scrape_reflections(reflections)
   db_entries = create_db_entries(keywords, reflections_dict)
-  return db_entries
+  return db_entries

server.py

@@ -1,21 +1,26 @@
-from flask import Flask, render_template, request, send_from_directory
+from flask import Flask, render_template, request
 from pymongo import Connection
 import json
 import os
+import re
+
+import reflection_scraper
 
 app = Flask(__name__)
 connection = Connection()
 db = connection.reflections
 collection = db.words
 
+reflections_dict = {}
+
 @app.route('/')
 def start():
 	return render_template('index.html')
 
 #How do we return an icon???
 @app.route('/favicon.ico')
 def get_icon():
-	return send_from_directory(os.path.join(app.root_path, 'static'), 'pw.ico')
+	return
 
 @app.route('/<name>', methods=["GET"])
 	#create match_data to return as json object for display
@@ -27,20 +32,14 @@ def get_icon():
 	#described above.
 
 def get_JSON(name):
-
-	print "got request for " + name
-
 	match_data = {}
 
 	match_data['name'] = name
 	match_data['children'] = []
 
 	doc = collection.find_one({'name':name})
-	print "name: " + name
 	my_keywords = doc['keywords'].keys()
 
-	kw_matches_by_name = {}
-
 	for keyword in my_keywords:
 		word_data = {}
 		word_data['name'] = keyword
@@ -50,10 +49,8 @@ def get_JSON(name):
 		match_docs = collection.find({'keywords.'+keyword:{'$exists':True}}).sort('keywords.'+keyword, -1)
 
 		for match_doc in match_docs[:10]:
-			match_name = match_doc['name']
-			if match_name != name:
-				match_names.append(match_name)
-				kw_matches_by_name[match_name] = kw_matches_by_name.get(match_name, 0) + 1
+			if match_doc['name'] != name:
+				match_names.append(match_doc['name'])
 
 		for person in match_names:
 			# INCLUDE TO CONCEAL LAST NAMES
@@ -70,12 +67,25 @@ def get_JSON(name):
 
 		match_data['children'].append(word_data)
 
-	match_data['top'] = max(kw_matches_by_name, key=kw_matches_by_name.get)
-
 	return json.dumps(match_data)
+
+@app.route('/<name>/<term>')
+def get_mentions(name, term):
+	global reflections_dict
+	if len(reflections_dict) == 0:
+		reflections_dict = get_ref_dict(name, term)
+	print reflections_dict
+	places = [m.start() for m in re.finditer(term, reflections_dict[name])]
+	snippets = []
+	for loc in places:
+		snippets.append(['...' + reflections_dict[name][loc-100:loc+100] + '...'])
+	return json.dumps(snippets)
+
+def get_ref_dict(name, term):
+	return reflection_scraper.scrape_reflections(reflection_scraper.get_reflections(reflection_scraper.get_file_names('html/')))
 
 if __name__ == '__main__':
-	port = int(os.environ.get('PORT', 5000))
-	app.run(host='0.0.0.0', port=port)
+	port = int(os.environ.get('PORT', 5757))
+	app.run(debug=True)
 
 

static/style.css

@@ -12,22 +12,14 @@ h1 {
 	letter-spacing: -1px;
 }
 
-h2 {
-	font-family: 'Arbutus Slab', serif;
-	font-size: 24px;
-	text-align: center;
- 	color: #574D37;
- 	letter-spacing: -1px;
-}
-
 form {
 	text-align: center;
 }
 
 label {
 	font-family: 'Arbutus Slab', serif;
 	font-size: 18px;
-	color: #F82CBF;
+	color: #F93;
 }
 
 circle {
@@ -43,11 +35,10 @@ circle {
 }
 
 #submit {
-	background-color: #F82CBF;
+	background-color: #F93;
 	padding: 5px;
 	color: white;
 	font-size: 14px;
-	border: none;
 }
 
 .gallery {
@@ -65,4 +56,42 @@ circle {
 	font-size: 14px;
 	letter-spacing: 1px;
 	color: #574D37;
+}
+
+.tooltip {
+	position: absolute;
+	background-color: white;
+	padding: 5px;
+	border-radius: 10px;
+	box-shadow: 0 0 15px 5px #E3E3E3;
+	width: 300px;
+
+}
+
+.close{
+	width:15px;
+	height:15px;
+	border-radius:50px;
+	font-size:.75em;
+	font-family: sans-serif;
+	color: white;
+	text-align:center;
+	text-decoration:none;
+	background:red;
+	display: inline-block;
+	margin: 5px;
+	float: left;
+}
+
+.close:hover {
+	background-color: rgb(175,10,10);
+	cursor: pointer;
+}
+
+.hidden {
+	display: none;
+}
+
+.not-option {
+	color: #999;
 }

static/tree.js

@@ -2,7 +2,6 @@
 //see original at http://mbostock.github.com/d3/ex/tree.html
 
 var generate_tree = function(){
-	console.log("entering generate tree")
 	var radius = 960 / 2;
 
 	var tree = d3.layout.tree()
@@ -16,10 +15,10 @@ var generate_tree = function(){
 	document.getElementById("chart").innerHTML = "";
 
 	var vis = d3.select("#chart").append("svg")
-	    .attr("width", radius * 2 + 400)
+	    .attr("width", radius * 2 + 150)
 	    .attr("height", radius * 2 + 200)
 	  .append("g")
-	    .attr("transform", "translate(" + (radius + 200) + "," + (radius + 100) + ")");
+	    .attr("transform", "translate(" + (radius + 50) + "," + (radius + 100) + ")");
 
 	//grab input name
 	var name = document.getElementById('name').value
@@ -37,8 +36,18 @@ var generate_tree = function(){
 	      .data(nodes)
 	    .enter().append("g")
 	      .attr("class", "node")
-	      .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; });
-
+	      .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; })
+	      .attr('id', function(d) {
+	      	if (d.parent){
+	      		return d.name + '-'+ d.parent.name
+	      	}
+	      	else {
+	      		return d.name
+	      	}
+
+	      	})
+	      .attr('title', function(d){return d.depth})
+
 	  node.append("circle")
 	      .attr("r", 4.5);
 
@@ -49,20 +58,55 @@ var generate_tree = function(){
 	      .attr("transform", function(d) { return d.x < 180 ? "translate(8)" : "rotate(180)translate(-8)"; })
 	      .text(function(d) { return d.name; });
 
-	  var top_match = (nodes[0].top)
-	  $("#top").html("Top Match: " + top_match)
+	   	$('.node').click(function(e) {
+	   		var name = $.trim($(this).attr('id').split('-')[0].split('(')[0])
+			var parent = $(this).attr('id').split('-')[1]
+			var closed = null;
+			//if something's already open
+			if ($('.tooltip').length > 0) {
+				closed = $('.tooltip').remove()
+			}
+			//if it doesn't belong to just clicked thing
+			if ((!closed) || (closed.attr('id') !== name + '-' + parent)){
+				var depth = $(this).attr('title')
+				var x = e.pageX
+				var y = e.pageY-100
+				if (depth == 2){
+					$.get('/' + name + '/' + parent, function(data){
+						var lines = $.parseJSON(data)
+						var currDisp = 0
+						d3.select('#chart').append('div')
+							.style('top', y + 'px')
+							.style('left', x + 'px')
+							.attr('class', 'tooltip')
+							.attr('id', name + '-' + parent)
+							.html("<span class='close'>x</span> <p id='snip'>" + lines[currDisp]+ "</p><div><span id ='prev'>Previous</span> <span id='next'>Next</span></div>")
+
+						$('.close').click(function(){
+							$(this).parent().remove()
+						})
+
+						$('#next').click(function() {
+							currDisp = Math.min(currDisp+1, lines.length-1)
+							$('#snip').text(lines[currDisp][0])
+						})
+
+						$('#prev').click(function() {
+							currDisp = Math.max(currDisp-1, 0)
+							$('#snip').text(lines[currDisp][0])
+						})
+					});
+				}
+			}
+		})
 	});
 };
 
+
 $(document).ready(function(){
 	$('#form').submit(function(){
 		generate_tree();
 		return false;
 	});
-	$('#name').keypress(function(e){
-		if (e.keyCode===13) {
-			generate_tree();
-		}
-	});
 	$('#name').focus()
 });

templates/index.html

@@ -14,7 +14,6 @@ <h1>PairWise</h1>
 			<input id="name" type="text" name="fname">
 			<input type="submit" id="submit" value="GO"></br>
 		</form>
-		<h2 id="top"></h2>
 		<div class="gallery" id='chart'></div>
 	</body>	
 </html>