Merge pull request #235 from domino14/bug/234/sqlite_chunks

Get SQLite queries in chunks.

djAerolith/lib/word_db_helper.py

@@ -13,6 +13,7 @@
 from django.conf import settings
 
 logger = logging.getLogger(__name__)
+MAX_CHUNK_SIZE = 999
 
 
 class BadInput(Exception):
@@ -185,7 +186,6 @@ def __init__(self, lexicon_name):
                            lexicon_name))
         self.conn = sqlite3.connect(file_path)
 
-    # XXX: Only used by tests.
     def get_word_data(self, word):
         """
         Gets data for the word passed in.
@@ -207,18 +207,27 @@ def get_words_data(self, words):
         """ Gets data for the words passed in. """
         logger.debug('Getting word data for %s words.', len(words))
         c = self.conn.cursor()
-        c.execute(""" SELECT lexicon_symbols, definition, front_hooks,
-                  back_hooks, inner_front_hook, inner_back_hook, alphagram,
-                  word FROM words WHERE word IN (%s) ORDER BY word""" %
-                  ','.join('?' * len(words)), words)
-        rows = c.fetchall()
-        words = []
-        for row in rows:
-            words.append(Word(word=row[7], definition=row[1],
-                              front_hooks=row[2], back_hooks=row[3],
-                              inner_front_hook=row[4], inner_back_hook=row[5],
-                              lexicon_symbols=row[0], alphagram=row[6]))
-        return words
+        idx = 0
+        word_data = []
+        while idx < len(words):
+            these_words = words[idx:idx+MAX_CHUNK_SIZE]
+            num_words = len(these_words)
+            query = """
+            SELECT lexicon_symbols, definition, front_hooks,
+            back_hooks, inner_front_hook, inner_back_hook, alphagram,
+            word FROM words WHERE word IN (%s) ORDER BY word
+            """ % ','.join('?' * num_words)
+            idx += MAX_CHUNK_SIZE
+            c.execute(query, these_words)
+            rows = c.fetchall()
+
+            for row in rows:
+                word_data.append(
+                    Word(word=row[7], definition=row[1],
+                         front_hooks=row[2], back_hooks=row[3],
+                         inner_front_hook=row[4], inner_back_hook=row[5],
+                         lexicon_symbols=row[0], alphagram=row[6]))
+        return word_data
 
     # XXX: Only used by tests.
     def get_alphagram_data(self, alphagram):
@@ -238,10 +247,21 @@ def get_alphagrams_data(self, alphagrams):
 
         """
         c = self.conn.cursor()
-        c.execute('SELECT alphagram, probability, combinations FROM alphagrams'
-                  ' WHERE alphagram IN (%s)' % ','.join('?' * len(alphagrams)),
-                  alphagrams)
-        return self._alphagrams(c)
+        ret_alphagrams = []
+        idx = 0
+        while idx < len(alphagrams):
+            these_alphagrams = alphagrams[idx:idx+MAX_CHUNK_SIZE]
+            num_alphas = len(these_alphagrams)
+            c.execute(
+                'SELECT alphagram, probability, combinations FROM alphagrams'
+                ' WHERE alphagram IN (%s)' % ','.join('?' * num_alphas),
+                these_alphagrams)
+            ret_alphagrams.extend(self._alphagrams(c))
+            idx += MAX_CHUNK_SIZE
+
+        logger.debug('get_alphagrams_data returned %s alphagrams',
+                     len(ret_alphagrams))
+        return ret_alphagrams
 
     # XXX: Only used by tests.
     def probability(self, alphagram):
@@ -388,9 +408,8 @@ def get_questions(self, alphagrams):
         # Handle in 1000-alphagram chunks.
 
         idx = 0
-        CHUNK_SIZE = 999
         while idx < len(alphagrams):
-            these_alphagrams = alphagrams[idx:idx+CHUNK_SIZE]
+            these_alphagrams = alphagrams[idx:idx+MAX_CHUNK_SIZE]
             num_alphas = len(these_alphagrams)
             query = """
             SELECT lexicon_symbols, definition, front_hooks, back_hooks,
@@ -399,7 +418,7 @@ def get_questions(self, alphagrams):
             INNER JOIN alphagrams ON words.alphagram = alphagrams.alphagram
             WHERE alphagrams.alphagram IN (%s) """ % ','.join('?' * num_alphas)
 
-            idx += CHUNK_SIZE
+            idx += MAX_CHUNK_SIZE
 
             c.execute(query, [a.alphagram for a in these_alphagrams])
 

djAerolith/wordwalls/api.py

@@ -175,11 +175,14 @@ def new_challenge(request, parsed_req_body):
             pk=parsed_req_body['challenge'])
     except DailyChallengeName.DoesNotExist:
         return bad_request('Bad challenge.')
-    tablenum = WordwallsGame().initialize_daily_challenge(
-        request.user, parsed_req_body['lexicon'],
-        challenge_name,
-        date_from_str(parsed_req_body['dt']),
-        use_table=parsed_req_body['tablenum'])
+    try:
+        tablenum = WordwallsGame().initialize_daily_challenge(
+            request.user, parsed_req_body['lexicon'],
+            challenge_name,
+            date_from_str(parsed_req_body['dt']),
+            use_table=parsed_req_body['tablenum'])
+    except GameInitException as e:
+        return bad_request(str(e))
 
     return table_response(tablenum)
 

djAerolith/wordwalls/management/commands/genNamedLists.py

@@ -69,6 +69,9 @@ def get_questions_by_condition(db, min_prob, max_prob, length, condition,
 
     """
     qs = Questions()
+    logger.debug('Condition: min_prob=%s max_prob=%s length=%s condition=%s '
+                 'condition_type=%s', min_prob, max_prob, length, condition,
+                 condition_type)
 
     to_filter = db.get_questions_for_probability_range(min_prob, max_prob,
                                                        length)

djAerolith/wordwalls/tests/test_api.py

@@ -4,10 +4,10 @@
 
 from django.test import TestCase, Client, RequestFactory
 from django.db import connection
+from django.utils import timezone
 
 from wordwalls.api import date_from_request_dict
 from wordwalls.game import WordwallsGame
-from base.models import WordList
 logger = logging.getLogger(__name__)
 
 
@@ -63,7 +63,7 @@ def test_bad_date_from_request(self):
             '/wordwalls/api/challenges_played/?lexicon=1&date=04032014')
         dt = date_from_request_dict(request.GET)
         # This test might fail if run exactly one nanosecond before midnight.
-        self.assertEqual(dt, date.today())
+        self.assertEqual(dt, timezone.localtime(timezone.now()).date())
 
     def test_future_date_from_request(self):
         """ Test that entering a future date results in today. """
@@ -72,7 +72,7 @@ def test_future_date_from_request(self):
             '/wordwalls/api/challenges_played/?lexicon=1&date=2900-01-03')
         dt = date_from_request_dict(request.GET)
         # This test might fail if run exactly one nanosecond before midnight.
-        self.assertEqual(dt, date.today())
+        self.assertEqual(dt, timezone.localtime(timezone.now()).date())
 
 
 class WordwallsNewChallengeTest(TestCase):

djAerolith/wordwalls/tests/test_game_logic.py

@@ -4,15 +4,16 @@
 A class mostly to test the logic for wordwalls/game.py
 
 """
-from datetime import date
 import mock
 import re
 import json
 import logging
+from datetime import date
 
 from django.test import TestCase
 from django.contrib.auth.models import User
 from django.test.utils import override_settings
+from django.utils import timezone
 from django.db import connection
 
 from base.forms import SavedListForm
@@ -591,7 +592,8 @@ def test_length_challenge(self):
         num_challenges = DailyChallenge.objects.count()
         challenge = DailyChallengeName.objects.get(name="Today's 6s")
         table_id = self.wwg.initialize_daily_challenge(
-            self.user, self.lex, challenge, date.today())
+            self.user, self.lex, challenge,
+            timezone.localtime(timezone.now()).date())
         # Assert that it created a challenge.
         self.assertEqual(num_challenges + 1, DailyChallenge.objects.count())
         wgm = self.wwg.get_wgm(table_id)
@@ -613,7 +615,8 @@ def test_bingo_marathon_challenge(self):
         """ Test bingo marathon challenge. """
         challenge = DailyChallengeName.objects.get(name='Bingo Marathon')
         table_id = self.wwg.initialize_daily_challenge(
-            self.user, self.lex, challenge, date.today())
+            self.user, self.lex, challenge,
+            timezone.localtime(timezone.now()).date())
         wgm = self.wwg.get_wgm(table_id)
         state = json.loads(wgm.currentGameState)
         self.assertTrue(state['qualifyForAward'])
@@ -647,7 +650,8 @@ def test_blank_bingos(self, mock_content):
         """ Test blank bingos. (This comment is unnecessary, right?)"""
         challenge = DailyChallengeName.objects.get(name='Blank Bingos')
         table_id = self.wwg.initialize_daily_challenge(
-            self.user, self.lex, challenge, date.today())
+            self.user, self.lex, challenge,
+            timezone.localtime(timezone.now()).date())
         wgm = self.wwg.get_wgm(table_id)
         state = json.loads(wgm.currentGameState)
         self.assertTrue(state['qualifyForAward'])