Background process to compute client-specific result scores

.gitignore

@@ -2,3 +2,4 @@
 *.pdf
 *.swp
 *debug.log
+.DS_Store

CFC_DataCollector/result_precompute/precompute_results.py

@@ -0,0 +1,41 @@
+import logging
+
+import sys
+import os
+
+from get_database import get_uuid_db
+
+sys.path.append("%s/../CFC_WebApp/" % os.getcwd())
+from main import userclient, carbon
+from dao.user import User
+
+class PrecomputeResults:
+    def __init__(self):
+        pass
+
+    # This should really be pulled out into a separate default client
+    def precomputeDefault(self, user_uuid):
+      user = User.fromUUID(user_uuid)
+      # carbon compare results is a tuple. Tuples are converted to arrays
+      # by mongodb
+      # In [44]: testUser.setScores(('a','b', 'c', 'd'), ('s', 't', 'u', 'v'))
+      # In [45]: testUser.getScore()
+      # Out[45]: ([u'a', u'b', u'c', u'd'], [u's', u't', u'u', u'v'])
+      carbonCompareResults = carbon.getFootprintCompare(user_uuid)
+      user.setScores(None, carbonCompareResults)
+
+    def precomputeResults(self):
+        for user_uuid_dict in get_uuid_db().find({}, {'uuid': 1, '_id': 0}):
+            logging.info("Computing precomputed results for ")
+            userclient.runClientSpecificBackgroundTasks(user_uuid_dict['uuid'], self.precomputeDefault)
+
+if __name__ == '__main__':
+    import json
+
+    config_data = json.load(open('config.json'))
+    log_base_dir = config_data['paths']['log_base_dir']
+    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
+                        filename="%s/precompute_results.log" % log_base_dir, level=logging.DEBUG)
+
+    pr = PrecomputeResults()
+    pr.precomputeResults()

CFC_DataCollector/tests/result_precompute/TestPrecomputeResults.py

@@ -0,0 +1,105 @@
+import unittest
+import json
+import logging
+from get_database import get_db, get_mode_db, get_section_db
+from result_precompute import precompute_results
+from datetime import datetime, timedelta
+import sys
+import os
+
+# print "old path is %s" % sys.path
+sys.path.append("%s/../CFC_WebApp/" % os.getcwd())
+sys.path.append("%s" % os.getcwd())
+# print "new path is %s" % sys.path
+from utils import load_database_json, purge_database_json
+
+from dao.user import User
+from dao.client import Client
+import tests.common
+
+logging.basicConfig(level=logging.DEBUG)
+
+class TestPrecomputeResults(unittest.TestCase):
+    def setUp(self):
+        self.testUsers = ["test@example.com", "best@example.com", "fest@example.com",
+                          "rest@example.com", "nest@example.com"]
+        self.serverName = 'localhost'
+
+        # Sometimes, we may have entries left behind in the database if one of the tests failed
+        # or threw an exception, so let us start by cleaning up all entries
+        tests.common.dropAllCollections(get_db())
+
+        self.ModesColl = get_mode_db()
+        self.assertEquals(self.ModesColl.find().count(), 0)
+
+        self.SectionsColl = get_section_db()
+        self.assertEquals(self.SectionsColl.find().count(), 0)
+
+        load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
+        load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testModeInferFile")
+
+        # Let's make sure that the users are registered so that they have profiles
+        for userEmail in self.testUsers:
+          User.register(userEmail)
+
+        self.now = datetime.now()
+        self.dayago = self.now - timedelta(days=1)
+        self.weekago = self.now - timedelta(weeks = 1)
+
+        for section in self.SectionsColl.find():
+          section['section_start_datetime'] = self.dayago
+          section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
+          if (section['confirmed_mode'] == 5):
+            # We only cluster bus and train trips
+            # And our test data only has bus trips
+            section['section_start_point'] = {u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728]}
+            section['section_end_point'] = {u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595]}
+          # print("Section start = %s, section end = %s" %
+          #   (section['section_start_datetime'], section['section_end_datetime']))
+          # Replace the user email with the UUID
+          section['user_id'] = User.fromEmail(section['user_id']).uuid
+          self.SectionsColl.save(section)
+          self.pr = precompute_results.PrecomputeResults()
+
+    def testDefaultPrecompute(self):
+        for email in self.testUsers:
+            currUser = User.fromEmail(email)
+            self.assertEqual(currUser.getScore(), (0, 0))
+
+        self.pr.precomputeResults()
+
+        for email in self.testUsers:
+            currUser = User.fromEmail(email)
+            (expectNone, carbonFootprint) = currUser.getScore()
+            self.assertEqual(expectNone, None)
+            self.assertEqual(len(carbonFootprint), 12)
+
+    def testClientSpecificPrecompute(self):
+        for email in self.testUsers:
+            currUser = User.fromEmail(email)
+            self.assertEqual(currUser.getScore(), (0, 0))
+
+        fakeEmail = "fest@example.com"
+
+        client = Client("testclient")
+        client.update(createKey = False)
+        tests.common.makeValid(client)
+
+        (resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
+        user = User.fromEmail(fakeEmail)
+        self.assertEqual(user.getFirstStudy(), 'testclient')
+
+        self.pr.precomputeResults()
+
+        self.assertEqual(user.getScore(), ('updatedPrev', 'updatedCurr'))
+
+        for email in self.testUsers:
+            if email != fakeEmail:
+                currUser = User.fromEmail(email)
+
+                (expectNone, carbonFootprint) = currUser.getScore()
+                self.assertEqual(expectNone, None)
+                self.assertEqual(len(carbonFootprint), 12)
+
+if __name__ == '__main__':
+    unittest.main()

CFC_WebApp/DesignDecisions.txt

@@ -67,4 +67,23 @@ to go with the second option.  In the future, we should consider having the
 static list periodically generated from the modules by an external script, in
 order to resolve the tradeoff
 
+--------------------------------------------------------------------------
+How do we run client specific background tasks?
 
+There are two options here:
+- Client specific: Keep track of a list of clients that require background
+  processing, and for each of those, call the client specific runBackgroundTasks method.
+  That method is responsible for determining the list of affected users and updating them.
+
+- User specific: Iterate over all users and call the corresponding client
+  specific runBackgroundTasks method, if the user has an associated client.
+
+The tradeoff here is between maintainability and performance. If there is a
+significant subset of users for whom we don't need to run a background task,
+then a client specific view might be more performant. But it requires clients
+that do need background tasks to add themselves to a separate list, and is a
+different mechanism than the one that we use for the filtering of the trips to
+confirm, for example.
+
+Since we currently anticipate using this for all clients, we will go with the
+user centered approach.

CFC_WebApp/api/cfc_webapp.py

@@ -260,13 +260,20 @@ def getCarbonCompare():
     return clientResult
   else:
     logging.debug("No overriding client result for user %s, returning default" % user_uuid)
-
+
+  user = User.fromUUID(user_uuid)
+  (ignore, currFootprint) = user.getScore()
+
+  if currFootprint == 0:
+    currFootprint = carbon.getFootprintCompare(user_uuid)
+    user.saveScores(None, currFootprint)
+
   (myModeShareCount, avgModeShareCount,
      myModeShareDistance, avgModeShareDistance,
      myModeCarbonFootprint, avgModeCarbonFootprint,
      myModeCarbonFootprintNoLongMotorized, avgModeCarbonFootprintNoLongMotorized, # ignored
      myOptimalCarbonFootprint, avgOptimalCarbonFootprint,
-     myOptimalCarbonFootprintNoLongMotorized, avgOptimalCarbonFootprintNoLongMotorized) = carbon.getFootprintCompare(user_uuid)
+     myOptimalCarbonFootprintNoLongMotorized, avgOptimalCarbonFootprintNoLongMotorized) = currFootprint
 
   renderedTemplate = template("compare.html",
                       myModeShareCount = json.dumps(myModeShareCount),

CFC_WebApp/clients/carshare/carshare.py

@@ -56,3 +56,6 @@ def clientSpecificSetters(uuid, sectionId, predictedModeMap):
 
 def getClientConfirmedModeField():
     return "auto_confirmed.mode"
+
+def runBackgroundTasks(uuid):
+  pass

CFC_WebApp/clients/gamified/gamified.py

@@ -121,3 +121,6 @@ def clientSpecificSetters(uuid, sectionId, predictedModeMap):
 
 def getClientConfirmedModeField():
   return None
+
+def runBackgroundTasks(uuid):
+  updateScore(uuid)

CFC_WebApp/clients/testclient/testclient.py

@@ -45,3 +45,10 @@ def clientSpecificSetters(uuid, sectionId, predictedModeMap):
 
 def getClientConfirmedModeField():
   return "test_auto_confirmed.mode"
+
+def runBackgroundTasks(uuid):
+  from dao.user import User
+
+  testuser = User.fromUUID(uuid)
+  testuser.setScores("updatedPrev", "updatedCurr")
+

CFC_WebApp/dao/client.py

@@ -126,6 +126,7 @@ def __validateKey(self, clientKey):
 # result will still be a NOP, though
   def __preRegister(self, userEmail):
     from dao.user import User
+    from main import userclient
 
     if User.isRegistered(userEmail):
       User.fromEmail(userEmail).setStudy(self.clientName)
@@ -146,7 +147,7 @@ def __preRegister(self, userEmail):
         e.msg = writeResult['err'][0]["errmsg"]
         raise e
     return (get_pending_signup_db().find({'study': self.clientName}).count(),
-            User.countForStudy(self.clientName))
+            userclient.countForStudy(self.clientName))
 
   def preRegister(self, clientKey, userEmail):
     if not self.__validateKey(clientKey):
@@ -188,6 +189,12 @@ def clientSpecificSetters(self, uuid, sectionId, predictedModeMap):
       return self.__loadModule().clientSpecificSetters(uuid, sectionId, predictedModeMap)
     else:
       return None
+
+  def runBackgroundTasks(self, uuid):
+    if self.isActive(datetime.now()):
+        self.__loadModule().runBackgroundTasks(uuid)
+    else:
+        logging.debug("Client is not active, skipping call...")
   # END: Standard customization hooks
 
   # This reads the combined set of queries from all clients

CFC_WebApp/dao/user.py

@@ -33,10 +33,6 @@ def fromUUID(user_uuid):
     user = User(user_uuid)
     return user
 
-  @staticmethod
-  def countForStudy(study):
-    return get_profile_db().find({'study_list': {'$in': [study]}}).count()
-
   def getProfile(self):
     return get_profile_db().find_one({'user_id': self.uuid})
 

CFC_WebApp/main/userclient.py

@@ -3,6 +3,7 @@
 # it into User or into Client.
 from dao.user import User
 from dao.client import Client
+from get_database import get_profile_db
 
 def getUserClient(user_uuid):
     study = User.fromUUID(user_uuid).getFirstStudy()
@@ -26,3 +27,25 @@ def getClientSpecificResult(user_uuid):
       return None
     else:
       return client.getResult(user_uuid)
+
+def runClientSpecificBackgroundTasks(user_uuid, defaultTasks):
+    client = getUserClient(user_uuid)
+    if client == None:
+      defaultTasks(user_uuid)
+    else:
+      client.runBackgroundTasks(user_uuid)
+
+def getClientQuery(clientName):
+    if clientName is None:
+        return {'study_list': {'$size': 0}}
+    else:
+        return {'study_list': {'$in': [clientName]}}
+
+def countForStudy(study):
+  return get_profile_db().find(getClientQuery(study)).count()
+
+def getUsersForClient(clientName):
+  # Find all users for this client
+  client_uuids = []
+  for user in get_profile_db().find(getClientQuery(clientName)):
+    client_uuids.append(user)

CFC_WebApp/tests/dao/TestUser.py

@@ -7,6 +7,7 @@
 from dao.user import User
 from dao.client import Client
 from tests import common
+from main import userclient
 
 import logging
 logging.basicConfig(level=logging.DEBUG)
@@ -24,7 +25,7 @@ def testIsNotRegistered(self):
     self.assertFalse(User.isRegistered('fake@fake.com'))
 
   def testCountForStudyZero(self):
-    self.assertEquals(User.countForStudy('testclient'), 0)
+    self.assertEquals(userclient.countForStudy('testclient'), 0)
 
   def testRegisterNonStudyUser(self):
     user = User.register('fake@fake.com')
@@ -46,35 +47,23 @@ def testIsRegistered(self):
     user = User.register('fake@fake.com')
     self.assertTrue(User.isRegistered('fake@fake.com'))
 
-  def testCountForStudy(self):
-    client = Client("testclient")
-    client.update(createKey = False)
-    common.makeValid(client)
-
-    (resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", "fake@fake.com")
-    self.assertEqual(resultPre, 1)
-    self.assertEqual(resultReg, 0)
-
-    user = User.register('fake@fake.com')
-    self.assertEquals(User.countForStudy('testclient'), 1)
-
   def testSetStudy(self):
     user = User.register('fake@fake.com')
     user.setStudy('testclient')
-    self.assertEquals(User.countForStudy('testclient'), 1)
+    self.assertEquals(userclient.countForStudy('testclient'), 1)
 
   def testUnsetStudyExists(self):
     user = User.register('fake@fake.com')
     user.setStudy('testclient')
-    self.assertEquals(User.countForStudy('testclient'), 1)
+    self.assertEquals(userclient.countForStudy('testclient'), 1)
 
     user.unsetStudy('testclient')
-    self.assertEquals(User.countForStudy('testclient'), 0)
+    self.assertEquals(userclient.countForStudy('testclient'), 0)
 
   def testUnsetStudyNotExists(self):
     user = User.register('fake@fake.com')
     user.unsetStudy('testclient')
-    self.assertEquals(User.countForStudy('testclient'), 0)
+    self.assertEquals(userclient.countForStudy('testclient'), 0)
 
   def testMergeDict(self):
     dict1 = {'a': 'a1', 'b': 'b1', 'c': 'c1'}