Add a web-based status page (PokemonGoF#908, PokemonGoF#1090)

hoanghuynh · Sep 2, 2016 · 5b57512 · 5b57512
1 parent cc81a96
commit 5b57512
Show file tree

Hide file tree

Showing 10 changed files with 570 additions and 11 deletions.
diff --git a/Gruntfile.js b/Gruntfile.js
@@ -11,7 +11,8 @@ module.exports = function(grunt) {
         files: {
           'static/dist/css/app.built.css': 'static/sass/main.scss',
           'static/dist/css/mobile.built.css': 'static/sass/mobile.scss',
-          'static/dist/css/statistics.built.css': 'static/css/statistics.css'
+          'static/dist/css/statistics.built.css': 'static/css/statistics.css',
+          'static/dist/css/status.built.css': 'static/sass/status.scss'
         }
       }
     },
@@ -29,7 +30,8 @@ module.exports = function(grunt) {
           'static/dist/js/map.built.js': 'static/js/map.js',
           'static/dist/js/mobile.built.js': 'static/js/mobile.js',
           'static/dist/js/stats.built.js': 'static/js/stats.js',
-          'static/dist/js/statistics.built.js': 'static/js/statistics.js'
+          'static/dist/js/statistics.built.js': 'static/js/statistics.js',
+          'static/dist/js/status.built.js': 'static/js/status.js'
         }
       }
     },
@@ -47,7 +49,8 @@ module.exports = function(grunt) {
           'static/dist/js/map.min.js': 'static/dist/js/map.built.js',
           'static/dist/js/mobile.min.js': 'static/dist/js/mobile.built.js',
           'static/dist/js/stats.min.js': 'static/dist/js/stats.built.js',
-          'static/dist/js/statistics.min.js': 'static/dist/js/statistics.built.js'
+          'static/dist/js/statistics.min.js': 'static/dist/js/statistics.built.js',
+          'static/dist/js/status.min.js': 'static/dist/js/status.built.js'
         }
       }
     },
@@ -102,7 +105,8 @@ module.exports = function(grunt) {
         files: {
           'static/dist/css/app.min.css': 'static/dist/css/app.built.css',
           'static/dist/css/mobile.min.css': 'static/dist/css/mobile.built.css',
-          'static/dist/css/statistics.min.css': 'static/dist/css/statistics.built.css'
+          'static/dist/css/statistics.min.css': 'static/dist/css/statistics.built.css',
+          'static/dist/css/status.min.css': 'static/dist/css/status.built.css'
         }
       }
     }

diff --git a/config/config.ini.example b/config/config.ini.example
@@ -21,6 +21,7 @@
 #step-limit:            # default 12
 #gym-info:              # enables detailed gym info collection (default false)
 #min-seconds-left:      # time that must be left on a spawn before considering it too late and skipping it (default 0)
+#status-name            # enables writing status updates to the database - if you use multiple processes, each needs a unique value
 
 # Misc
 #gmaps-key:             # your Google Maps API key
@@ -35,6 +36,7 @@
 #ssl-certificate:       # path to ssl certificate
 #ssl-privatekey:        # path to ssl private key
 #encrypt-lib:           # path to encrypt lib to be used instead of the shipped ones
+#status-page-password   # enables and protects the /status page to view status of all workers
 
 #Uncomment a line when you want to change its default value (Remove # at the beginning)
 #username, password, location and gmaps-key are required
diff --git a/docs/extras/status-page.md b/docs/extras/status-page.md
@@ -0,0 +1,17 @@
+# Status Page
+
+Keeping track of multiple processes and accounts can be a pain. To help, you can use the status page to view the status of each of your workers and their accounts.
+
+## Setup
+
+There are two steps to enable the status page:
+1. Set a password for the status page by adding the `-spp` argument to each worker running the web service or by setting the `status-page-password` field in `config.ini`. A password is required to enable the status page.
+2. Give each of your workers a unique "status name" to identify them on the status page by setting the `-sn` argument.
+
+## Accessing
+To view your status page, go to `<YourMapUrl>/status` (for example, `http://localhost:5050/status`) and enter the password you defined. The status of each of your workers will be displayed and continually update.
+
+## Screenshots
+
+![Example Login Page](https://i.imgur.com/TEBNprW.png)
+![Example Status Page](https://i.imgur.com/ieu5w1V.png)
diff --git a/pogom/app.py b/pogom/app.py
@@ -4,7 +4,7 @@
 import calendar
 import logging
 
-from flask import Flask, jsonify, render_template, request
+from flask import Flask, abort, jsonify, render_template, request
 from flask.json import JSONEncoder
 from flask_compress import Compress
 from datetime import datetime
@@ -14,7 +14,7 @@
 from collections import OrderedDict
 
 from . import config
-from .models import Pokemon, Gym, Pokestop, ScannedLocation
+from .models import Pokemon, Gym, Pokestop, ScannedLocation, MainWorker, WorkerStatus
 
 log = logging.getLogger(__name__)
 compress = Compress()
@@ -33,6 +33,8 @@ def __init__(self, import_name, **kwargs):
         self.route("/search_control", methods=['GET'])(self.get_search_control)
         self.route("/search_control", methods=['POST'])(self.post_search_control)
         self.route("/stats", methods=['GET'])(self.get_stats)
+        self.route("/status", methods=['GET'])(self.get_status)
+        self.route("/status", methods=['POST'])(self.post_status)
 
     def set_search_control(self, control):
         self.search_control = control
@@ -117,6 +119,15 @@ def raw_data(self):
         if request.args.get('spawnpoints', 'false') == 'true':
             d['spawnpoints'] = Pokemon.get_spawnpoints(swLat, swLng, neLat, neLng)
 
+        if request.args.get('status', 'false') == 'true':
+            args = get_args()
+            d = {}
+            if args.status_page_password is None:
+                d['error'] = 'Access denied'
+            elif request.args.get('password', None) == args.status_page_password:
+                d['main_workers'] = MainWorker.get_all()
+                d['workers'] = WorkerStatus.get_all()
+
         return jsonify(d)
 
     def loc(self):
@@ -232,6 +243,27 @@ def get_stats(self):
                                valid_input=self.get_valid_stat_input()
                                )
 
+    def get_status(self):
+        args = get_args()
+        if args.status_page_password is None:
+            abort(404)
+
+        return render_template('status.html')
+
+    def post_status(self):
+        args = get_args()
+        d = {}
+        if args.status_page_password is None:
+            abort(404)
+
+        if request.form.get('password', None) == args.status_page_password:
+            d['login'] = 'ok'
+            d['main_workers'] = MainWorker.get_all()
+            d['workers'] = WorkerStatus.get_all()
+        else:
+            d['login'] = 'failed'
+        return jsonify(d)
+
 
 class CustomJSONEncoder(JSONEncoder):
 

diff --git a/pogom/models.py b/pogom/models.py
@@ -452,6 +452,39 @@ def get_recent(swLat, swLng, neLat, neLng):
         return list(query)
 
 
+class MainWorker(BaseModel):
+    worker_name = CharField(primary_key=True, max_length=50)
+    message = CharField()
+    method = CharField(max_length=50)
+    last_modified = DateTimeField(index=True)
+
+
+class WorkerStatus(BaseModel):
+    username = CharField(primary_key=True, max_length=50)
+    worker_name = CharField()
+    success = IntegerField()
+    fail = IntegerField()
+    no_items = IntegerField()
+    skip = IntegerField()
+    last_modified = DateTimeField(index=True)
+    message = CharField(max_length=255)
+
+    @staticmethod
+    def get_recent():
+        query = (WorkerStatus
+                 .select()
+                 .where((WorkerStatus.last_modified >=
+                        (datetime.utcnow() - timedelta(minutes=5))))
+                 .order_by(WorkerStatus.username)
+                 .dicts())
+
+        status = []
+        for s in query:
+            status.append(s)
+
+        return status
+
+
 class Versions(flaskDb.Model):
     key = CharField()
     val = IntegerField()
@@ -809,12 +842,23 @@ def db_updater(args, q):
 def clean_db_loop(args):
     while True:
         try:
-
             # Clean out old scanned locations
             query = (ScannedLocation
                      .delete()
                      .where((ScannedLocation.last_modified <
-                            (datetime.utcnow() - timedelta(minutes=30)))))
+                             (datetime.utcnow() - timedelta(minutes=30)))))
+            query.execute()
+
+            query = (MainWorker
+                     .delete()
+                     .where((ScannedLocation.last_modified <
+                             (datetime.utcnow() - timedelta(minutes=30)))))
+            query.execute()
+
+            query = (WorkerStatus
+                     .delete()
+                     .where((ScannedLocation.last_modified <
+                             (datetime.utcnow() - timedelta(minutes=30)))))
             query.execute()
 
             # Remove active modifier from expired lured pokestops
@@ -855,13 +899,13 @@ def bulk_upsert(cls, data):
 def create_tables(db):
     db.connect()
     verify_database_schema(db)
-    db.create_tables([Pokemon, Pokestop, Gym, ScannedLocation, GymDetails, GymMember, GymPokemon, Trainer], safe=True)
+    db.create_tables([Pokemon, Pokestop, Gym, ScannedLocation, GymDetails, GymMember, GymPokemon, Trainer, MainWorker, WorkerStatus], safe=True)
     db.close()
 
 
 def drop_tables(db):
     db.connect()
-    db.drop_tables([Pokemon, Pokestop, Gym, ScannedLocation, Versions, GymDetails, GymMember, GymPokemon, Trainer], safe=True)
+    db.drop_tables([Pokemon, Pokestop, Gym, ScannedLocation, Versions, GymDetails, GymMember, GymPokemon, Trainer, MainWorker, WorkerStatus, Versions], safe=True)
     db.close()
 
 

diff --git a/pogom/search.py b/pogom/search.py
@@ -26,6 +26,7 @@
 import geopy
 import geopy.distance
 
+from datetime import datetime
 from operator import itemgetter
 from threading import Thread
 from queue import Queue, Empty
@@ -35,7 +36,7 @@
 from pgoapi import utilities as util
 from pgoapi.exceptions import AuthException
 
-from .models import parse_map, Pokemon, hex_bounds, GymDetails, parse_gyms
+from .models import parse_map, Pokemon, hex_bounds, GymDetails, parse_gyms, MainWorker, WorkerStatus
 from .transform import generate_location_steps
 from .fakePogoApi import FakePogoApi
 from .utils import now
@@ -169,6 +170,38 @@ def status_printer(threadStatus, search_items_queue, db_updates_queue, wh_queue)
         time.sleep(1)
 
 
+def worker_status_db_thread(threads_status, name, db_updates_queue):
+    log.info("Clearing previous statuses for '%s' worker", name)
+    WorkerStatus.delete().where(WorkerStatus.worker_name == name).execute()
+
+    while True:
+        workers = {}
+        overseer = None
+        for status in threads_status.values():
+            if status['type'] == 'Overseer':
+                overseer = {
+                    'worker_name': name,
+                    'message': status['message'],
+                    'method': status['method'],
+                    'last_modified': datetime.utcnow()
+                }
+            if status['type'] == 'Worker':
+                workers[status['user']] = {
+                    'username': status['user'],
+                    'worker_name': name,
+                    'success': status['success'],
+                    'fail': status['fail'],
+                    'no_items': status['noitems'],
+                    'skip': status['skip'],
+                    'last_modified': datetime.utcnow(),
+                    'message': status['message']
+                }
+        if overseer is not None:
+            db_updates_queue.put((MainWorker, {0: overseer}))
+            db_updates_queue.put((WorkerStatus, workers))
+        time.sleep(3)
+
+
 # The main search loop that keeps an eye on the over all process
 def search_overseer_thread(args, method, new_location_queue, pause_bit, encryption_lib_path, db_updates_queue, wh_queue):
 
@@ -191,6 +224,14 @@ def search_overseer_thread(args, method, new_location_queue, pause_bit, encrypti
         t.daemon = True
         t.start()
 
+    if args.status_name is not None:
+        log.info('Starting status database thread')
+        t = Thread(target=worker_status_db_thread,
+                   name='status_worker_db',
+                   args=(threadStatus, args.status_name, db_updates_queue))
+        t.daemon = True
+        t.start()
+
     # Create a search_worker_thread per account
     log.info('Starting search worker threads')
     for i, account in enumerate(args.accounts):

diff --git a/pogom/utils.py b/pogom/utils.py
@@ -154,6 +154,10 @@ def get_args():
     parser.add_argument('--ssl-privatekey', help='Path to SSL private key file')
     parser.add_argument('-ps', '--print-status', action='store_true',
                         help='Show a status screen instead of log messages. Can switch between status and logs by pressing enter.', default=False)
+    parser.add_argument('-sn', '--status-name', default=None,
+                        help='Enable status page database update using STATUS_NAME as main worker name')
+    parser.add_argument('-spp', '--status-page-password', default=None,
+                        help='Set the status page password')
     parser.add_argument('-el', '--encrypt-lib', help='Path to encrypt lib to be used instead of the shipped ones')
     verbosity = parser.add_mutually_exclusive_group()
     verbosity.add_argument('-v', '--verbose', help='Show debug messages from PomemonGo-Map and pgoapi. Optionally specify file to log to.', nargs='?', const='nofile', default=False, metavar='filename.log')