Fix #8 - Migration script to backup database file and restart the s…

dump_webcompat_to_db.py

@@ -4,13 +4,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import json
+import datetime
+
+from db import db_session
+from extract_id_title_url import get_webcompat_data
 from sqlalchemy import Column
 from sqlalchemy import String
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
-from extract_id_title_url import get_webcompat_data
-from db import db_session
-import json
 
 DB_PATH = 'sqlite:////tmp/test.db'
 
@@ -24,13 +26,24 @@ class Issue(Base):
     id = Column(String(128), unique=True, primary_key=True)
     summary = Column(String(256))
     url = Column(String(1024))
+    domain = Column(String(1024))
     body = Column(String(2048))
+    state = Column(String(128))
+    creation_time = Column(DateTime)
+    last_change_time = Column(DateTime)
 
-    def __init__(self, id, summary, url, body):
+    def __init__(self, id, summary, url, domain, body, state, creation_time, last_change_time):
         self.id = id
         self.summary = summary
         self.url = url
+        self.domain = domain
         self.body = body
+        self.state = state
+        self.creation_time = datetime.datetime.strptime(creation_time.split('Z')[0],
+                                                             "%Y-%m-%dT%H:%M:%S")
+        self.last_change_time = datetime.datetime.strptime(last_change_time.split('Z')[0],
+                                                             "%Y-%m-%dT%H:%M:%S")
+
 
 
 def main():
@@ -47,7 +60,7 @@ def main():
     # stuff data into database..
     for bug in data['bugs']:
         db_session.add(
-            Issue(bug['id'], bug['summary'], bug['url'], bug['body']))
+            Issue(bug['id'], bug['summary'], bug['url'], extract_domain_name(bug['url']), bug['body'], bug['state'], bug['creation_time'], bug['last_change_time']))
     db_session.commit()
 
 

extract_id_title_url.py

@@ -75,9 +75,12 @@ def extract_data(json_data, results_csv, results_bzlike):
         issue_state = issue['state'].encode('utf-8')
         cf_last_resolved = issue['closed_at']
         if issue_state == 'open':
-            status = 'OPEN'
+            state = 'OPEN'
         else:
-            status = 'RESOLVED'
+            state = 'RESOLVED'
+        for label in issue['labels']:
+            if 'status-' in label['name']:
+                label['name'] = label['name'].split('status-')[1]
         # Extracting the labels
         labels_list = [label['name'] for label in issue['labels']]
         # areWEcompatibleyet is only about mozilla bugs
@@ -109,7 +112,7 @@ def extract_data(json_data, results_csv, results_bzlike):
                       "op_sys": op_sys,
                       "creation_time": creation_time,
                       "last_change_time": last_change_time,
-                      "status": status,
+                      "state": state,
                       "cf_last_resolved": cf_last_resolved,
                       "resolution": resolution,
                       "body": body
@@ -142,7 +145,7 @@ def get_webcompat_data(url_repo=URL_REPO):
 
     Start with the first page and follow hypermedia links to explore the rest.
     '''
-    next_link = '%s/issues?per_page=100&page=1' % (url_repo)
+    next_link = '%s/issues?state=all&per_page=100&page=1' % (url_repo)
     results = []
     bzresults = []
 

webcompat_migration_backup_db.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+import datetime
+import time
+from subprocess import call
+
+import os
+from git import Repo, GitCommandError
+
+path = os.path.dirname(os.path.realpath(__file__))
+
+
+def run_db_migration():
+    current_ts = time.time()
+    time_stamp = datetime.datetime.fromtimestamp(current_ts).strftime('%Y-%m-%d-%H:%M:%S')
+    issue_backup_db = 'issues_backup_' + time_stamp + '.db'
+    repo = Repo(path)
+    assert not repo.bare
+    assert repo.refs['origin/master'] == repo.remotes.origin.refs.master
+    origin = repo.remotes.origin
+    index = repo.index
+    db_path = path + '/backup_db'
+    if os.path.exists(path + '/issues.db'):
+        if os.path.exists(db_path) and os.path.isdir(db_path):
+            #           check if a previous version of a backup exist and delete it
+            print 'Backup database already exists. Removing older version of the back up'
+            call('rm -r backup_db', shell=True)
+            call('git rm -r backup_db/.', shell=True)
+        print 'Switching to branch : master'
+        call('mkdir backup_db', shell=True)
+        try:
+            repo.heads.master.checkout()
+        except GitCommandError:
+            print 'Please make sure your current branch is clean and all the changes are either committed or stashed' \
+                  'before switching to master branch'
+
+        call('mv issues.db backup_db/' + issue_backup_db, shell=True)
+        index.add([path + '/backup_db/' + issue_backup_db])
+        index.commit('Adding new backup database at ' + time_stamp)
+        # push the backup database file to master branch of the repository
+        origin.push()
+        call('python extract_id_title_url.py 1', shell=True)
+        call('python run.py&', shell=True)
+        time.sleep(60)
+        call('python dump_webcompat_to_db.py 1', shell=True)
+    else:
+        raise IOError('FileNotFound : No database file found to backup. Please run python run.py to create the db file')
+
+if __name__ == '__main__':
+    WATCHED_FILES = [path + '/issues.db']
+    WATCHED_FILES_MTIMES = [(f, time.ctime(os.path.getmtime(f))) for f in WATCHED_FILES]
+    #   Read the last modified time property of the issues.db file. If there is any change, run the migration script
+    while True:
+        for f, mtime in WATCHED_FILES_MTIMES:
+            if time.ctime(os.path.getmtime(f)) != mtime:
+                run_db_migration()
+        time.sleep(36000)