skip message fetch if pre-persisted; add get-labels utility script

Signed-off-by: AbhishekKr <abhikumar163@gmail.com>

Author: abhishekkr

README.md

@@ -17,6 +17,8 @@
 > by default it reads mails first and stores it locally in a sqlite DB separated by year of mail, then deletes
 >
 > How to use: `python3 delete-mails.py ./config-yamls/delete-mails-config.yaml`
+>
+> GMail API doc: [developers.google.com/gmail/api/v1/reference/users/messages/delete](https://developers.google.com/gmail/api/v1/reference/users/messages/delete)
 
 ---
 

_config_/__init__.py

@@ -11,7 +11,10 @@ def config_yaml():
     Args: None
     """
     try:
-        return sys.argv[1]
+        if os.path.exists(sys.argv[1]):
+            return sys.argv[1]
+        else:
+            return "config.yaml"
     except:
         try:
             return os.environ("GMAIL_HELPER_CONFIG")
@@ -84,3 +87,10 @@ def message_ids_to_skip():
         return ret_val
     else:
         return ret_val.split(",")
+
+def labels_to_skip():
+    ret_val = env_else_yaml("labels_to_skip")
+    if isinstance(ret_val, list):
+        return ret_val
+    else:
+        return ret_val.split(",")

_dbms_/__init__.py

@@ -1,6 +1,7 @@
 import os
 import sqlite3
 from sqlite3 import Error
+import sys
 
 import _logging_ as _log
 import _config_ as _cfg
@@ -29,7 +30,7 @@ def create_schema_messages(db):
     Args:
         db: Local db connection object.
     """
-    _log.logger.debug("creating table message")
+    _log.logger.debug("creating table messages")
     cursorObj = db.cursor()
     cursorObj.execute("""CREATE TABLE IF NOT EXISTS messages(
         id text,
@@ -50,6 +51,23 @@ def create_schema_messages(db):
     db.commit()
 
 
+def create_schema_labels(db):
+    """Creates 'labels' schema on given DB connection.
+
+    Args:
+        db: Local db connection object.
+    """
+    _log.logger.debug("creating table labels")
+    cursorObj = db.cursor()
+    cursorObj.execute("""CREATE TABLE IF NOT EXISTS labels(
+        name text,
+        label blob,
+        CONSTRAINT name_pk PRIMARY KEY (name))""")
+    cursorObj.execute("""CREATE INDEX IF NOT EXISTS idx_labels_name
+        ON labels (name);""")
+    db.commit()
+
+
 def sender_of_message(message):
     try:
         return [item['value'] for item in message['payload']['headers'] if item['name'] == 'From'][0]
@@ -78,13 +96,26 @@ def date_of_message(message):
         return ""
 
 
+def message_exists(db, message_id):
+    sql_stmt = """SELECT * FROM messages WHERE id = '%s';""" % (message_id)
+    cursorObj = db.cursor()
+    result = cursorObj.execute(sql_stmt)
+    del cursorObj
+    if result.fetchone() == None:
+        return False
+    return True
+
+
 def add_message(db, message):
-    """Get and Delete a list of messages from GMail by Query.
+    """Persist a message from GMail if doesn't already exists locally.
 
     Args:
         db: Local db connection object.
         message: GMail message JSON object as read by get api.
     """
+    if message_exists(db, message['id']):
+        _log.logger.info("message:%s already exists, skipping db entry" % (message['id']))
+        return
     _log.logger.debug("[+] adding message: %s" % (message['id']))
 
     cursorObj = db.cursor()
@@ -113,12 +144,60 @@ def add_message(db, message):
             subject_of_message(message)))
         _log.logger.debug("---------------------")
         db.commit()
+        del cursorObj
+        return True
     except:
         _log.logger.error("failed to insert for %s" % (message['id']))
-        sys.exit(0)
+        return False
+
+
+def label_exists(db, label_name):
+    sql_stmt = """SELECT * FROM labels WHERE name = '%s';""" % (label_name)
+    cursorObj = db.cursor()
+    result = cursorObj.execute(sql_stmt)
+    del cursorObj
+    if result.fetchone() == None:
+        return False
+    return True
+
+
+def add_label(db, label):
+    """Persist a label from GMail if doesn't already exists locally.
+
+    Args:
+        db: Local db connection object.
+        label: GMail label
+    """
+    label_name = label['name']
+
+    if label_exists(db, label_name):
+        _log.logger.info("label:%s already exists, skipping db entry" % (label_name))
+        return
 
+    _log.logger.debug("[+] adding label: %s" % (label_name))
 
-def dbpath_by_year(year):
+    cursorObj = db.cursor()
+    sql_stmt = """INSERT INTO labels
+                        (name, label)
+                        VALUES (?,?)"""
+    values = (
+        label_name,
+        str(label),
+    )
+
+    try:
+        cursorObj.execute(sql_stmt, values)
+        _log.logger.debug("adding label: %s" % (label_name))
+        _log.logger.debug("---------------------")
+        db.commit()
+        del cursorObj
+        return True
+    except:
+        _log.logger.error("failed to insert for %s" % (label_name))
+        return False
+
+
+def dbpath_by_token(basename, token):
     """Returns DB path generated based on year.
 
     Args:
@@ -127,7 +206,7 @@ def dbpath_by_year(year):
     Returns:
         DB filesystem path.
     """
-    db_name = "gmail-to-delete-%d.db" % (year)
+    db_name = "%s-%s.db" % (str(basename), str(token))
     return os.path.join(_cfg.data_basepath(), db_name)
 
 
@@ -140,4 +219,16 @@ def connection_by_year(year):
     Returns:
         Local db connection object.
     """
-    return sql_connection(dbpath_by_year(year))
+    return sql_connection(dbpath_by_token("gmail-to-delete", year))
+
+
+def connection_by_feature(feature):
+    """Returns SQLite3 connection to db created at path for given year.
+
+    Args:
+        feature: Name of feature/construct; like mails, labels, spams, etc
+
+    Returns:
+        Local db connection object.
+    """
+    return sql_connection(dbpath_by_token("gmail", feature))

_google_/auth/__init__.py

@@ -48,3 +48,16 @@ def gmail_messages():
     creds = google_oauth_creds()
     service = build('gmail', 'v1', credentials=creds, cache_discovery=False)
     return service.users().messages()
+
+
+def gmail_labels():
+    """Returns GMail labels object exposing labels API.
+
+    Args: None
+
+    Returns:
+        GMail labels object.
+    """
+    creds = google_oauth_creds()
+    service = build('gmail', 'v1', credentials=creds, cache_discovery=False)
+    return service.users().labels()

_google_/gmail/__init__.py

@@ -5,6 +5,20 @@
 import _logging_ as _log
 
 
+def get_labels(db, labels_obj, user_id='me'):
+    results = labels_obj.list(userId='me').execute()
+    labels = results.get('labels', [])
+
+    if not labels:
+        _log.logger.info("No labels found.")
+        return
+
+    print('Labels:')
+    for label in labels:
+        if _db.add_label(db, label):
+            print("~ added: %s" % (label['name']))
+
+
 def get_mail(db, messages_obj, msg_id, user_id='me'):
     """Fetch a message from GMail by id and adds it to passed db.
 
@@ -18,8 +32,13 @@ def get_mail(db, messages_obj, msg_id, user_id='me'):
     message = messages_obj.get(
         userId=user_id, id=msg_id, format='full'
     ).execute()
+
+    if set.intersection(set(_cfg.labels_to_skip()), set(message['labelIds'])) != set():
+        return False
     _log.logger.debug("adding message: %s" % (message['id']))
-    _db.add_message(db, message)
+    if not _db.add_message(db, message):
+        return False
+    return True
 
 
 def delete_mail(messages_obj, msg_id, user_id='me'):
@@ -34,8 +53,8 @@ def delete_mail(messages_obj, msg_id, user_id='me'):
     try:
       messages_obj.delete(userId=user_id, id=msg_id).execute()
       _log.logger.info('Message with id: %s deleted successfully.' % msg_id)
-    except:
-      _log.logger.error('An error occurred: %s' % error)
+    except Exception as e:
+        _log.logger.error('An error occurred: %s' % str(e))
 
 
 def mails_by_query(messages_obj, user_id='me', query=''):
@@ -67,8 +86,8 @@ def mails_by_query(messages_obj, user_id='me', query=''):
       messages.extend(response['messages'])
 
     return messages
-  except:
-    _log.logger.error('An error occurred: %s' % error)
+  except Exception as e:
+    _log.logger.error('An error occurred: %s' % str(e))
     return []
 
 
@@ -87,5 +106,7 @@ def get_delete_mails_by_query(db, messages_obj, user_id='me', query=''):
     for message in mails_by_query(messages_obj, user_id, query):
         if message['id'] in message_ids_to_skip: continue
         _log.logger.debug(message['id'])
-        get_mail(db, messages_obj, message['id'])
-        delete_mail(messages_obj, message['id'])
+        if get_mail(db, messages_obj, message['id']):
+            delete_mail(messages_obj, message['id'])
+        else:
+            _log.logger.error("not deleting mail: %s" % (message['id']))

config-yamls/delete-mails-config.yaml

@@ -20,3 +20,5 @@ filters_to_delete:
   - "from:(googlegroups.com)"
 
 message_ids_to_skip: []
+
+labels_to_skip: []

get-labels.py

@@ -0,0 +1,48 @@
+# Copyright 2018 abhishekkr <abhikumar163@gmail.com>
+"""
+How to use:
+    python3 get-labels.py ./config-yamls/delete-mails-config.yaml
+
+
+How to configure:
+
+* using "get-labels-config.yaml", update all values to required configuration
+
+> * 'gmail_credential_jsonpath' is file path for OAuth2 credential json file downloaded from Google by following step from README
+> * 'gmail_auth_picklepath' need to be file paths where Google credentials can be stored for reuse
+> * 'scopes' is a list of GMail API scopes made available to Google Auth
+> * 'data_basepath' is under which all DBs would be created
+>
+> would be better if all above mentioned file paths should be at a secure location r/w only by your user
+
+
+* using env variables
+
+> all above mentioned config names prefixed with 'GMAIL_HELPER_' formulate their env var name
+"""
+
+from __future__ import print_function
+import base64
+import email
+import sys
+
+import _logging_ as _log
+import _config_ as _cfg
+import _google_.auth as _gauth
+import _google_.gmail as _gmail
+import _dbms_ as _db
+
+
+def main():
+    """Helps make a local backup of all lables."""
+    labels_obj = _gauth.gmail_labels()
+
+    db = _db.connection_by_feature("labels")
+    _db.create_schema_labels(db)
+    _gmail.get_labels(db, labels_obj, 'me')
+    db.close()
+    del db
+
+
+if __name__ == '__main__':
+    main()