Accumulator finished #4

TODO.md

@@ -12,7 +12,8 @@ See the [issues](https://github.com/drkostas/youbot/issues) too.
 - [X] Store comments in dropbox
 - [X] \[Merged\] Regularly backup logs files from logs/ to dropbox (for when running on Heroku) + Store errors in sql or dropbox
 - [X] Ensure code works without dropbox and emailer modules
-- [ ] Create the workflow for the accumulator
+- [X] Create the workflow for the accumulator
+- [ ] Load yt keys from Dropbox
 - [ ] Add SQL scripts for creating the tables needed
 - [ ] Update Readme
 - [ ] Recreate the Livestreaming module

confs/accumulator.yml

@@ -0,0 +1,17 @@
+tag: accumulator
+datastore:
+  - config:
+      hostname: !ENV ${MYSQL_HOST}
+      username: !ENV ${MYSQL_USERNAME}
+      password: !ENV ${MYSQL_PASSWORD}
+      db_name: !ENV ${MYSQL_DB_NAME}
+      port: 3306
+    type: mysql
+youtube:
+  - config:
+      client_id: !ENV ${CLIENT_ID}
+      client_secret: !ENV ${CLIENT_SECRET}
+      api_version: v3
+      read_only_scope: https://www.googleapis.com/auth/youtube.force-ssl
+    sleep_time: !ENV ${SLEEP_TIME}
+    type: simulated  # normal, simulated

youbot/run.py

@@ -66,7 +66,7 @@ def commenter(youtube: YoutubeManager, args: argparse.Namespace) -> None:
 
 
 def accumulator(youtube: YoutubeManager, args: argparse.Namespace) -> None:
-    raise NotImplementedError()
+    youtube.accumulator()
 
 
 def set_priority(youtube: YoutubeManager, args: argparse.Namespace) -> None:
@@ -106,24 +106,28 @@ def main():
     # Initializing
     args = get_args()
     ColorLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=False)
-    # Load the configurations
+    # Load configurations
     conf_obj = Configuration(config_src=args.config_file)
     tag = conf_obj.tag
-    logger = ColorLogger(logger_name=f'[{tag}] Main', color='yellow')
+    logger = ColorLogger(logger_name=f'[{tag}] Main', color='yellow')  # Reconfigure with the tag
     you_conf = conf_obj.get_config('youtube')[0]
+    sleep_time = int(you_conf['sleep_time'])
+    max_posted_hours = int(you_conf['max_posted_hours']) if 'max_posted_hours' in you_conf else -1
     db_conf = conf_obj.get_config('datastore')[0]
-    comments_conf = conf_obj.get_config('comments')[0]
+    comments_conf = None
+    if 'comments' in conf_obj.config:  # Optional
+        comments_conf = conf_obj.get_config('comments')[0]
     cloud_conf = None
-    if 'cloudstore' in conf_obj.config:
+    if 'cloudstore' in conf_obj.config:  # Optional
         cloud_conf = conf_obj.get_config('cloudstore')[0]
         emailer_conf = None
     if 'emailer' in conf_obj.config:  # Not implemented yet
         emailer_conf = conf_obj.get_config('emailer')[0]
     # Setup YouTube API
     youtube = YoutubeManager(config=you_conf['config'],
                              db_conf=db_conf, cloud_conf=cloud_conf, comments_conf=comments_conf,
-                             sleep_time=int(you_conf['sleep_time']),
-                             max_posted_hours=int(you_conf['max_posted_hours']),
+                             sleep_time=sleep_time,
+                             max_posted_hours=max_posted_hours,
                              api_type=you_conf['type'], tag=conf_obj.tag, log_path=args.log)
     # Run in the specified run mode
     func = globals()[args.run_mode]

youbot/youtube_utils/youtube_api.py

@@ -53,7 +53,7 @@ def __init__(self, config: Dict, tag: str):
     def _build_api(client_id: str, client_secret: str, api_version: str, read_only_scope: str,
                    tag: str) -> googleapiclient.discovery.Resource:
         """
-        Build a youtube api connection.
+        Build a YouTube api connection.
 
         Args:
             client_id:

youbot/youtube_utils/youtube_manager.py

@@ -24,15 +24,18 @@ def __init__(self, config: Dict, db_conf: Dict, cloud_conf: Dict, comments_conf:
         global logger
         logger = ColorLogger(logger_name=f'[{tag}] YoutubeManager', color='cyan')
         self.db = YoutubeMySqlDatastore(config=db_conf['config'], tag=tag)
-        self.comments_conf = comments_conf['config']
+        self.comments_conf = None
+        if comments_conf is not None:
+            self.comments_conf = comments_conf['config']
         self.dbox = None
         if cloud_conf is not None:
             self.dbox = DropboxCloudManager(config=cloud_conf['config'])
             self.dbox_logs_folder_path = cloud_conf['logs_folder_path']
             self.upload_logs_every = int(cloud_conf['upload_logs_every'])
-        elif self.comments_conf['type'] == 'dropbox':
-            raise YoutubeManagerError("Requested `dropbox` comments type "
-                                      "but `cloudstore` config is not set!")
+        elif self.comments_conf is not None:
+            if self.comments_conf['type'] == 'dropbox':
+                raise YoutubeManagerError("Requested `dropbox` comments type "
+                                          "but `cloudstore` config is not set!")
         self.default_sleep_time = sleep_time
         self.max_posted_hours = max_posted_hours
         self.api_type = api_type
@@ -96,6 +99,41 @@ def commenter(self):
                 logger.error(error_txt)
                 raise e
 
+    def accumulator(self):
+        # Initialize
+        sleep_time = 0
+        while True:
+            try:
+                time.sleep(sleep_time)
+                # Load recent comments
+                recent_commented_links = [comment["video_link"] for comment in
+                                          self.db.get_comments(n_recent=200)]
+                # Get info for recent comments with YT api
+                comments = []
+                exceptions = []
+                for cnt, link in enumerate(recent_commented_links):
+                    try:
+                        comments.extend(self.get_video_comments(link))
+                    except Exception as e:
+                        exceptions.append(e)
+                # Update comment data in the DB
+                for comment_dict in comments:
+                    self.db.update_comment(video_link=comment_dict['url'],
+                                           comment_id=comment_dict['comment_id'],
+                                           like_cnt=comment_dict['like_count'],
+                                           reply_cnt=comment_dict['reply_count'])
+                if len(exceptions) > 0:
+                    logger.error(f"{len(exceptions)} exceptions occurred! "
+                                 f"Will only print  the first one.")
+                    raise exceptions[0]
+            except Exception as e:
+                error_txt = f"Exception in the main loop of the Accumulator:\n{e}"
+                logger.error(error_txt)
+                sleep_time = self.seconds_until_next_hour()
+                logger.error(f"Will sleep until next hour ({sleep_time} seconds)")
+            else:
+                sleep_time = self.default_sleep_time
+
     def get_comments(self, n_recent, channel_ids):
         commented_comments = {}
         video_links_commented = []
@@ -175,6 +213,9 @@ def list_comments(self, n_recent: int = 50, min_likes: int = -1,
         self.pretty_print(headers, comments)
 
     def load_template_comments(self):
+        if self.comments_conf is None:
+            raise YoutubeManagerError("Tried to load template comments "
+                                      "but `comments` is not set in the config!")
         # Download files from dropbox
         if self.comments_conf['type'] == 'dropbox':
             # TODO: implement this in the dropbox lib