Merge rest of Andrew & Isaac Changes

augur/api/gunicorn_conf.py

@@ -6,9 +6,13 @@
 import shutil
 
 from augur.application.db.session import DatabaseSession
+from augur.application.config import AugurConfig
 
 logger = logging.getLogger(__name__)
 with DatabaseSession(logger) as session:
+
+    augur_config = AugurConfig(logger, session)
+
 
     # ROOT_AUGUR_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
@@ -21,21 +25,21 @@
     reload = True
 
     # set the log location for gunicorn    
-    logs_directory = session.config.get_value('Logging', 'logs_directory')
+    logs_directory = augur_config.get_value('Logging', 'logs_directory')
     accesslog = f"{logs_directory}/gunicorn.log"
     errorlog = f"{logs_directory}/gunicorn.log"
 
-    ssl_bool = session.config.get_value('Server', 'ssl')
+    ssl_bool = augur_config.get_value('Server', 'ssl')
 
     if ssl_bool is True: 
 
-        workers = int(session.config.get_value('Server', 'workers'))
-        bind = '%s:%s' % (session.config.get_value("Server", "host"), session.config.get_value("Server", "port"))
-        timeout = int(session.config.get_value('Server', 'timeout'))
-        certfile = str(session.config.get_value('Server', 'ssl_cert_file'))
-        keyfile = str(session.config.get_value('Server', 'ssl_key_file'))
+        workers = int(augur_config.get_value('Server', 'workers'))
+        bind = '%s:%s' % (augur_config.get_value("Server", "host"), augur_config.get_value("Server", "port"))
+        timeout = int(augur_config.get_value('Server', 'timeout'))
+        certfile = str(augur_config.get_value('Server', 'ssl_cert_file'))
+        keyfile = str(augur_config.get_value('Server', 'ssl_key_file'))
 
     else: 
-        workers = int(session.config.get_value('Server', 'workers'))
-        bind = '%s:%s' % (session.config.get_value("Server", "host"), session.config.get_value("Server", "port"))
-        timeout = int(session.config.get_value('Server', 'timeout'))
+        workers = int(augur_config.get_value('Server', 'workers'))
+        bind = '%s:%s' % (augur_config.get_value("Server", "host"), augur_config.get_value("Server", "port"))
+        timeout = int(augur_config.get_value('Server', 'timeout'))

augur/api/routes/config.py

@@ -12,6 +12,7 @@
 # Disable the requirement for SSL by setting env["AUGUR_DEV"] = True
 from augur.application.config import get_development_flag
 from augur.application.db.models import Config
+from augur.application.config import AugurConfig
 from augur.application.db.session import DatabaseSession
 
 logger = logging.getLogger(__name__)
@@ -35,7 +36,7 @@ def get_config():
 
         with DatabaseSession(logger) as session:
 
-            config_dict = session.config.load_config()
+            config_dict = AugurConfig(logger, session).config.load_config()
 
         return jsonify(config_dict), 200
 

augur/api/routes/user.py

@@ -101,6 +101,7 @@ def validate_user():
 
         return jsonify({"status": "Validated"})
 
+
     @server.app.route(f"/{AUGUR_API_VERSION}/user/logout", methods=['POST'])
     @login_required
     def logout_user_func():
@@ -447,13 +448,7 @@ def get_user_groups():
     @server.app.route(f"/{AUGUR_API_VERSION}/user/groups/repos/", methods=['GET', 'POST'])
     @login_required
     def get_user_groups_and_repos():
-        """Get a list of user groups and their repos
-
-        Returns
-        -------
-        list
-            A list with this strucutre : [{"<group_name>": <list_of_repos}, ...]
-        """
+        """Get a list of user groups and their repos"""
 
         if not development and not request.is_secure:
             return generate_upgrade_request()
@@ -463,7 +458,7 @@ def get_user_groups_and_repos():
             return {"status": "Missing argument columns"}
 
         # split list by , and remove whitespaces from edges
-        
+
         valid_columns = []
         columns =  columns.split(",")
         for column in columns:
@@ -542,3 +537,4 @@ def get_favorite_groups():
         group_names = [group.name for group in groups]
 
         return jsonify({"status": "success", "group_names": group_names})
+

augur/api/routes/util.py

@@ -8,6 +8,7 @@
 
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
+from augur.application.config import AugurConfig
 
 logger = AugurLogger("augur").get_logger()
 
@@ -220,7 +221,7 @@ def api_port():
 
         with DatabaseSession(logger) as session:
 
-            response = {'port': session.config.get_value('Server', 'port')}
+            response = {'port': AugurConfig(logger, session).get_value('Server', 'port')}
             return Response(response=json.dumps(response),
                             status=200,
                             mimetype="application/json")

augur/api/server.py

@@ -24,6 +24,7 @@
 
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
+from augur.application.config import AugurConfig
 from metadata import __version__ as augur_code_version
 
 from augur.api.routes import AUGUR_API_VERSION
@@ -48,9 +49,7 @@ def __init__(self):
         """Initialize the Server class."""
 
         self.logger = AugurLogger("server").get_logger()
-        self.session = DatabaseSession(self.logger)
-        self.config = self.session.config
-        self.engine = self.session.engine
+
 
         self.cache_manager = self.create_cache_manager()
         self.server_cache = self.get_server_cache()
@@ -436,9 +435,12 @@ def get_server_cache(self) -> Cache:
             server cache
         """
 
-        expire = int(self.config.get_value('Server', 'cache_expire'))
-        server_cache = self.cache_manager.get_cache('server', expire=expire)
-        server_cache.clear()
+        with DatabaseSession(self.logger) as session:
+            config = AugurConfig(self.logger, session)
+
+            expire = int(config.get_value('Server', 'cache_expire'))
+            server_cache = self.cache_manager.get_cache('server', expire=expire)
+            server_cache.clear()
 
         return server_cache
 

augur/api/view/api.py

@@ -1,6 +1,7 @@
 from flask import Flask, render_template, render_template_string, request, abort, jsonify, redirect, url_for, session, flash
 from flask_login import current_user, login_required
-from augur.util.repo_load_controller import parse_org_url, parse_repo_url
+from augur.application.db.models import Repo
+# from augur.util.repo_load_controller import parse_org_url, parse_repo_url
 from .utils import *
 
 def create_routes(server):
@@ -22,10 +23,10 @@ def av_add_user_repo():
 
         if not url or not group:
             flash("Repo or org URL must not be empty")
-        elif parse_org_url(url):
+        elif Repo.parse_github_org_url(url):
             current_user.add_org(group, url)
             flash("Successfully added org")
-        elif parse_repo_url(url):
+        elif Repo.parse_github_repo_url(url):
             current_user.add_repo(group, url)
             flash("Successfully added repo")
         else:

augur/api/view/routes.py

@@ -13,8 +13,6 @@
 
 logger = logging.getLogger(__name__)
 
-with DatabaseSession(logger) as db_session:
-    config = AugurConfig(logger, db_session)
 
 # ROUTES -----------------------------------------------------------------------
 
@@ -71,8 +69,12 @@ def repo_table_view():
                 rev = True
 
         direction = "DESC" if rev else "ASC"
-
-        pagination_offset = config.get_value("frontend", "pagination_offset")
+
+        with DatabaseSession(logger) as db_session:
+            config = AugurConfig(logger, db_session)
+
+            pagination_offset = config.get_value("frontend", "pagination_offset")
+
 
         if current_user.is_authenticated:
             data = current_user.get_repos(page = page, sort = sorting, direction = direction)[0]
@@ -153,26 +155,28 @@ def user_login():
                 if username is None:
                     raise LoginException("A login issue occurred")
 
-                user = User.get_user(username)
-                if not user and register is None:
-                    raise LoginException("Invalid login credentials")
-
-                # register a user
-                if register is not None:
-                    if user:
-                        raise LoginException("User already exists")
+                with DatabaseSession(logger) as db_session:
+                    user = User.get_user(db_session, username)
+
+                    if not user and register is None:
+                        raise LoginException("Invalid login credentials")
 
-                    email = request.form.get('email')
-                    first_name = request.form.get('first_name')
-                    last_name = request.form.get('last_name')
-                    admin = request.form.get('admin') or False
-
-                    result = User.create_user(username, password, email, first_name, last_name, admin)
-                    if not result[0]:
-                        raise LoginException("An error occurred registering your account")
-                    else:
-                        user = User.get_user(username)
-                        flash(result[1]["status"])
+                    # register a user
+                    if register is not None:
+                        if user:
+                            raise LoginException("User already exists")
+
+                        email = request.form.get('email')
+                        first_name = request.form.get('first_name')
+                        last_name = request.form.get('last_name')
+                        admin = request.form.get('admin') or False
+
+                        result = User.create_user(db_session, username, password, email, first_name, last_name, admin)
+                        if not result[0]:
+                            raise LoginException("An error occurred registering your account")
+                        else:
+                            user = User.get_user(username)
+                            flash(result[1]["status"])
 
                 # Log the user in if the password is valid
                 if user.validate(password) and login_user(user, remember = remember):
@@ -285,7 +289,10 @@ def user_group_view():
                 rev = True
                 params["direction"] = "DESC"
 
-        pagination_offset = config.get_value("frontend", "pagination_offset")
+        with DatabaseSession(logger) as db_session:
+            config = AugurConfig(logger, db_session)
+
+            pagination_offset = config.get_value("frontend", "pagination_offset")
 
         data = current_user.get_group_repos(group, **params)[0]
         page_count = (current_user.get_group_repo_count(group)[0]) or 0

augur/application/cli/__init__.py

@@ -19,7 +19,8 @@ def new_func(ctx, *args, **kwargs):
             #try to ping google's dns server
             socket.create_connection(("8.8.8.8",53))
             return ctx.invoke(function_internet_connection, *args, **kwargs)
-        except OSError:
+        except OSError as e:
+            print(e)
             print(f"\n\n{usage} command setup failed\nYou are not connect to the internet. Please connect to the internet to run Augur\n")
             sys.exit()        
 

augur/application/cli/backend.py

@@ -23,6 +23,7 @@
 from augur.application.db.models import Repo
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
+from augur.application.config import AugurConfig
 from augur.application.cli import test_connection, test_db_connection 
 
 
@@ -55,45 +56,50 @@ def start(disable_collection, development, port):
         os.environ["AUGUR_DEV"] = "1"
         logger.info("Starting in development mode")
 
-
-    with DatabaseSession(logger) as session:
-
+    try:
         gunicorn_location = os.getcwd() + "/augur/api/gunicorn_conf.py"
-        host = session.config.get_value("Server", "host")
+    except FileNotFoundError:
+        logger.error("\n\nPlease run augur commands in the root directory\n\n")
 
-        if not port:
-            port = session.config.get_value("Server", "port")
+    db_session = DatabaseSession(logger)
+    config = AugurConfig(logger, db_session)
+    host = config.get_value("Server", "host")
 
-        gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} --preload augur.api.server:app"
-        server = subprocess.Popen(gunicorn_command.split(" "))
+    if not port:
+        port = config.get_value("Server", "port")
+
+    db_session.invalidate()
+
+    gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} --preload augur.api.server:app"
+    server = subprocess.Popen(gunicorn_command.split(" "))
 
-        time.sleep(3)
-        logger.info('Gunicorn webserver started...')
-        logger.info(f'Augur is running at: http://127.0.0.1:{session.config.get_value("Server", "port")}')
+    time.sleep(3)
+    logger.info('Gunicorn webserver started...')
+    logger.info(f'Augur is running at: http://127.0.0.1:{port}')
 
-        worker_1_process = None
-        cpu_worker_process = None
-        celery_beat_process = None
-        if not disable_collection:
+    worker_1_process = None
+    cpu_worker_process = None
+    celery_beat_process = None
+    if not disable_collection:
 
-            if os.path.exists("celerybeat-schedule.db"):
-                logger.info("Deleting old task schedule")
-                os.remove("celerybeat-schedule.db")
+        if os.path.exists("celerybeat-schedule.db"):
+            logger.info("Deleting old task schedule")
+            os.remove("celerybeat-schedule.db")
 
-            worker_1 = f"celery -A augur.tasks.init.celery_app.celery_app worker -P eventlet -l info --concurrency=100 -n {uuid.uuid4().hex}@%h"
-            cpu_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=20 -n {uuid.uuid4().hex}@%h -Q cpu"
-            worker_1_process = subprocess.Popen(worker_1.split(" "))
+        worker_1 = f"celery -A augur.tasks.init.celery_app.celery_app worker -P eventlet -l info --concurrency=100 -n {uuid.uuid4().hex}@%h"
+        cpu_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=20 -n {uuid.uuid4().hex}@%h -Q cpu"
+        worker_1_process = subprocess.Popen(worker_1.split(" "))
 
-            cpu_worker_process = subprocess.Popen(cpu_worker.split(" "))
-            time.sleep(5)
+        cpu_worker_process = subprocess.Popen(cpu_worker.split(" "))
+        time.sleep(5)
 
-            start_task.si().apply_async()
+        start_task.si().apply_async()
 
-            celery_command = "celery -A augur.tasks.init.celery_app.celery_app beat -l debug"
-            celery_beat_process = subprocess.Popen(celery_command.split(" "))    
+        celery_command = "celery -A augur.tasks.init.celery_app.celery_app beat -l debug"
+        celery_beat_process = subprocess.Popen(celery_command.split(" "))    
 
-        else:
-            logger.info("Collection disabled")   
+    else:
+        logger.info("Collection disabled")   
 
     try:
         server.wait()
@@ -119,7 +125,8 @@ def start(disable_collection, development, port):
             clear_redis_caches()
             connection_string = ""
             with DatabaseSession(logger) as session:
-                connection_string = session.config.get_section("RabbitMQ")['connection_string']
+                config = AugurConfig(logger, session)
+                connection_string = config.get_section("RabbitMQ")['connection_string']
 
             clear_rabbitmq_messages(connection_string)
 
@@ -138,7 +145,8 @@ def stop():
     clear_redis_caches()
     connection_string = ""
     with DatabaseSession(logger) as session:
-        connection_string = session.config.get_section("RabbitMQ")['connection_string']
+        config = AugurConfig(logger, session)
+        connection_string = config.get_section("RabbitMQ")['connection_string']
 
     clear_rabbitmq_messages(connection_string)
 
@@ -154,7 +162,8 @@ def kill():
 
     connection_string = ""
     with DatabaseSession(logger) as session:
-        connection_string = session.config.get_section("RabbitMQ")['connection_string']
+        config = AugurConfig(logger, session)
+        connection_string = config.get_section("RabbitMQ")['connection_string']
 
     clear_rabbitmq_messages(connection_string)