From 42b0bc451df35f2994c4dd3cdba3b0813462696f Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 11 Jun 2024 12:22:20 -0500 Subject: [PATCH] move facade table refresh to refresh with the mat views Signed-off-by: Isaac Milarsky --- augur/tasks/db/refresh_materialized_views.py | 33 ++++++++ augur/tasks/git/facade_tasks.py | 86 +------------------- augur/tasks/start_tasks.py | 2 - 3 files changed, 34 insertions(+), 87 deletions(-) diff --git a/augur/tasks/db/refresh_materialized_views.py b/augur/tasks/db/refresh_materialized_views.py index 09faffe0cb..8a06ac7a61 100644 --- a/augur/tasks/db/refresh_materialized_views.py +++ b/augur/tasks/db/refresh_materialized_views.py @@ -4,6 +4,9 @@ from augur.tasks.init.celery_app import celery_app as celery from augur.application.db.lib import execute_sql +from augur.tasks.git.util.facade_worker.facade_worker.config import FacadeHelper +from augur.tasks.git.util.facade_worker.facade_worker.rebuildcache import invalidate_caches, rebuild_unknown_affiliation_and_web_caches + @celery.task(bind=True) def refresh_materialized_views(self): @@ -163,6 +166,36 @@ def refresh_materialized_views(self): logger.info(f"error is {e}") pass + #Now refresh facade tables + #Use this class to get all the settings and + #utility functions for facade + facade_helper = FacadeHelper(logger) + + if facade_helper.nuke_stored_affiliations: + logger.error("Nuke stored affiliations is deprecated!") + # deprecated because the UI component of facade where affiliations would be + # nuked upon change no longer exists, and this information can easily be derived + # from queries and materialized views in the current version of Augur. + # This method is also a major performance bottleneck with little value. + + if not facade_helper.limited_run or (facade_helper.limited_run and facade_helper.fix_affiliations): + logger.error("Fill empty affiliations is deprecated!") + # deprecated because the UI component of facade where affiliations would need + # to be fixed upon change no longer exists, and this information can easily be derived + # from queries and materialized views in the current version of Augur. + # This method is also a major performance bottleneck with little value. + + if facade_helper.force_invalidate_caches: + try: + invalidate_caches(facade_helper) + except Exception as e: + logger.info(f"error is {e}") + + if not facade_helper.limited_run or (facade_helper.limited_run and facade_helper.rebuild_caches): + try: + rebuild_unknown_affiliation_and_web_caches(facade_helper) + except Exception as e: + logger.info(f"error is {e}") diff --git a/augur/tasks/git/facade_tasks.py b/augur/tasks/git/facade_tasks.py index 97a69a7574..272c27afa2 100644 --- a/augur/tasks/git/facade_tasks.py +++ b/augur/tasks/git/facade_tasks.py @@ -10,8 +10,6 @@ from augur.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path, get_parent_commits_set, get_existing_commits_set from augur.tasks.git.util.facade_worker.facade_worker.analyzecommit import analyze_commit from augur.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_repo_commit_count, update_facade_scheduling_fields, get_facade_weight_with_commit_count -from augur.tasks.git.util.facade_worker.facade_worker.rebuildcache import fill_empty_affiliations, invalidate_caches, nuke_affiliations, rebuild_unknown_affiliation_and_web_caches - from augur.tasks.github.facade_github.tasks import * from augur.tasks.git.util.facade_worker.facade_worker.config import FacadeHelper @@ -235,37 +233,6 @@ def analyze_commits_in_parallel(repo_git, multithreaded: bool)-> None: logger.info("Analysis complete") return -@celery.task -def nuke_affiliations_facade_task(): - - logger = logging.getLogger(nuke_affiliations_facade_task.__name__) - - facade_helper = FacadeHelper(logger) - nuke_affiliations(facade_helper) - -@celery.task -def fill_empty_affiliations_facade_task(): - - logger = logging.getLogger(fill_empty_affiliations_facade_task.__name__) - facade_helper = FacadeHelper(logger) - fill_empty_affiliations(facade_helper) - -@celery.task -def invalidate_caches_facade_task(): - - logger = logging.getLogger(invalidate_caches_facade_task.__name__) - - facade_helper = FacadeHelper(logger) - invalidate_caches(facade_helper) - -@celery.task -def rebuild_unknown_affiliation_and_web_caches_facade_task(): - - logger = logging.getLogger(rebuild_unknown_affiliation_and_web_caches_facade_task.__name__) - - facade_helper = FacadeHelper(logger) - rebuild_unknown_affiliation_and_web_caches(facade_helper) - # retry this task indefinitely every 5 minutes if it errors. Since the only way it gets scheduled is by itself, so if it stops running no more clones will happen till the instance is restarted @celery.task(autoretry_for=(Exception,), retry_backoff=True, retry_backoff_max=300, retry_jitter=True, max_retries=None) def clone_repos(): @@ -464,55 +431,4 @@ def facade_phase(repo_git): ) logger.info(f"Facade sequence: {facade_sequence}") - return chain(*facade_sequence) - -def generate_non_repo_domain_facade_tasks(logger): - logger.info("Generating facade sequence") - facade_helper = FacadeHelper(logger) - - # Figure out what we need to do - limited_run = facade_helper.limited_run - delete_marked_repos = facade_helper.delete_marked_repos - pull_repos = facade_helper.pull_repos - # clone_repos = facade_helper.clone_repos - check_updates = facade_helper.check_updates - # force_updates = facade_helper.force_updates - run_analysis = facade_helper.run_analysis - # force_analysis = facade_helper.force_analysis - nuke_stored_affiliations = facade_helper.nuke_stored_affiliations - fix_affiliations = facade_helper.fix_affiliations - force_invalidate_caches = facade_helper.force_invalidate_caches - rebuild_caches = facade_helper.rebuild_caches - #if abs((datetime.datetime.strptime(session.cfg.get_setting('aliases_processed')[:-3], - # '%Y-%m-%d %I:%M:%S.%f') - datetime.datetime.now()).total_seconds()) // 3600 > int(session.cfg.get_setting( - # 'update_frequency')) else 0 - force_invalidate_caches = facade_helper.force_invalidate_caches - create_xlsx_summary_files = facade_helper.create_xlsx_summary_files - multithreaded = facade_helper.multithreaded - - facade_sequence = [] - - if nuke_stored_affiliations: - #facade_sequence.append(nuke_affiliations_facade_task.si().on_error(facade_error_handler.s()))#nuke_affiliations(session.cfg) - logger.info("Nuke stored affiliations is deprecated.") - # deprecated because the UI component of facade where affiliations would be - # nuked upon change no longer exists, and this information can easily be derived - # from queries and materialized views in the current version of Augur. - # This method is also a major performance bottleneck with little value. - - #logger.info(session.cfg) - if not limited_run or (limited_run and fix_affiliations): - #facade_sequence.append(fill_empty_affiliations_facade_task.si().on_error(facade_error_handler.s()))#fill_empty_affiliations(session) - logger.info("Fill empty affiliations is deprecated.") - # deprecated because the UI component of facade where affiliations would need - # to be fixed upon change no longer exists, and this information can easily be derived - # from queries and materialized views in the current version of Augur. - # This method is also a major performance bottleneck with little value. - - if force_invalidate_caches: - facade_sequence.append(invalidate_caches_facade_task.si().on_error(facade_error_handler.s()))#invalidate_caches(session.cfg) - - if not limited_run or (limited_run and rebuild_caches): - facade_sequence.append(rebuild_unknown_affiliation_and_web_caches_facade_task.si().on_error(facade_error_handler.s()))#rebuild_unknown_affiliation_and_web_caches(session.cfg) - - return facade_sequence + return chain(*facade_sequence) \ No newline at end of file diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py index 6b35881d60..f0b3450dfb 100644 --- a/augur/tasks/start_tasks.py +++ b/augur/tasks/start_tasks.py @@ -140,8 +140,6 @@ def non_repo_domain_tasks(self): enabled_tasks = [] - enabled_tasks.extend(generate_non_repo_domain_facade_tasks(logger)) - if machine_learning_phase.__name__ in enabled_phase_names: #enabled_tasks.extend(machine_learning_phase()) from augur.tasks.data_analysis.contributor_breadth_worker.contributor_breadth_worker import contributor_breadth_model