Remove empty files and move redundant folders to simplify project structure

augur/tasks/github/__init__.py

@@ -1,7 +1,7 @@
-from augur.tasks.github.contributors.tasks import *
-from augur.tasks.github.events.tasks import *
-from augur.tasks.github.issues.tasks import *
-from augur.tasks.github.messages.tasks import *
+from augur.tasks.github.contributors import *
+from augur.tasks.github.events import *
+from augur.tasks.github.issues import *
+from augur.tasks.github.messages import *
 from augur.tasks.github.pull_requests.tasks import *
 from augur.tasks.github.repo_info.tasks import *
 from augur.tasks.github.releases.tasks import *

augur/tasks/github/issues/tasks.py → augur/tasks/github/issues.py

@@ -1,4 +1,4 @@
 import logging
 import traceback
 from datetime import timedelta, timezone
 
@@ -7,7 +7,7 @@
 
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask
 from augur.application.db.data_parse import *
 from augur.tasks.github.util.github_data_access import GithubDataAccess
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
 from augur.tasks.github.util.util import add_key_value_pair_to_dicts, get_owner_repo
@@ -65,7 +65,7 @@
     url = f"https://api.github.com/repos/{owner}/{repo}/issues?state=all"
 
     if since:
-        url += since.isoformat()
+        url += f"&since={since.isoformat()}"
 
     github_data_access = GithubDataAccess(key_auth, logger)
 
@@ -76,7 +76,7 @@
 
     return list(issues_paginator)
 
 def process_issues(issues, task_name, repo_id, logger) -> None:
 
     # get repo_id or have it passed
     tool_source = "Issue Task"
@@ -87,7 +87,7 @@
     issue_mapping_data = {}
     issue_total = len(issues)
     contributors = []
     for index, issue in enumerate(issues):
 
         # calls is_valid_pr_block to see if the data is a pr.
         # if it is a pr we skip it because we don't need prs 
@@ -163,7 +163,7 @@
 
 
         # add the issue id to the lables and assignees, then add them to a list of dicts that will be inserted soon
         dict_key = "issue_id"
         issue_label_dicts += add_key_value_pair_to_dicts(other_issue_data["labels"], "issue_id", issue_id)
         issue_assignee_dicts += add_key_value_pair_to_dicts(other_issue_data["assignees"], "issue_id", issue_id)
 

augur/tasks/github/pull_requests/tasks.py

@@ -1,10 +1,10 @@
 import logging
 from datetime import datetime, timedelta, timezone
 
 from augur.tasks.github.pull_requests.core import extract_data_from_pr_list
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask, AugurSecondaryRepoCollectionTask
 from augur.application.db.data_parse import *
 from augur.tasks.github.util.github_data_access import GithubDataAccess
 from augur.tasks.github.util.github_paginator import GithubPaginator
 from augur.tasks.util.worker_util import remove_duplicate_dicts
@@ -14,7 +14,7 @@
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
 from augur.application.db.lib import get_session, get_repo_by_repo_git, bulk_insert_dicts, get_pull_request_reviews_by_repo_id
 from augur.application.db.util import execute_session_query
-from ..messages.tasks import process_github_comment_contributors
+from ..messages import process_github_comment_contributors
 from augur.application.db.lib import get_secondary_data_last_collected, get_updated_prs, get_core_data_last_collected
 
 from typing import Generator, List, Dict
@@ -54,7 +54,7 @@
                 total_count += len(all_data)
                 all_data.clear()
 
         if len(all_data):
             process_pull_requests(all_data, f"{owner}/{repo}: Pr task", repo_id, logger, augur_db)
             total_count += len(all_data)
 
@@ -66,8 +66,8 @@
 
 
 
 # TODO: Rename pull_request_reviewers table to pull_request_requested_reviewers
 # TODO: Fix column names in pull request labels table
 def retrieve_all_pr_data(repo_git: str, logger, key_auth, since): #-> Generator[List[Dict]]:
 
     owner, repo = get_owner_repo(repo_git)

augur/tasks/init/celery_app.py

@@ -26,18 +26,18 @@
                 'augur.tasks.data_analysis',
                 'augur.tasks.util.collection_util']
 
-github_tasks = ['augur.tasks.github.contributors.tasks',
-                'augur.tasks.github.issues.tasks',
+github_tasks = ['augur.tasks.github.contributors',
+                'augur.tasks.github.issues',
                 'augur.tasks.github.pull_requests.tasks',
-                'augur.tasks.github.events.tasks',
-                'augur.tasks.github.messages.tasks',
+                'augur.tasks.github.events',
+                'augur.tasks.github.messages',
                 'augur.tasks.github.facade_github.tasks',
                 'augur.tasks.github.releases.tasks',
                 'augur.tasks.github.repo_info.tasks',
                 'augur.tasks.github.detect_move.tasks',
                 'augur.tasks.github.pull_requests.files_model.tasks',
                 'augur.tasks.github.pull_requests.commits_model.tasks',
-                'augur.tasks.github.traffic.tasks']
+                'augur.tasks.github.traffic']
 
 gitlab_tasks = ['augur.tasks.gitlab.merge_request_task',
                 'augur.tasks.gitlab.issues_task',

augur/tasks/start_tasks.py

@@ -15,7 +15,7 @@
 from augur.tasks.github.pull_requests.files_model.tasks import process_pull_request_files
 from augur.tasks.github.pull_requests.commits_model.tasks import process_pull_request_commits
 from augur.tasks.git.dependency_tasks.tasks import process_ossf_dependency_metrics
-from augur.tasks.github.traffic.tasks import collect_github_repo_clones_data
+from augur.tasks.github.traffic import collect_github_repo_clones_data
 from augur.tasks.gitlab.merge_request_task import collect_gitlab_merge_requests, collect_merge_request_metadata, collect_merge_request_commits, collect_merge_request_files, collect_merge_request_comments
 from augur.tasks.gitlab.issues_task import collect_gitlab_issues, collect_gitlab_issue_comments
 from augur.tasks.gitlab.events_task import collect_gitlab_issue_events, collect_gitlab_merge_request_events