Merge Main into Dev

.docker-setup.sh

@@ -24,7 +24,6 @@ missingModules=""
 #Check everything that needs to be in the $PATH is in there.
 #Bash doesn't let this work if this is in an if statement for some reason it has to be chained
 type -P "docker" &>/dev/null && echo "docker found..." || missingModules="${missingModules} docker"
-type -P "docker-compose" &>/dev/null && echo "docker-compose found..." || missingModules="${missingModules} docker-compose"
 type -P "ifconfig" &>/dev/null && echo "ifconfig found..." || missingModules="${missingModules} ifconfig (part of net-tools)"
 type -P "psql" &>/dev/null && echo "psql found..." || missingModules="${missingModules} psql"
 type -P "watch" &>/dev/null && echo "watch found..." || missingModules="${missingModules} watch"

Makefile

@@ -143,16 +143,16 @@ docs-view: docs
 
 
 compose-run:
-	@ docker-compose -f docker-compose.yml up --build
+	@ docker compose -f docker-compose.yml up --build
 
 compose-run-database:
 	@ echo "**************************************************************************"
 	@ echo "Make sure there are no database credentials in docker_env.txt!"
 	@ echo "**************************************************************************"
 	@ echo
-	@ docker-compose -f docker-compose.yml -f database-compose.yml up --build
+	@ docker compose -f docker-compose.yml -f database-compose.yml up --build
 
-docker-build: docker-build-backend docker-build-frontend docker-build-database
+docker-build: docker-build-backend docker-build-frontend docker-build-database docker-build-rabbitmq
 
 docker-build-backend:
 	@ docker build -t augurlabs/augur:backend -f util/docker/backend/Dockerfile .
@@ -163,6 +163,8 @@ docker-build-frontend:
 docker-build-database:
 	@ docker build -t augurlabs/augur:database -f util/docker/database/Dockerfile .
 
+docker-build-rabbitmq:
+	@ docker build -t augurlabs/augur:rabbitmq -f util/docker/rabbitmq/Dockerfile .
 
 docker-run-backend:
 	@ - docker stop augur_backend
@@ -178,3 +180,8 @@ docker-run-database:
 	@ - docker stop augur_database
 	@ - docker rm augur_database
 	docker run -p 5434:5432 --name augur_database augurlabs/augur:database
+
+docker-run-rabbitmq:
+	@ - docker stop augur_rabbitmq
+	@ - docker rm augur_rabbitmq
+	docker run -p 5434:5432 --name augur_rabbitmq augurlabs/augur:rabbitmq

README.md

@@ -1,6 +1,7 @@
-# Augur NEW Release v0.62.0
+# Augur NEW Release v0.62.4
 
-CHECK out Augur's Public Instance at https://ai.chaoss.io 
+Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data. Less data carpentry for everyone else! 
+The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot) ... A public instance of 8Knot is available at https://metrix.chaoss.io ... That is tied to a public instance of Augur at https://ai.chaoss.io 
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
@@ -9,7 +10,7 @@ CHECK out Augur's Public Instance at https://ai.chaoss.io
 ## NEW RELEASE ALERT!
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.62.0
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.62.4
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard
@@ -96,6 +97,7 @@ Contributors
 - `Dawn Foster <https://github.com/geekygirldawn/>`_
 - `Ivana Atanasova <https://github.com/ivanayov/>`_
 - `Georg J.P. Link <https://github.com/GeorgLink/>`_
+- `Gary P White <https://github.com/garypwhite/>`_
 
 GSoC 2022 participants
 -----------------------

augur/api/routes/pull_request_reports.py

@@ -53,7 +53,7 @@ def pull_request_data_collection(repo_id, start_date, end_date):
                     ( EXTRACT ( EPOCH FROM last_response_time ) - EXTRACT ( EPOCH FROM pull_requests.pr_created_at ) ) / 86400 AS days_to_last_response, 
                     first_response_time,
                     last_response_time,
-                    average_time_between_responses,
+                    EXTRACT ( EPOCH FROM average_time_between_responses),
                     assigned_count,
                     review_requested_count,
                     labeled_count,
@@ -62,15 +62,15 @@ def pull_request_data_collection(repo_id, start_date, end_date):
                     referenced_count,
                     closed_count,
                     head_ref_force_pushed_count,
-                    merged_count,
+                    merged_count::INT,
                     milestoned_count,
                     unlabeled_count,
                     head_ref_deleted_count,
                     comment_count,
-                    lines_added, 
-                    lines_removed,
+                    COALESCE(lines_added, 0), 
+                    COALESCE(lines_removed, 0),
                     commit_count, 
-                    file_count
+                    COALESCE(file_count, 0)
                 FROM
                     repo,
                     repo_groups,
@@ -87,46 +87,47 @@ def pull_request_data_collection(repo_id, start_date, end_date):
                         count(*) FILTER (WHERE action = 'head_ref_force_pushed') AS head_ref_force_pushed_count,
                         count(*) FILTER (WHERE action = 'head_ref_deleted') AS head_ref_deleted_count,
                         count(*) FILTER (WHERE action = 'milestoned') AS milestoned_count,
-                        count(*) FILTER (WHERE action = 'merged') AS merged_count,
-                        MIN(message.msg_timestamp) AS first_response_time,
-                        COUNT(DISTINCT message.msg_timestamp) AS comment_count,
-                        MAX(message.msg_timestamp) AS last_response_time,
-                        (MAX(message.msg_timestamp) - MIN(message.msg_timestamp)) / COUNT(DISTINCT message.msg_timestamp) AS average_time_between_responses
-                        FROM pull_request_events, pull_requests, repo, pull_request_message_ref, message
-                        WHERE repo.repo_id = {repo_id}
-                        AND repo.repo_id = pull_requests.repo_id
-                        AND pull_requests.pull_request_id = pull_request_events.pull_request_id
-                        AND pull_requests.pull_request_id = pull_request_message_ref.pull_request_id
-                        AND pull_request_message_ref.msg_id = message.msg_id
+                        COALESCE(count(*) FILTER (WHERE action = 'merged'), 0) AS merged_count,
+                        COALESCE(MIN(message.msg_timestamp), pull_requests.pr_merged_at, pull_requests.pr_closed_at) AS first_response_time,
+                        COALESCE(COUNT(DISTINCT message.msg_timestamp), 0) AS comment_count,
+                        COALESCE(MAX(message.msg_timestamp), pull_requests.pr_closed_at) AS last_response_time,
+                        COALESCE((MAX(message.msg_timestamp) - MIN(message.msg_timestamp)) / COUNT(DISTINCT message.msg_timestamp), pull_requests.pr_created_at - pull_requests.pr_closed_at) AS average_time_between_responses
+                        FROM  pull_requests 
+												LEFT OUTER JOIN pull_request_events on pull_requests.pull_request_id = pull_request_events.pull_request_id 
+												JOIN repo on repo.repo_id = pull_requests.repo_id 
+												LEFT OUTER JOIN pull_request_message_ref on pull_requests.pull_request_id = pull_request_message_ref.pull_request_id
+												LEFT OUTER JOIN message on pull_request_message_ref.msg_id = message.msg_id
+                        WHERE repo.repo_id = 1
                         GROUP BY pull_requests.pull_request_id
                     ) response_times
                     ON pull_requests.pull_request_id = response_times.pull_request_id
-                    LEFT OUTER JOIN (
-                        SELECT pull_request_commits.pull_request_id, count(DISTINCT pr_cmt_sha) AS commit_count                                FROM pull_request_commits, pull_requests, pull_request_meta
+                    LEFT JOIN (
+                        SELECT pull_request_commits.pull_request_id, count(DISTINCT pr_cmt_sha) AS commit_count                                
+												FROM pull_request_commits, pull_requests, pull_request_meta
                         WHERE pull_requests.pull_request_id = pull_request_commits.pull_request_id
                         AND pull_requests.pull_request_id = pull_request_meta.pull_request_id
-                        AND pull_requests.repo_id = {repo_id}
+                        AND pull_requests.repo_id = 1
                         AND pr_cmt_sha <> pull_requests.pr_merge_commit_sha
                         AND pr_cmt_sha <> pull_request_meta.pr_sha
                         GROUP BY pull_request_commits.pull_request_id
                     ) all_commit_counts
                     ON pull_requests.pull_request_id = all_commit_counts.pull_request_id
-                    LEFT OUTER JOIN (
+                    LEFT JOIN (
                         SELECT MAX(pr_repo_meta_id), pull_request_meta.pull_request_id, pr_head_or_base, pr_src_meta_label
                         FROM pull_requests, pull_request_meta
                         WHERE pull_requests.pull_request_id = pull_request_meta.pull_request_id
-                        AND pull_requests.repo_id = {repo_id}
+                        AND pull_requests.repo_id = 1
                         AND pr_head_or_base = 'base'
                         GROUP BY pull_request_meta.pull_request_id, pr_head_or_base, pr_src_meta_label
                     ) base_labels
                     ON base_labels.pull_request_id = all_commit_counts.pull_request_id
-                    LEFT OUTER JOIN (
+                    LEFT JOIN (
                         SELECT sum(cmt_added) AS lines_added, sum(cmt_removed) AS lines_removed, pull_request_commits.pull_request_id, count(DISTINCT cmt_filename) AS file_count
                         FROM pull_request_commits, commits, pull_requests, pull_request_meta
                         WHERE cmt_commit_hash = pr_cmt_sha
                         AND pull_requests.pull_request_id = pull_request_commits.pull_request_id
                         AND pull_requests.pull_request_id = pull_request_meta.pull_request_id
-                        AND pull_requests.repo_id = {repo_id}
+                        AND pull_requests.repo_id = 1
                         AND commits.repo_id = pull_requests.repo_id
                         AND commits.cmt_commit_hash <> pull_requests.pr_merge_commit_sha
                         AND commits.cmt_commit_hash <> pull_request_meta.pr_sha
@@ -136,7 +137,7 @@ def pull_request_data_collection(repo_id, start_date, end_date):
                 WHERE 
                     repo.repo_group_id = repo_groups.repo_group_id 
                     AND repo.repo_id = pull_requests.repo_id 
-                    AND repo.repo_id = {repo_id} 
+                    AND repo.repo_id = 1 
                 ORDER BY
                     merged_count DESC
                     """)

augur/tasks/start_tasks.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 from typing import List
 import time
 import logging
@@ -10,19 +10,19 @@
 import random
 #from celery.result import AsyncResult
 from celery import signature
 from celery import group, chain, chord, signature
 from sqlalchemy import or_, and_,tuple_, update
 
 
 from augur.tasks.github import *
 if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
     from augur.tasks.data_analysis import *
 from augur.tasks.github.detect_move.tasks import detect_github_repo_move_core, detect_github_repo_move_secondary
 from augur.tasks.github.releases.tasks import collect_releases
 from augur.tasks.github.repo_info.tasks import collect_repo_info, collect_linux_badge_info
 from augur.tasks.github.pull_requests.files_model.tasks import process_pull_request_files
 from augur.tasks.github.pull_requests.commits_model.tasks import process_pull_request_commits
 from augur.tasks.git.dependency_tasks.tasks import process_ossf_dependency_metrics
 from augur.tasks.github.traffic.tasks import collect_github_repo_clones_data
 from augur.tasks.gitlab.merge_request_task import collect_gitlab_merge_requests, collect_merge_request_comments, collect_merge_request_metadata, collect_merge_request_reviewers, collect_merge_request_commits, collect_merge_request_files
 from augur.tasks.gitlab.issues_task import collect_gitlab_issues, collect_gitlab_issue_comments
@@ -339,7 +339,7 @@
     #TODO: Isaac needs to normalize the status's to be abstract in the 
     #collection_status table once augur dev is less unstable.
     with DatabaseSession(logger,engine) as session:
-        query = s.sql.text(f"""UPDATE repo SET secondary_staus = {CollectionState.PENDING.value}"""
+        query = s.sql.text(f"""UPDATE repo SET secondary_status = {CollectionState.PENDING.value}"""
         f""" WHERE secondary_status = '{CollectionState.ERROR.value}' ;"""
         f"""UPDATE repo SET core_status = {CollectionState.PENDING.value}"""
         f""" WHERE core_status = '{CollectionState.ERROR.value}' ;"""

docker-compose-externalDB.yml

@@ -6,6 +6,20 @@ services:
     ports:
      - 6379:6379
 
+  rabbitmq:
+  image: augur-rabbitmq
+  build:
+    context: .
+    dockerfile: ./docker/rabbitmq/Dockerfile
+  # ports for amqp connections / management api
+  ports:
+    - 5671:5671
+    - 5672:5672
+    - 15671:15671
+    - 15672:15672
+  environment:
+    - "RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur}"
+    - "RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123}"
 
   augur:
     image: augurlabs/augur-new:latest
@@ -27,6 +41,7 @@ services:
       - "AUGUR_GITHUB_USERNAME=${AUGUR_GITHUB_USERNAME}"
       - "AUGUR_GITLAB_USERNAME=${AUGUR_GITLAB_USERNAME}"
       - REDIS_CONN_STRING=redis://redis:6379
+      - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/augur_vhost
     depends_on:
       - redis
 

docker-compose.yml

@@ -19,6 +19,20 @@ services:
     ports:
      - 6379:6379
 
+  rabbitmq:
+    image: augur-rabbitmq
+    build:
+      context: .
+      dockerfile: ./docker/rabbitmq/Dockerfile
+    # ports for amqp connections / management api
+    ports:
+      - 5671:5671
+      - 5672:5672
+      - 15671:15671
+      - 15672:15672
+    environment:
+      - "RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur}"
+      - "RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123}"
 
   augur:
     image: augur-new:latest
@@ -40,10 +54,12 @@ services:
       - "AUGUR_GITLAB_API_KEY=${AUGUR_GITLAB_API_KEY}"
       - "AUGUR_GITHUB_USERNAME=${AUGUR_GITHUB_USERNAME}"
       - "AUGUR_GITLAB_USERNAME=${AUGUR_GITLAB_USERNAME}"
-      - "REDIS_CONN_STRING=redis://redis:6379"
+      - REDIS_CONN_STRING=redis://redis:6379
+      - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/augur_vhost
     depends_on:
       - augur-db
       - redis
+      - rabbitmq
 
 volumes:
   facade:

docker/backend/Dockerfile

@@ -2,7 +2,7 @@
 FROM python:3.10-slim-bullseye
 
 LABEL maintainer="outdoors@acm.org"
-LABEL version="0.62.0"
+LABEL version="0.62.4"
 
 ENV DEBIAN_FRONTEND=noninteractive
 

docker/database/Dockerfile

@@ -2,7 +2,7 @@
 FROM postgres:14
 
 LABEL maintainer="outdoors@acm.org"
-LABEL version="0.62.0"
+LABEL version="0.62.4"
 
 ENV POSTGRES_DB "test"
 ENV POSTGRES_USER "augur"

docker/rabbitmq/Dockerfile

@@ -0,0 +1,12 @@
+FROM rabbitmq:3.12-management-alpine
+
+LABEL maintainer="574/augur@simplelogin.com"
+LABEL version="0.62.4"
+
+COPY --chown=rabbitmq:rabbitmq ./docker/rabbitmq/augur.conf /etc/rabbitmq/conf.d/
+
+RUN chmod 777 /etc/rabbitmq/conf.d/augur.conf
+
+RUN touch /etc/rabbitmq/advanced.config \
+  && chmod 544 /etc/rabbitmq/advanced.config \
+  && echo '[ {rabbit, [ {consumer_timeout, undefined} ]} ].' >> /etc/rabbitmq/advanced.config

docker/rabbitmq/augur.conf

@@ -0,0 +1,10 @@
+default_vhost = augur_vhost
+
+default_permissions.configure = .*
+default_permissions.read = .*
+default_permissions.write = .*
+
+default_user_tags.administrator = true
+default_user_tags.augur = true
+default_user_tags.augurTag = true
+

docs/8knot.md

@@ -2,7 +2,7 @@
 ## Required Ubuntu Packages
 ```shell
 sudo apt install docker &&
-sudo apt install docker-compose 
+sudo apt install docker compose 
 ```
 
 If you are running Augur on the same computer, you should already have Python3.x installed. If not, for an Ubuntu 22.x Server also issue this command: 
@@ -21,11 +21,11 @@ git checkout redis-remap
 ## Start 8knot
 To see if everything works: 
 ```shell
-sudo docker-compose up --build
+sudo docker compose up --build
 ```
 
 If you want to run 8Knot in the background: 
 ```shell
-(nohup sudo docker-compose up --build &)
+(nohup sudo docker compose up --build &)
 ```
 

docs/dev-osx-install.md

@@ -428,11 +428,11 @@ Note: Augur will run on port 5000 by default (you probably need to change that i
 You can stop augur with `augur backend stop`, followed by `augur backend kill`. We recommend waiting 5 minutes between commands so Augur can shutdown more gently. There is no issue with data integrity if you issue them seconds apart, its just that stopping is nicer than killing. 
 
 ### Docker
-1. Make sure docker, and docker-compose are both installed
+1. Make sure docker, and docker compose are both installed
 2. Modify the `environment.txt` file in the root of the repository to include your GitHub and GitLab API keys.
 3. If you are already running postgresql on your server you have two choices: 
    - Change the port mappings in the `docker-compose.yml` file to match ports for Postgresql not currently in use.
    - Change to variables in `environment.txt` to include the correct values for your local, non-docker-container database.
 4. `sudo docker build -t augur-new -f docker/backend/Dockerfile .`
-5. `sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up` to run the database in a Docker Container or 
-   `sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up` to connect to an already running database. 
+5. `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to run the database in a Docker Container or 
+   `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to connect to an already running database. 

docs/new-install-ubuntu-python-3.10.md

@@ -368,11 +368,11 @@ Note: Augur will run on port 5000 by default (you probably need to change that i
 You can stop augur with `augur backend stop`, followed by `augur backend kill`. We recommend waiting 5 minutes between commands so Augur can shutdown more gently. There is no issue with data integrity if you issue them seconds apart, its just that stopping is nicer than killing. 
 
 ### Docker
-1. Make sure docker, and docker-compose are both installed
+1. Make sure docker, and docker compose are both installed
 2. Modify the `environment.txt` file in the root of the repository to include your GitHub and GitLab API keys.
 3. If you are already running postgresql on your server you have two choices: 
    - Change the port mappings in the `docker-compose.yml` file to match ports for Postgresql not currently in use.
    - Change to variables in `environment.txt` to include the correct values for your local, non-docker-container database.
 4. `sudo docker build -t augur-new -f docker/backend/Dockerfile .`
-5. `sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up` to run the database in a Docker Container or 
-   `sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up` to connect to an already running database. 
+5. `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to run the database in a Docker Container or 
+   `sudo docker compose --env-file ./environment.txt --file docker-compose.yml up` to connect to an already running database.