Merge pull request #108 from sommersoft/issuepr_metrics

kattni · web-flow · commit 9fb8d334aacb · 2019-09-17T19:33:09.000-04:00
Add "Days Open" Metrics; Validate Issue Labels; CirPy.org AWS Transition; Misc Maintenance
diff --git a/adabot/circuitpython_libraries.py b/adabot/circuitpython_libraries.py
@@ -99,6 +99,7 @@
     if vals[0].startswith("validate")
 ]
 
+pr_sort_re = re.compile("(?<=\(Open\s)(.+)(?=\sdays)")
 
 def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
     """runs the various library checking functions"""
@@ -204,7 +205,10 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
     output_handler("Core")
     print_pr_overview(core_insights)
     output_handler("* {} open pull requests".format(len(core_insights["open_prs"])))
-    for pr in core_insights["open_prs"]:
+    sorted_prs = sorted(core_insights["open_prs"],
+                        key=lambda days: int(pr_sort_re.search(days).group(1)),
+                        reverse=True)
+    for pr in sorted_prs:
         output_handler("  * {}".format(pr))
     print_issue_overview(core_insights)
     output_handler("* {} open issues".format(len(core_insights["open_issues"])))
@@ -223,7 +227,10 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
     output_handler("Libraries")
     print_pr_overview(lib_insights)
     output_handler("* {} open pull requests".format(len(lib_insights["open_prs"])))
-    for pr in lib_insights["open_prs"]:
+    sorted_prs = sorted(lib_insights["open_prs"],
+                        key=lambda days: int(pr_sort_re.search(days).group(1)),
+                        reverse=True)
+    for pr in sorted_prs:
         output_handler("  * {}".format(pr))
     print_issue_overview(lib_insights)
     output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
diff --git a/adabot/lib/circuitpython_library_validators.py b/adabot/lib/circuitpython_library_validators.py
@@ -56,6 +56,7 @@
 ERROR_MISSING_EXAMPLE_FOLDER = "Missing examples folder"
 ERROR_EXAMPLE_MISSING_SENSORNAME = "Example file(s) missing sensor/library name"
 ERROR_MISSING_EXAMPLE_SIMPLETEST = "Missing simpletest example."
+ERROR_MISSING_STANDARD_LABELS = "Missing one or more standard issue labels (bug, documentation, enhancement, good first issue)."
 ERROR_MISSING_LIBRARIANS = "CircuitPythonLibrarians team missing or does not have write access"
 ERROR_MISSING_LICENSE = "Missing license."
 ERROR_MISSING_LINT = "Missing lint config"
@@ -133,6 +134,8 @@
     "Adafruit_CircuitPython_miniQR",
 ]
 
+STD_REPO_LABELS = ("bug", "documentation", "enhancement", "good first issue")
+
 # Cache CircuitPython's subprojects on ReadTheDocs so its not fetched every repo check.
 rtd_subprojects = None
 
@@ -439,14 +442,24 @@ def validate_contents(self, repo):
             return []
 
         content_list = github.get("/repos/" + repo["full_name"] + "/contents/")
-        # Empty repos return an object with a "message" that the repo is empty.
-        if not content_list.ok or "message" in content_list.json():
-            if not self.validate_contents_quiet:
-                return [ERROR_UNABLE_PULL_REPO_CONTENTS]
-            return []
+        empty_repo = False
+        if not content_list.ok:
+            # Empty repos return:
+            #  - a 404 status code
+            #  - a "message" that the repo is empty.
+            if "message" in content_list.json():
+                if "empty" in content_list.json()["message"]:
+                    empty_repo = True
+            if not empty_repo:
+                if not self.validate_contents_quiet:
+                    return [ERROR_UNABLE_PULL_REPO_CONTENTS]
+                return []
 
         content_list = content_list.json()
-        files = [x["name"] for x in content_list]
+        files = []
+        # an empty repo will return a 'message'
+        if not empty_repo:
+            files = [x["name"] for x in content_list]
 
         # ignore new/in-work repos, which should have less than 8 files:
         # ___.py or folder, CoC, .travis.yml, .readthedocs.yml, docs/,
@@ -834,10 +847,17 @@ def gather_insights(self, repo, insights, since):
 
         for issue in issues:
             created = datetime.datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+            days_open = datetime.datetime.today() - created
+            if days_open.days < 0: # opened earlier today
+                days_open += datetime.timedelta(days=(days_open.days * -1))
             if "pull_request" in issue:
-                insights["open_prs"].append(issue["pull_request"]["html_url"])
+                pr_link = "{0} (Open {1} days)".format(issue["pull_request"]["html_url"],
+                                                       days_open.days)
+                insights["open_prs"].append(pr_link)
             else:
-                insights["open_issues"].append(issue["html_url"])
+                issue_link = "{0} (Open {1} days)".format(issue["html_url"],
+                                                          days_open.days)
+                insights["open_issues"].append(issue_link)
 
         # get milestones for core repo
         if repo["name"] == "circuitpython":
@@ -864,3 +884,22 @@ def validate_in_pypi(self, repo):
         if not common_funcs.repo_is_on_pypi(repo):
             return [ERROR_NOT_ON_PYPI]
         return []
+
+    def validate_labels(self, repo):
+        """ensures the repo has the standard labels available"""
+        response = github.get("/repos/" + repo["full_name"] + "/labels")
+        if not response.ok:
+            # replace 'output_handler' with ERROR_OUTPUT_HANDLER
+            self.output_file_data.append("Labels request failed: {}".format(repo["full_name"]))
+            return [ERROR_OUTPUT_HANDLER]
+
+        repo_labels = [label["name"] for label in response.json()]
+        has_all_labels = True
+        for label in STD_REPO_LABELS:
+            if not label in repo_labels:
+                has_all_labels = False
+
+        if not has_all_labels:
+            return [ERROR_MISSING_STANDARD_LABELS]
+        else:
+            return []
diff --git a/adabot/lib/common_funcs.py b/adabot/lib/common_funcs.py
@@ -234,6 +234,8 @@ def is_new_or_updated(repo):
     new_releases = 0
     releases = result.json()
     for release in releases:
+        if not release["published_at"]:
+            continue
         release_date = datetime.datetime.strptime(
             release["published_at"],
             "%Y-%m-%dT%H:%M:%SZ"
diff --git a/adabot/update_cp_org_libraries.py b/adabot/update_cp_org_libraries.py
@@ -26,6 +26,7 @@
 import inspect
 import json
 import os
+import re
 import sh
 from sh.contrib import git
 import sys
@@ -46,6 +47,8 @@
     dest="output_file"
 )
 
+sort_re = re.compile("(?<=\(Open\s)(.+)(?=\sdays)")
+
 def get_open_issues_and_prs(repo):
     """ Retreive all of the open issues (minus pull requests) for the repo.
     """
@@ -59,10 +62,17 @@ def get_open_issues_and_prs(repo):
 
     issues = result.json()
     for issue in issues:
+        created = datetime.datetime.strptime(issue["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+        days_open = datetime.datetime.today() - created
+        if days_open.days < 0: # opened earlier today
+            days_open += datetime.timedelta(days=(days_open.days * -1))
+
+        issue_title = "{0} (Open {1} days)".format(issue["title"],
+                                                   days_open.days)
         if "pull_request" not in issue: # ignore pull requests
-            open_issues.append({issue["html_url"]: issue["title"]})
+            open_issues.append({issue["html_url"]: issue_title})
         else:
-            open_pull_requests.append({issue["html_url"]: issue["title"]})
+            open_pull_requests.append({issue["html_url"]: issue_title})
 
     return open_issues, open_pull_requests
 
@@ -180,25 +190,6 @@ def update_json_file(json_string):
     print("Running circuitpython.org/libraries updater...")
 
     run_time = datetime.datetime.now()
-    # Travis CI weekly cron jobs do not allow or guarantee that they will be run
-    # on a specific day of the week. So, we set the cron to run daily, and then
-    # check for the day we want this to run.
-    if "TRAVIS" in os.environ:
-        should_run = int(os.environ["CP_ORG_UPDATER_RUN_DAY"])
-        if run_time.isoweekday() != should_run:
-            delta_days = should_run - run_time.isoweekday()
-            run_delta = datetime.timedelta(days=delta_days)
-            should_run_date = run_time + run_delta
-            msg = [
-                "Aborting...",
-                " - Today is not {}.".format(should_run_date.strftime("%A")),
-                " - Next scheduled run is: {}".format(should_run_date.strftime("%Y-%m-%d")),
-                " - To run the updater on a different day, change the",
-                "   'CP_ORG_UPDATER_RUN_DAY' environment variable in Travis.",
-                " - Day is a number between 1 & 7, with 1 being Monday."
-            ]
-            print("\n".join(msg))
-            sys.exit()
 
     working_directory = os.path.abspath(os.getcwd())
     #cp_org_dir = os.path.join(working_directory, ".cp_org")
@@ -319,11 +310,8 @@ def update_json_file(json_string):
     }
     json_obj = json.dumps(build_json, indent=2)
 
-    if "TRAVIS" in os.environ:
-        update_json_file(json_obj)
-    else:
-        #update_json_file(json_obj)
-        if local_file_output:
-            with open(output_filename, "w") as json_file:
-                json.dump(build_json, json_file, indent=2)
-        print(json_obj)
+    #update_json_file(json_obj)
+    if local_file_output:
+        with open(output_filename, "w") as json_file:
+            json.dump(build_json, json_file, indent=2)
+    print(json_obj)
