Revise release note scripting

utils/create_release_notes.py

@@ -1,38 +1,28 @@
 #!/usr/bin/env python
 
-# Coded for both python2 and python3.
+"""
+Create release notes for a new release of this GitHub repository.
+"""
 
-'''
-Create release notes for a new relase of the GitHub repository.
-'''
+# Requires: 
+#
+# * assumes current directory is within a repository clone
+# * pyGithub (conda or pip install) - https://pygithub.readthedocs.io/
+# * Github personal access token (https://github.com/settings/tokens)
+#
+# Github token access is needed or the GitHub API limit 
+# will likely interfere with making a complete report 
+# of the release.
 
-from datetime import datetime
-import os, sys
-import github
-import collections
 import argparse
+import datetime
+import github
+import logging
+import os
 
 
-CREDS_FILE_NAME = "__github_creds__.txt"        # put in ~/.config/ directory not readable by others
-GITHUB_PER_PAGE = 30
-
-
-def str2time(time_string):
-    # Tue, 20 Dec 2016 17:35:40 GMT
-    fmt = "%a, %d %b %Y %H:%M:%S %Z"
-    return datetime.strptime(time_string, fmt)
-
-
-def getCredentialsFile():
-    search_paths = [
-        os.path.dirname(__file__),
-        os.path.join(os.environ["HOME"], ".config"),    # TODO: robust?
-    ]
-    for path in search_paths:
-        full_name = os.path.join(path, CREDS_FILE_NAME)
-        if os.path.exists(full_name):
-            return full_name
-    raise ValueError('Cannot find credentials file: ' + CREDS_FILE_NAME)
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger('create_release_notes')
 
 
 def findGitConfigFile():
@@ -50,16 +40,19 @@ def findGitConfigFile():
     for i in range(99):
         config_file = os.path.join(path, ".git", "config")
         if os.path.exists(config_file):
-            return config_file
-        # TODO: what if we reach the root of the file system?
+            return config_file      # found it!
+
+        # next, look in the parent directory
         path = os.path.abspath(os.path.join(path, ".."))
+
     msg = "Could not find .git/config file in any parent directory."
+    logger.error(msg)
     raise ValueError(msg)
 
 
 def getRepositoryInfo():
     """
-    return organization, repository from .git/config file
+    return (organization, repository) tuple from .git/config file
 
     This is a simplistic search that could be improved by using 
     an open source package.
@@ -68,124 +61,81 @@ def getRepositoryInfo():
     """
     config_file = findGitConfigFile()
 
-    # found it!
-    found_origin = False
     with open(config_file, "r") as f:
         for line in f.readlines():
             line = line.strip()
-            if line == '[remote "origin"]':
-                found_origin = True
-            elif line.startswith("url"):
+            if line.startswith("url"):
                 url = line.split("=")[-1].strip()
                 if url.find("github.com") < 0:
                     msg = "Not a GitHub repo: " + url
+                    logger.error(msg)
                     raise ValueError(msg)
                 org, repo = url.rstrip(".git").split("/")[-2:]
                 return org, repo
 
 
-class ReleaseNotes(object):
-
-    def __init__(self, base, head=None, milestone=None):
-        self.base = base
-        self.head = head or "master"
-        self.milestone_title = milestone
-        self.milestone = None
-
-        self.commit_db = {}
-        self.db = dict(tags={}, pulls={}, issues={}, commits={})
-        self.organization, self.repository = getRepositoryInfo()
-        self.creds_file_name = getCredentialsFile()
-
-    def connect(self):
-        uname, pwd = open(self.creds_file_name, 'r').read().split()
-        self.gh = github.Github(uname, password=pwd, per_page=GITHUB_PER_PAGE)
-        self.user = self.gh.get_user(self.organization)
-        self.repo = self.user.get_repo(self.repository)
-
-    def learn(self):
-        base_commit = None
-        earliest = None
-        compare = self.repo.compare(self.base, self.head)
-        commits = self.db["commits"] = collections.OrderedDict()
-        for commit in compare.commits:
-            commits[commit.sha] = commit
-#         commits = self.db["commits"] = {commit.sha: commit for commit in compare.commits}
-
-        for milestone in self.repo.get_milestones(state="all"):
-            if milestone.title == self.milestone_title:
-                self.milestone = milestone
-                break
-        if self.milestone is None:
-            msg = "Could not find milestone: " + self.milestone
-            raise ValueError(msg)
-
-        tags = self.db["tags"]
-        for tag in self.repo.get_tags():
-            if tag.commit.sha in commits:
-                tags[tag.name] = tag
-            elif tag.name == self.base:
-                base_commit = self.repo.get_commit(tag.commit.sha)
-                earliest = str2time(base_commit.last_modified)
-
-        pulls = self.db["pulls"]
-        for pull in self.repo.get_pulls(state="closed"):
-            if pull.closed_at > earliest:
-                pulls[pull.number] = pull
-
-        issues = self.db["issues"]
-        for issue in self.repo.get_issues(milestone=self.milestone, state="closed"):
-            if self.milestone is not None or issue.closed_at > earliest:
-                if issue.number not in pulls:
-                    issues[issue.number] = issue
-
-    def print_report(self):
-        print("## " + self.milestone_title)
-        print("")
-        if self.milestone is not None:
-            print("**milestone**: [%s](%s)" % (self.milestone.title, self.milestone.url))
-            print("")
-        print("section | number")
-        print("-"*5, " | ", "-"*5)
-        print("New Tags | ", len(self.db["tags"]))
-        print("Pull Requests | ", len(self.db["pulls"]))
-        print("Issues | ", len(self.db["issues"]))
-        print("Commits | ", len(self.db["commits"]))
-        print("")
-        print("### Tags")
-        print("")
-        for k, tag in sorted(self.db["tags"].items()):
-            print("* [%s](%s) %s" % (tag.commit.sha[:7], tag.commit.html_url, k))
-        print("")
-        print("### Pull Requests")
-        print("")
-        for k, pull in sorted(self.db["pulls"].items()):
-            state = {True: "merged", False: "closed"}[pull.merged]
-            print("* [#%d](%s) (%s) %s" % (pull.number, pull.html_url, state, pull.title))
-        print("")
-        print("### Issues")
-        print("")
-        for k, issue in sorted(self.db["issues"].items()):
-            if k not in self.db["pulls"]:
-                print("* [#%d](%s) %s" % (issue.number, issue.html_url, issue.title))
-        print("")
-        print("### Commits")
-        print("")
-        for k, commit in self.db["commits"].items():
-            message = commit.commit.message.splitlines()[0]
-            print("* [%s](%s) %s" % (k[:7], commit.html_url, message))
-        print("")
+def get_release_info(token, base_tag_name, head_branch_name, milestone_name):
+    """mine the Github API for information about this release"""
+    organization_name, repository_name = getRepositoryInfo()
+    gh = github.Github(token)   # GitHub Personal Access Token
 
+    user = gh.get_user(organization_name)
+    logger.debug(f"user: {user}")
 
-def main(base="v3.2", head="master", milestone="NXDL 3.3"):
-    # github.enable_console_debug_logging()
-    notes = ReleaseNotes(base, head=head, milestone=milestone)
-    notes.connect()
-    notes.learn()
-    notes.print_report()
+    repo = user.get_repo(repository_name)
+    logger.debug(f"repo: {repo}")
+
+    milestones = [
+        m
+        for m in repo.get_milestones(state="all")
+        if m.title == milestone_name
+    ]
+    if len(milestones) == 0:
+        msg = f"Could not find milestone: {milestone_name}"
+        logger.error(msg)
+        raise ValueError(msg)
+    milestone = milestones[0]
+    logger.debug(f"milestone: {milestone}")
+
+    compare = repo.compare(base_tag_name, head_branch_name)
+    logger.debug(f"compare: {compare}")
+
+    commits = {c.sha: c for c in compare.commits}
+    logger.debug(f"# commits: {len(commits)}")
+
+    tags = {}
+    earliest = None
+    for t in repo.get_tags():
+        if t.commit.sha in commits:
+            tags[t.name] = t
+        elif t.name == base_tag_name:
+            base_commit = repo.get_commit(t.commit.sha)
+            earliest = str2time(base_commit.last_modified)
+    logger.debug(f"# tags: {len(tags)}")
+
+    pulls = {
+        p.number: p
+        for p in repo.get_pulls(state="closed")
+        if p.closed_at > earliest
+    }
+    logger.debug(f"# pulls: {len(pulls)}")
+
+    issues = {
+        i.number: i
+        for i in repo.get_issues(milestone=milestone, state="closed")
+        if (
+            (milestone is not None or i.closed_at > earliest)
+            and
+            i.number not in pulls
+        )
+    }
+    logger.debug(f"# issues: {len(issues)}")
+
+    return milestone, tags, pulls, issues, commits
 
 
 def parse_command_line():
+    """command line argument parser"""
     doc = __doc__.strip()
     parser = argparse.ArgumentParser(description=doc)
 
@@ -195,6 +145,13 @@ def parse_command_line():
     help_text = "name of milestone"
     parser.add_argument('milestone', action='store', help=help_text)
 
+    parser.add_argument(
+        'token', 
+        action='store', 
+        help=(
+            "personal access token "
+            "(see: https://github.com/settings/tokens)"))
+
     help_text = "name of tag, branch, SHA to end the range"
     help_text += ' (default="master")'
     parser.add_argument(
@@ -208,9 +165,94 @@ def parse_command_line():
     return parser.parse_args()
 
 
+def str2time(time_string):
+    """convert date/time string to datetime object
+
+    input string example: ``Tue, 20 Dec 2016 17:35:40 GMT``
+    """
+    if time_string is None:
+        msg = f"need valid date/time string, not: {time_string}"
+        logger.error(msg)
+        raise ValueError(msg)
+    return datetime.datetime.strptime(
+        time_string, 
+        "%a, %d %b %Y %H:%M:%S %Z")
+
+
+def report(title, milestone, tags, pulls, issues, commits):
+    print(f"## {title}")
+    print("")
+    print(f"* **date/time**: {datetime.datetime.now()}")
+    print("* **release**: ")
+    print("* **documentation**: [PDF]()")
+    if milestone is not None:
+        print(f"* **milestone**: [{milestone.title}]({milestone.url})")
+        print("")
+    print("section | quantity")
+    print("-"*5, " | ", "-"*5)
+    print(f"New Tags | {len(tags)}")
+    print(f"Pull Requests | {len(pulls)}")
+    print(f"Issues | {len(issues)}")
+    print(f"Commits | {len(commits)}")
+    print("")
+    print("### Tags")
+    print("")
+    print("tag | date | name")
+    print("-"*5, " | ", "-"*5, " | ", "-"*5)
+    for k, tag in sorted(tags.items()):
+        when = str2time(tag.last_modified).strftime("%Y-%m-%d")
+        print(f"[{tag.commit.sha[:7]}]({tag.commit.html_url}) | {when} | {k}")
+    print("")
+    print("### Pull Requests")
+    print("")
+    print("pull request | date | state | title")
+    print("-"*5, " | ", "-"*5, " | ", "-"*5, " | ", "-"*5)
+    for k, pull in sorted(pulls.items()):
+        state = {True: "merged", False: "closed"}[pull.merged]
+        when = str2time(pull.last_modified).strftime("%Y-%m-%d")
+        print(f"[#{pull.number}]({pull.html_url}) | {when} | {state} | {pull.title}")
+    print("")
+    print("### Issues")
+    print("")
+    print("issue | date | title")
+    print("-"*5, " | ", "-"*5, " | ", "-"*5)
+    for k, issue in sorted(issues.items()):
+        if k not in pulls:
+            when = issue.closed_at.strftime("%Y-%m-%d")
+            print(f"[#{issue.number}]({issue.html_url}) | {when} | {issue.title}")
+    print("")
+    print("### Commits")
+    print("")
+    print("commit | date | message")
+    print("-"*5, " | ", "-"*5, " | ", "-"*5)
+    for k, commit in commits.items():
+        message = commit.commit.message.splitlines()[0]
+        when = commit.raw_data['commit']['committer']['date'].split("T")[0]
+        print(f"[{k[:7]}]({commit.html_url}) | {when} | {message}")
+
+
+def main(base=None, head=None, milestone=None, token=None, debug=False):
+    if debug:
+        base_tag_name = base
+        head_branch_name = head
+        milestone_name = milestone
+        logger.setLevel(logging.DEBUG)
+    else:
+        cmd = parse_command_line()
+        base_tag_name = cmd.base
+        head_branch_name = cmd.head
+        milestone_name = cmd.milestone
+        token = cmd.token
+        logger.setLevel(logging.WARNING)
+
+    info = get_release_info(
+        token, base_tag_name, head_branch_name, milestone_name)
+    milestone, tags, pulls, issues, commits = info
+    report(milestone_name, *info)
+
+
 if __name__ == '__main__':
-    cmd = parse_command_line()
-    main(cmd.base, head=cmd.head, milestone=cmd.milestone)
+    main()
 
 
 # NeXus - Neutron and X-ray Common Data Format

utils/dev_create_release_notes.py

@@ -3,6 +3,21 @@
 '''
 Developers: use this code to develop and test create_release_notes.py
 '''
+import os
+
+CREDS_FILE = os.path.join(
+    os.environ["HOME"],
+    ".config",
+    "github_token",
+)
+
+with open(CREDS_FILE, "r") as cf:
+    token = cf.read().strip()
 
 from create_release_notes import main
-main("NXDL 2018.5", head="master", milestone="NXDL 2020.1")
+main(
+    base="v2018.5", 
+    head="master", 
+    milestone="NXDL 2020.1", 
+    token=token,
+    debug=True)