Sync: Try to fix the GitHub abuse mechanism coping code

* Fixes main bug that the default value for the Retry-After header was an int and not a str
* Make the default wait random, between 10 - 60 seconds
* Some tweaks, print the headers in more log messages, make them prettier.

Fixes nf-core#970

nf_core/sync.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import random
 import re
 import requests
 import requests_cache
@@ -331,29 +332,34 @@ def make_pull_request(self):
             try:
                 self.gh_pr_returned_data = json.loads(r.content)
                 returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4)
+                r_headers_pp = json.dumps(r.headers, indent=4)
             except:
                 self.gh_pr_returned_data = r.content
                 returned_data_prettyprint = r.content
+                r_headers_pp = r.headers
 
             # PR worked
             if r.status_code == 201:
                 self.pr_url = self.gh_pr_returned_data["html_url"]
-                log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}")
+                log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}\n\n{r_headers_pp}")
                 log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}")
                 break
 
             # Returned 403 error - too many simultaneous requests
             # https://github.com/nf-core/tools/issues/911
             if r.status_code == 403:
-                log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}")
-                wait_time = float(re.sub("[^0-9]", "", r.headers.get("Retry-After", 30)))
+                log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r_headers_pp}")
+                wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0))))
+                if wait_time == 0:
+                    log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait")
+                    wait_time = random.randrange(10, 60)
                 log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..")
                 time.sleep(wait_time)
 
             # Something went wrong
             else:
                 raise PullRequestException(
-                    f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}"
+                    f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}"
                 )
 
     def close_open_template_merge_prs(self):