nf-core · ewels · Mar 23, 2021 · Mar 23, 2021 · Mar 23, 2021 · Mar 23, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read
 * Added automated PR comments to the Markdown, YAML and Python lint CI tests to explain failures (tools and pipeline template)
 * Make `nf-core lint` summary table borders coloured according to overall pass / fail status
+* Attempted a fix for the automated sync when we submit too many PRs at once [[#911](https://github.com/nf-core/tools/issues/911)]
 
 ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19]
 

diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml
@@ -57,7 +57,7 @@ jobs:
       - name: Install yaml-lint
         run: npm install -g yaml-lint
       - name: Run yaml-lint
-        run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml")
+        run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml" -o -name "*.yaml")
 
       # If the above check failed, post a comment on the PR explaining the failure
       - name: Post PR comment
@@ -73,7 +73,7 @@ jobs:
             * Install `yaml-lint`
                 * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`)
             * Fix the markdown errors
-                * Run the test locally: `yamllint $(find . -type f -name "*.yml")`
+                * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml")`
                 * Fix any reported errors in your YAML files
 
             Once you push these changes the test should pass, and you can hide this comment :+1:

diff --git a/nf_core/sync.py b/nf_core/sync.py
@@ -6,9 +6,11 @@
 import json
 import logging
 import os
+import re
 import requests
 import requests_cache
 import shutil
+import time
 
 import nf_core
 import nf_core.create
@@ -320,27 +322,39 @@ def make_pull_request(self):
             "base": self.from_branch,
         }
 
-        r = requests.post(
-            url="https://api.github.com/repos/{}/pulls".format(self.gh_repo),
-            data=json.dumps(pr_content),
-            auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
-        )
-        try:
-            self.gh_pr_returned_data = json.loads(r.content)
-            returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4)
-        except:
-            self.gh_pr_returned_data = r.content
-            returned_data_prettyprint = r.content
-
-        # PR worked
-        if r.status_code == 201:
-            self.pr_url = self.gh_pr_returned_data["html_url"]
-            log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint))
-            log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"]))
-
-        # Something went wrong
-        else:
-            raise PullRequestException(f"GitHub API returned code {r.status_code}: \n{returned_data_prettyprint}")
+        while True:
+            r = requests.post(
+                url="https://api.github.com/repos/{}/pulls".format(self.gh_repo),
+                data=json.dumps(pr_content),
+                auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]),
+            )
+            try:
+                self.gh_pr_returned_data = json.loads(r.content)
+                returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4)
+            except:
+                self.gh_pr_returned_data = r.content
+                returned_data_prettyprint = r.content
+
+            # PR worked
+            if r.status_code == 201:
+                self.pr_url = self.gh_pr_returned_data["html_url"]
+                log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}")
+                log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}")
+                break
+
+            # Returned 403 error - too many simultaneous requests
+            # https://github.com/nf-core/tools/issues/911
+            if r.status_code == 403:
+                log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}")
+                wait_time = float(re.sub("[^0-9]", "", r.headers.get("Retry-After", 30)))
+                log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..")
+                time.sleep(wait_time)
+
+            # Something went wrong
+            else:
+                raise PullRequestException(
+                    f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}"
+                )
 
     def close_open_template_merge_prs(self):
         """Get all template merging branches (starting with 'nf-core-template-merge-')

diff --git a/tests/test_sync.py b/tests/test_sync.py
@@ -192,6 +192,7 @@ class MockResponse:
             def __init__(self, data, status_code):
                 self.status_code = status_code
                 self.content = json.dumps(data)
+                self.headers = {"content-encoding": "test", "connection": "fake"}
 
         if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls":
             response_data = {"html_url": "great_success"}