From a14af2f310c10f21f6673bc63dc44a4ee56d5e66 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:08:16 +0100 Subject: [PATCH 1/7] Find yml and yaml files for yaml linting --- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 5b09090281..4570f43bda 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -57,7 +57,7 @@ jobs: - name: Install yaml-lint run: npm install -g yaml-lint - name: Run yaml-lint - run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml") + run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml" -o -name "*.yaml") # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment @@ -73,7 +73,7 @@ jobs: * Install `yaml-lint` * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`) * Fix the markdown errors - * Run the test locally: `yamllint $(find . -type f -name "*.yml")` + * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml")` * Fix any reported errors in your YAML files Once you push these changes the test should pass, and you can hide this comment :+1: From f714baae3df276df21c3908504c2677094724aef Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:11:24 +0100 Subject: [PATCH 2/7] Log API headers on fail - create PR --- nf_core/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 68e9a9c5a4..c7520c131b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -340,7 +340,9 @@ def make_pull_request(self): # Something went wrong else: - raise PullRequestException(f"GitHub API returned code {r.status_code}: \n{returned_data_prettyprint}") + raise PullRequestException( + f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + ) def close_open_template_merge_prs(self): """Get all template merging branches (starting with 'nf-core-template-merge-') From 01dcdf23408ba7d20789f8eb3f1603f44dcca72a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:18:15 +0100 Subject: [PATCH 3/7] Sync - keep trying if we get 403 errors --- nf_core/sync.py | 75 +++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index c7520c131b..02ede2ceaa 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -6,9 +6,11 @@ import json import logging import os +import re import requests import requests_cache import shutil +import time import nf_core import nf_core.create @@ -311,38 +313,49 @@ def make_pull_request(self): "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) - # Make new pull-request - pr_content = { - "title": pr_title, - "body": pr_body_text, - "maintainer_can_modify": True, - "head": self.merge_branch, - "base": self.from_branch, - } - - r = requests.post( - url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), - data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) - try: - self.gh_pr_returned_data = json.loads(r.content) - returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4) - except: - self.gh_pr_returned_data = r.content - returned_data_prettyprint = r.content - - # PR worked - if r.status_code == 201: - self.pr_url = self.gh_pr_returned_data["html_url"] - log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) - log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) - - # Something went wrong - else: - raise PullRequestException( - f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + while True: + # Make new pull-request + pr_content = { + "title": pr_title, + "body": pr_body_text, + "maintainer_can_modify": True, + "head": self.merge_branch, + "base": self.from_branch, + } + + r = requests.post( + url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), + data=json.dumps(pr_content), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), ) + try: + self.gh_pr_returned_data = json.loads(r.content) + returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4) + except: + self.gh_pr_returned_data = r.content + returned_data_prettyprint = r.content + + # PR worked + if r.status_code == 201: + self.pr_url = self.gh_pr_returned_data["html_url"] + log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}") + log.info(f"GitHub PR created: {self.gh_pr_returned_data["html_url"]}") + break + + # Returned 403 error - too many simultaneous requests + # https://github.com/nf-core/tools/issues/911 + if r.status_code == 403: + log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}") + wait_time = float(re.sub('[^0-9]','', r.headers.get('Retry-After', 30))) + log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..") + time.sleep(wait_time) + + + # Something went wrong + else: + raise PullRequestException( + f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + ) def close_open_template_merge_prs(self): """Get all template merging branches (starting with 'nf-core-template-merge-') From 07c3978590496443de05f1519f7af42b114f681c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:19:00 +0100 Subject: [PATCH 4/7] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d13846bfb2..87f42a3026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdown, YAML and Python lint CI tests to explain failures (tools and pipeline template) * Make `nf-core lint` summary table borders coloured according to overall pass / fail status +* Attempted a fix for the automated sync when we submit too many PRs at once [[#911](https://github.com/nf-core/tools/issues/911)] ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] From d0633b2b4db0af115afc358dd9df1788b76c7db5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:20:52 +0100 Subject: [PATCH 5/7] Fix copypasta syntax error --- nf_core/sync.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 02ede2ceaa..468145ba0b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -339,18 +339,17 @@ def make_pull_request(self): if r.status_code == 201: self.pr_url = self.gh_pr_returned_data["html_url"] log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}") - log.info(f"GitHub PR created: {self.gh_pr_returned_data["html_url"]}") + log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}") break # Returned 403 error - too many simultaneous requests # https://github.com/nf-core/tools/issues/911 if r.status_code == 403: log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}") - wait_time = float(re.sub('[^0-9]','', r.headers.get('Retry-After', 30))) + wait_time = float(re.sub("[^0-9]", "", r.headers.get("Retry-After", 30))) log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..") time.sleep(wait_time) - # Something went wrong else: raise PullRequestException( From 3f15ce010ab28d35b452aeb51f65532ccda69f82 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:22:36 +0100 Subject: [PATCH 6/7] fix tests --- tests/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index 4a34e68a0e..3900a95d5d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -192,6 +192,7 @@ class MockResponse: def __init__(self, data, status_code): self.status_code = status_code self.content = json.dumps(data) + self.headers = {"content-encoding": "test", "connection": "fake"} if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} From fa00cde38cd2dcd9b3e45216698e2ea1f0d435e0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:24:03 +0100 Subject: [PATCH 7/7] Reduce diff - didn't need to indent that bit --- nf_core/sync.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 468145ba0b..ee8893089c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -313,16 +313,16 @@ def make_pull_request(self): "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) - while True: - # Make new pull-request - pr_content = { - "title": pr_title, - "body": pr_body_text, - "maintainer_can_modify": True, - "head": self.merge_branch, - "base": self.from_branch, - } + # Make new pull-request + pr_content = { + "title": pr_title, + "body": pr_body_text, + "maintainer_can_modify": True, + "head": self.merge_branch, + "base": self.from_branch, + } + while True: r = requests.post( url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), data=json.dumps(pr_content),