diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index b0dd5cc25a..d9b8623bec 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -24,9 +24,7 @@ jobs: message: | Hi @${{ github.event.pull_request.user.login }}, - It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. - The `master` branch on nf-core repositories should always contain code from the latest release. - Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. + It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 256a33570d..764649e5cc 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -4,13 +4,34 @@ on: types: [published] jobs: - sync-all: - name: Sync all pipelines + get-pipelines: runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + curl -O https://nf-co.re/pipeline_names.json + echo "::set-output name=matrix::$(cat pipeline_names.json)" + sync: + needs: get-pipelines + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} + fail-fast: false steps: + - uses: actions/checkout@v2 - name: Check out source-code repository + name: Check out nf-core/tools + + - uses: actions/checkout@v2 + name: Check out nf-core/${{ matrix.pipeline }} + with: + repository: nf-core/${{ matrix.pipeline }} + ref: dev + token: ${{ secrets.nf_core_bot_auth_token }} + path: nf-core/${{ matrix.pipeline }} - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -32,14 +53,20 @@ jobs: - name: Run synchronisation if: github.repository == 'nf-core/tools' env: - AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} + GITHUB_AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} run: | git config --global user.email "core@nf-co.re" git config --global user.name "nf-core-bot" - nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + nf-core --log-file sync_log_${{ matrix.pipeline }}.txt sync nf-core/${{ matrix.pipeline }} \ + --from-branch dev \ + --pull-request \ + --username nf-core-bot \ + --repository nf-core/${{ matrix.pipeline }} + - name: Upload sync log file artifact + if: ${{ always() }} uses: actions/upload-artifact@v2 with: - name: sync-log-file - path: sync_log.txt + name: sync_log_${{ matrix.pipeline }} + path: sync_log_${{ matrix.pipeline }}.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 69c9a9b2ad..2fcf3628f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,25 @@ Apologies for the inconvenience. * Fix syntax error in `/push_dockerhub.yml` GitHub Action workflow * Change `params.readPaths` -> `params.input_paths` in `test_full.config` * Check results when posting the lint results as a GitHub comment + * This feature is unfortunately not possible when making PRs from forks outside of the nf-core organisation for now. +* More major refactoring of the automated pipeline sync + * New GitHub Actions matrix parallelisation of sync jobs across pipelines [[#673](https://github.com/nf-core/tools/issues/673)] + * Removed the `--all` behaviour from `nf-core sync` as we no longer need it + * Sync now uses a new list of pipelines on the website which does not include archived pipelines [[#712](https://github.com/nf-core/tools/issues/712)] + * When making a PR it checks if a PR already exists - if so it updates it [[#710](https://github.com/nf-core/tools/issues/710)] + * More tests and code refactoring for more stable code. Hopefully fixes 404 error [[#711](https://github.com/nf-core/tools/issues/711)] ## [v1.10.1 - Copper Camel _(patch)_](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30] Patch release to fix the automatic template synchronisation, which failed in the v1.10 release. * Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk. -* GitHub actions sync now uploads verbose log as an artifact. -* Sync - fixed several minor bugs, improved logging. +* nf-core/tools GitHub Actions pipeline sync now uploads verbose log as an artifact. +* Sync - fixed several minor bugs, made logging less verbose. * Python Rich library updated to `>=4.2.1` +* Hopefully fix git config for pipeline sync so that commit comes from @nf-core-bot +* Fix sync auto-PR text indentation so that it doesn't all show as code +* Added explicit flag `--show-passed` for `nf-core lint` instead of taking logging verbosity ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a52b18a101..be9bdfaac9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -550,14 +550,12 @@ def bump_version(pipeline_dir, new_version, nextflow): @nf_core_cli.command("sync", help_priority=10) -@click.argument("pipeline_dir", type=click.Path(exists=True), nargs=-1, metavar="") +@click.argument("pipeline_dir", required=True, type=click.Path(exists=True), metavar="") @click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow vars.") @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") -@click.option("-u", "--username", type=str, help="GitHub username for the PR.") -@click.option("-r", "--repository", type=str, help="GitHub repository name for the PR.") -@click.option("-a", "--auth-token", type=str, help="GitHub API personal access token.") -@click.option("--all", is_flag=True, default=False, help="Sync template for all nf-core pipelines.") -def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_token, all): +@click.option("-r", "--repository", type=str, help="GitHub PR: target repository.") +@click.option("-u", "--username", type=str, help="GitHub PR: auth username.") +def sync(pipeline_dir, from_branch, pull_request, repository, username): """ Sync a pipeline TEMPLATE branch with the nf-core template. @@ -571,24 +569,13 @@ def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_tok new release of nf-core/tools (and the included template) is made. """ - # Pull and sync all nf-core pipelines - if all: - nf_core.sync.sync_all_pipelines(username, auth_token) - else: - # Manually check for the required parameter - if not pipeline_dir or len(pipeline_dir) != 1: - log.error("Either use --all or specify one ") - sys.exit(1) - else: - pipeline_dir = pipeline_dir[0] - - # Sync the given pipeline dir - sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request) - try: - sync_obj.sync() - except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: - log.error(e) - sys.exit(1) + # Sync the given pipeline dir + sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request, repository, username) + try: + sync_obj.sync() + except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + log.error(e) + sys.exit(1) if __name__ == "__main__": diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index bd292ca180..3836aa7637 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -46,7 +46,7 @@ These tests are run both with the latest available version of `Nextflow` and als ## Patch -: warning: Only in the unlikely and regretful event of a release happening with a bug. +:warning: Only in the unlikely and regretful event of a release happening with a bug. * On your own fork, make a new branch `patch` based on `upstream/master`. * Fix the bug, and bump version (X.Y.Z+1). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 94ec0a87ca..a561ad54f0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -17,6 +17,7 @@ jobs: {% raw %} # If the above check failed, post a comment on the PR explaining the failure + # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() uses: mshick/add-pr-comment@v1 diff --git a/nf_core/sync.py b/nf_core/sync.py index dc1fdeff56..0f48eb5161 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -44,7 +44,6 @@ class PipelineSync(object): make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes gh_username (str): GitHub username gh_repo (str): GitHub repository name - gh_auth_token (str): Authorisation token used to make PR with GitHub API Attributes: pipeline_dir (str): Path to target pipeline directory @@ -55,11 +54,10 @@ class PipelineSync(object): required_config_vars (list): List of nextflow variables required to make template pipeline gh_username (str): GitHub username gh_repo (str): GitHub repository name - gh_auth_token (str): Authorisation token used to make PR with GitHub API """ def __init__( - self, pipeline_dir, from_branch=None, make_pr=False, gh_username=None, gh_repo=None, gh_auth_token=None, + self, pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, ): """ Initialise syncing object """ @@ -73,17 +71,16 @@ def __init__( self.gh_username = gh_username self.gh_repo = gh_repo - self.gh_auth_token = gh_auth_token def sync(self): """ Find workflow attributes, create a new template pipeline on TEMPLATE """ - log.debug("Pipeline directory: {}".format(self.pipeline_dir)) + log.info("Pipeline directory: {}".format(self.pipeline_dir)) if self.from_branch: - log.debug("Using branch `{}` to fetch workflow variables".format(self.from_branch)) + log.info("Using branch `{}` to fetch workflow variables".format(self.from_branch)) if self.make_pr: - log.debug("Will attempt to automatically create a pull request") + log.info("Will attempt to automatically create a pull request") self.inspect_sync_dir() self.get_wf_config() @@ -124,7 +121,7 @@ def inspect_sync_dir(self): # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name - log.debug("Original pipeline repository branch is '{}'".format(self.original_branch)) + log.info("Original pipeline repository branch is '{}'".format(self.original_branch)) # Check to see if there are uncommitted changes on current branch if self.repo.is_dirty(untracked_files=True): @@ -139,7 +136,7 @@ def get_wf_config(self): # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: - log.debug("Checking out workflow branch '{}'".format(self.from_branch)) + log.info("Checking out workflow branch '{}'".format(self.from_branch)) self.repo.git.checkout(self.from_branch) except git.exc.GitCommandError: raise SyncException("Branch `{}` not found!".format(self.from_branch)) @@ -151,26 +148,6 @@ def get_wf_config(self): except git.exc.GitCommandError as e: log.error("Could not find active repo branch: ".format(e)) - # Figure out the GitHub username and repo name from the 'origin' remote if we can - try: - origin_url = self.repo.remotes.origin.url.rstrip(".git") - gh_origin_match = re.search(r"github\.com[:\/]([^\/]+)/([^\/]+)$", origin_url) - if gh_origin_match: - self.gh_username = gh_origin_match.group(1) - self.gh_repo = gh_origin_match.group(2) - else: - raise AttributeError - except AttributeError as e: - log.debug( - "Could not find repository URL for remote called 'origin' from remote: {}".format(self.repo.remotes) - ) - else: - log.debug( - "Found username and repo from remote: {}, {} - {}".format( - self.gh_username, self.gh_repo, self.repo.remotes.origin.url - ) - ) - # Fetch workflow variables log.debug("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) @@ -200,7 +177,7 @@ def delete_template_branch_files(self): Delete all files in the TEMPLATE branch """ # Delete everything - log.debug("Deleting all files in TEMPLATE branch") + log.info("Deleting all files in TEMPLATE branch") for the_file in os.listdir(self.pipeline_dir): if the_file == ".git": continue @@ -218,7 +195,7 @@ def make_template_pipeline(self): """ Delete all files and make a fresh template using the workflow variables """ - log.debug("Making a new template pipeline using pipeline variables") + log.info("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) @@ -245,7 +222,7 @@ def commit_template_changes(self): self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) self.made_changes = True - log.debug("Committed changes to TEMPLATE branch") + log.info("Committed changes to TEMPLATE branch") except Exception as e: raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) return True @@ -255,7 +232,7 @@ def push_template_branch(self): and try to make a PR. If we don't have the auth token, try to figure out a URL for the PR and print this to the console. """ - log.debug("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) + log.info("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) try: self.repo.git.push() except git.exc.GitCommandError as e: @@ -276,17 +253,18 @@ def make_pull_request(self): # If we've been asked to make a PR, check that we have the credentials try: - assert self.gh_auth_token is not None + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" except AssertionError: - log.info( - "Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format( - self.gh_username, self.gh_repo, self.original_branch + raise PullRequestException( + "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" + "Make a PR at the following URL:\n https://github.com/{}/compare/{}...TEMPLATE".format( + self.gh_repo, self.original_branch ) ) - raise PullRequestException("No GitHub authentication token set - cannot make PR") - log.debug("Submitting a pull request via the GitHub API") + log.info("Submitting a pull request via the GitHub API") + pr_title = "Important! Template update for nf-core/tools v{}".format(nf_core.__version__) pr_body_text = ( "A new release of the main template in nf-core/tools has just been released. " "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n" @@ -298,17 +276,90 @@ def make_pull_request(self): "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) + # Try to update an existing pull-request + if self.update_existing_pull_request(pr_title, pr_body_text) is False: + # None found - make a new pull-request + self.submit_pull_request(pr_title, pr_body_text) + + def update_existing_pull_request(self, pr_title, pr_body_text): + """ + List existing pull-requests between TEMPLATE and self.from_branch + + If one is found, attempt to update it with a new title and body text + If none are found, return False + """ + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" + # Look for existing pull-requests + list_prs_url = "https://api.github.com/repos/{}/pulls?head=nf-core:TEMPLATE&base={}".format( + self.gh_repo, self.from_branch + ) + r = requests.get( + url=list_prs_url, auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), + ) + try: + r_json = json.loads(r.content) + r_pp = json.dumps(r_json, indent=4) + except: + r_json = r.content + r_pp = r.content + + # PR worked + if r.status_code == 200: + log.debug("GitHub API listing existing PRs:\n{}".format(r_pp)) + + # No open PRs + if len(r_json) == 0: + log.info("No open PRs found between TEMPLATE and {}".format(self.from_branch)) + return False + + # Update existing PR + pr_update_api_url = r_json[0]["url"] + pr_content = {"title": pr_title, "body": pr_body_text} + + r = requests.patch( + url=pr_update_api_url, + data=json.dumps(pr_content), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), + ) + try: + r_json = json.loads(r.content) + r_pp = json.dumps(r_json, indent=4) + except: + r_json = r.content + r_pp = r.content + + # PR update worked + if r.status_code == 200: + log.debug("GitHub API PR-update worked:\n{}".format(r_pp)) + log.info("Updated GitHub PR: {}".format(r_json["html_url"])) + return True + # Something went wrong + else: + log.warn("Could not update PR ('{}'):\n{}\n{}".format(r.status_code, pr_update_api_url, r_pp)) + return False + + # Something went wrong + else: + log.warn("Could not list open PRs ('{}')\n{}\n{}".format(r.status_code, list_prs_url, r_pp)) + return False + + def submit_pull_request(self, pr_title, pr_body_text): + """ + Create a new pull-request on GitHub + """ + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" pr_content = { - "title": "Important! Template update for nf-core/tools v{}".format(nf_core.__version__), + "title": pr_title, "body": pr_body_text, "maintainer_can_modify": True, "head": "TEMPLATE", "base": self.from_branch, } + r = requests.post( - url="https://api.github.com/repos/{}/{}/pulls".format(self.gh_username, self.gh_repo), + url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, self.gh_auth_token), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), ) try: self.gh_pr_returned_data = json.loads(r.content) @@ -317,86 +368,23 @@ def make_pull_request(self): self.gh_pr_returned_data = r.content returned_data_prettyprint = r.content - if r.status_code != 201: + # PR worked + if r.status_code == 201: + log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) + log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) + + # Something went wrong + else: raise PullRequestException( "GitHub API returned code {}: \n{}".format(r.status_code, returned_data_prettyprint) ) - else: - log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) - log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) def reset_target_dir(self): """ Reset the target pipeline directory. Check out the original branch. """ - log.debug("Checking out original branch: '{}'".format(self.original_branch)) + log.info("Checking out original branch: '{}'".format(self.original_branch)) try: self.repo.git.checkout(self.original_branch) except git.exc.GitCommandError as e: raise SyncException("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) - - -def sync_all_pipelines(gh_username=None, gh_auth_token=None): - """Sync all nf-core pipelines - """ - - # Get remote workflows - wfs = nf_core.list.Workflows() - wfs.get_remote_workflows() - - successful_syncs = [] - failed_syncs = [] - - # Set up a working directory - tmpdir = tempfile.mkdtemp() - - # Let's do some updating! - for wf in wfs.remote_workflows: - - log.info("-" * 30) - log.info("Syncing {}".format(wf.full_name)) - - # Make a local working directory - wf_local_path = os.path.join(tmpdir, wf.name) - os.mkdir(wf_local_path) - log.debug("Sync working directory: {}".format(wf_local_path)) - - # Clone the repo - wf_remote_url = "https://{}@github.com/nf-core/{}".format(gh_auth_token, wf.name) - repo = git.Repo.clone_from(wf_remote_url, wf_local_path) - assert repo - - # Only show error messages from pipeline creation - logging.getLogger("nf_core.create").setLevel(logging.ERROR) - - # Sync the repo - log.debug("Running template sync") - sync_obj = nf_core.sync.PipelineSync( - pipeline_dir=wf_local_path, - from_branch="dev", - make_pr=True, - gh_username=gh_username, - gh_auth_token=gh_auth_token, - ) - try: - sync_obj.sync() - except (SyncException, PullRequestException) as e: - log.error("Sync failed for {}:\n{}".format(wf.full_name, e)) - failed_syncs.append(wf.name) - except Exception as e: - log.error("Something went wrong when syncing {}:\n{}".format(wf.full_name, e)) - failed_syncs.append(wf.name) - else: - log.info("[green]Sync successful for {}".format(wf.full_name)) - successful_syncs.append(wf.name) - - # Clean up - log.debug("Removing work directory: {}".format(wf_local_path)) - shutil.rmtree(wf_local_path) - - if len(successful_syncs) > 0: - log.info("[green]Finished. Successfully synchronised {} pipelines".format(len(successful_syncs))) - - if len(failed_syncs) > 0: - failed_list = "\n - ".join(failed_syncs) - log.error("[red]Errors whilst synchronising {} pipelines:\n - {}".format(len(failed_syncs), failed_list)) diff --git a/tests/test_sync.py b/tests/test_sync.py index 01b56ac358..c7726cfb7d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -30,6 +30,7 @@ def test_inspect_sync_dir_notgit(self): psync = nf_core.sync.PipelineSync(tempfile.mkdtemp()) try: psync.inspect_sync_dir() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: assert "does not appear to be a git repository" in e.args[0] @@ -42,6 +43,7 @@ def test_inspect_sync_dir_dirty(self): psync = nf_core.sync.PipelineSync(self.pipeline_dir) try: psync.inspect_sync_dir() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: os.remove(test_fn) assert e.args[0].startswith("Uncommitted changes found in pipeline directory!") @@ -56,24 +58,10 @@ def test_get_wf_config_no_branch(self): try: psync.inspect_sync_dir() psync.get_wf_config() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: assert e.args[0] == "Branch `foo` not found!" - def test_get_wf_config_fetch_origin(self): - """ - Try getting the GitHub username and repo from the git origin - - Also checks the fetched config variables, should pass - """ - # Try to sync, check we halt with the right error - psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.inspect_sync_dir() - # Add a remote to the git repo - psync.repo.create_remote("origin", "https://github.com/nf-core/demo.git") - psync.get_wf_config() - assert psync.gh_username == "nf-core" - assert psync.gh_repo == "demo" - def test_get_wf_config_missing_required_config(self): """ Try getting a workflow config, then make it miss a required config option """ # Try to sync, check we halt with the right error @@ -82,6 +70,7 @@ def test_get_wf_config_missing_required_config(self): try: psync.inspect_sync_dir() psync.get_wf_config() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: # Check that we did actually get some config back assert psync.wf_config["params.outdir"] == "'./results'" @@ -163,6 +152,7 @@ def test_push_template_branch_error(self): # Try to push changes try: psync.push_template_branch() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Could not push TEMPLATE branch") @@ -173,6 +163,7 @@ def test_make_pull_request_missing_username(self): psync.gh_repo = None try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0] == "Could not find GitHub username and repo name" @@ -180,14 +171,19 @@ def test_make_pull_request_missing_auth(self): """ Try making a PR without any auth """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.gh_username = "foo" - psync.gh_repo = "bar" - psync.gh_auth_token = None + psync.gh_repo = "foo/bar" + if "GITHUB_AUTH_TOKEN" in os.environ: + del os.environ["GITHUB_AUTH_TOKEN"] try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: - assert e.args[0] == "No GitHub authentication token set - cannot make PR" + assert e.args[0] == ( + "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" + "Make a PR at the following URL:\n https://github.com/foo/bar/compare/None...TEMPLATE" + ) - def mocked_requests_post(**kwargs): + def mocked_requests_get(**kwargs): """ Helper function to emulate POST requests responses from the web """ class MockResponse: @@ -195,31 +191,76 @@ def __init__(self, data, status_code): self.status_code = status_code self.content = json.dumps(data) - if kwargs["url"] == "https://api.github.com/repos/bad/response/pulls": - return MockResponse({}, 404) + url_template = "https://api.github.com/repos/{}/response/pulls?head=nf-core:TEMPLATE&base=None" + if kwargs["url"] == url_template.format("no_existing_pr"): + response_data = [] + return MockResponse(response_data, 200) + + if kwargs["url"] == url_template.format("existing_pr"): + response_data = [{"url": "url_to_update_pr"}] + return MockResponse(response_data, 200) + + return MockResponse({"get_url": kwargs["url"]}, 404) + + def mocked_requests_patch(**kwargs): + """ Helper function to emulate POST requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if kwargs["url"] == "url_to_update_pr": + response_data = {"html_url": "great_success"} + return MockResponse(response_data, 200) + + return MockResponse({"patch_url": kwargs["url"]}, 404) - if kwargs["url"] == "https://api.github.com/repos/good/response/pulls": + def mocked_requests_post(**kwargs): + """ Helper function to emulate POST requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} return MockResponse(response_data, 201) + return MockResponse({"post_url": kwargs["url"]}, 404) + + @mock.patch("requests.get", side_effect=mocked_requests_get) @mock.patch("requests.post", side_effect=mocked_requests_post) - def test_make_pull_request_bad_response(self, mock_post): - """ Try making a PR without any auth """ + def test_make_pull_request_success(self, mock_get, mock_post): + """ Try making a PR - successful response """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "bad" - psync.gh_repo = "response" - psync.gh_auth_token = "test" + psync.gh_username = "no_existing_pr" + psync.gh_repo = "no_existing_pr/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + psync.make_pull_request() + assert psync.gh_pr_returned_data["html_url"] == "great_success" + + @mock.patch("requests.get", side_effect=mocked_requests_get) + @mock.patch("requests.post", side_effect=mocked_requests_post) + def test_make_pull_request_bad_response(self, mock_get, mock_post): + """ Try making a PR and getting a 404 error """ + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.gh_username = "bad_url" + psync.gh_repo = "bad_url/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("GitHub API returned code 404:") - @mock.patch("requests.post", side_effect=mocked_requests_post) - def test_make_pull_request_bad_response(self, mock_post): - """ Try making a PR without any auth """ + @mock.patch("requests.get", side_effect=mocked_requests_get) + @mock.patch("requests.patch", side_effect=mocked_requests_patch) + def test_update_existing_pull_request(self, mock_get, mock_patch): + """ Try discovering a PR and updating it """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "good" - psync.gh_repo = "response" - psync.gh_auth_token = "test" - psync.make_pull_request() - assert psync.gh_pr_returned_data["html_url"] == "great_success" + psync.gh_username = "existing_pr" + psync.gh_repo = "existing_pr/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + assert psync.update_existing_pull_request("title", "body") is True