From 8a717da53914d3fc404d526a55b8d4d076b19cd4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:02:20 +0200 Subject: [PATCH 01/13] bump to version 1.11dev --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a2aefb90..adbd0b1215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # nf-core/tools: Changelog +## v1.11dev + +### Template + +### Linting + +### Other + ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30] ### Pipeline schema diff --git a/setup.py b/setup.py index 180f97f086..824a627373 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.10" +version = "1.11dev" with open("README.md") as f: readme = f.read() From cf9375fa6b7c1b49c8d1df3a136bad8e49817c3f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:16:43 +0200 Subject: [PATCH 02/13] Clean up lost variable name --- nf_core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 5dc109cbb8..c458bafe29 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -100,7 +100,7 @@ def sync(self): self.make_pull_request() except PullRequestException as e: self.reset_target_dir() - raise PullRequestException(pr_exception) + raise PullRequestException(e) self.reset_target_dir() From cb3532a74b925d1c807314f215392290f61906be Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:26:18 +0200 Subject: [PATCH 03/13] Fix bug that crashed the release sync --- nf_core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index c458bafe29..6c16f00226 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -395,7 +395,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): failed_syncs.append(wf.name) else: log.info( - "[green]Sync successful for {}:[/] [blue][link={1}]{1}[/link]".format( + "[green]Sync successful for {0}:[/] [blue][link={1}]{1}[/link]".format( wf.full_name, sync_obj.gh_pr_returned_data.get("html_url") ) ) From 49f009f4fd209e840b93b83054f6c083bc04cd3b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:28:25 +0200 Subject: [PATCH 04/13] Fix whitespace for PR body text --- nf_core/sync.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 6c16f00226..84e87c5fb9 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -289,21 +289,16 @@ def make_pull_request(self): log.info("Submitting a pull request via the GitHub API") - pr_body_text = """ - A new release of the main template in nf-core/tools has just been released. - This automated pull-request attempts to apply the relevant updates to this pipeline. - - Please make sure to merge this pull-request as soon as possible. - Once complete, make a new minor release of your pipeline. - - For instructions on how to merge this PR, please see - [https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs). - - For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), - please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag}). - """.format( - tag=nf_core.__version__ - ) + pr_body_text = ( + "A new release of the main template in nf-core/tools has just been released. " + "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n" + "Please make sure to merge this pull-request as soon as possible. " + "Once complete, make a new minor release of your pipeline. " + "For instructions on how to merge this PR, please see " + "[https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs).\n\n" + "For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), " + "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})." + ).format(tag=nf_core.__version__) pr_content = { "title": "Important! Template update for nf-core/tools v{}".format(nf_core.__version__), From a8784106f5284d5bf7536d4ba49bf076f01a225d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:32:27 +0200 Subject: [PATCH 05/13] Sync - make most log messages debug --- nf_core/sync.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 84e87c5fb9..37fea63fd9 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -140,7 +140,7 @@ def get_wf_config(self): # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: - log.info("Checking out workflow branch '{}'".format(self.from_branch)) + log.debug("Checking out workflow branch '{}'".format(self.from_branch)) self.repo.git.checkout(self.from_branch) except git.exc.GitCommandError: raise SyncException("Branch `{}` not found!".format(self.from_branch)) @@ -173,7 +173,7 @@ def get_wf_config(self): ) # Fetch workflow variables - log.info("Fetching workflow config variables") + log.debug("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) # Check that we have the required variables @@ -201,7 +201,7 @@ def delete_template_branch_files(self): Delete all files in the TEMPLATE branch """ # Delete everything - log.info("Deleting all files in TEMPLATE branch") + log.debug("Deleting all files in TEMPLATE branch") for the_file in os.listdir(self.pipeline_dir): if the_file == ".git": continue @@ -219,7 +219,7 @@ def make_template_pipeline(self): """ Delete all files and make a fresh template using the workflow variables """ - log.info("Making a new template pipeline using pipeline variables") + log.debug("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation if log.getEffectiveLevel() == logging.INFO: @@ -247,7 +247,7 @@ def commit_template_changes(self): self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) self.made_changes = True - log.info("Committed changes to TEMPLATE branch") + log.debug("Committed changes to TEMPLATE branch") except Exception as e: raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) return True @@ -257,7 +257,7 @@ def push_template_branch(self): and try to make a PR. If we don't have the auth token, try to figure out a URL for the PR and print this to the console. """ - log.info("Pushing TEMPLATE branch to remote") + log.debug("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) try: self.repo.git.push() except git.exc.GitCommandError as e: @@ -287,7 +287,7 @@ def make_pull_request(self): ) raise PullRequestException("No GitHub authentication token set - cannot make PR") - log.info("Submitting a pull request via the GitHub API") + log.debug("Submitting a pull request via the GitHub API") pr_body_text = ( "A new release of the main template in nf-core/tools has just been released. " @@ -355,6 +355,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): # Let's do some updating! for wf in wfs.remote_workflows: + log.info("-" * 30) log.info("Syncing {}".format(wf.full_name)) # Make a local working directory From 5419a7daccc4bcfdddcdf03394810b34f73f0859 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 16:41:25 +0200 Subject: [PATCH 06/13] Sync action - global git config --- .github/workflows/sync.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index f8063c53d8..690896b237 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -29,13 +29,11 @@ jobs: wget -qO- get.nextflow.io | bash sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - - name: Configure git - run: | - git config user.email "core@nf-co.re" - git config user.name "nf-core-bot" - - name: Run synchronisation if: github.repository == 'nf-core/tools' env: AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - run: nf-core sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + run: | + git config --global user.email "core@nf-co.re" + git config --global user.name "nf-core-bot" + nf-core sync --all --username nf-core-bot --auth-token $AUTH_TOKEN From 141ae8300ae12c924e4ca3274dd3141b24d1aae8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 17:21:32 +0200 Subject: [PATCH 07/13] Better logging - can now print a verbose log to a file --- nf_core/__main__.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ecbccbf6ee..8e0b64e7cd 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -102,15 +102,29 @@ def decorator(f): @click.group(cls=CustomHelpOrder) @click.version_option(nf_core.__version__) @click.option("-v", "--verbose", is_flag=True, default=False, help="Verbose output (print debug statements).") -def nf_core_cli(verbose): - stderr = rich.console.Console(file=sys.stderr) - logging.basicConfig( - level=logging.DEBUG if verbose else logging.INFO, - format="%(message)s", - datefmt=" ", - handlers=[rich.logging.RichHandler(console=stderr, markup=True)], +@click.option("-l", "--log-file", help="Filename to save a verbose log output to.") +def nf_core_cli(verbose, log_file): + + # Set the base logger to output DEBUG + log.setLevel(logging.DEBUG) + + # Set up logs to the console + log.addHandler( + rich.logging.RichHandler( + level=logging.DEBUG if verbose else logging.INFO, + console=rich.console.Console(file=sys.stderr), + show_time=False, + markup=True, + ) ) + # Set up logs to a file if we asked for one + if log_file: + log_fh = logging.FileHandler(log_file, encoding="utf-8") + log_fh.setLevel(logging.DEBUG) + log_fh.setFormatter(logging.Formatter("[%(asctime)s] %(name)-20s [%(levelname)-7s] %(message)s")) + log.addHandler(log_fh) + # nf-core list @nf_core_cli.command(help_priority=1) From 0c7075dfceff4089af1105281c89f7c1babee9ba Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 17:28:18 +0200 Subject: [PATCH 08/13] GitHub actions - upload sync log file artifact --- .github/workflows/sync.yml | 8 +++++++- CHANGELOG.md | 11 ++++++----- nf_core/__main__.py | 4 ++-- setup.py | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 690896b237..256a33570d 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -36,4 +36,10 @@ jobs: run: | git config --global user.email "core@nf-co.re" git config --global user.name "nf-core-bot" - nf-core sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + + - name: Upload sync log file artifact + uses: actions/upload-artifact@v2 + with: + name: sync-log-file + path: sync_log.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index adbd0b1215..f4d8d69258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # nf-core/tools: Changelog -## v1.11dev +## [v1.10.1](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30] -### Template - -### Linting +Patch release to fix the automatic template synchronisation, which failed in the v1.10 release. -### Other +* Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk. +* GitHub actions sync now uploads verbose log as an artifact. +* Sync - fixed several minor bugs, improved logging. +* Python Rich library updated to `>=4.2.1` ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8e0b64e7cd..98b5e50696 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -101,8 +101,8 @@ def decorator(f): @click.group(cls=CustomHelpOrder) @click.version_option(nf_core.__version__) -@click.option("-v", "--verbose", is_flag=True, default=False, help="Verbose output (print debug statements).") -@click.option("-l", "--log-file", help="Filename to save a verbose log output to.") +@click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.") +@click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="") def nf_core_cli(verbose, log_file): # Set the base logger to output DEBUG diff --git a/setup.py b/setup.py index 824a627373..6155104034 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "pyyaml", "requests", "requests_cache", - "rich>=4.0.0", + "rich>=4.2.1", "tabulate", ], setup_requires=["twine>=1.11.0", "setuptools>=38.6."], From 50c4e727dd8ba8f0528c2cc16e2690b550b75140 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 20:45:33 +0200 Subject: [PATCH 09/13] Init the root logger in __main__.py instead of named --- nf_core/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 98b5e50696..a46e1a03fe 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -24,7 +24,9 @@ import nf_core.sync import nf_core.utils -log = logging.getLogger(__name__) +# Set up logging as the root logger +# Submodules should all traverse back to this +log = logging.getLogger() def run_nf_core(): From 61f7496756317a20f3b4f2a384cd6b420c467006 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 20:49:12 +0200 Subject: [PATCH 10/13] Lint - add cli flag to show passed This is instead of taking the verbose log level. The results don't go through the logger, so this makes more sense. --- nf_core/__main__.py | 5 +++-- nf_core/lint.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a46e1a03fe..a52b18a101 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -290,9 +290,10 @@ def create(name, description, author, new_version, no_git, force, outdir): and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools", help="Execute additional checks for release-ready workflows.", ) +@click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line.") @click.option("--markdown", type=str, metavar="", help="File to write linting results to (Markdown)") @click.option("--json", type=str, metavar="", help="File to write linting results to (JSON)") -def lint(pipeline_dir, release, markdown, json): +def lint(pipeline_dir, release, show_passed, markdown, json): """ Check pipeline code against nf-core guidelines. @@ -302,7 +303,7 @@ def lint(pipeline_dir, release, markdown, json): """ # Run the lint tests! - lint_obj = nf_core.lint.run_linting(pipeline_dir, release, markdown, json) + lint_obj = nf_core.lint.run_linting(pipeline_dir, release, show_passed, markdown, json) if len(lint_obj.failed) > 0: sys.exit(1) diff --git a/nf_core/lint.py b/nf_core/lint.py index 9a77c991a2..fe3dca9e07 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -39,7 +39,7 @@ logging.getLogger("urllib3").setLevel(logging.WARNING) -def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None): +def run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None): """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object of type :class:`PipelineLint` after finished. @@ -65,7 +65,7 @@ def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None): return lint_obj # Print the results - lint_obj.print_results() + lint_obj.print_results(show_passed) # Save results to Markdown file if md_fn is not None: @@ -1273,7 +1273,9 @@ def check_schema_params(self): if len(removed_params) == 0 and len(added_params) == 0: self.passed.append((15, "Schema matched params returned from nextflow config")) - def print_results(self): + def print_results(self, show_passed=False): + + log.debug("Printing final results") console = Console() # Helper function to format test links nicely @@ -1294,7 +1296,7 @@ def _s(some_list): return "" # Table of passed tests - if len(self.passed) > 0 and log.getEffectiveLevel() == logging.DEBUG: + if len(self.passed) > 0 and show_passed: table = Table(style="green", box=rich.box.ROUNDED) table.add_column( "[[\u2714]] {} Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True, From 3dd46ac18ce1e265eaf83e5cfc1b962b3d9dbd27 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 20:58:28 +0200 Subject: [PATCH 11/13] Drop getEffectiveLevel() calls --- nf_core/lint.py | 4 ++-- nf_core/sync.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index fe3dca9e07..34f1ac5c2f 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -238,6 +238,7 @@ def lint_pipeline(self, release_mode=False): ) for fun_name in check_functions: progress.update(lint_progress, advance=1, func_name=fun_name) + log.debug("Running lint test: {}".format(fun_name)) getattr(self, fun_name)() if len(self.failed) > 0: log.error("Found test failures in `{}`, halting lint run.".format(fun_name)) @@ -1225,8 +1226,7 @@ def check_schema_lint(self): """ Lint the pipeline schema """ # Only show error messages from schema - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.schema").setLevel(logging.ERROR) + logging.getLogger("nf_core.schema").setLevel(logging.ERROR) # Lint the schema self.schema_obj = nf_core.schema.PipelineSchema() diff --git a/nf_core/sync.py b/nf_core/sync.py index 37fea63fd9..7e455b054f 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -222,8 +222,7 @@ def make_template_pipeline(self): log.debug("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.create").setLevel(logging.ERROR) + logging.getLogger("nf_core.create").setLevel(logging.ERROR) nf_core.create.PipelineCreate( name=self.wf_config["manifest.name"].strip('"').strip("'"), @@ -369,8 +368,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): assert repo # Only show error messages from pipeline creation - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.create").setLevel(logging.ERROR) + logging.getLogger("nf_core.create").setLevel(logging.ERROR) # Sync the repo log.debug("Running template sync") From d67d824346853e1a978b9a750130869dc73d70d2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 21:04:22 +0200 Subject: [PATCH 12/13] Log file artifact for create-lint-wf.yml too --- .github/workflows/create-lint-wf.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index ce6a3672fe..dc19a3c7f4 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -27,11 +27,17 @@ jobs: - name: Run nf-core/tools run: | - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline - nf-core list - nf-core licences nf-core-testpipeline - nf-core sync nf-core-testpipeline/ - nf-core schema build nf-core-testpipeline/ --no-prompts - nf-core bump-version nf-core-testpipeline/ 1.1 - nf-core modules install nf-core-testpipeline/ fastqc + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + nf-core --log-file log.txt lint nf-core-testpipeline + nf-core --log-file log.txt list + nf-core --log-file log.txt licences nf-core-testpipeline + nf-core --log-file log.txt sync nf-core-testpipeline/ + nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts + nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 + nf-core --log-file log.txt modules install nf-core-testpipeline/ fastqc + + - name: Upload log file artifact + uses: actions/upload-artifact@v2 + with: + name: nf-core-log-file + path: log.txt From 765d84e1c739fb8356d4e6713caa5a7c1f341e40 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 21:51:09 +0200 Subject: [PATCH 13/13] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6155104034..778d1a6d03 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.11dev" +version = "1.10.1" with open("README.md") as f: readme = f.read()