From 51587c06811f113d2352f5d941bca1e355b6507c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 14:13:46 +0100 Subject: [PATCH 01/32] bump to v2.8dev --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a96b0db398..7894ed3d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +# v2.8dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] - Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) diff --git a/setup.py b/setup.py index 046f5ddf76..38f36810b7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7.1" +version = "2.8dev" with open("README.md") as f: readme = f.read() From 46317bdcacd3321c23ed43e45345ee2e770d7f20 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 17:52:37 +0100 Subject: [PATCH 02/32] fix github_output syntax --- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 4 ++-- .../pipeline-template/.github/workflows/linting_comment.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 4c586a0929..f3dc3e50fe 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 585b933f1c..09f8c423e5 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 From 78d4a25ee2d48c9177a3155c779a2699aa07d378 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 17:55:42 +0100 Subject: [PATCH 03/32] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7894ed3d65..fd09bbde70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) + ### Linting ### Modules From 0d7925bc1e7e0084b7956a0386b1cf7316d05f44 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:35:19 +0000 Subject: [PATCH 04/32] Return from get_username and make sure it is assigned back to self.author --- nf_core/components/components_create.py | 2 ++ nf_core/modules/create.py | 2 +- nf_core/subworkflows/create.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index a8d56411ba..86c42c0373 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -173,3 +173,5 @@ def get_username(author): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + return author diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b4899750d7..019a77c71f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -123,7 +123,7 @@ def create(self): self._get_bioconda_tool() # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) self._get_module_structure_components() diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index d5f9a5efbb..e61f0c6c8d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -88,7 +88,7 @@ def create(self): ) # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) # Create subworkflow template with jinja2 nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) From 2cd002c218f81368a2789d61c171f148a66b4609 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Dec 2022 16:50:53 +0100 Subject: [PATCH 05/32] allow specifying only one container --- nf_core/modules/lint/main_nf.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c4f4bf9bc8..52aa30f7c3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,6 +271,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ( + "container_links", + "Too many double quotes found when specifying singularity container", + self.main_nf, + ) + ) if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -282,10 +291,14 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) + ) # lint double quotes if l.startswith("container"): - container_section = l + lines[i + 1] + lines[i + 2] - if container_section.count('"') > 2: + if l.count('"') > 2: self.failed.append( ("container_links", "Too many double quotes found when specifying containers", self.main_nf) ) From 96826eb00e832d51d543ddf78d805e7babb13cde Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Dec 2022 16:53:37 +0100 Subject: [PATCH 06/32] modify changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd09bbde70..f5584a9235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Linting +- Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) + ### Modules ### Subworkflows From a8cc7d9a10e01307c89e2f59af1cbba09c03ec69 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 9 Dec 2022 22:21:44 +0000 Subject: [PATCH 07/32] Ensure call to load_tools_config expects tuple return --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 3a5f1a502b..0bad1bec32 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -166,7 +166,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version - config_yml = nf_core.utils.load_tools_config() + _, config_yml = nf_core.utils.load_tools_config() if ( "lint" in config_yml and "nextflow_config" in config_yml["lint"] From a85b8962b94bd5854424cc2d08d9c97ebf1a3982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 12 Dec 2022 13:41:09 +0000 Subject: [PATCH 08/32] check only one container per line --- nf_core/modules/lint/main_nf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 52aa30f7c3..d44fe90f1e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -302,6 +302,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append( ("container_links", "Too many double quotes found when specifying containers", self.main_nf) ) + # lint more than one container in the same line + if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): + self.warned.append( + ( + "container_links", + "Docker and Singularity containers specified in the same line. Only first one checked.", + self.main_nf, + ) + ) # Try to connect to container URLs if url is None: continue From 6a7da79962da64c2e675a3efef012960a3299033 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 16:30:28 +0100 Subject: [PATCH 09/32] only check pipeline name without dashes if the name is provided by prompt --- nf_core/create.py | 7 ++++--- nf_core/sync.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 0bad1bec32..045c35d1b4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -173,9 +173,10 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa and "manifest.name" in config_yml["lint"]["nextflow_config"] ): return param_dict, skip_paths - # Check that the pipeline name matches the requirements - if not re.match(r"^[a-z]+$", param_dict["short_name"]): - raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") + if param_dict["prefix"] == "nf-core": + # Check that the pipeline name matches the requirements + if not re.match(r"^[a-z]+$", param_dict["short_name"]): + raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") return param_dict, skip_paths diff --git a/nf_core/sync.py b/nf_core/sync.py index 4d0e1e88ba..332b524cbb 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -252,8 +252,9 @@ def make_template_pipeline(self): plain=True, ).init_pipeline() except Exception as err: - # If sync fails, remove template_yaml_path before raising error. - os.remove(self.template_yaml_path) + if self.template_yaml_path: + # If sync fails, remove template_yaml_path before raising error. + os.remove(self.template_yaml_path) # Reset to where you were to prevent git getting messed up. self.repo.git.reset("--hard") raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") From f05b06e6f10bc7b971830aedeec2c827c5bd99ba Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:12:25 +0100 Subject: [PATCH 10/32] run prettier after dumping a json schema file --- nf_core/launch.py | 2 ++ nf_core/schema.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nf_core/launch.py b/nf_core/launch.py index d03e112c3e..ae4bb174ac 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,6 +17,7 @@ import nf_core.schema import nf_core.utils +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -703,6 +704,7 @@ def build_command(self): with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) fp.write("\n") + run_prettier_on_file(self.params_out) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags diff --git a/nf_core/schema.py b/nf_core/schema.py index d566f2b519..ee4a275c59 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,6 +16,7 @@ import nf_core.list import nf_core.utils +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -173,6 +174,7 @@ def save_schema(self, suppress_logging=False): with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) fh.write("\n") + run_prettier_on_file(self.schema_filename) def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) From a0f5d568ce01c6f7ac6fa0f3dba9dad42afeab4d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:14:17 +0100 Subject: [PATCH 11/32] modify changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5584a9235..82acae6845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Linting - Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) +- Run prettier after dumping a json schema file ([#2124](https://github.com/nf-core/tools/pull/2124)) ### Modules From 5ccbae835e6b536a4d5490f6904bead1f971395a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:15:36 +0100 Subject: [PATCH 12/32] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5584a9235..ef0cc5e053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ ### General +- Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) + ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] - Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) From e9e039a2679f487a861169caab147c5240c61b3d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 13 Dec 2022 15:55:08 +0100 Subject: [PATCH 13/32] =?UTF-8?q?add=20newline=20before=20table=20?= =?UTF-8?q?=F0=9F=A4=9E=F0=9F=8F=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ea83c3cca..8874837bc8 100644 --- a/README.md +++ b/README.md @@ -217,11 +217,12 @@ Please refer to the respective documentation for further details to manage packa ### Activate shell completions for nf-core/tools Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files. -shell | shell config file | command ---- | --- | --- -bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` -Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` -fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` + +| shell | shell config file | command | +| ----- | --------------------------------------- | -------------------------------------------------- | +| bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | +| Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From e3d987e2afaf76a0edc89c9a0ffb7e8fa970a220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 13 Dec 2022 16:04:08 +0100 Subject: [PATCH 14/32] Update README.md Co-authored-by: Phil Ewels --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8874837bc8..35a06028c9 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ Auto-completion for the `nf-core` command is available for bash, zsh and fish. T | shell | shell config file | command | | ----- | --------------------------------------- | -------------------------------------------------- | -| bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | -| Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | -| fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | +| bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | +| Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From 5338afbb3a6e27f01a4a344eb3c566691a89ae73 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 13 Dec 2022 15:10:23 +0000 Subject: [PATCH 15/32] [automated] Fix code linting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35a06028c9..480b966f92 100644 --- a/README.md +++ b/README.md @@ -218,8 +218,8 @@ Please refer to the respective documentation for further details to manage packa Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files. -| shell | shell config file | command | -| ----- | --------------------------------------- | -------------------------------------------------- | +| shell | shell config file | command | +| ----- | ----------------------------------------- | -------------------------------------------------- | | bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | | Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | | fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | From 6e7f87aae535110a0c29beadbca007418b9cd9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 13 Dec 2022 16:46:31 +0100 Subject: [PATCH 16/32] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 480b966f92..73d868bb32 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ Auto-completion for the `nf-core` command is available for bash, zsh and fish. T | shell | shell config file | command | | ----- | ----------------------------------------- | -------------------------------------------------- | | bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | -| Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | | fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From 47b4252840817a1f2b77e26c816c6151157fd263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 13 Dec 2022 17:31:19 +0100 Subject: [PATCH 17/32] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/launch.py | 1 - nf_core/schema.py | 1 - 2 files changed, 2 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index ae4bb174ac..0facca72dc 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -703,7 +703,6 @@ def build_command(self): if self.use_params_file: with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) - fp.write("\n") run_prettier_on_file(self.params_out) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' diff --git a/nf_core/schema.py b/nf_core/schema.py index ee4a275c59..92cbac852d 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -173,7 +173,6 @@ def save_schema(self, suppress_logging=False): log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) - fh.write("\n") run_prettier_on_file(self.schema_filename) def load_input_params(self, params_path): From 6eb466428600db57fa132ebfe28ffe0a0fe3f97c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Dec 2022 10:41:10 +0100 Subject: [PATCH 18/32] add json.dump into a helper function --- nf_core/launch.py | 6 ++---- nf_core/lint_utils.py | 12 ++++++++++++ nf_core/modules/modules_json.py | 7 ++----- nf_core/schema.py | 6 ++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 0facca72dc..87150172f7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,7 +17,7 @@ import nf_core.schema import nf_core.utils -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -701,9 +701,7 @@ def build_command(self): # Write the user selection to a file and run nextflow with that if self.use_params_file: - with open(self.params_out, "w") as fp: - json.dump(self.schema_obj.input_params, fp, indent=4) - run_prettier_on_file(self.params_out) + dump_json_with_prettier(self.params_out, self.schema_obj.input_params) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index f8ca563b00..c2fd75d375 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,3 +1,4 @@ +import json import logging import subprocess from pathlib import Path @@ -82,3 +83,14 @@ def run_prettier_on_file(file): "There was an error running the prettier pre-commit hook.\n" f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" ) + + +def dump_json_with_prettier(file_name, file_content): + """Dump a JSON file and run prettier on it. + Args: + file_name (Path | str): A file identifier as a string or pathlib.Path. + file_content (dict): Content to dump into the JSON file + """ + with open(file_name, "w") as fh: + json.dump(file_content, fh, indent=4) + run_prettier_on_file(file_name) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cda7c827dc..a1609e5fee 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -13,7 +13,7 @@ import nf_core.utils from nf_core.components.components_utils import get_components_to_install -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier from nf_core.modules.modules_repo import ( NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, @@ -1023,10 +1023,7 @@ def dump(self): """ # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) - fh.write("\n") - run_prettier_on_file(self.modules_json_path) + dump_json_with_prettier(self.modules_json_path, self.modules_json) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/schema.py b/nf_core/schema.py index 92cbac852d..04b17a9045 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,7 +16,7 @@ import nf_core.list import nf_core.utils -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -171,9 +171,7 @@ def save_schema(self, suppress_logging=False): num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) if not suppress_logging: log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") - with open(self.schema_filename, "w") as fh: - json.dump(self.schema, fh, indent=4) - run_prettier_on_file(self.schema_filename) + dump_json_with_prettier(self.schema_filename, self.schema) def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) From 433e5b2a9b532ccff164cf5e189bb32bf70a4a60 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 14:28:35 +0100 Subject: [PATCH 19/32] remove params.enable_conda --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 5 ----- nf_core/pipeline-template/nextflow.config | 3 --- nf_core/pipeline-template/nextflow_schema.json | 6 ------ 3 files changed, 14 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 714e659de9..c9c944b34a 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -75,11 +75,6 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) - // Check that conda channels are set-up correctly - if (params.enable_conda) { - Utils.checkCondaChannels(log) - } - // Check AWS batch settings NfcoreTemplate.awsBatch(workflow, params) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2a7df54878..5cbb076b46 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,7 +41,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false {% if nf_core_configs %} // Config options @@ -84,7 +83,6 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false @@ -93,7 +91,6 @@ profiles { charliecloud.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 72c4e163a9..2743562d6c 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -263,12 +263,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile conda' instead of providing this parameter.", - "hidden": true, - "fa_icon": "fas fa-bacon" } } } From 547076bb0c16261a980e2d3abaacccd013e4e7df Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 14:50:46 +0100 Subject: [PATCH 20/32] add params.enable_conda to deprecated vars --- nf_core/lint/nextflow_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 0cf98a5b39..79bce3e7f1 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -146,6 +146,7 @@ def nextflow_config(self): "params.singleEnd", "params.igenomesIgnore", "params.name", + "params.enable_conda", ] # Remove field that should be ignored according to the linting config From d9be489acc55d40b419cb29670c731be376ba04a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:06:32 +0100 Subject: [PATCH 21/32] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c40fb3a035..28a49b70b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### General - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) +- Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] From 4cf1fdc3dcb59a0b67c35a17c8e47f586fb1cb70 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:25:10 +0100 Subject: [PATCH 22/32] update pipeline template modules --- nf_core/pipeline-template/modules.json | 6 +- .../custom/dumpsoftwareversions/main.nf | 2 +- .../templates/dumpsoftwareversions.py | 99 ++++++++++--------- .../modules/nf-core/fastqc/main.nf | 40 +++----- .../modules/nf-core/multiqc/main.nf | 2 +- 5 files changed, 76 insertions(+), 73 deletions(-) mode change 100644 => 100755 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8618bacab6..08116ecbac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e0589..3df21765b9 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,7 +2,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index 787bdb7b1b..e55b8d43a9 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,5 +1,9 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import platform from textwrap import dedent @@ -7,6 +11,7 @@ def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -45,47 +50,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 05730368b2..9ae5838158 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -2,7 +2,7 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'quay.io/biocontainers/fastqc:0.11.9--0' }" @@ -20,30 +20,22 @@ process FASTQC { script: def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ stub: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index a8159a57bf..68f66bea74 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,7 +1,7 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" From c83965db55cf6a793c3525919b0fff05470af4c7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:49:55 +0100 Subject: [PATCH 23/32] fix test --- tests/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index 19b53717fd..9729ef2975 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -329,6 +329,12 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + # Modify fastqc module, it will have a patch which will be applied during update # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") From 3b2f53d1f6477ddef588ca6bdcc287253b7df626 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 16:29:17 +0100 Subject: [PATCH 24/32] fix test --- tests/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index 9729ef2975..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -303,6 +303,12 @@ def test_update_only_show_differences(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + tmpdir = tempfile.mkdtemp() shutil.rmtree(tmpdir) shutil.copytree(Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME), tmpdir) From 8acc9d67922180486749175c193ccabfcb57c03d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 15 Dec 2022 23:42:46 +0100 Subject: [PATCH 25/32] Revert unique() syntax change. Closes #2132 --- CHANGELOG.md | 1 + nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c40fb3a035..47bf9e8f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) +- Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) ### Linting diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 2b54789101..9bcc0086b5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -82,7 +82,7 @@ workflow {{ short_name|upper }} { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // From 8ce4d17bf3f24fff7175b0842ce5a2dca7b85884 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:49:00 +0100 Subject: [PATCH 26/32] handle json exception --- nf_core/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index b2431b3337..b60f61fff0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -248,7 +248,10 @@ def fetch_wf_config(wf_path, cache_config=True): if os.path.isfile(cache_path): log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: - config = json.load(fh) + try: + config = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{cache_path}' due to error {e}") return config log.debug("No config cache found") From 67c3656b60fd82c80a809198c0c621feb90d88bc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:52:54 +0100 Subject: [PATCH 27/32] handle all json-load exceptions --- nf_core/modules/modules_json.py | 6 +++++- nf_core/schema.py | 10 ++++++++-- tests/modules/modules_json.py | 10 ++++++++-- tests/test_launch.py | 6 +++++- tests/test_lint.py | 5 ++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a1609e5fee..2d95afd10c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -618,7 +618,11 @@ def load(self): """ try: with open(self.modules_json_path, "r") as fh: - self.modules_json = json.load(fh) + try: + self.modules_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.modules_json_path}' due to error {e}") + except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") diff --git a/nf_core/schema.py b/nf_core/schema.py index 04b17a9045..7ea9b972c0 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -105,7 +105,10 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" with open(self.schema_filename, "r") as fh: - self.schema = json.load(fh) + try: + self.schema = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.schema_filename}' due to error {e}") self.schema_defaults = {} self.schema_params = [] log.debug(f"JSON file loaded: {self.schema_filename}") @@ -182,7 +185,10 @@ def load_input_params(self, params_path): # First, try to load as JSON try: with open(params_path, "r") as fh: - params = json.load(fh) + try: + params = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{params_path}' due to error {e}") self.input_params.update(params) log.debug(f"Loaded JSON input params: {params_path}") except Exception as json_e: diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 67ee44f973..63ee4e743d 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -18,7 +18,10 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" mod_json_path = os.path.join(self.pipeline_dir, "modules.json") with open(mod_json_path, "r") as fh: - mod_json_sb = json.load(fh) + try: + mod_json_sb = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() @@ -212,7 +215,10 @@ def test_mod_json_dump(self): # Check that the dump function writes the correct content with open(mod_json_path, "r") as f: - mod_json_new = json.load(f) + try: + mod_json_new = json.load(f) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") assert mod_json == mod_json_new diff --git a/tests/test_launch.py b/tests/test_launch.py index de597127a2..d830311ba3 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -327,7 +327,11 @@ def test_build_command_params(self): ) # Check saved parameters file with open(self.nf_params_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}") + assert saved_json == {"input": "custom_input"} def test_build_command_params_cl(self): diff --git a/tests/test_lint.py b/tests/test_lint.py index 8160de9fbc..e4e93bd1f4 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -134,7 +134,10 @@ def test_json_output(self, tmp_dir): # Load created JSON file and check its contents with open(json_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{json_fn}' due to error {e}") assert saved_json["num_tests_pass"] > 0 assert saved_json["num_tests_warned"] > 0 assert saved_json["num_tests_ignored"] == 0 From 1625014620fd1202219369fa65b9daf6b8a49bad Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:54:36 +0100 Subject: [PATCH 28/32] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6842c3ffaa..ac0f8a6f6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) - Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) +- Handle `json.load()` exceptions ([#2134](https://github.com/nf-core/tools/pull/2134)) ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] From 632b675c15eed7c99fe9061c13be695347cf1c5f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:18:28 +0100 Subject: [PATCH 29/32] remove handling from linting as it's already handled later --- nf_core/schema.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 7ea9b972c0..a50a1ed789 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -105,10 +105,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" with open(self.schema_filename, "r") as fh: - try: - self.schema = json.load(fh) - except json.JSONDecodeError as e: - raise UserWarning(f"Unable to load JSON file '{self.schema_filename}' due to error {e}") + self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = [] log.debug(f"JSON file loaded: {self.schema_filename}") From 0cb35e64dbec3ab94d7207ecad221f4072cade92 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:48:09 +0100 Subject: [PATCH 30/32] check conda channels as it's done in rnaseq --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index c9c944b34a..05db418b2d 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -75,6 +75,11 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) + // Check that conda channels are set-up correctly + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + Utils.checkCondaChannels(log) + } + // Check AWS batch settings NfcoreTemplate.awsBatch(workflow, params) From 81fc96627bdc0af31bc186d2ef6acde7677e97fd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:50:07 +0100 Subject: [PATCH 31/32] remove params.enable_conda from modules template --- nf_core/module-template/modules/main.nf | 2 +- nf_core/pipeline-template/modules/local/samplesheet_check.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6cfd5af2d5..e8f043f083 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -23,7 +23,7 @@ process {{ tool_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - conda (params.enable_conda ? "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" : null) + conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" diff --git a/nf_core/pipeline-template/modules/local/samplesheet_check.nf b/nf_core/pipeline-template/modules/local/samplesheet_check.nf index 03a50c1040..5d25800775 100644 --- a/nf_core/pipeline-template/modules/local/samplesheet_check.nf +++ b/nf_core/pipeline-template/modules/local/samplesheet_check.nf @@ -2,7 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : 'quay.io/biocontainers/python:3.8.3' }" From 02acf6561aae7a9b40d4bcd5ffb3fdc9da818dd5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 09:45:03 +0100 Subject: [PATCH 32/32] bump to 2.7.2 for patch release --- CHANGELOG.md | 6 +----- setup.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0f8a6f6a..923a6c1ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -# v2.8dev +## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template @@ -12,10 +12,6 @@ - Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) - Run prettier after dumping a json schema file ([#2124](https://github.com/nf-core/tools/pull/2124)) -### Modules - -### Subworkflows - ### General - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) diff --git a/setup.py b/setup.py index 38f36810b7..cc83e90eed 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.8dev" +version = "2.7.2" with open("README.md") as f: readme = f.read()