diff --git a/CHANGELOG.md b/CHANGELOG.md index a96b0db398..923a6c1ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # nf-core/tools: Changelog +## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] + +### Template + +- Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) +- Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) + +### Linting + +- Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) +- Run prettier after dumping a json schema file ([#2124](https://github.com/nf-core/tools/pull/2124)) + +### General + +- Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) +- Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) +- Handle `json.load()` exceptions ([#2134](https://github.com/nf-core/tools/pull/2134)) + ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] - Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) diff --git a/README.md b/README.md index 0ea83c3cca..73d868bb32 100644 --- a/README.md +++ b/README.md @@ -217,11 +217,12 @@ Please refer to the respective documentation for further details to manage packa ### Activate shell completions for nf-core/tools Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files. -shell | shell config file | command ---- | --- | --- -bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` -Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` -fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` + +| shell | shell config file | command | +| ----- | ----------------------------------------- | -------------------------------------------------- | +| bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | +| zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index a8d56411ba..86c42c0373 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -173,3 +173,5 @@ def get_username(author): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + return author diff --git a/nf_core/create.py b/nf_core/create.py index 3a5f1a502b..045c35d1b4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -166,16 +166,17 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version - config_yml = nf_core.utils.load_tools_config() + _, config_yml = nf_core.utils.load_tools_config() if ( "lint" in config_yml and "nextflow_config" in config_yml["lint"] and "manifest.name" in config_yml["lint"]["nextflow_config"] ): return param_dict, skip_paths - # Check that the pipeline name matches the requirements - if not re.match(r"^[a-z]+$", param_dict["short_name"]): - raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") + if param_dict["prefix"] == "nf-core": + # Check that the pipeline name matches the requirements + if not re.match(r"^[a-z]+$", param_dict["short_name"]): + raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") return param_dict, skip_paths diff --git a/nf_core/launch.py b/nf_core/launch.py index d03e112c3e..87150172f7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,6 +17,7 @@ import nf_core.schema import nf_core.utils +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -700,9 +701,7 @@ def build_command(self): # Write the user selection to a file and run nextflow with that if self.use_params_file: - with open(self.params_out, "w") as fp: - json.dump(self.schema_obj.input_params, fp, indent=4) - fp.write("\n") + dump_json_with_prettier(self.params_out, self.schema_obj.input_params) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 0cf98a5b39..79bce3e7f1 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -146,6 +146,7 @@ def nextflow_config(self): "params.singleEnd", "params.igenomesIgnore", "params.name", + "params.enable_conda", ] # Remove field that should be ignored according to the linting config diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index f8ca563b00..c2fd75d375 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,3 +1,4 @@ +import json import logging import subprocess from pathlib import Path @@ -82,3 +83,14 @@ def run_prettier_on_file(file): "There was an error running the prettier pre-commit hook.\n" f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" ) + + +def dump_json_with_prettier(file_name, file_content): + """Dump a JSON file and run prettier on it. + Args: + file_name (Path | str): A file identifier as a string or pathlib.Path. + file_content (dict): Content to dump into the JSON file + """ + with open(file_name, "w") as fh: + json.dump(file_content, fh, indent=4) + run_prettier_on_file(file_name) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6cfd5af2d5..e8f043f083 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -23,7 +23,7 @@ process {{ tool_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - conda (params.enable_conda ? "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" : null) + conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b4899750d7..019a77c71f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -123,7 +123,7 @@ def create(self): self._get_bioconda_tool() # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) self._get_module_structure_components() diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c4f4bf9bc8..d44fe90f1e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,6 +271,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ( + "container_links", + "Too many double quotes found when specifying singularity container", + self.main_nf, + ) + ) if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -282,13 +291,26 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) + ) # lint double quotes if l.startswith("container"): - container_section = l + lines[i + 1] + lines[i + 2] - if container_section.count('"') > 2: + if l.count('"') > 2: self.failed.append( ("container_links", "Too many double quotes found when specifying containers", self.main_nf) ) + # lint more than one container in the same line + if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): + self.warned.append( + ( + "container_links", + "Docker and Singularity containers specified in the same line. Only first one checked.", + self.main_nf, + ) + ) # Try to connect to container URLs if url is None: continue diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cda7c827dc..2d95afd10c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -13,7 +13,7 @@ import nf_core.utils from nf_core.components.components_utils import get_components_to_install -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier from nf_core.modules.modules_repo import ( NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, @@ -618,7 +618,11 @@ def load(self): """ try: with open(self.modules_json_path, "r") as fh: - self.modules_json = json.load(fh) + try: + self.modules_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.modules_json_path}' due to error {e}") + except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") @@ -1023,10 +1027,7 @@ def dump(self): """ # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) - fh.write("\n") - run_prettier_on_file(self.modules_json_path) + dump_json_with_prettier(self.modules_json_path, self.modules_json) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 4c586a0929..f3dc3e50fe 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 585b933f1c..09f8c423e5 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 714e659de9..05db418b2d 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -76,7 +76,7 @@ class WorkflowMain { NfcoreTemplate.checkConfigProvided(workflow, log) // Check that conda channels are set-up correctly - if (params.enable_conda) { + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { Utils.checkCondaChannels(log) } diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8618bacab6..08116ecbac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/local/samplesheet_check.nf b/nf_core/pipeline-template/modules/local/samplesheet_check.nf index 03a50c1040..5d25800775 100644 --- a/nf_core/pipeline-template/modules/local/samplesheet_check.nf +++ b/nf_core/pipeline-template/modules/local/samplesheet_check.nf @@ -2,7 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : 'quay.io/biocontainers/python:3.8.3' }" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e0589..3df21765b9 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,7 +2,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index 787bdb7b1b..e55b8d43a9 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,5 +1,9 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import platform from textwrap import dedent @@ -7,6 +11,7 @@ def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -45,47 +50,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 05730368b2..9ae5838158 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -2,7 +2,7 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'quay.io/biocontainers/fastqc:0.11.9--0' }" @@ -20,30 +20,22 @@ process FASTQC { script: def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ stub: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index a8159a57bf..68f66bea74 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,7 +1,7 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2a7df54878..5cbb076b46 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,7 +41,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false {% if nf_core_configs %} // Config options @@ -84,7 +83,6 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false @@ -93,7 +91,6 @@ profiles { charliecloud.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 72c4e163a9..2743562d6c 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -263,12 +263,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile conda' instead of providing this parameter.", - "hidden": true, - "fa_icon": "fas fa-bacon" } } } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 2b54789101..9bcc0086b5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -82,7 +82,7 @@ workflow {{ short_name|upper }} { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // diff --git a/nf_core/schema.py b/nf_core/schema.py index d566f2b519..a50a1ed789 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,6 +16,7 @@ import nf_core.list import nf_core.utils +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -170,9 +171,7 @@ def save_schema(self, suppress_logging=False): num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) if not suppress_logging: log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") - with open(self.schema_filename, "w") as fh: - json.dump(self.schema, fh, indent=4) - fh.write("\n") + dump_json_with_prettier(self.schema_filename, self.schema) def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) @@ -183,7 +182,10 @@ def load_input_params(self, params_path): # First, try to load as JSON try: with open(params_path, "r") as fh: - params = json.load(fh) + try: + params = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{params_path}' due to error {e}") self.input_params.update(params) log.debug(f"Loaded JSON input params: {params_path}") except Exception as json_e: diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index d5f9a5efbb..e61f0c6c8d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -88,7 +88,7 @@ def create(self): ) # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) # Create subworkflow template with jinja2 nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) diff --git a/nf_core/sync.py b/nf_core/sync.py index 4d0e1e88ba..332b524cbb 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -252,8 +252,9 @@ def make_template_pipeline(self): plain=True, ).init_pipeline() except Exception as err: - # If sync fails, remove template_yaml_path before raising error. - os.remove(self.template_yaml_path) + if self.template_yaml_path: + # If sync fails, remove template_yaml_path before raising error. + os.remove(self.template_yaml_path) # Reset to where you were to prevent git getting messed up. self.repo.git.reset("--hard") raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") diff --git a/nf_core/utils.py b/nf_core/utils.py index b2431b3337..b60f61fff0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -248,7 +248,10 @@ def fetch_wf_config(wf_path, cache_config=True): if os.path.isfile(cache_path): log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: - config = json.load(fh) + try: + config = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{cache_path}' due to error {e}") return config log.debug("No config cache found") diff --git a/setup.py b/setup.py index 046f5ddf76..cc83e90eed 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7.1" +version = "2.7.2" with open("README.md") as f: readme = f.read() diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 67ee44f973..63ee4e743d 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -18,7 +18,10 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" mod_json_path = os.path.join(self.pipeline_dir, "modules.json") with open(mod_json_path, "r") as fh: - mod_json_sb = json.load(fh) + try: + mod_json_sb = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() @@ -212,7 +215,10 @@ def test_mod_json_dump(self): # Check that the dump function writes the correct content with open(mod_json_path, "r") as f: - mod_json_new = json.load(f) + try: + mod_json_new = json.load(f) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") assert mod_json == mod_json_new diff --git a/tests/modules/update.py b/tests/modules/update.py index 19b53717fd..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -303,6 +303,12 @@ def test_update_only_show_differences(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + tmpdir = tempfile.mkdtemp() shutil.rmtree(tmpdir) shutil.copytree(Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME), tmpdir) @@ -329,6 +335,12 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + # Modify fastqc module, it will have a patch which will be applied during update # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") diff --git a/tests/test_launch.py b/tests/test_launch.py index de597127a2..d830311ba3 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -327,7 +327,11 @@ def test_build_command_params(self): ) # Check saved parameters file with open(self.nf_params_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}") + assert saved_json == {"input": "custom_input"} def test_build_command_params_cl(self): diff --git a/tests/test_lint.py b/tests/test_lint.py index 8160de9fbc..e4e93bd1f4 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -134,7 +134,10 @@ def test_json_output(self, tmp_dir): # Load created JSON file and check its contents with open(json_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{json_fn}' due to error {e}") assert saved_json["num_tests_pass"] > 0 assert saved_json["num_tests_warned"] > 0 assert saved_json["num_tests_ignored"] == 0