From af93b2e57a8dcfdb4fe581f524fd76149d010ecc Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 5 Oct 2018 23:57:05 +0200 Subject: [PATCH 01/64] nf-core launch: new subcommand New subcommand to interactively prompt the user to set workflow flags before launching. --- nf_core/launch.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/nf-core | 12 ++++- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 nf_core/launch.py diff --git a/nf_core/launch.py b/nf_core/launch.py new file mode 100644 index 0000000000..323da0ec0f --- /dev/null +++ b/nf_core/launch.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +""" Launch a pipeline, interactively collecting params """ + +from __future__ import print_function + +import click +import logging +import os +import subprocess + +import nf_core.utils + +def launch_pipeline(workflow): + wf = Launch(workflow) + wf.collect_defaults() + wf.prompt_vars() + wf.build_command() + wf.launch_workflow() + +class Launch(object): + """ Class to hold config option to launch a pipeline """ + + def __init__(self, workflow): + """ Initialise the class with empty placeholder vars """ + + # Prepend nf-core/ if it seems sensible + if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): + workflow = "nf-core/{}".format(workflow) + logging.debug("Prepending nf-core/ to workflow") + logging.info("Launching {}\n".format(workflow)) + + self.workflow = workflow + self.nxf_flag_defaults = { + '-name': False, + '-r': False, + '-profile': 'standard', + '-w': os.getenv('NXF_WORK') if os.getenv('NXF_WORK') else 'work', + '-resume': False + } + self.nxf_flag_help = { + '-name': 'Unique name for this nextflow run', + '-r': 'Release / revision to use', + '-profile': 'Config profile to use', + '-w': 'Work directory for intermediate files', + '-resume': 'Resume a previous workflow run' + } + self.nxf_flags = {} + self.param_defaults = {} + self.params = {} + self.nextflow_cmd = "nextflow run {}".format(self.workflow) + + def collect_defaults(self): + """ Collect the default params and values from the workflow """ + config = nf_core.utils.fetch_wf_config(self.workflow) + for key, value in config.items(): + keys = key.split('.') + if keys[0] == 'params' and len(keys) == 2: + self.param_defaults[keys[1]] = value + + def prompt_vars(self): + """ Ask the user if they want to override any default values """ + # Main nextflow flags + logging.info("Main nextflow options:\n") + for flag, f_default in self.nxf_flag_defaults.items(): + f_user = click.prompt(self.nxf_flag_help[flag], default=f_default) + # Only save if we've changed the default + if f_user != f_default: + # Convert string bools to real bools + try: + f_user = f_user.strip('"').strip("'") + if f_user.lower() == 'true': f_user = True + if f_user.lower() == 'false': f_user = False + except AttributeError: + pass + self.nxf_flags[flag] = f_user + + # Pipeline params + logging.info("Pipeline specific parameters:\n") + for param, p_default in self.param_defaults.items(): + if not isinstance(p_default, dict) and not isinstance(p_default, list): + p_user = click.prompt("--{}".format(param), default=p_default) + # Only save if we've changed the default + if p_user != p_default: + # Convert string bools to real bools + try: + p_user = p_user.strip('"').strip("'") + if p_user.lower() == 'true': p_user = True + if p_user.lower() == 'false': p_user = False + except AttributeError: + pass + self.params[param] = p_user + + def build_command(self): + """ Build the nextflow run command based on what we know """ + for flag, val in self.nxf_flags.items(): + # Boolean flags like -resume + if isinstance(val, bool): + if val: + self.nextflow_cmd = "{} {}".format(self.nextflow_cmd, flag) + else: + logging.warn("TODO: Can't set false boolean flags currently.") + # String values + else: + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) + for param, val in self.params.items(): + # Boolean flags like --saveTrimmed + if isinstance(val, bool): + if val: + self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + else: + logging.warn("TODO: Can't set false boolean flags currently.") + # everything else + else: + self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) + + def launch_workflow(self): + """ Launch nextflow if required """ + logging.info("Nextflow command:\n {}\n\n".format(self.nextflow_cmd)) + if click.confirm('Do you want to run this command now?'): + logging.info("Launching!") + subprocess.call(self.nextflow_cmd, shell=True) diff --git a/scripts/nf-core b/scripts/nf-core index 26f30f20bd..5d3fd7a555 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -8,7 +8,7 @@ import sys import os import nf_core -import nf_core.lint, nf_core.list, nf_core.download, nf_core.licences, nf_core.bump_version, nf_core.create +import nf_core.bump_version, nf_core.create, nf_core.download, nf_core.launch, nf_core.licences, nf_core.lint, nf_core.list import logging @@ -114,6 +114,16 @@ def download(pipeline, release, singularity, outdir): dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir) dl.download_workflow() +@nf_core_cli.command() +@click.argument( + 'pipeline', + required = True, + metavar = "" +) +def launch(pipeline): + """ Interactively build a nextflow command and launch a pipeline """ + nf_core.launch.launch_pipeline(pipeline) + @nf_core_cli.command('bump-version') @click.argument( 'pipeline_dir', From 46fc0a736b6496cb7c15a646dc38c58aa53f929e Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Jan 2019 15:10:35 +0100 Subject: [PATCH 02/64] Adds workflow module --- nf_core/workflow/__init__.py | 0 nf_core/workflow/parameters.py | 179 +++++++++++++++++++++++++++++++++ nf_core/workflow/workflow.py | 23 +++++ 3 files changed, 202 insertions(+) create mode 100644 nf_core/workflow/__init__.py create mode 100644 nf_core/workflow/parameters.py create mode 100644 nf_core/workflow/workflow.py diff --git a/nf_core/workflow/__init__.py b/nf_core/workflow/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py new file mode 100644 index 0000000000..2167d3eaab --- /dev/null +++ b/nf_core/workflow/parameters.py @@ -0,0 +1,179 @@ +import copy +import json + + +class Parameters: + """Contains a static factory method + for :class:`Parameter` object creation. + """ + @staticmethod + def create_from_json(parameters_json): + """Creates a list of Parameter objects from + a description in JSON. + + Args: + parameters_json (str): Parameter(s) description in JSON. + + Returns: + list: Parameter objects. + + Raises: + IOError, if the JSON is of unknown schema to this parser. + """ + properties = json.loads(parameters_json) + parameters = [] + try: + for param in properties.get("parameters"): + parameter = Parameter.builder().name(param.get("name")) \ + .label(param.get("label")) \ + .usage(param.get("usage")) \ + .param_type(param.get("type")) \ + .choices(param.get("choices")) \ + .default(param.get("default_value")) \ + .pattern(param.get("pattern")) \ + .arity(param.get("arity")) \ + .build() + parameters.append(parameter) + except Exception as e: + raise IOError(e) + return parameters + + @staticmethod + def as_json(parameters, indent=0): + """Converts a list of Parameter objects into JSON. + + Returns: + list: JSON formatted parameters. + """ + params = {} + for p in parameters: + key = "params.{}".format(p.name) + params[key] = str(p.value) if p.value else p.default_value + return json.dumps(params, indent=indent) + + + +class Parameter(object): + """Holds information about a workflow parameter. + """ + def __init__(self, param_builder): + # Make some checks + + # Put content + self.name = param_builder.p_name + self.label = param_builder.p_label + self.usage = param_builder.p_usage + self.type = param_builder.p_type + self.value = param_builder.p_value + self.choices = copy.deepcopy(param_builder.p_choices) + self.default_value = param_builder.p_default_value + self.pattern = param_builder.p_pattern + self.arity = param_builder.p_arity + + @staticmethod + def builder(): + return ParameterBuilder() + +class ParameterBuilder: + """Parameter builder. + """ + def __init__(self): + self.p_name = "" + self.p_label = "" + self.p_usage = "" + self.p_type = "" + self.p_value = "" + self.p_choices = [] + self.p_default_value = "" + self.p_pattern = "" + self.p_arity = "" + + def name(self, name): + """Sets the parameter name. + + Args: + name (str): Parameter name. + """ + self.p_name = name + return self + + def label(self, label): + """Sets the parameter label. + + Args: + label (str): Parameter label. + """ + self.p_label = label + return self + + def usage(self, usage): + """Sets the parameter usage. + + Args: + usage (str): Parameter usage description. + """ + self.p_usage = usage + return self + + def value(self, value): + """Sets the parameter value. + + Args: + value (str): Parameter value. + """ + self.p_value = value + return self + + def choices(self, choices): + """Sets the parameter value choices. + + Args: + choices (list): Parameter value choices. + """ + self.p_choices = choices + return self + + def param_type(self, param_type): + """Sets the parameter type. + + Args: + param_type (str): Parameter type. + """ + self.p_type = param_type + return self + + def default(self, default): + """Sets the parameter default value. + + Args: + default (str): Parameter default value. + """ + self.p_default_value = default + return self + + def pattern(self, pattern): + """Sets the parameter regex pattern. + + Args: + pattern (str): Parameter regex pattern. + """ + self.p_pattern = pattern + return self + + def arity(self, arity): + """Sets the parameter regex pattern. + + Args: + pattern (str): Parameter regex pattern. + """ + self.p_arity = arity + return self + + def build(self): + """Builds parameter object. + + Returns: + Parameter: Fresh from the factory. + """ + return Parameter(self) + diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py new file mode 100644 index 0000000000..5ed65fc4f9 --- /dev/null +++ b/nf_core/workflow/workflow.py @@ -0,0 +1,23 @@ +from nf_core.parameters import Parameters +from nf_core.parameters import Parameter + +class Workflow(object): + """nf-core workflow object that holds run parameter information. + + Args: + name (str): Workflow name. + parameters_json (str): Workflow parameter data in JSON. + """ + def __init__(self, name, parameters_json): + self.name = name + self.parameters = Parameters.create_from_json(parameters_json) + + def as_params_json(self, indent=0): + """Converts the Parameter list in a workflow readable parameter + JSON file. + + Returns: + str: JSON formatted parameters. + """ + return Parameters.as_json(self.parameters, indent) + From a79cbb08ede5f9419ede75e03d5cf1ad4096687c Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Jan 2019 15:33:32 +0100 Subject: [PATCH 03/64] Adds tests for the new workflow module --- tests/workflow/example.json | 24 ++++++++++++++++++++++++ tests/workflow/test_parameters.py | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/workflow/example.json create mode 100644 tests/workflow/test_parameters.py diff --git a/tests/workflow/example.json b/tests/workflow/example.json new file mode 100644 index 0000000000..d03bc42f79 --- /dev/null +++ b/tests/workflow/example.json @@ -0,0 +1,24 @@ +{ + "parameters": [ + { + "name": "reads", + "label": "WGS single-end fastq file.", + "usage": "Needs to be provided as workflow input data.", + "type": "string", + "render": "file", + "default_value": "path/to/reads.fastq.gz", + "pattern": ".*(\\.fastq$|\\.fastq\\.gz$)", + "required": "True" + }, + { + "name": "norm_factor", + "label": "Normalization factor ", + "usage": "Integer value that will be applied against input reads.", + "type": "integer", + "render": "range", + "choices": ["1", "150"], + "default_value": "1", + "required:": "False" + } + ] +} \ No newline at end of file diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py new file mode 100644 index 0000000000..14ced903ea --- /dev/null +++ b/tests/workflow/test_parameters.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +"""Some tests covering the parameters code. +""" +import json +import os +import pytest +import shutil +import unittest +import nf_core.workflow.parameters as pms + +WD = os.path.dirname(__file__) +PATH_WORKING_EXAMPLE = os.path.join(WD, 'example.json') + +def test_creating_params_from_json(): + """Tests parsing of a parameter json.""" + assert os.path.isfile(PATH_WORKING_EXAMPLE) + with open(PATH_WORKING_EXAMPLE) as fp: + result = pms.Parameters.create_from_json(fp.read()) + assert len(result) == 2 \ No newline at end of file From 0d09ce9214b51e8e39520487999ced793ad6f7d7 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Jan 2019 16:35:12 +0100 Subject: [PATCH 04/64] Tests params dump --- tests/workflow/test_parameters.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py index 14ced903ea..d7b1e14997 100644 --- a/tests/workflow/test_parameters.py +++ b/tests/workflow/test_parameters.py @@ -11,9 +11,28 @@ WD = os.path.dirname(__file__) PATH_WORKING_EXAMPLE = os.path.join(WD, 'example.json') -def test_creating_params_from_json(): - """Tests parsing of a parameter json.""" +@pytest.fixture(scope="class") +def example_json(): assert os.path.isfile(PATH_WORKING_EXAMPLE) with open(PATH_WORKING_EXAMPLE) as fp: - result = pms.Parameters.create_from_json(fp.read()) - assert len(result) == 2 \ No newline at end of file + content = fp.read() + return content + +def test_creating_params_from_json(example_json): + """Tests parsing of a parameter json.""" + result = pms.Parameters.create_from_json(example_json) + assert len(result) == 2 + +def test_params_as_json_dump(example_json): + """Tests the JSON dump that can be consumed by Nextflow.""" + result = pms.Parameters.create_from_json(example_json) + parameter = result[0] + assert parameter.name == "reads" + expected_output = """ + { + "reads": "path/to/reads.fastq.gz" + }""" + parsed_output = json.loads(expected_output) + assert len(parsed_output.keys()) == 1 + assert parameter.name in parsed_output.keys() + assert parameter.default_value == parsed_output[parameter.name] From d3c94c452c63ab4b6188089be2fe3119824bb58d Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Jan 2019 16:45:01 +0100 Subject: [PATCH 05/64] ... Changelog --- CHANGELOG.md | 3 +++ nf_core/workflow/parameters.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3485b74aae..9c38cf3e4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## v1.5dev +#### Launch command +* Launch nf-core pipelines from command-line + #### Documentation * Added nf-core tools API description to assist developers with the classes and functions available. diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 2167d3eaab..e8bef34d56 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -47,8 +47,7 @@ def as_json(parameters, indent=0): """ params = {} for p in parameters: - key = "params.{}".format(p.name) - params[key] = str(p.value) if p.value else p.default_value + params[p.name] = str(p.value) if p.value else p.default_value return json.dumps(params, indent=indent) From 5361dd6af204f28cb01646c91089110a66a92856 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 13:26:18 +0100 Subject: [PATCH 06/64] Tests parameter builder --- tests/workflow/test_parameters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py index d7b1e14997..d4e0c76593 100644 --- a/tests/workflow/test_parameters.py +++ b/tests/workflow/test_parameters.py @@ -36,3 +36,9 @@ def test_params_as_json_dump(example_json): assert len(parsed_output.keys()) == 1 assert parameter.name in parsed_output.keys() assert parameter.default_value == parsed_output[parameter.name] + +def test_parameter_builder(): + """Tests the parameter builder.""" + parameter = pms.Parameter.builder().name("width").default(2).build() + assert parameter.name == "width" + assert parameter.default_value == 2 From e91e4ae7546f40752ab8c7b56244982a7a108137 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 15:15:40 +0100 Subject: [PATCH 07/64] Adds json convertion methods --- nf_core/workflow/parameters.py | 61 ++++++++++++++++++++++++++++++++-- nf_core/workflow/workflow.py | 12 +++++-- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index e8bef34d56..767ff348ab 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -31,6 +31,7 @@ def create_from_json(parameters_json): .choices(param.get("choices")) \ .default(param.get("default_value")) \ .pattern(param.get("pattern")) \ + .render(param.get("render")) \ .arity(param.get("arity")) \ .build() parameters.append(parameter) @@ -39,8 +40,12 @@ def create_from_json(parameters_json): return parameters @staticmethod - def as_json(parameters, indent=0): - """Converts a list of Parameter objects into JSON. + def in_nextflow_json(parameters, indent=0): + """Converts a list of Parameter objects into JSON, readable by Nextflow. + + Args: + parameters (list): List of :class:`Parameter` objects. + indent (integer): String output indentation. Defaults to 0. Returns: list: JSON formatted parameters. @@ -49,7 +54,22 @@ def as_json(parameters, indent=0): for p in parameters: params[p.name] = str(p.value) if p.value else p.default_value return json.dumps(params, indent=indent) + + @staticmethod + def in_full_json(parameters, indent=0): + """Converts a list of Parameter objects into JSON. All attributes + are written. + Args: + parameters (list): List of :class:`Parameter` objects. + indent (integer): String output indentation. Defaults to 0. + + Returns: + list: JSON formatted parameters. + """ + params_dict = {} + params_dict["parameters"] = [p.as_dict() for p in parameters] + return json.dumps(params_dict, indent=indent) class Parameter(object): @@ -68,11 +88,30 @@ def __init__(self, param_builder): self.default_value = param_builder.p_default_value self.pattern = param_builder.p_pattern self.arity = param_builder.p_arity + self.required = param_builder.p_required + self.render = param_builder.p_render @staticmethod def builder(): return ParameterBuilder() + def as_dict(self): + """Describes its attibutes in JSON. + + Args: + indent (integer): String output indentation. Defaults to 0. + + Returns: + str: Parameter object in JSON. + """ + params_dict = {} + for attribute in ['name', 'label', 'usage', 'required', + 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render']: + if getattr(self, attribute): + params_dict[attribute] = getattr(self, attribute) + params_dict['required'] = getattr(self, 'required') + return params_dict + class ParameterBuilder: """Parameter builder. """ @@ -85,7 +124,9 @@ def __init__(self): self.p_choices = [] self.p_default_value = "" self.p_pattern = "" - self.p_arity = "" + self.p_arity = 0 + self.p_render = "" + self.p_required = False def name(self, name): """Sets the parameter name. @@ -167,6 +208,20 @@ def arity(self, arity): """ self.p_arity = arity return self + + def render(self, render): + """Sets the parameter render type. + + Args: + render (str): UI render type. + """ + self.p_render = render + return self + + def required(self, required): + """Sets the required parameter flag.""" + self.p_required = required + return self def build(self): """Builds parameter object. diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index 5ed65fc4f9..3aa3c0f091 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -12,12 +12,20 @@ def __init__(self, name, parameters_json): self.name = name self.parameters = Parameters.create_from_json(parameters_json) - def as_params_json(self, indent=0): + def in_nextflow_json(self, indent=0): """Converts the Parameter list in a workflow readable parameter JSON file. Returns: str: JSON formatted parameters. """ - return Parameters.as_json(self.parameters, indent) + return Parameters.in_nextflow_json(self.parameters, indent) + def in_full_json(self, indent=0): + """Converts the Parameter list in a complete parameter JSON for + schema validation. + + Returns: + str: JSON formatted parameters. + """ + return Parameters.in_full_json(self.parameters, indent) \ No newline at end of file From 54a151760b98f10dab4c919414810bc94c15ce31 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 15:16:05 +0100 Subject: [PATCH 08/64] Adds vaidation test --- tests/workflow/test_parameters.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py index d4e0c76593..e0ed1860b9 100644 --- a/tests/workflow/test_parameters.py +++ b/tests/workflow/test_parameters.py @@ -2,14 +2,24 @@ """Some tests covering the parameters code. """ import json +import jsonschema +from jsonschema import ValidationError import os import pytest +import requests import shutil import unittest import nf_core.workflow.parameters as pms WD = os.path.dirname(__file__) PATH_WORKING_EXAMPLE = os.path.join(WD, 'example.json') +SCHEMA_URI = "https://nf-co.re/parameters.schema.json" + +@pytest.fixture(scope="class") +def schema(): + res = requests.get(SCHEMA_URI) + assert res.status_code == 200 + return res.text @pytest.fixture(scope="class") def example_json(): @@ -42,3 +52,17 @@ def test_parameter_builder(): parameter = pms.Parameter.builder().name("width").default(2).build() assert parameter.name == "width" assert parameter.default_value == 2 + +@pytest.mark.xfail(raises=ValidationError) +def test_validation(schema): + """Tests the parameter objects against the JSON schema.""" + parameter = pms.Parameter.builder().name("width").param_type("unknown").default(2).build() + params_in_json = pms.Parameters.in_full_json([parameter]) + jsonschema.validate(json.loads(pms.Parameters.in_full_json([parameter])), json.loads(schema)) + +def test_validation_with_success(schema): + """Tests the parameter objects against the JSON schema.""" + parameter = pms.Parameter.builder().name("width").param_type("integer") \ + .default(2).label("The width of a table.").render("textfield").required(False).build() + params_in_json = pms.Parameters.in_full_json([parameter]) + jsonschema.validate(json.loads(pms.Parameters.in_full_json([parameter])), json.loads(schema)) From f4cbecae8d42a17ef9639fdcad2071b6ef91272e Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 15:18:35 +0100 Subject: [PATCH 09/64] Refactors apidocs --- nf_core/workflow/parameters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 767ff348ab..4928dd2ce5 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -96,13 +96,13 @@ def builder(): return ParameterBuilder() def as_dict(self): - """Describes its attibutes in JSON. + """Describes its attibutes in a dictionary. Args: indent (integer): String output indentation. Defaults to 0. Returns: - str: Parameter object in JSON. + dict: Parameter object as key value pairs. """ params_dict = {} for attribute in ['name', 'label', 'usage', 'required', From 63db8c4da26c54aa59610f28ad0f6876f9a1c7ec Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 15:19:14 +0100 Subject: [PATCH 10/64] Removes args docs --- nf_core/workflow/parameters.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 4928dd2ce5..479a92fb18 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -98,9 +98,6 @@ def builder(): def as_dict(self): """Describes its attibutes in a dictionary. - Args: - indent (integer): String output indentation. Defaults to 0. - Returns: dict: Parameter object as key value pairs. """ From baeb5f8a6e8eb2b97fb538b98a9a9dd636984508 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Mon, 14 Jan 2019 15:29:53 +0100 Subject: [PATCH 11/64] Includes jsonschema package --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d56913c9a..93f40905b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,8 @@ install: - cd ${TRAVIS_BUILD_DIR} - pip install --upgrade pip # Get rid of dependency resolve issues from older pip versions - pip install . - - pip install codecov 'pytest==3.6.4' pytest-datafiles pytest-cov mock - - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation + - pip install codecov 'pytest==3.6.4' pytest-datafiles pytest-cov mock jsonschema + - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation script: - python -m pytest --cov=nf_core . From ff97393a878b32ec621f952dfe0d49288dc18f34 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 14:57:59 +0100 Subject: [PATCH 12/64] Implements basic validation logic --- nf_core/workflow/parameters.py | 4 +++ nf_core/workflow/validation.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 nf_core/workflow/validation.py diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 479a92fb18..a623b5381d 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -108,6 +108,10 @@ def as_dict(self): params_dict[attribute] = getattr(self, attribute) params_dict['required'] = getattr(self, 'required') return params_dict + + def validate(self): + Validator.check(self) + class ParameterBuilder: """Parameter builder. diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py new file mode 100644 index 0000000000..6c50a76a17 --- /dev/null +++ b/nf_core/workflow/validation.py @@ -0,0 +1,59 @@ +import abc +from nf_core.workflow.parameters import Parameter + +class Validators(object): + """ + """ + def __init__(self): + pass + + @staticmethod + def get_validator_for_param(parameter): + """Returns a validator object for a given parameter. + """ + if parameter.type == "integer": + return IntegerValidator(parameter) + raise LookupError("Cannot find a matching validator for type '{}'." + .format(parameter.type)) + + +class Validator(abc.ABC): + """Abstract base class for different parameter validators. + """ + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def __init__(self, parameter): + if not isinstance(parameter, Parameter): + raise (AttributeError("Argument must be of class {}" + .format(parameters.Parameter.__class__.__module__))) + self._param = parameter + + @abc.abstractmethod + def validate(self): + raise ValueError + + +class IntegerValidator(Validator): + """Implementation for parameters of type integer.""" + + def __init__(self, params): + super(IntegerValidator, self).__init__(params) + + def validate(self): + value = int(self._param.value) + choices = sorted([int(x) for x in self._param.choices]) + if not choices: + return + if len(choices) < 2: + raise AttributeError("The property 'choices' must have at least two entries.") + if not value >= choices[0] and value <= choices[-1]: + raise AttributeError("The value for parameter '{}' must be within range of [{},{}]" + .format(self.__param.name, choices[0], choices[-1])) + + + + + + + From 5f8b271e93bd7a752d4f47dd13fa41334c278213 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 14:58:41 +0100 Subject: [PATCH 13/64] Adds tests for validator class --- tests/workflow/test_validator.py | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/workflow/test_validator.py diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py new file mode 100644 index 0000000000..d946f7077b --- /dev/null +++ b/tests/workflow/test_validator.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +"""Some tests covering the parameters code. +""" +import os +import pytest +import requests +import shutil +import unittest +import nf_core.workflow.parameters as pms +import nf_core.workflow.validation as valid + + +WD = os.path.dirname(__file__) +PATH_WORKING_EXAMPLE = os.path.join(WD, 'example.json') +SCHEMA_URI = "https://nf-co.re/parameters.schema.json" + +@pytest.fixture(scope="class") +def valid_integer_param(): + param = pms.Parameter.builder().name("Fake Integer Param") \ + .default("0").value("10").choices(["0", "10"]).param_type("integer").build() + return param + +@pytest.fixture(scope="class") +def invalid_integer_param(): + param = pms.Parameter.builder().name("Fake Integer Param") \ + .default("0").value("20").choices(["0", "10"]).param_type("integer").build() + return param + +def test_simple_integer_validation(valid_integer_param): + validator = valid.Validators.get_validator_for_param(valid_integer_param) + validator.validate() + +@pytest.mark.xfail(raises=AttributeError) +def test_simple_integer_out_of_range(invalid_integer_param): + validator = valid.Validators.get_validator_for_param(invalid_integer_param) + validator.validate() \ No newline at end of file From 41ed743eec4dba35e131a027c68e7a74d9a761b1 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:25:14 +0100 Subject: [PATCH 14/64] Adds docstrings --- nf_core/workflow/validation.py | 35 ++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 6c50a76a17..cc7a8723e4 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -2,14 +2,23 @@ from nf_core.workflow.parameters import Parameter class Validators(object): - """ + """Gives access to a factory method for objects of instance + :class:`Validator` which returns the correct Validator for a + given parameter type. """ def __init__(self): pass @staticmethod def get_validator_for_param(parameter): - """Returns a validator object for a given parameter. + """Determines matching :class:`Validator` instance for a given parameter. + + Returns: + Validator: Matching validator for a given :class:`Parameter`. + + Raises: + LookupError: In case no matching validator for a given parameter type + can be determined. """ if parameter.type == "integer": return IntegerValidator(parameter) @@ -35,12 +44,26 @@ def validate(self): class IntegerValidator(Validator): - """Implementation for parameters of type integer.""" + """Implementation for parameters of type integer. + + Args: + parameter (:class:`Parameter`): A Parameter object. + + Raises: + AttributeError: In case the argument is not of instance :class:`Parameter`. - def __init__(self, params): - super(IntegerValidator, self).__init__(params) + """ + + def __init__(self, parameter): + super(IntegerValidator, self).__init__(parameter) def validate(self): + """Validates an parameter integer value against a given range (choices). + If the value is valid, no error is risen. + + Raises: + AtrributeError: Description of the value error. + """ value = int(self._param.value) choices = sorted([int(x) for x in self._param.choices]) if not choices: @@ -49,7 +72,7 @@ def validate(self): raise AttributeError("The property 'choices' must have at least two entries.") if not value >= choices[0] and value <= choices[-1]: raise AttributeError("The value for parameter '{}' must be within range of [{},{}]" - .format(self.__param.name, choices[0], choices[-1])) + .format(self._param.name, choices[0], choices[-1])) From 21da990d87473ec6f93bbb1cf4a3d7ef54e5d3bf Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:26:06 +0100 Subject: [PATCH 15/64] Refactors AttributeError message --- nf_core/workflow/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index cc7a8723e4..e6871fe3b2 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -35,7 +35,7 @@ class Validator(abc.ABC): def __init__(self, parameter): if not isinstance(parameter, Parameter): raise (AttributeError("Argument must be of class {}" - .format(parameters.Parameter.__class__.__module__))) + .format(Parameter.__class__))) self._param = parameter @abc.abstractmethod From 252b29fede6025eb74a1f98c68b64620031174a8 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:27:29 +0100 Subject: [PATCH 16/64] Adds apidoc for workflow module --- docs/api/_src/index.rst | 1 + docs/api/_src/workflow.rst | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 docs/api/_src/workflow.rst diff --git a/docs/api/_src/index.rst b/docs/api/_src/index.rst index 431056a0a2..35ee300346 100644 --- a/docs/api/_src/index.rst +++ b/docs/api/_src/index.rst @@ -17,6 +17,7 @@ Welcome to nf-core tools API documentation! lint list utils + workflow Indices and tables diff --git a/docs/api/_src/workflow.rst b/docs/api/_src/workflow.rst new file mode 100644 index 0000000000..7cf9afc711 --- /dev/null +++ b/docs/api/_src/workflow.rst @@ -0,0 +1,5 @@ +nf_core.list +============ + +.. automodule:: nf_core.workflow + :members: \ No newline at end of file From af0426646fee2a0accfdf9ba94d46bce99f3cbbd Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:34:29 +0100 Subject: [PATCH 17/64] Ensures backward compatibility for abc --- nf_core/workflow/validation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index e6871fe3b2..1421f8465f 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,6 +1,12 @@ import abc +import sys from nf_core.workflow.parameters import Parameter +if sys.version_info >= (3, 4): + ABC = abc.ABC +else: + ABC = abc.ABCMeta('ABC', (), {}) + class Validators(object): """Gives access to a factory method for objects of instance :class:`Validator` which returns the correct Validator for a @@ -26,7 +32,7 @@ def get_validator_for_param(parameter): .format(parameter.type)) -class Validator(abc.ABC): +class Validator(ABC): """Abstract base class for different parameter validators. """ __metaclass__ = abc.ABCMeta From 8121254e07fbfe53324fef1d7ca2c02ebb72e6b0 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:58:57 +0100 Subject: [PATCH 18/64] Adds StringValidator class --- nf_core/workflow/validation.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 1421f8465f..e7596db01d 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -28,6 +28,8 @@ def get_validator_for_param(parameter): """ if parameter.type == "integer": return IntegerValidator(parameter) + if parameter.type == "string": + return StringValidator(parameter) raise LookupError("Cannot find a matching validator for type '{}'." .format(parameter.type)) @@ -57,7 +59,6 @@ class IntegerValidator(Validator): Raises: AttributeError: In case the argument is not of instance :class:`Parameter`. - """ def __init__(self, parameter): @@ -81,6 +82,35 @@ def validate(self): .format(self._param.name, choices[0], choices[-1])) +class StringValidator(Validator): + """Implementation for parameters of type string. + + Args: + parameter (:class:`Parameter`): A Parameter object. + + Raises: + AttributeError: In case the argument is not of instance :class:`Parameter`. + """ + + def __init__(self, parameter): + super(StringValidator, self).__init__(parameter) + + def validate(self): + """Validates an parameter integer value against a given range (choices). + If the value is valid, no error is risen. + + Raises: + AtrributeError: Description of the value error. + """ + value = str(self._param.value) + choices = sorted([str(x) for x in self._param.choices]) + if not choices: + if not self._param.pattern: + raise AttributeError("Can't validate value for parameter {}," \ + "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) + + + From 054179a3b1a2b90f4f8fba14f7c3160579815803 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 15:59:17 +0100 Subject: [PATCH 19/64] Adds more tests for validator --- tests/workflow/test_validator.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py index d946f7077b..0bb432065a 100644 --- a/tests/workflow/test_validator.py +++ b/tests/workflow/test_validator.py @@ -26,6 +26,18 @@ def invalid_integer_param(): .default("0").value("20").choices(["0", "10"]).param_type("integer").build() return param +@pytest.fixture(scope="class") +def invalid_string_param_without_pattern_and_choices(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("Whatever").choices(["0", "10"]).param_type("integer").build() + return param + +@pytest.fixture(scope="class") +def param_with_unknown_type(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("Whatever").choices(["0", "10"]).param_type("unknown").build() + return param + def test_simple_integer_validation(valid_integer_param): validator = valid.Validators.get_validator_for_param(valid_integer_param) validator.validate() @@ -33,4 +45,14 @@ def test_simple_integer_validation(valid_integer_param): @pytest.mark.xfail(raises=AttributeError) def test_simple_integer_out_of_range(invalid_integer_param): validator = valid.Validators.get_validator_for_param(invalid_integer_param) + validator.validate() + +@pytest.mark.xfail(raises=AttributeError) +def test_string_with_empty_pattern_and_choices(invalid_string_param_without_pattern_and_choices): + validator = valid.Validators.get_validator_for_param(invalid_integer_param) + validator.validate() + +@pytest.mark.xfail(raises=LookupError) +def test_param_with_empty_type(param_with_unknown_type): + validator = valid.Validators.get_validator_for_param(param_with_unknown_type) validator.validate() \ No newline at end of file From 0e9f7d2400776b7ed9c216fec4f5efffd5f8e99d Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 16:29:21 +0100 Subject: [PATCH 20/64] Checks string pattern --- nf_core/workflow/validation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index e7596db01d..6b37c5a125 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,4 +1,5 @@ import abc +import re import sys from nf_core.workflow.parameters import Parameter @@ -108,6 +109,12 @@ def validate(self): if not self._param.pattern: raise AttributeError("Can't validate value for parameter {}," \ "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) + result = re.match(self._param.pattern, self._param.value) + if not result: + raise AttributeError("The value '{}' for parameter {}" \ + " did not match the regex pattern '{}' .".format( + self._param.value, self._param.name, self._param.pattern + )) From b51770a880571a2f252d6ddbf1b070fa0a3bd045 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 16:29:44 +0100 Subject: [PATCH 21/64] Adds tests for StringValidator --- tests/workflow/test_validator.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py index 0bb432065a..f077a2dd11 100644 --- a/tests/workflow/test_validator.py +++ b/tests/workflow/test_validator.py @@ -38,6 +38,20 @@ def param_with_unknown_type(): .default("Not empty!").value("Whatever").choices(["0", "10"]).param_type("unknown").build() return param +@pytest.fixture(scope="class") +def string_param_not_matching_pattern(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("id.123A").choices(["0", "10"])\ + .param_type("string").pattern(r"^id\.[0-9]*$").build() + return param + +@pytest.fixture(scope="class") +def string_param_matching_pattern(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("id.123").choices(["0", "10"])\ + .param_type("string").pattern(r"^id\.[0-9]*$").build() + return param + def test_simple_integer_validation(valid_integer_param): validator = valid.Validators.get_validator_for_param(valid_integer_param) validator.validate() @@ -55,4 +69,13 @@ def test_string_with_empty_pattern_and_choices(invalid_string_param_without_patt @pytest.mark.xfail(raises=LookupError) def test_param_with_empty_type(param_with_unknown_type): validator = valid.Validators.get_validator_for_param(param_with_unknown_type) + validator.validate() + +@pytest.mark.xfail(raises=AttributeError) +def test_string_param_not_matching_pattern(string_param_not_matching_pattern): + validator = valid.Validators.get_validator_for_param(string_param_not_matching_pattern) + validator.validate() + +def test_string_param_matching_pattern(string_param_matching_pattern): + validator = valid.Validators.get_validator_for_param(string_param_matching_pattern) validator.validate() \ No newline at end of file From 203151dfa5533898694f28c5781640c38b434fec Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 17:22:39 +0100 Subject: [PATCH 22/64] Validates string choices --- nf_core/workflow/validation.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 6b37c5a125..9188a57f0c 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -107,14 +107,20 @@ def validate(self): choices = sorted([str(x) for x in self._param.choices]) if not choices: if not self._param.pattern: - raise AttributeError("Can't validate value for parameter {}," \ + raise AttributeError("Can't validate value for parameter '{}'," \ "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) result = re.match(self._param.pattern, self._param.value) if not result: - raise AttributeError("The value '{}' for parameter {}" \ + raise AttributeError("The value '{}' for parameter '{}'" \ " did not match the regex pattern '{}' .".format( self._param.value, self._param.name, self._param.pattern )) + else: + if value not in choices: + raise AttributeError("The value '{}' for parameter '{}'" \ + " was not part of choices '{}'.".format( + value, self._param.name, choices + )) From 66e13a0f260632e01c8ec3e957be2e8ce5025b50 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 15 Jan 2019 17:22:53 +0100 Subject: [PATCH 23/64] Tests string choices validation --- tests/workflow/test_validator.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py index f077a2dd11..24bde72115 100644 --- a/tests/workflow/test_validator.py +++ b/tests/workflow/test_validator.py @@ -41,17 +41,31 @@ def param_with_unknown_type(): @pytest.fixture(scope="class") def string_param_not_matching_pattern(): param = pms.Parameter.builder().name("Fake String Param") \ - .default("Not empty!").value("id.123A").choices(["0", "10"])\ + .default("Not empty!").value("id.123A") \ .param_type("string").pattern(r"^id\.[0-9]*$").build() return param @pytest.fixture(scope="class") def string_param_matching_pattern(): param = pms.Parameter.builder().name("Fake String Param") \ - .default("Not empty!").value("id.123").choices(["0", "10"])\ + .default("Not empty!").value("id.123") \ .param_type("string").pattern(r"^id\.[0-9]*$").build() return param +@pytest.fixture(scope="class") +def string_param_not_matching_choices(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("snail").choices(["horse", "pig"])\ + .param_type("string").build() + return param + +@pytest.fixture(scope="class") +def string_param_matching_choices(): + param = pms.Parameter.builder().name("Fake String Param") \ + .default("Not empty!").value("horse").choices(["horse", "pig"])\ + .param_type("string").build() + return param + def test_simple_integer_validation(valid_integer_param): validator = valid.Validators.get_validator_for_param(valid_integer_param) validator.validate() @@ -78,4 +92,13 @@ def test_string_param_not_matching_pattern(string_param_not_matching_pattern): def test_string_param_matching_pattern(string_param_matching_pattern): validator = valid.Validators.get_validator_for_param(string_param_matching_pattern) + validator.validate() + +@pytest.mark.xfail(raises=AttributeError) +def test_string_param_not_matching_choices(string_param_not_matching_choices): + validator = valid.Validators.get_validator_for_param(string_param_not_matching_choices) + validator.validate() + +def test_string_param_matching_choices(string_param_matching_choices): + validator = valid.Validators.get_validator_for_param(string_param_matching_choices) validator.validate() \ No newline at end of file From e2f81805fe811c36fe890ba4055576e807857fe7 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Wed, 16 Jan 2019 11:28:39 +0100 Subject: [PATCH 24/64] Adds validate method for a parameter --- nf_core/workflow/parameters.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index a623b5381d..a57c6e2930 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -1,5 +1,6 @@ import copy import json +import nf_core.workflow.validation as vld class Parameters: @@ -110,7 +111,16 @@ def as_dict(self): return params_dict def validate(self): - Validator.check(self) + """Validates the parameter's value. If the value is within + the parameter requirements, no exception is thrown. + + Raises: + LookupError: Raised when no matching validator can be determined. + AttributeError: Raised with description, if a parameter value violates + the parameter constrains. + """ + validator = vld.Validators.get_validator_for_param(self) + validator.validate() class ParameterBuilder: From c95f1ebd69f8133cefbfac67240576d5be7c6a0b Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:15:44 +0100 Subject: [PATCH 25/64] Implements usage of pamareter JSON --- nf_core/launch.py | 45 +++++++++++++++++++++++++----- nf_core/workflow/parameters.py | 50 ++++++++++++++++++++++------------ nf_core/workflow/validation.py | 8 +++--- 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 323da0ec0f..c2a2ae14c2 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -9,13 +9,20 @@ import subprocess import nf_core.utils +import nf_core.workflow.parameters as pms -def launch_pipeline(workflow): - wf = Launch(workflow) - wf.collect_defaults() - wf.prompt_vars() - wf.build_command() - wf.launch_workflow() +def launch_pipeline(workflow, params_json): + launcher = Launch(workflow) + params_list = [] + if params_json: + with open(params_json, "r") as fp: + params_json_str=fp.read() + params_list = pms.Parameters.create_from_json(params_json_str) + if not params_json: + launcher.collect_defaults() # Fallback, calls Nextflow's config command + launcher.prompt_vars(params_list) + launcher.build_command(params_list) + launcher.launch_workflow() class Launch(object): """ Class to hold config option to launch a pipeline """ @@ -57,7 +64,7 @@ def collect_defaults(self): if keys[0] == 'params' and len(keys) == 2: self.param_defaults[keys[1]] = value - def prompt_vars(self): + def prompt_vars(self, params = None): """ Ask the user if they want to override any default values """ # Main nextflow flags logging.info("Main nextflow options:\n") @@ -76,6 +83,9 @@ def prompt_vars(self): # Pipeline params logging.info("Pipeline specific parameters:\n") + if params: + Launch.__prompt_defaults_from_param_objects(params) + return for param, p_default in self.param_defaults.items(): if not isinstance(p_default, dict) and not isinstance(p_default, list): p_user = click.prompt("--{}".format(param), default=p_default) @@ -89,6 +99,27 @@ def prompt_vars(self): except AttributeError: pass self.params[param] = p_user + + @classmethod + def __prompt_defaults_from_param_objects(cls, params): + for parameter in params: + value_is_valid = False + while(not value_is_valid): + desired_param_value = click.prompt("'--{name}'\n" + "\tUsage: {usage}\n" + "\tRange/Choices: {choices}.\n" + .format(name=parameter.name, + usage=parameter.usage, + choices=parameter.choices), + default=parameter.default_value) + parameter.value = desired_param_value + try: + parameter.validate() + except Exception as e: + print(e) + continue + else: + value_is_valid = True def build_command(self): """ Build the nextflow run command based on what we know """ diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index a57c6e2930..e5c01f1fa6 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -1,43 +1,50 @@ import copy import json +import requests +import requests_cache +from jsonschema import validate +from jsonschema.exceptions import ValidationError import nf_core.workflow.validation as vld +NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" class Parameters: """Contains a static factory method for :class:`Parameter` object creation. """ @staticmethod - def create_from_json(parameters_json): + def create_from_json(parameters_json, schema = ""): """Creates a list of Parameter objects from a description in JSON. Args: parameters_json (str): Parameter(s) description in JSON. + schema (str): Parameter schema in JSON. Returns: list: Parameter objects. Raises: - IOError, if the JSON is of unknown schema to this parser. + ValidationError: When the parameter JSON violates the schema. + LookupError: When the schema cannot be downloaded. """ + if not schema: + schema = Parameters.__download_schema_from_nf_core(NFCORE_PARAMS_SCHEMA_URI) + validate(json.loads(parameters_json), json.loads(schema)) # Throws a ValidationError when schema is violated properties = json.loads(parameters_json) parameters = [] - try: - for param in properties.get("parameters"): - parameter = Parameter.builder().name(param.get("name")) \ - .label(param.get("label")) \ - .usage(param.get("usage")) \ - .param_type(param.get("type")) \ - .choices(param.get("choices")) \ - .default(param.get("default_value")) \ - .pattern(param.get("pattern")) \ - .render(param.get("render")) \ - .arity(param.get("arity")) \ - .build() - parameters.append(parameter) - except Exception as e: - raise IOError(e) + for param in properties.get("parameters"): + parameter = Parameter.builder().name(param.get("name")) \ + .label(param.get("label")) \ + .usage(param.get("usage")) \ + .param_type(param.get("type")) \ + .choices(param.get("choices")) \ + .default(param.get("default_value")) \ + .pattern(param.get("pattern")) \ + .render(param.get("render")) \ + .arity(param.get("arity")) \ + .build() + parameters.append(parameter) return parameters @staticmethod @@ -71,6 +78,15 @@ def in_full_json(parameters, indent=0): params_dict = {} params_dict["parameters"] = [p.as_dict() for p in parameters] return json.dumps(params_dict, indent=indent) + + @classmethod + def __download_schema_from_nf_core(cls, url): + with requests_cache.disabled(): + result = requests.get(url, headers={'Cache-Control': 'no-cache'}) + if not result.status_code == 200: + raise LookupError("Could not fetch schema from {url}.\n{e}".format( + url, result.text)) + return result.text class Parameter(object): diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 9188a57f0c..24b3cc8d48 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,7 @@ import abc import re import sys -from nf_core.workflow.parameters import Parameter +import nf_core.workflow.parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC @@ -42,9 +42,9 @@ class Validator(ABC): @abc.abstractmethod def __init__(self, parameter): - if not isinstance(parameter, Parameter): + if not isinstance(parameter, pms.Parameter): raise (AttributeError("Argument must be of class {}" - .format(Parameter.__class__))) + .format(pms.Parameter.__class__))) self._param = parameter @abc.abstractmethod @@ -104,7 +104,7 @@ def validate(self): AtrributeError: Description of the value error. """ value = str(self._param.value) - choices = sorted([str(x) for x in self._param.choices]) + choices = sorted([str(x) for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: raise AttributeError("Can't validate value for parameter '{}'," \ From 2911737891d2a07c637193cd7112ed0c50d56b59 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:16:31 +0100 Subject: [PATCH 26/64] Provides option for JSON parameter --- scripts/nf-core | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/nf-core b/scripts/nf-core index 41317baaf4..d7063dcebe 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -18,6 +18,7 @@ import logging @click.option( '-v', '--verbose', is_flag = True, + default = False, help = "Verbose output (print debug statements)" ) def nf_core_cli(verbose): @@ -121,9 +122,16 @@ def download(pipeline, release, singularity, outdir): required = True, metavar = "" ) -def launch(pipeline): +@click.option( + '-p', '--params', + is_flag = False, + default = "", + type = str, + help = "Parameter file in JSON." +) +def launch(pipeline, params): """ Interactively build a nextflow command and launch a pipeline """ - nf_core.launch.launch_pipeline(pipeline) + nf_core.launch.launch_pipeline(pipeline, params) @nf_core_cli.command('bump-version') @click.argument( From 2ef4c68efe2a7e7c08791da81f40ddef7009b9f2 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:16:58 +0100 Subject: [PATCH 27/64] Extends example JSON --- tests/workflow/example.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/workflow/example.json b/tests/workflow/example.json index d03bc42f79..655ea5c36f 100644 --- a/tests/workflow/example.json +++ b/tests/workflow/example.json @@ -7,8 +7,7 @@ "type": "string", "render": "file", "default_value": "path/to/reads.fastq.gz", - "pattern": ".*(\\.fastq$|\\.fastq\\.gz$)", - "required": "True" + "pattern": ".*(\\.fastq$|\\.fastq\\.gz$)" }, { "name": "norm_factor", @@ -17,8 +16,7 @@ "type": "integer", "render": "range", "choices": ["1", "150"], - "default_value": "1", - "required:": "False" + "default_value": "1" } ] } \ No newline at end of file From 78215417c9070ae608c442d63e1649f584412003 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:17:21 +0100 Subject: [PATCH 28/64] Adds jsonschema as dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bf60879f6f..be7d8ba27c 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'cookiecutter', 'click', 'GitPython', + 'jsonschema', 'pyyaml', 'requests', 'requests_cache', From dcddde5a4598991288818c5e0a87e0890e1eea48 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:34:13 +0100 Subject: [PATCH 29/64] Provides direct mode, which will not ask for params command line input --- nf_core/launch.py | 11 ++++++++--- scripts/nf-core | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index c2a2ae14c2..f5cde1b1ad 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -11,7 +11,7 @@ import nf_core.utils import nf_core.workflow.parameters as pms -def launch_pipeline(workflow, params_json): +def launch_pipeline(workflow, params_json, direct): launcher = Launch(workflow) params_list = [] if params_json: @@ -20,7 +20,7 @@ def launch_pipeline(workflow, params_json): params_list = pms.Parameters.create_from_json(params_json_str) if not params_json: launcher.collect_defaults() # Fallback, calls Nextflow's config command - launcher.prompt_vars(params_list) + launcher.prompt_vars(params_list, direct) launcher.build_command(params_list) launcher.launch_workflow() @@ -64,7 +64,7 @@ def collect_defaults(self): if keys[0] == 'params' and len(keys) == 2: self.param_defaults[keys[1]] = value - def prompt_vars(self, params = None): + def prompt_vars(self, params = None, direct = False): """ Ask the user if they want to override any default values """ # Main nextflow flags logging.info("Main nextflow options:\n") @@ -80,6 +80,11 @@ def prompt_vars(self, params = None): except AttributeError: pass self.nxf_flags[flag] = f_user + + # Uses the parameter values from the JSON file + # and does not ask the user to set them explicitly + if direct: + return # Pipeline params logging.info("Pipeline specific parameters:\n") diff --git a/scripts/nf-core b/scripts/nf-core index d7063dcebe..d921718f8f 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -129,9 +129,15 @@ def download(pipeline, release, singularity, outdir): type = str, help = "Parameter file in JSON." ) -def launch(pipeline, params): +@click.option( + '--direct', + is_flag = True, + default = False, + help = "Uses the values from the parameter file directly." +) +def launch(pipeline, params, direct): """ Interactively build a nextflow command and launch a pipeline """ - nf_core.launch.launch_pipeline(pipeline, params) + nf_core.launch.launch_pipeline(pipeline, params, direct) @nf_core_cli.command('bump-version') @click.argument( From 58d04865b8aff04ff6ea2688bb6e04c87ee224ab Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:39:09 +0100 Subject: [PATCH 30/64] Remove init file from subdir --- nf_core/workflow/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 nf_core/workflow/__init__.py diff --git a/nf_core/workflow/__init__.py b/nf_core/workflow/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From d2bec743fdff2b36c1d3384e27f3c42ec4731310 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:42:11 +0100 Subject: [PATCH 31/64] Revert "Remove init file from subdir" This reverts commit 58d04865b8aff04ff6ea2688bb6e04c87ee224ab. --- nf_core/workflow/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 nf_core/workflow/__init__.py diff --git a/nf_core/workflow/__init__.py b/nf_core/workflow/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 4a96a3eb8574af5c996c74dfa931b754df4d976c Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 14:48:08 +0100 Subject: [PATCH 32/64] Refactors annoying import --- nf_core/workflow/parameters.py | 2 +- nf_core/workflow/validation.py | 2 +- tests/workflow/test_parameters.py | 2 +- tests/workflow/test_validator.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index e5c01f1fa6..1e11ae3a6b 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -4,7 +4,7 @@ import requests_cache from jsonschema import validate from jsonschema.exceptions import ValidationError -import nf_core.workflow.validation as vld +from nf_core.workflow import validation as vld NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 24b3cc8d48..03a4a2fe73 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,7 @@ import abc import re import sys -import nf_core.workflow.parameters as pms +from nf_core.workflow import parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py index e0ed1860b9..27ba0a3336 100644 --- a/tests/workflow/test_parameters.py +++ b/tests/workflow/test_parameters.py @@ -9,7 +9,7 @@ import requests import shutil import unittest -import nf_core.workflow.parameters as pms +from nf_core.workflow import parameters as pms WD = os.path.dirname(__file__) PATH_WORKING_EXAMPLE = os.path.join(WD, 'example.json') diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py index 24bde72115..30663cae89 100644 --- a/tests/workflow/test_validator.py +++ b/tests/workflow/test_validator.py @@ -6,8 +6,8 @@ import requests import shutil import unittest -import nf_core.workflow.parameters as pms -import nf_core.workflow.validation as valid +from nf_core.workflow import parameters as pms +from nf_core.workflow import validation as valid WD = os.path.dirname(__file__) From e94fd0411b559ceed5d216e95e91d5f2c301db4f Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:23:14 +0100 Subject: [PATCH 33/64] Builds command with parameter file --- nf_core/launch.py | 28 ++++++++++++++++++++++++++-- scripts/nf-core | 2 +- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index f5cde1b1ad..3c8e8fcab0 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -9,7 +9,7 @@ import subprocess import nf_core.utils -import nf_core.workflow.parameters as pms +from nf_core.workflow import parameters as pms def launch_pipeline(workflow, params_json, direct): launcher = Launch(workflow) @@ -126,7 +126,7 @@ def __prompt_defaults_from_param_objects(cls, params): else: value_is_valid = True - def build_command(self): + def build_command(self, params = None): """ Build the nextflow run command based on what we know """ for flag, val in self.nxf_flags.items(): # Boolean flags like -resume @@ -138,6 +138,13 @@ def build_command(self): # String values else: self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) + + if params: # When a parameter specification file was used, we can run Nextflow with it + path = Launch.__create_nfx_params_file(params) + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) + Launch.__write_params_as_full_json(params) + return + for param, val in self.params.items(): # Boolean flags like --saveTrimmed if isinstance(val, bool): @@ -149,6 +156,23 @@ def build_command(self): else: self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) + @classmethod + def __create_nfx_params_file(cls, params): + working_dir = os.getcwd() + output_file = os.path.join(working_dir, "nfx-params.json") + json_string = pms.Parameters.in_nextflow_json(params, indent=4) + with open(output_file, "w") as fp: + fp.write(json_string) + return output_file + + @classmethod + def __write_params_as_full_json(cls, params, outdir = os.getcwd()): + output_file = os.path.join(outdir, "full-params.json") + json_string = pms.Parameters.in_full_json(params, indent=4) + with open(output_file, "w") as fp: + fp.write(json_string) + return output_file + def launch_workflow(self): """ Launch nextflow if required """ logging.info("Nextflow command:\n {}\n\n".format(self.nextflow_cmd)) diff --git a/scripts/nf-core b/scripts/nf-core index d921718f8f..92e3d49a5b 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -133,7 +133,7 @@ def download(pipeline, release, singularity, outdir): '--direct', is_flag = True, default = False, - help = "Uses the values from the parameter file directly." + help = "Uses given values from the parameter file directly." ) def launch(pipeline, params, direct): """ Interactively build a nextflow command and launch a pipeline """ From f5b40bb9becd2d21fbb4f1c157f223e3a926dce3 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:24:03 +0100 Subject: [PATCH 34/64] Removes explicit type casting, as this destroys the purpose of the validation. Either the values are of the proper type, or not. --- nf_core/workflow/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 03a4a2fe73..166fe7d19a 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -73,7 +73,7 @@ def validate(self): AtrributeError: Description of the value error. """ value = int(self._param.value) - choices = sorted([int(x) for x in self._param.choices]) + choices = sorted([x for x in self._param.choices]) if not choices: return if len(choices) < 2: @@ -104,7 +104,7 @@ def validate(self): AtrributeError: Description of the value error. """ value = str(self._param.value) - choices = sorted([str(x) for x in self._param.choices]) if self._param.choices else [] + choices = sorted([x for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: raise AttributeError("Can't validate value for parameter '{}'," \ From 04d275e7fdeda384fe49976f677e094a95e64512 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:32:36 +0100 Subject: [PATCH 35/64] Checks instance for a set value --- nf_core/workflow/validation.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 166fe7d19a..516b01a2fb 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -72,7 +72,10 @@ def validate(self): Raises: AtrributeError: Description of the value error. """ - value = int(self._param.value) + value = self._param.value + if not isinstance(value, int): + raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) if not choices: return @@ -103,7 +106,10 @@ def validate(self): Raises: AtrributeError: Description of the value error. """ - value = str(self._param.value) + value = self._param.value + if not isinstance(value, str): + raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: @@ -121,11 +127,3 @@ def validate(self): " was not part of choices '{}'.".format( value, self._param.name, choices )) - - - - - - - - From 3891852ee1bda07162014e3542c24a9881b89872 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:43:34 +0100 Subject: [PATCH 36/64] Removes explicit type cast --- nf_core/workflow/parameters.py | 2 +- tests/workflow/example.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 1e11ae3a6b..278d0d7ccf 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -60,7 +60,7 @@ def in_nextflow_json(parameters, indent=0): """ params = {} for p in parameters: - params[p.name] = str(p.value) if p.value else p.default_value + params[p.name] = p.value if p.value else p.default_value return json.dumps(params, indent=indent) @staticmethod diff --git a/tests/workflow/example.json b/tests/workflow/example.json index 655ea5c36f..7f6d5c1486 100644 --- a/tests/workflow/example.json +++ b/tests/workflow/example.json @@ -15,8 +15,8 @@ "usage": "Integer value that will be applied against input reads.", "type": "integer", "render": "range", - "choices": ["1", "150"], - "default_value": "1" + "choices": [1, 150], + "default_value": 1 } ] } \ No newline at end of file From 6d9509cdc3268543594a0f14ae062d8fecccc977 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:47:05 +0100 Subject: [PATCH 37/64] Corrects Integer validation --- nf_core/workflow/validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 516b01a2fb..256f9125b3 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -77,11 +77,12 @@ def validate(self): raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) + print(choices) if not choices: return if len(choices) < 2: raise AttributeError("The property 'choices' must have at least two entries.") - if not value >= choices[0] and value <= choices[-1]: + if not (value >= choices[0] and value <= choices[-1]): raise AttributeError("The value for parameter '{}' must be within range of [{},{}]" .format(self._param.name, choices[0], choices[-1])) From 3d6e8a9b8570895ee80017f3b39fd08a89e9ef03 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 17 Jan 2019 17:48:27 +0100 Subject: [PATCH 38/64] Corrects validation test --- tests/workflow/test_validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflow/test_validator.py b/tests/workflow/test_validator.py index 30663cae89..f7c2e6cefa 100644 --- a/tests/workflow/test_validator.py +++ b/tests/workflow/test_validator.py @@ -17,13 +17,13 @@ @pytest.fixture(scope="class") def valid_integer_param(): param = pms.Parameter.builder().name("Fake Integer Param") \ - .default("0").value("10").choices(["0", "10"]).param_type("integer").build() + .default(0).value(10).choices([0, 10]).param_type("integer").build() return param @pytest.fixture(scope="class") def invalid_integer_param(): param = pms.Parameter.builder().name("Fake Integer Param") \ - .default("0").value("20").choices(["0", "10"]).param_type("integer").build() + .default(0).value(20).choices([0, 10]).param_type("integer").build() return param @pytest.fixture(scope="class") From 86c7a95fdf7df0d5ddc85645c23dcdabc9580b1d Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 22 Jan 2019 15:15:50 +0100 Subject: [PATCH 39/64] Reads params settings from Github --- nf_core/launch.py | 15 ++++++++++----- nf_core/utils.py | 25 ++++++++++++++++++++++++ nf_core/workflow/validation.py | 35 +++++++++++++++++++++++++++++++--- scripts/nf-core | 2 +- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 3c8e8fcab0..cc9bf816bb 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -11,14 +11,19 @@ import nf_core.utils from nf_core.workflow import parameters as pms -def launch_pipeline(workflow, params_json, direct): +def launch_pipeline(workflow, params_local_uri, direct): launcher = Launch(workflow) params_list = [] - if params_json: - with open(params_json, "r") as fp: - params_json_str=fp.read() + try: + if params_local_uri: + with open(params_local_uri, 'r') as fp: params_json_str = fp.read() + else: + params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow) params_list = pms.Parameters.create_from_json(params_json_str) - if not params_json: + except LookupError as e: + print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( + pipeline=workflow, exception=e)) + if not params_list: launcher.collect_defaults() # Fallback, calls Nextflow's config command launcher.prompt_vars(params_list, direct) launcher.build_command(params_list) diff --git a/nf_core/utils.py b/nf_core/utils.py index 315ef2a5db..28616b0f0f 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,9 +5,34 @@ import datetime import os +import requests +import requests_cache import subprocess import tempfile +PARAMETERS_URI_TEMPL = "https://raw.githubusercontent.com/nf-core/{pipeline}/master/parameters.settings.json" + + +def fetch_parameter_settings_from_github(pipeline): + """Requests the pipeline parameter settings from Github. + + Args: + pipeline (str): The nf-core pipeline name. + + Returns: + str: The raw JSON string of the file with the parameter settings. + + Raises: + LookupError: If for some reason the URI cannot be accessed. + """ + target_uri = PARAMETERS_URI_TEMPL.format(pipeline=pipeline) + try: + with requests_cache.disabled(): + result = requests.get(target_uri, headers={'Cache-Control': 'no-cache'}) + except (ConnectionError, TimeoutError) as e: + raise LookupError(e) + return result.text + def fetch_wf_config(wf_path): """Uses Nextflow to retrieve the the configuration variables diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 256f9125b3..fe9cf273bb 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -29,8 +29,10 @@ def get_validator_for_param(parameter): """ if parameter.type == "integer": return IntegerValidator(parameter) - if parameter.type == "string": + elif parameter.type == "string": return StringValidator(parameter) + elif parameter.type == "boolean": + return BooleanValidator(parameter) raise LookupError("Cannot find a matching validator for type '{}'." .format(parameter.type)) @@ -59,7 +61,7 @@ class IntegerValidator(Validator): parameter (:class:`Parameter`): A Parameter object. Raises: - AttributeError: In case the argument is not of instance :class:`Parameter`. + AttributeError: In case the argument is not of instance integer. """ def __init__(self, parameter): @@ -94,7 +96,7 @@ class StringValidator(Validator): parameter (:class:`Parameter`): A Parameter object. Raises: - AttributeError: In case the argument is not of instance :class:`Parameter`. + AttributeError: In case the argument is not of instance string. """ def __init__(self, parameter): @@ -128,3 +130,30 @@ def validate(self): " was not part of choices '{}'.".format( value, self._param.name, choices )) + + +class BooleanValidator(Validator): + """Implementation for parameters of type boolean. + + Args: + parameter (:class:`Parameter`): A Parameter object. + + Raises: + AttributeError: In case the argument is not of instance boolean. + """ + + def __init__(self, parameter): + super(BooleanValidator, self).__init__(parameter) + + def validate(self): + """Validates an parameter boolean value. + If the value is valid, no error is risen. + + Raises: + AtrributeError: Description of the value error. + """ + value = self._param.value + if not isinstance(self._param.value, bool): + raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" + .format(value, self._param.name, type(value))) + diff --git a/scripts/nf-core b/scripts/nf-core index 92e3d49a5b..b34ac7741c 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -127,7 +127,7 @@ def download(pipeline, release, singularity, outdir): is_flag = False, default = "", type = str, - help = "Parameter file in JSON." + help = "Local parameter settings file in JSON." ) @click.option( '--direct', From 2616b3ecbd0a8c9cebd50aee909745c4f844426e Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 22 Jan 2019 15:17:11 +0100 Subject: [PATCH 40/64] Import request_cache only if needed --- nf_core/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 28616b0f0f..7c7cdabab0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,8 +5,7 @@ import datetime import os -import requests -import requests_cache +import requests import subprocess import tempfile @@ -25,6 +24,8 @@ def fetch_parameter_settings_from_github(pipeline): Raises: LookupError: If for some reason the URI cannot be accessed. """ + import requests_cache + target_uri = PARAMETERS_URI_TEMPL.format(pipeline=pipeline) try: with requests_cache.disabled(): From d13af25b3b46b27986a510e611ae1cd5b2d10851 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 22 Jan 2019 15:39:34 +0100 Subject: [PATCH 41/64] Adds boolean and decimal validator --- nf_core/launch.py | 6 ++++++ nf_core/workflow/validation.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index cc9bf816bb..9de8223c94 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -122,6 +122,12 @@ def __prompt_defaults_from_param_objects(cls, params): usage=parameter.usage, choices=parameter.choices), default=parameter.default_value) + if parameter.type == "integer": + desired_param_value = int(desired_param_value) + elif parameter.type == "boolean": + desired_param_value = bool(desired_param_value) + elif parameter.type == "decimal": + desired_param_value = float(desired_param_value) parameter.value = desired_param_value try: parameter.validate() diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index fe9cf273bb..28efa7b599 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -32,7 +32,9 @@ def get_validator_for_param(parameter): elif parameter.type == "string": return StringValidator(parameter) elif parameter.type == "boolean": - return BooleanValidator(parameter) + return BooleanValidator(parameter) + elif parameter.type == "decimal": + return DecimalValidator(parameter) raise LookupError("Cannot find a matching validator for type '{}'." .format(parameter.type)) @@ -157,3 +159,28 @@ def validate(self): raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" .format(value, self._param.name, type(value))) + +class DecimalValidator(Validator): + """Implementation for parameters of type boolean. + + Args: + parameter (:class:`Parameter`): A Parameter object. + + Raises: + AttributeError: In case the argument is not of instance decimal. + """ + + def __init__(self, parameter): + super(DecimalValidator, self).__init__(parameter) + + def validate(self): + """Validates an parameter boolean value. + If the value is valid, no error is risen. + + Raises: + AtrributeError: Description of the value error. + """ + value = self._param.value + if not isinstance(self._param.value, float): + raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" + .format(value, self._param.name, type(value))) From 442ed872431b213fe391743e75e34b964d4a090c Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Wed, 23 Jan 2019 18:03:04 +0100 Subject: [PATCH 42/64] Adds group attribute to parameter object --- nf_core/workflow/parameters.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 278d0d7ccf..c3f91d463c 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -107,6 +107,7 @@ def __init__(self, param_builder): self.arity = param_builder.p_arity self.required = param_builder.p_required self.render = param_builder.p_render + self.group = param_builder.p_group @staticmethod def builder(): @@ -120,7 +121,7 @@ def as_dict(self): """ params_dict = {} for attribute in ['name', 'label', 'usage', 'required', - 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render']: + 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render', 'group']: if getattr(self, attribute): params_dict[attribute] = getattr(self, attribute) params_dict['required'] = getattr(self, 'required') @@ -154,6 +155,16 @@ def __init__(self): self.p_arity = 0 self.p_render = "" self.p_required = False + self.p_group = "" + + def group(self, group): + """Sets the parameter group tag + + Args: + group (str): Parameter group tag. + """ + self.p_group = group + return self def name(self, name): """Sets the parameter name. From eb43cdcd91d7612596e57a9b8376e21f1ac14a16 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 24 Jan 2019 13:01:41 +0100 Subject: [PATCH 43/64] Adds group property and tests --- nf_core/workflow/parameters.py | 1 + tests/workflow/example.json | 16 ++++++++++++++-- tests/workflow/test_parameters.py | 8 +++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index c3f91d463c..b4a3adc161 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -43,6 +43,7 @@ def create_from_json(parameters_json, schema = ""): .pattern(param.get("pattern")) \ .render(param.get("render")) \ .arity(param.get("arity")) \ + .group(param.get("group")) \ .build() parameters.append(parameter) return parameters diff --git a/tests/workflow/example.json b/tests/workflow/example.json index 7f6d5c1486..2a6b587158 100644 --- a/tests/workflow/example.json +++ b/tests/workflow/example.json @@ -7,7 +7,18 @@ "type": "string", "render": "file", "default_value": "path/to/reads.fastq.gz", - "pattern": ".*(\\.fastq$|\\.fastq\\.gz$)" + "pattern": ".*(\\.fastq$|\\.fastq\\.gz$)", + "group": "inputdata" + }, + { + "name": "index", + "label": "Mapper index", + "usage": "Needs to be provided for the mapping.", + "type": "string", + "render": "file", + "default_value": "path/to/index", + "pattern": ".*", + "group": "inputdata" }, { "name": "norm_factor", @@ -16,7 +27,8 @@ "type": "integer", "render": "range", "choices": [1, 150], - "default_value": 1 + "default_value": 1, + "group": "normalization" } ] } \ No newline at end of file diff --git a/tests/workflow/test_parameters.py b/tests/workflow/test_parameters.py index 27ba0a3336..d374c1256d 100644 --- a/tests/workflow/test_parameters.py +++ b/tests/workflow/test_parameters.py @@ -31,7 +31,13 @@ def example_json(): def test_creating_params_from_json(example_json): """Tests parsing of a parameter json.""" result = pms.Parameters.create_from_json(example_json) - assert len(result) == 2 + assert len(result) == 3 + +def test_groups_from_json(example_json): + """Tests group property of a parameter json.""" + result = pms.Parameters.create_from_json(example_json) + group_labels = set([ param.group for param in result ]) + assert len(group_labels) == 2 def test_params_as_json_dump(example_json): """Tests the JSON dump that can be consumed by Nextflow.""" From 9959bb619c6c6cdb473286f6ac0c5bff177321c8 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 24 Jan 2019 13:11:24 +0100 Subject: [PATCH 44/64] Provides several input option for boolean --- nf_core/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 9de8223c94..d730b2feaa 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -125,7 +125,7 @@ def __prompt_defaults_from_param_objects(cls, params): if parameter.type == "integer": desired_param_value = int(desired_param_value) elif parameter.type == "boolean": - desired_param_value = bool(desired_param_value) + desired_param_value = (str(desired_param_value).lower() in ['yes', 'y', 'true', 't', '1']) elif parameter.type == "decimal": desired_param_value = float(desired_param_value) parameter.value = desired_param_value From 2e8ade4a70653b7fe5fc63d9ebfe03215021355a Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 24 Jan 2019 13:54:32 +0100 Subject: [PATCH 45/64] Prompts for usage of default values --- nf_core/launch.py | 94 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index d730b2feaa..d5a8ed442c 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -94,7 +94,9 @@ def prompt_vars(self, params = None, direct = False): # Pipeline params logging.info("Pipeline specific parameters:\n") if params: - Launch.__prompt_defaults_from_param_objects(params) + Launch.__prompt_defaults_from_param_objects( + Launch.__group_parameters(params) + ) return for param, p_default in self.param_defaults.items(): if not isinstance(p_default, dict) and not isinstance(p_default, list): @@ -109,33 +111,71 @@ def prompt_vars(self, params = None, direct = False): except AttributeError: pass self.params[param] = p_user - + @classmethod - def __prompt_defaults_from_param_objects(cls, params): - for parameter in params: - value_is_valid = False - while(not value_is_valid): - desired_param_value = click.prompt("'--{name}'\n" - "\tUsage: {usage}\n" - "\tRange/Choices: {choices}.\n" - .format(name=parameter.name, - usage=parameter.usage, - choices=parameter.choices), - default=parameter.default_value) - if parameter.type == "integer": - desired_param_value = int(desired_param_value) - elif parameter.type == "boolean": - desired_param_value = (str(desired_param_value).lower() in ['yes', 'y', 'true', 't', '1']) - elif parameter.type == "decimal": - desired_param_value = float(desired_param_value) - parameter.value = desired_param_value - try: - parameter.validate() - except Exception as e: - print(e) - continue - else: - value_is_valid = True + def __group_parameters(cls, parameters): + """Groups parameters by their 'group' property. + + Args: + parameters (list): Collection of parameter objects. + + Returns: + dict: Parameter objects grouped by the `group` property. + """ + grouped_parameters = {} + for param in parameters: + if not grouped_parameters.get(param.group): + grouped_parameters[param.group] = [] + grouped_parameters[param.group].append(param) + return grouped_parameters + + @classmethod + def __prompt_defaults_from_param_objects(cls, params_grouped): + """Prompts the user for parameter input values and validates them. + + Args: + params_grouped (dict): A dictionary with parameter group labels + as keys and list of parameters as values. :: + + { + "group1": [param1, param2], + ... + } + """ + for group_label, params in params_grouped.items(): + print("{prespace}Parameter group: \'{label}\'{postspace}".format( + prespace='-'*5, + label=group_label, + postspace='-'*10) + ) + use_defaults = click.confirm("Do you want to use the groups defaults?", + default=True) + if use_defaults: + continue + for parameter in params: + value_is_valid = False + while(not value_is_valid): + desired_param_value = click.prompt("'--{name}'\n" + "\tUsage: {usage}\n" + "\tRange/Choices: {choices}.\n" + .format(name=parameter.name, + usage=parameter.usage, + choices=parameter.choices), + default=parameter.default_value) + if parameter.type == "integer": + desired_param_value = int(desired_param_value) + elif parameter.type == "boolean": + desired_param_value = (str(desired_param_value).lower() in ['yes', 'y', 'true', 't', '1']) + elif parameter.type == "decimal": + desired_param_value = float(desired_param_value) + parameter.value = desired_param_value + try: + parameter.validate() + except Exception as e: + print(e) + continue + else: + value_is_valid = True def build_command(self, params = None): """ Build the nextflow run command based on what we know """ From da62d5c4881c353a7dc81b75ffad94dfee77f342 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 24 Jan 2019 13:55:48 +0100 Subject: [PATCH 46/64] Corrects typo --- nf_core/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index d5a8ed442c..f3bf2cf2df 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -148,7 +148,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): label=group_label, postspace='-'*10) ) - use_defaults = click.confirm("Do you want to use the groups defaults?", + use_defaults = click.confirm("Do you want to use the group's defaults?", default=True) if use_defaults: continue From 16f3ee9251cb611db85517f2a9698d430972388c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 24 Jan 2019 19:28:40 +0100 Subject: [PATCH 47/64] Made nf-core launch prompts beautiful --- nf_core/launch.py | 129 +++++++++++++++++++++++---------- nf_core/workflow/parameters.py | 61 ++++++++-------- nf_core/workflow/validation.py | 46 ++++++------ 3 files changed, 145 insertions(+), 91 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index f3bf2cf2df..f527b75a0d 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -16,7 +16,7 @@ def launch_pipeline(workflow, params_local_uri, direct): params_list = [] try: if params_local_uri: - with open(params_local_uri, 'r') as fp: params_json_str = fp.read() + with open(params_local_uri, 'r') as fp: params_json_str = fp.read() else: params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow) params_list = pms.Parameters.create_from_json(params_json_str) @@ -43,10 +43,10 @@ def __init__(self, workflow): self.workflow = workflow self.nxf_flag_defaults = { - '-name': False, - '-r': False, + '-name': None, + '-r': None, '-profile': 'standard', - '-w': os.getenv('NXF_WORK') if os.getenv('NXF_WORK') else 'work', + '-w': os.getenv('NXF_WORK') if os.getenv('NXF_WORK') else './work', '-resume': False } self.nxf_flag_help = { @@ -72,9 +72,29 @@ def collect_defaults(self): def prompt_vars(self, params = None, direct = False): """ Ask the user if they want to override any default values """ # Main nextflow flags - logging.info("Main nextflow options:\n") + click.secho("Main nextflow options", bold=True, underline=True) for flag, f_default in self.nxf_flag_defaults.items(): - f_user = click.prompt(self.nxf_flag_help[flag], default=f_default) + + # Click prompts don't like None, so we have to use an empty string instead + f_default_print = f_default + if f_default is None: + f_default = '' + f_default_print = 'None' + + # Overwrite the default prompt for boolean + if isinstance(f_default, bool): + f_default_print = 'Y/n' if f_default else 'y/N' + + # Prompt for a response + f_user = click.prompt( + "\n{}\n {} {}".format( + self.nxf_flag_help[flag], + click.style(flag, fg='blue'), + click.style('[{}]'.format(str(f_default_print)), fg='green') + ), + default = f_default, + show_default = False + ) # Only save if we've changed the default if f_user != f_default: # Convert string bools to real bools @@ -85,14 +105,13 @@ def prompt_vars(self, params = None, direct = False): except AttributeError: pass self.nxf_flags[flag] = f_user - + # Uses the parameter values from the JSON file # and does not ask the user to set them explicitly if direct: return # Pipeline params - logging.info("Pipeline specific parameters:\n") if params: Launch.__prompt_defaults_from_param_objects( Launch.__group_parameters(params) @@ -143,39 +162,69 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): } """ for group_label, params in params_grouped.items(): - print("{prespace}Parameter group: \'{label}\'{postspace}".format( - prespace='-'*5, - label=group_label, - postspace='-'*10) - ) - use_defaults = click.confirm("Do you want to use the group's defaults?", - default=True) - if use_defaults: + click.echo("\n\n{}{}".format( + click.style('Parameter group: ', bold=True, underline=True), + click.style(group_label, bold=True, underline=True, fg='red'), + )) + use_defaults = click.confirm( + "Do you want to change the group's defaults? "+click.style('[y/N]', fg='green'), + default=False, show_default=False) + if not use_defaults: continue for parameter in params: value_is_valid = False - while(not value_is_valid): - desired_param_value = click.prompt("'--{name}'\n" - "\tUsage: {usage}\n" - "\tRange/Choices: {choices}.\n" - .format(name=parameter.name, - usage=parameter.usage, - choices=parameter.choices), - default=parameter.default_value) + first_attempt = True + while not value_is_valid: + # Start building the string to show to the user - label and usage + plines = ['', + click.style(parameter.label, bold=True), + click.style(parameter.usage, dim=True) + ] + # Add the choices / range if applicable + if parameter.choices: + rc = 'Choices' if parameter.type == 'string' else 'Range' + plines.append('{}: {}'.format(rc, str(parameter.choices))) + + # Reset the choice display if boolean + if parameter.type == "boolean": + pdef_val = 'Y/n' if parameter.default_value else 'y/N' + else: + pdef_val = parameter.default_value + + # Final line to print - command and default + flag_prompt = click.style(' --{} '.format(parameter.name), fg='blue') + \ + click.style('[{}]'.format(pdef_val), fg='green') + # Only show this final prompt if we're trying again + if first_attempt: + plines.append(flag_prompt) + else: + plines = [flag_prompt] + first_attempt = False + + # Use click.confirm if a boolean for default input handling + if parameter.type == "boolean": + parameter.value = click.confirm("\n".join(plines), + default=parameter.default_value, show_default=False) + # Use click.prompt if anything else + else: + parameter.value = click.prompt("\n".join(plines), + default=parameter.default_value, show_default=False) + + # Set input parameter types if parameter.type == "integer": - desired_param_value = int(desired_param_value) - elif parameter.type == "boolean": - desired_param_value = (str(desired_param_value).lower() in ['yes', 'y', 'true', 't', '1']) + parameter.value = int(parameter.value) elif parameter.type == "decimal": - desired_param_value = float(desired_param_value) - parameter.value = desired_param_value + parameter.value = float(parameter.value) + + # Validate the input try: parameter.validate() except Exception as e: - print(e) + click.secho("\nERROR: {}".format(e), fg='red') + click.secho("Please try again:") continue else: - value_is_valid = True + value_is_valid = True def build_command(self, params = None): """ Build the nextflow run command based on what we know """ @@ -189,13 +238,13 @@ def build_command(self, params = None): # String values else: self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) - + if params: # When a parameter specification file was used, we can run Nextflow with it path = Launch.__create_nfx_params_file(params) self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) Launch.__write_params_as_full_json(params) return - + for param, val in self.params.items(): # Boolean flags like --saveTrimmed if isinstance(val, bool): @@ -215,7 +264,7 @@ def __create_nfx_params_file(cls, params): with open(output_file, "w") as fp: fp.write(json_string) return output_file - + @classmethod def __write_params_as_full_json(cls, params, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") @@ -226,7 +275,13 @@ def __write_params_as_full_json(cls, params, outdir = os.getcwd()): def launch_workflow(self): """ Launch nextflow if required """ - logging.info("Nextflow command:\n {}\n\n".format(self.nextflow_cmd)) - if click.confirm('Do you want to run this command now?'): - logging.info("Launching!") + click.secho("\n\nNextflow command:", bold=True, underline=True) + click.secho(" {}\n\n".format(self.nextflow_cmd), fg='magenta') + + if click.confirm( + 'Do you want to run this command now? '+click.style('[y/N]', fg='green'), + default=False, + show_default=False + ): + logging.info("Launching workflow!") subprocess.call(self.nextflow_cmd, shell=True) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index b4a3adc161..6149237a1e 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -34,17 +34,17 @@ def create_from_json(parameters_json, schema = ""): properties = json.loads(parameters_json) parameters = [] for param in properties.get("parameters"): - parameter = Parameter.builder().name(param.get("name")) \ - .label(param.get("label")) \ - .usage(param.get("usage")) \ - .param_type(param.get("type")) \ - .choices(param.get("choices")) \ - .default(param.get("default_value")) \ - .pattern(param.get("pattern")) \ - .render(param.get("render")) \ - .arity(param.get("arity")) \ - .group(param.get("group")) \ - .build() + parameter = (Parameter.builder().name(param.get("name")) + .label(param.get("label")) + .usage(param.get("usage")) + .param_type(param.get("type")) + .choices(param.get("choices")) + .default(param.get("default_value")) + .pattern(param.get("pattern")) + .render(param.get("render")) + .arity(param.get("arity")) + .group(param.get("group")) + .build()) parameters.append(parameter) return parameters @@ -63,10 +63,10 @@ def in_nextflow_json(parameters, indent=0): for p in parameters: params[p.name] = p.value if p.value else p.default_value return json.dumps(params, indent=indent) - + @staticmethod def in_full_json(parameters, indent=0): - """Converts a list of Parameter objects into JSON. All attributes + """Converts a list of Parameter objects into JSON. All attributes are written. Args: @@ -79,7 +79,7 @@ def in_full_json(parameters, indent=0): params_dict = {} params_dict["parameters"] = [p.as_dict() for p in parameters] return json.dumps(params_dict, indent=indent) - + @classmethod def __download_schema_from_nf_core(cls, url): with requests_cache.disabled(): @@ -95,7 +95,7 @@ class Parameter(object): """ def __init__(self, param_builder): # Make some checks - + # Put content self.name = param_builder.p_name self.label = param_builder.p_label @@ -109,7 +109,7 @@ def __init__(self, param_builder): self.required = param_builder.p_required self.render = param_builder.p_render self.group = param_builder.p_group - + @staticmethod def builder(): return ParameterBuilder() @@ -121,13 +121,13 @@ def as_dict(self): dict: Parameter object as key value pairs. """ params_dict = {} - for attribute in ['name', 'label', 'usage', 'required', + for attribute in ['name', 'label', 'usage', 'required', 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render', 'group']: if getattr(self, attribute): params_dict[attribute] = getattr(self, attribute) params_dict['required'] = getattr(self, 'required') return params_dict - + def validate(self): """Validates the parameter's value. If the value is within the parameter requirements, no exception is thrown. @@ -135,7 +135,7 @@ def validate(self): Raises: LookupError: Raised when no matching validator can be determined. AttributeError: Raised with description, if a parameter value violates - the parameter constrains. + the parameter constrains. """ validator = vld.Validators.get_validator_for_param(self) validator.validate() @@ -166,7 +166,7 @@ def group(self, group): """ self.p_group = group return self - + def name(self, name): """Sets the parameter name. @@ -175,7 +175,7 @@ def name(self, name): """ self.p_name = name return self - + def label(self, label): """Sets the parameter label. @@ -184,7 +184,7 @@ def label(self, label): """ self.p_label = label return self - + def usage(self, usage): """Sets the parameter usage. @@ -193,7 +193,7 @@ def usage(self, usage): """ self.p_usage = usage return self - + def value(self, value): """Sets the parameter value. @@ -202,7 +202,7 @@ def value(self, value): """ self.p_value = value return self - + def choices(self, choices): """Sets the parameter value choices. @@ -211,7 +211,7 @@ def choices(self, choices): """ self.p_choices = choices return self - + def param_type(self, param_type): """Sets the parameter type. @@ -220,7 +220,7 @@ def param_type(self, param_type): """ self.p_type = param_type return self - + def default(self, default): """Sets the parameter default value. @@ -229,7 +229,7 @@ def default(self, default): """ self.p_default_value = default return self - + def pattern(self, pattern): """Sets the parameter regex pattern. @@ -238,7 +238,7 @@ def pattern(self, pattern): """ self.p_pattern = pattern return self - + def arity(self, arity): """Sets the parameter regex pattern. @@ -256,12 +256,12 @@ def render(self, render): """ self.p_render = render return self - + def required(self, required): """Sets the required parameter flag.""" self.p_required = required return self - + def build(self): """Builds parameter object. @@ -269,4 +269,3 @@ def build(self): Parameter: Fresh from the factory. """ return Parameter(self) - diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 28efa7b599..5954c690bf 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -9,20 +9,20 @@ ABC = abc.ABCMeta('ABC', (), {}) class Validators(object): - """Gives access to a factory method for objects of instance - :class:`Validator` which returns the correct Validator for a + """Gives access to a factory method for objects of instance + :class:`Validator` which returns the correct Validator for a given parameter type. """ def __init__(self): pass - + @staticmethod def get_validator_for_param(parameter): """Determines matching :class:`Validator` instance for a given parameter. Returns: Validator: Matching validator for a given :class:`Parameter`. - + Raises: LookupError: In case no matching validator for a given parameter type can be determined. @@ -34,11 +34,11 @@ def get_validator_for_param(parameter): elif parameter.type == "boolean": return BooleanValidator(parameter) elif parameter.type == "decimal": - return DecimalValidator(parameter) + return DecimalValidator(parameter) raise LookupError("Cannot find a matching validator for type '{}'." .format(parameter.type)) - + class Validator(ABC): """Abstract base class for different parameter validators. """ @@ -58,10 +58,10 @@ def validate(self): class IntegerValidator(Validator): """Implementation for parameters of type integer. - + Args: parameter (:class:`Parameter`): A Parameter object. - + Raises: AttributeError: In case the argument is not of instance integer. """ @@ -78,7 +78,7 @@ def validate(self): """ value = self._param.value if not isinstance(value, int): - raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + raise AttributeError("The value {} for parameter {} needs to be an Integer, but was a {}" .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) print(choices) @@ -87,16 +87,16 @@ def validate(self): if len(choices) < 2: raise AttributeError("The property 'choices' must have at least two entries.") if not (value >= choices[0] and value <= choices[-1]): - raise AttributeError("The value for parameter '{}' must be within range of [{},{}]" + raise AttributeError("'{}' must be within the range [{},{}]" .format(self._param.name, choices[0], choices[-1])) class StringValidator(Validator): """Implementation for parameters of type string. - + Args: parameter (:class:`Parameter`): A Parameter object. - + Raises: AttributeError: In case the argument is not of instance string. """ @@ -122,24 +122,24 @@ def validate(self): "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) result = re.match(self._param.pattern, self._param.value) if not result: - raise AttributeError("The value '{}' for parameter '{}'" \ - " did not match the regex pattern '{}' .".format( - self._param.value, self._param.name, self._param.pattern + raise AttributeError("'{}' doesn't match the regex pattern '{}'".format( + self._param.value, self._param.pattern )) else: if value not in choices: - raise AttributeError("The value '{}' for parameter '{}'" \ - " was not part of choices '{}'.".format( - value, self._param.name, choices - )) + raise AttributeError( + "'{}' is not not one of the choices {}".format( + value, str(choices) + ) + ) class BooleanValidator(Validator): """Implementation for parameters of type boolean. - + Args: parameter (:class:`Parameter`): A Parameter object. - + Raises: AttributeError: In case the argument is not of instance boolean. """ @@ -162,10 +162,10 @@ def validate(self): class DecimalValidator(Validator): """Implementation for parameters of type boolean. - + Args: parameter (:class:`Parameter`): A Parameter object. - + Raises: AttributeError: In case the argument is not of instance decimal. """ From 2770b31be3e28317758791728a93c716f3830d5b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Jan 2019 18:27:39 +0100 Subject: [PATCH 48/64] Colour: Once you pop, you can't stop --- nf_core/launch.py | 2 +- nf_core/lint.py | 12 ++++++------ scripts/nf-core | 14 ++++++-------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index f527b75a0d..4fc151c0be 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -164,7 +164,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): for group_label, params in params_grouped.items(): click.echo("\n\n{}{}".format( click.style('Parameter group: ', bold=True, underline=True), - click.style(group_label, bold=True, underline=True, fg='red'), + click.style(group_label, bold=True, underline=True, fg='red') )) use_defaults = click.confirm( "Do you want to change the group's defaults? "+click.style('[y/N]', fg='green'), diff --git a/nf_core/lint.py b/nf_core/lint.py index d4135439fc..2388e0f51d 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -818,13 +818,13 @@ def print_results(self): # Print results rl = "\n Using --release mode linting tests" if self.release_mode else '' logging.info("===========\n LINTING RESULTS\n=================\n" + - "{0:>4} tests passed".format(len(self.passed)) + - "{0:>4} tests had warnings".format(len(self.warned)) + - "{0:>4} tests failed".format(len(self.failed)) + rl + click.style("{0:>4} tests passed".format(len(self.passed)), fg='green') + + click.style("{0:>4} tests had warnings".format(len(self.warned)), fg='yellow') + + click.style("{0:>4} tests failed".format(len(self.failed)), fg='red') + rl ) if len(self.passed) > 0: - logging.debug("Test Passed:\n {}".format("\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.passed]))) + logging.debug("{}\n {}".format(click.style("Test Passed:", fg='green'), "\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.passed]))) if len(self.warned) > 0: - logging.warn("Test Warnings:\n {}".format("\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.warned]))) + logging.warn("{}\n {}".format(click.style("Test Warnings:", fg='yellow'), "\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.warned]))) if len(self.failed) > 0: - logging.error("Test Failures:\n {}".format("\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.failed]))) + logging.error("{}\n {}".format(click.style("Test Failures:", fg='red'), "\n ".join(["http://nf-co.re/errors#{}: {}".format(eid, msg) for eid, msg in self.failed]))) diff --git a/scripts/nf-core b/scripts/nf-core index b34ac7741c..fd692fa474 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -132,7 +132,7 @@ def download(pipeline, release, singularity, outdir): @click.option( '--direct', is_flag = True, - default = False, + default = False, help = "Uses given values from the parameter file directly." ) def launch(pipeline, params, direct): @@ -231,11 +231,9 @@ def create(name, description, author, new_version, no_git, force, outdir): if __name__ == '__main__': - print(""" - ,--./,-. - ___ __ __ __ ___ /,-._.--~\\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - """, file=sys.stderr) + click.echo(click.style("\n ,--.", fg='green')+click.style("/",fg='black')+click.style(",-.", fg='green')) + click.echo(click.style(" ___ __ __ __ ___ ", fg='blue')+click.style("/,-._.--~\\", fg='green')) + click.echo(click.style(" |\ | |__ __ / ` / \ |__) |__ ", fg='blue')+click.style(" } {", fg='yellow')) + click.echo(click.style(" | \| | \__, \__/ | \ |___ ", fg='blue')+click.style("\`-._,-`-,", fg='green')) + click.secho(" `._,._,'\n", fg='green') nf_core_cli() From 4c20b852168575fe4c9590d8a727b420b8e913f3 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 13:49:09 +0100 Subject: [PATCH 49/64] Refactors import statements --- nf_core/workflow/parameters.py | 2 +- nf_core/workflow/validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 6149237a1e..50c4de7fbd 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -4,7 +4,7 @@ import requests_cache from jsonschema import validate from jsonschema.exceptions import ValidationError -from nf_core.workflow import validation as vld +import nf_core.workflow.validation as vld NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 5954c690bf..36e0f4c50b 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,7 @@ import abc import re import sys -from nf_core.workflow import parameters as pms +import nf_core.workflow.parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC From b5d209694c12d6450d3dfbd21b5244680a7ca4c9 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 14:11:42 +0100 Subject: [PATCH 50/64] Updates import --- nf_core/workflow/parameters.py | 31 ++++++++++++++++++------------- nf_core/workflow/workflow.py | 4 ++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 50c4de7fbd..2720a98557 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -2,18 +2,21 @@ import json import requests import requests_cache + from jsonschema import validate -from jsonschema.exceptions import ValidationError + import nf_core.workflow.validation as vld NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" + class Parameters: """Contains a static factory method for :class:`Parameter` object creation. """ + @staticmethod - def create_from_json(parameters_json, schema = ""): + def create_from_json(parameters_json, schema=""): """Creates a list of Parameter objects from a description in JSON. @@ -35,16 +38,16 @@ def create_from_json(parameters_json, schema = ""): parameters = [] for param in properties.get("parameters"): parameter = (Parameter.builder().name(param.get("name")) - .label(param.get("label")) - .usage(param.get("usage")) - .param_type(param.get("type")) - .choices(param.get("choices")) - .default(param.get("default_value")) - .pattern(param.get("pattern")) - .render(param.get("render")) - .arity(param.get("arity")) - .group(param.get("group")) - .build()) + .label(param.get("label")) + .usage(param.get("usage")) + .param_type(param.get("type")) + .choices(param.get("choices")) + .default(param.get("default_value")) + .pattern(param.get("pattern")) + .render(param.get("render")) + .arity(param.get("arity")) + .group(param.get("group")) + .build()) parameters.append(parameter) return parameters @@ -93,6 +96,7 @@ def __download_schema_from_nf_core(cls, url): class Parameter(object): """Holds information about a workflow parameter. """ + def __init__(self, param_builder): # Make some checks @@ -122,7 +126,7 @@ def as_dict(self): """ params_dict = {} for attribute in ['name', 'label', 'usage', 'required', - 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render', 'group']: + 'type', 'value', 'choices', 'default_value', 'pattern', 'arity', 'render', 'group']: if getattr(self, attribute): params_dict[attribute] = getattr(self, attribute) params_dict['required'] = getattr(self, 'required') @@ -144,6 +148,7 @@ def validate(self): class ParameterBuilder: """Parameter builder. """ + def __init__(self): self.p_name = "" self.p_label = "" diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index 3aa3c0f091..6b66fa1c0e 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -1,5 +1,5 @@ -from nf_core.parameters import Parameters -from nf_core.parameters import Parameter +from nf_core.workflow.parameters import Parameters +from nf_core.workflow.parameters import Parameter class Workflow(object): """nf-core workflow object that holds run parameter information. From 6378d35980d3351ec84b7297655eff87e3db8125 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 14:23:50 +0100 Subject: [PATCH 51/64] Fixes wrong pytest command call --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 93f40905b4..165ef16590 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation script: - - python -m pytest --cov=nf_core . + - pytest --cov=nf_core . - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline From b32beb3f0fb8df6a05526413fbaa10b540e56a00 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 14:39:37 +0100 Subject: [PATCH 52/64] Reports coverage again --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 165ef16590..ebccb6a264 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation script: - - pytest --cov=nf_core . + - pytest . --cov - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline From 4c8a397d4f47fc7ba32944d002ede0d9646f8c6a Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 14:46:03 +0100 Subject: [PATCH 53/64] Revert "Reports coverage again" This reverts commit b32beb3f0fb8df6a05526413fbaa10b540e56a00. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ebccb6a264..165ef16590 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation script: - - pytest . --cov + - pytest --cov=nf_core . - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline From 04caa8c889570f7d6a0f01e14606302a8c29e4ff Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Sun, 27 Jan 2019 14:55:10 +0100 Subject: [PATCH 54/64] Removes pytests version pin --- .travis.yml | 2 +- nf_core/workflow/workflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 165ef16590..88ec742d4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - cd ${TRAVIS_BUILD_DIR} - pip install --upgrade pip # Get rid of dependency resolve issues from older pip versions - pip install . - - pip install codecov 'pytest==3.6.4' pytest-datafiles pytest-cov mock jsonschema + - pip install codecov pytest-datafiles pytest-cov mock jsonschema - pip install Sphinx sphinxcontrib-napoleon # For autodoc generation script: diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index 6b66fa1c0e..d0b606ba76 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -1,5 +1,5 @@ from nf_core.workflow.parameters import Parameters -from nf_core.workflow.parameters import Parameter + class Workflow(object): """nf-core workflow object that holds run parameter information. From 2b1a777fb232f08ef23261460a7c791a8e3b5a2f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Feb 2019 22:19:12 +0100 Subject: [PATCH 55/64] Remove circular import --- nf_core/workflow/parameters.py | 2 ++ nf_core/workflow/validation.py | 6 ++---- nf_core/workflow/workflow.py | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 2720a98557..179344bf49 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import copy import json import requests diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 36e0f4c50b..f321784553 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,8 @@ +#!/usr/bin/env python + import abc import re import sys -import nf_core.workflow.parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC @@ -46,9 +47,6 @@ class Validator(ABC): @abc.abstractmethod def __init__(self, parameter): - if not isinstance(parameter, pms.Parameter): - raise (AttributeError("Argument must be of class {}" - .format(pms.Parameter.__class__))) self._param = parameter @abc.abstractmethod diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index d0b606ba76..ae59c8958e 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from nf_core.workflow.parameters import Parameters @@ -6,12 +8,12 @@ class Workflow(object): Args: name (str): Workflow name. - parameters_json (str): Workflow parameter data in JSON. + parameters_json (str): Workflow parameter data in JSON. """ def __init__(self, name, parameters_json): self.name = name self.parameters = Parameters.create_from_json(parameters_json) - + def in_nextflow_json(self, indent=0): """Converts the Parameter list in a workflow readable parameter JSON file. @@ -28,4 +30,4 @@ def in_full_json(self, indent=0): Returns: str: JSON formatted parameters. """ - return Parameters.in_full_json(self.parameters, indent) \ No newline at end of file + return Parameters.in_full_json(self.parameters, indent) From a5cee6e64005820dd59b8472ee928f87dbebc3dd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Feb 2019 23:02:22 +0100 Subject: [PATCH 56/64] Fix py2 recursive import. Force local pull of workflow. --- nf_core/launch.py | 70 ++++++++++++++++++++++++++++++++++++++++------- nf_core/utils.py | 28 +------------------ scripts/nf-core | 5 +--- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 4fc151c0be..454c55c485 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -8,24 +8,55 @@ import os import subprocess -import nf_core.utils -from nf_core.workflow import parameters as pms +import nf_core.utils, nf_core.list +import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow def launch_pipeline(workflow, params_local_uri, direct): launcher = Launch(workflow) params_list = [] + + # Get nextflow to fetch the workflow if we don't already have it + if not launcher.local_wf: + logging.info("Downloading workflow: {}".format(launcher.workflow)) + try: + with open(os.devnull, 'w') as devnull: + nfconfig_raw = subprocess.check_output(['nextflow', 'pull', launcher.workflow], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) + else: + launcher.local_wf = nf_core.list.LocalWorkflow(launcher.workflow) + + # Get the pipeline default parameters try: + params_json_str = None + # Params file supplied to launch command if params_local_uri: - with open(params_local_uri, 'r') as fp: params_json_str = fp.read() + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + # Get workflow file from local cached copy else: - params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow) - params_list = pms.Parameters.create_from_json(params_json_str) + local_params_path = os.path.join(launcher.local_wf.local_path, 'parameters.settings.json') + if os.path.exists(local_params_path): + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + if not params_json_str: + raise LookupError + params_list = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) except LookupError as e: print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( - pipeline=workflow, exception=e)) + pipeline=launcher.workflow, exception=e)) + + # Fallback if parameters.settings.json not found, calls Nextflow's config command if not params_list: - launcher.collect_defaults() # Fallback, calls Nextflow's config command + launcher.collect_defaults() + + # Kick off the interactive wizard to collect user inputs launcher.prompt_vars(params_list, direct) + + # Build and launch the `nextflow run` command launcher.build_command(params_list) launcher.launch_workflow() @@ -41,6 +72,14 @@ def __init__(self, workflow): logging.debug("Prepending nf-core/ to workflow") logging.info("Launching {}\n".format(workflow)) + # Get local workflows to see if we have a cached version + self.local_wf = None + wfs = nf_core.list.Workflows() + wfs.get_local_nf_workflows() + for wf in wfs.local_workflows: + if workflow == wf.full_name: + self.local_wf = wf + self.workflow = workflow self.nxf_flag_defaults = { '-name': None, @@ -95,6 +134,7 @@ def prompt_vars(self, params = None, direct = False): default = f_default, show_default = False ) + # Only save if we've changed the default if f_user != f_default: # Convert string bools to real bools @@ -119,7 +159,17 @@ def prompt_vars(self, params = None, direct = False): return for param, p_default in self.param_defaults.items(): if not isinstance(p_default, dict) and not isinstance(p_default, list): - p_user = click.prompt("--{}".format(param), default=p_default) + + # Prompt for a response + p_user = click.prompt( + "\n --{} {}".format( + click.style(param, fg='blue'), + click.style('[{}]'.format(str(p_default)), fg='green') + ), + default = p_default, + show_default = False + ) + # Only save if we've changed the default if p_user != p_default: # Convert string bools to real bools @@ -260,7 +310,7 @@ def build_command(self, params = None): def __create_nfx_params_file(cls, params): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") - json_string = pms.Parameters.in_nextflow_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(params, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file @@ -268,7 +318,7 @@ def __create_nfx_params_file(cls, params): @classmethod def __write_params_as_full_json(cls, params, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") - json_string = pms.Parameters.in_full_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_full_json(params, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file diff --git a/nf_core/utils.py b/nf_core/utils.py index 7c7cdabab0..ddf0fc15e2 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,36 +5,10 @@ import datetime import os -import requests +import requests import subprocess import tempfile -PARAMETERS_URI_TEMPL = "https://raw.githubusercontent.com/nf-core/{pipeline}/master/parameters.settings.json" - - -def fetch_parameter_settings_from_github(pipeline): - """Requests the pipeline parameter settings from Github. - - Args: - pipeline (str): The nf-core pipeline name. - - Returns: - str: The raw JSON string of the file with the parameter settings. - - Raises: - LookupError: If for some reason the URI cannot be accessed. - """ - import requests_cache - - target_uri = PARAMETERS_URI_TEMPL.format(pipeline=pipeline) - try: - with requests_cache.disabled(): - result = requests.get(target_uri, headers={'Cache-Control': 'no-cache'}) - except (ConnectionError, TimeoutError) as e: - raise LookupError(e) - return result.text - - def fetch_wf_config(wf_path): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. diff --git a/scripts/nf-core b/scripts/nf-core index fd692fa474..8b7f95c985 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -124,15 +124,12 @@ def download(pipeline, release, singularity, outdir): ) @click.option( '-p', '--params', - is_flag = False, - default = "", type = str, help = "Local parameter settings file in JSON." ) @click.option( - '--direct', + '-d', '--direct', is_flag = True, - default = False, help = "Uses given values from the parameter file directly." ) def launch(pipeline, params, direct): From b1e86dcc025fd073c4c7ba07724a1a184ac1fba3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:16:03 +0100 Subject: [PATCH 57/64] Code testing on old pipelines, refactoring --- nf_core/launch.py | 236 ++++++++++++++++----------------- nf_core/workflow/parameters.py | 7 +- nf_core/workflow/validation.py | 6 +- 3 files changed, 120 insertions(+), 129 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 454c55c485..cafd84e13a 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -6,58 +6,37 @@ import click import logging import os +import re import subprocess import nf_core.utils, nf_core.list import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow def launch_pipeline(workflow, params_local_uri, direct): + + # Create a pipeline launch object launcher = Launch(workflow) - params_list = [] # Get nextflow to fetch the workflow if we don't already have it - if not launcher.local_wf: - logging.info("Downloading workflow: {}".format(launcher.workflow)) - try: - with open(os.devnull, 'w') as devnull: - nfconfig_raw = subprocess.check_output(['nextflow', 'pull', launcher.workflow], stderr=devnull) - except OSError as e: - if e.errno == os.errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - launcher.local_wf = nf_core.list.LocalWorkflow(launcher.workflow) + launcher.get_local_wf() # Get the pipeline default parameters - try: - params_json_str = None - # Params file supplied to launch command - if params_local_uri: - with open(params_local_uri, 'r') as fp: - params_json_str = fp.read() - # Get workflow file from local cached copy - else: - local_params_path = os.path.join(launcher.local_wf.local_path, 'parameters.settings.json') - if os.path.exists(local_params_path): - with open(params_local_uri, 'r') as fp: - params_json_str = fp.read() - if not params_json_str: - raise LookupError - params_list = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) - except LookupError as e: - print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( - pipeline=launcher.workflow, exception=e)) + launcher.parse_parameter_settings(params_local_uri) # Fallback if parameters.settings.json not found, calls Nextflow's config command - if not params_list: - launcher.collect_defaults() + if len(launcher.parameters) == 0: + launcher.collect_pipeline_param_defaults() + + # Group the parameters + launcher.group_parameters() # Kick off the interactive wizard to collect user inputs - launcher.prompt_vars(params_list, direct) + launcher.prompt_core_nxf_flags() + if not direct: + launcher.prompt_param_flags() # Build and launch the `nextflow run` command - launcher.build_command(params_list) + launcher.build_command() launcher.launch_workflow() class Launch(object): @@ -96,19 +75,74 @@ def __init__(self, workflow): '-resume': 'Resume a previous workflow run' } self.nxf_flags = {} - self.param_defaults = {} - self.params = {} + self.parameters = [] + self.grouped_parameters = {} + self.params_user = {} self.nextflow_cmd = "nextflow run {}".format(self.workflow) + self.use_params_file = True - def collect_defaults(self): + def get_local_wf(self): + """ + Check if this workflow has a local copy and use nextflow to pull it if not + """ + if not self.local_wf: + logging.info("Downloading workflow: {}".format(self.workflow)) + try: + with open(os.devnull, 'w') as devnull: + nfconfig_raw = subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) + else: + self.local_wf = nf_core.list.LocalWorkflow(self.workflow) + + def parse_parameter_settings(self, params_local_uri = None): + """ + Load full parameter info from the pipeline parameters.settings.json file + """ + try: + params_json_str = None + # Params file supplied to launch command + if params_local_uri: + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + # Get workflow file from local cached copy + else: + local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') + if os.path.exists(local_params_path): + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + if not params_json_str: + raise LookupError('parameters.settings.json file not found') + self.parameters = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) + except LookupError as e: + print("WARNING: Could not parse parameter settings file for `{pipeline}`:\n {exception}".format( + pipeline=self.workflow, exception=e)) + + def collect_pipeline_param_defaults(self): """ Collect the default params and values from the workflow """ + logging.info("Collecting pipeline parameter defaults\n") config = nf_core.utils.fetch_wf_config(self.workflow) for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: - self.param_defaults[keys[1]] = value - - def prompt_vars(self, params = None, direct = False): + parameter = (nf_core.workflow.parameters.Parameter.builder() + .name(keys[1]) + .label(None) + .usage(None) + .param_type("string") + .choices(None) + .default(str(value)) + .pattern(".*") + .render(None) + .arity(None) + .group("Pipeline parameters") + .build()) + self.parameters.append(parameter) + + def prompt_core_nxf_flags(self): """ Ask the user if they want to override any default values """ # Main nextflow flags click.secho("Main nextflow options", bold=True, underline=True) @@ -146,43 +180,7 @@ def prompt_vars(self, params = None, direct = False): pass self.nxf_flags[flag] = f_user - # Uses the parameter values from the JSON file - # and does not ask the user to set them explicitly - if direct: - return - - # Pipeline params - if params: - Launch.__prompt_defaults_from_param_objects( - Launch.__group_parameters(params) - ) - return - for param, p_default in self.param_defaults.items(): - if not isinstance(p_default, dict) and not isinstance(p_default, list): - - # Prompt for a response - p_user = click.prompt( - "\n --{} {}".format( - click.style(param, fg='blue'), - click.style('[{}]'.format(str(p_default)), fg='green') - ), - default = p_default, - show_default = False - ) - - # Only save if we've changed the default - if p_user != p_default: - # Convert string bools to real bools - try: - p_user = p_user.strip('"').strip("'") - if p_user.lower() == 'true': p_user = True - if p_user.lower() == 'false': p_user = False - except AttributeError: - pass - self.params[param] = p_user - - @classmethod - def __group_parameters(cls, parameters): + def group_parameters(self): """Groups parameters by their 'group' property. Args: @@ -191,27 +189,14 @@ def __group_parameters(cls, parameters): Returns: dict: Parameter objects grouped by the `group` property. """ - grouped_parameters = {} - for param in parameters: - if not grouped_parameters.get(param.group): - grouped_parameters[param.group] = [] - grouped_parameters[param.group].append(param) - return grouped_parameters - - @classmethod - def __prompt_defaults_from_param_objects(cls, params_grouped): - """Prompts the user for parameter input values and validates them. - - Args: - params_grouped (dict): A dictionary with parameter group labels - as keys and list of parameters as values. :: - - { - "group1": [param1, param2], - ... - } - """ - for group_label, params in params_grouped.items(): + for param in self.parameters: + if param.group not in self.grouped_parameters.keys(): + self.grouped_parameters[param.group] = [] + self.grouped_parameters[param.group].append(param) + + def prompt_param_flags(self): + """ Prompts the user for parameter input values and validates them. """ + for group_label, params in self.grouped_parameters.items(): click.echo("\n\n{}{}".format( click.style('Parameter group: ', bold=True, underline=True), click.style(group_label, bold=True, underline=True, fg='red') @@ -226,10 +211,12 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): first_attempt = True while not value_is_valid: # Start building the string to show to the user - label and usage - plines = ['', - click.style(parameter.label, bold=True), - click.style(parameter.usage, dim=True) - ] + plines = [''] + if parameter.label: + plines.append(click.style(parameter.label, bold=True)) + if parameter.usage: + plines.append(click.style(parameter.usage, dim=True)) + # Add the choices / range if applicable if parameter.choices: rc = 'Choices' if parameter.type == 'string' else 'Range' @@ -244,6 +231,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): # Final line to print - command and default flag_prompt = click.style(' --{} '.format(parameter.name), fg='blue') + \ click.style('[{}]'.format(pdef_val), fg='green') + # Only show this final prompt if we're trying again if first_attempt: plines.append(flag_prompt) @@ -265,6 +253,8 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): parameter.value = int(parameter.value) elif parameter.type == "decimal": parameter.value = float(parameter.value) + else: + parameter.value = str(parameter.value) # Validate the input try: @@ -276,7 +266,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): else: value_is_valid = True - def build_command(self, params = None): + def build_command(self): """ Build the nextflow run command based on what we know """ for flag, val in self.nxf_flags.items(): # Boolean flags like -resume @@ -289,36 +279,36 @@ def build_command(self, params = None): else: self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) - if params: # When a parameter specification file was used, we can run Nextflow with it - path = Launch.__create_nfx_params_file(params) + # Write the user selection to a file and run nextflow with that + if self.use_params_file: + path = self.create_nfx_params_file() self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) - Launch.__write_params_as_full_json(params) - return + self.write_params_as_full_json() - for param, val in self.params.items(): - # Boolean flags like --saveTrimmed - if isinstance(val, bool): - if val: - self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + # Call nextflow with a list of command line flags + else: + for param, val in self.params_user.items(): + # Boolean flags like --saveTrimmed + if isinstance(val, bool): + if val: + self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + else: + logging.error("Can't set false boolean flags.") + # everything else else: - logging.warn("TODO: Can't set false boolean flags currently.") - # everything else - else: - self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) + self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) - @classmethod - def __create_nfx_params_file(cls, params): + def create_nfx_params_file(self): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") - json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file - @classmethod - def __write_params_as_full_json(cls, params, outdir = os.getcwd()): + def write_params_as_full_json(self, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") - json_string = nf_core.workflow.parameters.Parameters.in_full_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_full_json(self.parameters, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 179344bf49..c56b891d9c 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -11,7 +11,6 @@ NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" - class Parameters: """Contains a static factory method for :class:`Parameter` object creation. @@ -39,7 +38,8 @@ def create_from_json(parameters_json, schema=""): properties = json.loads(parameters_json) parameters = [] for param in properties.get("parameters"): - parameter = (Parameter.builder().name(param.get("name")) + parameter = (Parameter.builder() + .name(param.get("name")) .label(param.get("label")) .usage(param.get("usage")) .param_type(param.get("type")) @@ -66,7 +66,8 @@ def in_nextflow_json(parameters, indent=0): """ params = {} for p in parameters: - params[p.name] = p.value if p.value else p.default_value + if p.value and p.value != p.default_value: + params[p.name] = p.value return json.dumps(params, indent=indent) @staticmethod diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index f321784553..c565af8458 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -111,12 +111,12 @@ def validate(self): """ value = self._param.value if not isinstance(value, str): - raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type String, but was {}" .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: - raise AttributeError("Can't validate value for parameter '{}'," \ + raise AttributeError("Can't validate value for parameter '{}', " \ "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) result = re.match(self._param.pattern, self._param.value) if not result: @@ -180,5 +180,5 @@ def validate(self): """ value = self._param.value if not isinstance(self._param.value, float): - raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type Decimal, but was {}" .format(value, self._param.name, type(value))) From 0d6ea8721682b10da36f43998424af07aa2c1d64 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:34:26 +0100 Subject: [PATCH 58/64] Try to guess types when handling param defaults --- nf_core/launch.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index cafd84e13a..738a416a77 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -49,7 +49,7 @@ def __init__(self, workflow): if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): workflow = "nf-core/{}".format(workflow) logging.debug("Prepending nf-core/ to workflow") - logging.info("Launching {}\n".format(workflow)) + logging.info("Launching {}".format(workflow)) # Get local workflows to see if we have a cached version self.local_wf = None @@ -97,6 +97,7 @@ def get_local_wf(self): raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: self.local_wf = nf_core.list.LocalWorkflow(self.workflow) + self.local_wf.get_local_nf_workflow_details() def parse_parameter_settings(self, params_local_uri = None): """ @@ -112,7 +113,7 @@ def parse_parameter_settings(self, params_local_uri = None): else: local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') if os.path.exists(local_params_path): - with open(params_local_uri, 'r') as fp: + with open(local_params_path, 'r') as fp: params_json_str = fp.read() if not params_json_str: raise LookupError('parameters.settings.json file not found') @@ -128,15 +129,36 @@ def collect_pipeline_param_defaults(self): for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: + + # Try to guess the variable type from the default value + p_type = 'string' + p_default = str(value) + # All digits - int + if value.isdigit(): + p_type = 'integer' + p_default = int(value) + else: + # Not just digis - try converting to a float + try: + p_default = float(value) + p_type = 'decimal' + except ValueError: + pass + # Strings 'true' and 'false' - booleans + if value == 'true' or value == 'false': + p_type = 'boolean' + p_default = True if value == 'true' else False + + # Build the Parameter object parameter = (nf_core.workflow.parameters.Parameter.builder() .name(keys[1]) .label(None) .usage(None) - .param_type("string") + .param_type(p_type) .choices(None) - .default(str(value)) + .default(p_default) .pattern(".*") - .render(None) + .render("textfield") .arity(None) .group("Pipeline parameters") .build()) From fab91b807ea737216b418ab31fb48b3617a9845f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:50:31 +0100 Subject: [PATCH 59/64] Fake a change in the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d1e3784e9..78fa358116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ pipelines from being synced if necessary. * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary. * Added nf-core tools API description to assist developers with the classes and functions available. * Docs are automatically built by Travis CI and updated on the nf-co.re website. -* Introduced test for filtering remote workflows by keyword +* Introduced test for filtering remote workflows by keyword. ## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 Tantalum Butterfly From 4b245aeb168a400db5908ea257558e06bad476ab Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:34:51 +0100 Subject: [PATCH 60/64] Fix LGTM warnings --- nf_core/launch.py | 3 +-- nf_core/utils.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 738a416a77..c889210806 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -6,7 +6,6 @@ import click import logging import os -import re import subprocess import nf_core.utils, nf_core.list @@ -89,7 +88,7 @@ def get_local_wf(self): logging.info("Downloading workflow: {}".format(self.workflow)) try: with open(os.devnull, 'w') as devnull: - nfconfig_raw = subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) + subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) except OSError as e: if e.errno == os.errno.ENOENT: raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") diff --git a/nf_core/utils.py b/nf_core/utils.py index ddf0fc15e2..ec9f807ba5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,7 +5,6 @@ import datetime import os -import requests import subprocess import tempfile From ef1546aa8935e28ca9522b6daf0472ceddceaed7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:36:39 +0100 Subject: [PATCH 61/64] =?UTF-8?q?Changelog=20=F0=9F=A4=A6=F0=9F=8F=BB?= =?UTF-8?q?=E2=80=8D=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f55130e6..8ef9d840fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary. * Added nf-core tools API description to assist developers with the classes and functions available. * Docs are automatically built by Travis CI and updated on the nf-co.re website. -* Introduced test for filtering remote workflows by keyword +* Introduced test for filtering remote workflows by keyword. * Build tools python API docs * Use Travis job for api doc generation and publish * Bump `conda` to 4.5.12 in base nf-core Dockerfile From 5d430a9e31e80d8eb99d228bcfa8a1fea2514c00 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:57:22 +0100 Subject: [PATCH 62/64] Testing and bugfixing --- nf_core/launch.py | 2 +- nf_core/workflow/validation.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index c889210806..7ec8a36fd7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -274,7 +274,7 @@ def prompt_param_flags(self): parameter.value = int(parameter.value) elif parameter.type == "decimal": parameter.value = float(parameter.value) - else: + elif parameter.type == "string": parameter.value = str(parameter.value) # Validate the input diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index c565af8458..fa8ac8ee3f 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -78,15 +78,13 @@ def validate(self): if not isinstance(value, int): raise AttributeError("The value {} for parameter {} needs to be an Integer, but was a {}" .format(value, self._param.name, type(value))) - choices = sorted([x for x in self._param.choices]) - print(choices) - if not choices: - return - if len(choices) < 2: - raise AttributeError("The property 'choices' must have at least two entries.") - if not (value >= choices[0] and value <= choices[-1]): - raise AttributeError("'{}' must be within the range [{},{}]" - .format(self._param.name, choices[0], choices[-1])) + if self._param.choices: + choices = sorted([x for x in self._param.choices]) + if len(choices) < 2: + raise AttributeError("The property 'choices' must have at least two entries.") + if not (value >= choices[0] and value <= choices[-1]): + raise AttributeError("'{}' must be within the range [{},{}]" + .format(self._param.name, choices[0], choices[-1])) class StringValidator(Validator): From 0d4ac5b5275f216ee2cfb3b9e9d8518505698361 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 18:57:58 +0100 Subject: [PATCH 63/64] utils.fetch_wf_config - save a json cache --- nf_core/launch.py | 4 ++-- nf_core/utils.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 7ec8a36fd7..0e66014d88 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -123,8 +123,8 @@ def parse_parameter_settings(self, params_local_uri = None): def collect_pipeline_param_defaults(self): """ Collect the default params and values from the workflow """ - logging.info("Collecting pipeline parameter defaults\n") - config = nf_core.utils.fetch_wf_config(self.workflow) + logging.debug("Collecting pipeline parameter defaults\n") + config = nf_core.utils.fetch_wf_config(self.workflow, self.local_wf) for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: diff --git a/nf_core/utils.py b/nf_core/utils.py index ec9f807ba5..227bc903ed 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -4,11 +4,13 @@ """ import datetime +import json +import logging import os import subprocess import tempfile -def fetch_wf_config(wf_path): +def fetch_wf_config(wf_path, wf=None): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -20,6 +22,27 @@ def fetch_wf_config(wf_path): """ config = dict() + cache_fn = None + cache_basedir = None + cache_path = None + + # Build a cache directory if we can + if os.path.isdir(os.path.join(os.getenv("HOME"), '.nextflow')): + cache_basedir = os.path.join(os.getenv("HOME"), '.nextflow', 'nf-core') + if not os.path.isdir(cache_basedir): + os.mkdir(cache_basedir) + + # If we're given a workflow object with a commit, see if we have a cached copy + if cache_basedir and wf and wf.full_name and wf.commit_sha: + cache_fn = '{}-{}.json'.format(wf.full_name.replace(os.path.sep, '-'), wf.commit_sha) + cache_path = os.path.join(cache_basedir, cache_fn) + if os.path.isfile(cache_path): + logging.debug("Found a config cache, loading: {}".format(cache_path)) + with open(cache_path, 'r') as fh: + config = json.load(fh) + return config + + # Call `nextflow config` and pipe stderr to /dev/null try: with open(os.devnull, 'w') as devnull: @@ -34,6 +57,13 @@ def fetch_wf_config(wf_path): ul = l.decode('utf-8') k, v = ul.split(' = ', 1) config[k] = v + + # If we can, save a cached copy + if cache_path: + logging.debug("Saving config cache: {}".format(cache_path)) + with open(cache_path, 'w') as fh: + json.dump(config, fh, indent=4) + return config From 9220525692796cd0d24b9e74e8ec99f56766bcd2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 19:10:09 +0100 Subject: [PATCH 64/64] Make the launch command work with local directories --- nf_core/launch.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 0e66014d88..998c6e0abc 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,7 +17,8 @@ def launch_pipeline(workflow, params_local_uri, direct): launcher = Launch(workflow) # Get nextflow to fetch the workflow if we don't already have it - launcher.get_local_wf() + if not launcher.wf_ispath: + launcher.get_local_wf() # Get the pipeline default parameters launcher.parse_parameter_settings(params_local_uri) @@ -44,19 +45,23 @@ class Launch(object): def __init__(self, workflow): """ Initialise the class with empty placeholder vars """ + # Check if the workflow name is actually a path + self.wf_ispath = os.path.exists(workflow) + # Prepend nf-core/ if it seems sensible - if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): + if 'nf-core' not in workflow and workflow.count('/') == 0 and not self.wf_ispath: workflow = "nf-core/{}".format(workflow) logging.debug("Prepending nf-core/ to workflow") logging.info("Launching {}".format(workflow)) - # Get local workflows to see if we have a cached version + # Get list of local workflows to see if we have a cached version self.local_wf = None - wfs = nf_core.list.Workflows() - wfs.get_local_nf_workflows() - for wf in wfs.local_workflows: - if workflow == wf.full_name: - self.local_wf = wf + if not self.wf_ispath: + wfs = nf_core.list.Workflows() + wfs.get_local_nf_workflows() + for wf in wfs.local_workflows: + if workflow == wf.full_name: + self.local_wf = wf self.workflow = workflow self.nxf_flag_defaults = { @@ -110,7 +115,10 @@ def parse_parameter_settings(self, params_local_uri = None): params_json_str = fp.read() # Get workflow file from local cached copy else: - local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') + if self.wf_ispath: + local_params_path = os.path.join(self.workflow, 'parameters.settings.json') + else: + local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') if os.path.exists(local_params_path): with open(local_params_path, 'r') as fp: params_json_str = fp.read() @@ -303,7 +311,8 @@ def build_command(self): # Write the user selection to a file and run nextflow with that if self.use_params_file: path = self.create_nfx_params_file() - self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) + if path is not None: + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) self.write_params_as_full_json() # Call nextflow with a list of command line flags @@ -323,6 +332,8 @@ def create_nfx_params_file(self): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4) + if json_string == '{}': + return None with open(output_file, "w") as fp: fp.write(json_string) return output_file