From 2b1a777fb232f08ef23261460a7c791a8e3b5a2f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Feb 2019 22:19:12 +0100 Subject: [PATCH 01/10] Remove circular import --- nf_core/workflow/parameters.py | 2 ++ nf_core/workflow/validation.py | 6 ++---- nf_core/workflow/workflow.py | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 2720a98557..179344bf49 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import copy import json import requests diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 36e0f4c50b..f321784553 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,8 @@ +#!/usr/bin/env python + import abc import re import sys -import nf_core.workflow.parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC @@ -46,9 +47,6 @@ class Validator(ABC): @abc.abstractmethod def __init__(self, parameter): - if not isinstance(parameter, pms.Parameter): - raise (AttributeError("Argument must be of class {}" - .format(pms.Parameter.__class__))) self._param = parameter @abc.abstractmethod diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index d0b606ba76..ae59c8958e 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from nf_core.workflow.parameters import Parameters @@ -6,12 +8,12 @@ class Workflow(object): Args: name (str): Workflow name. - parameters_json (str): Workflow parameter data in JSON. + parameters_json (str): Workflow parameter data in JSON. """ def __init__(self, name, parameters_json): self.name = name self.parameters = Parameters.create_from_json(parameters_json) - + def in_nextflow_json(self, indent=0): """Converts the Parameter list in a workflow readable parameter JSON file. @@ -28,4 +30,4 @@ def in_full_json(self, indent=0): Returns: str: JSON formatted parameters. """ - return Parameters.in_full_json(self.parameters, indent) \ No newline at end of file + return Parameters.in_full_json(self.parameters, indent) From a5cee6e64005820dd59b8472ee928f87dbebc3dd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Feb 2019 23:02:22 +0100 Subject: [PATCH 02/10] Fix py2 recursive import. Force local pull of workflow. --- nf_core/launch.py | 70 ++++++++++++++++++++++++++++++++++++++++------- nf_core/utils.py | 28 +------------------ scripts/nf-core | 5 +--- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 4fc151c0be..454c55c485 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -8,24 +8,55 @@ import os import subprocess -import nf_core.utils -from nf_core.workflow import parameters as pms +import nf_core.utils, nf_core.list +import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow def launch_pipeline(workflow, params_local_uri, direct): launcher = Launch(workflow) params_list = [] + + # Get nextflow to fetch the workflow if we don't already have it + if not launcher.local_wf: + logging.info("Downloading workflow: {}".format(launcher.workflow)) + try: + with open(os.devnull, 'w') as devnull: + nfconfig_raw = subprocess.check_output(['nextflow', 'pull', launcher.workflow], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) + else: + launcher.local_wf = nf_core.list.LocalWorkflow(launcher.workflow) + + # Get the pipeline default parameters try: + params_json_str = None + # Params file supplied to launch command if params_local_uri: - with open(params_local_uri, 'r') as fp: params_json_str = fp.read() + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + # Get workflow file from local cached copy else: - params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow) - params_list = pms.Parameters.create_from_json(params_json_str) + local_params_path = os.path.join(launcher.local_wf.local_path, 'parameters.settings.json') + if os.path.exists(local_params_path): + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + if not params_json_str: + raise LookupError + params_list = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) except LookupError as e: print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( - pipeline=workflow, exception=e)) + pipeline=launcher.workflow, exception=e)) + + # Fallback if parameters.settings.json not found, calls Nextflow's config command if not params_list: - launcher.collect_defaults() # Fallback, calls Nextflow's config command + launcher.collect_defaults() + + # Kick off the interactive wizard to collect user inputs launcher.prompt_vars(params_list, direct) + + # Build and launch the `nextflow run` command launcher.build_command(params_list) launcher.launch_workflow() @@ -41,6 +72,14 @@ def __init__(self, workflow): logging.debug("Prepending nf-core/ to workflow") logging.info("Launching {}\n".format(workflow)) + # Get local workflows to see if we have a cached version + self.local_wf = None + wfs = nf_core.list.Workflows() + wfs.get_local_nf_workflows() + for wf in wfs.local_workflows: + if workflow == wf.full_name: + self.local_wf = wf + self.workflow = workflow self.nxf_flag_defaults = { '-name': None, @@ -95,6 +134,7 @@ def prompt_vars(self, params = None, direct = False): default = f_default, show_default = False ) + # Only save if we've changed the default if f_user != f_default: # Convert string bools to real bools @@ -119,7 +159,17 @@ def prompt_vars(self, params = None, direct = False): return for param, p_default in self.param_defaults.items(): if not isinstance(p_default, dict) and not isinstance(p_default, list): - p_user = click.prompt("--{}".format(param), default=p_default) + + # Prompt for a response + p_user = click.prompt( + "\n --{} {}".format( + click.style(param, fg='blue'), + click.style('[{}]'.format(str(p_default)), fg='green') + ), + default = p_default, + show_default = False + ) + # Only save if we've changed the default if p_user != p_default: # Convert string bools to real bools @@ -260,7 +310,7 @@ def build_command(self, params = None): def __create_nfx_params_file(cls, params): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") - json_string = pms.Parameters.in_nextflow_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(params, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file @@ -268,7 +318,7 @@ def __create_nfx_params_file(cls, params): @classmethod def __write_params_as_full_json(cls, params, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") - json_string = pms.Parameters.in_full_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_full_json(params, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file diff --git a/nf_core/utils.py b/nf_core/utils.py index 7c7cdabab0..ddf0fc15e2 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,36 +5,10 @@ import datetime import os -import requests +import requests import subprocess import tempfile -PARAMETERS_URI_TEMPL = "https://raw.githubusercontent.com/nf-core/{pipeline}/master/parameters.settings.json" - - -def fetch_parameter_settings_from_github(pipeline): - """Requests the pipeline parameter settings from Github. - - Args: - pipeline (str): The nf-core pipeline name. - - Returns: - str: The raw JSON string of the file with the parameter settings. - - Raises: - LookupError: If for some reason the URI cannot be accessed. - """ - import requests_cache - - target_uri = PARAMETERS_URI_TEMPL.format(pipeline=pipeline) - try: - with requests_cache.disabled(): - result = requests.get(target_uri, headers={'Cache-Control': 'no-cache'}) - except (ConnectionError, TimeoutError) as e: - raise LookupError(e) - return result.text - - def fetch_wf_config(wf_path): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. diff --git a/scripts/nf-core b/scripts/nf-core index fd692fa474..8b7f95c985 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -124,15 +124,12 @@ def download(pipeline, release, singularity, outdir): ) @click.option( '-p', '--params', - is_flag = False, - default = "", type = str, help = "Local parameter settings file in JSON." ) @click.option( - '--direct', + '-d', '--direct', is_flag = True, - default = False, help = "Uses given values from the parameter file directly." ) def launch(pipeline, params, direct): From b1e86dcc025fd073c4c7ba07724a1a184ac1fba3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:16:03 +0100 Subject: [PATCH 03/10] Code testing on old pipelines, refactoring --- nf_core/launch.py | 236 ++++++++++++++++----------------- nf_core/workflow/parameters.py | 7 +- nf_core/workflow/validation.py | 6 +- 3 files changed, 120 insertions(+), 129 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 454c55c485..cafd84e13a 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -6,58 +6,37 @@ import click import logging import os +import re import subprocess import nf_core.utils, nf_core.list import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow def launch_pipeline(workflow, params_local_uri, direct): + + # Create a pipeline launch object launcher = Launch(workflow) - params_list = [] # Get nextflow to fetch the workflow if we don't already have it - if not launcher.local_wf: - logging.info("Downloading workflow: {}".format(launcher.workflow)) - try: - with open(os.devnull, 'w') as devnull: - nfconfig_raw = subprocess.check_output(['nextflow', 'pull', launcher.workflow], stderr=devnull) - except OSError as e: - if e.errno == os.errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - launcher.local_wf = nf_core.list.LocalWorkflow(launcher.workflow) + launcher.get_local_wf() # Get the pipeline default parameters - try: - params_json_str = None - # Params file supplied to launch command - if params_local_uri: - with open(params_local_uri, 'r') as fp: - params_json_str = fp.read() - # Get workflow file from local cached copy - else: - local_params_path = os.path.join(launcher.local_wf.local_path, 'parameters.settings.json') - if os.path.exists(local_params_path): - with open(params_local_uri, 'r') as fp: - params_json_str = fp.read() - if not params_json_str: - raise LookupError - params_list = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) - except LookupError as e: - print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( - pipeline=launcher.workflow, exception=e)) + launcher.parse_parameter_settings(params_local_uri) # Fallback if parameters.settings.json not found, calls Nextflow's config command - if not params_list: - launcher.collect_defaults() + if len(launcher.parameters) == 0: + launcher.collect_pipeline_param_defaults() + + # Group the parameters + launcher.group_parameters() # Kick off the interactive wizard to collect user inputs - launcher.prompt_vars(params_list, direct) + launcher.prompt_core_nxf_flags() + if not direct: + launcher.prompt_param_flags() # Build and launch the `nextflow run` command - launcher.build_command(params_list) + launcher.build_command() launcher.launch_workflow() class Launch(object): @@ -96,19 +75,74 @@ def __init__(self, workflow): '-resume': 'Resume a previous workflow run' } self.nxf_flags = {} - self.param_defaults = {} - self.params = {} + self.parameters = [] + self.grouped_parameters = {} + self.params_user = {} self.nextflow_cmd = "nextflow run {}".format(self.workflow) + self.use_params_file = True - def collect_defaults(self): + def get_local_wf(self): + """ + Check if this workflow has a local copy and use nextflow to pull it if not + """ + if not self.local_wf: + logging.info("Downloading workflow: {}".format(self.workflow)) + try: + with open(os.devnull, 'w') as devnull: + nfconfig_raw = subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) + else: + self.local_wf = nf_core.list.LocalWorkflow(self.workflow) + + def parse_parameter_settings(self, params_local_uri = None): + """ + Load full parameter info from the pipeline parameters.settings.json file + """ + try: + params_json_str = None + # Params file supplied to launch command + if params_local_uri: + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + # Get workflow file from local cached copy + else: + local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') + if os.path.exists(local_params_path): + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + if not params_json_str: + raise LookupError('parameters.settings.json file not found') + self.parameters = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) + except LookupError as e: + print("WARNING: Could not parse parameter settings file for `{pipeline}`:\n {exception}".format( + pipeline=self.workflow, exception=e)) + + def collect_pipeline_param_defaults(self): """ Collect the default params and values from the workflow """ + logging.info("Collecting pipeline parameter defaults\n") config = nf_core.utils.fetch_wf_config(self.workflow) for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: - self.param_defaults[keys[1]] = value - - def prompt_vars(self, params = None, direct = False): + parameter = (nf_core.workflow.parameters.Parameter.builder() + .name(keys[1]) + .label(None) + .usage(None) + .param_type("string") + .choices(None) + .default(str(value)) + .pattern(".*") + .render(None) + .arity(None) + .group("Pipeline parameters") + .build()) + self.parameters.append(parameter) + + def prompt_core_nxf_flags(self): """ Ask the user if they want to override any default values """ # Main nextflow flags click.secho("Main nextflow options", bold=True, underline=True) @@ -146,43 +180,7 @@ def prompt_vars(self, params = None, direct = False): pass self.nxf_flags[flag] = f_user - # Uses the parameter values from the JSON file - # and does not ask the user to set them explicitly - if direct: - return - - # Pipeline params - if params: - Launch.__prompt_defaults_from_param_objects( - Launch.__group_parameters(params) - ) - return - for param, p_default in self.param_defaults.items(): - if not isinstance(p_default, dict) and not isinstance(p_default, list): - - # Prompt for a response - p_user = click.prompt( - "\n --{} {}".format( - click.style(param, fg='blue'), - click.style('[{}]'.format(str(p_default)), fg='green') - ), - default = p_default, - show_default = False - ) - - # Only save if we've changed the default - if p_user != p_default: - # Convert string bools to real bools - try: - p_user = p_user.strip('"').strip("'") - if p_user.lower() == 'true': p_user = True - if p_user.lower() == 'false': p_user = False - except AttributeError: - pass - self.params[param] = p_user - - @classmethod - def __group_parameters(cls, parameters): + def group_parameters(self): """Groups parameters by their 'group' property. Args: @@ -191,27 +189,14 @@ def __group_parameters(cls, parameters): Returns: dict: Parameter objects grouped by the `group` property. """ - grouped_parameters = {} - for param in parameters: - if not grouped_parameters.get(param.group): - grouped_parameters[param.group] = [] - grouped_parameters[param.group].append(param) - return grouped_parameters - - @classmethod - def __prompt_defaults_from_param_objects(cls, params_grouped): - """Prompts the user for parameter input values and validates them. - - Args: - params_grouped (dict): A dictionary with parameter group labels - as keys and list of parameters as values. :: - - { - "group1": [param1, param2], - ... - } - """ - for group_label, params in params_grouped.items(): + for param in self.parameters: + if param.group not in self.grouped_parameters.keys(): + self.grouped_parameters[param.group] = [] + self.grouped_parameters[param.group].append(param) + + def prompt_param_flags(self): + """ Prompts the user for parameter input values and validates them. """ + for group_label, params in self.grouped_parameters.items(): click.echo("\n\n{}{}".format( click.style('Parameter group: ', bold=True, underline=True), click.style(group_label, bold=True, underline=True, fg='red') @@ -226,10 +211,12 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): first_attempt = True while not value_is_valid: # Start building the string to show to the user - label and usage - plines = ['', - click.style(parameter.label, bold=True), - click.style(parameter.usage, dim=True) - ] + plines = [''] + if parameter.label: + plines.append(click.style(parameter.label, bold=True)) + if parameter.usage: + plines.append(click.style(parameter.usage, dim=True)) + # Add the choices / range if applicable if parameter.choices: rc = 'Choices' if parameter.type == 'string' else 'Range' @@ -244,6 +231,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): # Final line to print - command and default flag_prompt = click.style(' --{} '.format(parameter.name), fg='blue') + \ click.style('[{}]'.format(pdef_val), fg='green') + # Only show this final prompt if we're trying again if first_attempt: plines.append(flag_prompt) @@ -265,6 +253,8 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): parameter.value = int(parameter.value) elif parameter.type == "decimal": parameter.value = float(parameter.value) + else: + parameter.value = str(parameter.value) # Validate the input try: @@ -276,7 +266,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): else: value_is_valid = True - def build_command(self, params = None): + def build_command(self): """ Build the nextflow run command based on what we know """ for flag, val in self.nxf_flags.items(): # Boolean flags like -resume @@ -289,36 +279,36 @@ def build_command(self, params = None): else: self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) - if params: # When a parameter specification file was used, we can run Nextflow with it - path = Launch.__create_nfx_params_file(params) + # Write the user selection to a file and run nextflow with that + if self.use_params_file: + path = self.create_nfx_params_file() self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) - Launch.__write_params_as_full_json(params) - return + self.write_params_as_full_json() - for param, val in self.params.items(): - # Boolean flags like --saveTrimmed - if isinstance(val, bool): - if val: - self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + # Call nextflow with a list of command line flags + else: + for param, val in self.params_user.items(): + # Boolean flags like --saveTrimmed + if isinstance(val, bool): + if val: + self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + else: + logging.error("Can't set false boolean flags.") + # everything else else: - logging.warn("TODO: Can't set false boolean flags currently.") - # everything else - else: - self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) + self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) - @classmethod - def __create_nfx_params_file(cls, params): + def create_nfx_params_file(self): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") - json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file - @classmethod - def __write_params_as_full_json(cls, params, outdir = os.getcwd()): + def write_params_as_full_json(self, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") - json_string = nf_core.workflow.parameters.Parameters.in_full_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_full_json(self.parameters, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 179344bf49..c56b891d9c 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -11,7 +11,6 @@ NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" - class Parameters: """Contains a static factory method for :class:`Parameter` object creation. @@ -39,7 +38,8 @@ def create_from_json(parameters_json, schema=""): properties = json.loads(parameters_json) parameters = [] for param in properties.get("parameters"): - parameter = (Parameter.builder().name(param.get("name")) + parameter = (Parameter.builder() + .name(param.get("name")) .label(param.get("label")) .usage(param.get("usage")) .param_type(param.get("type")) @@ -66,7 +66,8 @@ def in_nextflow_json(parameters, indent=0): """ params = {} for p in parameters: - params[p.name] = p.value if p.value else p.default_value + if p.value and p.value != p.default_value: + params[p.name] = p.value return json.dumps(params, indent=indent) @staticmethod diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index f321784553..c565af8458 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -111,12 +111,12 @@ def validate(self): """ value = self._param.value if not isinstance(value, str): - raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type String, but was {}" .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: - raise AttributeError("Can't validate value for parameter '{}'," \ + raise AttributeError("Can't validate value for parameter '{}', " \ "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) result = re.match(self._param.pattern, self._param.value) if not result: @@ -180,5 +180,5 @@ def validate(self): """ value = self._param.value if not isinstance(self._param.value, float): - raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type Decimal, but was {}" .format(value, self._param.name, type(value))) From 0d6ea8721682b10da36f43998424af07aa2c1d64 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:34:26 +0100 Subject: [PATCH 04/10] Try to guess types when handling param defaults --- nf_core/launch.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index cafd84e13a..738a416a77 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -49,7 +49,7 @@ def __init__(self, workflow): if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): workflow = "nf-core/{}".format(workflow) logging.debug("Prepending nf-core/ to workflow") - logging.info("Launching {}\n".format(workflow)) + logging.info("Launching {}".format(workflow)) # Get local workflows to see if we have a cached version self.local_wf = None @@ -97,6 +97,7 @@ def get_local_wf(self): raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: self.local_wf = nf_core.list.LocalWorkflow(self.workflow) + self.local_wf.get_local_nf_workflow_details() def parse_parameter_settings(self, params_local_uri = None): """ @@ -112,7 +113,7 @@ def parse_parameter_settings(self, params_local_uri = None): else: local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') if os.path.exists(local_params_path): - with open(params_local_uri, 'r') as fp: + with open(local_params_path, 'r') as fp: params_json_str = fp.read() if not params_json_str: raise LookupError('parameters.settings.json file not found') @@ -128,15 +129,36 @@ def collect_pipeline_param_defaults(self): for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: + + # Try to guess the variable type from the default value + p_type = 'string' + p_default = str(value) + # All digits - int + if value.isdigit(): + p_type = 'integer' + p_default = int(value) + else: + # Not just digis - try converting to a float + try: + p_default = float(value) + p_type = 'decimal' + except ValueError: + pass + # Strings 'true' and 'false' - booleans + if value == 'true' or value == 'false': + p_type = 'boolean' + p_default = True if value == 'true' else False + + # Build the Parameter object parameter = (nf_core.workflow.parameters.Parameter.builder() .name(keys[1]) .label(None) .usage(None) - .param_type("string") + .param_type(p_type) .choices(None) - .default(str(value)) + .default(p_default) .pattern(".*") - .render(None) + .render("textfield") .arity(None) .group("Pipeline parameters") .build()) From fab91b807ea737216b418ab31fb48b3617a9845f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 00:50:31 +0100 Subject: [PATCH 05/10] Fake a change in the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d1e3784e9..78fa358116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ pipelines from being synced if necessary. * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary. * Added nf-core tools API description to assist developers with the classes and functions available. * Docs are automatically built by Travis CI and updated on the nf-co.re website. -* Introduced test for filtering remote workflows by keyword +* Introduced test for filtering remote workflows by keyword. ## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 Tantalum Butterfly From 4b245aeb168a400db5908ea257558e06bad476ab Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:34:51 +0100 Subject: [PATCH 06/10] Fix LGTM warnings --- nf_core/launch.py | 3 +-- nf_core/utils.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 738a416a77..c889210806 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -6,7 +6,6 @@ import click import logging import os -import re import subprocess import nf_core.utils, nf_core.list @@ -89,7 +88,7 @@ def get_local_wf(self): logging.info("Downloading workflow: {}".format(self.workflow)) try: with open(os.devnull, 'w') as devnull: - nfconfig_raw = subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) + subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) except OSError as e: if e.errno == os.errno.ENOENT: raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") diff --git a/nf_core/utils.py b/nf_core/utils.py index ddf0fc15e2..ec9f807ba5 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,7 +5,6 @@ import datetime import os -import requests import subprocess import tempfile From ef1546aa8935e28ca9522b6daf0472ceddceaed7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:36:39 +0100 Subject: [PATCH 07/10] =?UTF-8?q?Changelog=20=F0=9F=A4=A6=F0=9F=8F=BB?= =?UTF-8?q?=E2=80=8D=E2=99=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f55130e6..8ef9d840fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary. * Added nf-core tools API description to assist developers with the classes and functions available. * Docs are automatically built by Travis CI and updated on the nf-co.re website. -* Introduced test for filtering remote workflows by keyword +* Introduced test for filtering remote workflows by keyword. * Build tools python API docs * Use Travis job for api doc generation and publish * Bump `conda` to 4.5.12 in base nf-core Dockerfile From 5d430a9e31e80d8eb99d228bcfa8a1fea2514c00 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 17:57:22 +0100 Subject: [PATCH 08/10] Testing and bugfixing --- nf_core/launch.py | 2 +- nf_core/workflow/validation.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index c889210806..7ec8a36fd7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -274,7 +274,7 @@ def prompt_param_flags(self): parameter.value = int(parameter.value) elif parameter.type == "decimal": parameter.value = float(parameter.value) - else: + elif parameter.type == "string": parameter.value = str(parameter.value) # Validate the input diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index c565af8458..fa8ac8ee3f 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -78,15 +78,13 @@ def validate(self): if not isinstance(value, int): raise AttributeError("The value {} for parameter {} needs to be an Integer, but was a {}" .format(value, self._param.name, type(value))) - choices = sorted([x for x in self._param.choices]) - print(choices) - if not choices: - return - if len(choices) < 2: - raise AttributeError("The property 'choices' must have at least two entries.") - if not (value >= choices[0] and value <= choices[-1]): - raise AttributeError("'{}' must be within the range [{},{}]" - .format(self._param.name, choices[0], choices[-1])) + if self._param.choices: + choices = sorted([x for x in self._param.choices]) + if len(choices) < 2: + raise AttributeError("The property 'choices' must have at least two entries.") + if not (value >= choices[0] and value <= choices[-1]): + raise AttributeError("'{}' must be within the range [{},{}]" + .format(self._param.name, choices[0], choices[-1])) class StringValidator(Validator): From 0d4ac5b5275f216ee2cfb3b9e9d8518505698361 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 18:57:58 +0100 Subject: [PATCH 09/10] utils.fetch_wf_config - save a json cache --- nf_core/launch.py | 4 ++-- nf_core/utils.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 7ec8a36fd7..0e66014d88 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -123,8 +123,8 @@ def parse_parameter_settings(self, params_local_uri = None): def collect_pipeline_param_defaults(self): """ Collect the default params and values from the workflow """ - logging.info("Collecting pipeline parameter defaults\n") - config = nf_core.utils.fetch_wf_config(self.workflow) + logging.debug("Collecting pipeline parameter defaults\n") + config = nf_core.utils.fetch_wf_config(self.workflow, self.local_wf) for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: diff --git a/nf_core/utils.py b/nf_core/utils.py index ec9f807ba5..227bc903ed 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -4,11 +4,13 @@ """ import datetime +import json +import logging import os import subprocess import tempfile -def fetch_wf_config(wf_path): +def fetch_wf_config(wf_path, wf=None): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -20,6 +22,27 @@ def fetch_wf_config(wf_path): """ config = dict() + cache_fn = None + cache_basedir = None + cache_path = None + + # Build a cache directory if we can + if os.path.isdir(os.path.join(os.getenv("HOME"), '.nextflow')): + cache_basedir = os.path.join(os.getenv("HOME"), '.nextflow', 'nf-core') + if not os.path.isdir(cache_basedir): + os.mkdir(cache_basedir) + + # If we're given a workflow object with a commit, see if we have a cached copy + if cache_basedir and wf and wf.full_name and wf.commit_sha: + cache_fn = '{}-{}.json'.format(wf.full_name.replace(os.path.sep, '-'), wf.commit_sha) + cache_path = os.path.join(cache_basedir, cache_fn) + if os.path.isfile(cache_path): + logging.debug("Found a config cache, loading: {}".format(cache_path)) + with open(cache_path, 'r') as fh: + config = json.load(fh) + return config + + # Call `nextflow config` and pipe stderr to /dev/null try: with open(os.devnull, 'w') as devnull: @@ -34,6 +57,13 @@ def fetch_wf_config(wf_path): ul = l.decode('utf-8') k, v = ul.split(' = ', 1) config[k] = v + + # If we can, save a cached copy + if cache_path: + logging.debug("Saving config cache: {}".format(cache_path)) + with open(cache_path, 'w') as fh: + json.dump(config, fh, indent=4) + return config From 9220525692796cd0d24b9e74e8ec99f56766bcd2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 7 Feb 2019 19:10:09 +0100 Subject: [PATCH 10/10] Make the launch command work with local directories --- nf_core/launch.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 0e66014d88..998c6e0abc 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,7 +17,8 @@ def launch_pipeline(workflow, params_local_uri, direct): launcher = Launch(workflow) # Get nextflow to fetch the workflow if we don't already have it - launcher.get_local_wf() + if not launcher.wf_ispath: + launcher.get_local_wf() # Get the pipeline default parameters launcher.parse_parameter_settings(params_local_uri) @@ -44,19 +45,23 @@ class Launch(object): def __init__(self, workflow): """ Initialise the class with empty placeholder vars """ + # Check if the workflow name is actually a path + self.wf_ispath = os.path.exists(workflow) + # Prepend nf-core/ if it seems sensible - if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): + if 'nf-core' not in workflow and workflow.count('/') == 0 and not self.wf_ispath: workflow = "nf-core/{}".format(workflow) logging.debug("Prepending nf-core/ to workflow") logging.info("Launching {}".format(workflow)) - # Get local workflows to see if we have a cached version + # Get list of local workflows to see if we have a cached version self.local_wf = None - wfs = nf_core.list.Workflows() - wfs.get_local_nf_workflows() - for wf in wfs.local_workflows: - if workflow == wf.full_name: - self.local_wf = wf + if not self.wf_ispath: + wfs = nf_core.list.Workflows() + wfs.get_local_nf_workflows() + for wf in wfs.local_workflows: + if workflow == wf.full_name: + self.local_wf = wf self.workflow = workflow self.nxf_flag_defaults = { @@ -110,7 +115,10 @@ def parse_parameter_settings(self, params_local_uri = None): params_json_str = fp.read() # Get workflow file from local cached copy else: - local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') + if self.wf_ispath: + local_params_path = os.path.join(self.workflow, 'parameters.settings.json') + else: + local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') if os.path.exists(local_params_path): with open(local_params_path, 'r') as fp: params_json_str = fp.read() @@ -303,7 +311,8 @@ def build_command(self): # Write the user selection to a file and run nextflow with that if self.use_params_file: path = self.create_nfx_params_file() - self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) + if path is not None: + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) self.write_params_as_full_json() # Call nextflow with a list of command line flags @@ -323,6 +332,8 @@ def create_nfx_params_file(self): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4) + if json_string == '{}': + return None with open(output_file, "w") as fp: fp.write(json_string) return output_file