diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f55130e6..8ef9d840fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary. * Added nf-core tools API description to assist developers with the classes and functions available. * Docs are automatically built by Travis CI and updated on the nf-co.re website. -* Introduced test for filtering remote workflows by keyword +* Introduced test for filtering remote workflows by keyword. * Build tools python API docs * Use Travis job for api doc generation and publish * Bump `conda` to 4.5.12 in base nf-core Dockerfile diff --git a/nf_core/launch.py b/nf_core/launch.py index 4fc151c0be..998c6e0abc 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -8,25 +8,35 @@ import os import subprocess -import nf_core.utils -from nf_core.workflow import parameters as pms +import nf_core.utils, nf_core.list +import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow def launch_pipeline(workflow, params_local_uri, direct): + + # Create a pipeline launch object launcher = Launch(workflow) - params_list = [] - try: - if params_local_uri: - with open(params_local_uri, 'r') as fp: params_json_str = fp.read() - else: - params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow) - params_list = pms.Parameters.create_from_json(params_json_str) - except LookupError as e: - print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format( - pipeline=workflow, exception=e)) - if not params_list: - launcher.collect_defaults() # Fallback, calls Nextflow's config command - launcher.prompt_vars(params_list, direct) - launcher.build_command(params_list) + + # Get nextflow to fetch the workflow if we don't already have it + if not launcher.wf_ispath: + launcher.get_local_wf() + + # Get the pipeline default parameters + launcher.parse_parameter_settings(params_local_uri) + + # Fallback if parameters.settings.json not found, calls Nextflow's config command + if len(launcher.parameters) == 0: + launcher.collect_pipeline_param_defaults() + + # Group the parameters + launcher.group_parameters() + + # Kick off the interactive wizard to collect user inputs + launcher.prompt_core_nxf_flags() + if not direct: + launcher.prompt_param_flags() + + # Build and launch the `nextflow run` command + launcher.build_command() launcher.launch_workflow() class Launch(object): @@ -35,11 +45,23 @@ class Launch(object): def __init__(self, workflow): """ Initialise the class with empty placeholder vars """ + # Check if the workflow name is actually a path + self.wf_ispath = os.path.exists(workflow) + # Prepend nf-core/ if it seems sensible - if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow): + if 'nf-core' not in workflow and workflow.count('/') == 0 and not self.wf_ispath: workflow = "nf-core/{}".format(workflow) logging.debug("Prepending nf-core/ to workflow") - logging.info("Launching {}\n".format(workflow)) + logging.info("Launching {}".format(workflow)) + + # Get list of local workflows to see if we have a cached version + self.local_wf = None + if not self.wf_ispath: + wfs = nf_core.list.Workflows() + wfs.get_local_nf_workflows() + for wf in wfs.local_workflows: + if workflow == wf.full_name: + self.local_wf = wf self.workflow = workflow self.nxf_flag_defaults = { @@ -57,19 +79,99 @@ def __init__(self, workflow): '-resume': 'Resume a previous workflow run' } self.nxf_flags = {} - self.param_defaults = {} - self.params = {} + self.parameters = [] + self.grouped_parameters = {} + self.params_user = {} self.nextflow_cmd = "nextflow run {}".format(self.workflow) + self.use_params_file = True - def collect_defaults(self): + def get_local_wf(self): + """ + Check if this workflow has a local copy and use nextflow to pull it if not + """ + if not self.local_wf: + logging.info("Downloading workflow: {}".format(self.workflow)) + try: + with open(os.devnull, 'w') as devnull: + subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) + else: + self.local_wf = nf_core.list.LocalWorkflow(self.workflow) + self.local_wf.get_local_nf_workflow_details() + + def parse_parameter_settings(self, params_local_uri = None): + """ + Load full parameter info from the pipeline parameters.settings.json file + """ + try: + params_json_str = None + # Params file supplied to launch command + if params_local_uri: + with open(params_local_uri, 'r') as fp: + params_json_str = fp.read() + # Get workflow file from local cached copy + else: + if self.wf_ispath: + local_params_path = os.path.join(self.workflow, 'parameters.settings.json') + else: + local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json') + if os.path.exists(local_params_path): + with open(local_params_path, 'r') as fp: + params_json_str = fp.read() + if not params_json_str: + raise LookupError('parameters.settings.json file not found') + self.parameters = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str) + except LookupError as e: + print("WARNING: Could not parse parameter settings file for `{pipeline}`:\n {exception}".format( + pipeline=self.workflow, exception=e)) + + def collect_pipeline_param_defaults(self): """ Collect the default params and values from the workflow """ - config = nf_core.utils.fetch_wf_config(self.workflow) + logging.debug("Collecting pipeline parameter defaults\n") + config = nf_core.utils.fetch_wf_config(self.workflow, self.local_wf) for key, value in config.items(): keys = key.split('.') if keys[0] == 'params' and len(keys) == 2: - self.param_defaults[keys[1]] = value - def prompt_vars(self, params = None, direct = False): + # Try to guess the variable type from the default value + p_type = 'string' + p_default = str(value) + # All digits - int + if value.isdigit(): + p_type = 'integer' + p_default = int(value) + else: + # Not just digis - try converting to a float + try: + p_default = float(value) + p_type = 'decimal' + except ValueError: + pass + # Strings 'true' and 'false' - booleans + if value == 'true' or value == 'false': + p_type = 'boolean' + p_default = True if value == 'true' else False + + # Build the Parameter object + parameter = (nf_core.workflow.parameters.Parameter.builder() + .name(keys[1]) + .label(None) + .usage(None) + .param_type(p_type) + .choices(None) + .default(p_default) + .pattern(".*") + .render("textfield") + .arity(None) + .group("Pipeline parameters") + .build()) + self.parameters.append(parameter) + + def prompt_core_nxf_flags(self): """ Ask the user if they want to override any default values """ # Main nextflow flags click.secho("Main nextflow options", bold=True, underline=True) @@ -95,6 +197,7 @@ def prompt_vars(self, params = None, direct = False): default = f_default, show_default = False ) + # Only save if we've changed the default if f_user != f_default: # Convert string bools to real bools @@ -106,33 +209,7 @@ def prompt_vars(self, params = None, direct = False): pass self.nxf_flags[flag] = f_user - # Uses the parameter values from the JSON file - # and does not ask the user to set them explicitly - if direct: - return - - # Pipeline params - if params: - Launch.__prompt_defaults_from_param_objects( - Launch.__group_parameters(params) - ) - return - for param, p_default in self.param_defaults.items(): - if not isinstance(p_default, dict) and not isinstance(p_default, list): - p_user = click.prompt("--{}".format(param), default=p_default) - # Only save if we've changed the default - if p_user != p_default: - # Convert string bools to real bools - try: - p_user = p_user.strip('"').strip("'") - if p_user.lower() == 'true': p_user = True - if p_user.lower() == 'false': p_user = False - except AttributeError: - pass - self.params[param] = p_user - - @classmethod - def __group_parameters(cls, parameters): + def group_parameters(self): """Groups parameters by their 'group' property. Args: @@ -141,27 +218,14 @@ def __group_parameters(cls, parameters): Returns: dict: Parameter objects grouped by the `group` property. """ - grouped_parameters = {} - for param in parameters: - if not grouped_parameters.get(param.group): - grouped_parameters[param.group] = [] - grouped_parameters[param.group].append(param) - return grouped_parameters - - @classmethod - def __prompt_defaults_from_param_objects(cls, params_grouped): - """Prompts the user for parameter input values and validates them. - - Args: - params_grouped (dict): A dictionary with parameter group labels - as keys and list of parameters as values. :: - - { - "group1": [param1, param2], - ... - } - """ - for group_label, params in params_grouped.items(): + for param in self.parameters: + if param.group not in self.grouped_parameters.keys(): + self.grouped_parameters[param.group] = [] + self.grouped_parameters[param.group].append(param) + + def prompt_param_flags(self): + """ Prompts the user for parameter input values and validates them. """ + for group_label, params in self.grouped_parameters.items(): click.echo("\n\n{}{}".format( click.style('Parameter group: ', bold=True, underline=True), click.style(group_label, bold=True, underline=True, fg='red') @@ -176,10 +240,12 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): first_attempt = True while not value_is_valid: # Start building the string to show to the user - label and usage - plines = ['', - click.style(parameter.label, bold=True), - click.style(parameter.usage, dim=True) - ] + plines = [''] + if parameter.label: + plines.append(click.style(parameter.label, bold=True)) + if parameter.usage: + plines.append(click.style(parameter.usage, dim=True)) + # Add the choices / range if applicable if parameter.choices: rc = 'Choices' if parameter.type == 'string' else 'Range' @@ -194,6 +260,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): # Final line to print - command and default flag_prompt = click.style(' --{} '.format(parameter.name), fg='blue') + \ click.style('[{}]'.format(pdef_val), fg='green') + # Only show this final prompt if we're trying again if first_attempt: plines.append(flag_prompt) @@ -215,6 +282,8 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): parameter.value = int(parameter.value) elif parameter.type == "decimal": parameter.value = float(parameter.value) + elif parameter.type == "string": + parameter.value = str(parameter.value) # Validate the input try: @@ -226,7 +295,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped): else: value_is_valid = True - def build_command(self, params = None): + def build_command(self): """ Build the nextflow run command based on what we know """ for flag, val in self.nxf_flags.items(): # Boolean flags like -resume @@ -239,36 +308,39 @@ def build_command(self, params = None): else: self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"')) - if params: # When a parameter specification file was used, we can run Nextflow with it - path = Launch.__create_nfx_params_file(params) - self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) - Launch.__write_params_as_full_json(params) - return + # Write the user selection to a file and run nextflow with that + if self.use_params_file: + path = self.create_nfx_params_file() + if path is not None: + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) + self.write_params_as_full_json() - for param, val in self.params.items(): - # Boolean flags like --saveTrimmed - if isinstance(val, bool): - if val: - self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + # Call nextflow with a list of command line flags + else: + for param, val in self.params_user.items(): + # Boolean flags like --saveTrimmed + if isinstance(val, bool): + if val: + self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param) + else: + logging.error("Can't set false boolean flags.") + # everything else else: - logging.warn("TODO: Can't set false boolean flags currently.") - # everything else - else: - self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) + self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"')) - @classmethod - def __create_nfx_params_file(cls, params): + def create_nfx_params_file(self): working_dir = os.getcwd() output_file = os.path.join(working_dir, "nfx-params.json") - json_string = pms.Parameters.in_nextflow_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4) + if json_string == '{}': + return None with open(output_file, "w") as fp: fp.write(json_string) return output_file - @classmethod - def __write_params_as_full_json(cls, params, outdir = os.getcwd()): + def write_params_as_full_json(self, outdir = os.getcwd()): output_file = os.path.join(outdir, "full-params.json") - json_string = pms.Parameters.in_full_json(params, indent=4) + json_string = nf_core.workflow.parameters.Parameters.in_full_json(self.parameters, indent=4) with open(output_file, "w") as fp: fp.write(json_string) return output_file diff --git a/nf_core/utils.py b/nf_core/utils.py index 7c7cdabab0..227bc903ed 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -4,38 +4,13 @@ """ import datetime +import json +import logging import os -import requests import subprocess import tempfile -PARAMETERS_URI_TEMPL = "https://raw.githubusercontent.com/nf-core/{pipeline}/master/parameters.settings.json" - - -def fetch_parameter_settings_from_github(pipeline): - """Requests the pipeline parameter settings from Github. - - Args: - pipeline (str): The nf-core pipeline name. - - Returns: - str: The raw JSON string of the file with the parameter settings. - - Raises: - LookupError: If for some reason the URI cannot be accessed. - """ - import requests_cache - - target_uri = PARAMETERS_URI_TEMPL.format(pipeline=pipeline) - try: - with requests_cache.disabled(): - result = requests.get(target_uri, headers={'Cache-Control': 'no-cache'}) - except (ConnectionError, TimeoutError) as e: - raise LookupError(e) - return result.text - - -def fetch_wf_config(wf_path): +def fetch_wf_config(wf_path, wf=None): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -47,6 +22,27 @@ def fetch_wf_config(wf_path): """ config = dict() + cache_fn = None + cache_basedir = None + cache_path = None + + # Build a cache directory if we can + if os.path.isdir(os.path.join(os.getenv("HOME"), '.nextflow')): + cache_basedir = os.path.join(os.getenv("HOME"), '.nextflow', 'nf-core') + if not os.path.isdir(cache_basedir): + os.mkdir(cache_basedir) + + # If we're given a workflow object with a commit, see if we have a cached copy + if cache_basedir and wf and wf.full_name and wf.commit_sha: + cache_fn = '{}-{}.json'.format(wf.full_name.replace(os.path.sep, '-'), wf.commit_sha) + cache_path = os.path.join(cache_basedir, cache_fn) + if os.path.isfile(cache_path): + logging.debug("Found a config cache, loading: {}".format(cache_path)) + with open(cache_path, 'r') as fh: + config = json.load(fh) + return config + + # Call `nextflow config` and pipe stderr to /dev/null try: with open(os.devnull, 'w') as devnull: @@ -61,6 +57,13 @@ def fetch_wf_config(wf_path): ul = l.decode('utf-8') k, v = ul.split(' = ', 1) config[k] = v + + # If we can, save a cached copy + if cache_path: + logging.debug("Saving config cache: {}".format(cache_path)) + with open(cache_path, 'w') as fh: + json.dump(config, fh, indent=4) + return config diff --git a/nf_core/workflow/parameters.py b/nf_core/workflow/parameters.py index 2720a98557..c56b891d9c 100644 --- a/nf_core/workflow/parameters.py +++ b/nf_core/workflow/parameters.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import copy import json import requests @@ -9,7 +11,6 @@ NFCORE_PARAMS_SCHEMA_URI = "https://nf-co.re/parameters.schema.json" - class Parameters: """Contains a static factory method for :class:`Parameter` object creation. @@ -37,7 +38,8 @@ def create_from_json(parameters_json, schema=""): properties = json.loads(parameters_json) parameters = [] for param in properties.get("parameters"): - parameter = (Parameter.builder().name(param.get("name")) + parameter = (Parameter.builder() + .name(param.get("name")) .label(param.get("label")) .usage(param.get("usage")) .param_type(param.get("type")) @@ -64,7 +66,8 @@ def in_nextflow_json(parameters, indent=0): """ params = {} for p in parameters: - params[p.name] = p.value if p.value else p.default_value + if p.value and p.value != p.default_value: + params[p.name] = p.value return json.dumps(params, indent=indent) @staticmethod diff --git a/nf_core/workflow/validation.py b/nf_core/workflow/validation.py index 36e0f4c50b..fa8ac8ee3f 100644 --- a/nf_core/workflow/validation.py +++ b/nf_core/workflow/validation.py @@ -1,7 +1,8 @@ +#!/usr/bin/env python + import abc import re import sys -import nf_core.workflow.parameters as pms if sys.version_info >= (3, 4): ABC = abc.ABC @@ -46,9 +47,6 @@ class Validator(ABC): @abc.abstractmethod def __init__(self, parameter): - if not isinstance(parameter, pms.Parameter): - raise (AttributeError("Argument must be of class {}" - .format(pms.Parameter.__class__))) self._param = parameter @abc.abstractmethod @@ -80,15 +78,13 @@ def validate(self): if not isinstance(value, int): raise AttributeError("The value {} for parameter {} needs to be an Integer, but was a {}" .format(value, self._param.name, type(value))) - choices = sorted([x for x in self._param.choices]) - print(choices) - if not choices: - return - if len(choices) < 2: - raise AttributeError("The property 'choices' must have at least two entries.") - if not (value >= choices[0] and value <= choices[-1]): - raise AttributeError("'{}' must be within the range [{},{}]" - .format(self._param.name, choices[0], choices[-1])) + if self._param.choices: + choices = sorted([x for x in self._param.choices]) + if len(choices) < 2: + raise AttributeError("The property 'choices' must have at least two entries.") + if not (value >= choices[0] and value <= choices[-1]): + raise AttributeError("'{}' must be within the range [{},{}]" + .format(self._param.name, choices[0], choices[-1])) class StringValidator(Validator): @@ -113,12 +109,12 @@ def validate(self): """ value = self._param.value if not isinstance(value, str): - raise AttributeError("The value {} for parameter {} needs to be of type Integer, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type String, but was {}" .format(value, self._param.name, type(value))) choices = sorted([x for x in self._param.choices]) if self._param.choices else [] if not choices: if not self._param.pattern: - raise AttributeError("Can't validate value for parameter '{}'," \ + raise AttributeError("Can't validate value for parameter '{}', " \ "because the value for 'choices' and 'pattern' were empty.".format(self._param.value)) result = re.match(self._param.pattern, self._param.value) if not result: @@ -182,5 +178,5 @@ def validate(self): """ value = self._param.value if not isinstance(self._param.value, float): - raise AttributeError("The value {} for parameter {} needs to be of type Boolean, but was {}" + raise AttributeError("The value {} for parameter {} needs to be of type Decimal, but was {}" .format(value, self._param.name, type(value))) diff --git a/nf_core/workflow/workflow.py b/nf_core/workflow/workflow.py index d0b606ba76..ae59c8958e 100644 --- a/nf_core/workflow/workflow.py +++ b/nf_core/workflow/workflow.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from nf_core.workflow.parameters import Parameters @@ -6,12 +8,12 @@ class Workflow(object): Args: name (str): Workflow name. - parameters_json (str): Workflow parameter data in JSON. + parameters_json (str): Workflow parameter data in JSON. """ def __init__(self, name, parameters_json): self.name = name self.parameters = Parameters.create_from_json(parameters_json) - + def in_nextflow_json(self, indent=0): """Converts the Parameter list in a workflow readable parameter JSON file. @@ -28,4 +30,4 @@ def in_full_json(self, indent=0): Returns: str: JSON formatted parameters. """ - return Parameters.in_full_json(self.parameters, indent) \ No newline at end of file + return Parameters.in_full_json(self.parameters, indent) diff --git a/scripts/nf-core b/scripts/nf-core index 14234b88c3..08228b607b 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -124,15 +124,12 @@ def download(pipeline, release, singularity, outdir): ) @click.option( '-p', '--params', - is_flag = False, - default = "", type = str, help = "Local parameter settings file in JSON." ) @click.option( - '--direct', + '-d', '--direct', is_flag = True, - default = False, help = "Uses given values from the parameter file directly." ) def launch(pipeline, params, direct):