nf-core · sven1103 · Feb 7, 2019 · Feb 6, 2019 · Feb 6, 2019 · Feb 6, 2019
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,7 +18,7 @@
 * Syncing now reads from a `blacklist.json` in order to exclude pipelines from being synced if necessary.
 * Added nf-core tools API description to assist developers with the classes and functions available.
   * Docs are automatically built by Travis CI and updated on the nf-co.re website.
-* Introduced test for filtering remote workflows by keyword
+* Introduced test for filtering remote workflows by keyword.
 * Build tools python API docs
   * Use Travis job for api doc generation and publish
 * Bump `conda` to 4.5.12 in base nf-core Dockerfile

diff --git a/nf_core/launch.py b/nf_core/launch.py
@@ -8,25 +8,35 @@
 import os
 import subprocess
 
-import nf_core.utils
-from nf_core.workflow import parameters as pms
+import nf_core.utils, nf_core.list
+import nf_core.workflow.parameters, nf_core.workflow.validation, nf_core.workflow.workflow
 
 def launch_pipeline(workflow, params_local_uri, direct):
+
+    # Create a pipeline launch object
     launcher = Launch(workflow)
-    params_list = []
-    try:
-        if params_local_uri:
-            with open(params_local_uri, 'r') as fp: params_json_str = fp.read()
-        else:
-            params_json_str = nf_core.utils.fetch_parameter_settings_from_github(workflow)
-        params_list = pms.Parameters.create_from_json(params_json_str)
-    except LookupError as e:
-        print("WARNING: No parameter settings file found for `{pipeline}`.\n{exception}".format(
-            pipeline=workflow, exception=e))
-    if not params_list:
-        launcher.collect_defaults()  # Fallback, calls Nextflow's config command
-    launcher.prompt_vars(params_list, direct)
-    launcher.build_command(params_list)
+
+    # Get nextflow to fetch the workflow if we don't already have it
+    if not launcher.wf_ispath:
+        launcher.get_local_wf()
+
+    # Get the pipeline default parameters
+    launcher.parse_parameter_settings(params_local_uri)
+
+    # Fallback if parameters.settings.json not found, calls Nextflow's config command
+    if len(launcher.parameters) == 0:
+        launcher.collect_pipeline_param_defaults()
+
+    # Group the parameters
+    launcher.group_parameters()
+
+    # Kick off the interactive wizard to collect user inputs
+    launcher.prompt_core_nxf_flags()
+    if not direct:
+        launcher.prompt_param_flags()
+
+    # Build and launch the `nextflow run` command
+    launcher.build_command()
     launcher.launch_workflow()
 
 class Launch(object):
@@ -35,11 +45,23 @@ class Launch(object):
     def __init__(self, workflow):
         """ Initialise the class with empty placeholder vars """
 
+        # Check if the workflow name is actually a path
+        self.wf_ispath = os.path.exists(workflow)
+
         # Prepend nf-core/ if it seems sensible
-        if 'nf-core' not in workflow and workflow.count('/') == 0 and not os.path.exists(workflow):
+        if 'nf-core' not in workflow and workflow.count('/') == 0 and not self.wf_ispath:
             workflow = "nf-core/{}".format(workflow)
             logging.debug("Prepending nf-core/ to workflow")
-        logging.info("Launching {}\n".format(workflow))
+        logging.info("Launching {}".format(workflow))
+
+        # Get list of local workflows to see if we have a cached version
+        self.local_wf = None
+        if not self.wf_ispath:
+            wfs = nf_core.list.Workflows()
+            wfs.get_local_nf_workflows()
+            for wf in wfs.local_workflows:
+                if workflow == wf.full_name:
+                    self.local_wf = wf
 
         self.workflow = workflow
         self.nxf_flag_defaults = {
@@ -57,19 +79,99 @@ def __init__(self, workflow):
             '-resume': 'Resume a previous workflow run'
         }
         self.nxf_flags = {}
-        self.param_defaults = {}
-        self.params = {}
+        self.parameters = []
+        self.grouped_parameters = {}
+        self.params_user = {}
         self.nextflow_cmd = "nextflow run {}".format(self.workflow)
+        self.use_params_file = True
 
-    def collect_defaults(self):
+    def get_local_wf(self):
+        """
+        Check if this workflow has a local copy and use nextflow to pull it if not
+        """
+        if not self.local_wf:
+            logging.info("Downloading workflow: {}".format(self.workflow))
+            try:
+                with open(os.devnull, 'w') as devnull:
+                    subprocess.check_output(['nextflow', 'pull', self.workflow], stderr=devnull)
+            except OSError as e:
+                if e.errno == os.errno.ENOENT:
+                    raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.")
+            except subprocess.CalledProcessError as e:
+                raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n   %s", e.returncode, e.output)
+            else:
+                self.local_wf = nf_core.list.LocalWorkflow(self.workflow)
+                self.local_wf.get_local_nf_workflow_details()
+
+    def parse_parameter_settings(self, params_local_uri = None):
+        """
+        Load full parameter info from the pipeline parameters.settings.json file
+        """
+        try:
+            params_json_str = None
+            # Params file supplied to launch command
+            if params_local_uri:
+                with open(params_local_uri, 'r') as fp:
+                    params_json_str = fp.read()
+            # Get workflow file from local cached copy
+            else:
+                if self.wf_ispath:
+                    local_params_path = os.path.join(self.workflow, 'parameters.settings.json')
+                else:
+                    local_params_path = os.path.join(self.local_wf.local_path, 'parameters.settings.json')
+                if os.path.exists(local_params_path):
+                    with open(local_params_path, 'r') as fp:
+                        params_json_str = fp.read()
+            if not params_json_str:
+                raise LookupError('parameters.settings.json file not found')
+            self.parameters = nf_core.workflow.parameters.Parameters.create_from_json(params_json_str)
+        except LookupError as e:
+            print("WARNING: Could not parse parameter settings file for `{pipeline}`:\n  {exception}".format(
+                pipeline=self.workflow, exception=e))
+
+    def collect_pipeline_param_defaults(self):
         """ Collect the default params and values from the workflow """
-        config = nf_core.utils.fetch_wf_config(self.workflow)
+        logging.debug("Collecting pipeline parameter defaults\n")
+        config = nf_core.utils.fetch_wf_config(self.workflow, self.local_wf)
         for key, value in config.items():
             keys = key.split('.')
             if keys[0] == 'params' and len(keys) == 2:
-                self.param_defaults[keys[1]] = value
 
-    def prompt_vars(self, params = None, direct = False):
+                # Try to guess the variable type from the default value
+                p_type = 'string'
+                p_default = str(value)
+                # All digits - int
+                if value.isdigit():
+                    p_type = 'integer'
+                    p_default = int(value)
+                else:
+                    # Not just digis - try converting to a float
+                    try:
+                        p_default = float(value)
+                        p_type = 'decimal'
+                    except ValueError:
+                        pass
+                # Strings 'true' and 'false' - booleans
+                if value == 'true' or value == 'false':
+                    p_type = 'boolean'
+                    p_default = True if value == 'true' else False
+
+                # Build the Parameter object
+                parameter = (nf_core.workflow.parameters.Parameter.builder()
+                             .name(keys[1])
+                             .label(None)
+                             .usage(None)
+                             .param_type(p_type)
+                             .choices(None)
+                             .default(p_default)
+                             .pattern(".*")
+                             .render("textfield")
+                             .arity(None)
+                             .group("Pipeline parameters")
+                             .build())
+                self.parameters.append(parameter)
+
+    def prompt_core_nxf_flags(self):
         """ Ask the user if they want to override any default values """
         # Main nextflow flags
         click.secho("Main nextflow options", bold=True, underline=True)
@@ -95,6 +197,7 @@ def prompt_vars(self, params = None, direct = False):
                 default = f_default,
                 show_default = False
             )
+
             # Only save if we've changed the default
             if f_user != f_default:
                 # Convert string bools to real bools
@@ -106,33 +209,7 @@ def prompt_vars(self, params = None, direct = False):
                     pass
                 self.nxf_flags[flag] = f_user
 
-        # Uses the parameter values from the JSON file
-        # and does not ask the user to set them explicitly
-        if direct:
-            return
-
-        # Pipeline params
-        if params:
-            Launch.__prompt_defaults_from_param_objects(
-                Launch.__group_parameters(params)
-                )
-            return
-        for param, p_default in self.param_defaults.items():
-            if not isinstance(p_default, dict) and not isinstance(p_default, list):
-                p_user = click.prompt("--{}".format(param), default=p_default)
-                # Only save if we've changed the default
-                if p_user != p_default:
-                    # Convert string bools to real bools
-                    try:
-                        p_user = p_user.strip('"').strip("'")
-                        if p_user.lower() == 'true': p_user = True
-                        if p_user.lower() == 'false': p_user = False
-                    except AttributeError:
-                        pass
-                    self.params[param] = p_user
-
-    @classmethod
-    def __group_parameters(cls, parameters):
+    def group_parameters(self):
         """Groups parameters by their 'group' property.
 
         Args:
@@ -141,27 +218,14 @@ def __group_parameters(cls, parameters):
         Returns:
             dict: Parameter objects grouped by the `group` property.
         """
-        grouped_parameters = {}
-        for param in parameters:
-            if not grouped_parameters.get(param.group):
-                grouped_parameters[param.group] = []
-            grouped_parameters[param.group].append(param)
-        return grouped_parameters
-
-    @classmethod
-    def __prompt_defaults_from_param_objects(cls, params_grouped):
-        """Prompts the user for parameter input values and validates them.
-
-        Args:
-            params_grouped (dict): A dictionary with parameter group labels
-                as keys and list of parameters as values. ::
-
-                {
-                    "group1": [param1, param2],
-                    ...
-                }
-        """
-        for group_label, params in params_grouped.items():
+        for param in self.parameters:
+            if param.group not in self.grouped_parameters.keys():
+                self.grouped_parameters[param.group] = []
+            self.grouped_parameters[param.group].append(param)
+
+    def prompt_param_flags(self):
+        """ Prompts the user for parameter input values and validates them. """
+        for group_label, params in self.grouped_parameters.items():
             click.echo("\n\n{}{}".format(
                 click.style('Parameter group: ', bold=True, underline=True),
                 click.style(group_label, bold=True, underline=True, fg='red')
@@ -176,10 +240,12 @@ def __prompt_defaults_from_param_objects(cls, params_grouped):
                 first_attempt = True
                 while not value_is_valid:
                     # Start building the string to show to the user - label and usage
-                    plines = ['',
-                        click.style(parameter.label, bold=True),
-                        click.style(parameter.usage, dim=True)
-                    ]
+                    plines = ['']
+                    if parameter.label:
+                        plines.append(click.style(parameter.label, bold=True))
+                    if parameter.usage:
+                        plines.append(click.style(parameter.usage, dim=True))
+
                     # Add the choices / range if applicable
                     if parameter.choices:
                         rc = 'Choices' if parameter.type == 'string' else 'Range'
@@ -194,6 +260,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped):
                     # Final line to print - command and default
                     flag_prompt = click.style(' --{} '.format(parameter.name), fg='blue') + \
                         click.style('[{}]'.format(pdef_val), fg='green')
+
                     # Only show this final prompt if we're trying again
                     if first_attempt:
                         plines.append(flag_prompt)
@@ -215,6 +282,8 @@ def __prompt_defaults_from_param_objects(cls, params_grouped):
                         parameter.value = int(parameter.value)
                     elif parameter.type == "decimal":
                         parameter.value = float(parameter.value)
+                    elif parameter.type == "string":
+                        parameter.value = str(parameter.value)
 
                     # Validate the input
                     try:
@@ -226,7 +295,7 @@ def __prompt_defaults_from_param_objects(cls, params_grouped):
                     else:
                         value_is_valid = True
 
-    def build_command(self, params = None):
+    def build_command(self):
         """ Build the nextflow run command based on what we know """
         for flag, val in self.nxf_flags.items():
             # Boolean flags like -resume
@@ -239,36 +308,39 @@ def build_command(self, params = None):
             else:
                 self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, flag, val.replace('"', '\\"'))
 
-        if params:  # When a parameter specification file was used, we can run Nextflow with it
-            path = Launch.__create_nfx_params_file(params)
-            self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path)
-            Launch.__write_params_as_full_json(params)
-            return
+        # Write the user selection to a file and run nextflow with that
+        if self.use_params_file:
+            path = self.create_nfx_params_file()
+            if path is not None:
+                self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path)
+            self.write_params_as_full_json()
 
-        for param, val in self.params.items():
-            # Boolean flags like --saveTrimmed
-            if isinstance(val, bool):
-                if val:
-                    self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param)
+        # Call nextflow with a list of command line flags
+        else:
+            for param, val in self.params_user.items():
+                # Boolean flags like --saveTrimmed
+                if isinstance(val, bool):
+                    if val:
+                        self.nextflow_cmd = "{} --{}".format(self.nextflow_cmd, param)
+                    else:
+                        logging.error("Can't set false boolean flags.")
+                # everything else
                 else:
-                    logging.warn("TODO: Can't set false boolean flags currently.")
-            # everything else
-            else:
-                self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"'))
+                    self.nextflow_cmd = '{} --{} "{}"'.format(self.nextflow_cmd, param, val.replace('"', '\\"'))
 
-    @classmethod
-    def __create_nfx_params_file(cls, params):
+    def create_nfx_params_file(self):
         working_dir = os.getcwd()
         output_file = os.path.join(working_dir, "nfx-params.json")
-        json_string = pms.Parameters.in_nextflow_json(params, indent=4)
+        json_string = nf_core.workflow.parameters.Parameters.in_nextflow_json(self.parameters, indent=4)
+        if json_string == '{}':
+            return None
         with open(output_file, "w") as fp:
             fp.write(json_string)
         return output_file
 
-    @classmethod
-    def __write_params_as_full_json(cls, params, outdir = os.getcwd()):
+    def write_params_as_full_json(self, outdir = os.getcwd()):
         output_file = os.path.join(outdir, "full-params.json")
-        json_string = pms.Parameters.in_full_json(params, indent=4)
+        json_string = nf_core.workflow.parameters.Parameters.in_full_json(self.parameters, indent=4)
         with open(output_file, "w") as fp:
             fp.write(json_string)
         return output_file