From 9388a46dabf4c1afc3c3c66011b57c31ea2fd165 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Aug 2023 13:24:12 +0200 Subject: [PATCH 1/5] remove option --template-yaml from sync command and use .nf-core.yml instead --- .../create-test-lint-wf-template.yml | 2 +- nf_core/__main__.py | 5 ++-- nf_core/create.py | 7 +++++- nf_core/sync.py | 23 ++++++------------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 62ec65bcd5..f4e4d8e528 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -97,7 +97,7 @@ jobs: # Try syncing it before we change anything - name: nf-core sync - run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/ --template-yaml ${{ matrix.TEMPLATE }} + run: nf-core --log-file log.txt sync --dir my-prefix-testpipeline/ # Run code style linting - name: Run Prettier --check diff --git a/nf_core/__main__.py b/nf_core/__main__.py index fd81b6105a..de14879935 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1654,8 +1654,7 @@ def bump_version(new_version, dir, nextflow): @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") -@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") -def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): +def sync(dir, from_branch, pull_request, github_repository, username): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -1675,7 +1674,7 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) + sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username) try: sync_obj.sync() except (SyncException, PullRequestException) as e: diff --git a/nf_core/create.py b/nf_core/create.py index 6b6f1e0b1b..a7ba8d14d8 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -96,10 +96,16 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa Args: template_yaml_path (str): Path to YAML file containing template parameters. """ + # Try reading config file + _, config_yml = nf_core.utils.load_tools_config() + + # Obtain template customization info from template yaml file or `.nf-core.yml` config file try: if template_yaml_path is not None: with open(template_yaml_path, "r") as f: template_yaml = yaml.safe_load(f) + elif "template" in config_yml: + template_yaml = config_yml["template"] else: template_yaml = {} except FileNotFoundError: @@ -169,7 +175,6 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version - _, config_yml = nf_core.utils.load_tools_config() if ( "lint" in config_yml and "nextflow_config" in config_yml["lint"] diff --git a/nf_core/sync.py b/nf_core/sync.py index 32b157b0a4..7c6a95e21b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -11,6 +11,7 @@ import requests import requests_cache import rich +import yaml from git import GitCommandError, InvalidGitRepositoryError import nf_core @@ -52,7 +53,6 @@ class PipelineSync: required_config_vars (list): List of nextflow variables required to make template pipeline gh_username (str): GitHub username gh_repo (str): GitHub repository name - template_yaml (str): Path to template.yml file for pipeline creation settings. """ def __init__( @@ -62,7 +62,6 @@ def __init__( make_pr=False, gh_repo=None, gh_username=None, - template_yaml_path=None, ): """Initialise syncing object""" @@ -80,11 +79,7 @@ def __init__( self.gh_repo = gh_repo self.pr_url = "" - self.template_yaml_path = template_yaml_path - # Save contents of template.yml for using outside of git. - if self.template_yaml_path is not None: - with open(self.template_yaml_path, "r") as template_yaml: - self.template_yaml_cache = template_yaml.read() + self.config_yml_path, self.config_yml = nf_core.utils.load_tools_config() # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api @@ -213,7 +208,7 @@ def delete_template_branch_files(self): # Delete everything log.info("Deleting all files in 'TEMPLATE' branch") for the_file in os.listdir(self.pipeline_dir): - if the_file == ".git" or the_file == self.template_yaml_path: + if the_file == ".git": continue file_path = os.path.join(self.pipeline_dir, the_file) log.debug(f"Deleting {file_path}") @@ -234,10 +229,10 @@ def make_template_pipeline(self): # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) - # Re-write the template yaml from cache which may have been updated - if self.template_yaml_path and self.template_yaml_cache: - with open(self.template_yaml_path, "w") as template_path: - template_path.write(self.template_yaml_cache) + # Re-write the template yaml info from .nf-core.yml config + if "template" in self.config_yml: + with open(self.config_yml_path, "w") as config_path: + yaml.safe_dump(self.config_yml, config_path) try: nf_core.create.PipelineCreate( @@ -248,13 +243,9 @@ def make_template_pipeline(self): force=True, outdir=self.pipeline_dir, author=self.wf_config["manifest.author"].strip('"').strip("'"), - template_yaml_path=self.template_yaml_path, plain=True, ).init_pipeline() except Exception as err: - if self.template_yaml_path: - # If sync fails, remove template_yaml_path before raising error. - os.remove(self.template_yaml_path) # Reset to where you were to prevent git getting messed up. self.repo.git.reset("--hard") raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") From 16ddb51ef858d4de10a7d64caa3bf1eaa862a656 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Aug 2023 15:00:14 +0200 Subject: [PATCH 2/5] throw deprecation warning if --template-yaml is set --- nf_core/__main__.py | 5 +++-- nf_core/sync.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index de14879935..fd81b6105a 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1654,7 +1654,8 @@ def bump_version(new_version, dir, nextflow): @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") -def sync(dir, from_branch, pull_request, github_repository, username): +@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") +def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -1674,7 +1675,7 @@ def sync(dir, from_branch, pull_request, github_repository, username): is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username) + sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) try: sync_obj.sync() except (SyncException, PullRequestException) as e: diff --git a/nf_core/sync.py b/nf_core/sync.py index 7c6a95e21b..d56f7d64c7 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -8,6 +8,7 @@ import shutil import git +import questionary import requests import requests_cache import rich @@ -43,6 +44,7 @@ class PipelineSync: make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes gh_username (str): GitHub username gh_repo (str): GitHub repository name + template_yaml_path (str): Path to template.yml file for pipeline creation settings. DEPRECATED Attributes: pipeline_dir (str): Path to target pipeline directory @@ -62,6 +64,7 @@ def __init__( make_pr=False, gh_repo=None, gh_username=None, + template_yaml_path=None, ): """Initialise syncing object""" @@ -81,6 +84,27 @@ def __init__( self.config_yml_path, self.config_yml = nf_core.utils.load_tools_config() + # Throw deprecation warning if template_yaml_path is set + if template_yaml_path is not None: + log.warning( + f"The `template_yaml_path` argument is deprecated. Saving pipeline creation settings in .nf-core.yml instead. Please remove {template_yaml_path} file." + ) + if "template" in self.config_yml: + overwrite_template = questionary.confirm( + f"A template section already exists in '{self.config_yml_path}'. Do you want to overwrite?", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() + if overwrite_template or "template" not in self.config_yml: + with open(template_yaml_path, "r") as f: + self.config_yml["template"] = yaml.safe_load(f) + with open(self.config_yml_path, "w") as fh: + yaml.safe_dump(self.config_yml, fh) + log.info(f"Saved pipeline creation settings to '{self.config_yml_path}'") + raise SystemExit( + f"Please commit your changes and delete {template_yaml_path} file. Then run the sync command again." + ) + # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api self.gh_api.lazy_init() From 7787f4b3bb0477c99197ea3cbe2a4b38647e5a35 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Aug 2023 15:03:39 +0200 Subject: [PATCH 3/5] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2a6931f11..c710ea5ba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Remove default false from nextflow_schema.json ([#2376](https://github.com/nf-core/tools/pull/2376)) - Add module MULTIQC to modules.config ([#2377](https://github.com/nf-core/tools/pull/2377)) - Update the Code of Conduct ([#2381](https://github.com/nf-core/tools/pull/2381)) +- Save template information to `.nf-core.yml` and deprecate argument `--template-yaml` for `nf-core sync` ([#2388](https://github.com/nf-core/tools/pull/2388) and [#2389](https://github.com/nf-core/tools/pull/2389)) ### Download From 86f662b43fa6db00113996d68b90a7f88df1d298 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Aug 2023 15:32:03 +0200 Subject: [PATCH 4/5] specify pipeline directory to obtain config file when sync --- nf_core/create.py | 12 +++++++++--- nf_core/sync.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index a7ba8d14d8..08b9676c9c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -57,7 +57,7 @@ def __init__( default_branch=None, ): self.template_params, skip_paths_keys, self.template_yaml = self.create_param_dict( - name, description, author, version, template_yaml_path, plain + name, description, author, version, template_yaml_path, plain, outdir if outdir else "." ) skippable_paths = { @@ -90,14 +90,20 @@ def __init__( outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) self.outdir = Path(outdir) - def create_param_dict(self, name, description, author, version, template_yaml_path, plain): + def create_param_dict(self, name, description, author, version, template_yaml_path, plain, pipeline_dir): """Creates a dictionary of parameters for the new pipeline. Args: + name (str): Name for the pipeline. + description (str): Description for the pipeline. + author (str): Authors name of the pipeline. + version (str): Version flag. template_yaml_path (str): Path to YAML file containing template parameters. + plain (bool): If true the pipeline template will be initialized plain, without customisation. + pipeline_dir (str): Path to the pipeline directory. """ # Try reading config file - _, config_yml = nf_core.utils.load_tools_config() + _, config_yml = nf_core.utils.load_tools_config(pipeline_dir) # Obtain template customization info from template yaml file or `.nf-core.yml` config file try: diff --git a/nf_core/sync.py b/nf_core/sync.py index d56f7d64c7..d1e73c786c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -82,7 +82,7 @@ def __init__( self.gh_repo = gh_repo self.pr_url = "" - self.config_yml_path, self.config_yml = nf_core.utils.load_tools_config() + self.config_yml_path, self.config_yml = nf_core.utils.load_tools_config(self.pipeline_dir) # Throw deprecation warning if template_yaml_path is set if template_yaml_path is not None: From 13cee01259f1afc7904fcab1e26a09a423f3429b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 7 Aug 2023 16:39:07 +0200 Subject: [PATCH 5/5] Update nf_core/sync.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index d1e73c786c..5402a6121d 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -102,7 +102,7 @@ def __init__( yaml.safe_dump(self.config_yml, fh) log.info(f"Saved pipeline creation settings to '{self.config_yml_path}'") raise SystemExit( - f"Please commit your changes and delete {template_yaml_path} file. Then run the sync command again." + f"Please commit your changes and delete the {template_yaml_path} file. Then run the sync command again." ) # Set up the API auth if supplied on the command line