Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Make remove command recursive #2049

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mirpedrol merged 8 commits into from
Nov 25, 2022
Merged

Make remove command recursive #2049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f188042
linting from pre-commit
mirpedrol Nov 24, 2022
fde173c
pre-commit changes
mirpedrol Nov 24, 2022
a5da0eb
pre-commit changes
mirpedrol Nov 24, 2022
c1e014d
pre-commit fixes
mirpedrol Nov 24, 2022
f65f4e2
check also if dependencies are installed in `check_component_installe…
mashehu Nov 25, 2022
1851c77
quiet down the reinstall log messages
mashehu Nov 25, 2022
65e6f73
Apply suggestions from code review
mirpedrol Nov 25, 2022
a5b5810
Merge branch 'dev' into remove-recursive
mirpedrol Nov 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions nf_core/components/install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,11 @@ def install(self, component, silent=False):
# Remove component if force is set and component is installed
install_track = None
if self.force:
log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'")
log.debug(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'")
self.clear_component_dir(component, component_dir)
install_track = self.clean_modules_json(component, self.modules_repo, modules_json)

log.info(f"{'Rei' if self.force else 'I'}nstalling '{component}'")
if not silent:
log.info(f"{'Rei' if self.force else 'I'}nstalling '{component}'")
log.debug(
f"Installing {self.component_type} '{component}' at modules hash {version} from {self.modules_repo.remote_url}"
)
Expand Down Expand Up @@ -186,7 +186,7 @@ def collect_and_verify_name(self, component, modules_repo):
if component is None:
component = questionary.autocomplete(
f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:",
choices=modules_repo.get_avail_components(self.component_type).sort(),
choices=sorted(modules_repo.get_avail_components(self.component_type)),
style=nf_core.utils.nfcore_question_style,
).unsafe_ask()

Expand Down Expand Up @@ -214,8 +214,11 @@ def check_component_installed(self, component, current_version, component_dir, m
False: if the component is installed
"""
if (current_version is not None and os.path.exists(component_dir)) and not force:
# make sure included components are also installed
if self.component_type == "subworkflows":
self.install_included_components(component_dir)
if not silent:
log.info(f"{self.component_type[:-1].title()} is already installed.")
log.info(f"{self.component_type[:-1].title()} '{component}' is already installed.")

if prompt:
message = (
Expand All @@ -235,7 +238,7 @@ def check_component_installed(self, component, current_version, component_dir, m
branch_flag = "" if modules_repo.branch == "master" else f"-b {modules_repo.branch} "

log.info(
f"To update '{component}' run 'nf-core {self.component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'"
f"To update '{component}' run 'nf-core {self.component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'."
)
return False

Expand Down Expand Up @@ -272,8 +275,8 @@ def clean_modules_json(self, component, modules_repo, modules_json):
for name, component_values in dir_components.items():
if name == component and dir == modules_repo.repo_path:
repo_to_remove = repo_url
log.info(
f"Removing {self.component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json"
log.debug(
f"Removing {self.component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json."
)
modules_json.remove_entry(
self.component_type, component, repo_to_remove, modules_repo.repo_path
Expand Down
151 changes: 81 additions & 70 deletions nf_core/components/remove.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class ComponentRemove(ComponentCommand):
def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False):
super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)

def remove(self, component, force=False):
def remove(self, component, removed_by=None, force=False):
"""
Remove an already installed module/subworkflow
This command only works for modules/subworkflows that are installed from 'nf-core/modules'
Expand Down Expand Up @@ -67,85 +67,96 @@ def remove(self, component, force=False):
modules_json.remove_entry(self.component_type, component, self.modules_repo.remote_url, repo_path)
return False

removed_by = None
dependent_components = {component: self.component_type}
if self.component_type == "subworkflows":
removed_by = component
dependent_components.update(
modules_json.get_dependent_components(
self.component_type, component, self.modules_repo.remote_url, repo_path, dependent_components
)
)
# remove all dependent components based on installed_by entry
# Remove entry from modules.json
removed = False
removed_components = []
for component_name, component_type in dependent_components.items():
removed_component = modules_json.remove_entry(
component_type,
component_name,
self.modules_repo.remote_url,
repo_path,
removed_by=removed_by,
)
removed_component_dir = Path(component_type, repo_path, component_name)
if removed_component:
# check if one of the dependent module/subworkflow has been manually included in the pipeline
include_stmts = self.check_if_in_include_stmts(str(removed_component_dir))
if include_stmts:
# print the include statements
log.warn(
f"The {component_type[:-1]} '{component_name}' is still included in the following workflow file{nf_core.utils.plural_s(include_stmts)}:"
)
console = Console()
for file, stmts in include_stmts.items():
renderables = []
for stmt in stmts:
renderables.append(
Syntax(
stmt["line"],
"groovy",
theme="ansi_dark",
line_numbers=True,
start_line=stmt["line_number"],
)
)
console.print(
Panel(
Group(*renderables),
title=f"{file}",
style="white",
title_align="center",
padding=1,
# Remove component from modules.json
removed_component = modules_json.remove_entry(
self.component_type,
component,
self.modules_repo.remote_url,
repo_path,
removed_by=removed_by,
)
removed_component_dir = Path(self.component_type, repo_path, component)
if removed_component:
# check if the module/subworkflow has been manually included in the pipeline
include_stmts = self.check_if_in_include_stmts(str(removed_component_dir))
if include_stmts:
# print the include statements
log.warn(
f"The {self.component_type[:-1]} '{component}' is still included in the following workflow file{nf_core.utils.plural_s(include_stmts)}:"
)
console = Console()
for file, stmts in include_stmts.items():
renderables = []
for stmt in stmts:
renderables.append(
Syntax(
stmt["line"],
"groovy",
theme="ansi_dark",
line_numbers=True,
start_line=stmt["line_number"],
)
)
# ask the user if they still want to remove the component, add it back otherwise
if not force:
if not questionary.confirm(
f"Do you still want to remove the {component_type[:-1]} '{component_name}'?",
style=nf_core.utils.nfcore_question_style,
).unsafe_ask():
# add the component back to modules.json
if not ComponentInstall(self.dir, component_type, force=True).install(component_name):
log.warn(
f"Could not install the {component_type[:-1]} '{component_name}', please install it manually with 'nf-core {component_type} install {component_name}'."
)
continue
# Remove the component files of all entries removed from modules.json
removed = (
True
if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed
else False
)
console.print(
Panel(
Group(*renderables),
title=f"{file}",
style="white",
title_align="center",
padding=1,
)
)
# ask the user if they still want to remove the component, add it back otherwise
if not force:
if not questionary.confirm(
f"Do you still want to remove the {self.component_type[:-1]} '{component}'?",
style=nf_core.utils.nfcore_question_style,
).unsafe_ask():
# add the component back to modules.json
if not ComponentInstall(self.dir, self.component_type, force=True).install(
component, silent=True
):
log.warn(
f"Could not install the {self.component_type[:-1]} '{component}', please install it manually with 'nf-core {component_type} install {component}'."
)
return removed
# Remove the component files of all entries removed from modules.json
removed = (
True if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed else False
)

if removed:
# remember removed dependencies
if component_name != component:
removed_components.append(component_name.replace("/", "_"))
if removed:
# remember removed dependencies
if self.component_type == "subworkflows":
removed_by = component
dependent_components = modules_json.get_dependent_components(
self.component_type, component, self.modules_repo.remote_url, repo_path, {}
)
for component_name, component_type in dependent_components.items():
original_component_tyoe = self.component_type
self.component_type = component_type
dependency_removed = self.remove(component_name, removed_by=removed_by)
self.component_type = original_component_tyoe
# remember removed dependencies
if dependency_removed:
removed_components.append(component_name.replace("/", "_"))
# print removed dependencies
if removed_components:
log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'.")
else:
log.info(f"Removed files for '{component}'.")
else:
installed_by = modules_json.modules_json["repos"][self.modules_repo.remote_url][self.component_type][
repo_path
][component]["installed_by"]
if installed_by == self.component_type:
log.error(
f"Did not remove '{component}', because it was also manually installed. Only updated 'installed_by' in modules.json."
)
log.info(
f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {', '.join(f"'{d}'" for d in installed_by)}."""
)
return removed
22 changes: 9 additions & 13 deletions nf_core/modules/modules_json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -647,7 +647,8 @@ def update(
except KeyError:
repo_component_entry[component_name]["installed_by"] = [installed_by]
finally:
repo_component_entry[component_name]["installed_by"].extend(installed_by_log)
new_installed_by = repo_component_entry[component_name]["installed_by"] + list(installed_by_log)
repo_component_entry[component_name]["installed_by"] = [*set(new_installed_by)]

# Sort the 'modules.json' repo entries
self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
Expand Down Expand Up @@ -693,7 +694,7 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N
# write the updated modules.json file
if removed_by == component_type:
log.info(
f"Updated the 'installed_by' list of {name}, but it is still installed, because it is required by {repo_entry[component_type][install_dir][name]['installed_by']}."
f"""Updated the 'installed_by' list for '{name}', but it is still installed, because it is required by {", ".join(f"'{d}'" for d in repo_entry[component_type][install_dir][name]['installed_by'])}."""
)
else:
log.info(
Expand Down Expand Up @@ -941,20 +942,15 @@ def get_dependent_components(
component_types = ["modules"] if component_type == "modules" else ["modules", "subworkflows"]
# Find all components that have an entry of install by of a given component, recursively call this function for subworkflows
for type in component_types:
components = self.modules_json["repos"][repo_url][type][install_dir].items()
try:
components = self.modules_json["repos"][repo_url][type][install_dir].items()
except KeyError as e:
# This exception will raise when there are only modules installed
log.debug(f"Trying to retrieve all {type}. There aren't {type} installed. Failed with error {e}")
continue
for component_name, component_entry in components:
if name in component_entry["installed_by"]:
dependent_components[component_name] = type
if type == "subworkflows":
dependent_components.update(
self.get_dependent_components(
type,
component_name,
repo_url,
install_dir,
dependent_components,
)
)

return dependent_components

Expand Down
1 change: 1 addition & 0 deletions requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ pytest-datafiles
requests-mock
Sphinx
sphinx-rtd-theme
requests_mock
19 changes: 19 additions & 0 deletions tests/subworkflows/remove.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,22 @@ def test_subworkflows_remove_one_of_two_subworkflow(self):
assert Path.exists(samtools_index_path) is False
assert Path.exists(samtools_stats_path) is True
self.subworkflow_remove.remove("bam_stats_samtools")


def test_subworkflows_remove_included_subworkflow(self):
"""Test removing subworkflow which is installed by another subworkflow and all it's dependencies."""
self.subworkflow_install.install("bam_sort_stats_samtools")
subworkflow_path = Path(self.subworkflow_install.dir, "subworkflows", "nf-core")
bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools")
bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools")
samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index")
samtools_stats_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "stats")

assert self.subworkflow_remove.remove("bam_stats_samtools") is False

assert Path.exists(subworkflow_path) is True
assert Path.exists(bam_sort_stats_samtools_path) is True
assert Path.exists(bam_stats_samtools_path) is True
assert Path.exists(samtools_index_path) is True
assert Path.exists(samtools_stats_path) is True
self.subworkflow_remove.remove("bam_sort_stats_samtools")
1 change: 1 addition & 0 deletions tests/test_subworkflows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ def tearDown(self):
test_subworkflows_list_remote_gitlab,
)
from .subworkflows.remove import (
test_subworkflows_remove_included_subworkflow,
test_subworkflows_remove_one_of_two_subworkflow,
test_subworkflows_remove_subworkflow,
test_subworkflows_remove_subworkflow_keep_installed_module,
Expand Down