From 4bad234df707bf886d172f97362ff4c2daa2a9c7 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 4 May 2023 16:34:13 +0100 Subject: [PATCH 1/7] error if quay.io found in docker container name Changes: - Linting error if container starts with quay.io/biocontainers but not quay.io/biocontainers/mulled Fixes #2276 --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c0792519..d64c03267e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Linting - Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) +- Error if module container specification has quay.io as prefix when it shouldn't have. ### Modules diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 102d5ead12..bfdb92c5cf 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -290,6 +290,14 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None + if l.startswith("quay.io/biocontainers") and not l.startswith("quay.io/biocontainers/mulled"): + self.failed.append( + ( + "container_links", + "quay.io/biocontainers prefix found, please use biocontainers instead", + self.main_nf, + ) + ) if l.startswith("biocontainers/"): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l From aea396a68e64dde41b1e00ce70e8da522bac828a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 5 May 2023 08:49:41 +0200 Subject: [PATCH 2/7] More verbose error explanation --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bfdb92c5cf..83ce4502ec 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -294,7 +294,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append( ( "container_links", - "quay.io/biocontainers prefix found, please use biocontainers instead", + "'quay.io/biocontainers/' container prefix found, please use just 'biocontainers/' instead", self.main_nf, ) ) From 86aee0caab95be421af718f96f9b85e74bc6f096 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 10:45:09 +0100 Subject: [PATCH 3/7] Add passed test if quay.io is correct --- nf_core/modules/lint/main_nf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 83ce4502ec..0af8e6357a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -298,6 +298,8 @@ def check_process_section(self, lines, fix_version, progress_bar): self.main_nf, ) ) + else: + self.passed.append(("container_links", "Container prefix is correct", self.main_nf)) if l.startswith("biocontainers/"): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l From 4351a34af621b087832c17b7d020197619e6e7f8 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 5 May 2023 10:47:39 +0100 Subject: [PATCH 4/7] Container status check only occurs once Puts container status check into a single if statement --- nf_core/modules/lint/main_nf.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0af8e6357a..96d03421eb 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,15 +271,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: - self.failed.append( - ( - "container_links", - "Too many double quotes found when specifying singularity container", - self.main_nf, - ) - ) + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -304,17 +296,26 @@ def check_process_section(self, lines, fix_version, progress_bar): # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix l = "quay.io/" + l url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: - self.failed.append( - ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) - ) + # lint double quotes - if l.startswith("container"): + if l.startswith("container") or _container_type(l) == "docker" or _container_type(l) == "singularity": if l.count('"') > 2: self.failed.append( - ("container_links", "Too many double quotes found when specifying containers", self.main_nf) + ( + "container_links", + f"Too many double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) + ) + else: + self.passed.append( + ( + "container_links", + f"Correct number of double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) ) + # lint more than one container in the same line if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): self.warned.append( From 46c1dea607937dfe710da12e9dd35809f88ff682 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 9 May 2023 13:32:32 +0100 Subject: [PATCH 5/7] Checks for any instance of quay.io in container name --- nf_core/modules/lint/main_nf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6530549efa..4f630bbb23 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -282,16 +282,16 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = NoneD - if l.startswith("quay.io/biocontainers") and not l.startswith("quay.io/biocontainers/mulled"): + if l.startswith("quay.io/"): self.failed.append( ( "container_links", - "'quay.io/biocontainers/' container prefix found, please use just 'biocontainers/' instead", + f"'quay.io//:' container name found, please use just '/:' instead. Offending container name: {l}", self.main_nf, ) ) else: - self.passed.append(("container_links", "Container prefix is correct", self.main_nf)) + self.passed.append(("container_links", f"Container prefix is correct", self.main_nf)) # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix From c786f2ac6bca26221e3e8bd6dc8825366fd09e6a Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Tue, 9 May 2023 14:46:55 +0100 Subject: [PATCH 6/7] Improved log message --- nf_core/modules/lint/main_nf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4f630bbb23..8150e7e839 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -283,10 +283,11 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = NoneD if l.startswith("quay.io/"): + l_stripped = re.sub("\W+$", "", l) self.failed.append( ( "container_links", - f"'quay.io//:' container name found, please use just '/:' instead. Offending container name: {l}", + f"{l_stripped} container name found, please use just 'organisation/container:tag' instead.", self.main_nf, ) ) From 00bd8efae41f63bb68a6d4b51f6beaaea0ab7878 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 9 May 2023 16:15:04 +0200 Subject: [PATCH 7/7] update gitlab sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 95cc2cad95..338d890f2f 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -18,7 +18,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8" +CORRECT_SHA = "1dff30bfca2d98eb7ac7b09269a15e822451d99f" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align"