From 65c2603bcc323c64e7d7428d783fc796d6d5b2c7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 20 Jan 2023 13:39:17 -0600 Subject: [PATCH 1/6] build: Replace requests-mock with responses --- requirements-dev.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 42ce780ce4..360f6ff87f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ isort myst_parser pytest-cov pytest-datafiles -requests-mock +responses Sphinx sphinx-rtd-theme -requests_mock From ebbce34f52475353f2e6f6767e6a22bd0be637f7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 21 Jan 2023 14:48:01 -0600 Subject: [PATCH 2/6] refactor: Replace requests_mock calls with responses --- tests/modules/create.py | 18 +++++++++--------- tests/test_modules.py | 6 +++--- tests/test_subworkflows.py | 5 +++-- tests/utils.py | 10 +++++----- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 61a8777b14..9e018e97b4 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,7 +1,7 @@ import os import pytest -import requests_mock +import responses import nf_core.modules from tests.utils import mock_api_calls @@ -9,8 +9,8 @@ def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "trim-galore", "0.6.7") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) @@ -20,8 +20,8 @@ def test_modules_create_succeed(self): def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "trim-galore", "0.6.7") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) @@ -33,8 +33,8 @@ def test_modules_create_fail_exists(self): def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "fastqc", "0.11.9") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "fastqc", "0.11.9") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) @@ -45,8 +45,8 @@ def test_modules_create_nfcore_modules(self): def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "star", "2.8.10a") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "star", "2.8.10a") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) diff --git a/tests/test_modules.py b/tests/test_modules.py index c50c1f2ba8..81d3f69c49 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,7 +6,7 @@ import tempfile import unittest -import requests_mock +import responses import nf_core.create import nf_core.modules @@ -35,8 +35,8 @@ def create_modules_repo_dummy(tmp_dir): fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) # mock biocontainers and anaconda response - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "bpipe", "0.9.11--hdfd78af_0") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) module_create.create() diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 552a2ab176..c07e5d92d8 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -6,7 +6,7 @@ import tempfile import unittest -import requests_mock +import responses import nf_core.create import nf_core.modules @@ -30,7 +30,8 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - with requests_mock.Mocker() as mock: + # FIXME This mock isn't used + with responses.RequestsMock() as rsps: subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) subworkflow_create.create() diff --git a/tests/utils.py b/tests/utils.py index 77e94be464..cb48ac347e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,8 @@ from contextlib import contextmanager from pathlib import Path +import responses + OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" @@ -68,14 +70,13 @@ def set_wd(path: Path): os.chdir(start_wd) -def mock_api_calls(mock, module, version): +def mock_api_calls(rsps: responses.RequestsMock, module, version): """Mock biocontainers and anaconda api calls for module""" biocontainers_api_url = ( f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" ) anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" anaconda_mock = { - "status_code": 200, "latest_version": version.split("--")[0], "summary": "", "doc_url": "", @@ -84,7 +85,6 @@ def mock_api_calls(mock, module, version): "license": "", } biocontainers_mock = { - "status_code": 200, "images": [ { "image_type": "Singularity", @@ -98,5 +98,5 @@ def mock_api_calls(mock, module, version): }, ], } - mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) - mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) + rsps.get(anaconda_api_url, json=anaconda_mock, status=200) + rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) From 887827284f86e7802132f0f72c73970f5eaa3a03 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 22 Jan 2023 14:48:53 -0600 Subject: [PATCH 3/6] refactor: Split biocontainer and anaconda mocks Responses actually throws an error if all the mocks aren't used. That allowed me to clean up a few tests that weren't actually using the mocks and find a few. --- tests/modules/create.py | 14 +++++++++----- tests/test_modules.py | 11 ++++------- tests/utils.py | 16 ++++++++++------ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 9e018e97b4..33764159f7 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -4,13 +4,14 @@ import responses import nf_core.modules -from tests.utils import mock_api_calls +from tests.utils import mock_anaconda_api_calls, mock_biocontainers_api_calls def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "trim-galore", "0.6.7") + mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") + mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) @@ -21,7 +22,8 @@ def test_modules_create_succeed(self): def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "trim-galore", "0.6.7") + mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") + mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) @@ -34,7 +36,8 @@ def test_modules_create_fail_exists(self): def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "fastqc", "0.11.9") + mock_anaconda_api_calls(rsps, "fastqc", "0.11.9") + mock_biocontainers_api_calls(rsps, "fastqc", "0.11.9") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) @@ -46,7 +49,8 @@ def test_modules_create_nfcore_modules(self): def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "star", "2.8.10a") + mock_anaconda_api_calls(rsps, "star", "2.8.10a") + mock_biocontainers_api_calls(rsps, "star", "2.8.10a") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) diff --git a/tests/test_modules.py b/tests/test_modules.py index 81d3f69c49..8d4a9c1c23 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -18,7 +18,6 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, - mock_api_calls, ) @@ -34,12 +33,10 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # mock biocontainers and anaconda response - with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) - module_create.create() + # FIXME Should use mock? + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) + module_create.create() return root_dir diff --git a/tests/utils.py b/tests/utils.py index cb48ac347e..0dd60dd051 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -70,11 +70,8 @@ def set_wd(path: Path): os.chdir(start_wd) -def mock_api_calls(rsps: responses.RequestsMock, module, version): - """Mock biocontainers and anaconda api calls for module""" - biocontainers_api_url = ( - f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" - ) +def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): + """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" anaconda_mock = { "latest_version": version.split("--")[0], @@ -84,6 +81,14 @@ def mock_api_calls(rsps: responses.RequestsMock, module, version): "files": [{"version": version.split("--")[0]}], "license": "", } + rsps.get(anaconda_api_url, json=anaconda_mock, status=200) + + +def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): + """Mock biocontainers api calls for module""" + biocontainers_api_url = ( + f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" + ) biocontainers_mock = { "images": [ { @@ -98,5 +103,4 @@ def mock_api_calls(rsps: responses.RequestsMock, module, version): }, ], } - rsps.get(anaconda_api_url, json=anaconda_mock, status=200) rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) From 9c2537fb1898344fb6ad540ebccb698f316bf125 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 12:17:16 -0500 Subject: [PATCH 4/6] test(modules): Turn off requests_cache for module creation --- tests/modules/create.py | 16 +++++++++++----- tests/test_modules.py | 15 +++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 33764159f7..98e498c1b0 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,6 +1,7 @@ import os import pytest +import requests_cache import responses import nf_core.modules @@ -15,7 +16,8 @@ def test_modules_create_succeed(self): module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) @@ -27,9 +29,11 @@ def test_modules_create_fail_exists(self): module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) - module_create.create() - with pytest.raises(UserWarning) as excinfo: + with requests_cache.disabled(): module_create.create() + with pytest.raises(UserWarning) as excinfo: + with requests_cache.disabled(): + module_create.create() assert "Module file exists already" in str(excinfo.value) @@ -41,7 +45,8 @@ def test_modules_create_nfcore_modules(self): module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "fastqc", "main.nf")) @@ -54,6 +59,7 @@ def test_modules_create_nfcore_modules_subtool(self): module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "star", "index", "main.nf")) diff --git a/tests/test_modules.py b/tests/test_modules.py index 8d4a9c1c23..21c003112e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,6 +6,7 @@ import tempfile import unittest +import requests_cache import responses import nf_core.create @@ -18,6 +19,8 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + mock_anaconda_api_calls, + mock_biocontainers_api_calls, ) @@ -33,10 +36,14 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # FIXME Should use mock? - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) - module_create.create() + # mock biocontainers and anaconda response + with responses.RequestsMock() as rsps: + mock_anaconda_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") + mock_biocontainers_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) + with requests_cache.disabled(): + module_create.create() return root_dir From cb60b1d2e10f80f2290f06f0285624dfc9b57dde Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 18:32:18 -0500 Subject: [PATCH 5/6] test(subworkflows): Remove unused mock --- tests/test_subworkflows.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index c07e5d92d8..c5d768ab10 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -30,10 +30,9 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # FIXME This mock isn't used - with responses.RequestsMock() as rsps: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) - subworkflow_create.create() + # TODO Add a mock here + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) + subworkflow_create.create() return root_dir From f4747bf32559fbe75a356c13ee70a2f43eba6d9a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 21:50:44 -0500 Subject: [PATCH 6/6] chore: Add #2165 to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2102b1c5a..4797f020e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) +- Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19]