From ed9c9dffa89e7adbcf6b52183d028f7f802302a2 Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:23:23 +0100 Subject: [PATCH 01/10] Add nf-core colors to rtd sphinx theme add sphinx to requirements file --- docs/api/_src/_static/css/custom.css | 200 +++++++++++++++++++++++++++ docs/api/_src/conf.py | 7 +- requirements-dev.txt | 2 + 3 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/_static/css/custom.css diff --git a/docs/api/_src/_static/css/custom.css b/docs/api/_src/_static/css/custom.css new file mode 100644 index 0000000000..2b7166ae22 --- /dev/null +++ b/docs/api/_src/_static/css/custom.css @@ -0,0 +1,200 @@ +@media (prefers-color-scheme: light) { + .wy-nav-side{ + background-color: #ededed; + } + .wy-nav-top, .wy-side-nav-search, .wy-menu-vertical a:active { + background-color: #32AD65; + } + .wy-menu-vertical a{ + color: #343434; + } + .wy-menu-vertical a:hover{ + background-color: #abacab85; + } + .wy-menu-vertical header, .wy-menu-vertical p.caption { + color:#32AD65; + } + + .wy-side-nav-search input[type=text]{ + border: none; + } + + code, .rst-content code.literal{ + background-color: rgba(220,220,220,0.4); + color: #db9444; + border: none; + } + .rst-content .note .admonition-title{ + background-color: #72757bfc + } + + html.writer-html4 .rst-content dl:not(.docutils)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple)>dt { + background-color: #32AD65; + border:none; + } + .rst-content .method>dt>code { + color: #f5f6f7; + } + .sig-paren{ + color: #db9444; + } + .rst-content div[class^=highlight], .rst-content pre.literal-block{ + border:none; + } + .highlight{ + background-color: #242424; + color: #e5e6e7; + } + .highlight .hll{ + color: #e5e6e7; + background-color: #585b60fc; + } + .highlight .k, .highlight .nt, .highlight .no{ + color: #6fb2e8; + } + .highlight .s, .highlight .s1, .highlight .s2{ + color: #32AD65; + } + .highlight .nb, .highlight .o{ + color: #db9444; + } + .highlight .c1{ + color: #88898afc; + } + .highlight .nv{ + color: #f9d977; + } + .btn.btn-neutral { + background-color: #e5e6e7d4 !important; + } + .rst-content .hint .admonition-title, .rst-content .hint .wy-alert-title, .rst-content .important .admonition-title, .rst-content .important .wy-alert-title, .rst-content .tip .admonition-title, .rst-content .tip .wy-alert-title, .rst-content .wy-alert-success.admonition-todo .admonition-title, .rst-content .wy-alert-success.admonition-todo .wy-alert-title, .rst-content .wy-alert-success.admonition .admonition-title, .rst-content .wy-alert-success.admonition .wy-alert-title, .rst-content .wy-alert-success.attention .admonition-title, .rst-content .wy-alert-success.attention .wy-alert-title, .rst-content .wy-alert-success.caution .admonition-title, .rst-content .wy-alert-success.caution .wy-alert-title, .rst-content .wy-alert-success.danger .admonition-title, .rst-content .wy-alert-success.danger .wy-alert-title, .rst-content .wy-alert-success.error .admonition-title, .rst-content .wy-alert-success.error .wy-alert-title, .rst-content .wy-alert-success.note .admonition-title, .rst-content .wy-alert-success.note .wy-alert-title, .rst-content .wy-alert-success.seealso .admonition-title, .rst-content .wy-alert-success.seealso .wy-alert-title, .rst-content .wy-alert-success.warning .admonition-title, .rst-content .wy-alert-success.warning .wy-alert-title, .rst-content .wy-alert.wy-alert-success .admonition-title, .wy-alert.wy-alert-success .rst-content .admonition-title, .wy-alert.wy-alert-success .wy-alert-title{ + background-color: #32AD65; + color: #e5e6e7; + } + + .rst-content .note, .rst-content .seealso, .rst-content .wy-alert-info.admonition, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .wy-alert.wy-alert-info, + .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .wy-alert-success.admonition, .rst-content .wy-alert-success.admonition-todo, .rst-content .wy-alert-success.attention, .rst-content .wy-alert-success.caution, .rst-content .wy-alert-success.danger, .rst-content .wy-alert-success.error, .rst-content .wy-alert-success.note, .rst-content .wy-alert-success.seealso, .rst-content .wy-alert-success.warning, .wy-alert.wy-alert-success{ + color: #343434; + background-color: #e3e3e3; + border: none; + } + + +.rst-content .admonition-todo .admonition-title, .rst-content .admonition-todo .wy-alert-title, .rst-content .attention .admonition-title, .rst-content .attention .wy-alert-title, .rst-content .caution .admonition-title, .rst-content .caution .wy-alert-title, .rst-content .warning .admonition-title, .rst-content .warning .wy-alert-title, .rst-content .wy-alert-warning.admonition .admonition-title, .rst-content .wy-alert-warning.admonition .wy-alert-title, .rst-content .wy-alert-warning.danger .admonition-title, .rst-content .wy-alert-warning.danger .wy-alert-title, .rst-content .wy-alert-warning.error .admonition-title, .rst-content .wy-alert-warning.error .wy-alert-title, .rst-content .wy-alert-warning.hint .admonition-title, .rst-content .wy-alert-warning.hint .wy-alert-title, .rst-content .wy-alert-warning.important .admonition-title, .rst-content .wy-alert-warning.important .wy-alert-title, .rst-content .wy-alert-warning.note .admonition-title, .rst-content .wy-alert-warning.note .wy-alert-title, .rst-content .wy-alert-warning.seealso .admonition-title, .rst-content .wy-alert-warning.seealso .wy-alert-title, .rst-content .wy-alert-warning.tip .admonition-title, .rst-content .wy-alert-warning.tip .wy-alert-title, .rst-content .wy-alert.wy-alert-warning .admonition-title, .wy-alert.wy-alert-warning .rst-content .admonition-title, .wy-alert.wy-alert-warning .wy-alert-title { + background-color: #f4a25b; + } +.rst-content .admonition-todo, .rst-content .attention, .rst-content .caution, .rst-content .warning, .rst-content .wy-alert-warning.admonition, .rst-content .wy-alert-warning.danger, .rst-content .wy-alert-warning.error, .rst-content .wy-alert-warning.hint, .rst-content .wy-alert-warning.important, .rst-content .wy-alert-warning.note, .rst-content .wy-alert-warning.seealso, .rst-content .wy-alert-warning.tip, .wy-alert.wy-alert-warning { + color: #343434; + background-color: #e3e3e3; + } + + +} +@media (prefers-color-scheme: dark) { + .wy-nav-content-wrap { + background-color: #181a1b + } + .wy-nav-top, .wy-side-nav-search, .wy-menu-vertical a:active { + background-color: #32AD65; + } + .wy-menu-vertical header, .wy-menu-vertical p.caption { + color:#32AD65; + } + .wy-nav-side{ + background-color: #2E2E2E; + } + .wy-nav-content{ + background-color: #343434; + color: #e5e6e7; + } + a, a:hover, a:focus, a:active { + color: #6fb2e8; + } + code, .rst-content code.literal{ + background-color: rgba(220,220,220,0.1); + color: #db9444; + border: none; + } + .wy-side-nav-search input[type=text]{ + border: none; + background-color: #e5e6e7; + } + .wy-side-nav-search>div.version{ + color: #e5e6e7c9; + } + .wy-side-nav-search .wy-dropdown>a, .wy-side-nav-search>a { + color: #e5e6e7; + } + + html.writer-html4 .rst-content dl:not(.docutils)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple)>dt { + background-color: #32AD65; + border:none; + } + html.writer-html4 .rst-content dl:not(.docutils) dl:not(.field-list)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple) dl:not(.field-list)>dt{ + background-color:#777; + color:#e5e6e7a6; + } + .rst-content code { + color: #e5e6e7d4; + } + .rst-content .method>dt>code { + color: #e5e6e7; + } + .sig-paren{ + color: #db9444; + } + .rst-content .note .admonition-title{ + background-color: #72757bfc + } + .rst-content div[class^=highlight], .rst-content pre.literal-block{ + border:none; + } + + .highlight{ + background-color: #242424; + color: #e5e6e7; + } + .highlight .hll{ + color: #e5e6e7; + background-color: #585b60fc; + } + .highlight .k, .highlight .nt, .highlight .no{ + color: #6fb2e8; + } + .highlight .s, .highlight .s1, .highlight .s2{ + color: #32AD65; + } + .highlight .nb, .highlight .o{ + color: #db9444; + } + .highlight .c1{ + color: #88898afc; + } + .highlight .nv{ + color: #f9d977; + } + .btn.btn-neutral { + background-color: #e5e6e7d4 !important; + } + + .rst-content .hint .admonition-title, .rst-content .hint .wy-alert-title, .rst-content .important .admonition-title, .rst-content .important .wy-alert-title, .rst-content .tip .admonition-title, .rst-content .tip .wy-alert-title, .rst-content .wy-alert-success.admonition-todo .admonition-title, .rst-content .wy-alert-success.admonition-todo .wy-alert-title, .rst-content .wy-alert-success.admonition .admonition-title, .rst-content .wy-alert-success.admonition .wy-alert-title, .rst-content .wy-alert-success.attention .admonition-title, .rst-content .wy-alert-success.attention .wy-alert-title, .rst-content .wy-alert-success.caution .admonition-title, .rst-content .wy-alert-success.caution .wy-alert-title, .rst-content .wy-alert-success.danger .admonition-title, .rst-content .wy-alert-success.danger .wy-alert-title, .rst-content .wy-alert-success.error .admonition-title, .rst-content .wy-alert-success.error .wy-alert-title, .rst-content .wy-alert-success.note .admonition-title, .rst-content .wy-alert-success.note .wy-alert-title, .rst-content .wy-alert-success.seealso .admonition-title, .rst-content .wy-alert-success.seealso .wy-alert-title, .rst-content .wy-alert-success.warning .admonition-title, .rst-content .wy-alert-success.warning .wy-alert-title, .rst-content .wy-alert.wy-alert-success .admonition-title, .wy-alert.wy-alert-success .rst-content .admonition-title, .wy-alert.wy-alert-success .wy-alert-title{ + background-color: #32AD65; + color: #e5e6e7; + } + + .rst-content .note, .rst-content .seealso, .rst-content .wy-alert-info.admonition, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .wy-alert.wy-alert-info, + .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .wy-alert-success.admonition, .rst-content .wy-alert-success.admonition-todo, .rst-content .wy-alert-success.attention, .rst-content .wy-alert-success.caution, .rst-content .wy-alert-success.danger, .rst-content .wy-alert-success.error, .rst-content .wy-alert-success.note, .rst-content .wy-alert-success.seealso, .rst-content .wy-alert-success.warning, .wy-alert.wy-alert-success{ + color: #343434; + background-color: #e3e3e3; + border: none; + } + + .rst-content .admonition-todo .admonition-title, .rst-content .admonition-todo .wy-alert-title, .rst-content .attention .admonition-title, .rst-content .attention .wy-alert-title, .rst-content .caution .admonition-title, .rst-content .caution .wy-alert-title, .rst-content .warning .admonition-title, .rst-content .warning .wy-alert-title, .rst-content .wy-alert-warning.admonition .admonition-title, .rst-content .wy-alert-warning.admonition .wy-alert-title, .rst-content .wy-alert-warning.danger .admonition-title, .rst-content .wy-alert-warning.danger .wy-alert-title, .rst-content .wy-alert-warning.error .admonition-title, .rst-content .wy-alert-warning.error .wy-alert-title, .rst-content .wy-alert-warning.hint .admonition-title, .rst-content .wy-alert-warning.hint .wy-alert-title, .rst-content .wy-alert-warning.important .admonition-title, .rst-content .wy-alert-warning.important .wy-alert-title, .rst-content .wy-alert-warning.note .admonition-title, .rst-content .wy-alert-warning.note .wy-alert-title, .rst-content .wy-alert-warning.seealso .admonition-title, .rst-content .wy-alert-warning.seealso .wy-alert-title, .rst-content .wy-alert-warning.tip .admonition-title, .rst-content .wy-alert-warning.tip .wy-alert-title, .rst-content .wy-alert.wy-alert-warning .admonition-title, .wy-alert.wy-alert-warning .rst-content .admonition-title, .wy-alert.wy-alert-warning .wy-alert-title { + background-color: #f4a25b; + } + .rst-content .admonition-todo, .rst-content .attention, .rst-content .caution, .rst-content .warning, .rst-content .wy-alert-warning.admonition, .rst-content .wy-alert-warning.danger, .rst-content .wy-alert-warning.error, .rst-content .wy-alert-warning.hint, .rst-content .wy-alert-warning.important, .rst-content .wy-alert-warning.note, .rst-content .wy-alert-warning.seealso, .rst-content .wy-alert-warning.tip, .wy-alert.wy-alert-warning { + color: #343434; + background-color: #e3e3e3; + } + +} \ No newline at end of file diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index d863a80d28..3ade174945 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,10 +14,9 @@ # import os import sys +sys.path.insert(0, os.path.abspath('../../../nf_core')) import nf_core -sys.path.insert(0, os.path.abspath("../../../nf_core")) - # -- Project information ----------------------------------------------------- project = "nf-core tools API" @@ -90,6 +89,10 @@ # Custom sidebar templates, must be a dictionary that maps document names # to template names. +html_css_files = [ + 'css/custom.css', +] + # # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by diff --git a/requirements-dev.txt b/requirements-dev.txt index 10e124429e..1ad518c05c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,5 @@ pytest-datafiles pytest-cov mock black +Sphinx +sphinx_rtd_theme \ No newline at end of file From 63b0997779a06f31c8256f3284fb79eebb07ff33 Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:27:17 +0100 Subject: [PATCH 02/10] fix for linting --- docs/api/_src/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 3ade174945..68023db2f3 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,7 +14,8 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../../../nf_core')) + +sys.path.insert(0, os.path.abspath("../../../nf_core")) import nf_core # -- Project information ----------------------------------------------------- @@ -90,7 +91,7 @@ # Custom sidebar templates, must be a dictionary that maps document names # to template names. html_css_files = [ - 'css/custom.css', + "css/custom.css", ] # From a7bea04637af2fbb5c582fdec3a329edc6603d25 Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:32:33 +0100 Subject: [PATCH 03/10] =?UTF-8?q?dummy=20commit=20to=20run=20the=20tests?= =?UTF-8?q?=20one=20more=20time=20=F0=9F=99=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/_src/_static/css/custom.css | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/_src/_static/css/custom.css b/docs/api/_src/_static/css/custom.css index 2b7166ae22..9bd4341651 100644 --- a/docs/api/_src/_static/css/custom.css +++ b/docs/api/_src/_static/css/custom.css @@ -196,5 +196,4 @@ color: #343434; background-color: #e3e3e3; } - -} \ No newline at end of file +} From 854e668cd85cc2cf68e17d450550dc73ff4386de Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 17:35:01 +0100 Subject: [PATCH 04/10] add pytest-workflow dep --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f2fa6aa93d..05dfc2de9c 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ "packaging", "prompt_toolkit>=3.0.3", "pyyaml", + "pytest-workflow", "questionary>=1.8.0", "requests_cache", "requests", From 170ee5865b1b104615c09bc04dcac6b580a9a40d Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 18:55:41 +0100 Subject: [PATCH 05/10] remove containsKey() --- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 78e8c65d7f..54935ec818 100644 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -219,7 +219,7 @@ class NfcoreSchema { } } } - if(rawSchema.keySet().contains('properties') && rawSchema.get('properties').containsKey(ignore_param)) { + if(rawSchema.keySet().contains('properties') && rawSchema.get('properties').keySet().contains(ignore_param)) { rawSchema.get("properties").remove(ignore_param) } if(rawSchema.keySet().contains('required') && rawSchema.required.contains(ignore_param)) { From e7d38968bf61b93645c226183edc76df143ff601 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 19:02:59 +0100 Subject: [PATCH 06/10] add 'params.name' to failing params in nextflow.config lint check --- nf_core/lint/nextflow_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index b458c257c0..19f10f045c 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -116,6 +116,7 @@ def nextflow_config(self): "params.container", "params.singleEnd", "params.igenomesIgnore", + "params.name", ] # Remove field that should be ignored according to the linting config From eb6d1eee40546af31053c8eb830fbff509059430 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:38:02 +0100 Subject: [PATCH 07/10] Split create-lint-wf commands into separate steps --- .github/workflows/create-lint-wf.yml | 39 ++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index eec5e9840c..569cac6599 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -4,6 +4,8 @@ on: [push, pull_request] jobs: MakeTestWorkflow: runs-on: ubuntu-latest + env: + NXF_VER: 21.03.0-edge steps: - uses: actions/checkout@v2 name: Check out source-code repository @@ -22,21 +24,32 @@ jobs: env: CAPSULE_LOG: none run: | - mkdir /tmp/nextflow - cd /tmp/nextflow wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + sudo mv nextflow /usr/local/bin/ - - name: Run nf-core/tools - run: | - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored - nf-core --log-file log.txt list - nf-core --log-file log.txt licences nf-core-testpipeline - nf-core --log-file log.txt sync nf-core-testpipeline/ - nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts - nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 - nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc + - name: nf-core create + run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + + - name: nf-core lint + run: nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored + + - name: nf-core list + run: nf-core --log-file log.txt list + + - name: nf-core licences + run: nf-core --log-file log.txt licences nf-core-testpipeline + + - name: nf-core sync + run: nf-core --log-file log.txt sync nf-core-testpipeline/ + + - name: nf-core schema + run: nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts + + - name: nf-core bump-version + run: nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 + + - name: nf-core modules install + run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc - name: Upload log file artifact if: ${{ always() }} From 39948be1461624ed9169809ec926a6187b936525 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:38:19 +0100 Subject: [PATCH 08/10] Automated PR comments for Python and Markdownlint CI test failures --- .github/workflows/markdown-lint.yml | 25 +++++++++++++++++++++++++ .github/workflows/python-lint.yml | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 5cdf07ec9a..5a590aff8e 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -16,3 +16,28 @@ jobs: - name: Run Markdownlint run: markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Markdown linting is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install `markdownlint-cli` + * On Mac: `brew install markdownlint-cli` + * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli): `npm install -g markdownlint-cli` + * Try to automatically fix errors in your pipeline: `markdownlint . --config .github/markdownlint.yml --fix` + * Check that there are no remaining problems: `markdownlint . --config .github/markdownlint.yml` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 4c8b5b00f9..cf41eb67da 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -15,3 +15,25 @@ jobs: - name: Check code lints with Black uses: jpetrucciani/black-check@master + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false From a407944f78c86e344c2098fc075c05f4a6176b19 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:40:15 +0100 Subject: [PATCH 09/10] Changelog, bump v1.13.2dev --- CHANGELOG.md | 7 +++++-- setup.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c56c0cb0e..3eb87806e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,11 @@ # nf-core/tools: Changelog -## v1.14dev +## v1.13.2dev -_..nothing yet.._ +### Tools code + +* Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read +* Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] diff --git a/setup.py b/setup.py index 05dfc2de9c..d5dc60e0d9 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.14dev" +version = "1.13.2dev" with open("README.md") as f: readme = f.read() From 396ebc25bb46322b8fcd6e2fd3818058af6e28f0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 21:23:58 +0100 Subject: [PATCH 10/10] @ewels implementation of nf-core/tools#936 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased to hide my lack of local testing. Hopefully Kevin doesn't read his GitHub email notifications.. 👀 --- CHANGELOG.md | 1 + nf_core/create.py | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eb87806e4..458b7900da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Tools code +* Fix `UnicodeDecodeError` with `nf-core create` on some systems [[#931]](https://github.com/nf-core/tools/issues/931) * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures diff --git a/nf_core/create.py b/nf_core/create.py index 6f23c99478..2492169486 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -83,7 +83,8 @@ def render_template(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") - binary_ftypes = ["image", "application/java-archive"] + binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] + binary_extensions = [".jpeg", ".jpg", ".png", ".zip", ".gz", ".jar", ".tar"] object_attrs = vars(self) object_attrs["nf_core_version"] = nf_core.__version__ @@ -106,23 +107,33 @@ def render_template(self): output_path = os.path.join(self.outdir, template_fn) os.makedirs(os.path.dirname(output_path), exist_ok=True) - # Just copy binary files - (ftype, encoding) = mimetypes.guess_type(template_fn_path) - if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): - log.debug(f"Copying binary file: '{output_path}'") + try: + # Just copy certain file extensions + filename, file_extension = os.path.splitext(template_fn_path) + if file_extension in binary_extensions: + raise AttributeError(f"File extension: {file_extension}") + + # Try to detect binary files + (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) + if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + raise AttributeError(f"Encoding: {encoding}") + + # Got this far - render the template + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) + + # Copy the file directly instead of using Jinja + except (AttributeError, UnicodeDecodeError) as e: + log.debug(f"Copying file without Jinja: '{output_path}' - {e}") shutil.copy(template_fn_path, output_path) continue - # Render the template - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - rendered_output = j_template.render(object_attrs) - - # Write to the pipeline output file - with open(output_path, "w") as fh: - log.debug(f"Writing to output file: '{output_path}'") - fh.write(rendered_output) - # Make a logo and save it self.make_pipeline_logo()