From 4d7e1c064cb49f59e2a193ac1c33ac8d241dd8d1 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Mon, 30 May 2022 12:34:31 +0200 Subject: [PATCH 1/4] Add memory to all java tools in get_software_versions --- CHANGELOG.md | 1 + main.nf | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a23c86b..c592eb687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#882](https://github.com/nf-core/eager/pull/882) Define DSL1 execution explicitly, as new versions Nextflow made DSL2 default (♥ to & fix from @Lehmann-Fabian) - [#879](https://github.com/nf-core/eager/issues/879) Add missing threads parameter for pre-clipping FastQC for single end data that caused insufficient memory in some cases (♥ to @marcel-keller for reporting) +- [#885](https://github.com/nf-core/eager/issues/879) Specify task memory for all tools in get_software_versions to account for java incompatibilty with some SGE clusters causing hanging of the process (♥ to @maxibor for reporting) ### `Dependencies` diff --git a/main.nf b/main.nf index 3ed1e2f3d..a41fb058c 100644 --- a/main.nf +++ b/main.nf @@ -3113,31 +3113,31 @@ process get_software_versions { echo $workflow.manifest.version &> v_pipeline.txt echo $workflow.nextflow.version &> v_nextflow.txt - fastqc --version &> v_fastqc.txt 2>&1 || true + fastqc -t ${task.cpus} --version &> v_fastqc.txt 2>&1 || true AdapterRemoval --version &> v_adapterremoval.txt 2>&1 || true fastp --version &> v_fastp.txt 2>&1 || true bwa &> v_bwa.txt 2>&1 || true - circulargenerator --help | head -n 1 &> v_circulargenerator.txt 2>&1 || true + circulargenerator -Xmx${task.memory.toGiga()}g --help | head -n 1 &> v_circulargenerator.txt 2>&1 || true samtools --version &> v_samtools.txt 2>&1 || true - dedup -v &> v_dedup.txt 2>&1 || true + dedup -Xmx${task.memory.toGiga()}g -v &> v_dedup.txt 2>&1 || true ## bioconda recipe of picard is incorrectly set up and extra warning made with stderr, this ugly command ensures only version exported - ( exec 7>&1; picard MarkDuplicates --version 2>&1 >&7 | grep -v '/' >&2 ) 2> v_markduplicates.txt || true - qualimap --version &> v_qualimap.txt 2>&1 || true + ( exec 7>&1; picard -Xmx${task.memory.toMega()}M MarkDuplicates --version 2>&1 >&7 | grep -v '/' >&2 ) 2> v_markduplicates.txt || true + qualimap --version --java-mem-size=${task.memory.toGiga()}G &> v_qualimap.txt 2>&1 || true preseq &> v_preseq.txt 2>&1 || true - gatk --version 2>&1 | grep '(GATK)' > v_gatk.txt 2>&1 || true - gatk3 --version 2>&1 | head -n 1 > v_gatk3.txt 2>&1 || true + gatk --java-options "-Xmx${task.memory.toGiga()}G" --version 2>&1 | grep '(GATK)' > v_gatk.txt 2>&1 || true + gatk3 -Xmx${task.memory.toGiga()}g --version 2>&1 | head -n 1 > v_gatk3.txt 2>&1 || true freebayes --version &> v_freebayes.txt 2>&1 || true bedtools --version &> v_bedtools.txt 2>&1 || true - damageprofiler --version &> v_damageprofiler.txt 2>&1 || true + damageprofiler -Xmx${task.memory.toGiga()}g --version &> v_damageprofiler.txt 2>&1 || true bam --version &> v_bamutil.txt 2>&1 || true pmdtools --version &> v_pmdtools.txt 2>&1 || true angsd -h |& head -n 1 | cut -d ' ' -f3-4 &> v_angsd.txt 2>&1 || true - multivcfanalyzer --help | head -n 1 &> v_multivcfanalyzer.txt 2>&1 || true - malt-run --help |& tail -n 3 | head -n 1 | cut -f 2 -d'(' | cut -f 1 -d ',' &> v_malt.txt 2>&1 || true - MaltExtract --help | head -n 2 | tail -n 1 &> v_maltextract.txt 2>&1 || true + multivcfanalyzer -Xmx${task.memory.toGiga()}g --help | head -n 1 &> v_multivcfanalyzer.txt 2>&1 || true + malt-run -J-Xmx${task.memory.toGiga()}g --help |& tail -n 3 | head -n 1 | cut -f 2 -d'(' | cut -f 1 -d ',' &> v_malt.txt 2>&1 || true + MaltExtract -Xmx${task.memory.toGiga()}g --help | head -n 2 | tail -n 1 &> v_maltextract.txt 2>&1 || true multiqc --version &> v_multiqc.txt 2>&1 || true - vcf2genome -h |& head -n 1 &> v_vcf2genome.txt || true - mtnucratio --help &> v_mtnucratiocalculator.txt || true + vcf2genome -Xmx${task.memory.toGiga()}g -h |& head -n 1 &> v_vcf2genome.txt || true + mtnucratio -Xmx${task.memory.toGiga()}g --help &> v_mtnucratiocalculator.txt || true sexdeterrmine --version &> v_sexdeterrmine.txt || true kraken2 --version | head -n 1 &> v_kraken.txt || true endorS.py --version &> v_endorSpy.txt || true @@ -3145,7 +3145,7 @@ process get_software_versions { bowtie2 --version | grep -a 'bowtie2-.* -fdebug' > v_bowtie2.txt || true eigenstrat_snp_coverage --version | cut -d ' ' -f2 >v_eigenstrat_snp_coverage.txt || true mapDamage --version > v_mapdamage.txt || true - bbduk.sh | grep 'Last modified' | cut -d ' ' -f 3-99 > v_bbduk.txt || true + bbduk.sh -Xmx${task.memory.toGiga()}g | grep 'Last modified' | cut -d ' ' -f 3-99 > v_bbduk.txt || true bcftools --version | grep 'bcftools' | cut -d ' ' -f 2 > v_bcftools.txt || true scrape_software_versions.py &> software_versions_mqc.yaml """ From 1dcd8822f5e5476dcb704795980c074ef0b807dc Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Mon, 30 May 2022 12:39:02 +0200 Subject: [PATCH 2/4] Phrasing Tweak --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c592eb687..0598f0460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#882](https://github.com/nf-core/eager/pull/882) Define DSL1 execution explicitly, as new versions Nextflow made DSL2 default (♥ to & fix from @Lehmann-Fabian) - [#879](https://github.com/nf-core/eager/issues/879) Add missing threads parameter for pre-clipping FastQC for single end data that caused insufficient memory in some cases (♥ to @marcel-keller for reporting) -- [#885](https://github.com/nf-core/eager/issues/879) Specify task memory for all tools in get_software_versions to account for java incompatibilty with some SGE clusters causing hanging of the process (♥ to @maxibor for reporting) +- [#885](https://github.com/nf-core/eager/issues/879) Specify task memory for all tools in get_software_versions to account for incompatibilty of java with some SGE clusters causing hanging of the process (♥ to @maxibor for reporting) ### `Dependencies` From ebba956e36e3d53deab82a693c194adfd73f1671 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Mon, 30 May 2022 12:39:22 +0200 Subject: [PATCH 3/4] Fix issue number in link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0598f0460..d495e9837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#882](https://github.com/nf-core/eager/pull/882) Define DSL1 execution explicitly, as new versions Nextflow made DSL2 default (♥ to & fix from @Lehmann-Fabian) - [#879](https://github.com/nf-core/eager/issues/879) Add missing threads parameter for pre-clipping FastQC for single end data that caused insufficient memory in some cases (♥ to @marcel-keller for reporting) -- [#885](https://github.com/nf-core/eager/issues/879) Specify task memory for all tools in get_software_versions to account for incompatibilty of java with some SGE clusters causing hanging of the process (♥ to @maxibor for reporting) +- [#885](https://github.com/nf-core/eager/issues/885) Specify task memory for all tools in get_software_versions to account for incompatibilty of java with some SGE clusters causing hanging of the process (♥ to @maxibor for reporting) ### `Dependencies` From b974bf12ddd7507e54d3a1fc270ab09395b22765 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 3 Jun 2022 20:59:51 +0200 Subject: [PATCH 4/4] Relace bbduk version with bbversion.sh --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index a41fb058c..5f424a5a2 100644 --- a/main.nf +++ b/main.nf @@ -3145,7 +3145,7 @@ process get_software_versions { bowtie2 --version | grep -a 'bowtie2-.* -fdebug' > v_bowtie2.txt || true eigenstrat_snp_coverage --version | cut -d ' ' -f2 >v_eigenstrat_snp_coverage.txt || true mapDamage --version > v_mapdamage.txt || true - bbduk.sh -Xmx${task.memory.toGiga()}g | grep 'Last modified' | cut -d ' ' -f 3-99 > v_bbduk.txt || true + bbversion.sh > v_bbduk.txt || true bcftools --version | grep 'bcftools' | cut -d ' ' -f 2 > v_bcftools.txt || true scrape_software_versions.py &> software_versions_mqc.yaml """