nf-core · muabnezor · Jun 6, 2025 · Nov 25, 2024 · Nov 25, 2024 · Nov 25, 2024
@@ -112,6 +112,7 @@ jobs:
             test_virus_identification,
             test_single_end,
             test_concoct,
+            test_longread,
           ]
     steps:
       - name: Free some space

@@ -1,13 +1,13 @@
 repos:
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.1.0"
+    rev: "v4.0.0-alpha.8"
     hooks:
       - id: prettier
         additional_dependencies:
           - [email protected]
 
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
-    rev: "3.1.2"
+    rev: "3.2.1"
     hooks:
       - id: editorconfig-checker
         alias: ec
@@ -7,12 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### `Added`
 
-- [#799](https://github.com/nf-core/mag/pull/799) - Add `--cat_classify_unbinned`, to enable taxonomic classification of unbinned contigs using CAT (requested by @amizeranschi, added by @dialvarezs)
+- [#718](https://github.com/nf-core/mag/pull/718) - Add support for independent long-read metagenomic assembly (requested by @ljmesi and many others, added by @muabnezor)
+- [#718](https://github.com/nf-core/mag/pull/718) - Added metaMDBG and (meta)Flye as long read assemblers (added by @muabnezor)
+- [#718](https://github.com/nf-core/mag/pull/718) - Added host removal for long reads using minimap2 as aligner (added by @muabnezor)
 
 ### `Changed`
 
-- [#799](https://github.comf/nf-core/mag/pull/799) - Upgraded to latest version of CAT_pack modules (requested by @maxibor, added by @dialvarezs)
-- [#811](https://github.comf/nf-core/mag/811) - Update util modules, and remove aria2 module to replace with native Nextflow downloading of CheckM database (by @dialvarezs)
+- [#718](https://github.com/nf-core/mag/pull/718) - Refactored all assembly steps into subworkflows (added by @muabnezor)
+- [#799](https://github.com/nf-core/mag/pull/799) - Add `--cat_classify_unbinned`, to enable taxonomic classification of unbinned contigs using CAT (requested by @amizeranschi, added by @dialvarezs)
+- [#799](https://github.com/nf-core/mag/pull/799) - Upgraded to latest version of CAT_pack modules (requested by @maxibor, added by @dialvarezs)
+- [#811](https://github.com/nf-core/mag/pull/811) - Update util modules, and remove aria2 module to replace with native Nextflow downloading of CheckM database (by @dialvarezs)
 - [#816](https://github.com/nf-core/mag/pull/816) - Removed all leftover references to conda 'defaults' channel (by @jfy133)
 
 ### `Fixed`
@@ -24,6 +28,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 | CAT        | 5.2.3            | 6.0.1       |
 | centrifuge | 1.0.4.1          | 1.0.4.2     |
 | nanolyse   | 1.41.6           | 1.44.1      |
+| flye       |                  | 2.9.5       |
+| metamdbg   |                  | 1.0         |
+| minimap2   |                  | 2.28        |
+| samtools   |                  | 1.21        |
 
 ### `Deprecated`
 

@@ -66,6 +66,10 @@
 
 - [Filtlong](https://github.com/rrwick/Filtlong)
 
+- [Flye](https://www.nature.com/articles/s41592-020-00971-x)
+
+  > Kolmogorov, M., Bickhart, D.M., Behsaz, B. et al. metaFlye: scalable long-read metagenome assembly using repeat graphs. Nat Methods 17, 1103–1110 (2020). doi: 10.1038/s41592-020-00971-x
+
 - [Freebayes](https://arxiv.org/abs/1207.3907)
 
   > Garrison E, Marth G. Haplotype-based variant detection from short-read sequencing. arXiv preprint arXiv:1207.3907 [q-bio.GN] 2012
@@ -106,6 +110,14 @@
 
   > Levy Karin, E., Mirdita, M. & Söding, J. MetaEuk—sensitive, high-throughput gene discovery, and annotation for large-scale eukaryotic metagenomics. Microbiome 8, 48 (2020). 10.1186/s40168-020-00808-x
 
+- [metaMDBG](https://doi.org/10.1038/s41587-023-01983-6)
+
+  > Benoit, G., Raguideau, S., James, R. et al. High-quality metagenome assembly from long accurate reads with metaMDBG. Nat Biotechnol 42, 1378–1383 (2024). doi:10.1038/s41587-023-01983-6
+
+- [minimap2](https://doi.org/10.1093/bioinformatics/bty191)
+
+  > Li, H. (2018). Minimap2: pairwise alignment for nucleotide sequences. Bioinformatics , 34(18), 3094–3100. doi: 10.1093/bioinformatics/bty191
+
 - [MMseqs2](https://www.nature.com/articles/nbt.3988)
 
   > Steinegger, M., Söding, J. MMseqs2 enables sensitive protein sequence searching for the analysis of massive data sets. Nat Biotechnol 35, 1026–1028 (2017).10.1038/nbt.3988

@@ -40,19 +40,42 @@
                 "pattern": "^\\S+\\.f(ast)?q\\.gz$",
                 "errorMessage": "short_reads_2 needs to be a file path with no spaces.\n\nFile needs to exist!"
             },
+            "short_reads_platform": {
+                "type": "string",
+                "enum": [
+                    "ILLUMINA",
+                    "BGISEQ",
+                    "LS454",
+                    "ION_TORRENT",
+                    "DNBSEQ",
+                    "ELEMENT",
+                    "ULTIMA",
+                    "VELA_DIAGNOSTICS",
+                    "GENAPSYS",
+                    "GENEMIND",
+                    "TAPESTRI"
+                ],
+                "meta": ["sr_platform"]
+            },
             "long_reads": {
                 "type": "string",
                 "format": "file-path",
                 "exists": true,
-                "pattern": "^\\S+\\.f(ast)?q\\.gz$",
-                "errorMessage": "long_reads needs to be a file path with no spaces.\n\nFile needs to exist!"
+                "pattern": "^\\S+\\.f(ast)?q\\.gz$"
+            },
+            "long_reads_platform": {
+                "type": "string",
+                "enum": ["OXFORD_NANOPORE", "OXFORD_NANOPORE_HQ", "PACBIO_CLR", "PACBIO_HIFI"],
+                "meta": ["lr_platform"]
             }
         },
-        "required": ["sample", "group", "short_reads_1"]
+        "required": ["sample", "group"],
+        "anyOf": [{ "required": ["short_reads_1"] }, { "required": ["long_reads"] }],
+        "dependentRequired": {
+            "short_reads_2": ["short_reads_1"],
+            "short_reads_1": ["short_reads_platform"],
+            "long_reads": ["long_reads_platform"]
+        }
     },
-    "uniqueEntries": ["sample", "run"],
-    "dependentRequired": {
-        "short_reads_2": ["short_reads_1"],
-        "long_reads": ["short_reads_1", "short_reads_2"]
-    }
+    "uniqueEntries": ["sample", "run"]
 }
@@ -138,6 +138,19 @@ process {
         errorStrategy = { task.exitStatus in ((130..145) + 104 + 21 + 12 + 1) ? 'retry' : 'finish' }
         maxRetries    = 5
     }
+
+    withName: METAMDBG_ASM {
+        cpus       = { 12 * task.attempt }
+        memory     = { 72.GB * (2 ** (task.attempt - 1)) }
+        time       = { 24.h * (2 ** (task.attempt - 1)) }
+    }
+
+    withName: FLYE {
+        cpus       = { 12 * task.attempt }
+        memory     = { 72.GB * (2 ** (task.attempt - 1)) }
+        time       = { 24.h * (2 ** (task.attempt - 1)) }
+    }
+
     //returns exit code 247 when running out of memory
     withName: BOWTIE2_ASSEMBLY_ALIGN {
         cpus          = { 2 * task.attempt }

@@ -199,13 +199,13 @@ process {
     }
 
     withName: FILTLONG {
-        ext.args   = [
+        ext.args  = { [
             "--min_length ${params.longreads_min_length}",
             "--keep_percent ${params.longreads_keep_percent}",
-            "--trim",
+            shortreads ? "--trim" : "",
             "--length_weight ${params.longreads_length_weight}",
-            params.longreads_min_quality ? "--min_mean_q ${params.longreads_min_quality}" : '',
-        ].join(' ').trim()
+            params.longreads_min_quality ? "--min_mean_q ${params.longreads_min_quality}" : ''
+        ].join(' ').trim() }
         publishDir = [
             path: { "${params.outdir}/QC_longreads/Filtlong" },
             mode: params.publish_dir_mode,
@@ -306,6 +306,66 @@ process {
         ]
     }
 
+    withName: MINIMAP2_HOST_INDEX {
+        publishDir = [
+            path: { "${params.outdir}/QC_longreads/minimap2/index" },
+            mode: params.publish_dir_mode,
+            pattern: '*.mmi',
+        ]
+    }
+
+    withName: MINIMAP2_HOST_ALIGN {
+        ext.args = {
+            meta.lr_platform == "OXFORD_NANOPORE" ? "-x map-ont" :
+            meta.lr_platform == "OXFORD_NANOPORE_HQ" ? "-x lr:hq" :
+            meta.lr_platform == "PACBIO_CLR" ? "-x map-pb" :
+            meta.lr_platform == "PACBIO_HIFI" ? "-x map-hifi" : ""
+        }
+        ext.prefix = { "${meta.id}_run${meta.run}_host_minimap" }
+        publishDir = [
+            path: { "${params.outdir}/QC_longreads/minimap2/align}" },
+            mode: params.publish_dir_mode,
+            pattern: "*.bam",
+            enabled: params.save_hostremoved_reads
+        ]
+    }
+
+    withName: MINIMAP2_ASSEMBLY_ALIGN {
+        ext.args = {
+            meta.lr_platform == "OXFORD_NANOPORE" ? "-x map-ont" :
+            meta.lr_platform == "OXFORD_NANOPORE_HQ" ? "-x lr:hq" :
+            meta.lr_platform == "PACBIO_CLR" ? "-x map-pb" :
+            meta.lr_platform == "PACBIO_HIFI" ? "-x map-hifi" : ""
+        }
+        ext.prefix = { "${meta.assembler}-${meta.id}-${meta2.id}" }
+        publishDir = [
+            path: { "${params.outdir}/Assembly/${meta2.assembler}/QC/${meta2.id}" },
+            mode: params.publish_dir_mode,
+            pattern: "*.{bam,bai}",
+            enabled: params.save_assembly_mapped_reads
+        ]
+    }
+
+    withName: SAMTOOLS_HOSTREMOVED_UNMAPPED {
+        ext.args = '-f 4'
+        ext.prefix = { "${meta.id}_${meta.run}_hostremoved" }
+        publishDir = [
+            path: { "${params.outdir}/QC_longreads/samtools/fastq" },
+            mode: params.publish_dir_mode,
+            pattern: '*_hostremoved.fastq.gz',
+            enabled: params.save_hostremoved_reads
+        ]
+    }
+
+    withName: SAMTOOLS_HOSTREMOVED_STATS {
+        ext.prefix = { "${meta.id}_${meta.run_accession}" }
+        publishDir = [
+            path: { "${params.outdir}/QC_longreads/samtools/stats" },
+            mode: params.publish_dir_mode,
+            pattern: '*stats'
+        ]
+    }
+
     withName: CENTRIFUGE_CENTRIFUGE {
         publishDir = [path: { "${params.outdir}/Taxonomy/centrifuge/${meta.id}" }, mode: params.publish_dir_mode, pattern: "*.txt"]
     }
@@ -350,6 +410,17 @@ process {
         publishDir = [path: { "${params.outdir}/Assembly/SPAdesHybrid" }, mode: params.publish_dir_mode, pattern: "*.{fasta.gz,gfa.gz,fa.gz,log}"]
     }
 
+    withName: FLYE {
+        ext.args   = ' --meta'
+        ext.prefix = { "FLYE-${meta.id}" }
+        publishDir = [path: { "${params.outdir}/Assembly/FLYE" }, mode: params.publish_dir_mode, pattern: "*.{fasta.gz,gfa.gz,log}"]
+    }
+
+    withName: METAMDBG_ASM {
+        ext.prefix = { "METAMDBG-${meta.id}" }
+        publishDir = [path: { "${params.outdir}/Assembly/METAMDBG" }, mode: params.publish_dir_mode, pattern: "*.{fasta.gz,log}"]
+    }
+
     withName: QUAST {
         publishDir = [path: { "${params.outdir}/Assembly/${meta.assembler}/QC/${meta.id}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename }]
     }
@@ -667,7 +738,14 @@ process {
         ]
     }
 
-    withName: METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS {
+    withName: METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS_SHORTREAD {
+        ext.args = params.shortread_percentidentity ? "--percentIdentity ${params.shortread_percentidentity}" : ''
+        publishDir = [path: { "${params.outdir}/GenomeBinning/depths/contigs" }, mode: params.publish_dir_mode, pattern: '*-depth.txt.gz']
+        ext.prefix = { "${meta.assembler}-${meta.id}-depth" }
+    }
+
+    withName: METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS_LONGREAD {
+        ext.args = params.longread_percentidentity ? "--percentIdentity ${params.longread_percentidentity}" : ''
         publishDir = [path: { "${params.outdir}/GenomeBinning/depths/contigs" }, mode: params.publish_dir_mode, pattern: '*-depth.txt.gz']
         ext.prefix = { "${meta.assembler}-${meta.id}-depth" }
     }

@@ -23,7 +23,7 @@ params {
     config_profile_description  = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.multirun.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.multirun.v4.csv'
     centrifuge_db               = params.pipelines_testdata_base_path + 'mag/test_data/minigut_cf.tar.gz'
     kraken2_db                  = params.pipelines_testdata_base_path + 'mag/test_data/minigut_kraken.tgz'
     skip_krona                  = false

@@ -24,7 +24,7 @@ params {
     config_profile_description  = 'Minimal test dataset to check pipeline function with AdapterRemoval data and domain classification.'
 
     // Input data
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.euk.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.euk.v4.csv'
     centrifuge_db               = params.pipelines_testdata_base_path + 'mag/test_data/minigut_cf.tar.gz'
     kraken2_db                  = params.pipelines_testdata_base_path + 'mag/test_data/minigut_kraken.tgz'
     metaeuk_db                  = params.pipelines_testdata_base_path + '/modules/data/proteomics/database/yeast_UPS.fasta'

@@ -25,7 +25,7 @@ params {
     config_profile_description           = 'Minimal test dataset to check pipeline function for ancient DNA step'
 
     // Input data
-    input                                = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.csv'
+    input                                = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     centrifuge_db                        = params.pipelines_testdata_base_path + 'mag/test_data/minigut_cf.tar.gz'
     kraken2_db                           = params.pipelines_testdata_base_path + 'mag/test_data/minigut_kraken.tgz'
     clip_tool                            = 'trimmomatic'

@@ -24,7 +24,7 @@ params {
     config_profile_description  = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     keep_phix                   = true
     skip_clipping               = true
     skip_prokka                 = true

@@ -24,7 +24,7 @@ params {
     config_profile_description    = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                         = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.csv'
+    input                         = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     assembly_input                = params.pipelines_testdata_base_path + 'mag/samplesheets/assembly_samplesheet.csv'
     centrifuge_db                 = params.pipelines_testdata_base_path + 'mag/test_data/minigut_cf.tar.gz'
     kraken2_db                    = params.pipelines_testdata_base_path + 'mag/test_data/minigut_kraken.tgz'

@@ -24,7 +24,7 @@ params {
     config_profile_description  = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     cat_db                      = params.pipelines_testdata_base_path + 'mag/test_data/minigut_cat.tar.gz'
     skip_spades                 = true
     min_length_unbinned_contigs = 1

@@ -25,7 +25,7 @@ params {
     config_profile_description = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                      = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.csv'
+    input                      = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.v4.csv'
     centrifuge_db              = null
     kraken2_db                 = null
     skip_krona                 = true

@@ -17,7 +17,7 @@ params {
     // Input data for full size test
     // hg19 reference with highly conserved and low-complexity regions masked by Brian Bushnell
     host_fasta                 = "s3://ngi-igenomes/test-data/mag/hg19_main_mask_ribo_animal_allplant_allfungus.fa.gz"
-    input                      = "s3://ngi-igenomes/test-data/mag/samplesheets/samplesheet.full.csv"
+    input                      = "s3://ngi-igenomes/test-data/mag/samplesheets/samplesheet.full.v4.csv"
 
     //centrifuge_db = "s3://ngi-igenomes/test-data/mag/p_compressed+h+v.tar.gz"
     kraken2_db                 = "s3://ngi-igenomes/test-data/mag/minikraken_8GB_202003.tgz"

@@ -25,7 +25,7 @@ params {
 
     // Input data
     host_fasta                  = params.pipelines_testdata_base_path + 'mag/host_reference/genome.hg38.chr21_10000bp_region.fa'
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.host_rm.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.host_rm.v4.csv'
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
     busco_db                    = params.pipelines_testdata_base_path + 'mag/databases/busco/bacteria_odb10.2024-01-08.tar.gz'

@@ -24,7 +24,7 @@ params {
     config_profile_description  = 'Minimal test dataset to check pipeline function'
 
     // Input data
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.hybrid.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.hybrid.v4.csv'
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
     busco_db                    = params.pipelines_testdata_base_path + 'mag/databases/busco/bacteria_odb10.2024-01-08.tar.gz'

@@ -25,7 +25,7 @@ params {
 
     // Input data
     host_fasta                  = params.pipelines_testdata_base_path + 'mag/host_reference/genome.hg38.chr21_10000bp_region.fa'
-    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.hybrid_host_rm.csv'
+    input                       = params.pipelines_testdata_base_path + 'mag/samplesheets/samplesheet.hybrid_host_rm.v4.csv'
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
     skip_binqc                  = true