From 8daafa38b5f60dd6654f87801fa6f71fbadffe33 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 19 Jan 2023 15:17:19 -0800 Subject: [PATCH 01/31] Adjust argument parser to have synteny mode in ntjoin_assemble.py --- bin/ntjoin_assemble.py | 59 ++++++++++++++++++++++++++---------------- ntJoin | 2 +- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index fcf7ebe..d7d48b6 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -951,46 +951,61 @@ def find_mx_min_max(graph, target): def parse_arguments(): "Parse ntJoin arguments" parser = argparse.ArgumentParser( - description="ntJoin: Scaffolding genome assemblies using reference assemblies and minimizer graphs", - epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" + description="ntJoin: Genome analysis using reference assemblies and minimizer graphs", + ) + parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + + subparsers = parser.add_subparsers() + scaffold_parser = subparsers.add_parser("scaffold", + help="Scaffold the input target assembly using the supplied reference(s)", + epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" "Example: myscaffolds.fa.k32.w1000.tsv - myscaffolds.fa is the expected matching fasta", formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") - parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) - parser.add_argument("-l", help="Weight of target genome assembly [1]", + scaffold_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") + scaffold_parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) + scaffold_parser.add_argument("-l", help="Weight of target genome assembly [1]", required=False, default=1, type=float) - parser.add_argument("-r", + scaffold_parser.add_argument("-r", help="List of reference assembly weights (in quotes, separated by spaces, " "in same order as minimizer TSV files)", required=True, type=str) - parser.add_argument("-p", help="Output prefix [out]", default="out", + scaffold_parser.add_argument("-p", help="Output prefix [out]", default="out", type=str, required=False) - parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) - parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) - parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, + scaffold_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) + scaffold_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) + scaffold_parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) + scaffold_parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, default=0, type=int) - parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", + scaffold_parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", action='store_true') - parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " + scaffold_parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " "increasing/decreasing to assign contig orientation [90]\n " "Note: Only used with --mkt is NOT specified", default=90, type=int) - parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) - parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") - parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", + scaffold_parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) + scaffold_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + scaffold_parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") + scaffold_parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", action="store_true") - parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", + scaffold_parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", action="store_true") - parser.add_argument("--overlap_gap", help="Length of gap introduced between overlapping, trimmed segments [20]", + scaffold_parser.add_argument("--overlap_gap", + help="Length of gap introduced between overlapping, trimmed segments [20]", type=int, default=20) - parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", + scaffold_parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", type=int, default=15) - parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", + scaffold_parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", type=int, default=10) - parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " + scaffold_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " "(computing minimizers, reading fasta file) [4]", type=int, default=4) + + synteny_parser = subparsers.add_parser("synteny", help="Extract syntenic blocks from input assemblies") + synteny_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of input assemblies") + synteny_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) + synteny_parser.add_argument("-p", help="Output prefix [out]", + default="out", type=str, required=False) + synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + return parser.parse_args() def print_parameters(self): diff --git a/ntJoin b/ntJoin index 4cc0737..a55b2ae 100755 --- a/ntJoin +++ b/ntJoin @@ -224,7 +224,7 @@ ifeq ($(overlap), True) endif $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa: $(target).k$(k).w$(w).tsv $(min_ref_targets) - $(log_time) $(assemble_path)/bin/ntjoin_assemble.py -p $(prefix) -n $(n) -s $< -l $(target_weight) \ + $(log_time) $(assemble_path)/bin/ntjoin_assemble.py scaffold -p $(prefix) -n $(n) -s $< -l $(target_weight) \ -r "$(reference_weights_list)" -k $(k) -g $(g) -G $(G) -t $(assemble_t) $(ntjoin_assembly_opts) $(min_ref_targets) $(target).k$(k).w$(w).n$(n).unassigned.scaffolds.fa: $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa From 9272a60f376a85cd9a12592455a0ecf6b36c22ab Mon Sep 17 00:00:00 2001 From: lcoombe Date: Mon, 6 Feb 2023 11:21:00 -0800 Subject: [PATCH 02/31] Integrate synteny option to ntjoin_assemble.py, print appropriate params --- bin/ntjoin_assemble.py | 90 +++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index d7d48b6..a66d514 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -21,6 +21,7 @@ from read_fasta import read_fasta import ntjoin_utils import ntjoin_overlap +import ntjoin_synteny warnings.simplefilter(action='ignore', category=RuntimeWarning) @@ -427,7 +428,10 @@ def find_paths_process(self, component): num_edges == len(subcomponent_graph.es()) and len(path) == len(set(path)): # All the nodes/edges from the graph are in the simple path, no repeated nodes path = self.convert_path_index_to_name(subcomponent_graph, path) - ctg_path = self.format_path(path, self.args.s, + if self.args.mode == "synteny": + ctg_path = ntjoin_synteny.find_synteny_blocks(path, Ntjoin.list_mx_info, self.args.k) + else: + ctg_path = self.format_path(path, self.args.s, subcomponent_graph) return_paths.append(ctg_path) return return_paths @@ -955,7 +959,7 @@ def parse_arguments(): ) parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - subparsers = parser.add_subparsers() + subparsers = parser.add_subparsers(dest="mode") scaffold_parser = subparsers.add_parser("scaffold", help="Scaffold the input target assembly using the supplied reference(s)", epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" @@ -1001,15 +1005,21 @@ def parse_arguments(): synteny_parser = subparsers.add_parser("synteny", help="Extract syntenic blocks from input assemblies") synteny_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of input assemblies") - synteny_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) + synteny_parser.add_argument("-n", help="Minimum edge weight [Number of input assemblies]", default=0, type=int) synteny_parser.add_argument("-p", help="Output prefix [out]", default="out", type=str, required=False) + synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + if len(sys.argv) == 1: + parser.print_help() + sys.exit() + return parser.parse_args() - def print_parameters(self): - "Print the set parameters for the ntJoin run" + def print_parameters_scaffold(self): + "Print the set parameters for the ntJoin scaffolding run" + print("Running ntJoin scaffolding..") print("Parameters:") print("\tReference TSV files: ", self.args.FILES) print("\t-s ", self.args.s) @@ -1036,11 +1046,41 @@ def print_parameters(self): print("\t--overlap_w", self.args.overlap_w) print("\t--btllib_t", self.args.btllib_t) + def print_parameters_synteny(self): + "Pring the set parameters for the ntJoin synteny run" + if self.args.n == 0: + self.args.n = len(self.args.FILES) + print("Running ntJoin synteny detection...") + print("Parameters:") + print("\tMinimizer TSV files: ", self.args.FILES) + print("\t-n", self.args.n) + print("\t-p", self.args.p) + print("\t-k", self.args.k) + + + def print_parameters(self): + "Print the parameters for the ntJoin in the specified mode" + if self.args.mode == "scaffold": + self.print_parameters_scaffold() + elif self.args.mode == "synteny": + self.print_parameters_synteny() + else: + raise ValueError(f"Unexpected mode: {self.args.mode}") + + def set_synteny_parameters(self): + "Set the default parameters for synteny mode" + self.args.r = " ".join(["1"] * len(self.args.FILES)) + self.args.t = 1 + def main(self): "Run ntJoin graph stage" print("Running ntJoin v1.1.1 ...\n") + print(self.args, self.args.n) self.print_parameters() + if self.args.mode == "synteny": + self.set_synteny_parameters() + # Parse the weights of each input reference assembly input_weights = [float(w) for w in re.split(r'\s+', self.args.r)] if len(input_weights) != len(self.args.FILES): @@ -1059,10 +1099,11 @@ def main(self): list_mx_info[assembly] = mxs_info list_mxs[assembly] = mxs weights[assembly] = input_weights.pop(0) - mxs_info, mxs = self.read_minimizers(self.args.s) - list_mx_info[self.args.s] = mxs_info - list_mxs[self.args.s] = mxs - weights[self.args.s] = self.args.l + if self.args.mode == "scaffold": + mxs_info, mxs = self.read_minimizers(self.args.s) + list_mx_info[self.args.s] = mxs_info + list_mxs[self.args.s] = mxs + weights[self.args.s] = self.args.l weight_str = "\n".join([f"{assembly}: {asm_weight}" for assembly, asm_weight in weights.items()]) print("\nWeights of assemblies:\n", weight_str, "\n", sep="") @@ -1082,22 +1123,34 @@ def main(self): graph = self.filter_graph_global(graph) # Find the min and max pos of minimizers for target assembly, per ctg - Ntjoin.mx_extremes = self.find_mx_min_max(graph, self.args.s) + if self.args.mode == "scaffold": + Ntjoin.mx_extremes = self.find_mx_min_max(graph, self.args.s) # Load target scaffolds into memory - min_match = re.search(r'^(\S+).k\d+.w\d+\.tsv', self.args.s) - if not min_match: - print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") - print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") - sys.exit(1) - assembly_fa = min_match.group(1) - scaffolds = self.read_fasta_file(assembly_fa) # scaffold_id -> Scaffold + if self.args.mode == "scaffold": + min_match = re.search(r'^(\S+).k\d+.w\d+\.tsv', self.args.s) + if not min_match: + print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") + print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") + sys.exit(1) + assembly_fa = min_match.group(1) + scaffolds = self.read_fasta_file(assembly_fa) # scaffold_id -> Scaffold - Ntjoin.scaffolds = scaffolds + Ntjoin.scaffolds = scaffolds # Find the paths through the graph paths, incorporated_segments = self.find_paths(graph) + if self.args.mode == "synteny": + with open(f"{self.args.p}.synteny_blocks.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + print(datetime.datetime.today(), ": DONE!", file=sys.stdout) + sys.exit() + Ntjoin.incorporated_segments = incorporated_segments if self.args.no_cut: @@ -1115,6 +1168,7 @@ def __init__(self): "Create an ntJoin instance" self.args = self.parse_arguments() + def main(): "Run ntJoin" Ntjoin().main() From 5572b238eeec969f3baae428b599c9fc8f0ae972 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 8 Feb 2023 15:25:46 -0800 Subject: [PATCH 03/31] Add synteny target to ntJoin makefile --- ntJoin | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ntJoin b/ntJoin index a55b2ae..48fc0f9 100755 --- a/ntJoin +++ b/ntJoin @@ -161,6 +161,8 @@ analysis: check_params_analysis \ all: check_params assemble analysis +synteny: $(prefix).synteny_blocks.tsv + check_params: ifeq ($(references), None) ifeq ($(reference_config), None) @@ -195,7 +197,7 @@ ifneq ($(shell command -v 'python3 -c "import btllib"'),) $(error ERROR: btllib not found. Please ensure btllib is installed) endif -.PHONY: help all version analysis assemble check_params jupiter check_install +.PHONY: help all version analysis assemble check_params jupiter check_install synteny .DELETE_ON_ERROR: $(prefix).n$(n).mx.dot .SECONDARY: @@ -248,3 +250,7 @@ else quast -t $(t) -o quast_$(prefix) -r $(ref) --fast --scaffold-gap-max-size 100000 --split-scaffolds \ $^ endif + +# Synteny +$(prefix).synteny_blocks.tsv: $(min_ref_targets) + $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) From 59352697fdf3b6d16a48ea9c80f201c34dda0f55 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 22 Feb 2023 16:34:36 -0800 Subject: [PATCH 04/31] After determining first synteny blocks, print out masked assembly files * Mask the regions of the contigs that are covered by synteny blocks * Allow for w-length wiggle room --- bin/ntjoin_assemble.py | 6 +- bin/ntjoin_synteny.py | 170 +++++++++++++++++++++++++++++++++++++++++ ntJoin | 5 +- 3 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 bin/ntjoin_synteny.py diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index a66d514..a4de262 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -1009,6 +1009,7 @@ def parse_arguments(): synteny_parser.add_argument("-p", help="Output prefix [out]", default="out", type=str, required=False) synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) + synteny_parser.add_argument("-w", help="Window sie used for minimizers", required=True, type=int) synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: @@ -1148,7 +1149,10 @@ def main(self): for block in subcomponent: outfile.write(block.get_block_string(block_num)) block_num += 1 - print(datetime.datetime.today(), ": DONE!", file=sys.stdout) + print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) + # Ready to start refining the synteny block coordinates + ntjoin_synteny.generate_additional_minimizers(paths, self.args.w) + sys.exit() Ntjoin.incorporated_segments = incorporated_segments diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py new file mode 100644 index 0000000..bcdebdc --- /dev/null +++ b/bin/ntjoin_synteny.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +ntJoin: Identifying synteny between genome assemblies using minimizer graphs +Written by Lauren Coombe @lcoombe +""" + +from collections import namedtuple +import ntjoin_utils +import pybedtools +from pybedtools.featurefuncs import extend_fields #pylint: disable=[no-name-in-module] +import re +import sys + +# Named tuples +Minimizer = namedtuple("Minimizer", ["mx", "position"]) + +# Regexes +fai_re = re.compile(r'^(\S+).k\d+.w\d+.tsv') + +class SyntenyBlock: + "A Synteny Block between the input assemblies" + def __init__(self, k, *assemblies): + "Instantiate a dictionary to keep track of assembly blocks for this synteny block" + self.assembly_blocks = {assembly: AssemblyBlock() for assembly in assemblies} + self.k = k # k-mer size used for minimizers, needed to adjust end coordinates + + + def continue_block(self, mx, list_mx_info): + "Given the minimizer and preliminary blocks, return if synteny block should extend, else False" + return all(mx_dict[mx][0] == self.assembly_blocks[assembly].contig_id \ + for assembly, mx_dict in list_mx_info.items()) + + def extend_block(self, mx, list_mx_jnfo): + "Extend the synteny block by extending each assembly block" + for assembly, mx_dict in list_mx_jnfo.items(): + ctg, pos = mx_dict[mx] + assert self.assembly_blocks[assembly].contig_id == ctg + self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) + + def start_block(self, mx, list_mx_info): + "Start the new synteny block" + for assembly, mx_dict in list_mx_info.items(): + ctg, pos = mx_dict[mx] + self.assembly_blocks[assembly].contig_id = ctg + self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) + + def determine_orientations(self): + "Determine the orientations of each assembly block" + for _, assembly_block in self.assembly_blocks.items(): + positions = [mx.position for mx in assembly_block.minimizers] + if all(x < y for x, y in zip(positions, positions[1:])): + assembly_block.ori = "+" + elif all(x > y for x, y in zip(positions, positions[1:])): + assembly_block.ori = "-" + else: + assembly_block.ori = "?" + + def all_oriented(self): + "Return true if all of the assembly blocks in the synteny block are oriented" + return all(assembly_block.ori in ["+", "-"] for _, assembly_block in self.assembly_blocks.items()) + + def get_block_string(self, num): + "Given the specified synteny block ID, print the synteny blocks" + return_str = "" + for assembly, assembly_block in self.assembly_blocks.items(): + start_pos = assembly_block.get_block_start() + end_pos = assembly_block.get_block_end() + self.k + block_string = f"{num}\t{assembly}\t{assembly_block.contig_id}\t{start_pos}" \ + f"\t{end_pos}\t{assembly_block.ori}\n" + return_str += block_string + return return_str + + +class AssemblyBlock: + "An assembly block for a given assembly. The AssemblyBlock objects per assembly make up a SyntenyBlock" + def __init__(self): + "Instantiate the AssemblyBlock" + self.contig_id = None + self.minimizers = [] + self.ori = None + + def get_block_start(self): + "Get the starting coordinate of the assembly block" + return min(self.minimizers[0].position, self.minimizers[-1].position) + + def get_block_end(self): + "Get the end coordinate of the assembly block" + return max(self.minimizers[0].position, self.minimizers[-1].position) + + +def find_synteny_blocks(path, list_mx_info, k): + "Given a path (sequence of mx), print the order/orientation/regions of contigs for an assembly" + out_blocks = [] # List of SyntenyBlock + prelim_blocks = SyntenyBlock(k, *list(list_mx_info.keys())) + past_start_flag = False + for mx in path: + if prelim_blocks.continue_block(mx, list_mx_info): + prelim_blocks.extend_block(mx, list_mx_info) + else: + # This is either the first mx, or we are past a stretch of repeating contigs + if past_start_flag: + prelim_blocks.determine_orientations() + if prelim_blocks.all_oriented(): + out_blocks.append(prelim_blocks) + prelim_blocks = SyntenyBlock(k, *list(list_mx_info.keys())) + prelim_blocks.start_block(mx, list_mx_info) + + prelim_blocks.determine_orientations() + if prelim_blocks.all_oriented(): + out_blocks.append(prelim_blocks) + + return out_blocks + +def find_fa_name(assembly_mx_name): + "Given the mx file name, return the corresponding fai file name" + if fai_match := re.search(fai_re, assembly_mx_name): + return f"{fai_match.group(1)}" + print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") + print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") + sys.exit(1) + +def add_name_bed(f): + "Add name feature to the BedTool" + f = extend_fields(f, 4) + f.name = "complement" + return f + +def get_synteny_bed_lists(paths, w): + "Given a set of synteny blocks, return a dictionary with a Bed interval lists per contig, per assembly" + synteny_beds = {} + for subcomponent in paths: + for block in subcomponent: + for assembly, assembly_block in block.assembly_blocks.items(): + if assembly not in synteny_beds: + synteny_beds[assembly] = {} + if assembly_block.contig_id not in synteny_beds[assembly]: + synteny_beds[assembly][assembly_block.contig_id] = [] + synteny_beds[assembly][assembly_block.contig_id].append( + ntjoin_utils.Bed(assembly_block.contig_id, + assembly_block.get_block_start() + w, + assembly_block.get_block_end() + block.k - w)) + return synteny_beds + +def mask_assemblies_with_synteny_extents(synteny_beds): + "Mask each reference assembly with determined synteny blocks" + mx_to_fa_dict = {} + for assembly, contig_dict in synteny_beds.items(): + bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ + for bed in contig_dict[ctg]] + bed_str = "\n".join(bed_str) + synteny_bed = pybedtools.BedTool(bed_str, from_string=True).sort() + fa_filename = find_fa_name(assembly) + synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") + mx_to_fa_dict[assembly] = fa_filename + return mx_to_fa_dict + + +def generate_additional_minimizers(paths, w): + "Given the existing synteny blocks, generate minimizers for increased block resolution" + synteny_beds = get_synteny_bed_lists(paths, w) + mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) + + + + + + + + + diff --git a/ntJoin b/ntJoin index 48fc0f9..b84d878 100755 --- a/ntJoin +++ b/ntJoin @@ -161,7 +161,8 @@ analysis: check_params_analysis \ all: check_params assemble analysis -synteny: $(prefix).synteny_blocks.tsv +synteny: $(fai_ref_targets) \ + $(prefix).synteny_blocks.tsv check_params: ifeq ($(references), None) @@ -253,4 +254,4 @@ endif # Synteny $(prefix).synteny_blocks.tsv: $(min_ref_targets) - $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) + $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) -w$(w) From aed8da436f5b6f29c6d19e6f1baadf5d2e12cb11 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 22 Feb 2023 16:41:25 -0800 Subject: [PATCH 05/31] Fixes for pylint - ntjoin_synteny.py --- bin/ntjoin_synteny.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index bcdebdc..19f3333 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -5,11 +5,10 @@ """ from collections import namedtuple -import ntjoin_utils -import pybedtools -from pybedtools.featurefuncs import extend_fields #pylint: disable=[no-name-in-module] import re import sys +import ntjoin_utils +import pybedtools # Named tuples Minimizer = namedtuple("Minimizer", ["mx", "position"]) @@ -26,7 +25,7 @@ def __init__(self, k, *assemblies): def continue_block(self, mx, list_mx_info): - "Given the minimizer and preliminary blocks, return if synteny block should extend, else False" + "Given minimizer and preliminary blocks, return if synteny block should extend, else False" return all(mx_dict[mx][0] == self.assembly_blocks[assembly].contig_id \ for assembly, mx_dict in list_mx_info.items()) @@ -78,7 +77,7 @@ def __init__(self): self.contig_id = None self.minimizers = [] self.ori = None - + def get_block_start(self): "Get the starting coordinate of the assembly block" return min(self.minimizers[0].position, self.minimizers[-1].position) @@ -119,12 +118,6 @@ def find_fa_name(assembly_mx_name): print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") sys.exit(1) -def add_name_bed(f): - "Add name feature to the BedTool" - f = extend_fields(f, 4) - f.name = "complement" - return f - def get_synteny_bed_lists(paths, w): "Given a set of synteny blocks, return a dictionary with a Bed interval lists per contig, per assembly" synteny_beds = {} @@ -159,12 +152,3 @@ def generate_additional_minimizers(paths, w): "Given the existing synteny blocks, generate minimizers for increased block resolution" synteny_beds = get_synteny_bed_lists(paths, w) mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) - - - - - - - - - From a09247c017317ebed591f818ee56d664e94d67bf Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 23 Feb 2023 09:27:16 -0800 Subject: [PATCH 06/31] Generate minimizers from masked fasta, read them in --- bin/ntjoin_assemble.py | 39 ++++++++------------------------------- bin/ntjoin_synteny.py | 18 ++++++++++++++++-- bin/ntjoin_utils.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index a4de262..5c3721b 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -55,33 +55,6 @@ def convert_path_index_to_name(graph, path): "Convert path of vertex indices to path of vertex names" return [Ntjoin.vertex_name(graph, vs) for vs in path] - @staticmethod - def read_minimizers(tsv_filename): - "Read the minimizers from a file, removing duplicate minimizers" - print(datetime.datetime.today(), ": Reading minimizers", tsv_filename, file=sys.stdout) - mx_info = {} # mx -> (contig, position) - mxs = [] # List of lists of minimizers - dup_mxs = set() # Set of minimizers identified as duplicates - with open(tsv_filename, 'r', encoding="utf-8") as tsv: - for line in tsv: - line = line.strip().split("\t") - if len(line) > 1: - mx_pos_split = line[1].split(" ") - mxs.append([mx_pos.split(":")[0] for mx_pos in mx_pos_split]) - for mx_pos in mx_pos_split: - mx, pos = mx_pos.split(":") - if mx in mx_info: # This is a duplicate, add to dup set, don't add to dict - dup_mxs.add(mx) - else: - mx_info[mx] = (line[0], int(pos)) - - mx_info = {mx: mx_entry_info for mx, mx_entry_info in mx_info.items() if mx not in dup_mxs} - mxs_filt = [] - for mx_list in mxs: - mx_list_filt = [mx for mx in mx_list if mx not in dup_mxs] - mxs_filt.append(mx_list_filt) - return mx_info, mxs_filt - @staticmethod def calc_total_weight(list_files, weights): "Calculate the total weight of an edge given the assembly support" @@ -1009,7 +982,9 @@ def parse_arguments(): synteny_parser.add_argument("-p", help="Output prefix [out]", default="out", type=str, required=False) synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - synteny_parser.add_argument("-w", help="Window sie used for minimizers", required=True, type=int) + synteny_parser.add_argument("-w", help="Window size used for minimizers", required=True, type=int) + synteny_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions "\ + "(computing minimizers, reading fasta file) [4]", type=int, default=4) synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: @@ -1057,6 +1032,8 @@ def print_parameters_synteny(self): print("\t-n", self.args.n) print("\t-p", self.args.p) print("\t-k", self.args.k) + print("\t-w", self.args.w) + print("\t--btllib_t", self.args.btllib_t) def print_parameters(self): @@ -1096,12 +1073,12 @@ def main(self): list_mxs = {} # Dictionary: assembly -> [lists of mx] weights = {} # Dictionary: assembly -> weight for assembly in self.args.FILES: - mxs_info, mxs = self.read_minimizers(assembly) + mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) list_mx_info[assembly] = mxs_info list_mxs[assembly] = mxs weights[assembly] = input_weights.pop(0) if self.args.mode == "scaffold": - mxs_info, mxs = self.read_minimizers(self.args.s) + mxs_info, mxs = ntjoin_utils.read_minimizers(self.args.s) list_mx_info[self.args.s] = mxs_info list_mxs[self.args.s] = mxs weights[self.args.s] = self.args.l @@ -1151,7 +1128,7 @@ def main(self): block_num += 1 print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) # Ready to start refining the synteny block coordinates - ntjoin_synteny.generate_additional_minimizers(paths, self.args.w) + ntjoin_synteny.generate_additional_minimizers(paths, self.args.w, self.args.btllib_t) sys.exit() diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 19f3333..f961586 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -132,6 +132,7 @@ def get_synteny_bed_lists(paths, w): ntjoin_utils.Bed(assembly_block.contig_id, assembly_block.get_block_start() + w, assembly_block.get_block_end() + block.k - w)) + return synteny_beds def mask_assemblies_with_synteny_extents(synteny_beds): @@ -144,11 +145,24 @@ def mask_assemblies_with_synteny_extents(synteny_beds): synteny_bed = pybedtools.BedTool(bed_str, from_string=True).sort() fa_filename = find_fa_name(assembly) synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") - mx_to_fa_dict[assembly] = fa_filename + mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" return mx_to_fa_dict +def generate_new_minimizers(tsv_to_fa_dict, k, w, t): + "Given the masked fasta files, generate minimizers at new w for each" + list_mx_info = {} + list_mxs = {} + for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): + indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, int(w/10), t) + mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) + list_mx_info[assembly_tsv] = mx_info + list_mxs[assembly_tsv] = mxs_filt + return list_mx_info, list_mxs + -def generate_additional_minimizers(paths, w): +def generate_additional_minimizers(paths, w, t): "Given the existing synteny blocks, generate minimizers for increased block resolution" + k = paths[0][0].k synteny_beds = get_synteny_bed_lists(paths, w) mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) + list_mx_info, list_mxs = generate_new_minimizers(mx_to_fa_dict, k, w, t) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index dd9985f..8a548e9 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -6,6 +6,8 @@ import datetime from collections import namedtuple +import shlex +import subprocess import sys import os @@ -45,6 +47,40 @@ def filter_minimizers(list_mxs): return return_mxs +def read_minimizers(tsv_filename): + "Read the minimizers from a file, removing duplicate minimizers" + print(datetime.datetime.today(), ": Reading minimizers", tsv_filename, file=sys.stdout) + mx_info = {} # mx -> (contig, position) + mxs = [] # List of lists of minimizers + dup_mxs = set() # Set of minimizers identified as duplicates + with open(tsv_filename, 'r', encoding="utf-8") as tsv: + for line in tsv: + line = line.strip().split("\t") + if len(line) > 1: + mx_pos_split = line[1].split(" ") + mxs.append([mx_pos.split(":")[0] for mx_pos in mx_pos_split]) + for mx_pos in mx_pos_split: + mx, pos = mx_pos.split(":") + if mx in mx_info: # This is a duplicate, add to dup set, don't add to dict + dup_mxs.add(mx) + else: + mx_info[mx] = (line[0], int(pos)) + + mx_info = {mx: mx_entry_info for mx, mx_entry_info in mx_info.items() if mx not in dup_mxs} + mxs_filt = [] + for mx_list in mxs: + mx_list_filt = [mx for mx in mx_list if mx not in dup_mxs] + mxs_filt.append(mx_list_filt) + return mx_info, mxs_filt + +def run_indexlr(assembly, k, w, t): + "Run indexlr on the given assembly with the specified k and w" + cmd = f"indexlr {assembly} --long --pos -k{k} -w{w} -t{t} -o {assembly}.k{k}.w{w}.tsv" + cmd = shlex.split(cmd) + ret_code = subprocess.call(cmd) + assert ret_code == 0 + return f"{assembly}.k{k}.w{w}.tsv" + # Defining helper classes class PathNode: From a36bb78f02fd75212ecebc9bfd1a369498dbb922 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 23 Feb 2023 16:02:56 -0800 Subject: [PATCH 07/31] Finishing first implementation of block extension --- bin/ntjoin_assemble.py | 39 ++++++++++++----- bin/ntjoin_synteny.py | 96 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 19 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 5c3721b..5973829 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -17,11 +17,11 @@ import igraph as ig import pybedtools import pymannkendall as mk -import btllib from read_fasta import read_fasta import ntjoin_utils import ntjoin_overlap import ntjoin_synteny +import btllib warnings.simplefilter(action='ignore', category=RuntimeWarning) @@ -61,10 +61,12 @@ def calc_total_weight(list_files, weights): return sum([weights[f] for f in list_files]) - def build_graph(self, list_mxs, weights): + def build_graph(self, list_mxs, weights, graph=None, black_list=None): "Builds an undirected graph: nodes=minimizers; edges=between adjacent minimizers" print(datetime.datetime.today(), ": Building graph", file=sys.stdout) - graph = ig.Graph() + + if graph is None: + graph = ig.Graph() vertices = set() edges = defaultdict(dict) # source -> target -> [list assembly support] @@ -81,9 +83,11 @@ def build_graph(self, list_mxs, weights): edges[assembly_mx_list[j]][assembly_mx_list[i]].append(assembly) else: edges[assembly_mx_list[i]][assembly_mx_list[j]] = [assembly] - vertices.add(assembly_mx_list[i]) + if black_list is None or assembly_mx_list[i] not in black_list: + vertices.add(assembly_mx_list[i]) if assembly_mx_list: - vertices.add(assembly_mx_list[-1]) + if black_list is None or assembly_mx_list[-1] not in black_list: + vertices.add(assembly_mx_list[-1]) formatted_edges = [(s, t) for s in edges for t in edges[s]] @@ -103,9 +107,13 @@ def build_graph(self, list_mxs, weights): return graph - def print_graph(self, graph): + def print_graph(self, graph, out_prefix=None): "Prints the minimizer graph in dot format" - out_graph = self.args.p + ".mx.dot" + if out_prefix is None: + out_graph = self.args.p + ".mx.dot" + else: + out_graph = out_prefix + "mx.dot" + with open(out_graph, 'w', encoding="utf-8") as outfile: print(datetime.datetime.today(), ": Printing graph", out_graph, sep=" ", file=sys.stdout) @@ -389,7 +397,6 @@ def find_paths_process(self, component): min_edge_weight <= max_edge_weight: component_graph = self.filter_graph(component_graph, min_edge_weight) min_edge_weight += 1 - for subcomponent in component_graph.components(): subcomponent_graph = component_graph.subgraph(subcomponent) source_nodes = [node.index for node in subcomponent_graph.vs() if node.degree() == 1] @@ -990,7 +997,7 @@ def parse_arguments(): if len(sys.argv) == 1: parser.print_help() sys.exit() - + return parser.parse_args() def print_parameters_scaffold(self): @@ -1128,7 +1135,19 @@ def main(self): block_num += 1 print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) # Ready to start refining the synteny block coordinates - ntjoin_synteny.generate_additional_minimizers(paths, self.args.w, self.args.btllib_t) + new_list_mxs, terminal_mxs = ntjoin_synteny.generate_additional_minimizers(paths, self.args.w, self.args.btllib_t, list_mx_info) + graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list = terminal_mxs) + self.print_graph(graph, out_prefix=f"{self.args.p}.extend.") + graph = self.filter_graph_global(graph) + paths, incorporated_segments = self.find_paths(graph) + print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) + with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + sys.exit() diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index f961586..78afe2a 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -4,9 +4,11 @@ Written by Lauren Coombe @lcoombe """ -from collections import namedtuple +from collections import namedtuple, defaultdict +import datetime import re import sys +import intervaltree import ntjoin_utils import pybedtools @@ -86,6 +88,14 @@ def get_block_end(self): "Get the end coordinate of the assembly block" return max(self.minimizers[0].position, self.minimizers[-1].position) + def get_block_terminal_mx(self): + "Return the terminal minimizer hashes for the assembly block" + return self.contig_id, self.minimizers[0], self.minimizers[-1] + + def get_block_internal_mx_hashes(self): + "Return the internal minimizer hashes for the assembly block" + return [mx_pos.mx for mx_pos in self.minimizers[1:-1]] + def find_synteny_blocks(path, list_mx_info, k): "Given a path (sequence of mx), print the order/orientation/regions of contigs for an assembly" @@ -107,7 +117,7 @@ def find_synteny_blocks(path, list_mx_info, k): prelim_blocks.determine_orientations() if prelim_blocks.all_oriented(): out_blocks.append(prelim_blocks) - + return out_blocks def find_fa_name(assembly_mx_name): @@ -150,19 +160,87 @@ def mask_assemblies_with_synteny_extents(synteny_beds): def generate_new_minimizers(tsv_to_fa_dict, k, w, t): "Given the masked fasta files, generate minimizers at new w for each" - list_mx_info = {} list_mxs = {} + new_list_mxs_info = {} for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): - indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, int(w/10), t) + indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, int(w/100), t) #!! TODO - fix magic number mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) - list_mx_info[assembly_tsv] = mx_info + new_list_mxs_info[assembly_tsv] = mx_info list_mxs[assembly_tsv] = mxs_filt - return list_mx_info, list_mxs - + return list_mxs, new_list_mxs_info + +def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): + "Update the given dictionary of trees with the new extent" + start_pos = min(mx1.position, mx2.position) + end_pos = max(mx1.position, mx2.position) + if assembly_name not in trees or ctg not in assembly_name[trees]: + trees[assembly_name][ctg] = intervaltree.IntervalTree() + trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) + +def find_mx_in_blocks(paths): + "Given the synteny blocks, find the minimizers at the terminal ends of each block, and internal" + terminal_mxs = set() + internal_mxs = set() + intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents -def generate_additional_minimizers(paths, w, t): + for subcomponent in paths: + for block in subcomponent: + curr_mx_len = len(terminal_mxs) + for assembly, assembly_block in block.assembly_blocks.items(): + contig, mx1, mx2 = assembly_block.get_block_terminal_mx() + terminal_mxs.add(mx1.mx) + terminal_mxs.add(mx2.mx) + update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) + internal = assembly_block.get_block_internal_mx_hashes() + internal_mxs = internal_mxs.union(internal) + assert len(terminal_mxs) == (curr_mx_len + 2) + return terminal_mxs, internal_mxs, intervaltrees + +def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_mx_info): + "Filter minimizers found in the mx black list" + return_mxs = {} + for assembly in list_mxs: + # assembly_mxs_filtered = [[mx for mx in mx_list if mx not in black_list and \ + # intervaltrees[assembly][list_mx_info[assembly][mx][0]] [list_mx_info[assembly][mx][1]] + # for mx_list in list_mxs[assembly]]] + assembly_mxs_filtered = [] + for mx_list in list_mxs[assembly]: + new_list = [] + for mx in mx_list: + ctg, pos = list_mx_info[assembly][mx] + i_tree = intervaltrees[assembly][ctg] + if new_list: + prev_pos = list_mx_info[assembly][new_list[-1]][1] + start = min(prev_pos, pos) + end = max(prev_pos, pos) + if i_tree[start:end]: # Split the mx adjacency if it spans over a known synteny block + assembly_mxs_filtered.append(new_list) + new_list = [] + if mx not in black_list and not i_tree[pos]: + new_list.append(mx) + assembly_mxs_filtered.append(new_list) + + return_mxs[assembly] = assembly_mxs_filtered + return return_mxs + +def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): + "Update the directory containing mx -> contig, position associations" + valid_mxs = set({mx for _, list_mx_val in list_mxs.items() \ + for list_mx in list_mx_val for mx in list_mx}) + for assembly, mx_dict in new_list_mx_info.items(): + for mx in mx_dict: + if mx in valid_mxs and mx not in list_mx_info[assembly]: + list_mx_info[assembly][mx] = mx_dict[mx] + + +def generate_additional_minimizers(paths, w, t, list_mx_info): "Given the existing synteny blocks, generate minimizers for increased block resolution" k = paths[0][0].k synteny_beds = get_synteny_bed_lists(paths, w) mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) - list_mx_info, list_mxs = generate_new_minimizers(mx_to_fa_dict, k, w, t) + list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, w, t) + terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) # !! TODO - only need internal? + list_mxs = filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) + list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies + update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info) + return list_mxs, terminal_mx From 4b7fcde4528c7b9bac4444e6d3032ce86e6b98b3 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 23 Feb 2023 16:28:35 -0800 Subject: [PATCH 08/31] Run rounds of synteny block extension with lowering w --- bin/ntjoin_assemble.py | 36 ++++++++++++++++++++++-------------- bin/ntjoin_synteny.py | 13 ++++++------- ntJoin | 5 ++++- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 5973829..8d5b71f 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -992,6 +992,8 @@ def parse_arguments(): synteny_parser.add_argument("-w", help="Window size used for minimizers", required=True, type=int) synteny_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions "\ "(computing minimizers, reading fasta file) [4]", type=int, default=4) + synteny_parser.add_argument("--w-rounds", help="decreasing list of 'w' values to use for refining ends", + default=[100, 10, 5], nargs="+", type=int) synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: @@ -1041,6 +1043,7 @@ def print_parameters_synteny(self): print("\t-k", self.args.k) print("\t-w", self.args.w) print("\t--btllib_t", self.args.btllib_t) + print("\t--w-rounds", self.args.w_rounds) def print_parameters(self): @@ -1134,20 +1137,7 @@ def main(self): outfile.write(block.get_block_string(block_num)) block_num += 1 print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) - # Ready to start refining the synteny block coordinates - new_list_mxs, terminal_mxs = ntjoin_synteny.generate_additional_minimizers(paths, self.args.w, self.args.btllib_t, list_mx_info) - graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list = terminal_mxs) - self.print_graph(graph, out_prefix=f"{self.args.p}.extend.") - graph = self.filter_graph_global(graph) - paths, incorporated_segments = self.find_paths(graph) - print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) - with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - + self.refine_block_coordinates(list_mx_info, graph, paths) sys.exit() @@ -1164,6 +1154,24 @@ def main(self): print(datetime.datetime.today(), ": DONE!", file=sys.stdout) + def refine_block_coordinates(self, list_mx_info, graph, paths): + "Ready to start refining the synteny block coordinates" + prev_w = self.args.w + for new_w in self.args.w_rounds: + new_list_mxs, terminal_mxs = ntjoin_synteny.generate_additional_minimizers( + paths, new_w, prev_w, self.args.btllib_t, list_mx_info) + graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list=terminal_mxs) + self.print_graph(graph, out_prefix=f"{self.args.p}.extend.") + graph = self.filter_graph_global(graph) + paths, _ = self.find_paths(graph) + print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) + with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + def __init__(self): "Create an ntJoin instance" self.args = self.parse_arguments() diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 78afe2a..39ac9e2 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -5,7 +5,6 @@ """ from collections import namedtuple, defaultdict -import datetime import re import sys import intervaltree @@ -117,7 +116,7 @@ def find_synteny_blocks(path, list_mx_info, k): prelim_blocks.determine_orientations() if prelim_blocks.all_oriented(): out_blocks.append(prelim_blocks) - + return out_blocks def find_fa_name(assembly_mx_name): @@ -163,7 +162,7 @@ def generate_new_minimizers(tsv_to_fa_dict, k, w, t): list_mxs = {} new_list_mxs_info = {} for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): - indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, int(w/100), t) #!! TODO - fix magic number + indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, w, t) #!! TODO - fix magic number mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) new_list_mxs_info[assembly_tsv] = mx_info list_mxs[assembly_tsv] = mxs_filt @@ -233,13 +232,13 @@ def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): list_mx_info[assembly][mx] = mx_dict[mx] -def generate_additional_minimizers(paths, w, t, list_mx_info): +def generate_additional_minimizers(paths, new_w, prev_w, t, list_mx_info): "Given the existing synteny blocks, generate minimizers for increased block resolution" k = paths[0][0].k - synteny_beds = get_synteny_bed_lists(paths, w) + synteny_beds = get_synteny_bed_lists(paths, prev_w) mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) - list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, w, t) - terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) # !! TODO - only need internal? + list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, new_w, t) + terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) list_mxs = filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info) diff --git a/ntJoin b/ntJoin index b84d878..ae0c055 100755 --- a/ntJoin +++ b/ntJoin @@ -22,6 +22,9 @@ references_list=$(references) reference_weights_list=$(reference_weights) endif +# synteny - w block extensions +w_sizes=$(synteny_ws) + min_ref_targets=$(addsuffix .k$(k).w$(w).tsv, $(references_list)) fai_ref_targets=$(addsuffix .fai, $(references_list)) @@ -254,4 +257,4 @@ endif # Synteny $(prefix).synteny_blocks.tsv: $(min_ref_targets) - $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) -w$(w) + $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) -w$(w) --w-rounds $(w_sizes) From 821f96740b826c4370bb25db3005dda770344506 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 28 Feb 2023 16:14:03 -0800 Subject: [PATCH 09/31] Correct edge attributes assignment, other bugfixes * When adding nodes to an established graph, need to take current edges into account when assigning the attributes --- bin/ntjoin_assemble.py | 13 +++++++++++++ bin/ntjoin_synteny.py | 4 +++- ntJoin | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 8d5b71f..a652e7b 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -67,6 +67,10 @@ def build_graph(self, list_mxs, weights, graph=None, black_list=None): if graph is None: graph = ig.Graph() + prev_edge_attributes = {} + else: + prev_edge_attributes = {e.index: {"support": e['support'], + "weight": e['weight']} for e in graph.es()} vertices = set() edges = defaultdict(dict) # source -> target -> [list assembly support] @@ -102,6 +106,7 @@ def build_graph(self, list_mxs, weights, graph=None, black_list=None): "weight": self.calc_total_weight(edges[s][t], weights)} for s in edges for t in edges[s]} + edge_attributes.update(prev_edge_attributes) self.set_edge_attributes(graph, edge_attributes) return graph @@ -1163,7 +1168,15 @@ def refine_block_coordinates(self, list_mx_info, graph, paths): graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list=terminal_mxs) self.print_graph(graph, out_prefix=f"{self.args.p}.extend.") graph = self.filter_graph_global(graph) + self.print_graph(graph, out_prefix=f"{self.args.p}.extend_filtered.") paths, _ = self.find_paths(graph) + with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: block_num = 0 diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 39ac9e2..cc6dca2 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -151,6 +151,8 @@ def mask_assemblies_with_synteny_extents(synteny_beds): bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ for bed in contig_dict[ctg]] bed_str = "\n".join(bed_str) + print(assembly, "++") + print(bed_str) synteny_bed = pybedtools.BedTool(bed_str, from_string=True).sort() fa_filename = find_fa_name(assembly) synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") @@ -172,7 +174,7 @@ def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): "Update the given dictionary of trees with the new extent" start_pos = min(mx1.position, mx2.position) end_pos = max(mx1.position, mx2.position) - if assembly_name not in trees or ctg not in assembly_name[trees]: + if assembly_name not in trees or ctg not in trees[assembly_name]: trees[assembly_name][ctg] = intervaltree.IntervalTree() trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) diff --git a/ntJoin b/ntJoin index ae0c055..944dc65 100755 --- a/ntJoin +++ b/ntJoin @@ -23,7 +23,7 @@ reference_weights_list=$(reference_weights) endif # synteny - w block extensions -w_sizes=$(synteny_ws) +w_sizes=100 10 5 min_ref_targets=$(addsuffix .k$(k).w$(w).tsv, $(references_list)) fai_ref_targets=$(addsuffix .fai, $(references_list)) From 5e918788a961a35ac33b944cefc66279c4ef8939 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 1 Mar 2023 10:57:06 -0800 Subject: [PATCH 10/31] Use bedtools slop for adjusting bed coordinates by w * Just controls in case adjustments go out of range, cleaner * Also bugfix - ensure that we update 'prev_w' with each iteration --- bin/ntjoin_assemble.py | 1 + bin/ntjoin_synteny.py | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index a652e7b..1dfb4ae 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -1176,6 +1176,7 @@ def refine_block_coordinates(self, list_mx_info, graph, paths): for block in subcomponent: outfile.write(block.get_block_string(block_num)) block_num += 1 + prev_w = new_w print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index cc6dca2..084ea87 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -127,7 +127,7 @@ def find_fa_name(assembly_mx_name): print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") sys.exit(1) -def get_synteny_bed_lists(paths, w): +def get_synteny_bed_lists(paths): "Given a set of synteny blocks, return a dictionary with a Bed interval lists per contig, per assembly" synteny_beds = {} for subcomponent in paths: @@ -139,22 +139,20 @@ def get_synteny_bed_lists(paths, w): synteny_beds[assembly][assembly_block.contig_id] = [] synteny_beds[assembly][assembly_block.contig_id].append( ntjoin_utils.Bed(assembly_block.contig_id, - assembly_block.get_block_start() + w, - assembly_block.get_block_end() + block.k - w)) + assembly_block.get_block_start(), + assembly_block.get_block_end() + block.k)) return synteny_beds -def mask_assemblies_with_synteny_extents(synteny_beds): +def mask_assemblies_with_synteny_extents(synteny_beds, w): "Mask each reference assembly with determined synteny blocks" mx_to_fa_dict = {} for assembly, contig_dict in synteny_beds.items(): bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ - for bed in contig_dict[ctg]] + for bed in contig_dict[ctg] if bed.end - bed.start > 2*w] bed_str = "\n".join(bed_str) - print(assembly, "++") - print(bed_str) - synteny_bed = pybedtools.BedTool(bed_str, from_string=True).sort() fa_filename = find_fa_name(assembly) + synteny_bed = pybedtools.BedTool(bed_str, from_string=True).slop(g=f"{fa_filename}.fai", l=-1*w, r=-1*w).sort() synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" return mx_to_fa_dict @@ -201,9 +199,6 @@ def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_m "Filter minimizers found in the mx black list" return_mxs = {} for assembly in list_mxs: - # assembly_mxs_filtered = [[mx for mx in mx_list if mx not in black_list and \ - # intervaltrees[assembly][list_mx_info[assembly][mx][0]] [list_mx_info[assembly][mx][1]] - # for mx_list in list_mxs[assembly]]] assembly_mxs_filtered = [] for mx_list in list_mxs[assembly]: new_list = [] @@ -237,8 +232,8 @@ def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): def generate_additional_minimizers(paths, new_w, prev_w, t, list_mx_info): "Given the existing synteny blocks, generate minimizers for increased block resolution" k = paths[0][0].k - synteny_beds = get_synteny_bed_lists(paths, prev_w) - mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds) + synteny_beds = get_synteny_bed_lists(paths) + mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds, prev_w) list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, new_w, t) terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) list_mxs = filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) From e7a3688bfe55ab14561a99fa557e9539acf026c1 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 1 Mar 2023 11:21:04 -0800 Subject: [PATCH 11/31] Remove intermediate files by default, add more log messages for w iterations --- bin/ntjoin_assemble.py | 7 +++---- bin/ntjoin_synteny.py | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 1dfb4ae..8c71e7d 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -999,6 +999,7 @@ def parse_arguments(): "(computing minimizers, reading fasta file) [4]", type=int, default=4) synteny_parser.add_argument("--w-rounds", help="decreasing list of 'w' values to use for refining ends", default=[100, 10, 5], nargs="+", type=int) + synteny_parser.add_argument("--dev", action="store_true", help="Developer mode - retain intermediate files") synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: @@ -1068,7 +1069,6 @@ def set_synteny_parameters(self): def main(self): "Run ntJoin graph stage" print("Running ntJoin v1.1.1 ...\n") - print(self.args, self.args.n) self.print_parameters() if self.args.mode == "synteny": @@ -1163,12 +1163,11 @@ def refine_block_coordinates(self, list_mx_info, graph, paths): "Ready to start refining the synteny block coordinates" prev_w = self.args.w for new_w in self.args.w_rounds: + print(datetime.datetime.today(), ": Extending synteny blocks with w =", new_w, file=sys.stdout) new_list_mxs, terminal_mxs = ntjoin_synteny.generate_additional_minimizers( - paths, new_w, prev_w, self.args.btllib_t, list_mx_info) + paths, new_w, prev_w, self.args.btllib_t, list_mx_info, self.args.dev) graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list=terminal_mxs) - self.print_graph(graph, out_prefix=f"{self.args.p}.extend.") graph = self.filter_graph_global(graph) - self.print_graph(graph, out_prefix=f"{self.args.p}.extend_filtered.") paths, _ = self.find_paths(graph) with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: block_num = 0 diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 084ea87..4b16bb3 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -6,6 +6,8 @@ from collections import namedtuple, defaultdict import re +import shlex +import subprocess import sys import intervaltree import ntjoin_utils @@ -128,7 +130,7 @@ def find_fa_name(assembly_mx_name): sys.exit(1) def get_synteny_bed_lists(paths): - "Given a set of synteny blocks, return a dictionary with a Bed interval lists per contig, per assembly" + "Given a set of synteny blocks, return a dictionary with a BED interval list per contig, per assembly" synteny_beds = {} for subcomponent in paths: for block in subcomponent: @@ -157,15 +159,24 @@ def mask_assemblies_with_synteny_extents(synteny_beds, w): mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" return mx_to_fa_dict -def generate_new_minimizers(tsv_to_fa_dict, k, w, t): +def delete_w_iteration_files(*filenames): + "Delete the given files for the specific lower w iteration" + for filename in filenames: + cmd = shlex.split(f"rm {filename}") + ret_code = subprocess.call(cmd) + assert ret_code == 0 + +def generate_new_minimizers(tsv_to_fa_dict, k, w, t, retain_files=False): "Given the masked fasta files, generate minimizers at new w for each" list_mxs = {} new_list_mxs_info = {} for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): - indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, w, t) #!! TODO - fix magic number + indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, w, t) mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) new_list_mxs_info[assembly_tsv] = mx_info list_mxs[assembly_tsv] = mxs_filt + if not retain_files: + delete_w_iteration_files(indexlr_filename, assembly_masked) return list_mxs, new_list_mxs_info def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): @@ -229,12 +240,12 @@ def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): list_mx_info[assembly][mx] = mx_dict[mx] -def generate_additional_minimizers(paths, new_w, prev_w, t, list_mx_info): +def generate_additional_minimizers(paths, new_w, prev_w, t, list_mx_info, dev=False): "Given the existing synteny blocks, generate minimizers for increased block resolution" k = paths[0][0].k synteny_beds = get_synteny_bed_lists(paths) mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds, prev_w) - list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, new_w, t) + list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, new_w, t, retain_files=dev) terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) list_mxs = filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies From 1e968e67d39603ab9e6cab68a33a05108e011fd4 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 1 Mar 2023 11:53:10 -0800 Subject: [PATCH 12/31] Add check for overlapping synteny blocks - print warning --- bin/ntjoin_synteny.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 4b16bb3..9e14d4a 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -185,6 +185,9 @@ def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): end_pos = max(mx1.position, mx2.position) if assembly_name not in trees or ctg not in trees[assembly_name]: trees[assembly_name][ctg] = intervaltree.IntervalTree() + if trees[assembly_name][ctg][start_pos+1:end_pos]: # Checking that this doesn't overlap with anything + print("WARNING: detected overlapping segments:", assembly_name, ctg, start_pos+1, end_pos, + file=sys.stderr) trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) def find_mx_in_blocks(paths): @@ -206,6 +209,16 @@ def find_mx_in_blocks(paths): assert len(terminal_mxs) == (curr_mx_len + 2) return terminal_mxs, internal_mxs, intervaltrees +def check_non_overlapping(paths): + "Given the paths, do final check to ensure intervals are not overlapping, will print warnings if that's the case" + intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents + for subcomponent in paths: + for block in subcomponent: + for assembly, assembly_block in block.assembly_blocks.items(): + contig, mx1, mx2 = assembly_block.get_block_terminal_mx() + update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) + + def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_mx_info): "Filter minimizers found in the mx black list" return_mxs = {} From e3335547e067202427156c2e002e74622c8d7ba5 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 2 Mar 2023 15:44:58 -0800 Subject: [PATCH 13/31] Major ntJoin code refactoring * Separated code into several files for better organization * Separated code that is specific to scaffolding and synteny modes * Retained a common class Ntjoin (ntjoin.py) which is a superclass for NtjoinSynteny and NtjoinScaffolder, which extend the functionality for their specific purposes * Extracted helper classes to their own dedicated files * Now, the pipeline is launched by the ntjoin_run.py script --- bin/assembly_block.py | 30 ++ bin/ntjoin.py | 186 +++++++++++++ bin/ntjoin_assemble.py | 618 ++++++++--------------------------------- bin/ntjoin_overlap.py | 11 +- bin/ntjoin_run.py | 94 +++++++ bin/ntjoin_synteny.py | 508 +++++++++++++++++---------------- bin/ntjoin_utils.py | 235 ++++++---------- bin/overlap_region.py | 91 ++++++ bin/path_node.py | 66 +++++ bin/synteny_block.py | 60 ++++ ntJoin | 4 +- 11 files changed, 995 insertions(+), 908 deletions(-) create mode 100644 bin/assembly_block.py create mode 100644 bin/ntjoin.py create mode 100755 bin/ntjoin_run.py create mode 100644 bin/overlap_region.py create mode 100644 bin/path_node.py create mode 100644 bin/synteny_block.py diff --git a/bin/assembly_block.py b/bin/assembly_block.py new file mode 100644 index 0000000..35ad987 --- /dev/null +++ b/bin/assembly_block.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +''' +Represents an assembly block for an assembly. +Multiple assembly blocks comprise a synteny block +''' + +class AssemblyBlock: + "An assembly block for a given assembly. The AssemblyBlock objects per assembly make up a SyntenyBlock" + def __init__(self): + "Instantiate the AssemblyBlock" + self.contig_id = None + self.minimizers = [] + self.ori = None + + def get_block_start(self): + "Get the starting coordinate of the assembly block" + return min(self.minimizers[0].position, self.minimizers[-1].position) + + def get_block_end(self): + "Get the end coordinate of the assembly block" + return max(self.minimizers[0].position, self.minimizers[-1].position) + + def get_block_terminal_mx(self): + "Return the terminal minimizer hashes for the assembly block" + return self.contig_id, self.minimizers[0], self.minimizers[-1] + + def get_block_internal_mx_hashes(self): + "Return the internal minimizer hashes for the assembly block" + return [mx_pos.mx for mx_pos in self.minimizers[1:-1]] diff --git a/bin/ntjoin.py b/bin/ntjoin.py new file mode 100644 index 0000000..b122763 --- /dev/null +++ b/bin/ntjoin.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +ntJoin: Scaffolding assemblies using reference assemblies and minimizer graphs +Written by Lauren Coombe (@lcoombe) +""" + +import datetime +import multiprocessing +import sys +import warnings +import ntjoin_utils +warnings.simplefilter(action='ignore', category=RuntimeWarning) + + +class Ntjoin: + "ntJoin: Scaffolding and analyzing synteny in assemblies using reference assemblies and minimizer graphs" + + @staticmethod + def convert_path_index_to_name(graph, path): + "Convert path of vertex indices to path of vertex names" + return [ntjoin_utils.vertex_name(graph, vs) for vs in path] + + def print_graph(self, graph, out_prefix=None): + "Prints the minimizer graph in dot format" + if out_prefix is None: + out_graph = self.args.p + ".mx.dot" + else: + out_graph = out_prefix + "mx.dot" + + with open(out_graph, 'w', encoding="utf-8") as outfile: + print(datetime.datetime.today(), ": Printing graph", out_graph, sep=" ", file=sys.stdout) + + outfile.write("graph G {\n") + + colours = ["red", "green", "blue", "purple", "orange", + "turquoise", "pink", "yellow", "orchid", "salmon"] + list_files = list(self.list_mx_info.keys()) + if len(list_files) > len(colours): + colours = ["red"]*len(list_files) + + for node in graph.vs(): + mx_ctg_pos_labels = "\n".join([str(asm_mx_info[node['name']]) + for _, asm_mx_info in self.list_mx_info.items()]) + node_label = f"\"{node['name']}\" [label=\"{node['name']}\n{mx_ctg_pos_labels}\"]" + outfile.write(f"{node_label}\n") + + for edge in graph.es(): + outfile.write(f"\"{ntjoin_utils.vertex_name(graph, edge.source)}\" --" \ + f"\"{ntjoin_utils.vertex_name(graph, edge.target)}\"") + weight = edge['weight'] + support = edge['support'] + if len(support) == 1: + colour = colours[list_files.index(support[0])] + elif len(support) == 2: + colour = "lightgrey" + else: + colour = "black" + outfile.write(f" [weight={weight} color={colour}]\n") + + outfile.write("}\n") + + print("\nfile_name\tnumber\tcolour") + for i, filename in enumerate(list_files): + print(filename, i, colours[i], sep="\t") + print("") + + + @staticmethod + def filter_graph(graph, min_weight): + "Filter the graph by edge weights on edges incident to branch nodes" + branch_nodes = [node.index for node in graph.vs() if node.degree() > 2] + to_remove_edges = [edge for node in branch_nodes for edge in graph.incident(node) + if graph.es()[edge]['weight'] < min_weight] + new_graph = graph.copy() + new_graph.delete_edges(to_remove_edges) + return new_graph + + def filter_graph_global(self, graph): + "Filter the graph globally based on minimum edge weight" + print(datetime.datetime.today(), ": Filtering the graph", file=sys.stdout) + if self.args.n <= min(self.weights.values()): + return graph + to_remove_edges = [edge.index for edge in graph.es() + if edge['weight'] < self.args.n] + new_graph = graph.copy() + new_graph.delete_edges(to_remove_edges) + return new_graph + + def determine_source_vertex(self, sources, graph): + '''Given the possible sources of the graph, determine which is the source and the target + Based on the assembly with the largest weight - orient others based on this assembly + ''' + max_wt_asm = [assembly for assembly, asm_weight in self.weights.items() + if asm_weight == max(self.weights.values())].pop() + list_mx_info_maxwt = self.list_mx_info[max_wt_asm] + min_pos = min([list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources]) + max_pos = max([list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources]) + source = [s for s in sources + if list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] == min_pos].pop() + target = [s for s in sources + if list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] == max_pos].pop() + return source, target + + @staticmethod + def is_graph_linear(graph): + "Given a graph, return True if all the components are linear" + for component in graph.components(): + component_graph = graph.subgraph(component) + if not all(u.degree() < 3 for u in component_graph.vs()): + return False + return True + + + def find_paths_process(self, component): + "Find paths given a component of the graph" + return_paths = [] + min_edge_weight = self.args.n + max_edge_weight = sum(self.weights.values()) + component_graph = self.graph.subgraph(component) + while not self.is_graph_linear(component_graph) and \ + min_edge_weight <= max_edge_weight: + component_graph = self.filter_graph(component_graph, min_edge_weight) + min_edge_weight += 1 + for subcomponent in component_graph.components(): + subcomponent_graph = component_graph.subgraph(subcomponent) + source_nodes = [node.index for node in subcomponent_graph.vs() if node.degree() == 1] + if len(source_nodes) == 2: + source, target = self.determine_source_vertex(source_nodes, subcomponent_graph) + path = subcomponent_graph.get_shortest_paths(source, target)[0] + num_edges = len(path) - 1 + if len(path) == len(subcomponent_graph.vs()) and \ + num_edges == len(subcomponent_graph.es()) and len(path) == len(set(path)): + # All the nodes/edges from the graph are in the simple path, no repeated nodes + path = self.convert_path_index_to_name(subcomponent_graph, path) + return_paths.append((path, subcomponent_graph)) + return return_paths + + + def find_paths(self): + "Finds paths through the minimizer graph" + print(datetime.datetime.today(), ": Finding paths", file=sys.stdout) + components = self.graph.components() + print("\nTotal number of components in graph:", len(components), "\n", sep=" ", file=sys.stdout) + + if self.args.t == 1: + paths = [self.find_paths_process(component) for component in components] + else: + with multiprocessing.Pool(self.args.t) as pool: + paths = pool.map(self.find_paths_process, components) + + return paths + + + def make_minimizer_graph_and_paths(self): + "Run ntJoin graph stage" + print(datetime.datetime.today(), ": Generating ntJoin minimizer graph ...\n") + + # Print the weights of the input assemblies + weight_str = "\n".join([f"{assembly}: {asm_weight}" for assembly, asm_weight in self.weights.items()]) + print("\nWeights of assemblies:\n", weight_str, "\n", sep="") + + # Filter minimizers - Keep only if found in all assemblies + list_mxs = ntjoin_utils.filter_minimizers(self.list_mxs) + + # Build a graph: Nodes = mx; Edges between adjacent mx in the assemblies + self.graph = ntjoin_utils.build_graph(list_mxs, self.weights) + + # Print the DOT graph + self.print_graph(self.graph) + + # Filter the graph edges by minimum weight + self.graph = self.filter_graph_global(self.graph) + + + def ntjoin_find_paths(self): + "Find the paths through the graph" + paths = self.find_paths() + return paths + + def __init__(self, args): + "Create an ntJoin instance" + self.list_mx_info = {} # Dictionary of dictionaries: assembly -> mx -> (contig, position) + self.list_mxs = {} # Dictionary: assembly -> [lists of mx] + self.graph = None + self.args = args + self.weights = {} diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 8c71e7d..3109f6c 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -4,158 +4,28 @@ Written by Lauren Coombe (@lcoombe) """ -import argparse import datetime -import multiprocessing import re -from collections import Counter -from collections import defaultdict +from collections import Counter, defaultdict import shlex import subprocess import sys import warnings -import igraph as ig import pybedtools import pymannkendall as mk from read_fasta import read_fasta import ntjoin_utils import ntjoin_overlap -import ntjoin_synteny +import ntjoin +import path_node +from overlap_region import OverlapRegion import btllib warnings.simplefilter(action='ignore', category=RuntimeWarning) -class Ntjoin: +class NtjoinScaffolder(ntjoin.Ntjoin): "ntJoin: Scaffolding assemblies using reference assemblies and minimizer graphs" - # Helper functions for interfacing with python-igraph - @staticmethod - def vertex_index(graph, name): - "Returns vertex index based on vertex name" - return graph.vs.find(name).index - - @staticmethod - def vertex_name(graph, index): - "Returns vertex name based on vertex id" - return graph.vs[index]['name'] - - @staticmethod - def edge_index(graph, source_name, target_name): - "Returns graph edge index based on source/target names" - return graph.get_eid(source_name, target_name) - - @staticmethod - def set_edge_attributes(graph, edge_attributes): - "Sets the edge attributes for a python-igraph graph" - graph.es()["support"] = [edge_attributes[e]['support'] for e in sorted(edge_attributes.keys())] - graph.es()["weight"] = [edge_attributes[e]['weight'] for e in sorted(edge_attributes.keys())] - - @staticmethod - def convert_path_index_to_name(graph, path): - "Convert path of vertex indices to path of vertex names" - return [Ntjoin.vertex_name(graph, vs) for vs in path] - - @staticmethod - def calc_total_weight(list_files, weights): - "Calculate the total weight of an edge given the assembly support" - return sum([weights[f] for f in list_files]) - - - def build_graph(self, list_mxs, weights, graph=None, black_list=None): - "Builds an undirected graph: nodes=minimizers; edges=between adjacent minimizers" - print(datetime.datetime.today(), ": Building graph", file=sys.stdout) - - if graph is None: - graph = ig.Graph() - prev_edge_attributes = {} - else: - prev_edge_attributes = {e.index: {"support": e['support'], - "weight": e['weight']} for e in graph.es()} - - vertices = set() - edges = defaultdict(dict) # source -> target -> [list assembly support] - - for assembly in list_mxs: - for assembly_mx_list in list_mxs[assembly]: - for i, j in zip(range(0, len(assembly_mx_list)), - range(1, len(assembly_mx_list))): - if assembly_mx_list[i] in edges and \ - assembly_mx_list[j] in edges[assembly_mx_list[i]]: - edges[assembly_mx_list[i]][assembly_mx_list[j]].append(assembly) - elif assembly_mx_list[j] in edges and \ - assembly_mx_list[i] in edges[assembly_mx_list[j]]: - edges[assembly_mx_list[j]][assembly_mx_list[i]].append(assembly) - else: - edges[assembly_mx_list[i]][assembly_mx_list[j]] = [assembly] - if black_list is None or assembly_mx_list[i] not in black_list: - vertices.add(assembly_mx_list[i]) - if assembly_mx_list: - if black_list is None or assembly_mx_list[-1] not in black_list: - vertices.add(assembly_mx_list[-1]) - - formatted_edges = [(s, t) for s in edges for t in edges[s]] - - print(datetime.datetime.today(), ": Adding vertices", file=sys.stdout) - graph.add_vertices(list(vertices)) - - print(datetime.datetime.today(), ": Adding edges", file=sys.stdout) - graph.add_edges(formatted_edges) - - print(datetime.datetime.today(), ": Adding attributes", file=sys.stdout) - edge_attributes = {self.edge_index(graph, s, t): {"support": edges[s][t], - "weight": self.calc_total_weight(edges[s][t], - weights)} - for s in edges for t in edges[s]} - edge_attributes.update(prev_edge_attributes) - self.set_edge_attributes(graph, edge_attributes) - - return graph - - - def print_graph(self, graph, out_prefix=None): - "Prints the minimizer graph in dot format" - if out_prefix is None: - out_graph = self.args.p + ".mx.dot" - else: - out_graph = out_prefix + "mx.dot" - - with open(out_graph, 'w', encoding="utf-8") as outfile: - print(datetime.datetime.today(), ": Printing graph", out_graph, sep=" ", file=sys.stdout) - - outfile.write("graph G {\n") - - colours = ["red", "green", "blue", "purple", "orange", - "turquoise", "pink", "yellow", "orchid", "salmon"] - list_files = list(Ntjoin.list_mx_info.keys()) - if len(list_files) > len(colours): - colours = ["red"]*len(list_files) - - for node in graph.vs(): - mx_ctg_pos_labels = "\n".join([str(asm_mx_info[node['name']]) - for _, asm_mx_info in Ntjoin.list_mx_info.items()]) - node_label = f"\"{node['name']}\" [label=\"{node['name']}\n{mx_ctg_pos_labels}\"]" - outfile.write(f"{node_label}\n") - - for edge in graph.es(): - outfile.write(f"\"{self.vertex_name(graph, edge.source)}\" --" \ - f"\"{self.vertex_name(graph, edge.target)}\"") - weight = edge['weight'] - support = edge['support'] - if len(support) == 1: - colour = colours[list_files.index(support[0])] - elif len(support) == 2: - colour = "lightgrey" - else: - colour = "black" - outfile.write(f" [weight={weight} color={colour}]\n") - - outfile.write("}\n") - - print("\nfile_name\tnumber\tcolour") - for i, filename in enumerate(list_files): - print(filename, i, colours[i], sep="\t") - print("") - def determine_orientation(self, positions): "Given a list of minimizer positions, determine the orientation of the contig" if len(positions) > 1: @@ -206,23 +76,23 @@ def calculate_gap_size(self, u, v, graph, cur_assembly): # Are situations where there is not a direct edge if an unoriented contig was in-between path = graph.get_shortest_paths(u_mx, v_mx, output="vpath")[0] supporting_assemblies = set.intersection( - *map(set, [graph.es()[self.edge_index(graph, s, t)]['support'] + *map(set, [graph.es()[ntjoin_utils.edge_index(graph, s, t)]['support'] for s, t in zip(path, path[1:])])) if not supporting_assemblies: return self.args.g, self.args.g - distances = [abs(Ntjoin.list_mx_info[assembly][v_mx][1] - Ntjoin.list_mx_info[assembly][u_mx][1]) + distances = [abs(self.list_mx_info[assembly][v_mx][1] - self.list_mx_info[assembly][u_mx][1]) for assembly in supporting_assemblies] mean_dist = int(sum(distances)/len(distances)) - self.args.k # Correct for the overhanging sequence before/after terminal minimizers if u.ori == "+": - a = u.end - Ntjoin.list_mx_info[cur_assembly][u_mx][1] - self.args.k + a = u.end - self.list_mx_info[cur_assembly][u_mx][1] - self.args.k else: - a = Ntjoin.list_mx_info[cur_assembly][u_mx][1] - u.start + a = self.list_mx_info[cur_assembly][u_mx][1] - u.start if v.ori == "+": - b = Ntjoin.list_mx_info[cur_assembly][v_mx][1] - v.start + b = self.list_mx_info[cur_assembly][v_mx][1] - v.start else: - b = v.end - Ntjoin.list_mx_info[cur_assembly][v_mx][1] - self.args.k + b = v.end - self.list_mx_info[cur_assembly][v_mx][1] - self.args.k try: assert a >= 0 @@ -230,8 +100,8 @@ def calculate_gap_size(self, u, v, graph, cur_assembly): except AssertionError as assert_error: print("ERROR: Gap distance estimation less than 0", "Vertex 1:", u, "Vertex 2:", v, sep="\n") - print("Minimizer positions:", Ntjoin.list_mx_info[cur_assembly][u_mx][1], - Ntjoin.list_mx_info[cur_assembly][v_mx][1]) + print("Minimizer positions:", self.list_mx_info[cur_assembly][u_mx][1], + self.list_mx_info[cur_assembly][v_mx][1]) print("Estimated distance: ", mean_dist) raise ValueError from assert_error @@ -252,7 +122,7 @@ def is_new_region_overlapping(start, end, node_i, node_j, incorporated_segments_ return False - def merge_relocations(self, path): + def merge_relocations(self, path, incorporated_segments): "If a path has adjacent collinear intervals of the same contig, merge them" if len(path) < 2: return path @@ -261,16 +131,16 @@ def merge_relocations(self, path): if node_i.contig == node_j.contig: if node_i.ori == "+" and node_j.ori == "+" and node_i.end <= node_j.start: if self.is_new_region_overlapping(node_i.start, node_j.end, node_i, node_j, - Ntjoin.incorporated_segments[node_i.contig]): + incorporated_segments[node_i.contig]): return_path.append(node_j) continue - Ntjoin.incorporated_segments[node_i.contig].add(ntjoin_utils.Bed(contig=return_path[-1].contig, + incorporated_segments[node_i.contig].add(ntjoin_utils.Bed(contig=return_path[-1].contig, start=return_path[-1].start, end=node_j.end)) - Ntjoin.incorporated_segments[node_i.contig].remove(ntjoin_utils.Bed(contig=return_path[-1].contig, + incorporated_segments[node_i.contig].remove(ntjoin_utils.Bed(contig=return_path[-1].contig, start=return_path[-1].start, end=return_path[-1].end)) - Ntjoin.incorporated_segments[node_j.contig].remove(ntjoin_utils.Bed(contig=node_j.contig, + incorporated_segments[node_j.contig].remove(ntjoin_utils.Bed(contig=node_j.contig, start=node_j.start, end=node_j.end)) return_path[-1].end = node_j.end @@ -278,16 +148,16 @@ def merge_relocations(self, path): return_path[-1].gap_size = node_j.gap_size elif node_i.ori == "-" and node_j.ori == "-" and node_i.start >= node_j.end: if self.is_new_region_overlapping(node_j.start, node_i.end, node_i, node_j, - Ntjoin.incorporated_segments[node_i.contig]): + incorporated_segments[node_i.contig]): return_path.append(node_j) continue - Ntjoin.incorporated_segments[node_i.contig].add(ntjoin_utils.Bed(contig=return_path[-1].contig, + incorporated_segments[node_i.contig].add(ntjoin_utils.Bed(contig=return_path[-1].contig, start=node_j.start, end=return_path[-1].end)) - Ntjoin.incorporated_segments[node_i.contig].remove(ntjoin_utils.Bed(contig=return_path[-1].contig, + incorporated_segments[node_i.contig].remove(ntjoin_utils.Bed(contig=return_path[-1].contig, start=return_path[-1].start, end=return_path[-1].end)) - Ntjoin.incorporated_segments[node_j.contig].remove(ntjoin_utils.Bed(contig=node_j.contig, + incorporated_segments[node_j.contig].remove(ntjoin_utils.Bed(contig=node_j.contig, start=node_j.start, end=node_j.end)) return_path[-1].start = node_j.start @@ -307,7 +177,7 @@ def format_path(self, path, assembly, component_graph): curr_ctg, prev_mx, first_mx = None, None, None positions = [] for mx in path: - ctg, pos = Ntjoin.list_mx_info[assembly][mx] + ctg, pos = self.list_mx_info[assembly][mx] if ctg is curr_ctg: positions.append(pos) else: @@ -315,13 +185,13 @@ def format_path(self, path, assembly, component_graph): if curr_ctg is not None: ori = self.determine_orientation(positions) if ori != "?": # Don't add to path if orientation couldn't be determined - out_path.append(ntjoin_utils.PathNode(contig=curr_ctg, ori=ori, + out_path.append(path_node.PathNode(contig=curr_ctg, ori=ori, start=self.calc_start_coord(positions, - Ntjoin.mx_extremes[curr_ctg][0]), + self.mx_extremes[curr_ctg][0]), end=self.calc_end_coord(positions, - Ntjoin.mx_extremes[curr_ctg][1], - Ntjoin.scaffolds[curr_ctg].length), - contig_size=Ntjoin.scaffolds[curr_ctg].length, + self.mx_extremes[curr_ctg][1], + self.scaffolds[curr_ctg].length), + contig_size=self.scaffolds[curr_ctg].length, first_mx=first_mx, terminal_mx=prev_mx)) curr_ctg = ctg @@ -330,13 +200,13 @@ def format_path(self, path, assembly, component_graph): prev_mx = mx ori = self.determine_orientation(positions) if ori != "?": - out_path.append(ntjoin_utils.PathNode(contig=curr_ctg, ori=ori, + out_path.append(path_node.PathNode(contig=curr_ctg, ori=ori, start=self.calc_start_coord(positions, - Ntjoin.mx_extremes[curr_ctg][0]), + self.mx_extremes[curr_ctg][0]), end=self.calc_end_coord(positions, - Ntjoin.mx_extremes[curr_ctg][1], - Ntjoin.scaffolds[curr_ctg].length), - contig_size=Ntjoin.scaffolds[curr_ctg].length, + self.mx_extremes[curr_ctg][1], + self.scaffolds[curr_ctg].length), + contig_size=self.scaffolds[curr_ctg].length, first_mx=first_mx, terminal_mx=prev_mx)) for u, v in zip(out_path, out_path[1:]): @@ -346,81 +216,6 @@ def format_path(self, path, assembly, component_graph): return out_path - @staticmethod - def filter_graph(graph, min_weight): - "Filter the graph by edge weights on edges incident to branch nodes" - branch_nodes = [node.index for node in graph.vs() if node.degree() > 2] - to_remove_edges = [edge for node in branch_nodes for edge in graph.incident(node) - if graph.es()[edge]['weight'] < min_weight] - new_graph = graph.copy() - new_graph.delete_edges(to_remove_edges) - return new_graph - - def filter_graph_global(self, graph): - "Filter the graph globally based on minimum edge weight" - print(datetime.datetime.today(), ": Filtering the graph", file=sys.stdout) - if self.args.n <= min(Ntjoin.weights.values()): - return graph - to_remove_edges = [edge.index for edge in graph.es() - if edge['weight'] < self.args.n] - new_graph = graph.copy() - new_graph.delete_edges(to_remove_edges) - return new_graph - - def determine_source_vertex(self, sources, graph): - '''Given the possible sources of the graph, determine which is the source and the target - Based on the assembly with the largest weight - orient others based on this assembly - ''' - max_wt_asm = [assembly for assembly, asm_weight in Ntjoin.weights.items() - if asm_weight == max(Ntjoin.weights.values())].pop() - list_mx_info_maxwt = Ntjoin.list_mx_info[max_wt_asm] - min_pos = min([list_mx_info_maxwt[self.vertex_name(graph, s)][1] for s in sources]) - max_pos = max([list_mx_info_maxwt[self.vertex_name(graph, s)][1] for s in sources]) - source = [s for s in sources - if list_mx_info_maxwt[self.vertex_name(graph, s)][1] == min_pos].pop() - target = [s for s in sources - if list_mx_info_maxwt[self.vertex_name(graph, s)][1] == max_pos].pop() - return source, target - - @staticmethod - def is_graph_linear(graph): - "Given a graph, return True if all the components are linear" - for component in graph.components(): - component_graph = graph.subgraph(component) - if not all(u.degree() < 3 for u in component_graph.vs()): - return False - return True - - - def find_paths_process(self, component): - "Find paths given a component of the graph" - return_paths = [] - min_edge_weight = self.args.n - max_edge_weight = sum(Ntjoin.weights.values()) - component_graph = Ntjoin.gin.subgraph(component) - while not self.is_graph_linear(component_graph) and \ - min_edge_weight <= max_edge_weight: - component_graph = self.filter_graph(component_graph, min_edge_weight) - min_edge_weight += 1 - for subcomponent in component_graph.components(): - subcomponent_graph = component_graph.subgraph(subcomponent) - source_nodes = [node.index for node in subcomponent_graph.vs() if node.degree() == 1] - if len(source_nodes) == 2: - source, target = self.determine_source_vertex(source_nodes, subcomponent_graph) - path = subcomponent_graph.get_shortest_paths(source, target)[0] - num_edges = len(path) - 1 - if len(path) == len(subcomponent_graph.vs()) and \ - num_edges == len(subcomponent_graph.es()) and len(path) == len(set(path)): - # All the nodes/edges from the graph are in the simple path, no repeated nodes - path = self.convert_path_index_to_name(subcomponent_graph, path) - if self.args.mode == "synteny": - ctg_path = ntjoin_synteny.find_synteny_blocks(path, Ntjoin.list_mx_info, self.args.k) - else: - ctg_path = self.format_path(path, self.args.s, - subcomponent_graph) - return_paths.append(ctg_path) - return return_paths - @staticmethod def tally_incorporated_segments(incorporated_list, path): "Keep track of contig segments incorporated into path" @@ -433,35 +228,6 @@ def tally_incorporated_segments(incorporated_list, path): start=path_node.start, end=path_node.end)) - def find_paths(self, graph): - "Finds paths per input assembly file" - print(datetime.datetime.today(), ": Finding paths", file=sys.stdout) - Ntjoin.gin = graph - components = graph.components() - print("\nTotal number of components in graph:", len(components), "\n", sep=" ", file=sys.stdout) - - if self.args.t == 1: - paths = [self.find_paths_process(component) for component in components] - else: - with multiprocessing.Pool(self.args.t) as pool: - paths = pool.map(self.find_paths_process, components) - - paths_return = [] - incorporated_segments = {} - for path_list in paths: - for path in path_list: - paths_return.append(path) - self.tally_incorporated_segments(incorporated_segments, path) - - Ntjoin.incorporated_segments = incorporated_segments - - paths_return_merged = [] - for path in paths_return: - path = self.merge_relocations(path) - paths_return_merged.append(path) - - return paths_return_merged, incorporated_segments - @staticmethod def is_best_region(path_nodes, query_node): @@ -496,7 +262,7 @@ def is_subsumed(i, path, contig_regions): return True return False - def adjust_paths(self, paths, scaffolds): + def adjust_paths(self, paths, scaffolds, incorporated_segments): "Given the found paths, removes duplicate regions to avoid cutting sequences (no_cut=True option)" contig_regions = {} # contig_id -> [list of PathNode] for path in paths: @@ -511,7 +277,7 @@ def adjust_paths(self, paths, scaffolds): for i, node in enumerate(path): if not self.is_subsumed(i, path, contig_regions): new_path.append(node) - new_path = self.merge_relocations(new_path) + new_path = self.merge_relocations(new_path, incorporated_segments) intermediate_paths.append(new_path) new_paths = [] @@ -555,19 +321,12 @@ def read_fasta_file(self, filename): sys.exit(1) return scaffolds - @staticmethod - def reverse_complement(sequence): - "Reverse complements a given sequence" - translation_table = str.maketrans( - "ACGTUNMRWSYKVHDBacgtunmrwsykvhdb", - "TGCAANKYWSRMBDHVtgcaankywsrmbdhv") - return sequence[::-1].translate(translation_table) @staticmethod def get_fasta_segment(path_node, sequence): "Given a PathNode and the contig sequence, return the corresponding sequence" if path_node.ori == "-": - return Ntjoin.reverse_complement(sequence[path_node.start:path_node.end]) + \ + return ntjoin_utils.reverse_complement(sequence[path_node.start:path_node.end]) + \ "N"*path_node.gap_size return sequence[path_node.start:path_node.end] + "N"*path_node.gap_size @@ -765,7 +524,7 @@ def get_adjusted_sequence(self, sequence, node): return return_sequence - def print_scaffolds(self, paths, intersecting_regions): + def print_scaffolds(self, paths, intersecting_regions, prev_incorporated_segments): "Given the paths, print out the scaffolds fasta" print(datetime.datetime.today(), ": Printing output scaffolds", file=sys.stdout) assembly = self.args.s @@ -785,7 +544,7 @@ def print_scaffolds(self, paths, intersecting_regions): # Deal with merging relocations for i, path in enumerate(paths): - new_path = self.merge_relocations(path) + new_path = self.merge_relocations(path, prev_incorporated_segments) new_path = self.remove_overlapping_regions(new_path, intersecting_regions) self.check_terminal_node_gap_zero(new_path) paths[i] = new_path @@ -800,7 +559,7 @@ def print_scaffolds(self, paths, intersecting_regions): for node in path: if node.ori == "?": continue - sequences.append(self.get_fasta_segment(node, Ntjoin.scaffolds[node.contig].sequence)) + sequences.append(self.get_fasta_segment(node, self.scaffolds[node.contig].sequence)) nodes.append(node) if len(sequences) < 2: continue @@ -823,7 +582,7 @@ def print_scaffolds(self, paths, intersecting_regions): for node in path: if node.ori == "?": continue - sequences.append(self.get_fasta_segment(node, Ntjoin.scaffolds[node.contig].sequence)) + sequences.append(self.get_fasta_segment(node, self.scaffolds[node.contig].sequence)) path_segments.append(ntjoin_utils.Bed(contig=node.contig, start=node.start, end=node.end)) nodes.append(node) @@ -869,7 +628,7 @@ def print_unassigned(self, assembly, assembly_fa, incorporated_segments, params, for chrom, s, e in incorporated_segments]) incorporated_segments_bed = pybedtools.BedTool(incorporated_segments_str, from_string=True).sort() - genome_bed, genome_dict = self.format_bedtools_genome(Ntjoin.scaffolds) + genome_bed, genome_dict = self.format_bedtools_genome(self.scaffolds) missing_bed = genome_bed.complement(i=incorporated_segments_bed, g=genome_dict) missing_bed.saveas(self.args.p + "." + assembly + ".unassigned.bed") @@ -891,10 +650,10 @@ def print_unassigned(self, assembly, assembly_fa, incorporated_segments, params, raise subprocess.CalledProcessError(out_fasta.returncode, cmd_shlex) @staticmethod - def tally_intersecting_segments(): + def tally_intersecting_segments(incorporated_segments): "Tally ctgs with intersecting segments, and keep track of 'best'" incorporated_bed_list = [] - for _, bed_entry_list in Ntjoin.incorporated_segments.items(): + for _, bed_entry_list in incorporated_segments.items(): for bed_entry in bed_entry_list: incorporated_bed_list.append(bed_entry) incorporated_bed_str = "\n".join([f"{chrom}\t{s}\t{e}" @@ -909,7 +668,7 @@ def tally_intersecting_segments(): for bed in bed_intersect: if bed.count > 1: if bed.chrom not in overlap_regions: - overlap_regions[bed.chrom] = ntjoin_utils.OverlapRegion() + overlap_regions[bed.chrom] = OverlapRegion() overlap_regions[bed.chrom].add_region(ntjoin_utils.Bed(contig=bed.chrom, start=bed.start, end=bed.end)) overlap_regions_fix = {} @@ -918,16 +677,15 @@ def tally_intersecting_segments(): return overlap_regions_fix - @staticmethod - def find_mx_min_max(graph, target): + def find_mx_min_max(self, target): "Given a dictionary in the form mx->(ctg, pos), find the min/max mx position per ctg" mx_extremes = {} # ctg -> (min_pos, max_pos) - for mx in Ntjoin.list_mx_info[target]: + for mx in self.list_mx_info[target]: try: - graph.vs().find(mx) + self.graph.vs().find(mx) except ValueError: continue - ctg, pos = Ntjoin.list_mx_info[target][mx] + ctg, pos = self.list_mx_info[target][mx] if ctg in mx_extremes: mx_extremes[ctg] = (min(mx_extremes[ctg][0], pos), max(mx_extremes[ctg][1], pos)) @@ -935,78 +693,23 @@ def find_mx_min_max(graph, target): mx_extremes[ctg] = (pos, pos) return mx_extremes + def format_adjust_paths(self, paths): + "Format and adjust the paths for relocations, incorporated sections" + return_paths = [] + incorporated_segments = {} + for path_list in paths: + for path, sub_graph in path_list: + ctg_path = self.format_path(path, self.args.s, sub_graph) + return_paths.append(ctg_path) + self.tally_incorporated_segments(incorporated_segments, ctg_path) + + paths_return_merged = [] + for path in return_paths: + path = self.merge_relocations(path, incorporated_segments) + paths_return_merged.append(path) + + return paths_return_merged, incorporated_segments - @staticmethod - def parse_arguments(): - "Parse ntJoin arguments" - parser = argparse.ArgumentParser( - description="ntJoin: Genome analysis using reference assemblies and minimizer graphs", - ) - parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - - subparsers = parser.add_subparsers(dest="mode") - scaffold_parser = subparsers.add_parser("scaffold", - help="Scaffold the input target assembly using the supplied reference(s)", - epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" - "Example: myscaffolds.fa.k32.w1000.tsv - myscaffolds.fa is the expected matching fasta", - formatter_class=argparse.RawTextHelpFormatter) - scaffold_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") - scaffold_parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) - scaffold_parser.add_argument("-l", help="Weight of target genome assembly [1]", - required=False, default=1, type=float) - scaffold_parser.add_argument("-r", - help="List of reference assembly weights (in quotes, separated by spaces, " - "in same order as minimizer TSV files)", - required=True, type=str) - scaffold_parser.add_argument("-p", help="Output prefix [out]", default="out", - type=str, required=False) - scaffold_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) - scaffold_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - scaffold_parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) - scaffold_parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, - default=0, type=int) - scaffold_parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", - action='store_true') - scaffold_parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " - "increasing/decreasing to assign contig orientation [90]\n " - "Note: Only used with --mkt is NOT specified", default=90, type=int) - scaffold_parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) - scaffold_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - scaffold_parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") - scaffold_parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", - action="store_true") - scaffold_parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", - action="store_true") - scaffold_parser.add_argument("--overlap_gap", - help="Length of gap introduced between overlapping, trimmed segments [20]", - type=int, default=20) - scaffold_parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", - type=int, default=15) - scaffold_parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", - type=int, default=10) - scaffold_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " - "(computing minimizers, reading fasta file) [4]", - type=int, default=4) - - synteny_parser = subparsers.add_parser("synteny", help="Extract syntenic blocks from input assemblies") - synteny_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of input assemblies") - synteny_parser.add_argument("-n", help="Minimum edge weight [Number of input assemblies]", default=0, type=int) - synteny_parser.add_argument("-p", help="Output prefix [out]", - default="out", type=str, required=False) - synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - synteny_parser.add_argument("-w", help="Window size used for minimizers", required=True, type=int) - synteny_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions "\ - "(computing minimizers, reading fasta file) [4]", type=int, default=4) - synteny_parser.add_argument("--w-rounds", help="decreasing list of 'w' values to use for refining ends", - default=[100, 10, 5], nargs="+", type=int) - synteny_parser.add_argument("--dev", action="store_true", help="Developer mode - retain intermediate files") - synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit() - - return parser.parse_args() def print_parameters_scaffold(self): "Print the set parameters for the ntJoin scaffolding run" @@ -1037,162 +740,73 @@ def print_parameters_scaffold(self): print("\t--overlap_w", self.args.overlap_w) print("\t--btllib_t", self.args.btllib_t) - def print_parameters_synteny(self): - "Pring the set parameters for the ntJoin synteny run" - if self.args.n == 0: - self.args.n = len(self.args.FILES) - print("Running ntJoin synteny detection...") - print("Parameters:") - print("\tMinimizer TSV files: ", self.args.FILES) - print("\t-n", self.args.n) - print("\t-p", self.args.p) - print("\t-k", self.args.k) - print("\t-w", self.args.w) - print("\t--btllib_t", self.args.btllib_t) - print("\t--w-rounds", self.args.w_rounds) - - - def print_parameters(self): - "Print the parameters for the ntJoin in the specified mode" - if self.args.mode == "scaffold": - self.print_parameters_scaffold() - elif self.args.mode == "synteny": - self.print_parameters_synteny() - else: - raise ValueError(f"Unexpected mode: {self.args.mode}") - - def set_synteny_parameters(self): - "Set the default parameters for synteny mode" - self.args.r = " ".join(["1"] * len(self.args.FILES)) - self.args.t = 1 - - def main(self): - "Run ntJoin graph stage" - print("Running ntJoin v1.1.1 ...\n") - self.print_parameters() - - if self.args.mode == "synteny": - self.set_synteny_parameters() - - # Parse the weights of each input reference assembly - input_weights = [float(w) for w in re.split(r'\s+', self.args.r)] - if len(input_weights) != len(self.args.FILES): - print("ERROR: The length of supplied reference weights (-r) and " - "number of assembly minimizer TSV inputs must be equal.") - print("Supplied lengths of arguments:") - print("Weights (-r):", len(input_weights), "Minimizer TSV files:", len(self.args.FILES), sep=" ") - sys.exit(1) - - # Read in the minimizers for each assembly - list_mx_info = {} # Dictionary of dictionaries: assembly -> mx -> (contig, position) - list_mxs = {} # Dictionary: assembly -> [lists of mx] - weights = {} # Dictionary: assembly -> weight - for assembly in self.args.FILES: - mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) - list_mx_info[assembly] = mxs_info - list_mxs[assembly] = mxs - weights[assembly] = input_weights.pop(0) - if self.args.mode == "scaffold": - mxs_info, mxs = ntjoin_utils.read_minimizers(self.args.s) - list_mx_info[self.args.s] = mxs_info - list_mxs[self.args.s] = mxs - weights[self.args.s] = self.args.l - weight_str = "\n".join([f"{assembly}: {asm_weight}" for assembly, asm_weight in weights.items()]) - print("\nWeights of assemblies:\n", weight_str, "\n", sep="") - - Ntjoin.list_mx_info = list_mx_info - Ntjoin.weights = weights - - # Filter minimizers - Keep only if found in all assemblies - list_mxs = ntjoin_utils.filter_minimizers(list_mxs) - # Build a graph: Nodes = mx; Edges between adjacent mx in the assemblies - graph = self.build_graph(list_mxs, Ntjoin.weights) + def main_scaffolder(self): + "Run ntJoin scaffolding stage" + print("Running ntJoin scaffolding ...\n") - # Print the DOT graph - self.print_graph(graph) + self.load_minimizers() - # Filter the graph edges by minimum weight - graph = self.filter_graph_global(graph) + # Generate minimizer graph, and get paths through the graph + paths = self.make_minimizer_graph_and_paths() - # Find the min and max pos of minimizers for target assembly, per ctg - if self.args.mode == "scaffold": - Ntjoin.mx_extremes = self.find_mx_min_max(graph, self.args.s) + self.mx_extremes = self.find_mx_min_max(self.args.s) # Load target scaffolds into memory - if self.args.mode == "scaffold": - min_match = re.search(r'^(\S+).k\d+.w\d+\.tsv', self.args.s) - if not min_match: - print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") - print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") - sys.exit(1) - assembly_fa = min_match.group(1) - scaffolds = self.read_fasta_file(assembly_fa) # scaffold_id -> Scaffold - - Ntjoin.scaffolds = scaffolds + min_match = re.search(r'^(\S+).k\d+.w\d+\.tsv', self.args.s) + if not min_match: + print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") + print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") + sys.exit(1) + assembly_fa = min_match.group(1) + self.scaffolds = self.read_fasta_file(assembly_fa) # scaffold_id -> Scaffold # Find the paths through the graph - paths, incorporated_segments = self.find_paths(graph) - - if self.args.mode == "synteny": - with open(f"{self.args.p}.synteny_blocks.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) - self.refine_block_coordinates(list_mx_info, graph, paths) - - sys.exit() + paths = self.find_paths() - Ntjoin.incorporated_segments = incorporated_segments + # Format the paths to PathNodes, tally incorporated segments + paths, incorporated_segments = self.format_adjust_paths(paths) if self.args.no_cut: - paths = self.adjust_paths(paths, scaffolds) + paths = self.adjust_paths(paths, self.scaffolds, incorporated_segments) # Tally any regions that overlap - intersecting_regions = self.tally_intersecting_segments() + intersecting_regions = self.tally_intersecting_segments(incorporated_segments) # Print the final scaffolds - self.print_scaffolds(paths, intersecting_regions) + self.print_scaffolds(paths, intersecting_regions, incorporated_segments) print(datetime.datetime.today(), ": DONE!", file=sys.stdout) - def refine_block_coordinates(self, list_mx_info, graph, paths): - "Ready to start refining the synteny block coordinates" - prev_w = self.args.w - for new_w in self.args.w_rounds: - print(datetime.datetime.today(), ": Extending synteny blocks with w =", new_w, file=sys.stdout) - new_list_mxs, terminal_mxs = ntjoin_synteny.generate_additional_minimizers( - paths, new_w, prev_w, self.args.btllib_t, list_mx_info, self.args.dev) - graph = self.build_graph(new_list_mxs, Ntjoin.weights, graph=graph, black_list=terminal_mxs) - graph = self.filter_graph_global(graph) - paths, _ = self.find_paths(graph) - with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - prev_w = new_w - - print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) - with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - - def __init__(self): - "Create an ntJoin instance" - self.args = self.parse_arguments() - - -def main(): - "Run ntJoin" - Ntjoin().main() - -if __name__ == "__main__": - main() + def set_weights(self): + "Parse the supplied weights" + weights = [float(w) for w in re.split(r'\s+', self.args.r)] + if len(weights) != len(self.args.FILES): + print("ERROR: The length of supplied reference weights (-r) and " + "number of assembly minimizer TSV inputs must be equal.") + print("Supplied lengths of arguments:") + print("Weights (-r):", len(weights), "Minimizer TSV files:", len(self.args.FILES), sep=" ") + sys.exit(1) + return weights + + def load_minimizers(self): + "Load in minimizers for ntJoin scaffolding mode" + weights = {} # Dictionary: assembly -> weight + for assembly in self.args.FILES: + mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) + self.list_mx_info[assembly] = mxs_info + self.list_mxs[assembly] = mxs + weights[assembly] = self.weights_list.pop(0) + mxs_info, mxs = ntjoin_utils.read_minimizers(self.args.s) + self.list_mx_info[self.args.s] = mxs_info + self.list_mxs[self.args.s] = mxs + weights[self.args.s] = self.args.l + self.weights = weights + + def __init__(self, args): + "Create an ntJoin instance for scaffolding" + super().__init__(args) + self.weights_list = self.set_weights() + self.print_parameters_scaffold() + self.mx_extremes = {} # ctg -> (min_pos, max_pos) + self.scaffolds = {} # scaffold_id -> Scaffold diff --git a/bin/ntjoin_overlap.py b/bin/ntjoin_overlap.py index f6d595a..731f0c1 100644 --- a/bin/ntjoin_overlap.py +++ b/bin/ntjoin_overlap.py @@ -5,7 +5,6 @@ from collections import namedtuple import numpy as np import ntjoin_utils -import ntjoin_assemble MappedPathInfo = namedtuple("MappedPathInfo", @@ -26,7 +25,7 @@ def merge_overlapping(list_mxs, list_mx_info, source, target, nodes): list_mxs_pair = filter_minimizers_position(list_mxs_pair, source, target, list_mx_info, nodes) with ntjoin_utils.HiddenPrints(): - graph = ntjoin_assemble.Ntjoin.build_graph(ntjoin_assemble.Ntjoin(), list_mxs_pair, weights) + graph = ntjoin_utils.build_graph(list_mxs_pair, weights) graph = filter_graph_global(graph, 2) paths_components = [] @@ -36,12 +35,12 @@ def merge_overlapping(list_mxs, list_mx_info, source, target, nodes): singleton_nodes = [node.index for node in component_graph.vs() if node.degree() == 0] if len(source_nodes) == 2: source_node, target_node = source_nodes - if ntjoin_assemble.Ntjoin.vertex_name(component_graph, source_node) > \ - ntjoin_assemble.Ntjoin.vertex_name(component_graph, target_node): + if ntjoin_utils.vertex_name(component_graph, source_node) > \ + ntjoin_utils.vertex_name(component_graph, target_node): source_node, target_node = target_node, source_node paths = component_graph.get_shortest_paths(source_node, target_node) assert len(paths) == 1 - path = [ntjoin_assemble.Ntjoin.vertex_name(component_graph, mx) for mx in paths[0]] + path = [ntjoin_utils.vertex_name(component_graph, mx) for mx in paths[0]] start_mx, end_mx = path[0], path[-1] source_start, target_start = [list_mx_info[assembly][start_mx] for assembly in [source, target]] @@ -64,7 +63,7 @@ def merge_overlapping(list_mxs, list_mx_info, source, target, nodes): [mid_mx_dist_end_source, mid_mx_dist_end_target]))) elif singleton_nodes: assert len(singleton_nodes) == 1 - mid_mx = ntjoin_assemble.Ntjoin.vertex_name(component_graph, singleton_nodes[0]) + mid_mx = ntjoin_utils.vertex_name(component_graph, singleton_nodes[0]) mid_mx_dist_end_source = get_dist_from_end(source, list_mx_info[source][mid_mx], nodes[source].get_aligned_length()) mid_mx_dist_end_target = get_dist_from_end(target, list_mx_info[target][mid_mx], diff --git a/bin/ntjoin_run.py b/bin/ntjoin_run.py new file mode 100755 index 0000000..9d34a35 --- /dev/null +++ b/bin/ntjoin_run.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +ntJoin: Scaffolding assemblies and analyzing synteny +using reference assemblies and minimizer graphs +Written by Lauren Coombe (@lcoombe) +""" +import argparse +import sys +from ntjoin_synteny import NtjoinSynteny +from ntjoin_assemble import NtjoinScaffolder + +def parse_arguments(): + "Parse ntJoin arguments" + parser = argparse.ArgumentParser( + description="ntJoin: Genome analysis using reference assemblies and minimizer graphs", + ) + parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + + subparsers = parser.add_subparsers(dest="mode") + scaffold_parser = subparsers.add_parser("scaffold", + help="Scaffold the input target assembly using the supplied reference(s)", + epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" + "Example: myscaffolds.fa.k32.w1000.tsv - myscaffolds.fa is the expected matching fasta", + formatter_class=argparse.RawTextHelpFormatter) + scaffold_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") + scaffold_parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) + scaffold_parser.add_argument("-l", help="Weight of target genome assembly [1]", + required=False, default=1, type=float) + scaffold_parser.add_argument("-r", + help="List of reference assembly weights (in quotes, separated by spaces, " + "in same order as minimizer TSV files)", + required=True, type=str) + scaffold_parser.add_argument("-p", help="Output prefix [out]", default="out", + type=str, required=False) + scaffold_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) + scaffold_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) + scaffold_parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) + scaffold_parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, + default=0, type=int) + scaffold_parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", + action='store_true') + scaffold_parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " + "increasing/decreasing to assign contig orientation [90]\n " + "Note: Only used with --mkt is NOT specified", default=90, type=int) + scaffold_parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) + scaffold_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + scaffold_parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") + scaffold_parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", + action="store_true") + scaffold_parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", + action="store_true") + scaffold_parser.add_argument("--overlap_gap", + help="Length of gap introduced between overlapping, trimmed segments [20]", + type=int, default=20) + scaffold_parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", + type=int, default=15) + scaffold_parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", + type=int, default=10) + scaffold_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " + "(computing minimizers, reading fasta file) [4]", + type=int, default=4) + + synteny_parser = subparsers.add_parser("synteny", help="Extract syntenic blocks from input assemblies") + synteny_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of input assemblies") + synteny_parser.add_argument("-n", help="Minimum edge weight [Number of input assemblies]", default=0, type=int) + synteny_parser.add_argument("-p", help="Output prefix [out]", + default="out", type=str, required=False) + synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) + synteny_parser.add_argument("-w", help="Window size used for minimizers", required=True, type=int) + synteny_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions "\ + "(computing minimizers, reading fasta file) [4]", type=int, default=4) + synteny_parser.add_argument("--w-rounds", help="decreasing list of 'w' values to use for refining ends", + default=[100, 10, 5], nargs="+", type=int) + synteny_parser.add_argument("--dev", action="store_true", help="Developer mode - retain intermediate files") + synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + + if len(sys.argv) == 1: + parser.print_help() + sys.exit() + + return parser.parse_args() + +def main(): + "Run ntJoin" + args = parse_arguments() + if args.mode == "scaffold": + NtjoinScaffolder(args).main_scaffolder() + elif args.mode == "synteny": + NtjoinSynteny(args).main_synteny() + else: + raise ValueError(f"Unexpected mode: {args.mode}") + +if __name__ == "__main__": + main() diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py index 9e14d4a..33adf35 100644 --- a/bin/ntjoin_synteny.py +++ b/bin/ntjoin_synteny.py @@ -4,263 +4,277 @@ Written by Lauren Coombe @lcoombe """ -from collections import namedtuple, defaultdict +from collections import defaultdict +import datetime import re import shlex import subprocess import sys import intervaltree import ntjoin_utils +import ntjoin import pybedtools +from synteny_block import SyntenyBlock -# Named tuples -Minimizer = namedtuple("Minimizer", ["mx", "position"]) # Regexes fai_re = re.compile(r'^(\S+).k\d+.w\d+.tsv') -class SyntenyBlock: - "A Synteny Block between the input assemblies" - def __init__(self, k, *assemblies): - "Instantiate a dictionary to keep track of assembly blocks for this synteny block" - self.assembly_blocks = {assembly: AssemblyBlock() for assembly in assemblies} - self.k = k # k-mer size used for minimizers, needed to adjust end coordinates - - - def continue_block(self, mx, list_mx_info): - "Given minimizer and preliminary blocks, return if synteny block should extend, else False" - return all(mx_dict[mx][0] == self.assembly_blocks[assembly].contig_id \ - for assembly, mx_dict in list_mx_info.items()) - - def extend_block(self, mx, list_mx_jnfo): - "Extend the synteny block by extending each assembly block" - for assembly, mx_dict in list_mx_jnfo.items(): - ctg, pos = mx_dict[mx] - assert self.assembly_blocks[assembly].contig_id == ctg - self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) - - def start_block(self, mx, list_mx_info): - "Start the new synteny block" - for assembly, mx_dict in list_mx_info.items(): - ctg, pos = mx_dict[mx] - self.assembly_blocks[assembly].contig_id = ctg - self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) - - def determine_orientations(self): - "Determine the orientations of each assembly block" - for _, assembly_block in self.assembly_blocks.items(): - positions = [mx.position for mx in assembly_block.minimizers] - if all(x < y for x, y in zip(positions, positions[1:])): - assembly_block.ori = "+" - elif all(x > y for x, y in zip(positions, positions[1:])): - assembly_block.ori = "-" +class NtjoinSynteny(ntjoin.Ntjoin): + "Instance for ntJoin synteny mode" + + def __init__(self, args): + super().__init__(args) + self.weights_list = [1] * len(self.args.FILES) + self.args.t = 1 + self.print_parameters_synteny() + + def print_parameters_synteny(self): + "Pring the set parameters for the ntJoin synteny run" + if self.args.n == 0: + self.args.n = len(self.args.FILES) + print("Running ntJoin synteny detection...") + print("Parameters:") + print("\tMinimizer TSV files: ", self.args.FILES) + print("\t-n", self.args.n) + print("\t-p", self.args.p) + print("\t-k", self.args.k) + print("\t-w", self.args.w) + print("\t--btllib_t", self.args.btllib_t) + print("\t--w-rounds", self.args.w_rounds) + + + def find_synteny_blocks(self, path): + "Given a path (sequence of mx), print the order/orientation/regions of contigs for an assembly" + out_blocks = [] # List of SyntenyBlock + prelim_blocks = SyntenyBlock(self.args.k, *list(self.list_mx_info.keys())) + past_start_flag = False + for mx in path: + if prelim_blocks.continue_block(mx, self.list_mx_info): + prelim_blocks.extend_block(mx, self.list_mx_info) else: - assembly_block.ori = "?" - - def all_oriented(self): - "Return true if all of the assembly blocks in the synteny block are oriented" - return all(assembly_block.ori in ["+", "-"] for _, assembly_block in self.assembly_blocks.items()) - - def get_block_string(self, num): - "Given the specified synteny block ID, print the synteny blocks" - return_str = "" - for assembly, assembly_block in self.assembly_blocks.items(): - start_pos = assembly_block.get_block_start() - end_pos = assembly_block.get_block_end() + self.k - block_string = f"{num}\t{assembly}\t{assembly_block.contig_id}\t{start_pos}" \ - f"\t{end_pos}\t{assembly_block.ori}\n" - return_str += block_string - return return_str - - -class AssemblyBlock: - "An assembly block for a given assembly. The AssemblyBlock objects per assembly make up a SyntenyBlock" - def __init__(self): - "Instantiate the AssemblyBlock" - self.contig_id = None - self.minimizers = [] - self.ori = None - - def get_block_start(self): - "Get the starting coordinate of the assembly block" - return min(self.minimizers[0].position, self.minimizers[-1].position) - - def get_block_end(self): - "Get the end coordinate of the assembly block" - return max(self.minimizers[0].position, self.minimizers[-1].position) - - def get_block_terminal_mx(self): - "Return the terminal minimizer hashes for the assembly block" - return self.contig_id, self.minimizers[0], self.minimizers[-1] - - def get_block_internal_mx_hashes(self): - "Return the internal minimizer hashes for the assembly block" - return [mx_pos.mx for mx_pos in self.minimizers[1:-1]] - - -def find_synteny_blocks(path, list_mx_info, k): - "Given a path (sequence of mx), print the order/orientation/regions of contigs for an assembly" - out_blocks = [] # List of SyntenyBlock - prelim_blocks = SyntenyBlock(k, *list(list_mx_info.keys())) - past_start_flag = False - for mx in path: - if prelim_blocks.continue_block(mx, list_mx_info): - prelim_blocks.extend_block(mx, list_mx_info) - else: - # This is either the first mx, or we are past a stretch of repeating contigs - if past_start_flag: - prelim_blocks.determine_orientations() - if prelim_blocks.all_oriented(): - out_blocks.append(prelim_blocks) - prelim_blocks = SyntenyBlock(k, *list(list_mx_info.keys())) - prelim_blocks.start_block(mx, list_mx_info) - - prelim_blocks.determine_orientations() - if prelim_blocks.all_oriented(): - out_blocks.append(prelim_blocks) - - return out_blocks - -def find_fa_name(assembly_mx_name): - "Given the mx file name, return the corresponding fai file name" - if fai_match := re.search(fai_re, assembly_mx_name): - return f"{fai_match.group(1)}" - print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") - print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") - sys.exit(1) - -def get_synteny_bed_lists(paths): - "Given a set of synteny blocks, return a dictionary with a BED interval list per contig, per assembly" - synteny_beds = {} - for subcomponent in paths: - for block in subcomponent: - for assembly, assembly_block in block.assembly_blocks.items(): - if assembly not in synteny_beds: - synteny_beds[assembly] = {} - if assembly_block.contig_id not in synteny_beds[assembly]: - synteny_beds[assembly][assembly_block.contig_id] = [] - synteny_beds[assembly][assembly_block.contig_id].append( - ntjoin_utils.Bed(assembly_block.contig_id, - assembly_block.get_block_start(), - assembly_block.get_block_end() + block.k)) - - return synteny_beds - -def mask_assemblies_with_synteny_extents(synteny_beds, w): - "Mask each reference assembly with determined synteny blocks" - mx_to_fa_dict = {} - for assembly, contig_dict in synteny_beds.items(): - bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ - for bed in contig_dict[ctg] if bed.end - bed.start > 2*w] - bed_str = "\n".join(bed_str) - fa_filename = find_fa_name(assembly) - synteny_bed = pybedtools.BedTool(bed_str, from_string=True).slop(g=f"{fa_filename}.fai", l=-1*w, r=-1*w).sort() - synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") - mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" - return mx_to_fa_dict - -def delete_w_iteration_files(*filenames): - "Delete the given files for the specific lower w iteration" - for filename in filenames: - cmd = shlex.split(f"rm {filename}") - ret_code = subprocess.call(cmd) - assert ret_code == 0 - -def generate_new_minimizers(tsv_to_fa_dict, k, w, t, retain_files=False): - "Given the masked fasta files, generate minimizers at new w for each" - list_mxs = {} - new_list_mxs_info = {} - for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): - indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, k, w, t) - mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) - new_list_mxs_info[assembly_tsv] = mx_info - list_mxs[assembly_tsv] = mxs_filt - if not retain_files: - delete_w_iteration_files(indexlr_filename, assembly_masked) - return list_mxs, new_list_mxs_info - -def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): - "Update the given dictionary of trees with the new extent" - start_pos = min(mx1.position, mx2.position) - end_pos = max(mx1.position, mx2.position) - if assembly_name not in trees or ctg not in trees[assembly_name]: - trees[assembly_name][ctg] = intervaltree.IntervalTree() - if trees[assembly_name][ctg][start_pos+1:end_pos]: # Checking that this doesn't overlap with anything - print("WARNING: detected overlapping segments:", assembly_name, ctg, start_pos+1, end_pos, - file=sys.stderr) - trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) - -def find_mx_in_blocks(paths): - "Given the synteny blocks, find the minimizers at the terminal ends of each block, and internal" - terminal_mxs = set() - internal_mxs = set() - intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents - - for subcomponent in paths: - for block in subcomponent: - curr_mx_len = len(terminal_mxs) - for assembly, assembly_block in block.assembly_blocks.items(): - contig, mx1, mx2 = assembly_block.get_block_terminal_mx() - terminal_mxs.add(mx1.mx) - terminal_mxs.add(mx2.mx) - update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) - internal = assembly_block.get_block_internal_mx_hashes() - internal_mxs = internal_mxs.union(internal) - assert len(terminal_mxs) == (curr_mx_len + 2) - return terminal_mxs, internal_mxs, intervaltrees - -def check_non_overlapping(paths): - "Given the paths, do final check to ensure intervals are not overlapping, will print warnings if that's the case" - intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents - for subcomponent in paths: - for block in subcomponent: - for assembly, assembly_block in block.assembly_blocks.items(): - contig, mx1, mx2 = assembly_block.get_block_terminal_mx() - update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) - - -def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_mx_info): - "Filter minimizers found in the mx black list" - return_mxs = {} - for assembly in list_mxs: - assembly_mxs_filtered = [] - for mx_list in list_mxs[assembly]: - new_list = [] - for mx in mx_list: - ctg, pos = list_mx_info[assembly][mx] - i_tree = intervaltrees[assembly][ctg] - if new_list: - prev_pos = list_mx_info[assembly][new_list[-1]][1] - start = min(prev_pos, pos) - end = max(prev_pos, pos) - if i_tree[start:end]: # Split the mx adjacency if it spans over a known synteny block - assembly_mxs_filtered.append(new_list) - new_list = [] - if mx not in black_list and not i_tree[pos]: - new_list.append(mx) - assembly_mxs_filtered.append(new_list) - - return_mxs[assembly] = assembly_mxs_filtered - return return_mxs - -def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): - "Update the directory containing mx -> contig, position associations" - valid_mxs = set({mx for _, list_mx_val in list_mxs.items() \ - for list_mx in list_mx_val for mx in list_mx}) - for assembly, mx_dict in new_list_mx_info.items(): - for mx in mx_dict: - if mx in valid_mxs and mx not in list_mx_info[assembly]: - list_mx_info[assembly][mx] = mx_dict[mx] - - -def generate_additional_minimizers(paths, new_w, prev_w, t, list_mx_info, dev=False): - "Given the existing synteny blocks, generate minimizers for increased block resolution" - k = paths[0][0].k - synteny_beds = get_synteny_bed_lists(paths) - mx_to_fa_dict = mask_assemblies_with_synteny_extents(synteny_beds, prev_w) - list_mxs, new_list_mx_info = generate_new_minimizers(mx_to_fa_dict, k, new_w, t, retain_files=dev) - terminal_mx, internal_mx, interval_trees = find_mx_in_blocks(paths) - list_mxs = filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) - list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies - update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info) - return list_mxs, terminal_mx + # This is either the first mx, or we are past a stretch of repeating contigs + if past_start_flag: + prelim_blocks.determine_orientations() + if prelim_blocks.all_oriented(): + out_blocks.append(prelim_blocks) + prelim_blocks = SyntenyBlock(self.args.k, *list(self.list_mx_info.keys())) + prelim_blocks.start_block(mx, self.list_mx_info) + + prelim_blocks.determine_orientations() + if prelim_blocks.all_oriented(): + out_blocks.append(prelim_blocks) + + return out_blocks + + @staticmethod + def find_fa_name(assembly_mx_name): + "Given the mx file name, return the corresponding fai file name" + if fai_match := re.search(fai_re, assembly_mx_name): + return f"{fai_match.group(1)}" + print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") + print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") + sys.exit(1) + + @staticmethod + def get_synteny_bed_lists(paths): + "Given a set of synteny blocks, return a dictionary with a BED interval list per contig, per assembly" + synteny_beds = {} + for subcomponent in paths: + for block in subcomponent: + for assembly, assembly_block in block.assembly_blocks.items(): + if assembly not in synteny_beds: + synteny_beds[assembly] = {} + if assembly_block.contig_id not in synteny_beds[assembly]: + synteny_beds[assembly][assembly_block.contig_id] = [] + synteny_beds[assembly][assembly_block.contig_id].append( + ntjoin_utils.Bed(assembly_block.contig_id, + assembly_block.get_block_start(), + assembly_block.get_block_end() + block.k)) + + return synteny_beds + + def mask_assemblies_with_synteny_extents(self, synteny_beds, w): + "Mask each reference assembly with determined synteny blocks" + mx_to_fa_dict = {} + for assembly, contig_dict in synteny_beds.items(): + bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ + for bed in contig_dict[ctg] if bed.end - bed.start > 2*w] + bed_str = "\n".join(bed_str) + fa_filename = self.find_fa_name(assembly) + synteny_bed = pybedtools.BedTool(bed_str, from_string=True).slop(g=f"{fa_filename}.fai", + l=-1*w, r=-1*w).sort() + synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") + mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" + return mx_to_fa_dict + + @staticmethod + def delete_w_iteration_files(*filenames): + "Delete the given files for the specific lower w iteration" + for filename in filenames: + cmd = shlex.split(f"rm {filename}") + ret_code = subprocess.call(cmd) + assert ret_code == 0 + + def generate_new_minimizers(self, tsv_to_fa_dict, w, retain_files=False): + "Given the masked fasta files, generate minimizers at new w for each" + list_mxs = {} + new_list_mxs_info = {} + for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): + indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, self.args.k, w, self.args.btllib_t) + mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) + new_list_mxs_info[assembly_tsv] = mx_info + list_mxs[assembly_tsv] = mxs_filt + if not retain_files: + self.delete_w_iteration_files(indexlr_filename, assembly_masked) + return list_mxs, new_list_mxs_info + + @staticmethod + def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): + "Update the given dictionary of trees with the new extent" + start_pos = min(mx1.position, mx2.position) + end_pos = max(mx1.position, mx2.position) + if assembly_name not in trees or ctg not in trees[assembly_name]: + trees[assembly_name][ctg] = intervaltree.IntervalTree() + if trees[assembly_name][ctg][start_pos+1:end_pos]: # Checking that this doesn't overlap with anything + print("WARNING: detected overlapping segments:", assembly_name, ctg, start_pos+1, end_pos, + file=sys.stderr) + trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) + + def find_mx_in_blocks(self, paths): + "Given the synteny blocks, find the minimizers at the terminal ends of each block, and internal mxs" + terminal_mxs = set() + internal_mxs = set() + intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents + + for subcomponent in paths: + for block in subcomponent: + curr_mx_len = len(terminal_mxs) + for assembly, assembly_block in block.assembly_blocks.items(): + contig, mx1, mx2 = assembly_block.get_block_terminal_mx() + terminal_mxs.add(mx1.mx) + terminal_mxs.add(mx2.mx) + self.update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) + internal = assembly_block.get_block_internal_mx_hashes() + internal_mxs = internal_mxs.union(internal) + assert len(terminal_mxs) == (curr_mx_len + 2) + return terminal_mxs, internal_mxs, intervaltrees + + def check_non_overlapping(self, paths): + "Given the paths, do final check to ensure intervals are not overlapping, will print warnings if so" + intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents + for subcomponent in paths: + for block in subcomponent: + for assembly, assembly_block in block.assembly_blocks.items(): + contig, mx1, mx2 = assembly_block.get_block_terminal_mx() + self.update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) + + @staticmethod + def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_mx_info): + "Filter minimizers found in the mx black list" + return_mxs = {} + for assembly in list_mxs: + assembly_mxs_filtered = [] + for mx_list in list_mxs[assembly]: + new_list = [] + for mx in mx_list: + ctg, pos = list_mx_info[assembly][mx] + i_tree = intervaltrees[assembly][ctg] + if new_list: + prev_pos = list_mx_info[assembly][new_list[-1]][1] + start = min(prev_pos, pos) + end = max(prev_pos, pos) + if i_tree[start:end]: # Split the mx adjacency if it spans over a known synteny block + assembly_mxs_filtered.append(new_list) + new_list = [] + if mx not in black_list and not i_tree[pos]: + new_list.append(mx) + assembly_mxs_filtered.append(new_list) + + return_mxs[assembly] = assembly_mxs_filtered + return return_mxs + + @staticmethod + def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): + "Update the directory containing mx -> contig, position associations" + valid_mxs = set({mx for _, list_mx_val in list_mxs.items() \ + for list_mx in list_mx_val for mx in list_mx}) + for assembly, mx_dict in new_list_mx_info.items(): + for mx in mx_dict: + if mx in valid_mxs and mx not in list_mx_info[assembly]: + list_mx_info[assembly][mx] = mx_dict[mx] + + def refine_block_coordinates(self, paths): + "Ready to start refining the synteny block coordinates" + prev_w = self.args.w + for new_w in self.args.w_rounds: + print(datetime.datetime.today(), ": Extending synteny blocks with w =", new_w, file=sys.stdout) + new_list_mxs, terminal_mxs = self.generate_additional_minimizers( + paths, new_w, prev_w, self.list_mx_info, self.args.dev) + graph = ntjoin_utils.build_graph(new_list_mxs, self.weights, graph=self.graph, black_list=terminal_mxs) + graph = self.filter_graph_global(graph) + paths = self.find_paths_synteny_blocks(self.find_paths()) + with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + prev_w = new_w + + print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) + with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + + def generate_additional_minimizers(self, paths, new_w, prev_w, list_mx_info, dev=False): + "Given the existing synteny blocks, generate minimizers for increased block resolution" + synteny_beds = self.get_synteny_bed_lists(paths) + mx_to_fa_dict = self.mask_assemblies_with_synteny_extents(synteny_beds, prev_w) + list_mxs, new_list_mx_info = self.generate_new_minimizers(mx_to_fa_dict, new_w, retain_files=dev) + terminal_mx, internal_mx, interval_trees = self.find_mx_in_blocks(paths) + list_mxs = self.filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) + list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies + self.update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info) + return list_mxs, terminal_mx + + def find_paths_synteny_blocks(self, paths): + "Given a list of paths, return a list of representative synteny blocks" + return [self.find_synteny_blocks(blocks) for path in paths for blocks, _ in path] + + def load_minimizers(self): + "Read the minimizers for the synteny mode" + weights = {} # Dictionary: assembly -> weight + for assembly in self.args.FILES: + mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) + self.list_mx_info[assembly] = mxs_info + self.list_mxs[assembly] = mxs + weights[assembly] = self.weights_list.pop(0) + self.weights = weights + + def main_synteny(self): + "Run the steps for ntJoin synteny mode" + print("Running ntJoin synteny detection ...\n", file=sys.stdout) + + # Run the common ntJoin steps + self.load_minimizers() + + self.make_minimizer_graph_and_paths() + + paths = self.ntjoin_find_paths() + + paths = self.find_paths_synteny_blocks(paths) + + with open(f"{self.args.p}.synteny_blocks.tsv", 'w', encoding="utf-8") as outfile: + block_num = 0 + for subcomponent in paths: + for block in subcomponent: + outfile.write(block.get_block_string(block_num)) + block_num += 1 + print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) + self.refine_block_coordinates(paths) + + print(datetime.datetime.today(), ": DONE!", file=sys.stdout) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 8a548e9..2e63e08 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -5,11 +5,12 @@ """ import datetime -from collections import namedtuple +from collections import namedtuple, defaultdict import shlex import subprocess import sys import os +import igraph as ig # Defining namedtuples @@ -17,6 +18,7 @@ Agp = namedtuple("Unassigned_bed", ["new_id", "contig", "start", "end"]) Scaffold = namedtuple("Scaffold", ["id", "length", "sequence"]) EdgeGraph = namedtuple("EdgeGraph", ["source", "target", "raw_gap_est"]) +Minimizer = namedtuple("Minimizer", ["mx", "position"]) class HiddenPrints: "Adapted from: https://stackoverflow.com/questions/8391411/how-to-block-calls-to-print" @@ -31,7 +33,87 @@ def __exit__(self, exc_type, exc_val, exc_tb): sys.stdout.close() sys.stdout = self._original_stdout -# Helper functions +# Helper functions for interfacing with python-igraph +def vertex_index(graph, name): + "Returns vertex index based on vertex name" + return graph.vs.find(name).index + +def vertex_name(graph, index): + "Returns vertex name based on vertex id" + return graph.vs[index]['name'] + +def edge_index(graph, source_name, target_name): + "Returns graph edge index based on source/target names" + return graph.get_eid(source_name, target_name) + +def set_edge_attributes(graph, edge_attributes): + "Sets the edge attributes for a python-igraph graph" + graph.es()["support"] = [edge_attributes[e]['support'] for e in sorted(edge_attributes.keys())] + graph.es()["weight"] = [edge_attributes[e]['weight'] for e in sorted(edge_attributes.keys())] + +def calc_total_weight(list_files, weights): + "Calculate the total weight of an edge given the assembly support" + return sum([weights[f] for f in list_files]) + +def build_graph(list_mxs, weights, graph=None, black_list=None): + "Builds an undirected graph: nodes=minimizers; edges=between adjacent minimizers" + print(datetime.datetime.today(), ": Building graph", file=sys.stdout) + + if graph is None: + graph = ig.Graph() + prev_edge_attributes = {} + else: + prev_edge_attributes = {e.index: {"support": e['support'], + "weight": e['weight']} for e in graph.es()} + + vertices = set() + edges = defaultdict(dict) # source -> target -> [list assembly support] + + for assembly in list_mxs: + for assembly_mx_list in list_mxs[assembly]: + for i, j in zip(range(0, len(assembly_mx_list)), + range(1, len(assembly_mx_list))): + if assembly_mx_list[i] in edges and \ + assembly_mx_list[j] in edges[assembly_mx_list[i]]: + edges[assembly_mx_list[i]][assembly_mx_list[j]].append(assembly) + elif assembly_mx_list[j] in edges and \ + assembly_mx_list[i] in edges[assembly_mx_list[j]]: + edges[assembly_mx_list[j]][assembly_mx_list[i]].append(assembly) + else: + edges[assembly_mx_list[i]][assembly_mx_list[j]] = [assembly] + if black_list is None or assembly_mx_list[i] not in black_list: + vertices.add(assembly_mx_list[i]) + if assembly_mx_list: + if black_list is None or assembly_mx_list[-1] not in black_list: + vertices.add(assembly_mx_list[-1]) + + formatted_edges = [(s, t) for s in edges for t in edges[s]] + + print(datetime.datetime.today(), ": Adding vertices", file=sys.stdout) + graph.add_vertices(list(vertices)) + + print(datetime.datetime.today(), ": Adding edges", file=sys.stdout) + graph.add_edges(formatted_edges) + + print(datetime.datetime.today(), ": Adding attributes", file=sys.stdout) + edge_attributes = {edge_index(graph, s, t): {"support": edges[s][t], + "weight": calc_total_weight(edges[s][t], + weights)} + for s in edges for t in edges[s]} + edge_attributes.update(prev_edge_attributes) + set_edge_attributes(graph, edge_attributes) + + return graph + +# Other helper functions + +def reverse_complement(sequence): + "Reverse complements a given sequence" + translation_table = str.maketrans( + "ACGTUNMRWSYKVHDBacgtunmrwsykvhdb", + "TGCAANKYWSRMBDHVtgcaankywsrmbdhv") + return sequence[::-1].translate(translation_table) + def filter_minimizers(list_mxs): "Filters out minimizers that are not found in all assemblies" print(datetime.datetime.today(), ": Filtering minimizers", file=sys.stdout) @@ -80,152 +162,3 @@ def run_indexlr(assembly, k, w, t): ret_code = subprocess.call(cmd) assert ret_code == 0 return f"{assembly}.k{k}.w{w}.tsv" - - -# Defining helper classes -class PathNode: - "Defines a node in a path of contig regions" - def __init__(self, contig, ori, start, end, contig_size, - first_mx, terminal_mx, gap_size=0, raw_gap_size=0): - self.contig = contig - self.ori = ori - self.start = start - self.end = end - self.contig_size = contig_size - self.first_mx = first_mx - self.terminal_mx = terminal_mx - self.gap_size = gap_size - self.raw_gap_size = raw_gap_size - self.start_adjust = 0 - self.end_adjust = 0 # Adjust for trimming - - def set_gap_size(self, gap_size): - "Set the gap size of the path node" - self.gap_size = gap_size - - def set_raw_gap_size(self, raw_gap_size): - "Set the 'raw' gap size. Equal to gap_size if > min_gap_size" - self.raw_gap_size = raw_gap_size - - def get_aligned_length(self): - "Get the aligned length based on start/end coordinates" - return self.end - self.start - - def get_end_adjusted_coordinate(self): - "Return the adjusted end coordinate" - if self.end_adjust == 0: - return self.get_aligned_length() - return self.end_adjust - - def get_adjusted_start(self): - "Return the start coordinate of segment, adjusted for any trimming" - if self.ori == "+": - return self.start + self.start_adjust - if self.ori == "-": - return self.start + (self.get_aligned_length() - self.get_end_adjusted_coordinate()) - raise OrientationError() - - def get_adjusted_end(self): - "Return the end coordinate of the segment, adjusted for any trimming" - if self.ori == "+": - return self.end - (self.get_aligned_length() - self.get_end_adjusted_coordinate()) - if self.ori == "-": - return self.end - self.start_adjust - raise OrientationError() - - def __str__(self): - return f"Contig:{self.contig}\tOrientation:{self.ori}\tStart-End:{self.start}-{self.end}\t"\ - f"Length:{self.contig_size}\tFirstmx:{self.first_mx}\tLastmx:{self.terminal_mx}\t" \ - f"Adjusted_start-end:{self.start_adjust}-{self.end_adjust}" - -class OrientationError(ValueError): - "Orientation type error" - def __init__(self): - self.message = "Orientation must be + or -" - super().__init__(self.message) - -class OverlapRegion: - "Overlapping regions in a contig to fix" - def __init__(self): - self.regions = [] - self.best_region = None - - def add_region(self, bed_region): - "Add a new region to the overlapping set" - if self.best_region is None or \ - (bed_region.end - bed_region.start) > \ - (self.best_region.end - self.best_region.start): - self.best_region = bed_region - self.regions.append(bed_region) - assert bed_region.contig == self.best_region.contig - - @staticmethod - def are_overlapping(region1, region2): - "Returns True if the given regions are overlapping" - return region1.start <= region2.end and region2.start <= region1.end - - @staticmethod - def is_subsumed(region1, region2): - "Returns True is region 1 is subsumed in region2" - return region1.start >= region2.start and region1.end <= region2.end - - def find_non_overlapping(self): - "Given overlapping regions, resolve to remove overlaps" - return_regions = {} # Bed -> (replacement Bed) or None - if not self.regions or self.best_region is None: - return None - for bed_region in self.regions: - if bed_region == self.best_region: - return_regions[bed_region] = bed_region - elif self.is_subsumed(bed_region, self.best_region): - # Subsumed region in the best region - return_regions[bed_region] = None - elif self.are_overlapping(bed_region, self.best_region): - # Overlaps with best region, but isn't subsumed - if bed_region.start <= self.best_region.start: - new_region = Bed(contig=bed_region.contig, start=bed_region.start, - end=self.best_region.start - 1) - elif bed_region.end >= self.best_region.end: - new_region = Bed(contig=bed_region.contig, start=self.best_region.end + 1, - end=bed_region.end) - return_regions[bed_region] = new_region - else: - return_regions[bed_region] = bed_region - - # Double check if any still overlaps. If so, adjust smaller of the overlapping regions. - existing_overlaps = True - while existing_overlaps: - sorted_regions = sorted([(b, a) for b, a in return_regions.items() if a is not None], key=lambda x: x[1]) - i, j = 0, 1 - existing_overlaps = False - while j < len(sorted_regions): - region1_before, region2_before = sorted_regions[i][0], sorted_regions[j][0] - region1_after, region2_after = sorted_regions[i][1], sorted_regions[j][1] - if region1_after is None or region2_after is None: - i += 1 - j += 1 - continue - if self.are_overlapping(region1_after, region2_after): - existing_overlaps = True - if self.is_subsumed(region1_after, region2_after): - # Region 1 is subsumed in region 2 - Remove region 1 - return_regions[region1_before] = None - elif self.is_subsumed(region2_after, region1_after): - # Region 2 is subsumed in region 1 - Remove region 2 - return_regions[region2_before] = None - elif (region1_after.end - region1_after.start) > (region2_after.end - region2_after.start): - # Adjust region 2 start - return_regions[region2_before] = Bed(contig=region2_after.contig, start=region1_after.end + 1, - end=region2_after.end) - elif (region1_after.end - region1_after.start) <= (region2_after.end - region2_after.start): - # Adjust region 1 end - return_regions[region1_before] = Bed(contig=region1_after.contig, start=region1_after.start, - end=region2_after.start - 1) - else: - print("Unexpected case!") - print(region1_before, region2_before, region1_after, region1_after) - - i += 1 - j += 1 - - return return_regions diff --git a/bin/overlap_region.py b/bin/overlap_region.py new file mode 100644 index 0000000..58aa7dc --- /dev/null +++ b/bin/overlap_region.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +''' +Representing an overlapping region in a contig to resolve +''' +from ntjoin_utils import Bed + +class OverlapRegion: + "Overlapping regions in a contig to fix" + def __init__(self): + self.regions = [] + self.best_region = None + + def add_region(self, bed_region): + "Add a new region to the overlapping set" + if self.best_region is None or \ + (bed_region.end - bed_region.start) > \ + (self.best_region.end - self.best_region.start): + self.best_region = bed_region + self.regions.append(bed_region) + assert bed_region.contig == self.best_region.contig + + @staticmethod + def are_overlapping(region1, region2): + "Returns True if the given regions are overlapping" + return region1.start <= region2.end and region2.start <= region1.end + + @staticmethod + def is_subsumed(region1, region2): + "Returns True is region 1 is subsumed in region2" + return region1.start >= region2.start and region1.end <= region2.end + + def find_non_overlapping(self): + "Given overlapping regions, resolve to remove overlaps" + return_regions = {} # Bed -> (replacement Bed) or None + if not self.regions or self.best_region is None: + return None + for bed_region in self.regions: + if bed_region == self.best_region: + return_regions[bed_region] = bed_region + elif self.is_subsumed(bed_region, self.best_region): + # Subsumed region in the best region + return_regions[bed_region] = None + elif self.are_overlapping(bed_region, self.best_region): + # Overlaps with best region, but isn't subsumed + if bed_region.start <= self.best_region.start: + new_region = Bed(contig=bed_region.contig, start=bed_region.start, + end=self.best_region.start - 1) + elif bed_region.end >= self.best_region.end: + new_region = Bed(contig=bed_region.contig, start=self.best_region.end + 1, + end=bed_region.end) + return_regions[bed_region] = new_region + else: + return_regions[bed_region] = bed_region + + # Double check if any still overlaps. If so, adjust smaller of the overlapping regions. + existing_overlaps = True + while existing_overlaps: + sorted_regions = sorted([(b, a) for b, a in return_regions.items() if a is not None], key=lambda x: x[1]) + i, j = 0, 1 + existing_overlaps = False + while j < len(sorted_regions): + region1_before, region2_before = sorted_regions[i][0], sorted_regions[j][0] + region1_after, region2_after = sorted_regions[i][1], sorted_regions[j][1] + if region1_after is None or region2_after is None: + i += 1 + j += 1 + continue + if self.are_overlapping(region1_after, region2_after): + existing_overlaps = True + if self.is_subsumed(region1_after, region2_after): + # Region 1 is subsumed in region 2 - Remove region 1 + return_regions[region1_before] = None + elif self.is_subsumed(region2_after, region1_after): + # Region 2 is subsumed in region 1 - Remove region 2 + return_regions[region2_before] = None + elif (region1_after.end - region1_after.start) > (region2_after.end - region2_after.start): + # Adjust region 2 start + return_regions[region2_before] = Bed(contig=region2_after.contig, start=region1_after.end + 1, + end=region2_after.end) + elif (region1_after.end - region1_after.start) <= (region2_after.end - region2_after.start): + # Adjust region 1 end + return_regions[region1_before] = Bed(contig=region1_after.contig, start=region1_after.start, + end=region2_after.start - 1) + else: + print("Unexpected case!") + print(region1_before, region2_before, region1_after, region1_after) + + i += 1 + j += 1 + + return return_regions diff --git a/bin/path_node.py b/bin/path_node.py new file mode 100644 index 0000000..23b26c1 --- /dev/null +++ b/bin/path_node.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +''' +Representing a ntJoin node in a scaffolding path +''' + +class OrientationError(ValueError): + "Orientation type error" + def __init__(self): + self.message = "Orientation must be + or -" + super().__init__(self.message) + +class PathNode: + "Defines a node in a path of contig regions" + def __init__(self, contig, ori, start, end, contig_size, + first_mx, terminal_mx, gap_size=0, raw_gap_size=0): + self.contig = contig + self.ori = ori + self.start = start + self.end = end + self.contig_size = contig_size + self.first_mx = first_mx + self.terminal_mx = terminal_mx + self.gap_size = gap_size + self.raw_gap_size = raw_gap_size + self.start_adjust = 0 + self.end_adjust = 0 # Adjust for trimming + + def set_gap_size(self, gap_size): + "Set the gap size of the path node" + self.gap_size = gap_size + + def set_raw_gap_size(self, raw_gap_size): + "Set the 'raw' gap size. Equal to gap_size if > min_gap_size" + self.raw_gap_size = raw_gap_size + + def get_aligned_length(self): + "Get the aligned length based on start/end coordinates" + return self.end - self.start + + def get_end_adjusted_coordinate(self): + "Return the adjusted end coordinate" + if self.end_adjust == 0: + return self.get_aligned_length() + return self.end_adjust + + def get_adjusted_start(self): + "Return the start coordinate of segment, adjusted for any trimming" + if self.ori == "+": + return self.start + self.start_adjust + if self.ori == "-": + return self.start + (self.get_aligned_length() - self.get_end_adjusted_coordinate()) + raise OrientationError() + + def get_adjusted_end(self): + "Return the end coordinate of the segment, adjusted for any trimming" + if self.ori == "+": + return self.end - (self.get_aligned_length() - self.get_end_adjusted_coordinate()) + if self.ori == "-": + return self.end - self.start_adjust + raise OrientationError() + + def __str__(self): + return f"Contig:{self.contig}\tOrientation:{self.ori}\tStart-End:{self.start}-{self.end}\t"\ + f"Length:{self.contig_size}\tFirstmx:{self.first_mx}\tLastmx:{self.terminal_mx}\t" \ + f"Adjusted_start-end:{self.start_adjust}-{self.end_adjust}" diff --git a/bin/synteny_block.py b/bin/synteny_block.py new file mode 100644 index 0000000..cf83b65 --- /dev/null +++ b/bin/synteny_block.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +''' +Represents an ntJoin synteny block +''' + +from assembly_block import AssemblyBlock +from ntjoin_utils import Minimizer + +class SyntenyBlock: + "A Synteny Block between the input assemblies" + def __init__(self, k, *assemblies): + "Instantiate a dictionary to keep track of assembly blocks for this synteny block" + self.assembly_blocks = {assembly: AssemblyBlock() for assembly in assemblies} + self.k = k # k-mer size used for minimizers, needed to adjust end coordinates + + + def continue_block(self, mx, list_mx_info): + "Given minimizer and preliminary blocks, return if synteny block should extend, else False" + return all(mx_dict[mx][0] == self.assembly_blocks[assembly].contig_id \ + for assembly, mx_dict in list_mx_info.items()) + + def extend_block(self, mx, list_mx_jnfo): + "Extend the synteny block by extending each assembly block" + for assembly, mx_dict in list_mx_jnfo.items(): + ctg, pos = mx_dict[mx] + assert self.assembly_blocks[assembly].contig_id == ctg + self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) + + def start_block(self, mx, list_mx_info): + "Start the new synteny block" + for assembly, mx_dict in list_mx_info.items(): + ctg, pos = mx_dict[mx] + self.assembly_blocks[assembly].contig_id = ctg + self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) + + def determine_orientations(self): + "Determine the orientations of each assembly block" + for _, assembly_block in self.assembly_blocks.items(): + positions = [mx.position for mx in assembly_block.minimizers] + if all(x < y for x, y in zip(positions, positions[1:])): + assembly_block.ori = "+" + elif all(x > y for x, y in zip(positions, positions[1:])): + assembly_block.ori = "-" + else: + assembly_block.ori = "?" + + def all_oriented(self): + "Return true if all of the assembly blocks in the synteny block are oriented" + return all(assembly_block.ori in ["+", "-"] for _, assembly_block in self.assembly_blocks.items()) + + def get_block_string(self, num): + "Given the specified synteny block ID, print the synteny blocks" + return_str = "" + for assembly, assembly_block in self.assembly_blocks.items(): + start_pos = assembly_block.get_block_start() + end_pos = assembly_block.get_block_end() + self.k + block_string = f"{num}\t{assembly}\t{assembly_block.contig_id}\t{start_pos}" \ + f"\t{end_pos}\t{assembly_block.ori}\n" + return_str += block_string + return return_str diff --git a/ntJoin b/ntJoin index 944dc65..9bbebe8 100755 --- a/ntJoin +++ b/ntJoin @@ -230,7 +230,7 @@ ifeq ($(overlap), True) endif $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa: $(target).k$(k).w$(w).tsv $(min_ref_targets) - $(log_time) $(assemble_path)/bin/ntjoin_assemble.py scaffold -p $(prefix) -n $(n) -s $< -l $(target_weight) \ + $(log_time) $(assemble_path)/bin/ntjoin_run.py scaffold -p $(prefix) -n $(n) -s $< -l $(target_weight) \ -r "$(reference_weights_list)" -k $(k) -g $(g) -G $(G) -t $(assemble_t) $(ntjoin_assembly_opts) $(min_ref_targets) $(target).k$(k).w$(w).n$(n).unassigned.scaffolds.fa: $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa @@ -257,4 +257,4 @@ endif # Synteny $(prefix).synteny_blocks.tsv: $(min_ref_targets) - $(assemble_path)/bin/ntjoin_assemble.py synteny $^ -k$(k) -w$(w) --w-rounds $(w_sizes) + $(assemble_path)/bin/ntjoin_run.py synteny $^ -k$(k) -w$(w) --w-rounds $(w_sizes) From 0f65a13c07219f93dbabc5ebac9a5678d9c95afe Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 15 Jun 2023 10:00:03 -0700 Subject: [PATCH 14/31] Refactoring ntJoin to allow better compatibility with synteny logic --- bin/assembly_block.py | 30 ----- bin/ntjoin_run.py | 22 +--- bin/ntjoin_synteny.py | 280 ------------------------------------------ bin/synteny_block.py | 60 --------- ntJoin | 9 +- 5 files changed, 3 insertions(+), 398 deletions(-) delete mode 100644 bin/assembly_block.py delete mode 100644 bin/ntjoin_synteny.py delete mode 100644 bin/synteny_block.py diff --git a/bin/assembly_block.py b/bin/assembly_block.py deleted file mode 100644 index 35ad987..0000000 --- a/bin/assembly_block.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 - -''' -Represents an assembly block for an assembly. -Multiple assembly blocks comprise a synteny block -''' - -class AssemblyBlock: - "An assembly block for a given assembly. The AssemblyBlock objects per assembly make up a SyntenyBlock" - def __init__(self): - "Instantiate the AssemblyBlock" - self.contig_id = None - self.minimizers = [] - self.ori = None - - def get_block_start(self): - "Get the starting coordinate of the assembly block" - return min(self.minimizers[0].position, self.minimizers[-1].position) - - def get_block_end(self): - "Get the end coordinate of the assembly block" - return max(self.minimizers[0].position, self.minimizers[-1].position) - - def get_block_terminal_mx(self): - "Return the terminal minimizer hashes for the assembly block" - return self.contig_id, self.minimizers[0], self.minimizers[-1] - - def get_block_internal_mx_hashes(self): - "Return the internal minimizer hashes for the assembly block" - return [mx_pos.mx for mx_pos in self.minimizers[1:-1]] diff --git a/bin/ntjoin_run.py b/bin/ntjoin_run.py index 9d34a35..5af1c8c 100755 --- a/bin/ntjoin_run.py +++ b/bin/ntjoin_run.py @@ -6,7 +6,6 @@ """ import argparse import sys -from ntjoin_synteny import NtjoinSynteny from ntjoin_assemble import NtjoinScaffolder def parse_arguments(): @@ -60,19 +59,6 @@ def parse_arguments(): "(computing minimizers, reading fasta file) [4]", type=int, default=4) - synteny_parser = subparsers.add_parser("synteny", help="Extract syntenic blocks from input assemblies") - synteny_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of input assemblies") - synteny_parser.add_argument("-n", help="Minimum edge weight [Number of input assemblies]", default=0, type=int) - synteny_parser.add_argument("-p", help="Output prefix [out]", - default="out", type=str, required=False) - synteny_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - synteny_parser.add_argument("-w", help="Window size used for minimizers", required=True, type=int) - synteny_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions "\ - "(computing minimizers, reading fasta file) [4]", type=int, default=4) - synteny_parser.add_argument("--w-rounds", help="decreasing list of 'w' values to use for refining ends", - default=[100, 10, 5], nargs="+", type=int) - synteny_parser.add_argument("--dev", action="store_true", help="Developer mode - retain intermediate files") - synteny_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: parser.print_help() @@ -83,12 +69,8 @@ def parse_arguments(): def main(): "Run ntJoin" args = parse_arguments() - if args.mode == "scaffold": - NtjoinScaffolder(args).main_scaffolder() - elif args.mode == "synteny": - NtjoinSynteny(args).main_synteny() - else: - raise ValueError(f"Unexpected mode: {args.mode}") + NtjoinScaffolder(args).main_scaffolder() + if __name__ == "__main__": main() diff --git a/bin/ntjoin_synteny.py b/bin/ntjoin_synteny.py deleted file mode 100644 index 33adf35..0000000 --- a/bin/ntjoin_synteny.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python3 -""" -ntJoin: Identifying synteny between genome assemblies using minimizer graphs -Written by Lauren Coombe @lcoombe -""" - -from collections import defaultdict -import datetime -import re -import shlex -import subprocess -import sys -import intervaltree -import ntjoin_utils -import ntjoin -import pybedtools -from synteny_block import SyntenyBlock - - -# Regexes -fai_re = re.compile(r'^(\S+).k\d+.w\d+.tsv') - -class NtjoinSynteny(ntjoin.Ntjoin): - "Instance for ntJoin synteny mode" - - def __init__(self, args): - super().__init__(args) - self.weights_list = [1] * len(self.args.FILES) - self.args.t = 1 - self.print_parameters_synteny() - - def print_parameters_synteny(self): - "Pring the set parameters for the ntJoin synteny run" - if self.args.n == 0: - self.args.n = len(self.args.FILES) - print("Running ntJoin synteny detection...") - print("Parameters:") - print("\tMinimizer TSV files: ", self.args.FILES) - print("\t-n", self.args.n) - print("\t-p", self.args.p) - print("\t-k", self.args.k) - print("\t-w", self.args.w) - print("\t--btllib_t", self.args.btllib_t) - print("\t--w-rounds", self.args.w_rounds) - - - def find_synteny_blocks(self, path): - "Given a path (sequence of mx), print the order/orientation/regions of contigs for an assembly" - out_blocks = [] # List of SyntenyBlock - prelim_blocks = SyntenyBlock(self.args.k, *list(self.list_mx_info.keys())) - past_start_flag = False - for mx in path: - if prelim_blocks.continue_block(mx, self.list_mx_info): - prelim_blocks.extend_block(mx, self.list_mx_info) - else: - # This is either the first mx, or we are past a stretch of repeating contigs - if past_start_flag: - prelim_blocks.determine_orientations() - if prelim_blocks.all_oriented(): - out_blocks.append(prelim_blocks) - prelim_blocks = SyntenyBlock(self.args.k, *list(self.list_mx_info.keys())) - prelim_blocks.start_block(mx, self.list_mx_info) - - prelim_blocks.determine_orientations() - if prelim_blocks.all_oriented(): - out_blocks.append(prelim_blocks) - - return out_blocks - - @staticmethod - def find_fa_name(assembly_mx_name): - "Given the mx file name, return the corresponding fai file name" - if fai_match := re.search(fai_re, assembly_mx_name): - return f"{fai_match.group(1)}" - print("ERROR: Target assembly minimizer TSV file must follow the naming convention:") - print("\ttarget_assembly.fa.k.w.tsv, where and are parameters used for minimizering") - sys.exit(1) - - @staticmethod - def get_synteny_bed_lists(paths): - "Given a set of synteny blocks, return a dictionary with a BED interval list per contig, per assembly" - synteny_beds = {} - for subcomponent in paths: - for block in subcomponent: - for assembly, assembly_block in block.assembly_blocks.items(): - if assembly not in synteny_beds: - synteny_beds[assembly] = {} - if assembly_block.contig_id not in synteny_beds[assembly]: - synteny_beds[assembly][assembly_block.contig_id] = [] - synteny_beds[assembly][assembly_block.contig_id].append( - ntjoin_utils.Bed(assembly_block.contig_id, - assembly_block.get_block_start(), - assembly_block.get_block_end() + block.k)) - - return synteny_beds - - def mask_assemblies_with_synteny_extents(self, synteny_beds, w): - "Mask each reference assembly with determined synteny blocks" - mx_to_fa_dict = {} - for assembly, contig_dict in synteny_beds.items(): - bed_str = [f"{ctg}\t{bed.start}\t{bed.end}\tSYNTENY" for ctg in contig_dict \ - for bed in contig_dict[ctg] if bed.end - bed.start > 2*w] - bed_str = "\n".join(bed_str) - fa_filename = self.find_fa_name(assembly) - synteny_bed = pybedtools.BedTool(bed_str, from_string=True).slop(g=f"{fa_filename}.fai", - l=-1*w, r=-1*w).sort() - synteny_bed.mask_fasta(fi=fa_filename, fo=f"{fa_filename}_masked.fa") - mx_to_fa_dict[assembly] = f"{fa_filename}_masked.fa" - return mx_to_fa_dict - - @staticmethod - def delete_w_iteration_files(*filenames): - "Delete the given files for the specific lower w iteration" - for filename in filenames: - cmd = shlex.split(f"rm {filename}") - ret_code = subprocess.call(cmd) - assert ret_code == 0 - - def generate_new_minimizers(self, tsv_to_fa_dict, w, retain_files=False): - "Given the masked fasta files, generate minimizers at new w for each" - list_mxs = {} - new_list_mxs_info = {} - for assembly_tsv, assembly_masked in tsv_to_fa_dict.items(): - indexlr_filename = ntjoin_utils.run_indexlr(assembly_masked, self.args.k, w, self.args.btllib_t) - mx_info, mxs_filt = ntjoin_utils.read_minimizers(indexlr_filename) - new_list_mxs_info[assembly_tsv] = mx_info - list_mxs[assembly_tsv] = mxs_filt - if not retain_files: - self.delete_w_iteration_files(indexlr_filename, assembly_masked) - return list_mxs, new_list_mxs_info - - @staticmethod - def update_interval_tree(trees, assembly_name, ctg, mx1, mx2): - "Update the given dictionary of trees with the new extent" - start_pos = min(mx1.position, mx2.position) - end_pos = max(mx1.position, mx2.position) - if assembly_name not in trees or ctg not in trees[assembly_name]: - trees[assembly_name][ctg] = intervaltree.IntervalTree() - if trees[assembly_name][ctg][start_pos+1:end_pos]: # Checking that this doesn't overlap with anything - print("WARNING: detected overlapping segments:", assembly_name, ctg, start_pos+1, end_pos, - file=sys.stderr) - trees[assembly_name][ctg][start_pos+1:end_pos] = (mx1, mx2) - - def find_mx_in_blocks(self, paths): - "Given the synteny blocks, find the minimizers at the terminal ends of each block, and internal mxs" - terminal_mxs = set() - internal_mxs = set() - intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents - - for subcomponent in paths: - for block in subcomponent: - curr_mx_len = len(terminal_mxs) - for assembly, assembly_block in block.assembly_blocks.items(): - contig, mx1, mx2 = assembly_block.get_block_terminal_mx() - terminal_mxs.add(mx1.mx) - terminal_mxs.add(mx2.mx) - self.update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) - internal = assembly_block.get_block_internal_mx_hashes() - internal_mxs = internal_mxs.union(internal) - assert len(terminal_mxs) == (curr_mx_len + 2) - return terminal_mxs, internal_mxs, intervaltrees - - def check_non_overlapping(self, paths): - "Given the paths, do final check to ensure intervals are not overlapping, will print warnings if so" - intervaltrees = defaultdict(dict) # assembly -> contig -> IntervalTree of synteny block extents - for subcomponent in paths: - for block in subcomponent: - for assembly, assembly_block in block.assembly_blocks.items(): - contig, mx1, mx2 = assembly_block.get_block_terminal_mx() - self.update_interval_tree(intervaltrees, assembly, contig, mx1, mx2) - - @staticmethod - def filter_minimizers_synteny_blocks(list_mxs, black_list, intervaltrees, list_mx_info): - "Filter minimizers found in the mx black list" - return_mxs = {} - for assembly in list_mxs: - assembly_mxs_filtered = [] - for mx_list in list_mxs[assembly]: - new_list = [] - for mx in mx_list: - ctg, pos = list_mx_info[assembly][mx] - i_tree = intervaltrees[assembly][ctg] - if new_list: - prev_pos = list_mx_info[assembly][new_list[-1]][1] - start = min(prev_pos, pos) - end = max(prev_pos, pos) - if i_tree[start:end]: # Split the mx adjacency if it spans over a known synteny block - assembly_mxs_filtered.append(new_list) - new_list = [] - if mx not in black_list and not i_tree[pos]: - new_list.append(mx) - assembly_mxs_filtered.append(new_list) - - return_mxs[assembly] = assembly_mxs_filtered - return return_mxs - - @staticmethod - def update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info): - "Update the directory containing mx -> contig, position associations" - valid_mxs = set({mx for _, list_mx_val in list_mxs.items() \ - for list_mx in list_mx_val for mx in list_mx}) - for assembly, mx_dict in new_list_mx_info.items(): - for mx in mx_dict: - if mx in valid_mxs and mx not in list_mx_info[assembly]: - list_mx_info[assembly][mx] = mx_dict[mx] - - def refine_block_coordinates(self, paths): - "Ready to start refining the synteny block coordinates" - prev_w = self.args.w - for new_w in self.args.w_rounds: - print(datetime.datetime.today(), ": Extending synteny blocks with w =", new_w, file=sys.stdout) - new_list_mxs, terminal_mxs = self.generate_additional_minimizers( - paths, new_w, prev_w, self.list_mx_info, self.args.dev) - graph = ntjoin_utils.build_graph(new_list_mxs, self.weights, graph=self.graph, black_list=terminal_mxs) - graph = self.filter_graph_global(graph) - paths = self.find_paths_synteny_blocks(self.find_paths()) - with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - prev_w = new_w - - print(datetime.datetime.today(), ": Done extended synteny blocks", file=sys.stdout) - with open(f"{self.args.p}.synteny_blocks.extended.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - - def generate_additional_minimizers(self, paths, new_w, prev_w, list_mx_info, dev=False): - "Given the existing synteny blocks, generate minimizers for increased block resolution" - synteny_beds = self.get_synteny_bed_lists(paths) - mx_to_fa_dict = self.mask_assemblies_with_synteny_extents(synteny_beds, prev_w) - list_mxs, new_list_mx_info = self.generate_new_minimizers(mx_to_fa_dict, new_w, retain_files=dev) - terminal_mx, internal_mx, interval_trees = self.find_mx_in_blocks(paths) - list_mxs = self.filter_minimizers_synteny_blocks(list_mxs, internal_mx, interval_trees, new_list_mx_info) - list_mxs = ntjoin_utils.filter_minimizers(list_mxs) # Filter for mx in all assemblies - self.update_list_mx_info(list_mxs, list_mx_info, new_list_mx_info) - return list_mxs, terminal_mx - - def find_paths_synteny_blocks(self, paths): - "Given a list of paths, return a list of representative synteny blocks" - return [self.find_synteny_blocks(blocks) for path in paths for blocks, _ in path] - - def load_minimizers(self): - "Read the minimizers for the synteny mode" - weights = {} # Dictionary: assembly -> weight - for assembly in self.args.FILES: - mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) - self.list_mx_info[assembly] = mxs_info - self.list_mxs[assembly] = mxs - weights[assembly] = self.weights_list.pop(0) - self.weights = weights - - def main_synteny(self): - "Run the steps for ntJoin synteny mode" - print("Running ntJoin synteny detection ...\n", file=sys.stdout) - - # Run the common ntJoin steps - self.load_minimizers() - - self.make_minimizer_graph_and_paths() - - paths = self.ntjoin_find_paths() - - paths = self.find_paths_synteny_blocks(paths) - - with open(f"{self.args.p}.synteny_blocks.tsv", 'w', encoding="utf-8") as outfile: - block_num = 0 - for subcomponent in paths: - for block in subcomponent: - outfile.write(block.get_block_string(block_num)) - block_num += 1 - print(datetime.datetime.today(), ": Done initial synteny blocks", file=sys.stdout) - self.refine_block_coordinates(paths) - - print(datetime.datetime.today(), ": DONE!", file=sys.stdout) diff --git a/bin/synteny_block.py b/bin/synteny_block.py deleted file mode 100644 index cf83b65..0000000 --- a/bin/synteny_block.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -''' -Represents an ntJoin synteny block -''' - -from assembly_block import AssemblyBlock -from ntjoin_utils import Minimizer - -class SyntenyBlock: - "A Synteny Block between the input assemblies" - def __init__(self, k, *assemblies): - "Instantiate a dictionary to keep track of assembly blocks for this synteny block" - self.assembly_blocks = {assembly: AssemblyBlock() for assembly in assemblies} - self.k = k # k-mer size used for minimizers, needed to adjust end coordinates - - - def continue_block(self, mx, list_mx_info): - "Given minimizer and preliminary blocks, return if synteny block should extend, else False" - return all(mx_dict[mx][0] == self.assembly_blocks[assembly].contig_id \ - for assembly, mx_dict in list_mx_info.items()) - - def extend_block(self, mx, list_mx_jnfo): - "Extend the synteny block by extending each assembly block" - for assembly, mx_dict in list_mx_jnfo.items(): - ctg, pos = mx_dict[mx] - assert self.assembly_blocks[assembly].contig_id == ctg - self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) - - def start_block(self, mx, list_mx_info): - "Start the new synteny block" - for assembly, mx_dict in list_mx_info.items(): - ctg, pos = mx_dict[mx] - self.assembly_blocks[assembly].contig_id = ctg - self.assembly_blocks[assembly].minimizers.append(Minimizer(mx, int(pos))) - - def determine_orientations(self): - "Determine the orientations of each assembly block" - for _, assembly_block in self.assembly_blocks.items(): - positions = [mx.position for mx in assembly_block.minimizers] - if all(x < y for x, y in zip(positions, positions[1:])): - assembly_block.ori = "+" - elif all(x > y for x, y in zip(positions, positions[1:])): - assembly_block.ori = "-" - else: - assembly_block.ori = "?" - - def all_oriented(self): - "Return true if all of the assembly blocks in the synteny block are oriented" - return all(assembly_block.ori in ["+", "-"] for _, assembly_block in self.assembly_blocks.items()) - - def get_block_string(self, num): - "Given the specified synteny block ID, print the synteny blocks" - return_str = "" - for assembly, assembly_block in self.assembly_blocks.items(): - start_pos = assembly_block.get_block_start() - end_pos = assembly_block.get_block_end() + self.k - block_string = f"{num}\t{assembly}\t{assembly_block.contig_id}\t{start_pos}" \ - f"\t{end_pos}\t{assembly_block.ori}\n" - return_str += block_string - return return_str diff --git a/ntJoin b/ntJoin index 9bbebe8..feedcbf 100755 --- a/ntJoin +++ b/ntJoin @@ -22,8 +22,6 @@ references_list=$(references) reference_weights_list=$(reference_weights) endif -# synteny - w block extensions -w_sizes=100 10 5 min_ref_targets=$(addsuffix .k$(k).w$(w).tsv, $(references_list)) fai_ref_targets=$(addsuffix .fai, $(references_list)) @@ -164,8 +162,6 @@ analysis: check_params_analysis \ all: check_params assemble analysis -synteny: $(fai_ref_targets) \ - $(prefix).synteny_blocks.tsv check_params: ifeq ($(references), None) @@ -201,7 +197,7 @@ ifneq ($(shell command -v 'python3 -c "import btllib"'),) $(error ERROR: btllib not found. Please ensure btllib is installed) endif -.PHONY: help all version analysis assemble check_params jupiter check_install synteny +.PHONY: help all version analysis assemble check_params jupiter check_install .DELETE_ON_ERROR: $(prefix).n$(n).mx.dot .SECONDARY: @@ -255,6 +251,3 @@ else $^ endif -# Synteny -$(prefix).synteny_blocks.tsv: $(min_ref_targets) - $(assemble_path)/bin/ntjoin_run.py synteny $^ -k$(k) -w$(w) --w-rounds $(w_sizes) From 5c00a596cde6189cf11f41de9a4677a89d4a90e9 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 15 Jun 2023 16:28:32 -0700 Subject: [PATCH 15/31] Adding helper functiosn to ntjoin_utils.py, flush stdout, add load_minimizers --- bin/ntjoin.py | 32 +++++++++++++-------- bin/ntjoin_utils.py | 68 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/bin/ntjoin.py b/bin/ntjoin.py index b122763..a2066b3 100644 --- a/bin/ntjoin.py +++ b/bin/ntjoin.py @@ -39,8 +39,8 @@ def print_graph(self, graph, out_prefix=None): colours = ["red"]*len(list_files) for node in graph.vs(): - mx_ctg_pos_labels = "\n".join([str(asm_mx_info[node['name']]) - for _, asm_mx_info in self.list_mx_info.items()]) + mx_ctg_pos_labels = "\n".join([f"{file_name}_{asm_mx_info[node['name']]}" + for file_name, asm_mx_info in self.list_mx_info.items()]) node_label = f"\"{node['name']}\" [label=\"{node['name']}\n{mx_ctg_pos_labels}\"]" outfile.write(f"{node_label}\n") @@ -62,7 +62,7 @@ def print_graph(self, graph, out_prefix=None): print("\nfile_name\tnumber\tcolour") for i, filename in enumerate(list_files): print(filename, i, colours[i], sep="\t") - print("") + print("", flush=True) @staticmethod @@ -77,7 +77,7 @@ def filter_graph(graph, min_weight): def filter_graph_global(self, graph): "Filter the graph globally based on minimum edge weight" - print(datetime.datetime.today(), ": Filtering the graph", file=sys.stdout) + print(datetime.datetime.today(), ": Filtering the graph", file=sys.stdout, flush=True) if self.args.n <= min(self.weights.values()): return graph to_remove_edges = [edge.index for edge in graph.es() @@ -93,8 +93,8 @@ def determine_source_vertex(self, sources, graph): max_wt_asm = [assembly for assembly, asm_weight in self.weights.items() if asm_weight == max(self.weights.values())].pop() list_mx_info_maxwt = self.list_mx_info[max_wt_asm] - min_pos = min([list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources]) - max_pos = max([list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources]) + min_pos = min((list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources)) + max_pos = max((list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] for s in sources)) source = [s for s in sources if list_mx_info_maxwt[ntjoin_utils.vertex_name(graph, s)][1] == min_pos].pop() target = [s for s in sources @@ -140,7 +140,7 @@ def find_paths(self): "Finds paths through the minimizer graph" print(datetime.datetime.today(), ": Finding paths", file=sys.stdout) components = self.graph.components() - print("\nTotal number of components in graph:", len(components), "\n", sep=" ", file=sys.stdout) + print("\nTotal number of components in graph:", len(components), "\n", sep=" ", file=sys.stdout, flush=True) if self.args.t == 1: paths = [self.find_paths_process(component) for component in components] @@ -150,14 +150,24 @@ def find_paths(self): return paths + def load_minimizers(self, repeat_bf=False): + "Load in minimizers for ntJoin scaffolding mode" + weights = {} # Dictionary: assembly -> weight + for assembly in self.args.FILES: + mxs_info, mxs = ntjoin_utils.read_minimizers(assembly, repeat_bf) + self.list_mx_info[assembly] = mxs_info + self.list_mxs[assembly] = mxs + weights[assembly] = self.weights_list.pop(0) + self.weights = weights - def make_minimizer_graph_and_paths(self): + + def make_minimizer_graph(self): "Run ntJoin graph stage" print(datetime.datetime.today(), ": Generating ntJoin minimizer graph ...\n") # Print the weights of the input assemblies weight_str = "\n".join([f"{assembly}: {asm_weight}" for assembly, asm_weight in self.weights.items()]) - print("\nWeights of assemblies:\n", weight_str, "\n", sep="") + print("\nWeights of assemblies:\n", weight_str, "\n", sep="", flush=True) # Filter minimizers - Keep only if found in all assemblies list_mxs = ntjoin_utils.filter_minimizers(self.list_mxs) @@ -168,9 +178,6 @@ def make_minimizer_graph_and_paths(self): # Print the DOT graph self.print_graph(self.graph) - # Filter the graph edges by minimum weight - self.graph = self.filter_graph_global(self.graph) - def ntjoin_find_paths(self): "Find the paths through the graph" @@ -184,3 +191,4 @@ def __init__(self, args): self.graph = None self.args = args self.weights = {} + self.weights_list = [] diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 2e63e08..a17289f 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -10,7 +10,10 @@ import subprocess import sys import os +import re +import math import igraph as ig +import btllib # Defining namedtuples @@ -53,7 +56,20 @@ def set_edge_attributes(graph, edge_attributes): def calc_total_weight(list_files, weights): "Calculate the total weight of an edge given the assembly support" - return sum([weights[f] for f in list_files]) + return sum((weights[f] for f in list_files)) + +def remove_flagged_edges(graph, remove_edges): + "Remove the listed edges from the given graph" + new_graph = graph.copy() + new_graph.delete_edges(remove_edges) + return new_graph + +def check_total_degree_vertex(vertex_id, graph, num_assemblies): + "Return the total weights of incident edges for the given vertex" + total_weight = sum((e["weight"] for e in graph.es()[graph.incident(vertex_id)])) + assert total_weight % num_assemblies == 0 + return total_weight + def build_graph(list_mxs, weights, graph=None, black_list=None): "Builds an undirected graph: nodes=minimizers; edges=between adjacent minimizers" @@ -90,16 +106,27 @@ def build_graph(list_mxs, weights, graph=None, black_list=None): formatted_edges = [(s, t) for s in edges for t in edges[s]] print(datetime.datetime.today(), ": Adding vertices", file=sys.stdout) + if prev_edge_attributes: + existing_vertices = {vertex['name'] for vertex in graph.vs()} + vertices = {vertex for vertex in vertices if vertex not in existing_vertices} graph.add_vertices(list(vertices)) print(datetime.datetime.today(), ": Adding edges", file=sys.stdout) + if prev_edge_attributes: + existing_edges = {(vertex_name(graph, edge.source), vertex_name(graph, edge.target)) + for edge in graph.es()} + max_expected_incident_weights = len(list_mxs)*2 + formatted_edges = [(s, t) for s, t in formatted_edges + if (s, t) not in existing_edges and (t, s) not in existing_edges and + check_total_degree_vertex(s, graph, len(list_mxs)) < max_expected_incident_weights and + check_total_degree_vertex(t, graph, len(list_mxs)) < max_expected_incident_weights] graph.add_edges(formatted_edges) print(datetime.datetime.today(), ": Adding attributes", file=sys.stdout) edge_attributes = {edge_index(graph, s, t): {"support": edges[s][t], "weight": calc_total_weight(edges[s][t], weights)} - for s in edges for t in edges[s]} + for s, t in formatted_edges} edge_attributes.update(prev_edge_attributes) set_edge_attributes(graph, edge_attributes) @@ -129,7 +156,7 @@ def filter_minimizers(list_mxs): return return_mxs -def read_minimizers(tsv_filename): +def read_minimizers(tsv_filename, repeat_bf=False): "Read the minimizers from a file, removing duplicate minimizers" print(datetime.datetime.today(), ": Reading minimizers", tsv_filename, file=sys.stdout) mx_info = {} # mx -> (contig, position) @@ -142,23 +169,50 @@ def read_minimizers(tsv_filename): mx_pos_split = line[1].split(" ") mxs.append([mx_pos.split(":")[0] for mx_pos in mx_pos_split]) for mx_pos in mx_pos_split: - mx, pos = mx_pos.split(":") - if mx in mx_info: # This is a duplicate, add to dup set, don't add to dict + mx, pos, seq = mx_pos.split(":") + if mx in mx_info or (repeat_bf and repeat_bf.contains(seq)): # Duplicate, add to dup set dup_mxs.add(mx) else: mx_info[mx] = (line[0], int(pos)) mx_info = {mx: mx_entry_info for mx, mx_entry_info in mx_info.items() if mx not in dup_mxs} + mxs_filt = [] for mx_list in mxs: mx_list_filt = [mx for mx in mx_list if mx not in dup_mxs] mxs_filt.append(mx_list_filt) return mx_info, mxs_filt -def run_indexlr(assembly, k, w, t): +def run_indexlr(assembly, k, w, t, **kwargs): "Run indexlr on the given assembly with the specified k and w" - cmd = f"indexlr {assembly} --long --pos -k{k} -w{w} -t{t} -o {assembly}.k{k}.w{w}.tsv" + extra_args = " ".join([f"-{key} {val}" for key, val in kwargs.items()]) + cmd = f"indexlr {assembly} --seq --long --pos -k{k} -w{w} -t{t} {extra_args} -o {assembly}.k{k}.w{w}.tsv" cmd = shlex.split(cmd) ret_code = subprocess.call(cmd) assert ret_code == 0 return f"{assembly}.k{k}.w{w}.tsv" + +def parse_bf_size(bf_size_str, parser): + "Parse the BF size from the given string" + str_match = re.search(r'^(\d+)([BkMG])$', bf_size_str) + if not str_match: + parser.print_help() + parser.error(f"Invalid input value for --bf: {bf_size_str}") + num, units = int(str_match.group(1)), str_match.group(2) + if units == "B": + return num + if units == "k": + return int(num*1e3) + if units == "M": + return int(num*1e6) + return int(num*1e9) + +def approximate_bf_size(genome_file, fpr, threads): + "Approximate the BF size to use based on the genome size and provided FPR" + genome_size = 0 + with btllib.SeqReader(genome_file, btllib.SeqReaderFlag.LONG_MODE, threads) as reader: + for record in reader: + genome_size += len(record.seq) + size_bits = math.ceil((-1*genome_size) / (math.log(1 - fpr))) + print(f"Calculated Bloom filter size: {int(size_bits/8)} bytes") + return int(size_bits/8) From 10961764706cb93bf0e1be114ad1286907485f47 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Thu, 15 Jun 2023 16:33:46 -0700 Subject: [PATCH 16/31] Updates to ntjoin_assemble.py and ntJoin to ensure compatibility with ntSynt --- bin/ntjoin_assemble.py | 35 ++++++++++++++++------------------- ntJoin | 2 +- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/bin/ntjoin_assemble.py b/bin/ntjoin_assemble.py index 3109f6c..97fa414 100755 --- a/bin/ntjoin_assemble.py +++ b/bin/ntjoin_assemble.py @@ -13,13 +13,13 @@ import warnings import pybedtools import pymannkendall as mk +import btllib from read_fasta import read_fasta import ntjoin_utils import ntjoin_overlap import ntjoin -import path_node +from path_node import PathNode from overlap_region import OverlapRegion -import btllib warnings.simplefilter(action='ignore', category=RuntimeWarning) @@ -185,7 +185,7 @@ def format_path(self, path, assembly, component_graph): if curr_ctg is not None: ori = self.determine_orientation(positions) if ori != "?": # Don't add to path if orientation couldn't be determined - out_path.append(path_node.PathNode(contig=curr_ctg, ori=ori, + out_path.append(PathNode(contig=curr_ctg, ori=ori, start=self.calc_start_coord(positions, self.mx_extremes[curr_ctg][0]), end=self.calc_end_coord(positions, @@ -200,7 +200,7 @@ def format_path(self, path, assembly, component_graph): prev_mx = mx ori = self.determine_orientation(positions) if ori != "?": - out_path.append(path_node.PathNode(contig=curr_ctg, ori=ori, + out_path.append(PathNode(contig=curr_ctg, ori=ori, start=self.calc_start_coord(positions, self.mx_extremes[curr_ctg][0]), end=self.calc_end_coord(positions, @@ -414,10 +414,10 @@ def join_sequences(sequences_list, path, path_segments): node.start == path_segments[0].start and \ node.end == path_segments[0].end: if node.ori == "+": - path[i].start += len_diff + node.start += len_diff else: - path[i].end -= len_diff - assert len(sequence_start_strip) - path[i].gap_size == path[i].end - path[i].start + node.end -= len_diff + assert len(sequence_start_strip) - node.gap_size == node.end - node.start break sequence_end_strip = sequences_list[-1].rstrip("Nn") # Strip from 3' @@ -743,12 +743,13 @@ def print_parameters_scaffold(self): def main_scaffolder(self): "Run ntJoin scaffolding stage" - print("Running ntJoin scaffolding ...\n") - self.load_minimizers() + self.load_minimizers_scaffold() # Generate minimizer graph, and get paths through the graph - paths = self.make_minimizer_graph_and_paths() + self.make_minimizer_graph() + + self.graph = self.filter_graph_global(self.graph) self.mx_extremes = self.find_mx_min_max(self.args.s) @@ -789,19 +790,15 @@ def set_weights(self): sys.exit(1) return weights - def load_minimizers(self): + def load_minimizers_scaffold(self): "Load in minimizers for ntJoin scaffolding mode" - weights = {} # Dictionary: assembly -> weight - for assembly in self.args.FILES: - mxs_info, mxs = ntjoin_utils.read_minimizers(assembly) - self.list_mx_info[assembly] = mxs_info - self.list_mxs[assembly] = mxs - weights[assembly] = self.weights_list.pop(0) + # Load in minimizers for references + self.load_minimizers() + # Now, add minimizers for reference mxs_info, mxs = ntjoin_utils.read_minimizers(self.args.s) self.list_mx_info[self.args.s] = mxs_info self.list_mxs[self.args.s] = mxs - weights[self.args.s] = self.args.l - self.weights = weights + self.weights[self.args.s] = self.args.l def __init__(self, args): "Create an ntJoin instance for scaffolding" diff --git a/ntJoin b/ntJoin index feedcbf..7bddaac 100755 --- a/ntJoin +++ b/ntJoin @@ -202,7 +202,7 @@ endif .SECONDARY: %.k$(k).w$(w).tsv: % - $(log_time) indexlr --long --pos -k $(k) -w $(w) -t $(t) $< > $@ + $(log_time) indexlr --seq --long --pos -k $(k) -w $(w) -t $(t) $< > $@ %.fai: % $(log_time) samtools faidx $< From 34a0cdcdc422ca4c37ee3b8fac5ad99042572ad9 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 21 Jun 2023 11:14:48 -0700 Subject: [PATCH 17/31] Remove synteny-specific code --- bin/ntjoin_utils.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index a17289f..b36f24f 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -191,28 +191,3 @@ def run_indexlr(assembly, k, w, t, **kwargs): ret_code = subprocess.call(cmd) assert ret_code == 0 return f"{assembly}.k{k}.w{w}.tsv" - -def parse_bf_size(bf_size_str, parser): - "Parse the BF size from the given string" - str_match = re.search(r'^(\d+)([BkMG])$', bf_size_str) - if not str_match: - parser.print_help() - parser.error(f"Invalid input value for --bf: {bf_size_str}") - num, units = int(str_match.group(1)), str_match.group(2) - if units == "B": - return num - if units == "k": - return int(num*1e3) - if units == "M": - return int(num*1e6) - return int(num*1e9) - -def approximate_bf_size(genome_file, fpr, threads): - "Approximate the BF size to use based on the genome size and provided FPR" - genome_size = 0 - with btllib.SeqReader(genome_file, btllib.SeqReaderFlag.LONG_MODE, threads) as reader: - for record in reader: - genome_size += len(record.seq) - size_bits = math.ceil((-1*genome_size) / (math.log(1 - fpr))) - print(f"Calculated Bloom filter size: {int(size_bits/8)} bytes") - return int(size_bits/8) From e4b75a390fb00df295560ae544a88f0d6bfa8897 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Wed, 21 Jun 2023 11:16:25 -0700 Subject: [PATCH 18/31] Fixes for pylint --- bin/ntjoin_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index b36f24f..1642ff1 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -10,10 +10,7 @@ import subprocess import sys import os -import re -import math import igraph as ig -import btllib # Defining namedtuples From 04ad9960c76163bd63ac4be6e8131c66a1d09153 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 11:41:55 -0700 Subject: [PATCH 19/31] Update to dealing with building graph with additional minimizers * Saw more issues where nodes in the graph had unexpectedly large incident edge weights * Now, do the filter afterward initial addition (to avoid case where added edges together cause the issue) --- bin/ntjoin_utils.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 1642ff1..2e50473 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -68,6 +68,19 @@ def check_total_degree_vertex(vertex_id, graph, num_assemblies): return total_weight +def check_added_edges_incident_weights(graph, edges, num_assemblies): + "Checks the added edges in the graph, filtering any that have too many incident edges, if needed" + max_expected_edges = num_assemblies*2 + flagged_edges = [] + for s, t in edges: + if check_total_degree_vertex(s, graph, num_assemblies) > max_expected_edges or \ + check_total_degree_vertex(t, graph, num_assemblies) > max_expected_edges: + flagged_edges.append(edge_index(graph, s, t)) + if flagged_edges: + return remove_flagged_edges(graph, flagged_edges) + return graph + + def build_graph(list_mxs, weights, graph=None, black_list=None): "Builds an undirected graph: nodes=minimizers; edges=between adjacent minimizers" print(datetime.datetime.today(), ": Building graph", file=sys.stdout) @@ -112,11 +125,8 @@ def build_graph(list_mxs, weights, graph=None, black_list=None): if prev_edge_attributes: existing_edges = {(vertex_name(graph, edge.source), vertex_name(graph, edge.target)) for edge in graph.es()} - max_expected_incident_weights = len(list_mxs)*2 formatted_edges = [(s, t) for s, t in formatted_edges - if (s, t) not in existing_edges and (t, s) not in existing_edges and - check_total_degree_vertex(s, graph, len(list_mxs)) < max_expected_incident_weights and - check_total_degree_vertex(t, graph, len(list_mxs)) < max_expected_incident_weights] + if (s, t) not in existing_edges and (t, s) not in existing_edges] graph.add_edges(formatted_edges) print(datetime.datetime.today(), ": Adding attributes", file=sys.stdout) @@ -127,6 +137,9 @@ def build_graph(list_mxs, weights, graph=None, black_list=None): edge_attributes.update(prev_edge_attributes) set_edge_attributes(graph, edge_attributes) + if prev_edge_attributes: + graph = check_added_edges_incident_weights(graph, formatted_edges, len(list_mxs)) + return graph # Other helper functions From cdd224ea8d21805a9f2a3eb5d36a8963ffafef6e Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 13:35:48 -0700 Subject: [PATCH 20/31] Change location of assert statements, check for edges inducing incident weights unbalanced --- bin/ntjoin_utils.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 2e50473..5696b06 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -61,20 +61,30 @@ def remove_flagged_edges(graph, remove_edges): new_graph.delete_edges(remove_edges) return new_graph -def check_total_degree_vertex(vertex_id, graph, num_assemblies): +def check_total_degree_vertex(vertex_id, graph): "Return the total weights of incident edges for the given vertex" total_weight = sum((e["weight"] for e in graph.es()[graph.incident(vertex_id)])) - assert total_weight % num_assemblies == 0 return total_weight +def check_graph_incident_weights(graph, num_assemblies): + "Check that the incident edges per vertex are as expected" + for node_idx in graph.vs(): + sum_incident_edges = check_total_degree_vertex(node_idx, graph) + assert sum_incident_edges <= 2*num_assemblies + assert sum_incident_edges % num_assemblies == 0 + def check_added_edges_incident_weights(graph, edges, num_assemblies): "Checks the added edges in the graph, filtering any that have too many incident edges, if needed" max_expected_edges = num_assemblies*2 flagged_edges = [] for s, t in edges: - if check_total_degree_vertex(s, graph, num_assemblies) > max_expected_edges or \ - check_total_degree_vertex(t, graph, num_assemblies) > max_expected_edges: + source_sum_edges = check_total_degree_vertex(s, graph) + target_sum_edges = check_total_degree_vertex(t, graph) + if source_sum_edges > max_expected_edges or \ + target_sum_edges > max_expected_edges or \ + source_sum_edges % num_assemblies != 0 or \ + target_sum_edges % num_assemblies != 0: flagged_edges.append(edge_index(graph, s, t)) if flagged_edges: return remove_flagged_edges(graph, flagged_edges) @@ -139,7 +149,7 @@ def build_graph(list_mxs, weights, graph=None, black_list=None): if prev_edge_attributes: graph = check_added_edges_incident_weights(graph, formatted_edges, len(list_mxs)) - + check_graph_incident_weights(graph, len(list_mxs)) return graph # Other helper functions @@ -176,6 +186,7 @@ def read_minimizers(tsv_filename, repeat_bf=False): for line in tsv: line = line.strip().split("\t") if len(line) > 1: + contig = line[0] mx_pos_split = line[1].split(" ") mxs.append([mx_pos.split(":")[0] for mx_pos in mx_pos_split]) for mx_pos in mx_pos_split: @@ -183,7 +194,7 @@ def read_minimizers(tsv_filename, repeat_bf=False): if mx in mx_info or (repeat_bf and repeat_bf.contains(seq)): # Duplicate, add to dup set dup_mxs.add(mx) else: - mx_info[mx] = (line[0], int(pos)) + mx_info[mx] = (contig, int(pos)) mx_info = {mx: mx_entry_info for mx, mx_entry_info in mx_info.items() if mx not in dup_mxs} From 83eba11fdac5a3f59a53b0545f06ccd991c2b44b Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 14:02:15 -0700 Subject: [PATCH 21/31] Remove sum edge weight modulus num_assemblies check --- bin/ntjoin_utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 5696b06..3cb9904 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -70,8 +70,10 @@ def check_graph_incident_weights(graph, num_assemblies): "Check that the incident edges per vertex are as expected" for node_idx in graph.vs(): sum_incident_edges = check_total_degree_vertex(node_idx, graph) - assert sum_incident_edges <= 2*num_assemblies - assert sum_incident_edges % num_assemblies == 0 + try: + assert sum_incident_edges <= 2*num_assemblies + except AssertionError: + print(vertex_name(graph, node_idx)) def check_added_edges_incident_weights(graph, edges, num_assemblies): @@ -79,12 +81,8 @@ def check_added_edges_incident_weights(graph, edges, num_assemblies): max_expected_edges = num_assemblies*2 flagged_edges = [] for s, t in edges: - source_sum_edges = check_total_degree_vertex(s, graph) - target_sum_edges = check_total_degree_vertex(t, graph) - if source_sum_edges > max_expected_edges or \ - target_sum_edges > max_expected_edges or \ - source_sum_edges % num_assemblies != 0 or \ - target_sum_edges % num_assemblies != 0: + if check_total_degree_vertex(s, graph) > max_expected_edges or \ + check_total_degree_vertex(t, graph) > max_expected_edges: flagged_edges.append(edge_index(graph, s, t)) if flagged_edges: return remove_flagged_edges(graph, flagged_edges) From 663b59ebda705ff5e8b10dc98bc0d415981c702b Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 14:17:32 -0700 Subject: [PATCH 22/31] Bugfix for incident edge check --- bin/ntjoin_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 3cb9904..25c67bf 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -69,7 +69,7 @@ def check_total_degree_vertex(vertex_id, graph): def check_graph_incident_weights(graph, num_assemblies): "Check that the incident edges per vertex are as expected" for node_idx in graph.vs(): - sum_incident_edges = check_total_degree_vertex(node_idx, graph) + sum_incident_edges = check_total_degree_vertex(node_idx.index, graph) try: assert sum_incident_edges <= 2*num_assemblies except AssertionError: From 685a8946a685bb00865554e7b7a66d1375b8641c Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 14:20:53 -0700 Subject: [PATCH 23/31] Better log message --- bin/ntjoin_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 25c67bf..5c4f42b 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -73,7 +73,7 @@ def check_graph_incident_weights(graph, num_assemblies): try: assert sum_incident_edges <= 2*num_assemblies except AssertionError: - print(vertex_name(graph, node_idx)) + print("Edge weights incident to", vertex_name(graph, node_idx), "higher than expected") def check_added_edges_incident_weights(graph, edges, num_assemblies): From f3ebd65b4bcde3f6b826de90842d5b3307fa3469 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 14:28:36 -0700 Subject: [PATCH 24/31] Fix handling of assertion error --- bin/ntjoin_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 5c4f42b..a7d5969 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -73,7 +73,7 @@ def check_graph_incident_weights(graph, num_assemblies): try: assert sum_incident_edges <= 2*num_assemblies except AssertionError: - print("Edge weights incident to", vertex_name(graph, node_idx), "higher than expected") + print("Edge weights incident to", vertex_name(graph, node_idx.index), "higher than expected") def check_added_edges_incident_weights(graph, edges, num_assemblies): From c9465135fe42d39b0a5f5b0e63db3b66c45dc127 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 14:33:56 -0700 Subject: [PATCH 25/31] Remove extra check, use weights for max_expected_incident_weight --- bin/ntjoin_utils.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index a7d5969..621f873 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -66,19 +66,10 @@ def check_total_degree_vertex(vertex_id, graph): total_weight = sum((e["weight"] for e in graph.es()[graph.incident(vertex_id)])) return total_weight -def check_graph_incident_weights(graph, num_assemblies): - "Check that the incident edges per vertex are as expected" - for node_idx in graph.vs(): - sum_incident_edges = check_total_degree_vertex(node_idx.index, graph) - try: - assert sum_incident_edges <= 2*num_assemblies - except AssertionError: - print("Edge weights incident to", vertex_name(graph, node_idx.index), "higher than expected") - -def check_added_edges_incident_weights(graph, edges, num_assemblies): +def check_added_edges_incident_weights(graph, edges, weights): "Checks the added edges in the graph, filtering any that have too many incident edges, if needed" - max_expected_edges = num_assemblies*2 + max_expected_edges = sum(weights.keys())*2 flagged_edges = [] for s, t in edges: if check_total_degree_vertex(s, graph) > max_expected_edges or \ @@ -146,8 +137,7 @@ def build_graph(list_mxs, weights, graph=None, black_list=None): set_edge_attributes(graph, edge_attributes) if prev_edge_attributes: - graph = check_added_edges_incident_weights(graph, formatted_edges, len(list_mxs)) - check_graph_incident_weights(graph, len(list_mxs)) + graph = check_added_edges_incident_weights(graph, formatted_edges, weights) return graph # Other helper functions From 4b201ad0bed5a105d0351bb2854cb58b09eebff8 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Tue, 10 Oct 2023 15:08:58 -0700 Subject: [PATCH 26/31] Fix weight summation --- bin/ntjoin_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ntjoin_utils.py b/bin/ntjoin_utils.py index 621f873..04bf028 100644 --- a/bin/ntjoin_utils.py +++ b/bin/ntjoin_utils.py @@ -69,7 +69,7 @@ def check_total_degree_vertex(vertex_id, graph): def check_added_edges_incident_weights(graph, edges, weights): "Checks the added edges in the graph, filtering any that have too many incident edges, if needed" - max_expected_edges = sum(weights.keys())*2 + max_expected_edges = sum(weights.values())*2 flagged_edges = [] for s, t in edges: if check_total_degree_vertex(s, graph) > max_expected_edges or \ From c23b2677946247f009127a6989d5f52cc0d2bc5f Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 27 Oct 2023 16:07:29 -0700 Subject: [PATCH 27/31] Remove subparser --- bin/ntjoin_run.py | 55 +++++++++++++++++++++-------------------------- ntJoin | 2 +- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/bin/ntjoin_run.py b/bin/ntjoin_run.py index 5af1c8c..9ef6b28 100755 --- a/bin/ntjoin_run.py +++ b/bin/ntjoin_run.py @@ -11,53 +11,48 @@ def parse_arguments(): "Parse ntJoin arguments" parser = argparse.ArgumentParser( - description="ntJoin: Genome analysis using reference assemblies and minimizer graphs", - ) - parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - - subparsers = parser.add_subparsers(dest="mode") - scaffold_parser = subparsers.add_parser("scaffold", - help="Scaffold the input target assembly using the supplied reference(s)", - epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" - "Example: myscaffolds.fa.k32.w1000.tsv - myscaffolds.fa is the expected matching fasta", - formatter_class=argparse.RawTextHelpFormatter) - scaffold_parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") - scaffold_parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) - scaffold_parser.add_argument("-l", help="Weight of target genome assembly [1]", + description="ntJoin: Scaffolding genome assemblies using reference assemblies and minimizer graphs", + epilog="Note: Script expects that each input minimizer TSV file has a matching fasta file.\n" + "Example: myscaffolds.fa.k32.w1000.tsv - myscaffolds.fa is the expected matching fasta", + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("FILES", nargs="+", help="Minimizer TSV files of references") + parser.add_argument("-s", help="Target scaffolds minimizer TSV file", required=True) + parser.add_argument("-l", help="Weight of target genome assembly [1]", required=False, default=1, type=float) - scaffold_parser.add_argument("-r", + parser.add_argument("-r", help="List of reference assembly weights (in quotes, separated by spaces, " "in same order as minimizer TSV files)", required=True, type=str) - scaffold_parser.add_argument("-p", help="Output prefix [out]", default="out", + parser.add_argument("-p", help="Output prefix [out]", default="out", type=str, required=False) - scaffold_parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) - scaffold_parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) - scaffold_parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) - scaffold_parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, + parser.add_argument("-n", help="Minimum edge weight [1]", default=1, type=int) + parser.add_argument("-k", help="Kmer size used for minimizer step", required=True, type=int) + parser.add_argument("-g", help="Minimum gap size (bp)", required=False, default=20, type=int) + parser.add_argument("-G", help="Maximum gap size (bp) (0 if no maximum threshold)", required=False, default=0, type=int) - scaffold_parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", + parser.add_argument("--mkt", help="Use Mann-Kendall Test to orient contigs (slower, overrides m)", action='store_true') - scaffold_parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " + parser.add_argument('-m', help="Require at least m %% of minimizer positions to be " "increasing/decreasing to assign contig orientation [90]\n " "Note: Only used with --mkt is NOT specified", default=90, type=int) - scaffold_parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) - scaffold_parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') - scaffold_parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") - scaffold_parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", + parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) + parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") + parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", action="store_true") - scaffold_parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", + parser.add_argument("--overlap", help="Attempt to detect and trim overlapping joined sequences", action="store_true") - scaffold_parser.add_argument("--overlap_gap", + parser.add_argument("--overlap_gap", help="Length of gap introduced between overlapping, trimmed segments [20]", type=int, default=20) - scaffold_parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", + parser.add_argument("--overlap_k", help="Kmer size used for overlap minimizer step [15]", type=int, default=15) - scaffold_parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", + parser.add_argument("--overlap_w", help="Window size used for overlap minimizer step [10]", type=int, default=10) - scaffold_parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " + parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " "(computing minimizers, reading fasta file) [4]", type=int, default=4) + parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: diff --git a/ntJoin b/ntJoin index 7bddaac..f77b547 100755 --- a/ntJoin +++ b/ntJoin @@ -226,7 +226,7 @@ ifeq ($(overlap), True) endif $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa: $(target).k$(k).w$(w).tsv $(min_ref_targets) - $(log_time) $(assemble_path)/bin/ntjoin_run.py scaffold -p $(prefix) -n $(n) -s $< -l $(target_weight) \ + $(log_time) $(assemble_path)/bin/ntjoin_run.py -p $(prefix) -n $(n) -s $< -l $(target_weight) \ -r "$(reference_weights_list)" -k $(k) -g $(g) -G $(G) -t $(assemble_t) $(ntjoin_assembly_opts) $(min_ref_targets) $(target).k$(k).w$(w).n$(n).unassigned.scaffolds.fa: $(target).k$(k).w$(w).n$(n).assigned.scaffolds.fa From 5e9757070904d175847943b4bf1f45d9cb70998a Mon Sep 17 00:00:00 2001 From: lcoombe Date: Fri, 27 Oct 2023 16:16:44 -0700 Subject: [PATCH 28/31] Fix -v duplication --- bin/ntjoin_run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/ntjoin_run.py b/bin/ntjoin_run.py index 9ef6b28..45cea9a 100755 --- a/bin/ntjoin_run.py +++ b/bin/ntjoin_run.py @@ -52,7 +52,6 @@ def parse_arguments(): parser.add_argument("--btllib_t", help="Number of threads for btllib wrapper functions " "(computing minimizers, reading fasta file) [4]", type=int, default=4) - parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') if len(sys.argv) == 1: From 71670027961f8f5daa6cd5a31d86684339956cc5 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Mon, 30 Oct 2023 09:15:55 -0700 Subject: [PATCH 29/31] Remove --quiet in requirements install for azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c490c7c..3833e33 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,7 +60,7 @@ jobs: source activate ntjoin_CI conda install --yes --quiet --name ntjoin_CI -c conda-forge -c bioconda python=3.9 mamba mamba install --yes --quiet -c conda-forge -c bioconda pylint samtools - mamba install --yes --quiet -c conda-forge -c bioconda --file requirements.txt + mamba install --yes -c conda-forge -c bioconda --file requirements.txt displayName: Install Anaconda packages - script: | source activate ntjoin_CI From 8b474d600bdd26146dace974cca1c87e23d032b0 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Mon, 30 Oct 2023 09:56:27 -0700 Subject: [PATCH 30/31] Update version, macOS image --- azure-pipelines.yml | 2 +- bin/ntjoin_run.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3833e33..e3430a6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ jobs: - job: macOS_default_clang pool: - vmImage: macOS-latest + vmImage: macOS-13 steps: - script: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH diff --git a/bin/ntjoin_run.py b/bin/ntjoin_run.py index 45cea9a..271c5f8 100755 --- a/bin/ntjoin_run.py +++ b/bin/ntjoin_run.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """ -ntJoin: Scaffolding assemblies and analyzing synteny -using reference assemblies and minimizer graphs +ntJoin: Scaffolding assemblies using reference assemblies and minimizer graphs Written by Lauren Coombe (@lcoombe) """ import argparse @@ -36,7 +35,7 @@ def parse_arguments(): "increasing/decreasing to assign contig orientation [90]\n " "Note: Only used with --mkt is NOT specified", default=90, type=int) parser.add_argument('-t', help="Number of threads for multiprocessing [1]", default=1, type=int) - parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.1') + parser.add_argument("-v", "--version", action='version', version='ntJoin v1.1.3') parser.add_argument("--agp", help="Output AGP file describing scaffolds", action="store_true") parser.add_argument("--no_cut", help="Do not cut input contigs, place in most representative path", action="store_true") From 7ecd361b2e4fc7a50b5bdb3e90282f999fbeccc7 Mon Sep 17 00:00:00 2001 From: lcoombe Date: Mon, 30 Oct 2023 10:09:14 -0700 Subject: [PATCH 31/31] Use macOS-latest --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e3430a6..3833e33 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ jobs: - job: macOS_default_clang pool: - vmImage: macOS-13 + vmImage: macOS-latest steps: - script: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH