From 9734b63855dc557de9ebd51b2c3cccfb3095ca8a Mon Sep 17 00:00:00 2001 From: Zilan Xin Date: Fri, 10 Jul 2026 01:20:13 -0700 Subject: [PATCH 1/5] new vg combine, adding new options --- src/subcommand/combine_main.cpp | 947 +++++++++++++++++++++++++++++++- 1 file changed, 927 insertions(+), 20 deletions(-) diff --git a/src/subcommand/combine_main.cpp b/src/subcommand/combine_main.cpp index b96665ecc29..979b90aa921 100644 --- a/src/subcommand/combine_main.cpp +++ b/src/subcommand/combine_main.cpp @@ -8,7 +8,11 @@ #include #include +#include #include +#include +#include +#include #include #include @@ -38,10 +42,52 @@ void help_combine(char** argv) { << " ex: If path x is present in graphs N-1 and N, then" << endl << " an edge connecting the last node of x in N-1 " << endl << " and the first node of x in N will be added." << endl + << " -f, --connect-fragments" << endl + << " [DEFAULT MODE] treat paths sharing" << endl + << " sample/locus/haplotype/phase block but differing only by" << endl + << " subrange (e.g. CHM13#0#chr22 and" << endl + << " CHM13#0#chr22[17475777]) as fragments of one path. Sort" << endl + << " fragments by start offset, concatenate in order, add" << endl + << " the missing inter-fragment edges, and rewrite the path" << endl + << " name to a single [start-end] subrange covering the" << endl + << " union. This runs by default whenever none of -c/-p/-m" << endl + << " is given." << endl + << " -s, --shared-nodes modify the default (-f) mode: do NOT renumber" << endl + << " IDs when merging. Node IDs that appear in both" << endl + << " inputs must carry the same sequence; one copy is" << endl + << " kept (intended for `vg chunk` output that shares" << endl + << " boundary nodes across chunks). Fragments are merged" << endl + << " into one path, trimming duplicated boundary steps." << endl + << " Valid only with the default mode; cannot be" << endl + << " combined with -c/-p/-m." << endl + << " -m, --merge combine chunked-assembly graphs that share" << endl + << " REFERENCE-sense paths (e.g. CHM13#0#chr22)" << endl + << " and overlap on them. Buckets inputs by" << endl + << " REFERENCE identity (sample/locus/haplotype)," << endl + << " so chunks of different chromosomes (e.g." << endl + << " CHM13#0#chr21 and CHM13#0#chr22) can be passed" << endl + << " together; each reference is stitched on its" << endl + << " own and kept as a separate path in the output." << endl + << " Within a reference, sorts inputs by REFERENCE" << endl + << " start offset, validates that each chunk's" << endl + << " boundary is a single chain node shared by" << endl + << " every path (and equal to the REFERENCE end)," << endl + << " trims each right chunk's leading node by the" << endl + << " REFERENCE-offset overlap, connects the left" << endl + << " end to the trimmed right start, then merges" << endl + << " fragments like -f." << endl + << " -u, --fusion requires -m. Instead of just adding an edge" << endl + << " between the left end node and the right's" << endl + << " trimmed start node, fuse them into a single" << endl + << " node carrying the concatenated sequence." << endl << " -h, --help print this help message to stderr and exit" << endl; } static int cat_proto_graphs(int argc, char** argv, const Logger& logger); +static int connect_paths_combine(int argc, char** argv, const Logger& logger); +static int connect_fragments_combine(int argc, char** argv, const Logger& logger, + bool shared_nodes); +static int merge_combine(int argc, char** argv, const Logger& logger, bool fusion); int main_combine(int argc, char** argv) { Logger logger("vg combine"); @@ -52,8 +98,12 @@ int main_combine(int argc, char** argv) { } bool connect_paths = false; + bool connect_fragments = false; + bool shared_nodes = false; bool cat_proto = false; - + bool merge = false; + bool fusion = false; + int c; optind = 2; // force optind past command positional argument while (true) { @@ -61,12 +111,16 @@ int main_combine(int argc, char** argv) { { {"help", no_argument, 0, 'h'}, {"connect-paths", no_argument, 0, 'p'}, + {"connect-fragments", no_argument, 0, 'f'}, + {"shared-nodes", no_argument, 0, 's'}, {"cat-proto", no_argument, 0, 'c'}, + {"merge", no_argument, 0, 'm'}, + {"fusion", no_argument, 0, 'u'}, {0, 0, 0, 0} }; int option_index = 0; - c = getopt_long (argc, argv, "h?pc", + c = getopt_long (argc, argv, "h?pfscmu", long_options, &option_index); // Detect the end of the options. @@ -78,9 +132,21 @@ int main_combine(int argc, char** argv) { case 'p': connect_paths = true; break; + case 'f': + connect_fragments = true; + break; + case 's': + shared_nodes = true; + break; case 'c': cat_proto = true; break; + case 'm': + merge = true; + break; + case 'u': + fusion = true; + break; case 'h': case '?': help_combine(argv); @@ -92,13 +158,58 @@ int main_combine(int argc, char** argv) { } } + if (fusion && !merge) { + logger.error() << "--fusion/-u requires --merge/-m." << endl; + } + if (cat_proto) { - if (connect_paths) + if (connect_paths) logger.warn() << "--cat-proto/-c option is deprecated " << "and will be removed in a future version of vg." << endl; return cat_proto_graphs(argc, argv, logger); } - + + if (merge) { + if (connect_paths || connect_fragments || shared_nodes) { + logger.error() << "--merge/-m is mutually exclusive with --connect-paths/-p, " + << "--connect-fragments/-f, and --shared-nodes/-s." << endl; + } + return merge_combine(argc, argv, logger, fusion); + } + + if (connect_paths) { + // --connect-paths/-p mode (explicit), mutually exclusive with -f and -s. + if (connect_fragments || shared_nodes) { + logger.error() << "--connect-paths/-p is mutually exclusive with " + << "--connect-fragments/-f and --shared-nodes/-s." << endl; + } + return connect_paths_combine(argc, argv, logger); + } + + // --connect-fragments/-f is the DEFAULT mode. Unless the user explicitly + // selects another mode (--cat-proto/-c and --merge/-m are handled above, + // --connect-paths/-p is handled just above), fragment merging runs. Passing + // -f explicitly is equivalent to the default. + // + // --shared-nodes/-s is a modifier of this default mode: it is forwarded so + // the per-graph copy keeps source IDs intact and shared boundary nodes are + // deduplicated before fragments are merged into a single coordinate-spanning + // path. It is therefore only valid here (not with -c/-p/-m). + return connect_fragments_combine(argc, argv, logger, shared_nodes); +} + +// Register subcommand +static Subcommand vg_combine("combine", "merge multiple graph files together", main_combine); + +// --connect-paths / -p mode. +// +// Loads the first graph as the destination accumulator, then for each +// subsequent graph renumbers its node IDs out of the way and appends it, +// connecting same-named paths end-to-end across graphs (an edge is added from +// the last node of a path in the accumulator to the first node of the same-named +// path in the incoming graph). See handlealgs::append_path_handle_graph. +int connect_paths_combine(int argc, char** argv, const Logger& logger) { + unique_ptr first_graph; string first_graph_filename = get_input_file_name(optind, argc, argv); first_graph = vg::io::VPKG::load_one(first_graph_filename); @@ -117,19 +228,8 @@ int main_combine(int argc, char** argv) { } max_node_id = graph->max_node_id(); - if (connect_paths) { - handlealgs::append_path_handle_graph(graph.get(), first_graph.get(), true); - } else { - graph->for_each_path_handle([&](path_handle_t path_handle) { - string path_name = graph->get_path_name(path_handle); - if (first_graph->has_path(path_name)) { - logger.error() << "Paths with name \"" << path_name - << "\" found in multiple input graphs. If they are consecutive" - << "subpath ranges, they can be connected by using the -p option." << endl; - } - }); - handlealgs::copy_path_handle_graph(graph.get(), first_graph.get()); - } + // -p: append, connecting same-named paths end-to-end across graphs. + handlealgs::append_path_handle_graph(graph.get(), first_graph.get(), true); } // Serialize the graph using VPKG. @@ -138,8 +238,310 @@ int main_combine(int argc, char** argv) { return 0; } -// Register subcommand -static Subcommand vg_combine("combine", "merge multiple graph files together", main_combine); +// Helper: compute the total sequence length spanned by a path in `graph`. +static size_t compute_path_length(const PathHandleGraph* graph, const path_handle_t& path) { + size_t length = 0; + for (handle_t h : graph->scan_path(path)) { + length += graph->get_length(h); + } + return length; +} + +// Bucket paths in `dest` by their non-coordinate metadata +// (sense, sample, locus, haplotype, circular) -- phase_block is intentionally +// excluded so paths that use phase_block as a start coordinate land in the +// same bucket -- then sort each bucket by start offset, drop the original +// fragment paths, and re-create a single coordinate-spanning path per group. +// Inter-fragment edges are added where missing; runs of duplicated handles +// at the seam (suffix-of-prev == prefix-of-next) are trimmed so the merged +// path doesn't revisit shared boundary nodes. +// +// This is the second pass of -f / -s+-f / -m: it expects `dest` to already +// contain every input graph's nodes, edges, and per-fragment paths. +static void merge_path_fragments(MutablePathMutableHandleGraph* dest, + const Logger& logger) { + using handlegraph::PathSense; + using handlegraph::PathMetadata; + using handlegraph::subrange_t; + using handlegraph::offset_t; + + struct Fragment { + path_handle_t handle; + string name; + subrange_t subrange; + size_t phase_block; + size_t length; + }; + using GroupKey = std::tuple; + std::map> groups; + + dest->for_each_path_handle([&](const path_handle_t& path_handle) { + GroupKey key{ + dest->get_sense(path_handle), + dest->get_sample_name(path_handle), + dest->get_locus_name(path_handle), + dest->get_haplotype(path_handle), + dest->get_is_circular(path_handle) + }; + Fragment f; + f.handle = path_handle; + f.name = dest->get_path_name(path_handle); + f.subrange = dest->get_subrange(path_handle); + f.phase_block = dest->get_phase_block(path_handle); + f.length = compute_path_length(dest, path_handle); + groups[key].push_back(std::move(f)); + }); + + auto fragment_start = [](const Fragment& f) -> offset_t { + if (f.subrange != PathMetadata::NO_SUBRANGE) { + return f.subrange.first; + } + if (f.phase_block != PathMetadata::NO_PHASE_BLOCK) { + return (offset_t)f.phase_block; + } + return 0; + }; + auto fragment_end = [&](const Fragment& f) -> offset_t { + if (f.subrange != PathMetadata::NO_SUBRANGE && + f.subrange.second != PathMetadata::NO_END_POSITION) { + return f.subrange.second; + } + return fragment_start(f) + (offset_t)f.length; + }; + + for (auto& kv : groups) { + auto& frags = kv.second; + if (frags.size() <= 1) { + continue; + } + const auto& key = kv.first; + PathSense input_sense = std::get<0>(key); + const string& sample = std::get<1>(key); + const string& locus = std::get<2>(key); + size_t haplotype = std::get<3>(key); + bool is_circular = std::get<4>(key); + + std::sort(frags.begin(), frags.end(), + [&](const Fragment& a, const Fragment& b) { + return fragment_start(a) < fragment_start(b); + }); + + offset_t combined_start = fragment_start(frags.front()); + offset_t combined_end = 0; + for (const auto& f : frags) { + combined_end = std::max(combined_end, fragment_end(f)); + } + subrange_t combined_subrange{combined_start, combined_end}; + + // Method A: preserve the input sense. A HAPLOTYPE path cannot exist + // without a phase_block (create_path_name enforces this), so instead of + // demoting it to REFERENCE/GENERIC we keep it HAPLOTYPE and attach a + // placeholder phase_block of 0. The real coordinates live in the + // subrange, yielding names like HG002#2#chr22#0[26540133-27549291]. + // REFERENCE/GENERIC paths must NOT carry a phase_block, so they keep + // NO_PHASE_BLOCK as before. + PathSense merged_sense = input_sense; + size_t merged_phase_block = (input_sense == PathSense::HAPLOTYPE) + ? 0 + : PathMetadata::NO_PHASE_BLOCK; + + string merged_name = PathMetadata::create_path_name( + merged_sense, sample, locus, haplotype, + merged_phase_block, combined_subrange); + + if (dest->has_path(merged_name)) { + bool is_own_fragment = false; + for (const auto& f : frags) { + if (f.name == merged_name) { is_own_fragment = true; break; } + } + if (!is_own_fragment) { + logger.error() << "Merged fragment name \"" << merged_name + << "\" collides with an existing unrelated path in the combined graph." + << endl; + } + } + + std::vector> fragment_steps; + fragment_steps.reserve(frags.size()); + for (const auto& f : frags) { + std::vector steps; + for (handle_t h : dest->scan_path(f.handle)) { + steps.push_back(h); + } + fragment_steps.push_back(std::move(steps)); + } + + for (const auto& f : frags) { + dest->destroy_path(f.handle); + } + + path_handle_t merged = dest->create_path( + merged_sense, sample, locus, haplotype, + merged_phase_block, combined_subrange, is_circular); + + bool have_prev = false; + handle_t prev_tail{}; + const std::vector* prev_steps_ptr = nullptr; + for (size_t i = 0; i < fragment_steps.size(); ++i) { + const auto& steps = fragment_steps[i]; + if (steps.empty()) { + continue; + } + size_t skip = 0; + if (have_prev && prev_steps_ptr != nullptr) { + const auto& prev_steps = *prev_steps_ptr; + size_t max_overlap = std::min(prev_steps.size(), steps.size()); + for (size_t k = max_overlap; k >= 1; --k) { + bool match = true; + for (size_t off = 0; off < k; ++off) { + if (prev_steps[prev_steps.size() - k + off] != steps[off]) { + match = false; + break; + } + } + if (match) { + skip = k; + break; + } + } + } + if (have_prev && skip == 0 && !dest->has_edge(prev_tail, steps.front())) { + dest->create_edge(prev_tail, steps.front()); + } + for (size_t j = skip; j < steps.size(); ++j) { + dest->append_step(merged, steps[j]); + } + prev_tail = steps.back(); + prev_steps_ptr = &steps; + have_prev = true; + } + + logger.info() << "Merged " << frags.size() << " fragments into \"" + << merged_name << "\"" << endl; + } +} + +// --connect-fragments / -f mode. +// +// Combines multiple graphs and merges paths that are fragments of the same +// logical path. Paths are bucketed by (sense, sample, locus, haplotype, +// circular). phase_block is *not* part of the key: when it varies inside a +// bucket it is interpreted as a start coordinate (the PathMetadata spec +// explicitly allows phase_block to be "a number or a start coordinate"), so +// e.g. `HG002#1#chr22#0` and `HG002#1#chr22#15419374` are recognized as +// fragments of the same logical chr22 and merged. +// +// Each fragment's effective coordinate range on the merged path is derived as: +// - No subrange, no phase_block -> [0, path-length) +// - No subrange, phase_block=s -> [s, s + path-length) (single offset) +// - subrange=[s] -> [s, s + path-length) (single offset) +// - subrange=[s-e] -> [s, e) (range) +// Fragments are then sorted by start, the missing inter-fragment edges are +// added, and the union [min start, max end-exclusive) becomes the merged path's +// subrange. The merged path drops phase_block entirely; if the inputs were +// HAPLOTYPE-sense purely because of phase_block, the merged path becomes +// REFERENCE-sense (or GENERIC if there is no sample). +// +// When `shared_nodes` is true (-s + -f), the per-graph copy preserves source +// node IDs and deduplicates nodes that already exist in the accumulator +// (sequence-verified). The fragment-stitching loop also checks for shared +// boundary handles between adjacent fragments and trims the duplicate steps, +// so the merged path traverses each shared node only once. +int connect_fragments_combine(int argc, char** argv, const Logger& logger, + bool shared_nodes) { + + using handlegraph::nid_t; + + // (1) Load the first graph; it doubles as the destination accumulator. + string first_graph_filename = get_input_file_name(optind, argc, argv); + unique_ptr dest = + vg::io::VPKG::load_one(first_graph_filename); + int64_t max_node_id = dest->max_node_id(); + + // (2) For each subsequent graph: copy nodes/edges into the destination, + // then copy each path with its original (per-fragment) name. The + // fragment merging happens in a second pass. + // + // Default behavior renumbers the incoming graph's IDs out of the way. + // Under -s (shared_nodes) we keep source IDs and dedupe colliding + // nodes by ID after verifying their sequences match. + while (optind < argc) { + unique_ptr graph; + string graph_filename = get_input_file_name(optind, argc, argv); + graph = vg::io::VPKG::load_one(graph_filename); + + if (shared_nodes) { + graph->for_each_handle([&](const handle_t& src_h) { + nid_t id = graph->get_id(src_h); + string src_seq = graph->get_sequence(src_h); + if (dest->has_node(id)) { + if (dest->get_sequence(dest->get_handle(id)) != src_seq) { + logger.error() << "Node " << id << " has conflicting sequences across " + << "inputs; cannot combine in --shared-nodes mode." << endl; + } + // else: shared node already in dest; skip creation. + } else { + dest->create_handle(src_seq, id); + } + }); + graph->for_each_edge([&](const edge_t& src_edge) { + handle_t a = dest->get_handle(graph->get_id(src_edge.first), + graph->get_is_reverse(src_edge.first)); + handle_t b = dest->get_handle(graph->get_id(src_edge.second), + graph->get_is_reverse(src_edge.second)); + if (!dest->has_edge(a, b)) { + dest->create_edge(a, b); + } + }); + max_node_id = std::max(max_node_id, (int64_t)graph->max_node_id()); + } else { + int64_t delta = max_node_id - graph->min_node_id(); + if (delta >= 0) { + graph->increment_node_ids(delta + 1); + } + max_node_id = graph->max_node_id(); + + // Copy nodes + edges only -- paths handled below so we can detect + // name collisions explicitly. + handlealgs::copy_handle_graph(graph.get(), dest.get()); + } + + graph->for_each_path_handle([&](const path_handle_t& src_path) { + string path_name = graph->get_path_name(src_path); + if (dest->has_path(path_name)) { + logger.error() << "Paths with name \"" << path_name + << "\" found in multiple input graphs. Two fragments cannot share " + << "the same subrange suffix; please disambiguate the inputs." << endl; + } + if (shared_nodes) { + // IDs match between graph and dest; replay steps directly. + path_handle_t new_path = dest->create_path( + graph->get_sense(src_path), + graph->get_sample_name(src_path), + graph->get_locus_name(src_path), + graph->get_haplotype(src_path), + graph->get_phase_block(src_path), + graph->get_subrange(src_path), + graph->get_is_circular(src_path) + ); + for (handle_t src_step : graph->scan_path(src_path)) { + handle_t dest_step = dest->get_handle(graph->get_id(src_step), + graph->get_is_reverse(src_step)); + dest->append_step(new_path, dest_step); + } + } else { + handlealgs::copy_path(graph.get(), src_path, dest.get()); + } + }); + } + + // (3) Bucket and merge fragments (shared with -m). + merge_path_fragments(dest.get(), logger); + + // (4) Serialize the merged graph. + vg::io::save_handle_graph(dest.get(), cout); + return 0; +} // This is the original vg combine logic, which itself mimics using "cat" to join up protobuf files @@ -228,8 +630,513 @@ int cat_proto_graphs(int argc, char** argv, const Logger& logger) { if (!cout) { logger.error() << "Could not write converted graph to output stream." << endl; } - + + }); + } + return 0; +} + + +// Per-input REFERENCE metadata for --merge/-m, shared between the bucket +// stitcher and the top-level driver. +struct MergeInput { + string filename; + unique_ptr graph; + path_handle_t ref_path; + string ref_name; + handle_t ref_start_handle; + handle_t ref_end_handle; + handlegraph::offset_t ref_start_offset = 0; + handlegraph::offset_t ref_end_offset = 0; + string sample; + string locus; + size_t haplotype; + bool is_circular = false; +}; + +// Stitch one bucket of inputs that all share the same REFERENCE identity +// (sample, locus, haplotype, circular) -- i.e. chunks of a single chromosome -- +// into one accumulator graph. This is steps (2)-(4) of --merge/-m: +// +// (2) Sort inputs by REFERENCE start offset. +// (3) Validate the chain boundary on every input: +// - every non-first input's paths all start at its REF start node +// - every non-last input's paths all end at its REF end node +// Boundary nodes must be visited in forward orientation. +// (4) Walk pairs left->right. For each right chunk: +// - overlap = dest_REF_end_offset - src_REF_start_offset +// (error on negative gap or if overlap >= start node length) +// - if overlap > 0: divide_handle(start, overlap), pop_front_step on +// every path in the right chunk, destroy the head piece. The new +// start node is the tail piece. +// - renumber the right chunk's IDs out of dest's range, copy +// nodes/edges/paths into dest under their per-fragment names. +// - connect: if --fusion, glue the left end node and the new right +// start node into a single node with concatenated sequence; +// otherwise just create_edge(left_end, new_right_start). +// +// The returned graph keeps each chunk's per-fragment path names; the caller +// runs merge_path_fragments once across all buckets. A bucket of one input is +// returned as-is (its graph is moved out, no stitching needed). +static unique_ptr +stitch_reference_bucket(std::vector& inputs, const Logger& logger, bool fusion) { + + using handlegraph::PathSense; + using handlegraph::PathMetadata; + using handlegraph::subrange_t; + using handlegraph::offset_t; + using handlegraph::nid_t; + + // (2) Sort by REFERENCE start offset. + std::sort(inputs.begin(), inputs.end(), + [](const MergeInput& a, const MergeInput& b) { + return a.ref_start_offset < b.ref_start_offset; + }); + + // (3) Validate that every chunk's paths share the appropriate boundary. + for (size_t i = 0; i < inputs.size(); ++i) { + auto& in = inputs[i]; + bool need_end_check = (i + 1 < inputs.size()); + bool need_start_check = (i > 0); + in.graph->for_each_path_handle([&](const path_handle_t& p) { + if (need_end_check) { + step_handle_t last_step = in.graph->path_back(p); + handle_t last_h = in.graph->get_handle_of_step(last_step); + if (last_h != in.ref_end_handle) { + logger.error() << "Path \"" << in.graph->get_path_name(p) + << "\" in \"" << in.filename + << "\" does not end at the REFERENCE end node (id " + << in.graph->get_id(in.ref_end_handle) + << "); --merge/-m requires every path in a left/middle " + << "chunk to terminate on the REFERENCE boundary." << endl; + } + } + if (need_start_check) { + step_handle_t first_step = in.graph->path_begin(p); + handle_t first_h = in.graph->get_handle_of_step(first_step); + if (first_h != in.ref_start_handle) { + logger.error() << "Path \"" << in.graph->get_path_name(p) + << "\" in \"" << in.filename + << "\" does not start at the REFERENCE start node (id " + << in.graph->get_id(in.ref_start_handle) + << "); --merge/-m requires every path in a right/middle " + << "chunk to begin on the REFERENCE boundary." << endl; + } + } }); } + + // (4) Pairwise stitch. inputs[0]'s graph becomes the destination accumulator. + unique_ptr dest = std::move(inputs[0].graph); + int64_t max_node_id = dest->max_node_id(); + + // Rightmost boundary handle, tracked in dest's ID space. + handle_t dest_end_handle = inputs[0].ref_end_handle; + offset_t dest_ref_end_offset = inputs[0].ref_end_offset; + + for (size_t i = 1; i < inputs.size(); ++i) { + auto& in = inputs[i]; + unique_ptr src = std::move(in.graph); + + // Capture src's REF endpoint identity BEFORE any mutation (divide / + // increment may invalidate the original handles). + nid_t src_ref_start_id = src->get_id(in.ref_start_handle); + bool src_ref_start_rev = src->get_is_reverse(in.ref_start_handle); + nid_t src_ref_end_id = src->get_id(in.ref_end_handle); + bool src_ref_end_rev = src->get_is_reverse(in.ref_end_handle); + size_t src_start_len = src->get_length(in.ref_start_handle); + + // Length of the left chunk's boundary node (only matters for fusion: + // under -u every right-chunk path's first node grows by this many bp, + // so the path metadata must shift back by this amount so the merged + // subrange end still reflects the original genomic coordinate). + size_t left_boundary_len = dest->get_length(dest_end_handle); + + // Overlap from REF offsets. + offset_t overlap = dest_ref_end_offset - in.ref_start_offset; + if (overlap < 0) { + logger.error() << "REFERENCE offset gap between \"" << inputs[i-1].filename + << "\" (REF ends at offset " << dest_ref_end_offset + << ") and \"" << in.filename << "\" (REF starts at offset " + << in.ref_start_offset << "). --merge/-m does not insert gaps." + << endl; + } + + // Trim the right chunk's start node by `overlap` bp. + nid_t new_start_id_in_src = src_ref_start_id; + if (overlap > 0) { + if ((size_t)overlap > src_start_len) { + logger.error() << "REFERENCE overlap of " << overlap << " bp between \"" + << inputs[i-1].filename << "\" and \"" << in.filename + << "\" exceeds the right chunk's first node length (" + << src_start_len << " bp on node " << src_ref_start_id + << ")." << endl; + } + if ((size_t)overlap == src_start_len) { + logger.error() << "REFERENCE overlap of " << overlap + << " bp exactly equals the right chunk's first node length; " + << "trim would yield a zero-length start node. Re-chunk so the " + << "first node of \"" << in.filename + << "\" extends past the overlap." << endl; + } + // Split the start node. divide_handle keeps paths consistent: + // every path now visits piece_A then piece_B at its head. + auto pieces = src->divide_handle(in.ref_start_handle, + std::vector{(size_t)overlap}); + handle_t piece_A = pieces[0]; + handle_t piece_B = pieces[1]; + new_start_id_in_src = src->get_id(piece_B); + + // Drop piece_A from every path's front. We materialize the path + // list first to avoid iterator invalidation. + std::vector all_paths; + src->for_each_path_handle([&](const path_handle_t& p) { + all_paths.push_back(p); + }); + for (path_handle_t p : all_paths) { + src->pop_front_step(p); + } + + // piece_A has no remaining path steps and (because the boundary + // node is the true left edge of the chunk) no inbound edges. + // Its sole outbound edge piece_A->piece_B vanishes with the node. + src->destroy_handle(piece_A); + } + + // Reconcile each path's offset metadata with what its first node + // actually represents in the destination accumulator: + // - Trimming removes `overlap` bp from the front, so every path + // "starts" `overlap` bp later in its own coordinate system. + // - Under --fusion, the right chunk's first node will gain + // `left_boundary_len` bp on its left (the left chunk's boundary + // sequence is fused in), so the path now "starts" that many bp + // earlier than just-after-trim. + // The combined shift makes fragment_start + path_length come out to + // the correct genomic end after fragment-merge. + int64_t meta_shift = (int64_t)overlap - (fusion ? (int64_t)left_boundary_len : 0); + if (meta_shift != 0) { + struct PathSnapshot { + PathSense sense; + string sample; + string locus; + size_t haplotype; + size_t phase_block; + subrange_t subrange; + bool is_circular; + std::vector steps; + }; + std::vector snapshots; + src->for_each_path_handle([&](const path_handle_t& sp) { + PathSnapshot s; + s.sense = src->get_sense(sp); + s.sample = src->get_sample_name(sp); + s.locus = src->get_locus_name(sp); + s.haplotype = src->get_haplotype(sp); + s.phase_block = src->get_phase_block(sp); + s.subrange = src->get_subrange(sp); + s.is_circular = src->get_is_circular(sp); + for (handle_t h : src->scan_path(sp)) { + s.steps.push_back(h); + } + snapshots.push_back(std::move(s)); + }); + std::vector to_destroy; + src->for_each_path_handle([&](const path_handle_t& sp) { + to_destroy.push_back(sp); + }); + for (path_handle_t sp : to_destroy) { + src->destroy_path(sp); + } + for (auto& s : snapshots) { + if (s.subrange != PathMetadata::NO_SUBRANGE) { + int64_t new_first = (int64_t)s.subrange.first + meta_shift; + if (new_first < 0) new_first = 0; + s.subrange.first = (offset_t)new_first; + } else if (s.phase_block != PathMetadata::NO_PHASE_BLOCK) { + int64_t new_pb = (int64_t)s.phase_block + meta_shift; + if (new_pb < 0) new_pb = 0; + s.phase_block = (size_t)new_pb; + } else { + int64_t new_first = meta_shift; + if (new_first < 0) new_first = 0; + s.subrange = subrange_t{(offset_t)new_first, + PathMetadata::NO_END_POSITION}; + } + path_handle_t new_p = src->create_path( + s.sense, s.sample, s.locus, s.haplotype, + s.phase_block, s.subrange, s.is_circular); + for (handle_t h : s.steps) { + src->append_step(new_p, h); + } + } + } + + // Renumber src out of dest's range. + int64_t shift = 0; + int64_t delta = max_node_id - (int64_t)src->min_node_id(); + if (delta >= 0) { + src->increment_node_ids(delta + 1); + shift = delta + 1; + } + max_node_id = std::max(max_node_id, (int64_t)src->max_node_id()); + + // Copy nodes/edges, then each path under its original (per-fragment) + // name. The fragment-merge pass at the end re-collapses by metadata. + handlealgs::copy_handle_graph(src.get(), dest.get()); + src->for_each_path_handle([&](const path_handle_t& src_path) { + string path_name = src->get_path_name(src_path); + if (dest->has_path(path_name)) { + logger.error() << "Path \"" << path_name + << "\" exists in multiple inputs to --merge/-m; " + << "fragments must carry distinct subrange-suffixed names." + << endl; + } + handlealgs::copy_path(src.get(), src_path, dest.get()); + }); + + // Locate the right chunk's new start and REF end in dest's ID space. + handle_t src_new_start_in_dest = + dest->get_handle((nid_t)((int64_t)new_start_id_in_src + shift), src_ref_start_rev); + handle_t src_ref_end_in_dest = + dest->get_handle((nid_t)((int64_t)src_ref_end_id + shift), src_ref_end_rev); + + if (fusion) { + // Glue dest_end_handle (left chunk's end) and src_new_start_in_dest + // (right chunk's trimmed start) into a single node with the + // concatenated forward-strand sequence. Both are visited only in + // forward orientation by the paths that touch them (we asserted + // forward boundaries above), so the concatenation order matches + // what every path expects. + string fused_seq = dest->get_sequence(dest_end_handle) + + dest->get_sequence(src_new_start_in_dest); + handle_t fused = dest->create_handle(fused_seq); + max_node_id = std::max(max_node_id, (int64_t)dest->get_id(fused)); + + // Preserve every edge that wasn't the implicit seam. + // - inbound to the left side of dest_end_handle -> inbound to fused + // - outbound from the right side of src_new_start_in_dest -> + // outbound from fused + std::vector preds; + dest->follow_edges(dest_end_handle, true, [&](const handle_t& p) { + preds.push_back(p); + }); + std::vector succs; + dest->follow_edges(src_new_start_in_dest, false, [&](const handle_t& n) { + succs.push_back(n); + }); + for (handle_t p : preds) { + if (!dest->has_edge(p, fused)) dest->create_edge(p, fused); + } + for (handle_t n : succs) { + if (!dest->has_edge(fused, n)) dest->create_edge(fused, n); + } + + // Rewrite every step on the two originals to point at fused, + // preserving the step's orientation defensively (we already + // asserted forward but the code stays correct either way). + auto replace_steps_on = [&](handle_t old_h) { + std::vector steps; + dest->for_each_step_on_handle(old_h, [&](const step_handle_t& s) { + steps.push_back(s); + }); + for (const step_handle_t& s : steps) { + handle_t h = dest->get_handle_of_step(s); + handle_t replacement = dest->get_is_reverse(h) + ? dest->flip(fused) : fused; + step_handle_t next = dest->get_next_step(s); + dest->rewrite_segment(s, next, std::vector{replacement}); + } + }; + replace_steps_on(dest_end_handle); + replace_steps_on(src_new_start_in_dest); + + // Destroy originals (also removes any stray edges we didn't pick up). + dest->destroy_handle(dest_end_handle); + dest->destroy_handle(src_new_start_in_dest); + + // If the right chunk's REF is a single node, the new boundary IS + // the fused node; otherwise it's the right chunk's REF end. + dest_end_handle = (src_ref_end_id == new_start_id_in_src) + ? fused + : src_ref_end_in_dest; + } else { + if (!dest->has_edge(dest_end_handle, src_new_start_in_dest)) { + dest->create_edge(dest_end_handle, src_new_start_in_dest); + } + dest_end_handle = src_ref_end_in_dest; + } + + // The right chunk's REF end offset becomes the new dest end offset. + // (Trimming only removed bases from the front, so the end is unchanged.) + dest_ref_end_offset = in.ref_end_offset; + + logger.info() << "Stitched \"" << in.filename << "\": overlap=" << overlap + << " bp, " << (fusion ? "fused" : "edged") << " boundary." + << endl; + } + + return dest; +} + +// --merge / -m mode (optionally with --fusion / -u). +// +// Combines chunked-assembly graphs that share PathSense::REFERENCE paths +// (e.g. CHM13#0#chr22 / CHM13#0#chr22[17475777]) and overlap on them. The +// REFERENCE-path offsets pin each chunk to a coordinate; the overlap between +// adjacent chunks is computed from those offsets and trimmed off the right +// chunk's leading node before stitching. +// +// Inputs may describe more than one reference (e.g. some chunks of CHM13#0#chr21 +// and some of CHM13#0#chr22). They are bucketed by REFERENCE identity (sample, +// locus, haplotype, circular); each bucket is stitched independently, then the +// per-bucket graphs are combined into one output (node IDs renumbered to avoid +// collisions, no edges added between distinct references). The final output +// carries one REFERENCE path per chromosome (e.g. both CHM13#0#chr21 and +// CHM13#0#chr22). +// +// Pipeline: +// (1) Load all inputs and locate each graph's unique REFERENCE-sense path. +// Require exactly one REFERENCE path per graph. +// (2) Bucket inputs by REFERENCE identity. +// (3) Stitch each bucket (see stitch_reference_bucket: sort, validate +// boundaries, pairwise trim+connect/fuse), then copy the stitched bucket +// graphs into a single destination, renumbering IDs as needed. +// (4) Run the same fragment-merge pass as --connect-fragments to collapse +// each reference's chunk paths into one coordinate-spanning path. Because +// that pass buckets by (sense, sample, locus, haplotype, circular), +// different chromosomes stay separate. +int merge_combine(int argc, char** argv, const Logger& logger, bool fusion) { + + using handlegraph::PathSense; + using handlegraph::PathMetadata; + using handlegraph::subrange_t; + using handlegraph::offset_t; + using handlegraph::nid_t; + + // (1) Load all inputs. + std::vector inputs; + while (optind < argc) { + MergeInput in; + in.filename = get_input_file_name(optind, argc, argv); + in.graph = vg::io::VPKG::load_one(in.filename); + inputs.push_back(std::move(in)); + } + if (inputs.size() < 2) { + logger.error() << "--merge/-m requires at least two input graphs." << endl; + } + + // Find the unique REFERENCE path in each graph and pull its endpoint and + // offset metadata. + for (auto& in : inputs) { + std::vector refs; + in.graph->for_each_path_handle([&](const path_handle_t& p) { + if (in.graph->get_sense(p) == PathSense::REFERENCE) { + refs.push_back(p); + } + }); + if (refs.empty()) { + logger.error() << "No REFERENCE-sense path found in \"" << in.filename + << "\"; --merge/-m requires exactly one per graph." << endl; + } + if (refs.size() > 1) { + logger.error() << refs.size() << " REFERENCE-sense paths found in \"" + << in.filename << "\"; --merge/-m requires exactly one." << endl; + } + in.ref_path = refs[0]; + in.ref_name = in.graph->get_path_name(in.ref_path); + in.sample = in.graph->get_sample_name(in.ref_path); + in.locus = in.graph->get_locus_name(in.ref_path); + in.haplotype = in.graph->get_haplotype(in.ref_path); + in.is_circular = in.graph->get_is_circular(in.ref_path); + + subrange_t sr = in.graph->get_subrange(in.ref_path); + size_t pb = in.graph->get_phase_block(in.ref_path); + if (sr != PathMetadata::NO_SUBRANGE) { + in.ref_start_offset = sr.first; + } else if (pb != PathMetadata::NO_PHASE_BLOCK) { + in.ref_start_offset = (offset_t)pb; + } else { + in.ref_start_offset = 0; + } + size_t ref_len = compute_path_length(in.graph.get(), in.ref_path); + in.ref_end_offset = in.ref_start_offset + (offset_t)ref_len; + + step_handle_t first_step = in.graph->path_begin(in.ref_path); + step_handle_t last_step = in.graph->path_back(in.ref_path); + in.ref_start_handle = in.graph->get_handle_of_step(first_step); + in.ref_end_handle = in.graph->get_handle_of_step(last_step); + + if (in.graph->get_is_reverse(in.ref_start_handle) + || in.graph->get_is_reverse(in.ref_end_handle)) { + logger.error() << "REFERENCE path \"" << in.ref_name << "\" in \"" + << in.filename << "\" visits its boundary node in reverse " + << "orientation; --merge/-m only supports forward-strand boundaries." + << endl; + } + } + + // (2) Bucket inputs by REFERENCE identity (sample, locus, haplotype, + // circular). Chunks of the same chromosome land in one bucket; different + // chromosomes land in different buckets and are stitched separately. + // bucket_order preserves first-seen order for deterministic output. + using BucketKey = std::tuple; + std::map> buckets; + std::vector bucket_order; + for (size_t i = 0; i < inputs.size(); ++i) { + BucketKey key{inputs[i].sample, inputs[i].locus, + inputs[i].haplotype, inputs[i].is_circular}; + if (!buckets.count(key)) { + bucket_order.push_back(key); + } + buckets[key].push_back(i); + } + + // (3) Stitch each bucket, then combine the per-bucket graphs into a single + // destination accumulator. The first bucket's stitched graph is the + // destination; each subsequent one is renumbered out of the way and its + // nodes/edges/paths copied in. No edges are created between buckets, so + // distinct references remain disconnected components. + unique_ptr dest; + int64_t max_node_id = 0; + for (const auto& key : bucket_order) { + std::vector bucket; + bucket.reserve(buckets[key].size()); + for (size_t i : buckets[key]) { + bucket.push_back(std::move(inputs[i])); + } + + unique_ptr stitched = + stitch_reference_bucket(bucket, logger, fusion); + + if (!dest) { + dest = std::move(stitched); + max_node_id = dest->max_node_id(); + continue; + } + + int64_t delta = max_node_id - (int64_t)stitched->min_node_id(); + if (delta >= 0) { + stitched->increment_node_ids(delta + 1); + } + max_node_id = std::max(max_node_id, (int64_t)stitched->max_node_id()); + + handlealgs::copy_handle_graph(stitched.get(), dest.get()); + stitched->for_each_path_handle([&](const path_handle_t& src_path) { + string path_name = stitched->get_path_name(src_path); + if (dest->has_path(path_name)) { + logger.error() << "Path \"" << path_name + << "\" exists in multiple --merge/-m references; inputs for " + << "different chromosomes must not share path names." << endl; + } + handlealgs::copy_path(stitched.get(), src_path, dest.get()); + }); + } + + // (4) Bucket and merge fragments into single coordinate-spanning paths + // (one per reference). + merge_path_fragments(dest.get(), logger); + + // (5) Serialize. + vg::io::save_handle_graph(dest.get(), cout); return 0; } From d9ea751b97022094f41c85a13d9888e86067e27a Mon Sep 17 00:00:00 2001 From: faithokamoto Date: Fri, 17 Jul 2026 14:48:20 -0700 Subject: [PATCH 2/5] remove concat --- autocomp.bash | 2 +- autocomplete.sh | 2 +- src/subcommand/concat_main.cpp | 111 --------------------------------- src/subcommand/subcommand.hpp | 4 +- test/t/09_vg_concat.t | 32 ---------- 5 files changed, 5 insertions(+), 146 deletions(-) delete mode 100644 src/subcommand/concat_main.cpp delete mode 100644 test/t/09_vg_concat.t diff --git a/autocomp.bash b/autocomp.bash index b1c227e34e2..9295d8f89aa 100755 --- a/autocomp.bash +++ b/autocomp.bash @@ -3,7 +3,7 @@ _vg_complete() local cur prev COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" - opts="add align annotate augment autoindex bench-dist-query benchmark call chain chains chunk circularize clip cluster combine concat construct convert deconstruct depth describe dotplot filter find gamcompare gampcompare gamsort gbwt genotype giraffe haplotypes ids index inject kmers map mask mcmc minimizer mod mpmap pack paths primers prune rna sim simplify snarls sort stats surject trace translate validate vectorize version view viz zipcode" + opts="add align annotate augment autoindex bench-dist-query benchmark call chain chains chunk circularize clip cluster combine construct convert deconstruct depth describe dotplot filter find gamcompare gampcompare gamsort gbwt genotype giraffe haplotypes ids index inject kmers map mask mcmc minimizer mod mpmap pack paths primers prune rna sim simplify snarls sort stats surject trace translate validate vectorize version view viz zipcode" if [ $COMP_CWORD -eq 1 ] then diff --git a/autocomplete.sh b/autocomplete.sh index 627017b4ea9..c24d746ba49 100644 --- a/autocomplete.sh +++ b/autocomplete.sh @@ -6,7 +6,7 @@ then local cur prev COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" - opts="add align annotate augment autoindex bench-dist-query benchmark call chain chains chunk circularize clip cluster combine concat construct convert deconstruct depth describe dotplot filter find gamcompare gampcompare gamsort gbwt genotype giraffe haplotypes ids index inject kmers map mask mcmc minimizer mod mpmap pack paths primers prune rna sim simplify snarls sort stats surject trace translate validate vectorize version view viz zipcode" + opts="add align annotate augment autoindex bench-dist-query benchmark call chain chains chunk circularize clip cluster combine construct convert deconstruct depth describe dotplot filter find gamcompare gampcompare gamsort gbwt genotype giraffe haplotypes ids index inject kmers map mask mcmc minimizer mod mpmap pack paths primers prune rna sim simplify snarls sort stats surject trace translate validate vectorize version view viz zipcode" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 diff --git a/src/subcommand/concat_main.cpp b/src/subcommand/concat_main.cpp deleted file mode 100644 index 27c3d09d556..00000000000 --- a/src/subcommand/concat_main.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/** \file concat_main.cpp - * - * Defines the "vg concat" subcommand - */ - - -#include -#include -#include - -#include - -#include "subcommand.hpp" -#include "../option.hpp" -#include "../xg.hpp" -#include "../vg.hpp" -#include -#include -#include "../io/save_handle_graph.hpp" -#include - -using namespace std; -using namespace vg; -using namespace vg::subcommand; - -void help_concat(char** argv) { - cerr << "usage: " << argv[0] << " concat [options] [graph2.vg ...] >merged.vg" << endl - << "Concatenates graphs in order by adding edges from the tail nodes of the" << endl - << "predecessor to the head nodes of the following graph. If node ID spaces overlap" << endl - << "between graphs, they will be resolved (as in vg ids -j)" << endl - << endl - << "Options:" << endl - << " -p, --only-join-paths only add edges necessary to join up appended paths" << endl - << " (as opposed between all heads/tails)" << endl - << " -h, --help print this help message to stderr and exit" << endl; -} - -int main_concat(int argc, char** argv) { - - if (argc == 2) { - help_concat(argv); - return 1; - } - - bool only_join_paths = false; - - int c; - optind = 2; // force optind past command positional argument - while (true) { - static struct option long_options[] = - { - {"help", no_argument, 0, 'h'}, - {"only-join-paths", no_argument, 0, 'p'}, - {0, 0, 0, 0} - }; - - int option_index = 0; - c = getopt_long (argc, argv, "h?p", - long_options, &option_index); - - // Detect the end of the options. - if (c == -1) - break; - - switch (c) - { - case 'p': - only_join_paths = true; - break; - case 'h': - case '?': - help_concat(argv); - exit(1); - break; - default: - abort (); - } - } - - unique_ptr first_graph; - get_input_file(optind, argc, argv, [&](istream& in) { - first_graph = vg::io::VPKG::load_one(in); - }); - int64_t max_node_id = first_graph->max_node_id(); - - while (optind < argc) { - - unique_ptr graph; - get_input_file(optind, argc, argv, [&](istream& in) { - graph = vg::io::VPKG::load_one(in); - }); - - // join the id spaces if necessary - int64_t delta = max_node_id - graph->min_node_id(); - if (delta >= 0) { - graph->increment_node_ids(delta + 1); - } - max_node_id = graph->max_node_id(); - - handlealgs::append_path_handle_graph(graph.get(), first_graph.get(), only_join_paths); - } - - // Serialize the graph using VPKG. - vg::io::save_handle_graph(first_graph.get(), cout); - - return 0; -} - -// Register subcommand -static Subcommand vg_concat("concat", "concatenate graphs tail-to-head", DEPRECATED, main_concat); - diff --git a/src/subcommand/subcommand.hpp b/src/subcommand/subcommand.hpp index 9a119ffc167..bfaa0a592ad 100644 --- a/src/subcommand/subcommand.hpp +++ b/src/subcommand/subcommand.hpp @@ -74,7 +74,9 @@ const static std::map REMOVED_CMD_MESSAGES{ {"msga", std::string("vg msga was an early prototype for constructing genome graphs " "from multiple sequence alignments, but VG team members have developed " "improved graph construction algorithms in Cactus and PGGB, " - "and several other tools have been developed by other groups.")} + "and several other tools have been developed by other groups.") + }, + {"concat", "Please use the more flexible and updated 'vg combine' to merge graphs"} }; /// Define a way to print the titles of the different categories diff --git a/test/t/09_vg_concat.t b/test/t/09_vg_concat.t deleted file mode 100644 index a4fcbe89cc3..00000000000 --- a/test/t/09_vg_concat.t +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -BASH_TAP_ROOT=../deps/bash-tap -. ../deps/bash-tap/bash-tap-bootstrap - -PATH=../bin:$PATH # for vg - -plan tests 4 - -vg construct -r small/x.fa -v small/x.vcf.gz >x.vg - -num_nodes=$(vg view -g x.vg | grep ^S | wc -l) -num_edges=$(vg view -g x.vg | grep ^L | wc -l) - -#echo $num_nodes - -is $(vg concat x.vg x.vg | vg view -g - | grep ^S | wc -l) $(echo "$num_nodes * 2" | bc) "concat doubles the number of nodes" -is $(vg concat x.vg x.vg | vg view -g - | grep ^L | wc -l) $(echo "$num_edges * 2 + 1" | bc) "concat doubles the number of edges + 1" - -rm -f x.vg - -vg view -Jv ./reversing/reversing_path.json > reversing.vg - -num_nodes=$(vg view -g reversing.vg | grep ^S | wc -l) -num_edges=$(vg view -g reversing.vg | grep ^L | wc -l) - -is $(vg concat reversing.vg reversing.vg -p | vg view -g - | grep ^S | wc -l) $(echo "$num_nodes * 2" | bc) "concat -p doubles the number of nodes on reversing graph" -# without -p, the heads/tails are backwards so you get something uglier -is $(vg concat reversing.vg reversing.vg -p | vg view -g - | grep ^L | wc -l) $(echo "$num_edges * 2 + 1" | bc) "concat -p doubles the number of edges + 1 on reversing graph" - -rm -f reversing.vg - From 37de3a1c7ea3c5be5880c140eb8701fb6ebd51cb Mon Sep 17 00:00:00 2001 From: Zilan Xin Date: Wed, 16 Sep 2026 03:43:07 -0700 Subject: [PATCH 3/5] Refactor the vg combine and add tests --- doc/vgmanmd.desc.md | 2 +- src/algorithms/fuse_nodes.cpp | 118 ++++ src/algorithms/fuse_nodes.hpp | 20 + src/combine.cpp | 808 ++++++++++++++++++++++ src/combine.hpp | 139 ++++ src/subcommand/combine_main.cpp | 1117 ++----------------------------- src/unittest/combine.cpp | 1100 ++++++++++++++++++++++++++++++ src/unittest/fuse_nodes.cpp | 117 ++++ test/t/05_vg_find.t | 7 +- test/t/51_vg_combine.t | 386 ++++++++++- 10 files changed, 2717 insertions(+), 1097 deletions(-) create mode 100644 src/algorithms/fuse_nodes.cpp create mode 100644 src/algorithms/fuse_nodes.hpp create mode 100644 src/combine.cpp create mode 100644 src/combine.hpp create mode 100644 src/unittest/combine.cpp create mode 100644 src/unittest/fuse_nodes.cpp diff --git a/doc/vgmanmd.desc.md b/doc/vgmanmd.desc.md index 22379c97379..45ffb4a4115 100644 --- a/doc/vgmanmd.desc.md +++ b/doc/vgmanmd.desc.md @@ -45,7 +45,7 @@ This is an incomplete list of vg subcommands. For a complete list, run `vg help` - [`vg mod`](#mod): filter, transform, and edit the graph. - [`vg prune`](#prune): prune the graph for GCSA2 indexing. - [`vg ids`](#ids): manipulate graph node ids. - - [`vg combine`](#combine): merge graphs into a combined graph. Useful for per-chromosome graphs. + - [`vg combine`](#combine): merge graphs holding fragments of the same paths (e.g. chunks of one chromosome) into one graph. - [`vg paths`](#paths): manipulate paths in a graph. - [`vg gbwt`](#gbwt): manipulate GBWTs and associated indexes. [wiki page](https://github.com/vgteam/vg/wiki/VG-GBWT-Subcommand) - [`vg annotate`](#annotate): annotate a graph or alignments. diff --git a/src/algorithms/fuse_nodes.cpp b/src/algorithms/fuse_nodes.cpp new file mode 100644 index 00000000000..ae07e9f8c25 --- /dev/null +++ b/src/algorithms/fuse_nodes.cpp @@ -0,0 +1,118 @@ +/** + * \file fuse_nodes.cpp + * + * Defines an algorithm to fuse one pair of adjacent nodes into one node. + */ + +#include "fuse_nodes.hpp" + +#include +#include + +namespace vg { +namespace algorithms { + +using namespace std; + +/// Check whether two nodes can be fused. +bool can_fuse_nodes(const HandleGraph& graph, handle_t left, handle_t right) { + if (graph.get_id(left) == graph.get_id(right)) { + // A node cannot be concatenated onto itself. + return false; + } + bool clean = true; + // left must have nothing attached on its right side. + graph.follow_edges(left, false, [&](const handle_t&) { + clean = false; + return false; + }); + if (!clean) { + return false; + } + // right must have nothing attached on its left side. + graph.follow_edges(right, true, [&](const handle_t&) { + clean = false; + return false; + }); + return clean; +} + +/// Fuse two nodes into one node. +handle_t fuse_nodes(handlegraph::MutablePathDeletableHandleGraph* graph, + handle_t left, handle_t right) { + + // Create the fused node. + handle_t fused = graph->create_handle(graph->get_sequence(left) + + graph->get_sequence(right)); + + // Collect all neighbours of left and right. + unordered_set predecessors; + unordered_set successors; + graph->follow_edges(left, true, [&](const handle_t& p) { + predecessors.insert(p); + }); + graph->follow_edges(right, false, [&](const handle_t& n) { + successors.insert(n); + }); + + // Handle a neighbour that is left or right itself, so the edges built below + // land on fused. + auto translate = [&](const handle_t& h) { + if (h == left || h == right) { + return fused; + } + if (h == graph->flip(left) || h == graph->flip(right)) { + return graph->flip(fused); + } + return h; + }; + // Build the edges to the neighbours. + for (const handle_t& p : predecessors) { + handle_t from = translate(p); + if (!graph->has_edge(from, fused)) { + graph->create_edge(from, fused); + } + } + for (const handle_t& n : successors) { + handle_t to = translate(n); + if (!graph->has_edge(fused, to)) { + graph->create_edge(fused, to); + } + } + + // Collect every step that visits left or right. + vector> to_rewrite; + for (handle_t original : {left, right}) { + graph->for_each_step_on_handle(original, [&](const step_handle_t& s) { + bool flipped = graph->get_is_reverse(original) + != graph->get_is_reverse(graph->get_handle_of_step(s)); + to_rewrite.emplace_back(s, flipped); + }); + } + // Rewrite each collected step to point at fused. + for (const auto& step_and_flip : to_rewrite) { + graph->rewrite_segment(step_and_flip.first, + graph->get_next_step(step_and_flip.first), + {step_and_flip.second ? graph->flip(fused) : fused}); + } + + // Tear down the old nodes and the edges on them. + for (handle_t original : {left, right}) { + unordered_set to_remove; + graph->follow_edges(original, false, [&](const handle_t& h) { + to_remove.insert(graph->edge_handle(original, h)); + }); + graph->follow_edges(original, true, [&](const handle_t& h) { + to_remove.insert(graph->edge_handle(h, original)); + }); + for (const edge_t& e : to_remove) { + graph->destroy_edge(e); + } + graph->destroy_handle(original); + } + + return fused; +} + +} +} diff --git a/src/algorithms/fuse_nodes.hpp b/src/algorithms/fuse_nodes.hpp new file mode 100644 index 00000000000..d8dce512aa1 --- /dev/null +++ b/src/algorithms/fuse_nodes.hpp @@ -0,0 +1,20 @@ +#ifndef VG_ALGORITHMS_FUSE_NODES_HPP_INCLUDED +#define VG_ALGORITHMS_FUSE_NODES_HPP_INCLUDED + +#include "../handle.hpp" +#include + +namespace vg { +namespace algorithms { + +/// Check whether two nodes can be fused. +bool can_fuse_nodes(const HandleGraph& graph, handle_t left, handle_t right); + +/// Fuse two nodes into one node. +handle_t fuse_nodes(handlegraph::MutablePathDeletableHandleGraph* graph, + handle_t left, handle_t right); + +} +} + +#endif diff --git a/src/combine.cpp b/src/combine.cpp new file mode 100644 index 00000000000..e4a78237b8c --- /dev/null +++ b/src/combine.cpp @@ -0,0 +1,808 @@ +/** + * \file combine.cpp + * + * Implements GraphCombiner. + */ + +#include "combine.hpp" + +#include + +#include "algorithms/fuse_nodes.hpp" + +namespace vg { + +using namespace std; +using handlegraph::nid_t; +using handlegraph::offset_t; +using handlegraph::PathMetadata; +using handlegraph::PathSense; +using handlegraph::subrange_t; + +/// Compute the length of a path. +size_t GraphCombiner::compute_path_length(const PathHandleGraph* graph, + const path_handle_t& path) { + // A path-position graph already knows this. + if (const PathPositionHandleGraph* positional + = dynamic_cast(graph)) { + return positional->get_path_length(path); + } + size_t length = 0; + for (handle_t h : graph->scan_path(path)) { + length += graph->get_length(h); + } + return length; +} + +/// Return a path's start offset. +offset_t GraphCombiner::path_start_offset(const subrange_t& subrange, size_t phase_block, + bool phase_block_is_offset) { + if (subrange != PathMetadata::NO_SUBRANGE) { + return subrange.first; + } + if (phase_block_is_offset && phase_block != PathMetadata::NO_PHASE_BLOCK) { + return (offset_t)phase_block; + } + return 0; +} + +/// Raise every node ID above the previous graph's max ID; return the shift. +int64_t GraphCombiner::renumber_out_of_way(MutableHandleGraph* graph, int64_t above) { + int64_t delta = above - (int64_t)graph->min_node_id(); + if (delta < 0) { + return 0; + } + graph->increment_node_ids(delta + 1); + return delta + 1; +} + +GraphCombiner::GroupKey GraphCombiner::group_key_of(const PathHandleGraph& graph, + const path_handle_t& path, + bool phase_block_is_offset) { + // If phase blocks are coordinates, the key folds them out. + size_t phase_block = graph.get_phase_block(path); + + return GroupKey{graph.get_sense(path), + graph.get_sample_name(path), + graph.get_locus_name(path), + graph.get_haplotype(path), + graph.get_is_circular(path), + phase_block_is_offset ? PathMetadata::NO_PHASE_BLOCK : phase_block}; +} + +/// Read up to `length` bp off the front of a path. +string GraphCombiner::path_prefix(const PathHandleGraph& graph, const path_handle_t& path, + size_t length) { + string sequence; + for (handle_t h : graph.scan_path(path)) { + if (sequence.size() >= length) { + break; + } + sequence += graph.get_sequence(h); + } + // The last node read can overshoot; a short path just yields all of itself. + sequence.resize(std::min(sequence.size(), length)); + return sequence; +} + +/// Read up to `length` bp off the back of a path. +string GraphCombiner::path_suffix(const PathHandleGraph& graph, const path_handle_t& path, + size_t length) { + if (graph.is_empty(path)) { + return string(); + } + string sequence; + step_handle_t step = graph.path_back(path); + while (sequence.size() < length) { + sequence.insert(0, graph.get_sequence(graph.get_handle_of_step(step))); + if (!graph.has_previous_step(step)) { + break; + } + step = graph.get_previous_step(step); + } + // Drop whatever the first node read overshot by, keeping the tail. + if (sequence.size() > length) { + sequence.erase(0, sequence.size() - length); + } + return sequence; +} + +/// Constructor. +GraphCombiner::GraphCombiner(SeamPolicy seam, bool fuse, bool phase_block_is_offset) + : seam(seam), fuse(fuse), phase_block_is_offset(phase_block_is_offset), + logger("vg combine") { +} + +/// Combine the input graphs. +unique_ptr GraphCombiner::combine(vector&& inputs) { + + // renumber and shared need at least one input graph; trim needs at least two. + if (inputs.empty()) { + logger.error() << "At least one input graph is required." << endl; + } + if (seam == SeamPolicy::COORD_TRIM && inputs.size() < 2) { + logger.error() << "--seam trim measures overlap between chunks, so it needs " + << "at least two input graphs." << endl; + } + + // Turn each public Input into a private Chunk. + vector chunks; + chunks.reserve(inputs.size()); + for (Input& in : inputs) { + Chunk chunk; + chunk.name = std::move(in.name); + chunk.graph = std::move(in.graph); + chunks.push_back(std::move(chunk)); + } + inputs.clear(); + + // Check each input: no path may have multiple fragments within one graph. + for (const Chunk& chunk : chunks) { + check_no_split_paths(chunk); + } + + // For COORD_TRIM, read each chunk's REFERENCE identity and sort by reference + // then start offset. + if (seam == SeamPolicy::COORD_TRIM) { + for (Chunk& chunk : chunks) { + describe_reference(chunk); + } + sort_by_reference_offset(chunks); + check_overlaps_agree(chunks); + } + + for (size_t i = 0; i < chunks.size(); ++i) { + Chunk& chunk = chunks[i]; + + // Non-trim modes just ingest each chunk in turn. + if (seam != SeamPolicy::COORD_TRIM) { + ingest(chunk); + continue; + } + + // The sort made each reference's chunks contiguous, so two bools settle + // whether this is that reference's first or last chunk. + auto found = seams.find(chunk.ref_key); + bool is_first_of_ref = (found == seams.end()); + bool is_last_of_ref = (i + 1 == chunks.size()) + || chunks[i + 1].ref_key != chunk.ref_key; + + if (!is_last_of_ref) { + check_all_paths_end_at(chunk); + } + if (!is_first_of_ref) { + check_all_paths_start_at(chunk); + } + + // Handles will be invalidated, so record IDs and orientations first. + nid_t new_start_id = chunk.graph->get_id(chunk.ref_start_handle); + bool start_is_reverse = chunk.graph->get_is_reverse(chunk.ref_start_handle); + nid_t ref_end_id = chunk.graph->get_id(chunk.ref_end_handle); + bool end_is_reverse = chunk.graph->get_is_reverse(chunk.ref_end_handle); + + // offset_t is unsigned, so check for a gap before subtracting; then + // trim_and_shift cuts the overlap away and gives the new start ID. + if (!is_first_of_ref) { + const SeamState& running = found->second; + + if (running.end_offset < chunk.ref_start_offset) { + logger.error() << "REFERENCE offset gap of " + << (chunk.ref_start_offset - running.end_offset) + << " bp between \"" << running.source + << "\" (REFERENCE ends at offset " << running.end_offset + << ") and \"" << chunk.name << "\" (REFERENCE starts at offset " + << chunk.ref_start_offset + << "). --seam trim does not insert gaps." << endl; + } + offset_t overlap = running.end_offset - chunk.ref_start_offset; + size_t left_boundary_len = dest->get_length( + dest->get_handle(running.end_id, running.end_is_reverse)); + new_start_id = trim_and_shift(chunk, overlap, left_boundary_len); + } + + // Ingesting renumbers IDs, so re-fetch the handles by the shift, then + // seam onto the chunk to the left. + int64_t shift = ingest(chunk); + + handle_t chunk_ref_end = dest->get_handle((nid_t)((int64_t)ref_end_id + shift), + end_is_reverse); + handle_t new_end = chunk_ref_end; + if (!is_first_of_ref) { + handle_t new_start = dest->get_handle((nid_t)((int64_t)new_start_id + shift), + start_is_reverse); + new_end = connect_seam(found->second, chunk.name, new_start, + chunk_ref_end); + } + + // Update the SeamState for this chunk. + seams[chunk.ref_key] = SeamState{dest->get_id(new_end), + dest->get_is_reverse(new_end), + chunk.ref_end_offset, + chunk.name}; + } + + merge_path_fragments(); + return std::move(dest); +} + +/// Check whether a chunk holds multiple fragments of a logical path. +void GraphCombiner::check_no_split_paths(const Chunk& chunk) const { + + // Group by GroupKey, recording each path's start offset and name. + map>> groups; + chunk.graph->for_each_path_handle([&](const path_handle_t& path) { + groups[group_key_of(*chunk.graph, path, phase_block_is_offset)] + .emplace_back(path_start_offset(chunk.graph->get_subrange(path), + chunk.graph->get_phase_block(path), + phase_block_is_offset), + chunk.graph->get_path_name(path)); + }); + + for (auto& group : groups) { + vector>& pieces = group.second; + if (pieces.size() <= 1) { + continue; + } + + // Sort by offset then name, and build the list of names for the message. + std::sort(pieces.begin(), pieces.end()); + string names; + for (const auto& piece : pieces) { + names += (names.empty() ? "\"" : ", \"") + piece.second + "\""; + } + logger.error() << "\"" << chunk.name << "\" holds " << pieces.size() + << " pieces of one path (" << names << "). vg combine joins pieces " + << "that arrive in separate inputs; a path already split inside one " + << "input is not something it will reassemble." << endl; + } +} + +/// Fill in `chunk`'s REFERENCE description and do some checks. +void GraphCombiner::describe_reference(Chunk& chunk) { + + // Collect every REFERENCE-sense path into refs. + vector refs; + chunk.graph->for_each_path_handle([&](const path_handle_t& p) { + if (chunk.graph->get_sense(p) == PathSense::REFERENCE) { + refs.push_back(p); + } + }); + + // Two checks: there must be exactly one REFERENCE path. + if (refs.empty()) { + logger.error() << "No REFERENCE-sense path found in \"" << chunk.name + << "\"; --seam trim reads each chunk's position off its " + << "REFERENCE path, and needs exactly one per graph." << endl; + } + if (refs.size() > 1) { + logger.error() << refs.size() << " REFERENCE-sense paths found in \"" << chunk.name + << "\"; --seam trim needs exactly one, so it knows which one " + << "gives the chunk its position." << endl; + } + path_handle_t ref = refs[0]; + + // Record the two identity fields, ref_name and ref_key, and the REFERENCE + // path's start and end. + chunk.ref_name = chunk.graph->get_path_name(ref); + chunk.ref_key = RefKey{chunk.graph->get_sample_name(ref), + chunk.graph->get_locus_name(ref), + chunk.graph->get_haplotype(ref), + chunk.graph->get_is_circular(ref)}; + chunk.ref_start_offset = path_start_offset(chunk.graph->get_subrange(ref), + chunk.graph->get_phase_block(ref), + phase_block_is_offset); + chunk.ref_end_offset = chunk.ref_start_offset + + (offset_t)compute_path_length(chunk.graph.get(), ref); + + chunk.ref_start_handle = chunk.graph->get_handle_of_step(chunk.graph->path_begin(ref)); + chunk.ref_end_handle = chunk.graph->get_handle_of_step(chunk.graph->path_back(ref)); + + // Check boundary orientation: the REFERENCE path must visit its first and last + // nodes forward. + if (chunk.graph->get_is_reverse(chunk.ref_start_handle) + || chunk.graph->get_is_reverse(chunk.ref_end_handle)) { + logger.error() << "REFERENCE path \"" << chunk.ref_name << "\" in \"" << chunk.name + << "\" visits its boundary node in reverse orientation; --seam trim " + << "only supports forward-strand boundaries." << endl; + } +} + +/// Group each reference's chunks together and sort them by start position. +void GraphCombiner::sort_by_reference_offset(vector& chunks) { + + // Different references keep first-seen order; chunks of one reference sort by + // ascending offset. + map first_seen; + for (const Chunk& chunk : chunks) { + first_seen.emplace(chunk.ref_key, first_seen.size()); + } + std::stable_sort(chunks.begin(), chunks.end(), + [&](const Chunk& a, const Chunk& b) { + size_t ra = first_seen.at(a.ref_key); + size_t rb = first_seen.at(b.ref_key); + if (ra != rb) { + return ra < rb; + } + return a.ref_start_offset < b.ref_start_offset; + }); +} + +/// Check that overlapping chunks spell the same sequence over the overlap. +void GraphCombiner::check_overlaps_agree(const vector& chunks) const { + + // The sort made each reference's chunks contiguous and offset-ascending, so only + // neighbours can overlap. + for (size_t i = 1; i < chunks.size(); ++i) { + const Chunk& left = chunks[i - 1]; + const Chunk& right = chunks[i]; + + // Skip a different reference, and a gap, which combine() reports itself. + if (right.ref_key != left.ref_key + || left.ref_end_offset <= right.ref_start_offset) { + continue; + } + offset_t overlap = left.ref_end_offset - right.ref_start_offset; + + // Read the shared span off each side. A nested chunk reads short; the checks + // in trim_and_shift catch that. + string from_left = path_suffix(*left.graph, + left.graph->get_path_handle(left.ref_name), + overlap); + string from_right = path_prefix(*right.graph, + right.graph->get_path_handle(right.ref_name), + overlap); + + // Find the first base they disagree on, if any. + size_t shared = std::min(from_left.size(), from_right.size()); + size_t at = 0; + while (at < shared && from_left[at] == from_right[at]) { + ++at; + } + if (at < shared) { + logger.error() << "Chunks \"" << left.name << "\" and \"" << right.name + << "\" overlap on REFERENCE coordinates [" + << right.ref_start_offset << ", " << left.ref_end_offset + << ") but disagree there: at offset " + << (right.ref_start_offset + at) << " \"" << left.name + << "\" has " << from_left[at] << " and \"" << right.name + << "\" has " << from_right[at] + << ". --seam trim drops the right chunk's copy of the overlap, " + << "so the two have to spell the same sequence." << endl; + } + } +} + +/// Check that every path in the chunk starts at its REFERENCE start node. +void GraphCombiner::check_all_paths_start_at(const Chunk& chunk) const { + chunk.graph->for_each_path_handle([&](const path_handle_t& p) { + handle_t first = chunk.graph->get_handle_of_step(chunk.graph->path_begin(p)); + if (first != chunk.ref_start_handle) { + logger.error() << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" + << chunk.name << "\" does not start at the REFERENCE start node " + << "(id " << chunk.graph->get_id(chunk.ref_start_handle) + << "). --seam trim trims and joins at that one node, so every " + << "path in a chunk that has a chunk to its left has to begin " + << "there." << endl; + } + }); +} + +/// Check that every path in the chunk ends at its REFERENCE end node. +void GraphCombiner::check_all_paths_end_at(const Chunk& chunk) const { + chunk.graph->for_each_path_handle([&](const path_handle_t& p) { + handle_t last = chunk.graph->get_handle_of_step(chunk.graph->path_back(p)); + if (last != chunk.ref_end_handle) { + logger.error() << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" + << chunk.name << "\" does not end at the REFERENCE end node " + << "(id " << chunk.graph->get_id(chunk.ref_end_handle) + << "). --seam trim trims and joins at that one node, so every " + << "path in a chunk that has a chunk to its right has to end " + << "there." << endl; + } + }); +} + +/// Trim `chunk`'s overlap with the chunk to its left, and fix up path coordinates. +nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, + size_t left_boundary_len) { + + // A raw pointer to this chunk's graph, and the ID of its REFERENCE start node. + MutablePathDeletableHandleGraph* graph = chunk.graph.get(); + nid_t new_start_id = graph->get_id(chunk.ref_start_handle); + + // Two checks: the overlap must be strictly shorter than the first node. + if (overlap > 0) { + size_t start_len = graph->get_length(chunk.ref_start_handle); + if (overlap > start_len) { + logger.error() << "REFERENCE overlap of " << overlap << " bp before \"" + << chunk.name << "\" exceeds the length of its first node (" + << start_len << " bp on node " << new_start_id + << "), so the overlap is not all on that node. Re-chunk so it " + << "is." << endl; + } + if (overlap == start_len) { + logger.error() << "REFERENCE overlap of " << overlap + << " bp exactly equals the length of the first node of \"" + << chunk.name << "\"; trimming it would leave a zero-length " + << "node. Re-chunk so that first node extends past the overlap." + << endl; + } + + // Split the first node in two: the overlapping piece, and the remainder, + // which becomes the new start. + vector pieces = graph->divide_handle(chunk.ref_start_handle, + vector{overlap}); + handle_t overlapping = pieces[0]; + handle_t remainder = pieces[1]; + new_start_id = graph->get_id(remainder); + + // Drop every path's first step, so each now starts at the remainder. + vector all_paths; + graph->for_each_path_handle([&](const path_handle_t& p) { + all_paths.push_back(p); + }); + for (path_handle_t p : all_paths) { + graph->pop_front_step(p); + } + + // Check that no edge enters overlapping from the left; if none does, + // destroy it. + size_t inbound = 0; + graph->follow_edges(overlapping, true, [&](const handle_t&) { inbound++; }); + if (inbound > 0) { + logger.error() << "The first " << overlap << " bp of \"" << chunk.name + << "\" overlap the chunk to its left and have to be trimmed, " + << "but " << inbound << " edge(s) arrive at node " + << graph->get_id(overlapping) + << " there and would be lost. --seam trim needs the chunk " + << "boundary to be a clean left edge." << endl; + } + graph->destroy_handle(overlapping); + } + + // Compute the metadata shift. + int64_t meta_shift = (int64_t)overlap - (fuse ? (int64_t)left_boundary_len : 0); + if (meta_shift == 0) { + return new_start_id; + } + + // Collect every path's metadata and all steps. + struct PathSnapshot { + PathSense sense; ///< Sense of the path. + string sample; ///< Sample name. + string locus; ///< Locus name. + size_t haplotype; ///< Haplotype number. + size_t phase_block; ///< Phase block. + subrange_t subrange; ///< Coordinate range. + bool is_circular; ///< Whether it is circular. + vector steps; ///< Nodes visited, in order. + }; + + vector snapshots; + vector to_destroy; + graph->for_each_path_handle([&](const path_handle_t& p) { + PathSnapshot s; + s.sense = graph->get_sense(p); + s.sample = graph->get_sample_name(p); + s.locus = graph->get_locus_name(p); + s.haplotype = graph->get_haplotype(p); + s.phase_block = graph->get_phase_block(p); + s.subrange = graph->get_subrange(p); + s.is_circular = graph->get_is_circular(p); + for (handle_t h : graph->scan_path(p)) { + s.steps.push_back(h); + } + snapshots.push_back(std::move(s)); + to_destroy.push_back(p); + }); + for (path_handle_t p : to_destroy) { + graph->destroy_path(p); + } + for (PathSnapshot& s : snapshots) { + // Shift each path's coordinates by meta_shift, and rebuild the path. + if (s.subrange != PathMetadata::NO_SUBRANGE) { + s.subrange.first = + (offset_t)max(0, (int64_t)s.subrange.first + meta_shift); + } else if (phase_block_is_offset && s.phase_block != PathMetadata::NO_PHASE_BLOCK) { + s.phase_block = (size_t)max(0, (int64_t)s.phase_block + meta_shift); + } else { + s.subrange = subrange_t{(offset_t)max(0, meta_shift), + PathMetadata::NO_END_POSITION}; + } + path_handle_t rebuilt = graph->create_path(s.sense, s.sample, s.locus, s.haplotype, + s.phase_block, s.subrange, s.is_circular); + for (handle_t h : s.steps) { + graph->append_step(rebuilt, h); + } + } + return new_start_id; +} + +/// Move `chunk` into the accumulator; return how far its IDs were shifted. +int64_t GraphCombiner::ingest(Chunk& chunk) { + + if (!dest) { + // The first graph becomes the accumulator: nothing to copy or renumber. + dest = std::move(chunk.graph); + max_node_id = dest->max_node_id(); + return 0; + } + + int64_t shift = 0; + if (seam == SeamPolicy::SHARED_IDS) { + share_nodes_by_id(chunk); + max_node_id = std::max(max_node_id, (int64_t)chunk.graph->max_node_id()); + } else { + shift = renumber_out_of_way(chunk.graph.get(), max_node_id); + max_node_id = std::max(max_node_id, (int64_t)chunk.graph->max_node_id()); + handlealgs::copy_handle_graph(chunk.graph.get(), dest.get()); + } + copy_paths_checked(chunk); + // Free the source rather than holding every input until combine finishes. + chunk.graph.reset(); + return shift; +} + +/// Copy nodes and edges, keeping IDs as given and distinguishing nodes by ID. +void GraphCombiner::share_nodes_by_id(Chunk& chunk) { + + chunk.graph->for_each_handle([&](const handle_t& source) { + nid_t id = chunk.graph->get_id(source); + string sequence = chunk.graph->get_sequence(source); + if (!dest->has_node(id)) { + dest->create_handle(sequence, id); + return; + } + // Under this policy a repeated ID is the same node, so two sequences for + // one ID is a contradiction with no way to keep both. + if (dest->get_sequence(dest->get_handle(id)) != sequence) { + logger.error() << "Node " << id << " has different sequences in \"" << chunk.name + << "\" and in an earlier input. Under --seam shared a repeated " + << "node ID means the same node, so the sequences have to match." + << endl; + } + }); + chunk.graph->for_each_edge([&](const edge_t& source) { + handle_t from = dest->get_handle(chunk.graph->get_id(source.first), + chunk.graph->get_is_reverse(source.first)); + handle_t to = dest->get_handle(chunk.graph->get_id(source.second), + chunk.graph->get_is_reverse(source.second)); + if (!dest->has_edge(from, to)) { + dest->create_edge(from, to); + } + }); +} + +/// Copy paths, rejecting duplicate names. +void GraphCombiner::copy_paths_checked(Chunk& chunk) { + + chunk.graph->for_each_path_handle([&](const path_handle_t& source) { + string path_name = chunk.graph->get_path_name(source); + if (dest->has_path(path_name)) { + logger.error() << "Path \"" << path_name << "\" is in \"" << chunk.name + << "\" and in an earlier input. Pieces of one path have to be " + << "told apart by name, so give them distinct subranges." << endl; + } + handlealgs::copy_path(chunk.graph.get(), source, dest.get()); + }); +} + +/// Join the left chunk's end to this chunk's start; return the reference's new end. +handle_t GraphCombiner::connect_seam(const SeamState& running, const string& chunk_name, + handle_t new_start, handle_t chunk_ref_end) { + + handle_t left = dest->get_handle(running.end_id, running.end_is_reverse); + + // If not fusing nodes, just add an edge. + if (!fuse) { + if (!dest->has_edge(left, new_start)) { + dest->create_edge(left, new_start); + } + return chunk_ref_end; + } + + // can_fuse_nodes decides whether the two nodes can be fused. + if (!algorithms::can_fuse_nodes(*dest, left, new_start)) { + // Welding buries both facing ends, and something is attached to one. + logger.error() << "--fuse cannot weld node " << dest->get_id(left) << " from \"" + << running.source << "\" to node " << dest->get_id(new_start) + << " from \"" << chunk_name << "\": one of them still has an edge " + << "on the side being welded, which the weld would have to drop. " + << "--seam trim without --fuse joins them with an edge instead and " + << "keeps everything." << endl; + } + + // Note the end node before fusing, fuse into a new node, and update the max ID. + bool ref_ends_at_seam = (dest->get_id(chunk_ref_end) == dest->get_id(new_start)); + handle_t fused = algorithms::fuse_nodes(dest.get(), left, new_start); + max_node_id = std::max(max_node_id, (int64_t)dest->get_id(fused)); + return ref_ends_at_seam ? fused : chunk_ref_end; +} + +/// Merge fragments from the same path into a single path spanning their entire +/// coordinate range. +void GraphCombiner::merge_path_fragments() { + + struct Fragment { + path_handle_t handle; ///< Handle to the path. + subrange_t subrange; ///< Coordinate range. + size_t phase_block; ///< Phase block. + size_t length; ///< Sequence length. + bool positioned_by_phase_block; ///< Whether the phase block gives its position. + }; + map> groups; + + // Walk every path in dest, filling in a Fragment for each. + dest->for_each_path_handle([&](const path_handle_t& path) { + Fragment f; + f.handle = path; + f.subrange = dest->get_subrange(path); + f.phase_block = dest->get_phase_block(path); + f.length = compute_path_length(dest.get(), path); + f.positioned_by_phase_block = phase_block_is_offset + && f.subrange == PathMetadata::NO_SUBRANGE + && f.phase_block != PathMetadata::NO_PHASE_BLOCK; + groups[group_key_of(*dest, path, phase_block_is_offset)].push_back(std::move(f)); + }); + + // Lambdas computing a fragment's start and end coordinates. + auto fragment_start = [&](const Fragment& f) -> offset_t { + return path_start_offset(f.subrange, f.phase_block, phase_block_is_offset); + }; + auto fragment_end = [&](const Fragment& f) -> offset_t { + if (f.subrange != PathMetadata::NO_SUBRANGE + && f.subrange.second != PathMetadata::NO_END_POSITION) { + return f.subrange.second; + } + return fragment_start(f) + (offset_t)f.length; + }; + + // Merge each group in turn. + for (auto& group : groups) { + vector& frags = group.second; + if (frags.size() <= 1) { + // Nothing to merge, and nothing to rename: a lone path is untouched. + continue; + } + PathSense sense = std::get<0>(group.first); + const string& sample = std::get<1>(group.first); + const string& locus = std::get<2>(group.first); + size_t haplotype = std::get<3>(group.first); + bool is_circular = std::get<4>(group.first); + + std::sort(frags.begin(), frags.end(), + [&](const Fragment& a, const Fragment& b) { + return fragment_start(a) < fragment_start(b); + }); + + // Detect any gap, updating the furthest reference offset reached so far. + offset_t covered_end = fragment_end(frags.front()); + for (size_t i = 1; i < frags.size(); ++i) { + offset_t next_start = fragment_start(frags[i]); + if (covered_end < next_start) { + logger.error() << "Gap of " << (next_start - covered_end) + << " bp before fragment \"" + << dest->get_path_name(frags[i].handle) + << "\"; the fragments before it only reach offset " + << covered_end << ". vg combine does not insert gaps." + << endl; + } + covered_end = std::max(covered_end, fragment_end(frags[i])); + } + + // With phase blocks read as coordinates, check they run continuously. + for (size_t i = 0; phase_block_is_offset && i + 1 < frags.size(); ++i) { + if (!frags[i].positioned_by_phase_block + || !frags[i + 1].positioned_by_phase_block) { + continue; + } + offset_t expected = (offset_t)frags[i].phase_block + (offset_t)frags[i].length; + if (expected != (offset_t)frags[i + 1].phase_block) { + logger.error() << "--phase-block-is-offset given, but phase blocks of \"" + << dest->get_path_name(frags[i].handle) + << "\" do not line up as coordinates (block " + << frags[i].phase_block << " + length " << frags[i].length + << " != block " << frags[i + 1].phase_block + << "). Are these phase block numbers rather than offsets?" + << endl; + } + } + + // Compute the merged subrange, phase block, and name. + offset_t combined_start = fragment_start(frags.front()); + offset_t combined_end = 0; + for (const Fragment& f : frags) { + combined_end = std::max(combined_end, fragment_end(f)); + } + subrange_t combined_subrange{combined_start, combined_end}; + + // HAPLOTYPE requires a phase block. Keep the group's own; treating phase blocks + // as coordinates folded it into the subrange, so 0 stands in there. + size_t group_phase_block = std::get<5>(group.first); + size_t merged_phase_block = PathMetadata::NO_PHASE_BLOCK; + if (sense == PathSense::HAPLOTYPE) { + merged_phase_block = (group_phase_block == PathMetadata::NO_PHASE_BLOCK) + ? 0 : group_phase_block; + } + string merged_name = PathMetadata::create_path_name(sense, sample, locus, haplotype, + merged_phase_block, + combined_subrange); + + // Can't lean on create_path(): PackedGraph throws on a duplicate, HashGraph + // quietly orphans the old path, and VG just appends to it. So check here. + if (dest->has_path(merged_name)) { + bool is_own_fragment = false; + for (const Fragment& f : frags) { + if (dest->get_path_name(f.handle) == merged_name) { + is_own_fragment = true; + break; + } + } + if (!is_own_fragment) { + logger.error() << "The merge of " << frags.size() << " fragments would be " + << "named \"" << merged_name << "\", which is already an " + << "unrelated path in the output." << endl; + } + } + + // Save each fragment's node steps into routes. + vector> routes; + routes.reserve(frags.size()); + for (const Fragment& f : frags) { + vector steps; + for (handle_t h : dest->scan_path(f.handle)) { + steps.push_back(h); + } + routes.push_back(std::move(steps)); + } + + // Destroy the old fragments, and create the merged path. + for (const Fragment& f : frags) { + dest->destroy_path(f.handle); + } + + path_handle_t merged = dest->create_path(sense, sample, locus, haplotype, + merged_phase_block, combined_subrange, + is_circular); + const vector* previous = nullptr; + // Walk each fragment's steps in order, appending them to the merged path. + for (const vector& steps : routes) { + if (steps.empty()) { + continue; + } + // `skip` is the shared run at the seam: `run` is the candidate length, tried + // longest first down to 1 (a run of 0 would match trivially), and `k` compares + // `previous`'s tail against `steps`'s head. + size_t skip = 0; + if (previous != nullptr) { + size_t most = std::min(previous->size(), steps.size()); + for (size_t run = most; run >= 1; --run) { + bool same = true; + for (size_t k = 0; k < run; ++k) { + if ((*previous)[previous->size() - run + k] != steps[k]) { + same = false; + break; + } + } + if (same) { + skip = run; + break; + } + } + if (skip == 0 && !dest->has_edge(previous->back(), steps.front())) { + // Nothing shared, so they abut and the graph must say so. + dest->create_edge(previous->back(), steps.front()); + } + } + for (size_t k = skip; k < steps.size(); ++k) { + dest->append_step(merged, steps[k]); + } + previous = &steps; + } + + logger.info() << "Merged " << frags.size() << " fragments into \"" << merged_name + << "\"" << endl; + } +} + +} diff --git a/src/combine.hpp b/src/combine.hpp new file mode 100644 index 00000000000..6ebc02a3064 --- /dev/null +++ b/src/combine.hpp @@ -0,0 +1,139 @@ +#ifndef VG_COMBINE_HPP_INCLUDED +#define VG_COMBINE_HPP_INCLUDED + +/** \file combine.hpp + * + * Combining graphs that hold fragments of the same logical paths. + */ + +#include +#include +#include +#include +#include + +#include "handle.hpp" +#include "log.hpp" + +namespace vg { + +namespace unittest { +/// Reaches into GraphCombiner's internals; see src/unittest/combine.cpp. +class TestGraphCombiner; +} + +/// The three modes of vg combine, selected by --seam. +enum class SeamPolicy { + RENUMBER, ///< --seam renumber + SHARED_IDS, ///< --seam shared + COORD_TRIM ///< --seam trim +}; + +/// Combine fragmented graphs into a whole graph. +class GraphCombiner { +public: + struct Input { + std::string name; + std::unique_ptr graph; + }; + + /// Constructor. + GraphCombiner(SeamPolicy seam, bool fuse, bool phase_block_is_offset); + + /// Combine the input graphs. + std::unique_ptr combine(std::vector&& inputs); + +private: + /// Identity of a reference: sample, locus, haplotype, circular. + using RefKey = std::tuple; + + /// Seaming state. + struct SeamState { + handlegraph::nid_t end_id; ///< Node the reference so far ends on. + bool end_is_reverse; ///< Its orientation. + handlegraph::offset_t end_offset; ///< REFERENCE offset it ends at. + std::string source; ///< Name of chunk/inputs. + }; + + /// An input, plus the REFERENCE description COORD_TRIM positions it by. + struct Chunk { + std::string name; ///< Name of the input. + std::unique_ptr graph; ///< The input graph. + + std::string ref_name; ///< Its REFERENCE path. + handle_t ref_start_handle; ///< First node of that path. + handle_t ref_end_handle; ///< Last node of that path. + handlegraph::offset_t ref_start_offset = 0; ///< Where that path starts. + handlegraph::offset_t ref_end_offset = 0; ///< Where that path ends. + RefKey ref_key; ///< Which reference it belongs to. + }; + + /// A path's identity as a comparable tuple: sense, sample, locus, haplotype, circular, phase block. + using GroupKey = std::tuple; + + /// Compute the length of a path. + static size_t compute_path_length(const PathHandleGraph* graph, const path_handle_t& path); + /// Return a path's start offset. + static handlegraph::offset_t path_start_offset(const handlegraph::subrange_t& subrange, + size_t phase_block, bool phase_block_is_offset); + /// Raise every node ID above the previous graph's max ID; return the shift. + static int64_t renumber_out_of_way(MutableHandleGraph* graph, int64_t above); + /// Build a path's GroupKey. + static GroupKey group_key_of(const PathHandleGraph& graph, const path_handle_t& path, + bool phase_block_is_offset); + /// Read up to `length` bp off the front of a path. + static std::string path_prefix(const PathHandleGraph& graph, const path_handle_t& path, + size_t length); + /// Read up to `length` bp off the back of a path. + static std::string path_suffix(const PathHandleGraph& graph, const path_handle_t& path, + size_t length); + + /// Check whether a chunk holds multiple fragments of a logical path. + void check_no_split_paths(const Chunk& chunk) const; + /// Fill in `chunk`'s REFERENCE description and do some checks. + void describe_reference(Chunk& chunk); + /// Group each reference's chunks together and sort them by start position. + void sort_by_reference_offset(std::vector& chunks); + /// Check that overlapping chunks spell the same sequence over the overlap. + void check_overlaps_agree(const std::vector& chunks) const; + + /// Check that every path in the chunk starts at its REFERENCE start node. + void check_all_paths_start_at(const Chunk& chunk) const; + /// Check that every path in the chunk ends at its REFERENCE end node. + void check_all_paths_end_at(const Chunk& chunk) const; + + /// Trim `chunk`'s overlap with the chunk to its left, and fix up path coordinates. + handlegraph::nid_t trim_and_shift(Chunk& chunk, handlegraph::offset_t overlap, + size_t left_boundary_len); + + /// Move `chunk` into the accumulator; return how far its IDs were shifted. + int64_t ingest(Chunk& chunk); + /// Copy nodes and edges, keeping IDs as given and distinguishing nodes by ID. + void share_nodes_by_id(Chunk& chunk); + /// Copy paths, rejecting duplicate names. + void copy_paths_checked(Chunk& chunk); + + /// Join the left chunk's end to this chunk's start; return the reference's new end. + handle_t connect_seam(const SeamState& seam, const std::string& chunk_name, + handle_t new_start, handle_t chunk_ref_end); + + /// Merge fragments from the same path into a single path spanning their entire + /// coordinate range. + void merge_path_fragments(); + + SeamPolicy seam; + bool fuse; + bool phase_block_is_offset; + Logger logger; + + std::unique_ptr dest; + int64_t max_node_id = 0; + std::map seams; + + friend class unittest::TestGraphCombiner; +}; + +} + +#endif diff --git a/src/subcommand/combine_main.cpp b/src/subcommand/combine_main.cpp index 979b90aa921..59d28cbbe21 100644 --- a/src/subcommand/combine_main.cpp +++ b/src/subcommand/combine_main.cpp @@ -4,24 +4,19 @@ */ -#include #include #include -#include #include -#include -#include +#include #include #include -#include -#include #include "subcommand.hpp" +#include "../combine.hpp" #include "../handle.hpp" -#include "../vg.hpp" #include "../io/save_handle_graph.hpp" using namespace std; @@ -30,64 +25,37 @@ using namespace vg::subcommand; void help_combine(char** argv) { cerr << "usage: " << argv[0] << " combine [options] [graph2.vg ...] >merged.vg" << endl - << "Combines one or more graphs into a single file, regardless of input format." << endl - << "Node IDs will be modified as needed to resolve conflicts (same as vg ids -j)." << endl + << "Combines one or more graphs into a single file." << endl + << "Input graphs must not share any path name." << endl << endl << "Options:" << endl - << " -c, --cat-proto merge by converting each to Protobuf (if not already)" << endl - << " and catting the results." << endl - << " node IDs not modified [DEPRECATED]" << endl - << " -p, --connect-paths add edges necessary to connect paths with the same name" << endl - << " which are present in different graphs." << endl - << " ex: If path x is present in graphs N-1 and N, then" << endl - << " an edge connecting the last node of x in N-1 " << endl - << " and the first node of x in N will be added." << endl - << " -f, --connect-fragments" << endl - << " [DEFAULT MODE] treat paths sharing" << endl - << " sample/locus/haplotype/phase block but differing only by" << endl - << " subrange (e.g. CHM13#0#chr22 and" << endl - << " CHM13#0#chr22[17475777]) as fragments of one path. Sort" << endl - << " fragments by start offset, concatenate in order, add" << endl - << " the missing inter-fragment edges, and rewrite the path" << endl - << " name to a single [start-end] subrange covering the" << endl - << " union. This runs by default whenever none of -c/-p/-m" << endl - << " is given." << endl - << " -s, --shared-nodes modify the default (-f) mode: do NOT renumber" << endl - << " IDs when merging. Node IDs that appear in both" << endl - << " inputs must carry the same sequence; one copy is" << endl - << " kept (intended for `vg chunk` output that shares" << endl - << " boundary nodes across chunks). Fragments are merged" << endl - << " into one path, trimming duplicated boundary steps." << endl - << " Valid only with the default mode; cannot be" << endl - << " combined with -c/-p/-m." << endl - << " -m, --merge combine chunked-assembly graphs that share" << endl - << " REFERENCE-sense paths (e.g. CHM13#0#chr22)" << endl - << " and overlap on them. Buckets inputs by" << endl - << " REFERENCE identity (sample/locus/haplotype)," << endl - << " so chunks of different chromosomes (e.g." << endl - << " CHM13#0#chr21 and CHM13#0#chr22) can be passed" << endl - << " together; each reference is stitched on its" << endl - << " own and kept as a separate path in the output." << endl - << " Within a reference, sorts inputs by REFERENCE" << endl - << " start offset, validates that each chunk's" << endl - << " boundary is a single chain node shared by" << endl - << " every path (and equal to the REFERENCE end)," << endl - << " trims each right chunk's leading node by the" << endl - << " REFERENCE-offset overlap, connects the left" << endl - << " end to the trimmed right start, then merges" << endl - << " fragments like -f." << endl - << " -u, --fusion requires -m. Instead of just adding an edge" << endl - << " between the left end node and the right's" << endl - << " trimmed start node, fuse them into a single" << endl - << " node carrying the concatenated sequence." << endl + << " -s, --seam POLICY how to resolve overlap between adjacent chunks:" << endl + << " renumber assume no overlap [default]" << endl + << " shared overlap by shared nodes" << endl + << " trim overlap in REFERENCE coordinates" << endl + << " -u, --fuse requires --seam trim. Fuse the boundary nodes into" << endl + << " one node instead of adding an edge." << endl + << " -P, --phase-block-is-offset" << endl + << " treat a path's phase block as a start coordinate, so" << endl + << " fragments differing only by phase block are merged." << endl << " -h, --help print this help message to stderr and exit" << endl; } -static int cat_proto_graphs(int argc, char** argv, const Logger& logger); -static int connect_paths_combine(int argc, char** argv, const Logger& logger); -static int connect_fragments_combine(int argc, char** argv, const Logger& logger, - bool shared_nodes); -static int merge_combine(int argc, char** argv, const Logger& logger, bool fusion); +/// Map a --seam argument to its policy, or exit naming the legal values. +static SeamPolicy parse_seam_policy(const string& value, const Logger& logger) { + if (value == "renumber") { + return SeamPolicy::RENUMBER; + } + if (value == "shared") { + return SeamPolicy::SHARED_IDS; + } + if (value == "trim") { + return SeamPolicy::COORD_TRIM; + } + logger.error() << "unrecognized --seam policy \"" << value + << "\"; expected one of: renumber, shared, trim." << endl; + return SeamPolicy::RENUMBER; // not reached; logger.error() exits +} int main_combine(int argc, char** argv) { Logger logger("vg combine"); @@ -97,30 +65,26 @@ int main_combine(int argc, char** argv) { return 1; } - bool connect_paths = false; - bool connect_fragments = false; - bool shared_nodes = false; - bool cat_proto = false; - bool merge = false; - bool fusion = false; + SeamPolicy seam = SeamPolicy::RENUMBER; + bool fuse = false; + bool phase_block_is_offset = false; int c; optind = 2; // force optind past command positional argument + + // Parse command line options. while (true) { static struct option long_options[] = { {"help", no_argument, 0, 'h'}, - {"connect-paths", no_argument, 0, 'p'}, - {"connect-fragments", no_argument, 0, 'f'}, - {"shared-nodes", no_argument, 0, 's'}, - {"cat-proto", no_argument, 0, 'c'}, - {"merge", no_argument, 0, 'm'}, - {"fusion", no_argument, 0, 'u'}, + {"seam", required_argument, 0, 's'}, + {"fuse", no_argument, 0, 'u'}, + {"phase-block-is-offset", no_argument, 0, 'P'}, {0, 0, 0, 0} }; int option_index = 0; - c = getopt_long (argc, argv, "h?pfscmu", + c = getopt_long (argc, argv, "h?s:uP", long_options, &option_index); // Detect the end of the options. @@ -129,23 +93,14 @@ int main_combine(int argc, char** argv) { switch (c) { - case 'p': - connect_paths = true; - break; - case 'f': - connect_fragments = true; - break; case 's': - shared_nodes = true; - break; - case 'c': - cat_proto = true; - break; - case 'm': - merge = true; + seam = parse_seam_policy(optarg, logger); break; case 'u': - fusion = true; + fuse = true; + break; + case 'P': + phase_block_is_offset = true; break; case 'h': case '?': @@ -158,985 +113,25 @@ int main_combine(int argc, char** argv) { } } - if (fusion && !merge) { - logger.error() << "--fusion/-u requires --merge/-m." << endl; - } - - if (cat_proto) { - if (connect_paths) - logger.warn() << "--cat-proto/-c option is deprecated " - << "and will be removed in a future version of vg." << endl; - return cat_proto_graphs(argc, argv, logger); - } - - if (merge) { - if (connect_paths || connect_fragments || shared_nodes) { - logger.error() << "--merge/-m is mutually exclusive with --connect-paths/-p, " - << "--connect-fragments/-f, and --shared-nodes/-s." << endl; - } - return merge_combine(argc, argv, logger, fusion); - } - - if (connect_paths) { - // --connect-paths/-p mode (explicit), mutually exclusive with -f and -s. - if (connect_fragments || shared_nodes) { - logger.error() << "--connect-paths/-p is mutually exclusive with " - << "--connect-fragments/-f and --shared-nodes/-s." << endl; - } - return connect_paths_combine(argc, argv, logger); - } - - // --connect-fragments/-f is the DEFAULT mode. Unless the user explicitly - // selects another mode (--cat-proto/-c and --merge/-m are handled above, - // --connect-paths/-p is handled just above), fragment merging runs. Passing - // -f explicitly is equivalent to the default. - // - // --shared-nodes/-s is a modifier of this default mode: it is forwarded so - // the per-graph copy keeps source IDs intact and shared boundary nodes are - // deduplicated before fragments are merged into a single coordinate-spanning - // path. It is therefore only valid here (not with -c/-p/-m). - return connect_fragments_combine(argc, argv, logger, shared_nodes); -} - -// Register subcommand -static Subcommand vg_combine("combine", "merge multiple graph files together", main_combine); - -// --connect-paths / -p mode. -// -// Loads the first graph as the destination accumulator, then for each -// subsequent graph renumbers its node IDs out of the way and appends it, -// connecting same-named paths end-to-end across graphs (an edge is added from -// the last node of a path in the accumulator to the first node of the same-named -// path in the incoming graph). See handlealgs::append_path_handle_graph. -int connect_paths_combine(int argc, char** argv, const Logger& logger) { - - unique_ptr first_graph; - string first_graph_filename = get_input_file_name(optind, argc, argv); - first_graph = vg::io::VPKG::load_one(first_graph_filename); - int64_t max_node_id = first_graph->max_node_id(); - - while (optind < argc) { - - unique_ptr graph; - string graph_filename = get_input_file_name(optind, argc, argv); - graph = vg::io::VPKG::load_one(graph_filename); - - // join the id spaces if necessary - int64_t delta = max_node_id - graph->min_node_id(); - if (delta >= 0) { - graph->increment_node_ids(delta + 1); - } - max_node_id = graph->max_node_id(); - - // -p: append, connecting same-named paths end-to-end across graphs. - handlealgs::append_path_handle_graph(graph.get(), first_graph.get(), true); - } - - // Serialize the graph using VPKG. - vg::io::save_handle_graph(first_graph.get(), cout); - - return 0; -} - -// Helper: compute the total sequence length spanned by a path in `graph`. -static size_t compute_path_length(const PathHandleGraph* graph, const path_handle_t& path) { - size_t length = 0; - for (handle_t h : graph->scan_path(path)) { - length += graph->get_length(h); - } - return length; -} - -// Bucket paths in `dest` by their non-coordinate metadata -// (sense, sample, locus, haplotype, circular) -- phase_block is intentionally -// excluded so paths that use phase_block as a start coordinate land in the -// same bucket -- then sort each bucket by start offset, drop the original -// fragment paths, and re-create a single coordinate-spanning path per group. -// Inter-fragment edges are added where missing; runs of duplicated handles -// at the seam (suffix-of-prev == prefix-of-next) are trimmed so the merged -// path doesn't revisit shared boundary nodes. -// -// This is the second pass of -f / -s+-f / -m: it expects `dest` to already -// contain every input graph's nodes, edges, and per-fragment paths. -static void merge_path_fragments(MutablePathMutableHandleGraph* dest, - const Logger& logger) { - using handlegraph::PathSense; - using handlegraph::PathMetadata; - using handlegraph::subrange_t; - using handlegraph::offset_t; - - struct Fragment { - path_handle_t handle; - string name; - subrange_t subrange; - size_t phase_block; - size_t length; - }; - using GroupKey = std::tuple; - std::map> groups; - - dest->for_each_path_handle([&](const path_handle_t& path_handle) { - GroupKey key{ - dest->get_sense(path_handle), - dest->get_sample_name(path_handle), - dest->get_locus_name(path_handle), - dest->get_haplotype(path_handle), - dest->get_is_circular(path_handle) - }; - Fragment f; - f.handle = path_handle; - f.name = dest->get_path_name(path_handle); - f.subrange = dest->get_subrange(path_handle); - f.phase_block = dest->get_phase_block(path_handle); - f.length = compute_path_length(dest, path_handle); - groups[key].push_back(std::move(f)); - }); - - auto fragment_start = [](const Fragment& f) -> offset_t { - if (f.subrange != PathMetadata::NO_SUBRANGE) { - return f.subrange.first; - } - if (f.phase_block != PathMetadata::NO_PHASE_BLOCK) { - return (offset_t)f.phase_block; - } - return 0; - }; - auto fragment_end = [&](const Fragment& f) -> offset_t { - if (f.subrange != PathMetadata::NO_SUBRANGE && - f.subrange.second != PathMetadata::NO_END_POSITION) { - return f.subrange.second; - } - return fragment_start(f) + (offset_t)f.length; - }; - - for (auto& kv : groups) { - auto& frags = kv.second; - if (frags.size() <= 1) { - continue; - } - const auto& key = kv.first; - PathSense input_sense = std::get<0>(key); - const string& sample = std::get<1>(key); - const string& locus = std::get<2>(key); - size_t haplotype = std::get<3>(key); - bool is_circular = std::get<4>(key); - - std::sort(frags.begin(), frags.end(), - [&](const Fragment& a, const Fragment& b) { - return fragment_start(a) < fragment_start(b); - }); - - offset_t combined_start = fragment_start(frags.front()); - offset_t combined_end = 0; - for (const auto& f : frags) { - combined_end = std::max(combined_end, fragment_end(f)); - } - subrange_t combined_subrange{combined_start, combined_end}; - - // Method A: preserve the input sense. A HAPLOTYPE path cannot exist - // without a phase_block (create_path_name enforces this), so instead of - // demoting it to REFERENCE/GENERIC we keep it HAPLOTYPE and attach a - // placeholder phase_block of 0. The real coordinates live in the - // subrange, yielding names like HG002#2#chr22#0[26540133-27549291]. - // REFERENCE/GENERIC paths must NOT carry a phase_block, so they keep - // NO_PHASE_BLOCK as before. - PathSense merged_sense = input_sense; - size_t merged_phase_block = (input_sense == PathSense::HAPLOTYPE) - ? 0 - : PathMetadata::NO_PHASE_BLOCK; - - string merged_name = PathMetadata::create_path_name( - merged_sense, sample, locus, haplotype, - merged_phase_block, combined_subrange); - - if (dest->has_path(merged_name)) { - bool is_own_fragment = false; - for (const auto& f : frags) { - if (f.name == merged_name) { is_own_fragment = true; break; } - } - if (!is_own_fragment) { - logger.error() << "Merged fragment name \"" << merged_name - << "\" collides with an existing unrelated path in the combined graph." - << endl; - } - } - - std::vector> fragment_steps; - fragment_steps.reserve(frags.size()); - for (const auto& f : frags) { - std::vector steps; - for (handle_t h : dest->scan_path(f.handle)) { - steps.push_back(h); - } - fragment_steps.push_back(std::move(steps)); - } - - for (const auto& f : frags) { - dest->destroy_path(f.handle); - } - - path_handle_t merged = dest->create_path( - merged_sense, sample, locus, haplotype, - merged_phase_block, combined_subrange, is_circular); - - bool have_prev = false; - handle_t prev_tail{}; - const std::vector* prev_steps_ptr = nullptr; - for (size_t i = 0; i < fragment_steps.size(); ++i) { - const auto& steps = fragment_steps[i]; - if (steps.empty()) { - continue; - } - size_t skip = 0; - if (have_prev && prev_steps_ptr != nullptr) { - const auto& prev_steps = *prev_steps_ptr; - size_t max_overlap = std::min(prev_steps.size(), steps.size()); - for (size_t k = max_overlap; k >= 1; --k) { - bool match = true; - for (size_t off = 0; off < k; ++off) { - if (prev_steps[prev_steps.size() - k + off] != steps[off]) { - match = false; - break; - } - } - if (match) { - skip = k; - break; - } - } - } - if (have_prev && skip == 0 && !dest->has_edge(prev_tail, steps.front())) { - dest->create_edge(prev_tail, steps.front()); - } - for (size_t j = skip; j < steps.size(); ++j) { - dest->append_step(merged, steps[j]); - } - prev_tail = steps.back(); - prev_steps_ptr = &steps; - have_prev = true; - } - - logger.info() << "Merged " << frags.size() << " fragments into \"" - << merged_name << "\"" << endl; - } -} - -// --connect-fragments / -f mode. -// -// Combines multiple graphs and merges paths that are fragments of the same -// logical path. Paths are bucketed by (sense, sample, locus, haplotype, -// circular). phase_block is *not* part of the key: when it varies inside a -// bucket it is interpreted as a start coordinate (the PathMetadata spec -// explicitly allows phase_block to be "a number or a start coordinate"), so -// e.g. `HG002#1#chr22#0` and `HG002#1#chr22#15419374` are recognized as -// fragments of the same logical chr22 and merged. -// -// Each fragment's effective coordinate range on the merged path is derived as: -// - No subrange, no phase_block -> [0, path-length) -// - No subrange, phase_block=s -> [s, s + path-length) (single offset) -// - subrange=[s] -> [s, s + path-length) (single offset) -// - subrange=[s-e] -> [s, e) (range) -// Fragments are then sorted by start, the missing inter-fragment edges are -// added, and the union [min start, max end-exclusive) becomes the merged path's -// subrange. The merged path drops phase_block entirely; if the inputs were -// HAPLOTYPE-sense purely because of phase_block, the merged path becomes -// REFERENCE-sense (or GENERIC if there is no sample). -// -// When `shared_nodes` is true (-s + -f), the per-graph copy preserves source -// node IDs and deduplicates nodes that already exist in the accumulator -// (sequence-verified). The fragment-stitching loop also checks for shared -// boundary handles between adjacent fragments and trims the duplicate steps, -// so the merged path traverses each shared node only once. -int connect_fragments_combine(int argc, char** argv, const Logger& logger, - bool shared_nodes) { - - using handlegraph::nid_t; - - // (1) Load the first graph; it doubles as the destination accumulator. - string first_graph_filename = get_input_file_name(optind, argc, argv); - unique_ptr dest = - vg::io::VPKG::load_one(first_graph_filename); - int64_t max_node_id = dest->max_node_id(); - - // (2) For each subsequent graph: copy nodes/edges into the destination, - // then copy each path with its original (per-fragment) name. The - // fragment merging happens in a second pass. - // - // Default behavior renumbers the incoming graph's IDs out of the way. - // Under -s (shared_nodes) we keep source IDs and dedupe colliding - // nodes by ID after verifying their sequences match. - while (optind < argc) { - unique_ptr graph; - string graph_filename = get_input_file_name(optind, argc, argv); - graph = vg::io::VPKG::load_one(graph_filename); - - if (shared_nodes) { - graph->for_each_handle([&](const handle_t& src_h) { - nid_t id = graph->get_id(src_h); - string src_seq = graph->get_sequence(src_h); - if (dest->has_node(id)) { - if (dest->get_sequence(dest->get_handle(id)) != src_seq) { - logger.error() << "Node " << id << " has conflicting sequences across " - << "inputs; cannot combine in --shared-nodes mode." << endl; - } - // else: shared node already in dest; skip creation. - } else { - dest->create_handle(src_seq, id); - } - }); - graph->for_each_edge([&](const edge_t& src_edge) { - handle_t a = dest->get_handle(graph->get_id(src_edge.first), - graph->get_is_reverse(src_edge.first)); - handle_t b = dest->get_handle(graph->get_id(src_edge.second), - graph->get_is_reverse(src_edge.second)); - if (!dest->has_edge(a, b)) { - dest->create_edge(a, b); - } - }); - max_node_id = std::max(max_node_id, (int64_t)graph->max_node_id()); - } else { - int64_t delta = max_node_id - graph->min_node_id(); - if (delta >= 0) { - graph->increment_node_ids(delta + 1); - } - max_node_id = graph->max_node_id(); - - // Copy nodes + edges only -- paths handled below so we can detect - // name collisions explicitly. - handlealgs::copy_handle_graph(graph.get(), dest.get()); - } - - graph->for_each_path_handle([&](const path_handle_t& src_path) { - string path_name = graph->get_path_name(src_path); - if (dest->has_path(path_name)) { - logger.error() << "Paths with name \"" << path_name - << "\" found in multiple input graphs. Two fragments cannot share " - << "the same subrange suffix; please disambiguate the inputs." << endl; - } - if (shared_nodes) { - // IDs match between graph and dest; replay steps directly. - path_handle_t new_path = dest->create_path( - graph->get_sense(src_path), - graph->get_sample_name(src_path), - graph->get_locus_name(src_path), - graph->get_haplotype(src_path), - graph->get_phase_block(src_path), - graph->get_subrange(src_path), - graph->get_is_circular(src_path) - ); - for (handle_t src_step : graph->scan_path(src_path)) { - handle_t dest_step = dest->get_handle(graph->get_id(src_step), - graph->get_is_reverse(src_step)); - dest->append_step(new_path, dest_step); - } - } else { - handlealgs::copy_path(graph.get(), src_path, dest.get()); - } - }); - } - - // (3) Bucket and merge fragments (shared with -m). - merge_path_fragments(dest.get(), logger); - - // (4) Serialize the merged graph. - vg::io::save_handle_graph(dest.get(), cout); - return 0; -} - - -// This is the original vg combine logic, which itself mimics using "cat" to join up protobuf files -// Since it relies on the Protobuf format itself, particular the ability to stream together chunks that -// would otherwise be invalid individually, it is probably never going to be ported to the handle graph -// api, which is why it's been relegated to the deprecated bin -int cat_proto_graphs(int argc, char** argv, const Logger& logger) { - - while (optind < argc) { - get_input_file(optind, argc, argv, [&](istream& in) { - // We're producing output in uncompressed, "VG"-type-tagged, VPKG Protobuf format. - // We will check if this file is uncompressed or compressed VG-type-tagged data. - - if (vg::io::BlockedGzipInputStream::SmellsLikeGzip(in)) { - // It is compressed. - - // Save our start position - auto start = in.tellg(); - - { - // Try decompressing. - vg::io::BlockedGzipInputStream decompressed(in); - if (decompressed.IsBGZF() && vg::io::MessageIterator::sniff_tag(decompressed) == "VG") { - // We have Blocked GZIP which we can potentially just forward. - // It looks like compressed VG Protobuf data. - - // Decompress it all to stdout, using the ZeroCopyInputStream API. - char* buffer = nullptr; - int bytes = 0; - while (cout && decompressed.Next((const void**) &buffer, &bytes)) { - // Each time we get bytes, write them to stdout. - cout.write(buffer, bytes); - } - - if (!cout) { - logger.error() << "Could not write decompressed data to output stream." << endl; - } - - // Do the next input file - return; - } - } - - // We may have hit EOF. - in.clear(); - - // If we get here, it wasn't compressed VG Protobuf. - // So we need to go back to the start of the file, since the decompressor read some. - in.seekg(start); - - } else if (vg::io::MessageIterator::sniff_tag(in) == "VG") { - // It isn't compressed, but it looks like uncompressed VG Protobuf. - // Send the uncompressed data to stdout. - cout << in.rdbuf(); - - if (!cout) { - logger.error() << "Could not write raw data to output stream." << endl; - } - - // Do the next input file - return; - } - - // If we get here, it isn't compressed or uncompressed VG protobuf. - // Read it as a PathHandleGraph - unique_ptr graph = vg::io::VPKG::load_one(in); - - // Convert to vg::VG - VG* vg_graph = dynamic_cast(graph.get()); - if (vg_graph == nullptr) { - vg_graph = new vg::VG(); - handlealgs::copy_path_handle_graph(graph.get(), vg_graph); - // Give the unique_ptr ownership and delete the graph we loaded. - graph.reset(vg_graph); - // Make sure the paths are all synced up - vg_graph->paths.to_graph(vg_graph->graph); - } - - { - // Save to stdout, uncompressed - vg::io::ProtobufEmitter emitter(cout, false); - vg_graph->serialize_to_emitter(emitter); - // Make sure the emitter goes away and writes before we check on the stream. - } - - if (!cout) { - logger.error() << "Could not write converted graph to output stream." << endl; - } - - }); - } - return 0; -} - - -// Per-input REFERENCE metadata for --merge/-m, shared between the bucket -// stitcher and the top-level driver. -struct MergeInput { - string filename; - unique_ptr graph; - path_handle_t ref_path; - string ref_name; - handle_t ref_start_handle; - handle_t ref_end_handle; - handlegraph::offset_t ref_start_offset = 0; - handlegraph::offset_t ref_end_offset = 0; - string sample; - string locus; - size_t haplotype; - bool is_circular = false; -}; - -// Stitch one bucket of inputs that all share the same REFERENCE identity -// (sample, locus, haplotype, circular) -- i.e. chunks of a single chromosome -- -// into one accumulator graph. This is steps (2)-(4) of --merge/-m: -// -// (2) Sort inputs by REFERENCE start offset. -// (3) Validate the chain boundary on every input: -// - every non-first input's paths all start at its REF start node -// - every non-last input's paths all end at its REF end node -// Boundary nodes must be visited in forward orientation. -// (4) Walk pairs left->right. For each right chunk: -// - overlap = dest_REF_end_offset - src_REF_start_offset -// (error on negative gap or if overlap >= start node length) -// - if overlap > 0: divide_handle(start, overlap), pop_front_step on -// every path in the right chunk, destroy the head piece. The new -// start node is the tail piece. -// - renumber the right chunk's IDs out of dest's range, copy -// nodes/edges/paths into dest under their per-fragment names. -// - connect: if --fusion, glue the left end node and the new right -// start node into a single node with concatenated sequence; -// otherwise just create_edge(left_end, new_right_start). -// -// The returned graph keeps each chunk's per-fragment path names; the caller -// runs merge_path_fragments once across all buckets. A bucket of one input is -// returned as-is (its graph is moved out, no stitching needed). -static unique_ptr -stitch_reference_bucket(std::vector& inputs, const Logger& logger, bool fusion) { - - using handlegraph::PathSense; - using handlegraph::PathMetadata; - using handlegraph::subrange_t; - using handlegraph::offset_t; - using handlegraph::nid_t; - - // (2) Sort by REFERENCE start offset. - std::sort(inputs.begin(), inputs.end(), - [](const MergeInput& a, const MergeInput& b) { - return a.ref_start_offset < b.ref_start_offset; - }); - - // (3) Validate that every chunk's paths share the appropriate boundary. - for (size_t i = 0; i < inputs.size(); ++i) { - auto& in = inputs[i]; - bool need_end_check = (i + 1 < inputs.size()); - bool need_start_check = (i > 0); - in.graph->for_each_path_handle([&](const path_handle_t& p) { - if (need_end_check) { - step_handle_t last_step = in.graph->path_back(p); - handle_t last_h = in.graph->get_handle_of_step(last_step); - if (last_h != in.ref_end_handle) { - logger.error() << "Path \"" << in.graph->get_path_name(p) - << "\" in \"" << in.filename - << "\" does not end at the REFERENCE end node (id " - << in.graph->get_id(in.ref_end_handle) - << "); --merge/-m requires every path in a left/middle " - << "chunk to terminate on the REFERENCE boundary." << endl; - } - } - if (need_start_check) { - step_handle_t first_step = in.graph->path_begin(p); - handle_t first_h = in.graph->get_handle_of_step(first_step); - if (first_h != in.ref_start_handle) { - logger.error() << "Path \"" << in.graph->get_path_name(p) - << "\" in \"" << in.filename - << "\" does not start at the REFERENCE start node (id " - << in.graph->get_id(in.ref_start_handle) - << "); --merge/-m requires every path in a right/middle " - << "chunk to begin on the REFERENCE boundary." << endl; - } - } - }); + // --fuse requires --seam trim. + if (fuse && seam != SeamPolicy::COORD_TRIM) { + logger.error() << "--fuse/-u requires --seam trim." << endl; } - // (4) Pairwise stitch. inputs[0]'s graph becomes the destination accumulator. - unique_ptr dest = std::move(inputs[0].graph); - int64_t max_node_id = dest->max_node_id(); - - // Rightmost boundary handle, tracked in dest's ID space. - handle_t dest_end_handle = inputs[0].ref_end_handle; - offset_t dest_ref_end_offset = inputs[0].ref_end_offset; - - for (size_t i = 1; i < inputs.size(); ++i) { - auto& in = inputs[i]; - unique_ptr src = std::move(in.graph); - - // Capture src's REF endpoint identity BEFORE any mutation (divide / - // increment may invalidate the original handles). - nid_t src_ref_start_id = src->get_id(in.ref_start_handle); - bool src_ref_start_rev = src->get_is_reverse(in.ref_start_handle); - nid_t src_ref_end_id = src->get_id(in.ref_end_handle); - bool src_ref_end_rev = src->get_is_reverse(in.ref_end_handle); - size_t src_start_len = src->get_length(in.ref_start_handle); - - // Length of the left chunk's boundary node (only matters for fusion: - // under -u every right-chunk path's first node grows by this many bp, - // so the path metadata must shift back by this amount so the merged - // subrange end still reflects the original genomic coordinate). - size_t left_boundary_len = dest->get_length(dest_end_handle); - - // Overlap from REF offsets. - offset_t overlap = dest_ref_end_offset - in.ref_start_offset; - if (overlap < 0) { - logger.error() << "REFERENCE offset gap between \"" << inputs[i-1].filename - << "\" (REF ends at offset " << dest_ref_end_offset - << ") and \"" << in.filename << "\" (REF starts at offset " - << in.ref_start_offset << "). --merge/-m does not insert gaps." - << endl; - } - - // Trim the right chunk's start node by `overlap` bp. - nid_t new_start_id_in_src = src_ref_start_id; - if (overlap > 0) { - if ((size_t)overlap > src_start_len) { - logger.error() << "REFERENCE overlap of " << overlap << " bp between \"" - << inputs[i-1].filename << "\" and \"" << in.filename - << "\" exceeds the right chunk's first node length (" - << src_start_len << " bp on node " << src_ref_start_id - << ")." << endl; - } - if ((size_t)overlap == src_start_len) { - logger.error() << "REFERENCE overlap of " << overlap - << " bp exactly equals the right chunk's first node length; " - << "trim would yield a zero-length start node. Re-chunk so the " - << "first node of \"" << in.filename - << "\" extends past the overlap." << endl; - } - // Split the start node. divide_handle keeps paths consistent: - // every path now visits piece_A then piece_B at its head. - auto pieces = src->divide_handle(in.ref_start_handle, - std::vector{(size_t)overlap}); - handle_t piece_A = pieces[0]; - handle_t piece_B = pieces[1]; - new_start_id_in_src = src->get_id(piece_B); - - // Drop piece_A from every path's front. We materialize the path - // list first to avoid iterator invalidation. - std::vector all_paths; - src->for_each_path_handle([&](const path_handle_t& p) { - all_paths.push_back(p); - }); - for (path_handle_t p : all_paths) { - src->pop_front_step(p); - } - - // piece_A has no remaining path steps and (because the boundary - // node is the true left edge of the chunk) no inbound edges. - // Its sole outbound edge piece_A->piece_B vanishes with the node. - src->destroy_handle(piece_A); - } - - // Reconcile each path's offset metadata with what its first node - // actually represents in the destination accumulator: - // - Trimming removes `overlap` bp from the front, so every path - // "starts" `overlap` bp later in its own coordinate system. - // - Under --fusion, the right chunk's first node will gain - // `left_boundary_len` bp on its left (the left chunk's boundary - // sequence is fused in), so the path now "starts" that many bp - // earlier than just-after-trim. - // The combined shift makes fragment_start + path_length come out to - // the correct genomic end after fragment-merge. - int64_t meta_shift = (int64_t)overlap - (fusion ? (int64_t)left_boundary_len : 0); - if (meta_shift != 0) { - struct PathSnapshot { - PathSense sense; - string sample; - string locus; - size_t haplotype; - size_t phase_block; - subrange_t subrange; - bool is_circular; - std::vector steps; - }; - std::vector snapshots; - src->for_each_path_handle([&](const path_handle_t& sp) { - PathSnapshot s; - s.sense = src->get_sense(sp); - s.sample = src->get_sample_name(sp); - s.locus = src->get_locus_name(sp); - s.haplotype = src->get_haplotype(sp); - s.phase_block = src->get_phase_block(sp); - s.subrange = src->get_subrange(sp); - s.is_circular = src->get_is_circular(sp); - for (handle_t h : src->scan_path(sp)) { - s.steps.push_back(h); - } - snapshots.push_back(std::move(s)); - }); - std::vector to_destroy; - src->for_each_path_handle([&](const path_handle_t& sp) { - to_destroy.push_back(sp); - }); - for (path_handle_t sp : to_destroy) { - src->destroy_path(sp); - } - for (auto& s : snapshots) { - if (s.subrange != PathMetadata::NO_SUBRANGE) { - int64_t new_first = (int64_t)s.subrange.first + meta_shift; - if (new_first < 0) new_first = 0; - s.subrange.first = (offset_t)new_first; - } else if (s.phase_block != PathMetadata::NO_PHASE_BLOCK) { - int64_t new_pb = (int64_t)s.phase_block + meta_shift; - if (new_pb < 0) new_pb = 0; - s.phase_block = (size_t)new_pb; - } else { - int64_t new_first = meta_shift; - if (new_first < 0) new_first = 0; - s.subrange = subrange_t{(offset_t)new_first, - PathMetadata::NO_END_POSITION}; - } - path_handle_t new_p = src->create_path( - s.sense, s.sample, s.locus, s.haplotype, - s.phase_block, s.subrange, s.is_circular); - for (handle_t h : s.steps) { - src->append_step(new_p, h); - } - } - } - - // Renumber src out of dest's range. - int64_t shift = 0; - int64_t delta = max_node_id - (int64_t)src->min_node_id(); - if (delta >= 0) { - src->increment_node_ids(delta + 1); - shift = delta + 1; - } - max_node_id = std::max(max_node_id, (int64_t)src->max_node_id()); - - // Copy nodes/edges, then each path under its original (per-fragment) - // name. The fragment-merge pass at the end re-collapses by metadata. - handlealgs::copy_handle_graph(src.get(), dest.get()); - src->for_each_path_handle([&](const path_handle_t& src_path) { - string path_name = src->get_path_name(src_path); - if (dest->has_path(path_name)) { - logger.error() << "Path \"" << path_name - << "\" exists in multiple inputs to --merge/-m; " - << "fragments must carry distinct subrange-suffixed names." - << endl; - } - handlealgs::copy_path(src.get(), src_path, dest.get()); - }); - - // Locate the right chunk's new start and REF end in dest's ID space. - handle_t src_new_start_in_dest = - dest->get_handle((nid_t)((int64_t)new_start_id_in_src + shift), src_ref_start_rev); - handle_t src_ref_end_in_dest = - dest->get_handle((nid_t)((int64_t)src_ref_end_id + shift), src_ref_end_rev); - - if (fusion) { - // Glue dest_end_handle (left chunk's end) and src_new_start_in_dest - // (right chunk's trimmed start) into a single node with the - // concatenated forward-strand sequence. Both are visited only in - // forward orientation by the paths that touch them (we asserted - // forward boundaries above), so the concatenation order matches - // what every path expects. - string fused_seq = dest->get_sequence(dest_end_handle) - + dest->get_sequence(src_new_start_in_dest); - handle_t fused = dest->create_handle(fused_seq); - max_node_id = std::max(max_node_id, (int64_t)dest->get_id(fused)); - - // Preserve every edge that wasn't the implicit seam. - // - inbound to the left side of dest_end_handle -> inbound to fused - // - outbound from the right side of src_new_start_in_dest -> - // outbound from fused - std::vector preds; - dest->follow_edges(dest_end_handle, true, [&](const handle_t& p) { - preds.push_back(p); - }); - std::vector succs; - dest->follow_edges(src_new_start_in_dest, false, [&](const handle_t& n) { - succs.push_back(n); - }); - for (handle_t p : preds) { - if (!dest->has_edge(p, fused)) dest->create_edge(p, fused); - } - for (handle_t n : succs) { - if (!dest->has_edge(fused, n)) dest->create_edge(fused, n); - } - - // Rewrite every step on the two originals to point at fused, - // preserving the step's orientation defensively (we already - // asserted forward but the code stays correct either way). - auto replace_steps_on = [&](handle_t old_h) { - std::vector steps; - dest->for_each_step_on_handle(old_h, [&](const step_handle_t& s) { - steps.push_back(s); - }); - for (const step_handle_t& s : steps) { - handle_t h = dest->get_handle_of_step(s); - handle_t replacement = dest->get_is_reverse(h) - ? dest->flip(fused) : fused; - step_handle_t next = dest->get_next_step(s); - dest->rewrite_segment(s, next, std::vector{replacement}); - } - }; - replace_steps_on(dest_end_handle); - replace_steps_on(src_new_start_in_dest); - - // Destroy originals (also removes any stray edges we didn't pick up). - dest->destroy_handle(dest_end_handle); - dest->destroy_handle(src_new_start_in_dest); - - // If the right chunk's REF is a single node, the new boundary IS - // the fused node; otherwise it's the right chunk's REF end. - dest_end_handle = (src_ref_end_id == new_start_id_in_src) - ? fused - : src_ref_end_in_dest; - } else { - if (!dest->has_edge(dest_end_handle, src_new_start_in_dest)) { - dest->create_edge(dest_end_handle, src_new_start_in_dest); - } - dest_end_handle = src_ref_end_in_dest; - } - - // The right chunk's REF end offset becomes the new dest end offset. - // (Trimming only removed bases from the front, so the end is unchanged.) - dest_ref_end_offset = in.ref_end_offset; - - logger.info() << "Stitched \"" << in.filename << "\": overlap=" << overlap - << " bp, " << (fusion ? "fused" : "edged") << " boundary." - << endl; - } - - return dest; -} - -// --merge / -m mode (optionally with --fusion / -u). -// -// Combines chunked-assembly graphs that share PathSense::REFERENCE paths -// (e.g. CHM13#0#chr22 / CHM13#0#chr22[17475777]) and overlap on them. The -// REFERENCE-path offsets pin each chunk to a coordinate; the overlap between -// adjacent chunks is computed from those offsets and trimmed off the right -// chunk's leading node before stitching. -// -// Inputs may describe more than one reference (e.g. some chunks of CHM13#0#chr21 -// and some of CHM13#0#chr22). They are bucketed by REFERENCE identity (sample, -// locus, haplotype, circular); each bucket is stitched independently, then the -// per-bucket graphs are combined into one output (node IDs renumbered to avoid -// collisions, no edges added between distinct references). The final output -// carries one REFERENCE path per chromosome (e.g. both CHM13#0#chr21 and -// CHM13#0#chr22). -// -// Pipeline: -// (1) Load all inputs and locate each graph's unique REFERENCE-sense path. -// Require exactly one REFERENCE path per graph. -// (2) Bucket inputs by REFERENCE identity. -// (3) Stitch each bucket (see stitch_reference_bucket: sort, validate -// boundaries, pairwise trim+connect/fuse), then copy the stitched bucket -// graphs into a single destination, renumbering IDs as needed. -// (4) Run the same fragment-merge pass as --connect-fragments to collapse -// each reference's chunk paths into one coordinate-spanning path. Because -// that pass buckets by (sense, sample, locus, haplotype, circular), -// different chromosomes stay separate. -int merge_combine(int argc, char** argv, const Logger& logger, bool fusion) { - - using handlegraph::PathSense; - using handlegraph::PathMetadata; - using handlegraph::subrange_t; - using handlegraph::offset_t; - using handlegraph::nid_t; - - // (1) Load all inputs. - std::vector inputs; + vector inputs; while (optind < argc) { - MergeInput in; - in.filename = get_input_file_name(optind, argc, argv); - in.graph = vg::io::VPKG::load_one(in.filename); + GraphCombiner::Input in; + in.name = get_input_file_name(optind, argc, argv); + in.graph = vg::io::VPKG::load_one(in.name); inputs.push_back(std::move(in)); } - if (inputs.size() < 2) { - logger.error() << "--merge/-m requires at least two input graphs." << endl; - } - - // Find the unique REFERENCE path in each graph and pull its endpoint and - // offset metadata. - for (auto& in : inputs) { - std::vector refs; - in.graph->for_each_path_handle([&](const path_handle_t& p) { - if (in.graph->get_sense(p) == PathSense::REFERENCE) { - refs.push_back(p); - } - }); - if (refs.empty()) { - logger.error() << "No REFERENCE-sense path found in \"" << in.filename - << "\"; --merge/-m requires exactly one per graph." << endl; - } - if (refs.size() > 1) { - logger.error() << refs.size() << " REFERENCE-sense paths found in \"" - << in.filename << "\"; --merge/-m requires exactly one." << endl; - } - in.ref_path = refs[0]; - in.ref_name = in.graph->get_path_name(in.ref_path); - in.sample = in.graph->get_sample_name(in.ref_path); - in.locus = in.graph->get_locus_name(in.ref_path); - in.haplotype = in.graph->get_haplotype(in.ref_path); - in.is_circular = in.graph->get_is_circular(in.ref_path); - subrange_t sr = in.graph->get_subrange(in.ref_path); - size_t pb = in.graph->get_phase_block(in.ref_path); - if (sr != PathMetadata::NO_SUBRANGE) { - in.ref_start_offset = sr.first; - } else if (pb != PathMetadata::NO_PHASE_BLOCK) { - in.ref_start_offset = (offset_t)pb; - } else { - in.ref_start_offset = 0; - } - size_t ref_len = compute_path_length(in.graph.get(), in.ref_path); - in.ref_end_offset = in.ref_start_offset + (offset_t)ref_len; - - step_handle_t first_step = in.graph->path_begin(in.ref_path); - step_handle_t last_step = in.graph->path_back(in.ref_path); - in.ref_start_handle = in.graph->get_handle_of_step(first_step); - in.ref_end_handle = in.graph->get_handle_of_step(last_step); - - if (in.graph->get_is_reverse(in.ref_start_handle) - || in.graph->get_is_reverse(in.ref_end_handle)) { - logger.error() << "REFERENCE path \"" << in.ref_name << "\" in \"" - << in.filename << "\" visits its boundary node in reverse " - << "orientation; --merge/-m only supports forward-strand boundaries." - << endl; - } - } - - // (2) Bucket inputs by REFERENCE identity (sample, locus, haplotype, - // circular). Chunks of the same chromosome land in one bucket; different - // chromosomes land in different buckets and are stitched separately. - // bucket_order preserves first-seen order for deterministic output. - using BucketKey = std::tuple; - std::map> buckets; - std::vector bucket_order; - for (size_t i = 0; i < inputs.size(); ++i) { - BucketKey key{inputs[i].sample, inputs[i].locus, - inputs[i].haplotype, inputs[i].is_circular}; - if (!buckets.count(key)) { - bucket_order.push_back(key); - } - buckets[key].push_back(i); - } - - // (3) Stitch each bucket, then combine the per-bucket graphs into a single - // destination accumulator. The first bucket's stitched graph is the - // destination; each subsequent one is renumbered out of the way and its - // nodes/edges/paths copied in. No edges are created between buckets, so - // distinct references remain disconnected components. - unique_ptr dest; - int64_t max_node_id = 0; - for (const auto& key : bucket_order) { - std::vector bucket; - bucket.reserve(buckets[key].size()); - for (size_t i : buckets[key]) { - bucket.push_back(std::move(inputs[i])); - } - - unique_ptr stitched = - stitch_reference_bucket(bucket, logger, fusion); - - if (!dest) { - dest = std::move(stitched); - max_node_id = dest->max_node_id(); - continue; - } - - int64_t delta = max_node_id - (int64_t)stitched->min_node_id(); - if (delta >= 0) { - stitched->increment_node_ids(delta + 1); - } - max_node_id = std::max(max_node_id, (int64_t)stitched->max_node_id()); - - handlealgs::copy_handle_graph(stitched.get(), dest.get()); - stitched->for_each_path_handle([&](const path_handle_t& src_path) { - string path_name = stitched->get_path_name(src_path); - if (dest->has_path(path_name)) { - logger.error() << "Path \"" << path_name - << "\" exists in multiple --merge/-m references; inputs for " - << "different chromosomes must not share path names." << endl; - } - handlealgs::copy_path(stitched.get(), src_path, dest.get()); - }); - } - - // (4) Bucket and merge fragments into single coordinate-spanning paths - // (one per reference). - merge_path_fragments(dest.get(), logger); - - // (5) Serialize. - vg::io::save_handle_graph(dest.get(), cout); + GraphCombiner combiner(seam, fuse, phase_block_is_offset); + unique_ptr combined = + combiner.combine(std::move(inputs)); + vg::io::save_handle_graph(combined.get(), cout); return 0; } + +// Register subcommand +static Subcommand vg_combine("combine", "merge multiple graph files together", main_combine); diff --git a/src/unittest/combine.cpp b/src/unittest/combine.cpp new file mode 100644 index 00000000000..c1223ae7b77 --- /dev/null +++ b/src/unittest/combine.cpp @@ -0,0 +1,1100 @@ +/// \file combine.cpp +/// +/// Unit tests for GraphCombiner's internals. +/// +/// GraphCombiner::combine() is covered end to end by test/t/51_vg_combine.t. +/// These tests drive the steps underneath it, whose intermediate results -- the ID +/// shifts, the trimmed boundary, the merged coordinates -- the command line cannot see. +/// +/// Error branches are absent on purpose: logger.error() exits the process, which would +/// take the whole test binary with it. They live in test/t/51_vg_combine.t. + +#include "catch.hpp" +#include "../combine.hpp" + +#include "bdsg/hash_graph.hpp" +#include "bdsg/overlays/path_position_overlays.hpp" + +namespace vg { +namespace unittest { + +using namespace std; +using handlegraph::nid_t; +using handlegraph::offset_t; +using handlegraph::PathMetadata; +using handlegraph::PathSense; +using handlegraph::subrange_t; + +// A child class to expose the internals for testing. +class TestGraphCombiner : public GraphCombiner { +public: + using GraphCombiner::GraphCombiner; + using GraphCombiner::Chunk; + using GraphCombiner::SeamState; + using GraphCombiner::RefKey; + using GraphCombiner::GroupKey; + using GraphCombiner::compute_path_length; + using GraphCombiner::path_start_offset; + using GraphCombiner::renumber_out_of_way; + using GraphCombiner::group_key_of; + using GraphCombiner::path_prefix; + using GraphCombiner::path_suffix; + using GraphCombiner::check_no_split_paths; + using GraphCombiner::describe_reference; + using GraphCombiner::sort_by_reference_offset; + using GraphCombiner::check_overlaps_agree; + using GraphCombiner::check_all_paths_start_at; + using GraphCombiner::check_all_paths_end_at; + using GraphCombiner::trim_and_shift; + using GraphCombiner::ingest; + using GraphCombiner::share_nodes_by_id; + using GraphCombiner::copy_paths_checked; + using GraphCombiner::connect_seam; + using GraphCombiner::merge_path_fragments; + using GraphCombiner::dest; + using GraphCombiner::max_node_id; + using GraphCombiner::seams; +}; + +/// Spell out the sequence a path visits, so tests can assert on what a path +/// means rather than on which nodes happen to carry it. +static string spell_path(const PathHandleGraph& graph, const string& name) { + string sequence; + path_handle_t path = graph.get_path_handle(name); + for (handle_t h : graph.scan_path(path)) { + sequence += graph.get_sequence(h); + } + return sequence; +} + +/// Make a chunk holding an empty HashGraph. +static TestGraphCombiner::Chunk make_chunk(const string& name) { + TestGraphCombiner::Chunk chunk; + chunk.name = name; + chunk.graph = make_unique(); + return chunk; +} + +/// Create a path with full metadata and walk it over the given handles. +static path_handle_t add_path(MutablePathHandleGraph& graph, const PathSense& sense, + const string& sample, const string& locus, size_t haplotype, + size_t phase_block, const subrange_t& subrange, + const vector& steps, bool is_circular = false) { + path_handle_t path = graph.create_path(sense, sample, locus, haplotype, phase_block, + subrange, is_circular); + for (handle_t h : steps) { + graph.append_step(path, h); + } + return path; +} + +/// Create a REFERENCE-sense path, the sense --seam trim positions a chunk by. +static path_handle_t add_ref_path(MutablePathHandleGraph& graph, const string& sample, + const string& locus, const subrange_t& subrange, + const vector& steps) { + return add_path(graph, PathSense::REFERENCE, sample, locus, 0, + PathMetadata::NO_PHASE_BLOCK, subrange, steps); +} + +/// Collect a graph's path names, sorted so tests do not depend on iteration order. +static vector sorted_path_names(const PathHandleGraph& graph) { + vector names; + graph.for_each_path_handle([&](const path_handle_t& p) { + names.push_back(graph.get_path_name(p)); + }); + std::sort(names.begin(), names.end()); + return names; +} + +TEST_CASE("compute_path_length adds up what a path visits", "[combine]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGGG"); + graph.create_edge(n1, n2); + graph.create_edge(n2, n3); + add_ref_path(graph, "GRCh38", "chr1", subrange_t{0, 11}, {n1, n2, n3}); + + SECTION("a graph without positions is scanned step by step") { + // HashGraph is not a PathPositionHandleGraph, so this takes the scan_path branch. + path_handle_t path = graph.get_path_handle("GRCh38#0#chr1[0-11]"); + REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 11); + } + + SECTION("a positional graph is asked directly, and agrees") { + bdsg::PositionOverlay positional(&graph); + path_handle_t path = positional.get_path_handle("GRCh38#0#chr1[0-11]"); + REQUIRE(TestGraphCombiner::compute_path_length(&positional, path) == 11); + } + + SECTION("an empty path has no length") { + graph.create_path_handle("empty"); + path_handle_t path = graph.get_path_handle("empty"); + REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 0); + } + + SECTION("a step's orientation does not change how much it contributes") { + handle_t n4 = graph.create_handle("TTT"); + path_handle_t path = graph.create_path_handle("flipped"); + graph.append_step(path, graph.flip(n4)); + REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 3); + } +} + +TEST_CASE("path_start_offset reads a start offset off path metadata", "[combine]") { + SECTION("a subrange gives the offset outright") { + REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, 200}, 7, true) == 100); + REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, 200}, 7, false) == 100); + } + + SECTION("a subrange wins over a phase block even when blocks are offsets") { + REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, PathMetadata::NO_END_POSITION}, + 7, true) == 100); + } + + SECTION("without a subrange, -P reads the phase block as the offset") { + REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, 7, true) == 7); + } + + SECTION("without -P a phase block is an opaque identifier, not a position") { + REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, 7, false) == 0); + } + + SECTION("no subrange and no phase block starts at zero") { + REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, + PathMetadata::NO_PHASE_BLOCK, true) == 0); + REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, + PathMetadata::NO_PHASE_BLOCK, false) == 0); + } +} + +TEST_CASE("renumber_out_of_way shifts node IDs so they do not collide with IDs already used", "[combine]") { + SECTION("IDs already above the mark are left alone") { + bdsg::HashGraph graph; + graph.create_handle("A", 10); + graph.create_handle("C", 12); + REQUIRE(TestGraphCombiner::renumber_out_of_way(&graph, 3) == 0); + REQUIRE(graph.min_node_id() == 10); + REQUIRE(graph.max_node_id() == 12); + } + + SECTION("overlapping IDs are shifted clear, and stay clear") { + bdsg::HashGraph graph; + graph.create_handle("A", 1); + graph.create_handle("C", 5); + int64_t shift = TestGraphCombiner::renumber_out_of_way(&graph, 10); + REQUIRE(shift == 10); + REQUIRE(graph.min_node_id() == 11); + REQUIRE(graph.max_node_id() == 15); + } + + SECTION("a graph starting exactly on the mark still moves off it") { + // delta is 0 here, which is not negative, so the shift is 1: IDs have to end + // up strictly above the mark, not on it. + bdsg::HashGraph graph; + graph.create_handle("A", 7); + REQUIRE(TestGraphCombiner::renumber_out_of_way(&graph, 7) == 1); + REQUIRE(graph.min_node_id() == 8); + } + + SECTION("the shift the caller gets back is the one that was applied") { + bdsg::HashGraph graph; + graph.create_handle("A", 4); + graph.create_handle("C", 9); + int64_t shift = TestGraphCombiner::renumber_out_of_way(&graph, 20); + REQUIRE(graph.get_id(graph.get_handle(4 + shift)) == 4 + shift); + REQUIRE(graph.has_node(9 + shift)); + REQUIRE(!graph.has_node(4)); + } +} + +TEST_CASE("group_key_of decides which paths are pieces of the same thing", "[combine]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAAA"); + + add_path(graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 10, + PathMetadata::NO_SUBRANGE, {n1}); + + path_handle_t block_0 = graph.get_path_handle("HG002#1#chr1#0"); + path_handle_t block_10 = graph.get_path_handle("HG002#1#chr1#10"); + + SECTION("phase blocks tell paths apart by default") { + REQUIRE(TestGraphCombiner::group_key_of(graph, block_0, false) + != TestGraphCombiner::group_key_of(graph, block_10, false)); + } + + SECTION("-P folds the phase block out, so the two become one group") { + REQUIRE(TestGraphCombiner::group_key_of(graph, block_0, true) + == TestGraphCombiner::group_key_of(graph, block_10, true)); + // Folded out means replaced by the sentinel, not kept as some other value. + REQUIRE(std::get<5>(TestGraphCombiner::group_key_of(graph, block_0, true)) + == PathMetadata::NO_PHASE_BLOCK); + } + + SECTION("a subrange is not part of the key, so fragments group together") { + add_path(graph, PathSense::REFERENCE, "GRCh38", "chr1", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}); + add_path(graph, PathSense::REFERENCE, "GRCh38", "chr1", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 8}, {n1}); + REQUIRE(TestGraphCombiner::group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[0-4]"), false) + == TestGraphCombiner::group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[4-8]"), false)); + } + + SECTION("each identity field on its own is enough to split the group") { + add_path(graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, + PathMetadata::NO_SUBRANGE, {n1}); // other haplotype + add_path(graph, PathSense::HAPLOTYPE, "HG002", "chr2", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); // other locus + add_path(graph, PathSense::HAPLOTYPE, "HG005", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); // other sample + add_path(graph, PathSense::REFERENCE, "HG002", "chr1", 1, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}); // other sense + + auto key = [&](const string& name) { + return TestGraphCombiner::group_key_of(graph, graph.get_path_handle(name), false); + }; + REQUIRE(key("HG002#1#chr1#0") != key("HG002#2#chr1#0")); + REQUIRE(key("HG002#1#chr1#0") != key("HG002#1#chr2#0")); + REQUIRE(key("HG002#1#chr1#0") != key("HG005#1#chr1#0")); + REQUIRE(key("HG002#1#chr1#0") != key("HG002#1#chr1[0-4]")); + } +} + +TEST_CASE("path_prefix and path_suffix read the ends of a path", "[combine]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGTT"); + graph.create_edge(n1, n2); + graph.create_edge(n2, n3); + path_handle_t path = add_ref_path(graph, "GRCh38", "chr1", subrange_t{0, 11}, + {n1, n2, n3}); + + SECTION("a span inside the first or last node stops there") { + REQUIRE(TestGraphCombiner::path_prefix(graph, path, 3) == "AAA"); + REQUIRE(TestGraphCombiner::path_suffix(graph, path, 3) == "GTT"); + } + + SECTION("a span crossing node boundaries keeps reading") { + REQUIRE(TestGraphCombiner::path_prefix(graph, path, 7) == "AAAACCG"); + REQUIRE(TestGraphCombiner::path_suffix(graph, path, 7) == "CCGGGTT"); + } + + SECTION("asking for more than the path holds yields all of it") { + REQUIRE(TestGraphCombiner::path_prefix(graph, path, 20) == "AAAACCGGGTT"); + REQUIRE(TestGraphCombiner::path_suffix(graph, path, 20) == "AAAACCGGGTT"); + } + + SECTION("asking for nothing yields nothing, even on an empty path") { + path_handle_t empty = add_ref_path(graph, "GRCh38", "chr2", subrange_t{0, 0}, {}); + REQUIRE(TestGraphCombiner::path_prefix(graph, path, 0) == ""); + REQUIRE(TestGraphCombiner::path_suffix(graph, path, 0) == ""); + REQUIRE(TestGraphCombiner::path_prefix(graph, empty, 5) == ""); + REQUIRE(TestGraphCombiner::path_suffix(graph, empty, 5) == ""); + } + + SECTION("a reverse step reads in the path's orientation") { + handle_t n4 = graph.create_handle("TTGCA"); + graph.create_edge(n3, graph.flip(n4)); + path_handle_t flipped = add_ref_path(graph, "GRCh38", "chr3", subrange_t{0, 16}, + {n1, n2, n3, graph.flip(n4)}); + REQUIRE(TestGraphCombiner::path_suffix(graph, flipped, 6) == "TTGCAA"); + } +} + +TEST_CASE("check_no_split_paths passes graphs whose paths are each whole", "[combine]") { + // Paths that merely look alike are not pieces of one path. + // Rejection exits the process, so it is tested in test/t/51_vg_combine.t. + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + + SECTION("a single path is not a split path") { + TestGraphCombiner::Chunk chunk = make_chunk("one.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 1); + } + + SECTION("paths differing only by haplotype are separate paths") { + TestGraphCombiner::Chunk chunk = make_chunk("haps.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{0, 4}, {n1}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, + subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); + } + + SECTION("paths differing only by locus are separate paths") { + TestGraphCombiner::Chunk chunk = make_chunk("loci.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*chunk.graph, "GRCh38", "chr2", subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); + } + + SECTION("phase blocks are separate paths until -P says otherwise") { + TestGraphCombiner::Chunk chunk = make_chunk("blocks.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 10, + PathMetadata::NO_SUBRANGE, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); + } +} + +TEST_CASE("describe_reference reads a chunk's position off its REFERENCE path", "[combine]") { + + SECTION("every field is filled in from the path's metadata") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("left.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CC"); + handle_t n3 = chunk.graph->create_handle("GGGGG"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n2, n3); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 111}, {n1, n2, n3}); + // A haplotype alongside it must not be mistaken for the reference. + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 111}, {n1, n2, n3}); + + combiner.describe_reference(chunk); + + REQUIRE(chunk.ref_name == "GRCh38#0#chr1[100-111]"); + REQUIRE(chunk.ref_key == TestGraphCombiner::RefKey{"GRCh38", "chr1", 0, false}); + REQUIRE(chunk.ref_start_offset == 100); + // The end is the start plus the path's own length, not the subrange's end. + REQUIRE(chunk.ref_end_offset == 111); + REQUIRE(chunk.graph->get_id(chunk.ref_start_handle) == chunk.graph->get_id(n1)); + REQUIRE(chunk.graph->get_id(chunk.ref_end_handle) == chunk.graph->get_id(n3)); + REQUIRE(!chunk.graph->get_is_reverse(chunk.ref_start_handle)); + REQUIRE(!chunk.graph->get_is_reverse(chunk.ref_end_handle)); + } + + SECTION("the end offset follows the sequence, not the subrange's claim") { + // A subrange claiming 100-999 over 11bp of sequence still ends at 111. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("claim.pg"); + handle_t n1 = chunk.graph->create_handle("AAAAAAAAAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 999}, {n1}); + + combiner.describe_reference(chunk); + REQUIRE(chunk.ref_start_offset == 100); + REQUIRE(chunk.ref_end_offset == 111); + } + + SECTION("a single-node chunk starts and ends on the same node") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("solo.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + + combiner.describe_reference(chunk); + REQUIRE(chunk.ref_start_handle == chunk.ref_end_handle); + REQUIRE(chunk.ref_start_offset == 0); + REQUIRE(chunk.ref_end_offset == 4); + } + + SECTION("a reference with no subrange starts at zero, with or without -P") { + // A REFERENCE-sense path may not carry a phase block, so -P has nothing to + // read here and the chunk can only be placed at the origin. + for (bool phase_block_is_offset : {false, true}) { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, phase_block_is_offset); + TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", PathMetadata::NO_SUBRANGE, {n1}); + + combiner.describe_reference(chunk); + REQUIRE(chunk.ref_start_offset == 0); + REQUIRE(chunk.ref_end_offset == 4); + } + } + + SECTION("a circular reference is a different reference from a linear one") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("circle.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); + + combiner.describe_reference(chunk); + REQUIRE(chunk.ref_key == TestGraphCombiner::RefKey{"GRCh38", "chrM", 0, true}); + } +} + +TEST_CASE("sort_by_reference_offset groups references and orders each by offset", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + + // Build chunks by hand: the sort reads only ref_key and ref_start_offset, so + // these need no graphs. + auto positioned = [](const string& name, const string& locus, offset_t start) { + TestGraphCombiner::Chunk chunk; + chunk.name = name; + chunk.ref_key = TestGraphCombiner::RefKey{"GRCh38", locus, 0, false}; + chunk.ref_start_offset = start; + return chunk; + }; + // Same, for chunks whose reference differs in something other than locus. + auto keyed = [](const string& name, const TestGraphCombiner::RefKey& ref_key, + offset_t start) { + TestGraphCombiner::Chunk chunk; + chunk.name = name; + chunk.ref_key = ref_key; + chunk.ref_start_offset = start; + return chunk; + }; + + SECTION("chunks of one reference come out in ascending offset order") { + vector chunks; + chunks.push_back(positioned("c", "chr1", 200)); + chunks.push_back(positioned("a", "chr1", 0)); + chunks.push_back(positioned("b", "chr1", 100)); + + combiner.sort_by_reference_offset(chunks); + REQUIRE(chunks[0].name == "a"); + REQUIRE(chunks[1].name == "b"); + REQUIRE(chunks[2].name == "c"); + } + + SECTION("references keep first-seen order, and their chunks stay contiguous") { + // chr2 is seen first, so it stays first even though chr1 sorts earlier. + vector chunks; + chunks.push_back(positioned("y2", "chr2", 100)); + chunks.push_back(positioned("x2", "chr1", 100)); + chunks.push_back(positioned("y1", "chr2", 0)); + chunks.push_back(positioned("x1", "chr1", 0)); + + combiner.sort_by_reference_offset(chunks); + REQUIRE(chunks[0].name == "y1"); + REQUIRE(chunks[1].name == "y2"); + REQUIRE(chunks[2].name == "x1"); + REQUIRE(chunks[3].name == "x2"); + } + + SECTION("equal offsets keep the order they arrived in") { + vector chunks; + chunks.push_back(positioned("first", "chr1", 50)); + chunks.push_back(positioned("second", "chr1", 50)); + chunks.push_back(positioned("third", "chr1", 50)); + + combiner.sort_by_reference_offset(chunks); + REQUIRE(chunks[0].name == "first"); + REQUIRE(chunks[1].name == "second"); + REQUIRE(chunks[2].name == "third"); + } + + SECTION("haplotype and circularity make separate references of one locus") { + // All four share sample and locus, so only the other two RefKey fields can + // tell the three references apart. + vector chunks; + chunks.push_back(keyed("hap2", {"GRCh38", "chr1", 2, false}, 50)); + chunks.push_back(positioned("hap0_late", "chr1", 100)); + chunks.push_back(positioned("hap0_early", "chr1", 0)); + chunks.push_back(keyed("circular", {"GRCh38", "chr1", 0, true}, 50)); + + combiner.sort_by_reference_offset(chunks); + // References lead in first-seen order; hap0's two chunks follow in offset order. + REQUIRE(chunks[0].name == "hap2"); + REQUIRE(chunks[1].name == "hap0_early"); + REQUIRE(chunks[2].name == "hap0_late"); + REQUIRE(chunks[3].name == "circular"); + } +} + +TEST_CASE("check_all_paths_start_at and check_all_paths_end_at pass chunks with flush boundaries", "[combine]") { + // Legitimate chunks are not turned away. + // Rejection exits the process, so it is tested in test/t/51_vg_combine.t. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + + TestGraphCombiner::Chunk chunk = make_chunk("flush.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("C"); + handle_t n3 = chunk.graph->create_handle("G"); + handle_t n4 = chunk.graph->create_handle("TTTTT"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n1, n3); + chunk.graph->create_edge(n2, n4); + chunk.graph->create_edge(n3, n4); + // Three paths through a bubble: they take different alleles but share both ends. + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 10}, {n1, n2, n4}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{0, 10}, {n1, n3, n4}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, + subrange_t{0, 10}, {n1, n2, n4}); + chunk.ref_start_handle = n1; + chunk.ref_end_handle = n4; + + SECTION("all paths beginning on the boundary node are accepted") { + combiner.check_all_paths_start_at(chunk); + REQUIRE(chunk.graph->get_path_count() == 3); + } + + SECTION("all paths ending on the boundary node are accepted") { + combiner.check_all_paths_end_at(chunk); + REQUIRE(chunk.graph->get_path_count() == 3); + } + + SECTION("a boundary the paths cross in reverse is matched by orientation too") { + // The check compares handles, so a reverse boundary needs reverse steps. + TestGraphCombiner::Chunk flipped = make_chunk("flipped.pg"); + handle_t a = flipped.graph->create_handle("AAAA"); + handle_t b = flipped.graph->create_handle("CCC"); + flipped.graph->create_edge(flipped.graph->flip(a), b); + add_ref_path(*flipped.graph, "GRCh38", "chr1", subrange_t{0, 7}, + {flipped.graph->flip(a), b}); + flipped.ref_start_handle = flipped.graph->flip(a); + flipped.ref_end_handle = b; + + combiner.check_all_paths_start_at(flipped); + combiner.check_all_paths_end_at(flipped); + REQUIRE(flipped.graph->get_path_count() == 1); + } +} + +TEST_CASE("trim_and_shift cuts the overlap away and moves coordinates with it", "[combine]") { + + SECTION("no overlap and no fuse leaves the chunk exactly as it was") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("clean.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); + chunk.ref_start_handle = n1; + nid_t before = chunk.graph->get_id(n1); + + nid_t after = combiner.trim_and_shift(chunk, 0, 5); + + REQUIRE(after == before); + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[100-107]"}); + } + + SECTION("no overlap but fusing pulls every coordinate back by the welded node") { + // Welding buries the left chunk's last node inside this chunk's first, so + // this chunk's coordinates move back by that node's length. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + TestGraphCombiner::Chunk chunk = make_chunk("weld.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); + chunk.ref_start_handle = n1; + + combiner.trim_and_shift(chunk, 0, 5); + + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[95-107]"}); + } + + SECTION("an overlap splits the first node, drops the front, and shifts coordinates") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("overlap.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + handle_t n2 = chunk.graph->create_handle("GGG"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 109}, {n1, n2}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 109}, {n1, n2}); + chunk.ref_start_handle = n1; + + nid_t new_start = combiner.trim_and_shift(chunk, 4, 5); + + // The overlapping "AAAA" is gone; "CC" remains and is the new start. + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "CC"); + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-109]") == "CCGGG"); + REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-109]") == "CCGGG"); + } + + SECTION("an overlap of one base still leaves the rest of the node behind") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("thin.pg"); + handle_t n1 = chunk.graph->create_handle("AC"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 102}, {n1}); + chunk.ref_start_handle = n1; + + nid_t new_start = combiner.trim_and_shift(chunk, 1, 5); + + REQUIRE(chunk.graph->get_node_count() == 1); + REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "C"); + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[101-102]") == "C"); + } + + SECTION("with -P the phase block moves instead of a subrange") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, true); + TestGraphCombiner::Chunk chunk = make_chunk("pblock.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 100, + PathMetadata::NO_SUBRANGE, {n1}); + chunk.ref_start_handle = n1; + + combiner.trim_and_shift(chunk, 4, 5); + + REQUIRE(sorted_path_names(*chunk.graph) == vector{"HG002#1#chr1#104"}); + } + + SECTION("a path with no coordinates at all is given a subrange to carry the shift") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + add_path(*chunk.graph, PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, "scaffold", + PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, + PathMetadata::NO_SUBRANGE, {n1}); + chunk.ref_start_handle = n1; + + combiner.trim_and_shift(chunk, 4, 0); + + REQUIRE(sorted_path_names(*chunk.graph) == vector{"scaffold[4]"}); + } + + SECTION("a shift that would run past zero stops at zero") { + // Fusing pulls coordinates back by 50, but the path only starts at 10. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + TestGraphCombiner::Chunk chunk = make_chunk("clamp.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{10, 14}, {n1}); + chunk.ref_start_handle = n1; + + combiner.trim_and_shift(chunk, 0, 50); + + REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[0-14]"}); + } + + SECTION("trimming keeps each path on its own alleles through a bubble") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("bubble.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + handle_t n2 = chunk.graph->create_handle("G"); + handle_t n3 = chunk.graph->create_handle("T"); + handle_t n4 = chunk.graph->create_handle("TTTTT"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n1, n3); + chunk.graph->create_edge(n2, n4); + chunk.graph->create_edge(n3, n4); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 112}, {n1, n2, n4}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 112}, {n1, n3, n4}); + chunk.ref_start_handle = n1; + + combiner.trim_and_shift(chunk, 4, 0); + + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-112]") == "CCGTTTTT"); + REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-112]") == "CCTTTTTT"); + } +} + +TEST_CASE("ingest moves a chunk into the accumulator", "[combine]") { + + SECTION("the first chunk becomes the accumulator untouched") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("first.pg"); + chunk.graph->create_handle("AAAA", 5); + chunk.graph->create_handle("CCC", 9); + + int64_t shift = combiner.ingest(chunk); + + REQUIRE(shift == 0); + REQUIRE(combiner.max_node_id == 9); + REQUIRE(combiner.dest != nullptr); + REQUIRE(combiner.dest->has_node(5)); + REQUIRE(combiner.dest->has_node(9)); + // The graph was moved out of the chunk, not copied. + REQUIRE(chunk.graph == nullptr); + } + + SECTION("renumber lifts a second chunk's IDs clear of the first") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk left = make_chunk("left.pg"); + handle_t l1 = left.graph->create_handle("AAAA", 1); + add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 4}, {l1}); + combiner.ingest(left); + + TestGraphCombiner::Chunk right = make_chunk("right.pg"); + handle_t r1 = right.graph->create_handle("CCC", 1); + handle_t r2 = right.graph->create_handle("GG", 2); + right.graph->create_edge(r1, r2); + add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r1, r2}); + + int64_t shift = combiner.ingest(right); + + REQUIRE(shift > 0); + // Both chunks' nodes are present and none was overwritten. + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1)) == "AAAA"); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1 + shift)) == "CCC"); + REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(1 + shift), + combiner.dest->get_handle(2 + shift))); + REQUIRE(combiner.max_node_id == 2 + shift); + REQUIRE(sorted_path_names(*combiner.dest) + == vector{"GRCh38#0#chr1[0-4]", "GRCh38#0#chr1[4-9]"}); + // The source is released rather than held until combine finishes. + REQUIRE(right.graph == nullptr); + } + + SECTION("shared keeps IDs as they are, so nothing is shifted") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + TestGraphCombiner::Chunk left = make_chunk("left.pg"); + handle_t l1 = left.graph->create_handle("AAAA", 1); + handle_t l2 = left.graph->create_handle("CCC", 2); + left.graph->create_edge(l1, l2); + add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 7}, {l1, l2}); + combiner.ingest(left); + + TestGraphCombiner::Chunk right = make_chunk("right.pg"); + handle_t r2 = right.graph->create_handle("CCC", 2); // the shared node + handle_t r3 = right.graph->create_handle("GG", 3); + right.graph->create_edge(r2, r3); + add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r2, r3}); + + int64_t shift = combiner.ingest(right); + + REQUIRE(shift == 0); + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.max_node_id == 3); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[4-9]") == "CCCGG"); + } +} + +TEST_CASE("share_nodes_by_id treats a repeated ID as the same node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + handle_t d1 = combiner.dest->create_handle("AAAA", 1); + handle_t d2 = combiner.dest->create_handle("CCC", 2); + combiner.dest->create_edge(d1, d2); + + SECTION("an ID already present is reused, and a new one is created") { + TestGraphCombiner::Chunk chunk = make_chunk("more.pg"); + chunk.graph->create_handle("CCC", 2); // already in dest + chunk.graph->create_handle("GG", 3); // new + + combiner.share_nodes_by_id(chunk); + + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(2)) == "CCC"); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(3)) == "GG"); + } + + SECTION("an edge that is already there is not created twice") { + TestGraphCombiner::Chunk chunk = make_chunk("dup_edge.pg"); + handle_t c1 = chunk.graph->create_handle("AAAA", 1); + handle_t c2 = chunk.graph->create_handle("CCC", 2); + chunk.graph->create_edge(c1, c2); + + combiner.share_nodes_by_id(chunk); + + size_t edges = 0; + combiner.dest->for_each_edge([&](const edge_t&) { edges++; }); + REQUIRE(edges == 1); + REQUIRE(combiner.dest->get_node_count() == 2); + } + + SECTION("an edge's orientation survives the copy") { + TestGraphCombiner::Chunk chunk = make_chunk("flip_edge.pg"); + handle_t c2 = chunk.graph->create_handle("CCC", 2); + handle_t c3 = chunk.graph->create_handle("GG", 3); + // 2 to the reverse of 3: the copy must not quietly straighten this out. + chunk.graph->create_edge(c2, chunk.graph->flip(c3)); + + combiner.share_nodes_by_id(chunk); + + REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(2), + combiner.dest->get_handle(3, true))); + REQUIRE(!combiner.dest->has_edge(combiner.dest->get_handle(2), + combiner.dest->get_handle(3, false))); + } + + SECTION("nodes arriving on their own, with no edges, still land") { + TestGraphCombiner::Chunk chunk = make_chunk("island.pg"); + chunk.graph->create_handle("TTTT", 7); + + combiner.share_nodes_by_id(chunk); + + REQUIRE(combiner.dest->has_node(7)); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(7)) == "TTTT"); + } +} + +TEST_CASE("copy_paths_checked carries a path's metadata across with it", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + handle_t d1 = combiner.dest->create_handle("AAAA", 1); + handle_t d2 = combiner.dest->create_handle("CCC", 2); + combiner.dest->create_edge(d1, d2); + + TestGraphCombiner::Chunk chunk = make_chunk("paths.pg"); + handle_t c1 = chunk.graph->create_handle("AAAA", 1); + handle_t c2 = chunk.graph->create_handle("CCC", 2); + chunk.graph->create_edge(c1, c2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 7}, {c1, c2}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 3, + PathMetadata::NO_SUBRANGE, {c1, c2}); + add_path(*chunk.graph, PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, "scaffold", + PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, + PathMetadata::NO_SUBRANGE, {c2}, true); + + combiner.copy_paths_checked(chunk); + + SECTION("every path arrives, under its own name") { + REQUIRE(sorted_path_names(*combiner.dest) + == vector{"GRCh38#0#chr1[0-7]", "HG002#1#chr1#3", "scaffold"}); + } + + SECTION("the metadata fields come across one for one") { + path_handle_t hap = combiner.dest->get_path_handle("HG002#1#chr1#3"); + REQUIRE(combiner.dest->get_sense(hap) == PathSense::HAPLOTYPE); + REQUIRE(combiner.dest->get_sample_name(hap) == "HG002"); + REQUIRE(combiner.dest->get_locus_name(hap) == "chr1"); + REQUIRE(combiner.dest->get_haplotype(hap) == 1); + REQUIRE(combiner.dest->get_phase_block(hap) == 3); + + path_handle_t ref = combiner.dest->get_path_handle("GRCh38#0#chr1[0-7]"); + REQUIRE(combiner.dest->get_sense(ref) == PathSense::REFERENCE); + REQUIRE(combiner.dest->get_subrange(ref) == subrange_t{0, 7}); + + REQUIRE(combiner.dest->get_is_circular(combiner.dest->get_path_handle("scaffold"))); + } + + SECTION("the steps land in order, on dest's own nodes") { + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); + REQUIRE(combiner.dest->get_step_count( + combiner.dest->get_path_handle("GRCh38#0#chr1[0-7]")) == 2); + } +} + +TEST_CASE("connect_seam joins the running reference to the next chunk", "[combine]") { + + SECTION("without fusing, the two are joined by an edge") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + handle_t end = combiner.dest->create_handle("GG", 3); + combiner.dest->create_edge(start, end); + + TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; + handle_t new_end = combiner.connect_seam(seam, "right.pg", start, end); + + REQUIRE(combiner.dest->has_edge(left, start)); + REQUIRE(combiner.dest->get_node_count() == 3); + // Nothing was welded, so the chunk's own end is still the reference's end. + REQUIRE(new_end == end); + } + + SECTION("an edge that is already there is not added again") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + combiner.dest->create_edge(left, start); + + TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; + combiner.connect_seam(seam, "right.pg", start, start); + + size_t edges = 0; + combiner.dest->for_each_edge([&](const edge_t&) { edges++; }); + REQUIRE(edges == 1); + } + + SECTION("fusing welds the two boundary nodes into one") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + combiner.dest = make_unique(); + handle_t before = combiner.dest->create_handle("TT", 1); + handle_t left = combiner.dest->create_handle("AAAA", 2); + handle_t start = combiner.dest->create_handle("CCC", 3); + handle_t end = combiner.dest->create_handle("GG", 4); + combiner.dest->create_edge(before, left); + combiner.dest->create_edge(start, end); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {before, left}); + combiner.max_node_id = 4; + + TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 6, "left.pg"}; + handle_t new_end = combiner.connect_seam(seam, "right.pg", start, end); + + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-6]") == "TTAAAACCC"); + REQUIRE(new_end == end); + REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); + } + + SECTION("fusing a single-node chunk returns the weld, not the stale handle") { + // When the chunk's reference ends on the very node being welded, the old + // handle is destroyed by the fuse; the caller must get the new one back. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + combiner.max_node_id = 2; + + TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; + handle_t new_end = combiner.connect_seam(seam, "right.pg", start, start); + + REQUIRE(combiner.dest->get_node_count() == 1); + REQUIRE(combiner.dest->has_node(combiner.dest->get_id(new_end))); + REQUIRE(combiner.dest->get_sequence(new_end) == "AAAACCC"); + REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); + } +} + +TEST_CASE("merge_path_fragments joins pieces of one path back together", "[combine]") { + + SECTION("two abutting fragments become one path spanning both") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); + // Nothing joined the two before, so the merge has to say they abut. + REQUIRE(combiner.dest->has_edge(n1, n2)); + } + + SECTION("a lone fragment is left alone, and keeps its name") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-4]"}); + } + + SECTION("fragments sharing a run of steps at the seam do not repeat it") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CC"); + handle_t n3 = combiner.dest->create_handle("GGG"); + combiner.dest->create_edge(n1, n2); + combiner.dest->create_edge(n2, n3); + // Both fragments cross n2: it must appear once in the merge, not twice. + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {n1, n2}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 9}, {n2, n3}); + + combiner.merge_path_fragments(); + + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); + REQUIRE(combiner.dest->get_step_count( + combiner.dest->get_path_handle("GRCh38#0#chr1[0-9]")) == 3); + } + + SECTION("fragments of different paths do not bleed into each other") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t x1 = combiner.dest->create_handle("AAAA"); + handle_t x2 = combiner.dest->create_handle("CCC"); + handle_t y1 = combiner.dest->create_handle("TTTT"); + handle_t y2 = combiner.dest->create_handle("GGG"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {x1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {x2}); + add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{0, 4}, {y1}); + add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{4, 7}, {y2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) + == vector{"GRCh38#0#chr1[0-7]", "GRCh38#0#chr2[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr2[0-7]") == "TTTTGGG"); + REQUIRE(!combiner.dest->has_edge(x2, y1)); + } + + SECTION("fragments arriving out of order are put back in coordinate order") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n3 = combiner.dest->create_handle("GGG"); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CC"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, {n3}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 6}, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); + } + + SECTION("a merged HAPLOTYPE path keeps the phase block it needs") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, + subrange_t{0, 4}, {n1}); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, + subrange_t{4, 7}, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#2#chr22#4[0-7]"}); + path_handle_t merged = combiner.dest->get_path_handle("HG002#2#chr22#4[0-7]"); + REQUIRE(combiner.dest->get_phase_block(merged) == 4); + REQUIRE(combiner.dest->get_sense(merged) == PathSense::HAPLOTYPE); + } + + SECTION("with -P, phase blocks position the fragments and fold into a subrange") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, true); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 4, + PathMetadata::NO_SUBRANGE, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#1#chr1#0[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "HG002#1#chr1#0[0-7]") == "AAAACCC"); + } + + SECTION("a circular path stays circular through the merge") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 7}, {n2}, true); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chrM[0-7]"}); + REQUIRE(combiner.dest->get_is_circular( + combiner.dest->get_path_handle("GRCh38#0#chrM[0-7]"))); + } + + SECTION("three fragments merge in one pass") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CC"); + handle_t n3 = combiner.dest->create_handle("GGG"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 6}, {n2}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, {n3}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-9]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); + REQUIRE(combiner.dest->has_edge(n1, n2)); + REQUIRE(combiner.dest->has_edge(n2, n3)); + } +} + +} +} diff --git a/src/unittest/fuse_nodes.cpp b/src/unittest/fuse_nodes.cpp new file mode 100644 index 00000000000..f791bf736c1 --- /dev/null +++ b/src/unittest/fuse_nodes.cpp @@ -0,0 +1,117 @@ +/// \file fuse_nodes.cpp +/// +/// Unit tests for the fuse_nodes algorithm. + +#include "catch.hpp" +#include "../handle.hpp" +#include "../algorithms/fuse_nodes.hpp" + +#include "bdsg/hash_graph.hpp" + +namespace vg { +namespace unittest { + +using namespace std; + +/// Spell out the sequence a path visits, so tests can assert on what a path +/// means rather than on which nodes happen to carry it. +static string spell_path(const PathHandleGraph& graph, const string& name) { + string sequence; + path_handle_t path = graph.get_path_handle(name); + for (handle_t h : graph.scan_path(path)) { + sequence += graph.get_sequence(h); + } + return sequence; +} + +TEST_CASE("can_fuse_nodes accepts a clean boundary and rejects the rest", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGG"); + handle_t n4 = graph.create_handle("TT"); + graph.create_edge(n1, n2); + graph.create_edge(n3, n4); + + SECTION("nothing on the inner sides of the pair") { + REQUIRE(algorithms::can_fuse_nodes(graph, n2, n3)); + } + SECTION("the left node has something on its right") { + REQUIRE(!algorithms::can_fuse_nodes(graph, n1, n3)); + } + SECTION("the right node has something on its left") { + REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n4)); + } + SECTION("a node cannot be fused with itself") { + REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n2)); + } + SECTION("an edge between the pair would be swallowed, so it is refused") { + graph.create_edge(n2, n3); + REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n3)); + } +} + +TEST_CASE("fuse_nodes concatenates sequence, re-hangs edges, and keeps paths", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGG"); + handle_t n4 = graph.create_handle("TT"); + graph.create_edge(n1, n2); + graph.create_edge(n3, n4); + + // Three paths over the seam: one ends on it, one starts on it, and one + // crosses it backwards, an orientation the rewrite has to preserve. + path_handle_t left_path = graph.create_path_handle("left"); + graph.append_step(left_path, n1); + graph.append_step(left_path, n2); + path_handle_t right_path = graph.create_path_handle("right"); + graph.append_step(right_path, n3); + graph.append_step(right_path, n4); + path_handle_t back_path = graph.create_path_handle("backwards"); + graph.append_step(back_path, graph.flip(n2)); + graph.append_step(back_path, graph.flip(n1)); + + REQUIRE(algorithms::can_fuse_nodes(graph, n2, n3)); + handle_t fused = algorithms::fuse_nodes(&graph, n2, n3); + + REQUIRE(graph.get_sequence(fused) == "CCGGGG"); + REQUIRE(graph.get_node_count() == 3); + + size_t edge_count = 0; + graph.for_each_edge([&](const edge_t&) { edge_count++; }); + REQUIRE(edge_count == 2); + REQUIRE(graph.has_edge(n1, fused)); + REQUIRE(graph.has_edge(fused, n4)); + + REQUIRE(spell_path(graph, "left") == "AAACCGGGG"); + REQUIRE(spell_path(graph, "right") == "CCGGGGTT"); + REQUIRE(spell_path(graph, "backwards") == "CCCCGGTTT"); + REQUIRE(graph.get_step_count(graph.get_path_handle("left")) == 2); +} + +TEST_CASE("fuse_nodes turns a cycle through the pair into a self-loop", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGG"); + graph.create_edge(n1, n2); + // Back edge closing a cycle over the pair. Neither end of it sits on the + // inner side of the seam, so the pair is still fusable. + graph.create_edge(n3, n2); + + REQUIRE(algorithms::can_fuse_nodes(graph, n2, n3)); + handle_t fused = algorithms::fuse_nodes(&graph, n2, n3); + + REQUIRE(graph.get_sequence(fused) == "CCGGGG"); + REQUIRE(graph.get_node_count() == 2); + REQUIRE(graph.has_edge(n1, fused)); + REQUIRE(graph.has_edge(fused, fused)); + + size_t edge_count = 0; + graph.for_each_edge([&](const edge_t&) { edge_count++; }); + REQUIRE(edge_count == 2); +} + +} +} diff --git a/test/t/05_vg_find.t b/test/t/05_vg_find.t index 346826959a0..5e2ab55621e 100644 --- a/test/t/05_vg_find.t +++ b/test/t/05_vg_find.t @@ -91,10 +91,11 @@ rm -f test.vg test.xg vg construct -m 1000 -r small/xy.fa -v small/xy2.vcf.gz -R x -C -a 2> /dev/null | vg view -j - | sed s/_alt/alt/g | vg view -Jv - >w.vg vg index -x w.xg w.vg -cat w.vg | vg paths -d -v - > part1.tmp vg find -x w.xg -Q alt > part2.tmp -is "$(vg combine -c part1.tmp part2.tmp | vg paths -L -v - | wc -l)" "$(vg view -j w.vg | jq -c '.path[] | select(.name | contains("alt"))' | wc -l)" "pattern based path extraction works" -rm -f w.xg w.vg part1.tmp part2.tmp +# vg find -Q emits paths without the nodes they visit, so the result only loads +# as a raw message stream (vg view -c), not as a graph. +is "$(vg view -c part2.tmp | jq -c '.path[].name' | wc -l)" "$(vg view -j w.vg | jq -c '.path[] | select(.name | contains("alt"))' | wc -l)" "pattern based path extraction works" +rm -f w.xg w.vg part2.tmp vg construct -r tiny/tiny.fa -v tiny/tiny.vcf.gz >t.vg vg index -x t.xg t.vg diff --git a/test/t/51_vg_combine.t b/test/t/51_vg_combine.t index 58b15437fe6..79db325a596 100644 --- a/test/t/51_vg_combine.t +++ b/test/t/51_vg_combine.t @@ -5,49 +5,371 @@ BASH_TAP_ROOT=../deps/bash-tap PATH=../bin:$PATH # for vg -plan tests 10 +plan tests 115 -vg construct -r small/x.fa -v small/x.vcf.gz > x.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg paths -d -v - > y.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg paths -d -v - > z.vg +C=combine +mkdir -p $C -vg combine x.vg y.vg z.vg > xyz.vg +# Construct chunks holding a reference plus two haplotypes. +RS='H\tVN:Z:1.1\tRS:Z:GRCh38\n' -is $(vg stats -z xyz.vg | grep nodes | awk '{print $2}') 645 "combined graph as correct total nodes" -is $(vg stats -z xyz.vg | grep edges | awk '{print $2}') 888 "combined graph as correct total edges" -vg paths -Ev x.vg > x.paths -vg paths -Ev xyz.vg > xyz.paths -diff x.paths xyz.paths -is $? 0 "combined graph has same input path" +# Left chunk including HG002 and HG005#1 +printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-10]\t1+,3+,4+\t*\nP\tHG005#1#chr1#0[0-10]\t1+,3+,4+\t*\n' > $C/ml.gfa +# Right chunk including only HG002 +printf "$RS"'S\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[10-20]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[10-20]\t5+,6+,8+\t*\n' > $C/mr.gfa +vg convert -p $C/ml.gfa > $C/ml.pg 2>/dev/null +vg convert -p $C/mr.gfa > $C/mr.pg 2>/dev/null -rm -f x.vg y.vg z.vg xyz.vg x.paths xyz.paths +is "$(vg paths -M -v $C/ml.pg | tail -n +2 | cut -f2 | grep -c HAPLOTYPE)" "3" "the chunk carries three haplotype paths" +is "$(vg paths -M -v $C/ml.pg | tail -n +2 | cut -f2 | grep -c REFERENCE)" "1" "the chunk carries one reference path" -vg construct -r small/x.fa -v small/x.vcf.gz | vg convert -p - > x.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg paths -d -v - | vg convert -a - > y.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg paths -d -v - | vg convert -p - > z.vg +vg combine $C/ml.pg $C/mr.pg > $C/o_def.vg 2>/dev/null +is "$(vg paths -Ev $C/o_def.vg | wc -l)" "4" "every path is considered" +is "$(vg paths -Ev $C/o_def.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20],HG005#1#chr1#0[0-10]" "each sample and haplotype merges within its own group" +is "$(vg paths -Ev $C/o_def.vg | cut -f2 | sort -n | paste -sd,)" "10,20,20,20" "the merged paths span both chunks and the lone one does not" +is "$(vg stats -N $C/o_def.vg)" "8" "merging keeps every node of both chunks, 4 for each" +is "$(vg stats -E $C/o_def.vg)" "9" "the three paths cross the seam on one shared edge, so the edges count increase by 1(8 -> 9)" +is "$(vg stats -s $C/o_def.vg | wc -l)" "1" "the two chunks end up in one component" -vg combine x.vg y.vg z.vg > xyz.vg +# Only fragments of the same logical paths are merged. +is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>GRCh38#0#chr1\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTAGGGGG" "the reference only merge its own fragments" +is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG002#1#chr1#0\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTCGGGGG" "HG002#1#chr1 only merge its own fragments" +is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 only merge its own fragments" +is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG005#1#chr1#0\[0-10\]$' | tail -1)" "AAAAGGGGGG" "HG005#1#chr1 is the lone fragment untouched" -is $(vg stats -z xyz.vg | grep nodes | awk '{print $2}') 645 "combined handle graph as correct total nodes" -is $(vg stats -z xyz.vg | grep edges | awk '{print $2}') 888 "combined handle graph as correct total edges" -vg paths -Ev x.vg > x.paths -vg paths -Ev xyz.vg > xyz.paths -diff x.paths xyz.paths -is $? 0 "combined handle graph has same input path" +# renumber is the default, so naming it explicitly must change nothing. +vg combine -s renumber $C/ml.pg $C/mr.pg > $C/o_ren.vg 2>/dev/null +is "$(vg paths -Ev $C/o_ren.vg)" "$(vg paths -Ev $C/o_def.vg)" "--seam renumber is the default" +is "$(vg paths -Fv $C/o_ren.vg | sort)" "$(vg paths -Fv $C/o_def.vg | sort)" "and spells out the same sequences" -rm -f x.vg y.vg z.vg xyz.vg x.paths xyz.paths +# A path with no subrange is not a fragment of anything, so its name must come through untouched. +printf "$RS"'S\t1\tAAAACCCC\nP\tGRCh38#0#chr1\t1+\t*\n' > $C/solo.gfa +vg convert -p $C/solo.gfa > $C/solo.pg 2>/dev/null -vg construct -r small/x.fa -v small/x.vcf.gz | vg convert -p - > x.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg convert -a - > y.vg -vg construct -r small/x.fa -v small/x.vcf.gz | vg convert -v - > z.vg +vg combine $C/solo.pg > $C/o_solo.vg 2>/dev/null +is "$(vg paths -Ev $C/o_solo.vg)" "$(printf 'GRCh38#0#chr1\t8')" "a lone path with no subrange keeps its name" -vg combine -p x.vg y.vg z.vg > xyz.vg +# Fragments from different haplotype must not be merged. +printf "$RS"'S\t1\tAAAA\nS\t2\tCGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+\t*\n' > $C/h1.gfa +printf "$RS"'S\t3\tTTTT\nS\t4\tAGGGGG\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t3+,4+\t*\nP\tHG002#2#chr1#0[10-20]\t3+,4+\t*\n' > $C/h2.gfa +vg convert -p $C/h1.gfa > $C/h1.pg 2>/dev/null +vg convert -p $C/h2.gfa > $C/h2.pg 2>/dev/null -is $(vg stats -z xyz.vg | grep nodes | awk '{print $2}') 645 "path combined graph as correct total nodes" -is $(vg stats -z xyz.vg | grep edges | awk '{print $2}') 890 "path combined path graph as correct total edges" -is $(vg paths -Ev xyz.vg | wc -l) 1 "path combined graph as correct number of paths" -is $(vg paths -Ev xyz.vg | awk '{print $2}') 3003 "combined path has correct size" +vg combine $C/h1.pg $C/h2.pg > $C/o_hap.vg 2>/dev/null +is "$(vg paths -Ev $C/o_hap.vg | wc -l)" "3" "only the reference path has fragments to merge" +is "$(vg paths -Ev $C/o_hap.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-10],HG002#2#chr1#0[10-20]" "fragments of different haplotypes stay apart" -rm -f x.vg y.vg z.vg xyz.vg +# Shuffled fragments of two contigs must not splice into each other. +printf "$RS"'S\t1\tAAAA\nS\t2\tCCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/mc_chr1l.gfa +printf "$RS"'S\t1\tGGGG\nS\t2\tTTTTTT\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t1+,2+\t*\n' > $C/mc_chr1r.gfa +printf "$RS"'S\t1\tTTTT\nS\t2\tGGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[0-10]\t1+,2+\t*\n' > $C/mc_chr2l.gfa +printf "$RS"'S\t1\tCCCC\nS\t2\tAAAAAA\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[10-20]\t1+,2+\t*\n' > $C/mc_chr2r.gfa +for g in mc_chr1l mc_chr1r mc_chr2l mc_chr2r; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done +vg combine $C/mc_chr2r.pg $C/mc_chr1l.pg $C/mc_chr2l.pg $C/mc_chr1r.pg > $C/o_multi.vg 2>/dev/null +is "$(vg paths -Ev $C/o_multi.vg | wc -l)" "2" "each contig yields its own merged path" +is "$(vg paths -Ev $C/o_multi.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],GRCh38#0#chr2[0-20]" "both contigs span both of their chunks" +is "$(vg paths -Ev $C/o_multi.vg | cut -f2 | sort -u)" "20" "neither contig grew by taking the other's chunks" +is "$(vg stats -s $C/o_multi.vg | wc -l)" "2" "the contigs stay in separate components" +is "$(vg stats -E $C/o_multi.vg)" "6" "only the two intra-contig seams are edged" +is "$(vg paths -Fv $C/o_multi.vg | grep -A1 '^>GRCh38#0#chr1\[0-20\]$' | tail -1)" "AAAACCCCCCGGGGTTTTTT" "chr1 spells out only its own chunks" +is "$(vg paths -Fv $C/o_multi.vg | grep -A1 '^>GRCh38#0#chr2\[0-20\]$' | tail -1)" "TTTTGGGGGGCCCCAAAAAA" "chr2 spells out only its own chunks" +# --seam shared: the two chunks sharing nodes 4 and 5. +printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nS\t5\tTTTT\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nL\t4\t+\t5\t+\t0M\nP\tGRCh38#0#chr1[0-14]\t1+,2+,4+,5+\t*\nP\tHG002#1#chr1#0[0-14]\t1+,2+,4+,5+\t*\nP\tHG002#2#chr1#0[0-14]\t1+,3+,4+,5+\t*\n' > $C/sl.gfa +printf "$RS"'S\t4\tGGGGG\nS\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t4\t+\t5\t+\t0M\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[5-20]\t4+,5+,6+,8+\t*\nP\tHG002#1#chr1#0[5-20]\t4+,5+,7+,8+\t*\nP\tHG002#2#chr1#0[5-20]\t4+,5+,6+,8+\t*\n' > $C/sr.gfa +vg convert -p $C/sl.gfa > $C/sl.pg 2>/dev/null +vg convert -p $C/sr.gfa > $C/sr.pg 2>/dev/null + +vg combine -s shared $C/sl.pg $C/sr.pg > $C/o_sh.vg 2>/dev/null +is "$(vg paths -Ev $C/o_sh.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20]" "--seam shared merges all three paths over their shared nodes" +is "$(vg paths -Ev $C/o_sh.vg | cut -f2 | sort -u)" "20" "no path counts the same nodes twice" +is "$(vg stats -N $C/o_sh.vg)" "8" "the shared nodes are deduplicated rather than copied" +is "$(vg stats -E $C/o_sh.vg)" "9" "the edges between them are also deduplicated" +is "$(vg paths -Fv $C/o_sh.vg | grep -A1 '^>HG002#1#chr1#0\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTCGGGGG" "HG002#1#chr1 keeps its own sequences over the shared nodes" +is "$(vg paths -Fv $C/o_sh.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 keeps its own sequences over the shared nodes" + +# --seam shared: different paths share different nodes. +printf "$RS"'S\t4\tGGGGG\nS\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t4\t+\t5\t+\t0M\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[5-20]\t4+,5+,6+,8+\t*\nP\tHG002#1#chr1#0[5-20]\t4+,5+,7+,8+\t*\nP\tHG002#2#chr1#0[10-20]\t5+,6+,8+\t*\n' > $C/sr2.gfa +vg convert -p $C/sr2.gfa > $C/sr2.pg 2>/dev/null + +vg combine -s shared $C/sl.pg $C/sr2.pg > $C/o_sh2.vg 2>/dev/null +is "$(vg paths -Ev $C/o_sh2.vg | cut -f2 | sort -u)" "20" "HG002#2#chr1 has a shorter overlap but it still keeps the same total length" +is "$(vg paths -Fv $C/o_sh2.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 also keeps the same sequence" + +# --seam trim: on a 2bp overlap. +printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-10]\t1+,3+,4+\t*\n' > $C/tl.gfa +printf "$RS"'S\t5\tGGTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[8-18]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[8-18]\t5+,6+,8+\t*\n' > $C/tr.gfa +vg convert -p $C/tl.gfa > $C/tl.pg 2>/dev/null +vg convert -p $C/tr.gfa > $C/tr.pg 2>/dev/null + +vg combine -s trim $C/tl.pg $C/tr.pg > $C/o_trim.vg 2>/dev/null +is "$(vg paths -Ev $C/o_trim.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "all three paths report the same trimmed span" +is "$(vg paths -Ev $C/o_trim.vg | cut -f2 | sort -u)" "18" "the overlap of three paths has been trimmed" +is "$(vg stats -N $C/o_trim.vg)" "8" "trimming splits the boundary node and drops the overlapping half, leaving the node counts unchanged" +is "$(vg stats -E $C/o_trim.vg)" "9" "seaming create one edge at the boundary" +is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>GRCh38#0#chr1\[0-18\]$' | tail -1)" "AAAACGGGGGTTAGGGGG" "the overlap of the reference path is trimmed" +is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>HG002#1#chr1#0\[0-18\]$' | tail -1)" "AAAACGGGGGTTCGGGGG" "the overlap of the HG002#1#chr1 is trimmed" +is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>HG002#2#chr1#0\[0-18\]$' | tail -1)" "AAAAGGGGGGTTAGGGGG" "the overlap of the HG002#2#chr1 is trimmed" + +# --seam trim --fuse: must rewrite the step of every path, replace the original nodes with the fused nodes. +vg combine -s trim -u $C/tl.pg $C/tr.pg > $C/o_fuse.vg 2>/dev/null +is "$(vg stats -N $C/o_fuse.vg)" "7" "the two boundary nodes become one" +is "$(vg stats -E $C/o_fuse.vg)" "8" "the edge between two boundary nodes is removed" +is "$(vg paths -Ev $C/o_fuse.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "welding does not change any path's coordinate" +is "$(vg paths -Ev $C/o_fuse.vg | cut -f2 | sort -u)" "18" "welding doesn't change any path's length" +is "$(vg paths -Fv $C/o_fuse.vg | sort)" "$(vg paths -Fv $C/o_trim.vg | sort)" "welding doesn't change any path's sequence" + +# --seam trim on two contigs interleaved: each must trim to its own span. +printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/mt_chr1l.gfa +printf "$RS"'S\t1\tCCGGG\nS\t2\tTTTTT\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t1+,2+\t*\n' > $C/mt_chr1r.gfa +printf "$RS"'S\t1\tTTTTT\nS\t2\tGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[0-10]\t1+,2+\t*\n' > $C/mt_chr2l.gfa +printf "$RS"'S\t1\tGGAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[8-18]\t1+,2+\t*\n' > $C/mt_chr2r.gfa +for g in mt_chr1l mt_chr1r mt_chr2l mt_chr2r; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done + +vg combine -s trim $C/mt_chr2r.pg $C/mt_chr1l.pg $C/mt_chr2l.pg $C/mt_chr1r.pg > $C/o_mt.pg 2>/dev/null +is "$(vg paths -Ev $C/o_mt.pg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],GRCh38#0#chr2[0-18]" "each contig trims to its own span" +is "$(vg paths -Ev $C/o_mt.pg | cut -f2 | sort -u)" "18" "each contig lost only its own overlap" +is "$(vg stats -s $C/o_mt.pg | wc -l)" "2" "the contigs stay in separate components" +is "$(vg stats -E $C/o_mt.pg)" "6" "only the two intra-contig seams are edged" +is "$(vg paths -Fv $C/o_mt.pg | grep -A1 '^>GRCh38#0#chr1\[0-18\]$' | tail -1)" "AAAAACCCCCGGGTTTTT" "chr1 keeps its own sequence through the trim" +is "$(vg paths -Fv $C/o_mt.pg | grep -A1 '^>GRCh38#0#chr2\[0-18\]$' | tail -1)" "TTTTTGGGGGAAACCCCC" "chr2 keeps its own" + +# --fuse welds one seam per contig, not one overall. +vg combine -s trim -u $C/mt_chr2r.pg $C/mt_chr1l.pg $C/mt_chr2l.pg $C/mt_chr1r.pg > $C/o_mtu.pg 2>/dev/null +is "$(vg stats -N $C/o_mtu.pg)" "6" "each contig's boundary nodes weld into one" +is "$(vg stats -E $C/o_mtu.pg)" "4" "and each contig's seam edge goes with them" +is "$(vg paths -Fv $C/o_mtu.pg | sort)" "$(vg paths -Fv $C/o_mt.pg | sort)" "welding leaves both contigs spelling the same sequence" + +# Another single chunk. +printf "$RS"'S\t1\tAAAA\nS\t2\tCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-7]\t1+,2+\t*\n' > $C/one.gfa +vg convert -p $C/one.gfa > $C/one.pg 2>/dev/null + +# Error: nothing to combine. +vg combine -s renumber 2>$C/none.err >/dev/null +is "$?" "1" "combining nothing is rejected" +is "$(grep -c 'At least one input graph' $C/none.err)" "1" "the error says at least one input graph is required" + +# Error: --seam trim needs more than one chunk. +vg combine -s trim $C/one.pg 2>$C/lone.err >/dev/null +is "$?" "1" "--seam trim on a single input is rejected" +is "$(grep -c 'at least two input graphs' $C/lone.err)" "1" "the error says trim needs at least two inputs" + +# Error: --fuse only makes sense when --seam trim leaves a boundary to weld. +vg combine -u $C/tl.pg $C/tr.pg 2>$C/fusearg.err >/dev/null +is "$?" "1" "--fuse without --seam trim is rejected" +is "$(grep -c 'requires --seam trim' $C/fusearg.err)" "1" "the error says --fuse needs --seam trim" + +# Error: --seam only takes the policies it knows. +vg combine -s bogus $C/tl.pg $C/tr.pg 2>$C/seam.err >/dev/null +is "$?" "1" "an unrecognized --seam policy is rejected" +is "$(grep -c 'renumber, shared, trim' $C/seam.err)" "1" "the error lists the policies that do exist" + +# Error: vg combine joins pieces from separate inputs, not pieces inside one input. +printf "$RS"'S\t1\tAAAAAAAAAA\nS\t2\tCCCCCCCCCC\nP\tGRCh38#0#chr1[0-10]\t1+\t*\nP\tGRCh38#0#chr1[10-20]\t2+\t*\n' > $C/split.gfa +# Same input, plus the edge that already joins the two pieces. +printf "$RS"'S\t1\tAAAAAAAAAA\nS\t2\tCCCCCCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+\t*\nP\tGRCh38#0#chr1[10-20]\t2+\t*\n' > $C/split_joined.gfa +vg convert -p $C/split.gfa > $C/split.pg 2>/dev/null +vg convert -p $C/split_joined.gfa > $C/split_joined.pg 2>/dev/null + +vg combine $C/split.pg 2>$C/split.err >/dev/null +is "$?" "1" "an input holding a path in pieces is rejected" +is "$(grep -c 'GRCh38#0#chr1\[0-10\].*GRCh38#0#chr1\[10-20\]' $C/split.err)" "1" "the error reports the pieces" +is "$(grep -c 'split.pg' $C/split.err)" "1" "the error reports the input holding them" + +vg combine $C/split_joined.pg 2>/dev/null >/dev/null +is "$?" "1" "and is rejected even when an edge already joins the pieces" + +# Error: a read-only graph format cannot be combined; VPKG reports this itself. +vg gbwt -g $C/m.gbz --gbz-format -G graphs/gfa_with_reference.gfa 2>/dev/null +vg combine $C/m.gbz $C/one.pg 2>/dev/null >/dev/null +is "$?" "1" "a GBZ input is rejected rather than crashing" + +# Error: --seam trim joins chunks end to end, so a gap between them is rejected. +printf "$RS"'S\t1\tAAAAAAAAAA\nP\tGRCh38#0#chr1[0-10]\t1+\t*\n' > $C/gapl.gfa +printf "$RS"'S\t2\tCCCCCCCCCC\nP\tGRCh38#0#chr1[20-30]\t2+\t*\n' > $C/gapr.gfa +vg convert -p $C/gapl.gfa > $C/gapl.pg 2>/dev/null +vg convert -p $C/gapr.gfa > $C/gapr.pg 2>/dev/null +vg combine -s trim $C/gapl.pg $C/gapr.pg 2>$C/tgap.err >/dev/null +is "$?" "1" "--seam trim rejects a gap between chunks" +is "$(grep -c 'REFERENCE offset gap of 10 bp' $C/tgap.err)" "1" "--seam trim also reports the size of the gap" + +# Error: the default policy refuses the same gap, caught by a different check. +vg combine $C/gapl.pg $C/gapr.pg 2>$C/gap.err >/dev/null +is "$?" "1" "the default policy rejects a gap between fragments too" +is "$(grep -c 'Gap of 10 bp' $C/gap.err)" "1" "the default policy reports the size of the gap" + +# Error: --seam trim needs at least one reference. +printf 'S\t1\tAAAA\nP\tplain_contig\t1+\t*\n' > $C/noref.gfa +vg convert -p $C/noref.gfa > $C/noref.pg 2>/dev/null +vg combine -s trim $C/noref.pg $C/one.pg 2>$C/noref.err >/dev/null +is "$?" "1" "--seam trim rejects a chunk with no REFERENCE path" +is "$(grep -c 'No REFERENCE-sense path found' $C/noref.err)" "1" "the error says a reference path is needed" + +# Error: --seam trim rejects more than one reference. +printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\nP\tGRCh38#0#chr2[0-4]\t1+\t*\n' > $C/tworef.gfa +vg convert -p $C/tworef.gfa > $C/tworef.pg 2>/dev/null +vg combine -s trim $C/tworef.pg $C/one.pg 2>$C/tworef.err >/dev/null +is "$?" "1" "--seam trim rejects a chunk with two REFERENCE paths" +is "$(grep -c '2 REFERENCE-sense paths found' $C/tworef.err)" "1" "the error counts the paths it found" + +# Error: --seam trim requires trimming on the forward strand of the boundary node. +printf "$RS"'S\t1\tAAAA\nS\t2\tCCC\nL\t1\t-\t2\t+\t0M\nP\tGRCh38#0#chr1[0-7]\t1-,2+\t*\n' > $C/rev.gfa +vg convert -p $C/rev.gfa > $C/rev.pg 2>/dev/null +vg combine -s trim $C/rev.pg $C/one.pg 2>$C/rev.err >/dev/null +is "$?" "1" "--seam trim rejects a reverse-strand boundary" +is "$(grep -c 'forward-strand boundaries' $C/rev.err)" "1" "the error says only forward boundaries work" + +# The left chunk shared by three --seam trim error cases below. +printf "$RS"'S\t1\tAAAAAAAAAA\nP\tGRCh38#0#chr1[0-10]\t1+\t*\n' > $C/bigl.gfa +vg convert -p $C/bigl.gfa > $C/bigl.pg 2>/dev/null + +# Error: --seam trim requires the left and right chunks to share the same sequence. +printf "$RS"'S\t2\tAACCC\nS\t3\tGGGGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[6-16]\t2+,3+\t*\n' > $C/mmr.gfa +vg convert -p $C/mmr.gfa > $C/mmr.pg 2>/dev/null +vg combine -s trim $C/bigl.pg $C/mmr.pg 2>$C/mm.err >/dev/null +is "$?" "1" "chunks that disagree on the overlapping sequence are rejected" +is "$(grep -c 'but disagree there' $C/mm.err)" "1" "the error says the overlap sequence does not match" +is "$(grep -c 'at offset 8' $C/mm.err)" "1" "the error reports the first position they differ at" + +# Error: --seam trim requires that all paths start from the reference path. +printf "$RS"'S\t5\tGGTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[8-18]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[12-18]\t6+,8+\t*\n' > $C/trb.gfa +printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-5]\t1+,3+\t*\n' > $C/tlb.gfa +vg convert -p $C/trb.gfa > $C/trb.pg 2>/dev/null +vg convert -p $C/tlb.gfa > $C/tlb.pg 2>/dev/null + +vg combine -s trim $C/tl.pg $C/trb.pg >/dev/null 2>$C/start.err +is "$?" "1" "a path that does not start at the REFERENCE start node is rejected" +is "$(grep -c 'does not start at the REFERENCE start node (id 5)' $C/start.err)" "1" "the error reports the path and the REFERENCE start node it should have started on" + +# Error: --seam trim requires that all paths end at the reference path. +vg combine -s trim $C/tlb.pg $C/tr.pg >/dev/null 2>$C/end.err +is "$?" "1" "a path that does not end at the REFERENCE end node is rejected" +is "$(grep -c 'does not end at the REFERENCE end node (id 4)' $C/end.err)" "1" "the error reports the path and the REFERENCE end node it should have ended on" + +# Error: --seam trim does not allow the overlap to exceed the length of the first node of the right chunk. +printf "$RS"'S\t2\tAA\nS\t3\tAAAAGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[4-12]\t2+,3+\t*\n' > $C/bigr.gfa +vg convert -p $C/bigr.gfa > $C/bigr.pg 2>/dev/null +vg combine -s trim $C/bigl.pg $C/bigr.pg 2>$C/big.err >/dev/null +is "$?" "1" "an overlap longer than the first node is rejected" +is "$(grep -c 'exceeds the length of its first node' $C/big.err)" "1" "the error says the overlap does not fit" + +# Error: --seam trim does not allow the overlap to equal the length of the first node of the right chunk. +printf "$RS"'S\t2\tAAA\nS\t3\tGGGGGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[7-16]\t2+,3+\t*\n' > $C/eqr.gfa +vg convert -p $C/eqr.gfa > $C/eqr.pg 2>/dev/null +vg combine -s trim $C/bigl.pg $C/eqr.pg 2>$C/eq.err >/dev/null +is "$?" "1" "an overlap that would empty the first node is rejected" +is "$(grep -c 'would leave a zero-length' $C/eq.err)" "1" "the error says the node would be left empty" + +# Error: --seam trim needs a clean chunk boundary. +printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/bl.gfa +printf "$RS"'S\t3\tCCGGG\nS\t4\tTTTTT\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t3+,4+\t*\n' > $C/br.gfa +# Same right chunk, plus an edge back into the node the trim want to remove. +printf "$RS"'S\t3\tCCGGG\nS\t4\tTTTTT\nL\t3\t+\t4\t+\t0M\nL\t4\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t3+,4+\t*\n' > $C/br_back.gfa +# Same left chunk, plus a node hanging off the right of its boundary node. +printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nS\t5\tAC\nL\t1\t+\t2\t+\t0M\nL\t2\t+\t5\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/bl_dangle.gfa +for g in bl br br_back bl_dangle; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done + +vg combine -s trim $C/bl.pg $C/br.pg > $C/o_b.pg 2>/dev/null +is "$(vg paths -Ev $C/o_b.pg)" "$(printf 'GRCh38#0#chr1[0-18]\t18')" "the two chunks are joined into one path" +is "$(vg paths -Fv $C/o_b.pg | tail -1)" "AAAAACCCCCGGGTTTTT" "the overlapped CC is trimmed away and the two chunks are joined into one sequence" + +vg combine -s trim $C/bl.pg $C/br_back.pg >/dev/null 2>$C/back.err +is "$?" "1" "an edge into the node that would be trimmed is rejected" +is "$(grep -c 'would be lost' $C/back.err)" "1" "the error says the edge would be lost" + +# An edge off the left boundary is fine when seaming, but not for fusing. +vg combine -s trim $C/bl_dangle.pg $C/br.pg > $C/o_bd.pg 2>/dev/null +is "$(vg paths -Ev $C/o_bd.pg)" "$(printf 'GRCh38#0#chr1[0-18]\t18')" "an edge off the left boundary is fine for seaming" + +vg combine -s trim -u $C/bl_dangle.pg $C/br.pg >/dev/null 2>$C/fuse.err +is "$?" "1" "--fuse rejects a boundary node with an edge on the welded side" +is "$(grep -c 'weld would have to drop' $C/fuse.err)" "1" "the error says what the weld would drop" + +vg combine -s trim -u $C/bl.pg $C/br.pg > $C/o_bu.pg 2>/dev/null +is "$(vg stats -N $C/o_bu.pg)" "3" "a clean boundary is required for fusing into one node" + +# Error: --seam shared requires that all shared nodes have the same sequence. +printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\n' > $C/c1.gfa +printf "$RS"'S\t1\tCCCC\nP\tGRCh38#0#chr1[4-8]\t1+\t*\n' > $C/c2.gfa +vg convert -p $C/c1.gfa > $C/c1.pg 2>/dev/null +vg convert -p $C/c2.gfa > $C/c2.pg 2>/dev/null +vg combine -s shared $C/c1.pg $C/c2.pg 2>$C/conflict.err >/dev/null +is "$?" "1" "--seam shared rejects same ID carrying different sequences" +is "$(grep -c 'has different sequences' $C/conflict.err)" "1" "the error reports the node that disagrees" + +# Error: pieces of one path are told apart by name, so the same name cannot arrive twice. +printf "$RS"'S\t2\tGGGG\nP\tGRCh38#0#chr1[0-4]\t2+\t*\n' > $C/dup.gfa +vg convert -p $C/dup.gfa > $C/dup.pg 2>/dev/null +vg combine $C/c1.pg $C/dup.pg 2>$C/dup.err >/dev/null +is "$?" "1" "a path with the same name in both chunks is rejected" +is "$(grep -c 'in an earlier input' $C/dup.err)" "1" "the error reports the path that was duplicated" + +# Phase-block fragments: blocks 0 and 10 read as offsets, block 1 does not. +printf 'H\tVN:Z:1.0\nS\t1\tACGTA\nS\t2\tCCGGT\nP\tHG002#1#x#0\t1+,2+\t*\nL\t1\t+\t2\t+\t0M\n' > $C/pb1.gfa +printf 'H\tVN:Z:1.0\nS\t3\tTTACG\nS\t4\tGGCAT\nP\tHG002#1#x#10\t3+,4+\t*\nL\t3\t+\t4\t+\t0M\n' > $C/pb2.gfa +printf 'H\tVN:Z:1.0\nS\t3\tTTACG\nS\t4\tGGCAT\nP\tHG002#1#x#1\t3+,4+\t*\nL\t3\t+\t4\t+\t0M\n' > $C/pbnum.gfa +vg convert -p $C/pb1.gfa > $C/pb1.pg 2>/dev/null +vg convert -p $C/pb2.gfa > $C/pb2.pg 2>/dev/null +vg convert -p $C/pbnum.gfa > $C/pbnum.pg 2>/dev/null + +# By default a phase block is just a phase block, not an offset, so these fragments do not merge. +vg combine $C/pb1.pg $C/pb2.pg > $C/o_pb.vg 2>/dev/null +is "$(vg paths -Ev $C/o_pb.vg | wc -l)" "2" "phase blocks is not merged by default" +is "$(vg paths -Ev $C/o_pb.vg | cut -f1 | sort | paste -sd,)" "HG002#1#x#0,HG002#1#x#10" "the untouched fragments keep their names" + +# -P reads the phase block as a start offset instead. +vg combine -P $C/pb1.pg $C/pb2.pg > $C/o_pbP.vg 2>/dev/null +is "$(vg paths -Ev $C/o_pbP.vg | cut -f1)" "HG002#1#x#0[0-20]" "-P merges by phase block into one spanning path" +is "$(vg paths -Ev $C/o_pbP.vg | cut -f2)" "20" "the merged length matches the span the blocks imply" + +# The phase block as a start offset is not continuous. +vg combine -P $C/pb1.pg $C/pbnum.pg > $C/o_pbnum.vg 2>$C/pb.err +is "$?" "1" "-P reads the phase block as a start offset, but they are not continuous" +is "$(grep -c 'rather than offsets' $C/pb.err)" "1" "the error asks whether these are numbers, not offsets" + +# A phase block that is not 0 has to survive the merge intact. +printf "$RS"'S\t1\tACGTA\nS\t2\tCCGGT\nL\t1\t+\t2\t+\t0M\nP\tHG002#2#chr22#4[0-10]\t1+,2+\t*\n' > $C/hb1.gfa +printf "$RS"'S\t3\tTTACG\nS\t4\tGGCAT\nL\t3\t+\t4\t+\t0M\nP\tHG002#2#chr22#4[10-20]\t3+,4+\t*\n' > $C/hb2.gfa +vg convert -p $C/hb1.gfa > $C/hb1.pg 2>/dev/null +vg convert -p $C/hb2.gfa > $C/hb2.pg 2>/dev/null + +is "$(vg paths -M -v $C/hb1.pg | tail -1 | cut -f2,6)" "$(printf 'HAPLOTYPE\t4')" "the fixture really is a HAPLOTYPE path with phase block 4" +vg combine $C/hb1.pg $C/hb2.pg > $C/o_hb.pg 2>/dev/null +is "$(vg paths -Ev $C/o_hb.pg)" "$(printf 'HG002#2#chr22#4[0-20]\t20')" "merging keeps the phase block and spans both fragments" + +# Error: a merged name that is already taken is rejected. +printf "$RS"'S\t1\tAAAACCCC\nP\tGRCh38#0#chr1[0-8]\t1+\t*\n' > $C/whole.gfa +vg convert -v $C/whole.gfa > $C/whole_lin.vg 2>/dev/null +vg circularize -p 'GRCh38#0#chr1[0-8]' $C/whole_lin.vg > $C/whole.vg 2>/dev/null +printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\n' > $C/fa.gfa +printf "$RS"'S\t1\tCCCC\nP\tGRCh38#0#chr1[4-8]\t1+\t*\n' > $C/fb.gfa +vg convert -p $C/fa.gfa > $C/fa.pg 2>/dev/null +vg convert -p $C/fb.gfa > $C/fb.pg 2>/dev/null +vg combine $C/whole.vg $C/fa.pg $C/fb.pg 2>$C/collide.err >/dev/null +is "$?" "1" "a merged name that is already taken is rejected" +is "$(grep -c 'already an unrelated path' $C/collide.err)" "1" "the error names the path standing in the way" + +# vg combine has to work on graphs vg itself produces, not just hand-built GFA. +sed 's/^>x/>SAMPLE#0#x/' small/x.fa > $C/pansn.fa +vg construct -r $C/pansn.fa > $C/base.vg +vg index -x $C/base.xg $C/base.vg +is "$(vg paths -M -v $C/base.vg | tail -1 | cut -f2)" "REFERENCE" "PanSN naming yields a REFERENCE-sense path" + +# Chunking by node range gives two chunks that meet end to end and share no node IDs, +# which is what the default policy expects. +vg chunk -x $C/base.xg -r 1:16 -c 0 > $C/adj1.vg +vg chunk -x $C/base.xg -r 17:32 -c 0 > $C/adj2.vg +vg combine $C/adj1.vg $C/adj2.vg > $C/o_real.vg 2>/dev/null +is "$(vg paths -Ev $C/o_real.vg)" "$(printf 'SAMPLE#0#x[0-1001]\t1001')" "two node-range chunks merge into one contig-spanning path" +is "$(vg stats -N $C/o_real.vg)" "32" "the merged graph keeps every input node" +is "$(vg stats -E $C/o_real.vg)" "31" "merging adds exactly one inter-fragment edge" + +# Whatever VPKG can load, combine can merge. +vg convert -p $C/adj1.vg > $C/adj1.pg +vg convert -a $C/adj2.vg > $C/adj2.hg +vg combine $C/adj1.pg $C/adj2.hg > $C/o_fmt.vg 2>/dev/null +is "$(vg paths -Ev $C/o_fmt.vg)" "$(vg paths -Ev $C/o_real.vg)" "PackedGraph and HashGraph inputs merge the same way" +is "$(vg stats -N $C/o_fmt.vg)" "32" "and keep the same nodes" + +rm -rf $C From 11d332729ef019876fc3578ef4b9435d9de8b0cc Mon Sep 17 00:00:00 2001 From: Zilan Xin Date: Sat, 10 Oct 2026 02:31:46 -0700 Subject: [PATCH 4/5] Update index test to use vg combine instead of removed vg concat Co-Authored-By: Claude Opus 5.5 (1M context) --- test/t/06_vg_index.t | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/t/06_vg_index.t b/test/t/06_vg_index.t index 466a802ecf0..f753feefc76 100644 --- a/test/t/06_vg_index.t +++ b/test/t/06_vg_index.t @@ -143,16 +143,15 @@ rm -rf x1337.gam x1337.sorted.gam.gai2 x1337.sorted.gam.gai x1337.sorted.gam vg construct -r small/x.fa -v small/x.vcf.gz >x.vg -vg construct -r small/x.fa -v small/x.vcf.gz >y.vg -vg construct -r small/x.fa -v small/x.vcf.gz >z.vg +vg construct -r small/y.fa -v small/y.vcf.gz >y.vg -vg concat x.vg y.vg z.vg >q.vg +vg combine x.vg y.vg >q.vg vg index -x q.xg q.vg is $? 0 "storage of multiple graphs in an index succeeds" -rm x.vg y.vg z.vg q.vg +rm x.vg y.vg q.vg rm -rf q.xg # Now test backward nodes From b2f290fc983bb7dc6ed09892b6704777648a9402 Mon Sep 17 00:00:00 2001 From: Zilan Xin Date: Mon, 26 Oct 2026 09:56:08 -0700 Subject: [PATCH 5/5] Address review comments on vg combine --- deps/libhandlegraph | 2 +- src/algorithms/fuse_nodes.cpp | 2 +- src/combine.cpp | 682 ++++++------- src/combine.hpp | 207 ++-- src/subcommand/combine_main.cpp | 30 +- src/unittest/combine.cpp | 1603 +++++++++++++++++-------------- src/unittest/fuse_nodes.cpp | 73 +- src/unittest/path_length.cpp | 54 ++ test/t/51_vg_combine.t | 542 ++++------- 9 files changed, 1716 insertions(+), 1479 deletions(-) create mode 100644 src/unittest/path_length.cpp diff --git a/deps/libhandlegraph b/deps/libhandlegraph index 07fd981f0f4..3d8c3c9b1e3 160000 --- a/deps/libhandlegraph +++ b/deps/libhandlegraph @@ -1 +1 @@ -Subproject commit 07fd981f0f43e9d4d8aa61a84169a9eeda69152a +Subproject commit 3d8c3c9b1e3f31b2f5ea785f68b1d06e5611d58f diff --git a/src/algorithms/fuse_nodes.cpp b/src/algorithms/fuse_nodes.cpp index ae07e9f8c25..743fe1c95c4 100644 --- a/src/algorithms/fuse_nodes.cpp +++ b/src/algorithms/fuse_nodes.cpp @@ -56,7 +56,7 @@ handle_t fuse_nodes(handlegraph::MutablePathDeletableHandleGraph* graph, }); // Handle a neighbour that is left or right itself, so the edges built below - // land on fused. + // connect to fused. auto translate = [&](const handle_t& h) { if (h == left || h == right) { return fused; diff --git a/src/combine.cpp b/src/combine.cpp index e4a78237b8c..2819b80bcde 100644 --- a/src/combine.cpp +++ b/src/combine.cpp @@ -7,6 +7,10 @@ #include "combine.hpp" #include +#include +#include + +#include #include "algorithms/fuse_nodes.hpp" @@ -18,25 +22,10 @@ using handlegraph::offset_t; using handlegraph::PathMetadata; using handlegraph::PathSense; using handlegraph::subrange_t; +using handlegraph::algorithms::get_path_length; -/// Compute the length of a path. -size_t GraphCombiner::compute_path_length(const PathHandleGraph* graph, - const path_handle_t& path) { - // A path-position graph already knows this. - if (const PathPositionHandleGraph* positional - = dynamic_cast(graph)) { - return positional->get_path_length(path); - } - size_t length = 0; - for (handle_t h : graph->scan_path(path)) { - length += graph->get_length(h); - } - return length; -} - -/// Return a path's start offset. -offset_t GraphCombiner::path_start_offset(const subrange_t& subrange, size_t phase_block, - bool phase_block_is_offset) { +offset_t GraphCombiner::path_start_offset(const subrange_t& subrange, + size_t phase_block) const { if (subrange != PathMetadata::NO_SUBRANGE) { return subrange.first; } @@ -46,8 +35,7 @@ offset_t GraphCombiner::path_start_offset(const subrange_t& subrange, size_t pha return 0; } -/// Raise every node ID above the previous graph's max ID; return the shift. -int64_t GraphCombiner::renumber_out_of_way(MutableHandleGraph* graph, int64_t above) { +int64_t GraphCombiner::shift_ids_above(MutableHandleGraph* graph, int64_t above) { int64_t delta = above - (int64_t)graph->min_node_id(); if (delta < 0) { return 0; @@ -57,20 +45,17 @@ int64_t GraphCombiner::renumber_out_of_way(MutableHandleGraph* graph, int64_t ab } GraphCombiner::GroupKey GraphCombiner::group_key_of(const PathHandleGraph& graph, - const path_handle_t& path, - bool phase_block_is_offset) { - // If phase blocks are coordinates, the key folds them out. + const path_handle_t& path) const { + // If phase blocks are coordinates, the key leaves them out. size_t phase_block = graph.get_phase_block(path); return GroupKey{graph.get_sense(path), graph.get_sample_name(path), graph.get_locus_name(path), graph.get_haplotype(path), - graph.get_is_circular(path), phase_block_is_offset ? PathMetadata::NO_PHASE_BLOCK : phase_block}; } -/// Read up to `length` bp off the front of a path. string GraphCombiner::path_prefix(const PathHandleGraph& graph, const path_handle_t& path, size_t length) { string sequence; @@ -85,7 +70,6 @@ string GraphCombiner::path_prefix(const PathHandleGraph& graph, const path_handl return sequence; } -/// Read up to `length` bp off the back of a path. string GraphCombiner::path_suffix(const PathHandleGraph& graph, const path_handle_t& path, size_t length) { if (graph.is_empty(path)) { @@ -107,135 +91,137 @@ string GraphCombiner::path_suffix(const PathHandleGraph& graph, const path_handl return sequence; } -/// Constructor. -GraphCombiner::GraphCombiner(SeamPolicy seam, bool fuse, bool phase_block_is_offset) - : seam(seam), fuse(fuse), phase_block_is_offset(phase_block_is_offset), - logger("vg combine") { +GraphCombiner::GraphCombiner(SeamPolicy seam_policy, bool fuse, bool phase_block_is_offset, + const string& log_context) + : seam_policy(seam_policy), fuse(fuse), phase_block_is_offset(phase_block_is_offset), + logger(log_context) { } -/// Combine the input graphs. unique_ptr GraphCombiner::combine(vector&& inputs) { // renumber and shared need at least one input graph; trim needs at least two. if (inputs.empty()) { - logger.error() << "At least one input graph is required." << endl; + throw std::invalid_argument("At least one input graph is required."); } - if (seam == SeamPolicy::COORD_TRIM && inputs.size() < 2) { - logger.error() << "--seam trim measures overlap between chunks, so it needs " - << "at least two input graphs." << endl; + if (seam_policy == SeamPolicy::COORD_TRIM && inputs.size() < 2) { + throw std::invalid_argument("Trimming measures overlap between inputs, so it needs " + "at least two input graphs."); } - // Turn each public Input into a private Chunk. - vector chunks; - chunks.reserve(inputs.size()); - for (Input& in : inputs) { - Chunk chunk; - chunk.name = std::move(in.name); - chunk.graph = std::move(in.graph); - chunks.push_back(std::move(chunk)); + // Check each input: no path may have multiple fragments within one graph, and + // every subrange end must match the path's sequence length. + for (const Input& in : inputs) { + check_no_split_paths(in); + check_subrange_matches_length(in); } - inputs.clear(); - // Check each input: no path may have multiple fragments within one graph. - for (const Chunk& chunk : chunks) { - check_no_split_paths(chunk); - } - - // For COORD_TRIM, read each chunk's REFERENCE identity and sort by reference - // then start offset. - if (seam == SeamPolicy::COORD_TRIM) { - for (Chunk& chunk : chunks) { - describe_reference(chunk); + if (seam_policy != SeamPolicy::COORD_TRIM) { + // Non-trim modes just ingest each input in turn. + for (Input& in : inputs) { + ingest(in); } - sort_by_reference_offset(chunks); + } else { + // Describe each input's reference path, and sort by base path then start offset. + vector chunks; + chunks.reserve(inputs.size()); + for (Input& in : inputs) { + ReferenceDescription ref = describe_reference(in); + chunks.push_back(Chunk{std::move(in), std::move(ref)}); + } + sort_by_base_path_offset(chunks); check_overlaps_agree(chunks); - } - for (size_t i = 0; i < chunks.size(); ++i) { - Chunk& chunk = chunks[i]; + for (size_t i = 0; i < chunks.size(); ++i) { + Chunk& chunk = chunks[i]; - // Non-trim modes just ingest each chunk in turn. - if (seam != SeamPolicy::COORD_TRIM) { - ingest(chunk); - continue; - } - - // The sort made each reference's chunks contiguous, so two bools settle - // whether this is that reference's first or last chunk. - auto found = seams.find(chunk.ref_key); - bool is_first_of_ref = (found == seams.end()); - bool is_last_of_ref = (i + 1 == chunks.size()) - || chunks[i + 1].ref_key != chunk.ref_key; - - if (!is_last_of_ref) { - check_all_paths_end_at(chunk); - } - if (!is_first_of_ref) { - check_all_paths_start_at(chunk); - } + // The sort made each base path's chunks contiguous, so two bools tell + // whether this is that base path's first or last chunk. + auto found = seams.find(chunk.ref.base_path_key); + bool is_first_of_ref = (found == seams.end()); + bool is_last_of_ref = (i + 1 == chunks.size()) + || chunks[i + 1].ref.base_path_key != chunk.ref.base_path_key; - // Handles will be invalidated, so record IDs and orientations first. - nid_t new_start_id = chunk.graph->get_id(chunk.ref_start_handle); - bool start_is_reverse = chunk.graph->get_is_reverse(chunk.ref_start_handle); - nid_t ref_end_id = chunk.graph->get_id(chunk.ref_end_handle); - bool end_is_reverse = chunk.graph->get_is_reverse(chunk.ref_end_handle); + if (!is_last_of_ref) { + check_all_paths_end_at_reference_last_node(chunk); + } + if (!is_first_of_ref) { + check_all_paths_start_at_reference_first_node(chunk); + } - // offset_t is unsigned, so check for a gap before subtracting; then - // trim_and_shift cuts the overlap away and gives the new start ID. - if (!is_first_of_ref) { - const SeamState& running = found->second; + // Handles will be invalidated, so record IDs and orientations first. + nid_t new_start_id = chunk.graph->get_id(chunk.ref.start_handle); + bool start_is_reverse = chunk.graph->get_is_reverse(chunk.ref.start_handle); + nid_t ref_end_id = chunk.graph->get_id(chunk.ref.end_handle); + bool end_is_reverse = chunk.graph->get_is_reverse(chunk.ref.end_handle); + + // offset_t is unsigned, so check for a gap before subtracting; then + // trim_and_shift cuts the overlap away and gives the new start ID. + if (!is_first_of_ref) { + const SeamState& running = found->second; + + if (running.is_circular != chunk.ref.is_circular) { + stringstream message; + message << "Reference path \"" << chunk.ref.path_name << "\" in \"" + << chunk.name << "\" is " + << (chunk.ref.is_circular ? "circular" : "not circular") + << ", but its base path in \"" << running.last_input_name + << "\" is " << (running.is_circular ? "circular" : "not circular") + << ". Pieces of one path have to agree on whether it is circular."; + throw InputConflictError(message.str()); + } + if (running.end_offset < chunk.ref.start_offset) { + stringstream message; + message << "Base path offset gap of " + << (chunk.ref.start_offset - running.end_offset) + << " bp between \"" << running.last_input_name + << "\" (its reference path ends at offset " << running.end_offset + << ") and \"" << chunk.name << "\" (its reference path starts at offset " + << chunk.ref.start_offset + << "). Trimming does not fill in gaps."; + throw ChunkOverlapError(message.str()); + } + offset_t overlap = running.end_offset - chunk.ref.start_offset; + size_t left_end_node_len = dest->get_length( + dest->get_handle(running.end_id, running.end_is_reverse)); + new_start_id = trim_and_shift(chunk, overlap, left_end_node_len); + } - if (running.end_offset < chunk.ref_start_offset) { - logger.error() << "REFERENCE offset gap of " - << (chunk.ref_start_offset - running.end_offset) - << " bp between \"" << running.source - << "\" (REFERENCE ends at offset " << running.end_offset - << ") and \"" << chunk.name << "\" (REFERENCE starts at offset " - << chunk.ref_start_offset - << "). --seam trim does not insert gaps." << endl; + // ingest() shifts the chunk's node IDs, so look up its end nodes in dest by + // their shifted IDs, then join it onto the end of the base path built so far. + int64_t shift = ingest(chunk); + + handle_t chunk_ref_end = dest->get_handle((nid_t)((int64_t)ref_end_id + shift), + end_is_reverse); + handle_t new_end = chunk_ref_end; + if (!is_first_of_ref) { + handle_t new_start = dest->get_handle((nid_t)((int64_t)new_start_id + shift), + start_is_reverse); + new_end = connect_seam(found->second, new_start, chunk_ref_end, chunk.name); } - offset_t overlap = running.end_offset - chunk.ref_start_offset; - size_t left_boundary_len = dest->get_length( - dest->get_handle(running.end_id, running.end_is_reverse)); - new_start_id = trim_and_shift(chunk, overlap, left_boundary_len); - } - - // Ingesting renumbers IDs, so re-fetch the handles by the shift, then - // seam onto the chunk to the left. - int64_t shift = ingest(chunk); - - handle_t chunk_ref_end = dest->get_handle((nid_t)((int64_t)ref_end_id + shift), - end_is_reverse); - handle_t new_end = chunk_ref_end; - if (!is_first_of_ref) { - handle_t new_start = dest->get_handle((nid_t)((int64_t)new_start_id + shift), - start_is_reverse); - new_end = connect_seam(found->second, chunk.name, new_start, - chunk_ref_end); - } - - // Update the SeamState for this chunk. - seams[chunk.ref_key] = SeamState{dest->get_id(new_end), - dest->get_is_reverse(new_end), - chunk.ref_end_offset, - chunk.name}; + + // Update the SeamState for this chunk. + seams[chunk.ref.base_path_key] = SeamState{dest->get_id(new_end), + dest->get_is_reverse(new_end), + chunk.ref.end_offset, + chunk.name, + chunk.ref.is_circular}; + } } + inputs.clear(); merge_path_fragments(); return std::move(dest); } -/// Check whether a chunk holds multiple fragments of a logical path. -void GraphCombiner::check_no_split_paths(const Chunk& chunk) const { +void GraphCombiner::check_no_split_paths(const Input& input) const { // Group by GroupKey, recording each path's start offset and name. map>> groups; - chunk.graph->for_each_path_handle([&](const path_handle_t& path) { - groups[group_key_of(*chunk.graph, path, phase_block_is_offset)] - .emplace_back(path_start_offset(chunk.graph->get_subrange(path), - chunk.graph->get_phase_block(path), - phase_block_is_offset), - chunk.graph->get_path_name(path)); + input.graph->for_each_path_handle([&](const path_handle_t& path) { + groups[group_key_of(*input.graph, path)] + .emplace_back(path_start_offset(input.graph->get_subrange(path), + input.graph->get_phase_block(path)), + input.graph->get_path_name(path)); }); for (auto& group : groups) { @@ -250,106 +236,129 @@ void GraphCombiner::check_no_split_paths(const Chunk& chunk) const { for (const auto& piece : pieces) { names += (names.empty() ? "\"" : ", \"") + piece.second + "\""; } - logger.error() << "\"" << chunk.name << "\" holds " << pieces.size() - << " pieces of one path (" << names << "). vg combine joins pieces " - << "that arrive in separate inputs; a path already split inside one " - << "input is not something it will reassemble." << endl; + stringstream message; + message << "\"" << input.name << "\" holds " << pieces.size() + << " pieces of one path (" << names << "). Only pieces that are in " + << "separate inputs are joined; a path already split inside one input is " + << "not reassembled."; + throw InputConflictError(message.str()); } } -/// Fill in `chunk`'s REFERENCE description and do some checks. -void GraphCombiner::describe_reference(Chunk& chunk) { +void GraphCombiner::check_subrange_matches_length(const Input& input) { + input.graph->for_each_path_handle([&](const path_handle_t& path) { + subrange_t subrange = input.graph->get_subrange(path); + if (subrange == PathMetadata::NO_SUBRANGE + || subrange.second == PathMetadata::NO_END_POSITION) { + return; + } + offset_t actual_end = subrange.first + (offset_t)get_path_length(input.graph.get(), path); + if (subrange.second != actual_end) { + stringstream message; + message << "Path \"" << input.graph->get_path_name(path) << "\" in \"" << input.name + << "\" claims to end at offset " << subrange.second + << ", but its sequence ends at offset " << actual_end << "."; + throw PathLengthMismatchError(message.str()); + } + }); +} + +GraphCombiner::ReferenceDescription GraphCombiner::describe_reference(const Input& input) const { // Collect every REFERENCE-sense path into refs. vector refs; - chunk.graph->for_each_path_handle([&](const path_handle_t& p) { - if (chunk.graph->get_sense(p) == PathSense::REFERENCE) { + input.graph->for_each_path_handle([&](const path_handle_t& p) { + if (input.graph->get_sense(p) == PathSense::REFERENCE) { refs.push_back(p); } }); - // Two checks: there must be exactly one REFERENCE path. + // Two checks: there must be exactly one REFERENCE-sense path. if (refs.empty()) { - logger.error() << "No REFERENCE-sense path found in \"" << chunk.name - << "\"; --seam trim reads each chunk's position off its " - << "REFERENCE path, and needs exactly one per graph." << endl; + stringstream message; + message << "No REFERENCE-sense path found in \"" << input.name + << "\"; trimming reads each input's position off its REFERENCE-sense path, " + << "and needs exactly one per graph."; + throw ReferencePathError(message.str()); } if (refs.size() > 1) { - logger.error() << refs.size() << " REFERENCE-sense paths found in \"" << chunk.name - << "\"; --seam trim needs exactly one, so it knows which one " - << "gives the chunk its position." << endl; + stringstream message; + message << refs.size() << " REFERENCE-sense paths found in \"" << input.name + << "\"; trimming needs exactly one, so it knows which one gives the " + << "input its position."; + throw ReferencePathError(message.str()); } - path_handle_t ref = refs[0]; - - // Record the two identity fields, ref_name and ref_key, and the REFERENCE - // path's start and end. - chunk.ref_name = chunk.graph->get_path_name(ref); - chunk.ref_key = RefKey{chunk.graph->get_sample_name(ref), - chunk.graph->get_locus_name(ref), - chunk.graph->get_haplotype(ref), - chunk.graph->get_is_circular(ref)}; - chunk.ref_start_offset = path_start_offset(chunk.graph->get_subrange(ref), - chunk.graph->get_phase_block(ref), - phase_block_is_offset); - chunk.ref_end_offset = chunk.ref_start_offset - + (offset_t)compute_path_length(chunk.graph.get(), ref); - - chunk.ref_start_handle = chunk.graph->get_handle_of_step(chunk.graph->path_begin(ref)); - chunk.ref_end_handle = chunk.graph->get_handle_of_step(chunk.graph->path_back(ref)); - - // Check boundary orientation: the REFERENCE path must visit its first and last + path_handle_t path = refs[0]; + + // Record the reference path's name, the base path it is part of, and where + // it starts and ends. + ReferenceDescription ref; + ref.path_name = input.graph->get_path_name(path); + ref.base_path_key = BasePathKey{input.graph->get_sample_name(path), + input.graph->get_locus_name(path), + input.graph->get_haplotype(path)}; + ref.is_circular = input.graph->get_is_circular(path); + ref.start_offset = path_start_offset(input.graph->get_subrange(path), + input.graph->get_phase_block(path)); + ref.end_offset = ref.start_offset + (offset_t)get_path_length(input.graph.get(), path); + + ref.start_handle = input.graph->get_handle_of_step(input.graph->path_begin(path)); + ref.end_handle = input.graph->get_handle_of_step(input.graph->path_back(path)); + + // Check boundary orientation: the reference path must visit its first and last // nodes forward. - if (chunk.graph->get_is_reverse(chunk.ref_start_handle) - || chunk.graph->get_is_reverse(chunk.ref_end_handle)) { - logger.error() << "REFERENCE path \"" << chunk.ref_name << "\" in \"" << chunk.name - << "\" visits its boundary node in reverse orientation; --seam trim " - << "only supports forward-strand boundaries." << endl; + if (input.graph->get_is_reverse(ref.start_handle) + || input.graph->get_is_reverse(ref.end_handle)) { + stringstream message; + message << "Reference path \"" << ref.path_name << "\" in \"" << input.name + << "\" visits its boundary node in reverse orientation; trimming only " + << "supports forward-strand boundaries."; + throw ReferencePathError(message.str()); } + return ref; } -/// Group each reference's chunks together and sort them by start position. -void GraphCombiner::sort_by_reference_offset(vector& chunks) { +void GraphCombiner::sort_by_base_path_offset(vector& chunks) { - // Different references keep first-seen order; chunks of one reference sort by + // Different base paths keep first-seen order; chunks of one base path sort by // ascending offset. - map first_seen; + map first_seen; for (const Chunk& chunk : chunks) { - first_seen.emplace(chunk.ref_key, first_seen.size()); + first_seen.emplace(chunk.ref.base_path_key, first_seen.size()); } std::stable_sort(chunks.begin(), chunks.end(), [&](const Chunk& a, const Chunk& b) { - size_t ra = first_seen.at(a.ref_key); - size_t rb = first_seen.at(b.ref_key); + size_t ra = first_seen.at(a.ref.base_path_key); + size_t rb = first_seen.at(b.ref.base_path_key); if (ra != rb) { return ra < rb; } - return a.ref_start_offset < b.ref_start_offset; + return a.ref.start_offset < b.ref.start_offset; }); } -/// Check that overlapping chunks spell the same sequence over the overlap. void GraphCombiner::check_overlaps_agree(const vector& chunks) const { - // The sort made each reference's chunks contiguous and offset-ascending, so only + // The sort made each base path's chunks contiguous and offset-ascending, so only // neighbours can overlap. for (size_t i = 1; i < chunks.size(); ++i) { const Chunk& left = chunks[i - 1]; const Chunk& right = chunks[i]; - // Skip a different reference, and a gap, which combine() reports itself. - if (right.ref_key != left.ref_key - || left.ref_end_offset <= right.ref_start_offset) { + // Skip a different base path, and a gap, which combine() reports itself. + if (right.ref.base_path_key != left.ref.base_path_key + || left.ref.end_offset <= right.ref.start_offset) { continue; } - offset_t overlap = left.ref_end_offset - right.ref_start_offset; + offset_t overlap = left.ref.end_offset - right.ref.start_offset; // Read the shared span off each side. A nested chunk reads short; the checks // in trim_and_shift catch that. string from_left = path_suffix(*left.graph, - left.graph->get_path_handle(left.ref_name), + left.graph->get_path_handle(left.ref.path_name), overlap); string from_right = path_prefix(*right.graph, - right.graph->get_path_handle(right.ref_name), + right.graph->get_path_handle(right.ref.path_name), overlap); // Find the first base they disagree on, if any. @@ -359,78 +368,81 @@ void GraphCombiner::check_overlaps_agree(const vector& chunks) const { ++at; } if (at < shared) { - logger.error() << "Chunks \"" << left.name << "\" and \"" << right.name - << "\" overlap on REFERENCE coordinates [" - << right.ref_start_offset << ", " << left.ref_end_offset - << ") but disagree there: at offset " - << (right.ref_start_offset + at) << " \"" << left.name - << "\" has " << from_left[at] << " and \"" << right.name - << "\" has " << from_right[at] - << ". --seam trim drops the right chunk's copy of the overlap, " - << "so the two have to spell the same sequence." << endl; + stringstream message; + message << "Chunks \"" << left.name << "\" and \"" << right.name + << "\" overlap on base path coordinates [" + << right.ref.start_offset << ", " << left.ref.end_offset + << ") but disagree there: at offset " + << (right.ref.start_offset + at) << " \"" << left.name + << "\" has " << from_left[at] << " and \"" << right.name + << "\" has " << from_right[at] + << ". Trimming drops the right chunk's copy of the overlap, " + << "so the two have to spell the same sequence."; + throw ChunkOverlapError(message.str()); } } } -/// Check that every path in the chunk starts at its REFERENCE start node. -void GraphCombiner::check_all_paths_start_at(const Chunk& chunk) const { +void GraphCombiner::check_all_paths_start_at_reference_first_node(const Chunk& chunk) const { chunk.graph->for_each_path_handle([&](const path_handle_t& p) { handle_t first = chunk.graph->get_handle_of_step(chunk.graph->path_begin(p)); - if (first != chunk.ref_start_handle) { - logger.error() << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" - << chunk.name << "\" does not start at the REFERENCE start node " - << "(id " << chunk.graph->get_id(chunk.ref_start_handle) - << "). --seam trim trims and joins at that one node, so every " - << "path in a chunk that has a chunk to its left has to begin " - << "there." << endl; + if (first != chunk.ref.start_handle) { + stringstream message; + message << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" + << chunk.name << "\" does not start at the reference path's start node " + << "(id " << chunk.graph->get_id(chunk.ref.start_handle) + << "). Trimming cuts and joins at that one node, so every path in a " + << "chunk that has a chunk to its left has to begin there."; + throw ChunkOverlapError(message.str()); } }); } -/// Check that every path in the chunk ends at its REFERENCE end node. -void GraphCombiner::check_all_paths_end_at(const Chunk& chunk) const { +void GraphCombiner::check_all_paths_end_at_reference_last_node(const Chunk& chunk) const { chunk.graph->for_each_path_handle([&](const path_handle_t& p) { handle_t last = chunk.graph->get_handle_of_step(chunk.graph->path_back(p)); - if (last != chunk.ref_end_handle) { - logger.error() << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" - << chunk.name << "\" does not end at the REFERENCE end node " - << "(id " << chunk.graph->get_id(chunk.ref_end_handle) - << "). --seam trim trims and joins at that one node, so every " - << "path in a chunk that has a chunk to its right has to end " - << "there." << endl; + if (last != chunk.ref.end_handle) { + stringstream message; + message << "Path \"" << chunk.graph->get_path_name(p) << "\" in \"" + << chunk.name << "\" does not end at the reference path's end node " + << "(id " << chunk.graph->get_id(chunk.ref.end_handle) + << "). Trimming cuts and joins at that one node, so every path in a " + << "chunk that has a chunk to its right has to end there."; + throw ChunkOverlapError(message.str()); } }); } -/// Trim `chunk`'s overlap with the chunk to its left, and fix up path coordinates. nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, - size_t left_boundary_len) { + size_t left_end_node_len) { - // A raw pointer to this chunk's graph, and the ID of its REFERENCE start node. + // A raw pointer to this chunk's graph, and the ID of its reference path's start node. MutablePathDeletableHandleGraph* graph = chunk.graph.get(); - nid_t new_start_id = graph->get_id(chunk.ref_start_handle); + nid_t new_start_id = graph->get_id(chunk.ref.start_handle); // Two checks: the overlap must be strictly shorter than the first node. if (overlap > 0) { - size_t start_len = graph->get_length(chunk.ref_start_handle); + size_t start_len = graph->get_length(chunk.ref.start_handle); if (overlap > start_len) { - logger.error() << "REFERENCE overlap of " << overlap << " bp before \"" - << chunk.name << "\" exceeds the length of its first node (" - << start_len << " bp on node " << new_start_id - << "), so the overlap is not all on that node. Re-chunk so it " - << "is." << endl; + stringstream message; + message << "Overlap of " << overlap << " bp before \"" + << chunk.name << "\" exceeds the length of its first node (" + << start_len << " bp on node " << new_start_id + << "), so the overlap is not all on that node. Re-chunk so it is."; + throw ChunkOverlapError(message.str()); } if (overlap == start_len) { - logger.error() << "REFERENCE overlap of " << overlap - << " bp exactly equals the length of the first node of \"" - << chunk.name << "\"; trimming it would leave a zero-length " - << "node. Re-chunk so that first node extends past the overlap." - << endl; + stringstream message; + message << "Overlap of " << overlap + << " bp exactly equals the length of the first node of \"" + << chunk.name << "\"; trimming it would leave a zero-length " + << "node. Re-chunk so that first node extends past the overlap."; + throw ChunkOverlapError(message.str()); } // Split the first node in two: the overlapping piece, and the remainder, // which becomes the new start. - vector pieces = graph->divide_handle(chunk.ref_start_handle, + vector pieces = graph->divide_handle(chunk.ref.start_handle, vector{overlap}); handle_t overlapping = pieces[0]; handle_t remainder = pieces[1]; @@ -450,18 +462,20 @@ nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, size_t inbound = 0; graph->follow_edges(overlapping, true, [&](const handle_t&) { inbound++; }); if (inbound > 0) { - logger.error() << "The first " << overlap << " bp of \"" << chunk.name - << "\" overlap the chunk to its left and have to be trimmed, " - << "but " << inbound << " edge(s) arrive at node " - << graph->get_id(overlapping) - << " there and would be lost. --seam trim needs the chunk " - << "boundary to be a clean left edge." << endl; + stringstream message; + message << "The first " << overlap << " bp of \"" << chunk.name + << "\" overlap the chunk to its left and have to be trimmed, " + << "but " << inbound << " edge(s) attach to node " + << graph->get_id(overlapping) + << " there and would be lost. Trimming needs the chunk " + << "boundary to be a clean left edge."; + throw ChunkOverlapError(message.str()); } graph->destroy_handle(overlapping); } // Compute the metadata shift. - int64_t meta_shift = (int64_t)overlap - (fuse ? (int64_t)left_boundary_len : 0); + int64_t meta_shift = (int64_t)overlap - (fuse ? (int64_t)left_end_node_len : 0); if (meta_shift == 0) { return new_start_id; } @@ -489,6 +503,21 @@ nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, s.phase_block = graph->get_phase_block(p); s.subrange = graph->get_subrange(p); s.is_circular = graph->get_is_circular(p); + // A path without coordinates starts at 0. + int64_t start = 0; + if (s.subrange != PathMetadata::NO_SUBRANGE) { + start = (int64_t)s.subrange.first; + } else if (phase_block_is_offset && s.phase_block != PathMetadata::NO_PHASE_BLOCK) { + start = (int64_t)s.phase_block; + } + if (start + meta_shift < 0) { + stringstream message; + message << "Path \"" << graph->get_path_name(p) << "\" in \"" << chunk.name + << "\" starts at offset " << start << ", but joining it on moves it back by " + << -meta_shift << " bp, which would put its start before 0. Its " + << "coordinates do not match the chunk to its left."; + throw ChunkOverlapError(message.str()); + } for (handle_t h : graph->scan_path(p)) { s.steps.push_back(h); } @@ -501,13 +530,11 @@ nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, for (PathSnapshot& s : snapshots) { // Shift each path's coordinates by meta_shift, and rebuild the path. if (s.subrange != PathMetadata::NO_SUBRANGE) { - s.subrange.first = - (offset_t)max(0, (int64_t)s.subrange.first + meta_shift); + s.subrange.first = (offset_t)((int64_t)s.subrange.first + meta_shift); } else if (phase_block_is_offset && s.phase_block != PathMetadata::NO_PHASE_BLOCK) { - s.phase_block = (size_t)max(0, (int64_t)s.phase_block + meta_shift); + s.phase_block = (size_t)((int64_t)s.phase_block + meta_shift); } else { - s.subrange = subrange_t{(offset_t)max(0, meta_shift), - PathMetadata::NO_END_POSITION}; + s.subrange = subrange_t{(offset_t)meta_shift, PathMetadata::NO_END_POSITION}; } path_handle_t rebuilt = graph->create_path(s.sense, s.sample, s.locus, s.haplotype, s.phase_block, s.subrange, s.is_circular); @@ -518,37 +545,35 @@ nid_t GraphCombiner::trim_and_shift(Chunk& chunk, offset_t overlap, return new_start_id; } -/// Move `chunk` into the accumulator; return how far its IDs were shifted. -int64_t GraphCombiner::ingest(Chunk& chunk) { +int64_t GraphCombiner::ingest(Input& input) { if (!dest) { - // The first graph becomes the accumulator: nothing to copy or renumber. - dest = std::move(chunk.graph); + // The first graph becomes dest: nothing to copy or renumber. + dest = std::move(input.graph); max_node_id = dest->max_node_id(); return 0; } int64_t shift = 0; - if (seam == SeamPolicy::SHARED_IDS) { - share_nodes_by_id(chunk); - max_node_id = std::max(max_node_id, (int64_t)chunk.graph->max_node_id()); + if (seam_policy == SeamPolicy::SHARED_IDS) { + share_nodes_by_id(input); + max_node_id = std::max(max_node_id, (int64_t)input.graph->max_node_id()); } else { - shift = renumber_out_of_way(chunk.graph.get(), max_node_id); - max_node_id = std::max(max_node_id, (int64_t)chunk.graph->max_node_id()); - handlealgs::copy_handle_graph(chunk.graph.get(), dest.get()); + shift = shift_ids_above(input.graph.get(), max_node_id); + max_node_id = std::max(max_node_id, (int64_t)input.graph->max_node_id()); + handlealgs::copy_handle_graph(input.graph.get(), dest.get()); } - copy_paths_checked(chunk); + copy_paths_checked(input); // Free the source rather than holding every input until combine finishes. - chunk.graph.reset(); + input.graph.reset(); return shift; } -/// Copy nodes and edges, keeping IDs as given and distinguishing nodes by ID. -void GraphCombiner::share_nodes_by_id(Chunk& chunk) { +void GraphCombiner::share_nodes_by_id(Input& input) { - chunk.graph->for_each_handle([&](const handle_t& source) { - nid_t id = chunk.graph->get_id(source); - string sequence = chunk.graph->get_sequence(source); + input.graph->for_each_handle([&](const handle_t& source) { + nid_t id = input.graph->get_id(source); + string sequence = input.graph->get_sequence(source); if (!dest->has_node(id)) { dest->create_handle(sequence, id); return; @@ -556,71 +581,66 @@ void GraphCombiner::share_nodes_by_id(Chunk& chunk) { // Under this policy a repeated ID is the same node, so two sequences for // one ID is a contradiction with no way to keep both. if (dest->get_sequence(dest->get_handle(id)) != sequence) { - logger.error() << "Node " << id << " has different sequences in \"" << chunk.name - << "\" and in an earlier input. Under --seam shared a repeated " - << "node ID means the same node, so the sequences have to match." - << endl; + stringstream message; + message << "Node " << id << " has different sequences in \"" << input.name + << "\" and in an earlier input. When node IDs are shared, a repeated " + << "node ID means the same node, so the sequences have to match."; + throw InputConflictError(message.str()); } }); - chunk.graph->for_each_edge([&](const edge_t& source) { - handle_t from = dest->get_handle(chunk.graph->get_id(source.first), - chunk.graph->get_is_reverse(source.first)); - handle_t to = dest->get_handle(chunk.graph->get_id(source.second), - chunk.graph->get_is_reverse(source.second)); - if (!dest->has_edge(from, to)) { - dest->create_edge(from, to); - } + input.graph->for_each_edge([&](const edge_t& source) { + handle_t from = dest->get_handle(input.graph->get_id(source.first), + input.graph->get_is_reverse(source.first)); + handle_t to = dest->get_handle(input.graph->get_id(source.second), + input.graph->get_is_reverse(source.second)); + dest->create_edge(from, to); }); } -/// Copy paths, rejecting duplicate names. -void GraphCombiner::copy_paths_checked(Chunk& chunk) { +void GraphCombiner::copy_paths_checked(Input& input) { - chunk.graph->for_each_path_handle([&](const path_handle_t& source) { - string path_name = chunk.graph->get_path_name(source); + input.graph->for_each_path_handle([&](const path_handle_t& source) { + string path_name = input.graph->get_path_name(source); if (dest->has_path(path_name)) { - logger.error() << "Path \"" << path_name << "\" is in \"" << chunk.name - << "\" and in an earlier input. Pieces of one path have to be " - << "told apart by name, so give them distinct subranges." << endl; + stringstream message; + message << "Path \"" << path_name << "\" is in \"" << input.name + << "\" and in an earlier input. Pieces of one path have to be " + << "told apart by name, so give them distinct subranges."; + throw InputConflictError(message.str()); } - handlealgs::copy_path(chunk.graph.get(), source, dest.get()); + handlealgs::copy_path(input.graph.get(), source, dest.get()); }); } -/// Join the left chunk's end to this chunk's start; return the reference's new end. -handle_t GraphCombiner::connect_seam(const SeamState& running, const string& chunk_name, - handle_t new_start, handle_t chunk_ref_end) { +handle_t GraphCombiner::connect_seam(const SeamState& base_path_end, handle_t chunk_start, + handle_t chunk_end, const string& input_name) { - handle_t left = dest->get_handle(running.end_id, running.end_is_reverse); + handle_t left = dest->get_handle(base_path_end.end_id, base_path_end.end_is_reverse); // If not fusing nodes, just add an edge. if (!fuse) { - if (!dest->has_edge(left, new_start)) { - dest->create_edge(left, new_start); - } - return chunk_ref_end; + dest->create_edge(left, chunk_start); + return chunk_end; } // can_fuse_nodes decides whether the two nodes can be fused. - if (!algorithms::can_fuse_nodes(*dest, left, new_start)) { - // Welding buries both facing ends, and something is attached to one. - logger.error() << "--fuse cannot weld node " << dest->get_id(left) << " from \"" - << running.source << "\" to node " << dest->get_id(new_start) - << " from \"" << chunk_name << "\": one of them still has an edge " - << "on the side being welded, which the weld would have to drop. " - << "--seam trim without --fuse joins them with an edge instead and " - << "keeps everything." << endl; + if (!algorithms::can_fuse_nodes(*dest, left, chunk_start)) { + // Fusing removes both facing ends, and something is attached to one. + stringstream message; + message << "Cannot fuse node " << dest->get_id(left) << " from \"" + << base_path_end.last_input_name << "\" to node " << dest->get_id(chunk_start) + << " from \"" << input_name << "\": one of them still has an edge " + << "on the side being fused, which fusing would have to drop."; + throw FuseError(message.str()); } // Note the end node before fusing, fuse into a new node, and update the max ID. - bool ref_ends_at_seam = (dest->get_id(chunk_ref_end) == dest->get_id(new_start)); - handle_t fused = algorithms::fuse_nodes(dest.get(), left, new_start); + bool ref_ends_at_seam = (dest->get_id(chunk_end) == dest->get_id(chunk_start)); + handle_t fused = algorithms::fuse_nodes(dest.get(), left, chunk_start); max_node_id = std::max(max_node_id, (int64_t)dest->get_id(fused)); - return ref_ends_at_seam ? fused : chunk_ref_end; + return ref_ends_at_seam ? fused : chunk_end; } -/// Merge fragments from the same path into a single path spanning their entire -/// coordinate range. void GraphCombiner::merge_path_fragments() { struct Fragment { @@ -629,6 +649,7 @@ void GraphCombiner::merge_path_fragments() { size_t phase_block; ///< Phase block. size_t length; ///< Sequence length. bool positioned_by_phase_block; ///< Whether the phase block gives its position. + bool is_circular; ///< Whether it is circular. }; map> groups; @@ -638,18 +659,21 @@ void GraphCombiner::merge_path_fragments() { f.handle = path; f.subrange = dest->get_subrange(path); f.phase_block = dest->get_phase_block(path); - f.length = compute_path_length(dest.get(), path); + f.length = get_path_length(dest.get(), path); f.positioned_by_phase_block = phase_block_is_offset && f.subrange == PathMetadata::NO_SUBRANGE && f.phase_block != PathMetadata::NO_PHASE_BLOCK; - groups[group_key_of(*dest, path, phase_block_is_offset)].push_back(std::move(f)); + f.is_circular = dest->get_is_circular(path); + groups[group_key_of(*dest, path)].push_back(std::move(f)); }); // Lambdas computing a fragment's start and end coordinates. auto fragment_start = [&](const Fragment& f) -> offset_t { - return path_start_offset(f.subrange, f.phase_block, phase_block_is_offset); + return path_start_offset(f.subrange, f.phase_block); }; auto fragment_end = [&](const Fragment& f) -> offset_t { + // Fusing lengthens the left fragment's last node, so its measured length can + // overshoot; the subrange end, checked against the input, is the real end. if (f.subrange != PathMetadata::NO_SUBRANGE && f.subrange.second != PathMetadata::NO_END_POSITION) { return f.subrange.second; @@ -668,24 +692,36 @@ void GraphCombiner::merge_path_fragments() { const string& sample = std::get<1>(group.first); const string& locus = std::get<2>(group.first); size_t haplotype = std::get<3>(group.first); - bool is_circular = std::get<4>(group.first); + + // Circularity is not part of the key, so the fragments have to agree on it. + bool is_circular = frags.front().is_circular; + for (const Fragment& f : frags) { + if (f.is_circular != is_circular) { + stringstream message; + message << "Fragments \"" << dest->get_path_name(frags.front().handle) + << "\" and \"" << dest->get_path_name(f.handle) + << "\" are pieces of one path, but only one of them is circular."; + throw InputConflictError(message.str()); + } + } std::sort(frags.begin(), frags.end(), [&](const Fragment& a, const Fragment& b) { return fragment_start(a) < fragment_start(b); }); - // Detect any gap, updating the furthest reference offset reached so far. + // Detect any gap, updating the furthest offset along the base path reached so far. offset_t covered_end = fragment_end(frags.front()); for (size_t i = 1; i < frags.size(); ++i) { offset_t next_start = fragment_start(frags[i]); if (covered_end < next_start) { - logger.error() << "Gap of " << (next_start - covered_end) - << " bp before fragment \"" - << dest->get_path_name(frags[i].handle) - << "\"; the fragments before it only reach offset " - << covered_end << ". vg combine does not insert gaps." - << endl; + stringstream message; + message << "Gap of " << (next_start - covered_end) + << " bp before fragment \"" + << dest->get_path_name(frags[i].handle) + << "\"; the fragments before it only reach offset " + << covered_end << ". Gaps between fragments are not filled in."; + throw FragmentMergeError(message.str()); } covered_end = std::max(covered_end, fragment_end(frags[i])); } @@ -698,13 +734,14 @@ void GraphCombiner::merge_path_fragments() { } offset_t expected = (offset_t)frags[i].phase_block + (offset_t)frags[i].length; if (expected != (offset_t)frags[i + 1].phase_block) { - logger.error() << "--phase-block-is-offset given, but phase blocks of \"" - << dest->get_path_name(frags[i].handle) - << "\" do not line up as coordinates (block " - << frags[i].phase_block << " + length " << frags[i].length - << " != block " << frags[i + 1].phase_block - << "). Are these phase block numbers rather than offsets?" - << endl; + stringstream message; + message << "Phase blocks are read as offsets, but phase blocks of \"" + << dest->get_path_name(frags[i].handle) + << "\" do not line up as coordinates (block " + << frags[i].phase_block << " + length " << frags[i].length + << " != block " << frags[i + 1].phase_block + << "). Are these phase block numbers rather than offsets?"; + throw FragmentMergeError(message.str()); } } @@ -717,8 +754,8 @@ void GraphCombiner::merge_path_fragments() { subrange_t combined_subrange{combined_start, combined_end}; // HAPLOTYPE requires a phase block. Keep the group's own; treating phase blocks - // as coordinates folded it into the subrange, so 0 stands in there. - size_t group_phase_block = std::get<5>(group.first); + // as coordinates added it to the subrange, so 0 is used there. + size_t group_phase_block = std::get<4>(group.first); size_t merged_phase_block = PathMetadata::NO_PHASE_BLOCK; if (sense == PathSense::HAPLOTYPE) { merged_phase_block = (group_phase_block == PathMetadata::NO_PHASE_BLOCK) @@ -729,7 +766,7 @@ void GraphCombiner::merge_path_fragments() { combined_subrange); // Can't lean on create_path(): PackedGraph throws on a duplicate, HashGraph - // quietly orphans the old path, and VG just appends to it. So check here. + // leaves the old path unreachable by name, and VG just appends to it. So check here. if (dest->has_path(merged_name)) { bool is_own_fragment = false; for (const Fragment& f : frags) { @@ -739,9 +776,8 @@ void GraphCombiner::merge_path_fragments() { } } if (!is_own_fragment) { - logger.error() << "The merge of " << frags.size() << " fragments would be " - << "named \"" << merged_name << "\", which is already an " - << "unrelated path in the output." << endl; + // A path with this name has the same GroupKey, so it is one of frags. + throw std::logic_error("merged path name collides with a path outside its group"); } } @@ -770,9 +806,9 @@ void GraphCombiner::merge_path_fragments() { if (steps.empty()) { continue; } - // `skip` is the shared run at the seam: `run` is the candidate length, tried - // longest first down to 1 (a run of 0 would match trivially), and `k` compares - // `previous`'s tail against `steps`'s head. + // `skip` is how many steps at the start of `steps` repeat the end of `previous`. + // Try each length `run`, longest first, down to 1 (a run of 0 would always match); + // `k` goes through the run, comparing step k of `steps` with step k of `previous`'s last `run` steps. size_t skip = 0; if (previous != nullptr) { size_t most = std::min(previous->size(), steps.size()); @@ -789,7 +825,7 @@ void GraphCombiner::merge_path_fragments() { break; } } - if (skip == 0 && !dest->has_edge(previous->back(), steps.front())) { + if (skip == 0) { // Nothing shared, so they abut and the graph must say so. dest->create_edge(previous->back(), steps.front()); } diff --git a/src/combine.hpp b/src/combine.hpp index 6ebc02a3064..1c155bb43be 100644 --- a/src/combine.hpp +++ b/src/combine.hpp @@ -4,10 +4,14 @@ /** \file combine.hpp * * Combining graphs that hold fragments of the same logical paths. + * Seaming joins the next input onto the end of a base path built so far: the + * overlap is trimmed off, and the two pieces are joined at the seam by an edge + * or by fusing nodes. */ #include #include +#include #include #include #include @@ -22,66 +26,117 @@ namespace unittest { class TestGraphCombiner; } -/// The three modes of vg combine, selected by --seam. +/// How overlap between the inputs is resolved. enum class SeamPolicy { - RENUMBER, ///< --seam renumber - SHARED_IDS, ///< --seam shared - COORD_TRIM ///< --seam trim + RENUMBER, ///< Inputs don't overlap; node IDs are shifted to avoid collisions. + SHARED_IDS, ///< Nodes with the same ID are the same node. + COORD_TRIM ///< Overlap in reference coordinates is trimmed off. +}; + +/// Thrown when GraphCombiner's inputs cannot be combined. +class CombineError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/// An input's reference path is missing, ambiguous, or cannot be trimmed at. +class ReferencePathError : public CombineError { +public: + using CombineError::CombineError; +}; + +/// Neighbouring chunks of a base path do not fit together where they meet. +class ChunkOverlapError : public CombineError { +public: + using CombineError::CombineError; +}; + +/// Inputs contradict each other, or one input contradicts itself. +class InputConflictError : public CombineError { +public: + using CombineError::CombineError; +}; + +/// Two boundary nodes cannot be fused into one. +class FuseError : public CombineError { +public: + using CombineError::CombineError; +}; + +/// Fragments of one path cannot be merged into a single path. +class FragmentMergeError : public CombineError { +public: + using CombineError::CombineError; +}; + +/// A path's subrange claims a different end than its sequence gives it. +class PathLengthMismatchError : public CombineError { +public: + using CombineError::CombineError; }; /// Combine fragmented graphs into a whole graph. class GraphCombiner { public: + /// A graph to combine. struct Input { - std::string name; - std::unique_ptr graph; + std::string name; ///< Identifies the graph in error messages, usually the original filename. + std::unique_ptr graph; ///< The graph itself. }; - /// Constructor. - GraphCombiner(SeamPolicy seam, bool fuse, bool phase_block_is_offset); + /// Make a combiner. seam_policy is the mode for resolving overlap between the inputs; + /// fuse merges the two boundary nodes into one where trimmed inputs are joined, + /// instead of adding an edge; phase_block_is_offset reads a path's phase block + /// as its start coordinate; log_context prefixes progress messages. + GraphCombiner(SeamPolicy seam_policy, bool fuse, bool phase_block_is_offset, + const std::string& log_context); /// Combine the input graphs. + /// Throws a CombineError if the graphs cannot be combined, or std::invalid_argument + /// if there are not enough graphs: none, or only one when trimming. std::unique_ptr combine(std::vector&& inputs); private: - /// Identity of a reference: sample, locus, haplotype, circular. - using RefKey = std::tuple; + /// Identity of a base path: sample, locus, haplotype. + using BasePathKey = std::tuple; - /// Seaming state. + /// The current end of a base path in the combined graph, where the next input is seamed on. struct SeamState { - handlegraph::nid_t end_id; ///< Node the reference so far ends on. + handlegraph::nid_t end_id; ///< Node the base path so far ends on. bool end_is_reverse; ///< Its orientation. - handlegraph::offset_t end_offset; ///< REFERENCE offset it ends at. - std::string source; ///< Name of chunk/inputs. + handlegraph::offset_t end_offset; ///< Base path offset it ends at. + std::string last_input_name; ///< Name of the input that last extended this base path, for error messages. + bool is_circular = false; ///< Whether this base path is circular. }; - /// An input, plus the REFERENCE description COORD_TRIM positions it by. - struct Chunk { - std::string name; ///< Name of the input. - std::unique_ptr graph; ///< The input graph. - - std::string ref_name; ///< Its REFERENCE path. - handle_t ref_start_handle; ///< First node of that path. - handle_t ref_end_handle; ///< Last node of that path. - handlegraph::offset_t ref_start_offset = 0; ///< Where that path starts. - handlegraph::offset_t ref_end_offset = 0; ///< Where that path ends. - RefKey ref_key; ///< Which reference it belongs to. + /// An input's reference path: its name, its base path, its first and last nodes, where it starts and ends on that base path, and whether it is circular. + struct ReferenceDescription { + std::string path_name; ///< Name of its reference path in this chunk's graph, such as GRCh38#0#chr1[100-200], for looking it up and for error reporting. + BasePathKey base_path_key; ///< The base path its reference path is a piece of, such as {"GRCh38", "chr1", 0}. + handle_t start_handle; ///< First node of that path. + handle_t end_handle; ///< Last node of that path. + handlegraph::offset_t start_offset = 0; ///< Where that path starts on the base path. + handlegraph::offset_t end_offset = 0; ///< Where that path ends on the base path. + bool is_circular = false; ///< Whether that path is circular. }; - /// A path's identity as a comparable tuple: sense, sample, locus, haplotype, circular, phase block. + /// An input, plus a description of its reference path. + struct Chunk : Input { + ReferenceDescription ref; + }; + + /// A path's identity as a comparable tuple: sense, sample, locus, haplotype, phase block. using GroupKey = std::tuple; + size_t, size_t>; + /// Build a path's GroupKey. + GroupKey group_key_of(const PathHandleGraph& graph, const path_handle_t& path) const; - /// Compute the length of a path. - static size_t compute_path_length(const PathHandleGraph* graph, const path_handle_t& path); /// Return a path's start offset. - static handlegraph::offset_t path_start_offset(const handlegraph::subrange_t& subrange, - size_t phase_block, bool phase_block_is_offset); - /// Raise every node ID above the previous graph's max ID; return the shift. - static int64_t renumber_out_of_way(MutableHandleGraph* graph, int64_t above); - /// Build a path's GroupKey. - static GroupKey group_key_of(const PathHandleGraph& graph, const path_handle_t& path, - bool phase_block_is_offset); + handlegraph::offset_t path_start_offset(const handlegraph::subrange_t& subrange, + size_t phase_block) const; + /// Increase graph nodes IDs so that the minimum node ID is greater than "above". + /// Returns the amount IDs were incremented by, which is 0 if no IDs needed to change. + static int64_t shift_ids_above(MutableHandleGraph* graph, int64_t above); /// Read up to `length` bp off the front of a path. static std::string path_prefix(const PathHandleGraph& graph, const path_handle_t& path, size_t length); @@ -89,47 +144,63 @@ class GraphCombiner { static std::string path_suffix(const PathHandleGraph& graph, const path_handle_t& path, size_t length); - /// Check whether a chunk holds multiple fragments of a logical path. - void check_no_split_paths(const Chunk& chunk) const; - /// Fill in `chunk`'s REFERENCE description and do some checks. - void describe_reference(Chunk& chunk); - /// Group each reference's chunks together and sort them by start position. - void sort_by_reference_offset(std::vector& chunks); - /// Check that overlapping chunks spell the same sequence over the overlap. + /// Throw InputConflictError if the input holds more than one fragment of a logical path. + void check_no_split_paths(const Input& input) const; + /// Throw PathLengthMismatchError if a path in the input has a subrange end other than its start plus its sequence length. + static void check_subrange_matches_length(const Input& input); + /// Describe `input`'s reference path. Throw ReferencePathError if it does not have + /// exactly one, or if that path visits its first or last node in reverse. + ReferenceDescription describe_reference(const Input& input) const; + /// Group each base path's chunks together and sort them by start position. + void sort_by_base_path_offset(std::vector& chunks); + /// Throw ChunkOverlapError if two neighbouring chunks of one base path spell different sequences where they overlap. + /// The chunks must already be sorted by sort_by_base_path_offset(). void check_overlaps_agree(const std::vector& chunks) const; - /// Check that every path in the chunk starts at its REFERENCE start node. - void check_all_paths_start_at(const Chunk& chunk) const; - /// Check that every path in the chunk ends at its REFERENCE end node. - void check_all_paths_end_at(const Chunk& chunk) const; - - /// Trim `chunk`'s overlap with the chunk to its left, and fix up path coordinates. + /// Throw ChunkOverlapError if any path in the chunk does not start at its reference path's first node. + void check_all_paths_start_at_reference_first_node(const Chunk& chunk) const; + /// Throw ChunkOverlapError if any path in the chunk does not end at its reference path's last node. + void check_all_paths_end_at_reference_last_node(const Chunk& chunk) const; + + /// Cut the overlap off the front of `chunk` and shift its paths' coordinates back to match. + /// `chunk` is the right chunk, being joined onto the end of the base path built so far. + /// `overlap` is how many bp of `chunk` the base path already covers; it must be shorter than `chunk`'s first node. + /// `left_end_node_len` is the length of the base path's current last node, which fusing adds in front of `chunk`. + /// Returns the ID of `chunk`'s new first node. + /// Throws ChunkOverlapError if the shift would move a path's start coordinate below 0. handlegraph::nid_t trim_and_shift(Chunk& chunk, handlegraph::offset_t overlap, - size_t left_boundary_len); - - /// Move `chunk` into the accumulator; return how far its IDs were shifted. - int64_t ingest(Chunk& chunk); - /// Copy nodes and edges, keeping IDs as given and distinguishing nodes by ID. - void share_nodes_by_id(Chunk& chunk); - /// Copy paths, rejecting duplicate names. - void copy_paths_checked(Chunk& chunk); - - /// Join the left chunk's end to this chunk's start; return the reference's new end. - handle_t connect_seam(const SeamState& seam, const std::string& chunk_name, - handle_t new_start, handle_t chunk_ref_end); + size_t left_end_node_len); + + /// Add `input`'s nodes, edges and paths to dest, and free its graph. + /// Unless nodes are shared by ID, its node IDs are first shifted above every ID already in dest; returns the shift, or 0 if none. + int64_t ingest(Input& input); + /// Copy `input`'s nodes and edges into dest, keeping their IDs; a node whose ID is already in dest is the same node and is not copied again. + /// Throw InputConflictError if such a node has a different sequence. + void share_nodes_by_id(Input& input); + /// Copy `input`'s paths into dest; throw InputConflictError if a path name is already in dest. + void copy_paths_checked(Input& input); + + /// Join a chunk already copied into dest onto the end of the base path built so far, by an edge or, when fusing, by fusing the two nodes. + /// `base_path_end` is where that base path ends. + /// `chunk_start` and `chunk_end` are the chunk's reference path's first and last nodes in dest. + /// `input_name` names the chunk's input in error messages. + /// Returns the base path's new last node. + /// Throws FuseError if fusing is on and the two nodes cannot be fused. + handle_t connect_seam(const SeamState& base_path_end, handle_t chunk_start, + handle_t chunk_end, const std::string& input_name); /// Merge fragments from the same path into a single path spanning their entire /// coordinate range. void merge_path_fragments(); - SeamPolicy seam; - bool fuse; - bool phase_block_is_offset; - Logger logger; + SeamPolicy seam_policy; ///< The mode for resolving overlap between the inputs. + bool fuse; ///< Merge the two boundary nodes into one where trimmed inputs are joined, instead of adding an edge. + bool phase_block_is_offset; ///< Read a path's phase block as its start coordinate. + Logger logger; ///< Reports progress. - std::unique_ptr dest; - int64_t max_node_id = 0; - std::map seams; + std::unique_ptr dest; ///< The combined graph built so far. + int64_t max_node_id = 0; ///< Largest node ID used in dest so far. + std::map seams; ///< For each base path, the node, orientation, offset and input it ends at so far. friend class unittest::TestGraphCombiner; }; diff --git a/src/subcommand/combine_main.cpp b/src/subcommand/combine_main.cpp index 59d28cbbe21..004af71682e 100644 --- a/src/subcommand/combine_main.cpp +++ b/src/subcommand/combine_main.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -27,12 +28,16 @@ void help_combine(char** argv) { cerr << "usage: " << argv[0] << " combine [options] [graph2.vg ...] >merged.vg" << endl << "Combines one or more graphs into a single file." << endl << "Input graphs must not share any path name." << endl + << "Pieces of one path in different inputs must all run in the same direction." << endl << endl << "Options:" << endl << " -s, --seam POLICY how to resolve overlap between adjacent chunks:" << endl << " renumber assume no overlap [default]" << endl << " shared overlap by shared nodes" << endl - << " trim overlap in REFERENCE coordinates" << endl + << " trim overlap in reference path coordinates." << endl + << " Every path must run in the same direction" << endl + << " as the reference path, and start and end" << endl + << " on its end nodes where chunks are joined." << endl << " -u, --fuse requires --seam trim. Fuse the boundary nodes into" << endl << " one node instead of adding an edge." << endl << " -P, --phase-block-is-offset" << endl @@ -54,7 +59,7 @@ static SeamPolicy parse_seam_policy(const string& value, const Logger& logger) { } logger.error() << "unrecognized --seam policy \"" << value << "\"; expected one of: renumber, shared, trim." << endl; - return SeamPolicy::RENUMBER; // not reached; logger.error() exits + throw std::logic_error("unreachable"); // logger.error() stream exits after logging } int main_combine(int argc, char** argv) { @@ -65,7 +70,7 @@ int main_combine(int argc, char** argv) { return 1; } - SeamPolicy seam = SeamPolicy::RENUMBER; + SeamPolicy seam_policy = SeamPolicy::RENUMBER; bool fuse = false; bool phase_block_is_offset = false; @@ -94,7 +99,7 @@ int main_combine(int argc, char** argv) { switch (c) { case 's': - seam = parse_seam_policy(optarg, logger); + seam_policy = parse_seam_policy(optarg, logger); break; case 'u': fuse = true; @@ -114,7 +119,7 @@ int main_combine(int argc, char** argv) { } // --fuse requires --seam trim. - if (fuse && seam != SeamPolicy::COORD_TRIM) { + if (fuse && seam_policy != SeamPolicy::COORD_TRIM) { logger.error() << "--fuse/-u requires --seam trim." << endl; } @@ -126,9 +131,18 @@ int main_combine(int argc, char** argv) { inputs.push_back(std::move(in)); } - GraphCombiner combiner(seam, fuse, phase_block_is_offset); - unique_ptr combined = - combiner.combine(std::move(inputs)); + GraphCombiner combiner(seam_policy, fuse, phase_block_is_offset, "vg combine"); + unique_ptr combined; + try { + combined = combiner.combine(std::move(inputs)); + } catch (const FuseError& e) { + logger.error() << e.what() << " Without --fuse, --seam trim joins them with an " + << "edge instead and keeps everything." << endl; + } catch (const CombineError& e) { + logger.error() << e.what() << endl; + } catch (const std::invalid_argument& e) { + logger.error() << e.what() << endl; + } vg::io::save_handle_graph(combined.get(), cout); return 0; } diff --git a/src/unittest/combine.cpp b/src/unittest/combine.cpp index c1223ae7b77..a1e1be69c2d 100644 --- a/src/unittest/combine.cpp +++ b/src/unittest/combine.cpp @@ -13,7 +13,6 @@ #include "../combine.hpp" #include "bdsg/hash_graph.hpp" -#include "bdsg/overlays/path_position_overlays.hpp" namespace vg { namespace unittest { @@ -28,23 +27,25 @@ using handlegraph::subrange_t; // A child class to expose the internals for testing. class TestGraphCombiner : public GraphCombiner { public: - using GraphCombiner::GraphCombiner; + TestGraphCombiner(SeamPolicy seam_policy, bool fuse, bool phase_block_is_offset) + : GraphCombiner(seam_policy, fuse, phase_block_is_offset, "test") { + } using GraphCombiner::Chunk; using GraphCombiner::SeamState; - using GraphCombiner::RefKey; + using GraphCombiner::BasePathKey; using GraphCombiner::GroupKey; - using GraphCombiner::compute_path_length; using GraphCombiner::path_start_offset; - using GraphCombiner::renumber_out_of_way; + using GraphCombiner::shift_ids_above; using GraphCombiner::group_key_of; using GraphCombiner::path_prefix; using GraphCombiner::path_suffix; using GraphCombiner::check_no_split_paths; + using GraphCombiner::check_subrange_matches_length; using GraphCombiner::describe_reference; - using GraphCombiner::sort_by_reference_offset; + using GraphCombiner::sort_by_base_path_offset; using GraphCombiner::check_overlaps_agree; - using GraphCombiner::check_all_paths_start_at; - using GraphCombiner::check_all_paths_end_at; + using GraphCombiner::check_all_paths_start_at_reference_first_node; + using GraphCombiner::check_all_paths_end_at_reference_last_node; using GraphCombiner::trim_and_shift; using GraphCombiner::ingest; using GraphCombiner::share_nodes_by_id; @@ -75,7 +76,7 @@ static TestGraphCombiner::Chunk make_chunk(const string& name) { return chunk; } -/// Create a path with full metadata and walk it over the given handles. +/// Create a path with the given metadata, that visits the given steps. static path_handle_t add_path(MutablePathHandleGraph& graph, const PathSense& sense, const string& sample, const string& locus, size_t haplotype, size_t phase_block, const subrange_t& subrange, @@ -88,7 +89,7 @@ static path_handle_t add_path(MutablePathHandleGraph& graph, const PathSense& se return path; } -/// Create a REFERENCE-sense path, the sense --seam trim positions a chunk by. +/// Create a REFERENCE-sense path. static path_handle_t add_ref_path(MutablePathHandleGraph& graph, const string& sample, const string& locus, const subrange_t& subrange, const vector& steps) { @@ -106,109 +107,98 @@ static vector sorted_path_names(const PathHandleGraph& graph) { return names; } -TEST_CASE("compute_path_length adds up what a path visits", "[combine]") { - bdsg::HashGraph graph; - handle_t n1 = graph.create_handle("AAAA"); - handle_t n2 = graph.create_handle("CC"); - handle_t n3 = graph.create_handle("GGGGG"); - graph.create_edge(n1, n2); - graph.create_edge(n2, n3); - add_ref_path(graph, "GRCh38", "chr1", subrange_t{0, 11}, {n1, n2, n3}); - - SECTION("a graph without positions is scanned step by step") { - // HashGraph is not a PathPositionHandleGraph, so this takes the scan_path branch. - path_handle_t path = graph.get_path_handle("GRCh38#0#chr1[0-11]"); - REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 11); - } - - SECTION("a positional graph is asked directly, and agrees") { - bdsg::PositionOverlay positional(&graph); - path_handle_t path = positional.get_path_handle("GRCh38#0#chr1[0-11]"); - REQUIRE(TestGraphCombiner::compute_path_length(&positional, path) == 11); - } - - SECTION("an empty path has no length") { - graph.create_path_handle("empty"); - path_handle_t path = graph.get_path_handle("empty"); - REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 0); - } +/// Build a chunk by hand on a locus of GRCh38, without a graph; sorting doesn't look at it. +static TestGraphCombiner::Chunk chunk_on_locus(const string& name, const string& locus, + offset_t start) { + TestGraphCombiner::Chunk chunk; + chunk.name = name; + chunk.ref.base_path_key = TestGraphCombiner::BasePathKey{"GRCh38", locus, 0}; + chunk.ref.start_offset = start; + return chunk; +} - SECTION("a step's orientation does not change how much it contributes") { - handle_t n4 = graph.create_handle("TTT"); - path_handle_t path = graph.create_path_handle("flipped"); - graph.append_step(path, graph.flip(n4)); - REQUIRE(TestGraphCombiner::compute_path_length(&graph, path) == 3); - } +/// Same, for chunks that differ in sample or haplotype. +static TestGraphCombiner::Chunk chunk_on_base_path(const string& name, + const TestGraphCombiner::BasePathKey& base_path_key, + offset_t start) { + TestGraphCombiner::Chunk chunk; + chunk.name = name; + chunk.ref.base_path_key = base_path_key; + chunk.ref.start_offset = start; + return chunk; } -TEST_CASE("path_start_offset reads a start offset off path metadata", "[combine]") { - SECTION("a subrange gives the offset outright") { - REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, 200}, 7, true) == 100); - REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, 200}, 7, false) == 100); - } +TEST_CASE("path_start_offset reads the offset from a subrange", "[combine]") { + TestGraphCombiner offsets(SeamPolicy::RENUMBER, false, true); + TestGraphCombiner not_offsets(SeamPolicy::RENUMBER, false, false); + REQUIRE(offsets.path_start_offset(subrange_t{100, 200}, 7) == 100); + REQUIRE(not_offsets.path_start_offset(subrange_t{100, 200}, 7) == 100); +} - SECTION("a subrange wins over a phase block even when blocks are offsets") { - REQUIRE(TestGraphCombiner::path_start_offset(subrange_t{100, PathMetadata::NO_END_POSITION}, - 7, true) == 100); - } +TEST_CASE("path_start_offset prefers a subrange over a phase block when phase blocks are offsets", "[combine]") { + TestGraphCombiner offsets(SeamPolicy::RENUMBER, false, true); + REQUIRE(offsets.path_start_offset(subrange_t{100, PathMetadata::NO_END_POSITION}, + 7) == 100); +} - SECTION("without a subrange, -P reads the phase block as the offset") { - REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, 7, true) == 7); - } +TEST_CASE("path_start_offset reads the phase block when there is no subrange and phase blocks are offsets", "[combine]") { + TestGraphCombiner offsets(SeamPolicy::RENUMBER, false, true); + REQUIRE(offsets.path_start_offset(PathMetadata::NO_SUBRANGE, 7) == 7); +} - SECTION("without -P a phase block is an opaque identifier, not a position") { - REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, 7, false) == 0); - } +TEST_CASE("path_start_offset ignores the phase block when phase blocks are not offsets", "[combine]") { + TestGraphCombiner not_offsets(SeamPolicy::RENUMBER, false, false); + REQUIRE(not_offsets.path_start_offset(PathMetadata::NO_SUBRANGE, 7) == 0); +} - SECTION("no subrange and no phase block starts at zero") { - REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, - PathMetadata::NO_PHASE_BLOCK, true) == 0); - REQUIRE(TestGraphCombiner::path_start_offset(PathMetadata::NO_SUBRANGE, - PathMetadata::NO_PHASE_BLOCK, false) == 0); - } +TEST_CASE("path_start_offset returns zero with no subrange and no phase block", "[combine]") { + TestGraphCombiner offsets(SeamPolicy::RENUMBER, false, true); + TestGraphCombiner not_offsets(SeamPolicy::RENUMBER, false, false); + REQUIRE(offsets.path_start_offset(PathMetadata::NO_SUBRANGE, + PathMetadata::NO_PHASE_BLOCK) == 0); + REQUIRE(not_offsets.path_start_offset(PathMetadata::NO_SUBRANGE, + PathMetadata::NO_PHASE_BLOCK) == 0); } -TEST_CASE("renumber_out_of_way shifts node IDs so they do not collide with IDs already used", "[combine]") { - SECTION("IDs already above the mark are left alone") { - bdsg::HashGraph graph; - graph.create_handle("A", 10); - graph.create_handle("C", 12); - REQUIRE(TestGraphCombiner::renumber_out_of_way(&graph, 3) == 0); - REQUIRE(graph.min_node_id() == 10); - REQUIRE(graph.max_node_id() == 12); - } +TEST_CASE("shift_ids_above leaves IDs already above the given ID alone", "[combine]") { + bdsg::HashGraph graph; + graph.create_handle("A", 10); + graph.create_handle("C", 12); + REQUIRE(TestGraphCombiner::shift_ids_above(&graph, 3) == 0); + REQUIRE(graph.min_node_id() == 10); + REQUIRE(graph.max_node_id() == 12); +} - SECTION("overlapping IDs are shifted clear, and stay clear") { - bdsg::HashGraph graph; - graph.create_handle("A", 1); - graph.create_handle("C", 5); - int64_t shift = TestGraphCombiner::renumber_out_of_way(&graph, 10); - REQUIRE(shift == 10); - REQUIRE(graph.min_node_id() == 11); - REQUIRE(graph.max_node_id() == 15); - } +TEST_CASE("shift_ids_above shifts overlapping IDs above the given ID", "[combine]") { + bdsg::HashGraph graph; + graph.create_handle("A", 1); + graph.create_handle("C", 5); + int64_t shift = TestGraphCombiner::shift_ids_above(&graph, 10); + REQUIRE(shift == 10); + REQUIRE(graph.min_node_id() == 11); + REQUIRE(graph.max_node_id() == 15); +} - SECTION("a graph starting exactly on the mark still moves off it") { - // delta is 0 here, which is not negative, so the shift is 1: IDs have to end - // up strictly above the mark, not on it. - bdsg::HashGraph graph; - graph.create_handle("A", 7); - REQUIRE(TestGraphCombiner::renumber_out_of_way(&graph, 7) == 1); - REQUIRE(graph.min_node_id() == 8); - } +TEST_CASE("shift_ids_above moves IDs that start exactly at the given ID above it", "[combine]") { + // delta is 0 here, which is not negative, so the shift is 1: IDs have to end + // up strictly above the mark, not on it. + bdsg::HashGraph graph; + graph.create_handle("A", 7); + REQUIRE(TestGraphCombiner::shift_ids_above(&graph, 7) == 1); + REQUIRE(graph.min_node_id() == 8); +} - SECTION("the shift the caller gets back is the one that was applied") { - bdsg::HashGraph graph; - graph.create_handle("A", 4); - graph.create_handle("C", 9); - int64_t shift = TestGraphCombiner::renumber_out_of_way(&graph, 20); - REQUIRE(graph.get_id(graph.get_handle(4 + shift)) == 4 + shift); - REQUIRE(graph.has_node(9 + shift)); - REQUIRE(!graph.has_node(4)); - } +TEST_CASE("shift_ids_above returns the shift it applied", "[combine]") { + bdsg::HashGraph graph; + graph.create_handle("A", 4); + graph.create_handle("C", 9); + int64_t shift = TestGraphCombiner::shift_ids_above(&graph, 20); + REQUIRE(graph.get_id(graph.get_handle(4 + shift)) == 4 + shift); + REQUIRE(graph.has_node(9 + shift)); + REQUIRE(!graph.has_node(4)); } -TEST_CASE("group_key_of decides which paths are pieces of the same thing", "[combine]") { +TEST_CASE("group_key_of groups pieces of one path together", "[combine]") { bdsg::HashGraph graph; handle_t n1 = graph.create_handle("AAAA"); @@ -220,26 +210,29 @@ TEST_CASE("group_key_of decides which paths are pieces of the same thing", "[com path_handle_t block_0 = graph.get_path_handle("HG002#1#chr1#0"); path_handle_t block_10 = graph.get_path_handle("HG002#1#chr1#10"); + TestGraphCombiner offsets(SeamPolicy::RENUMBER, false, true); + TestGraphCombiner not_offsets(SeamPolicy::RENUMBER, false, false); + SECTION("phase blocks tell paths apart by default") { - REQUIRE(TestGraphCombiner::group_key_of(graph, block_0, false) - != TestGraphCombiner::group_key_of(graph, block_10, false)); + REQUIRE(not_offsets.group_key_of(graph, block_0) + != not_offsets.group_key_of(graph, block_10)); } - SECTION("-P folds the phase block out, so the two become one group") { - REQUIRE(TestGraphCombiner::group_key_of(graph, block_0, true) - == TestGraphCombiner::group_key_of(graph, block_10, true)); - // Folded out means replaced by the sentinel, not kept as some other value. - REQUIRE(std::get<5>(TestGraphCombiner::group_key_of(graph, block_0, true)) + SECTION("when phase blocks are offsets the phase block is left out of the key") { + REQUIRE(offsets.group_key_of(graph, block_0) + == offsets.group_key_of(graph, block_10)); + // Left out means replaced by NO_PHASE_BLOCK, not kept as some other value. + REQUIRE(std::get<4>(offsets.group_key_of(graph, block_0)) == PathMetadata::NO_PHASE_BLOCK); } - SECTION("a subrange is not part of the key, so fragments group together") { + SECTION("fragments with different subranges share a key") { add_path(graph, PathSense::REFERENCE, "GRCh38", "chr1", 0, PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}); add_path(graph, PathSense::REFERENCE, "GRCh38", "chr1", 0, PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 8}, {n1}); - REQUIRE(TestGraphCombiner::group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[0-4]"), false) - == TestGraphCombiner::group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[4-8]"), false)); + REQUIRE(not_offsets.group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[0-4]")) + == not_offsets.group_key_of(graph, graph.get_path_handle("GRCh38#0#chr1[4-8]"))); } SECTION("each identity field on its own is enough to split the group") { @@ -253,7 +246,7 @@ TEST_CASE("group_key_of decides which paths are pieces of the same thing", "[com PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}); // other sense auto key = [&](const string& name) { - return TestGraphCombiner::group_key_of(graph, graph.get_path_handle(name), false); + return not_offsets.group_key_of(graph, graph.get_path_handle(name)); }; REQUIRE(key("HG002#1#chr1#0") != key("HG002#2#chr1#0")); REQUIRE(key("HG002#1#chr1#0") != key("HG002#1#chr2#0")); @@ -287,7 +280,7 @@ TEST_CASE("path_prefix and path_suffix read the ends of a path", "[combine]") { REQUIRE(TestGraphCombiner::path_suffix(graph, path, 20) == "AAAACCGGGTT"); } - SECTION("asking for nothing yields nothing, even on an empty path") { + SECTION("asking for zero bases or reading an empty path yields nothing") { path_handle_t empty = add_ref_path(graph, "GRCh38", "chr2", subrange_t{0, 0}, {}); REQUIRE(TestGraphCombiner::path_prefix(graph, path, 0) == ""); REQUIRE(TestGraphCombiner::path_suffix(graph, path, 0) == ""); @@ -304,212 +297,257 @@ TEST_CASE("path_prefix and path_suffix read the ends of a path", "[combine]") { } } -TEST_CASE("check_no_split_paths passes graphs whose paths are each whole", "[combine]") { - // Paths that merely look alike are not pieces of one path. - // Rejection exits the process, so it is tested in test/t/51_vg_combine.t. +TEST_CASE("check_no_split_paths accepts a graph with a single path", "[combine]") { TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("one.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 1); +} - SECTION("a single path is not a split path") { - TestGraphCombiner::Chunk chunk = make_chunk("one.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - combiner.check_no_split_paths(chunk); - REQUIRE(chunk.graph->get_path_count() == 1); - } +TEST_CASE("check_no_split_paths accepts paths that differ only by haplotype", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("haps.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{0, 4}, {n1}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, + subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); +} - SECTION("paths differing only by haplotype are separate paths") { - TestGraphCombiner::Chunk chunk = make_chunk("haps.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - subrange_t{0, 4}, {n1}); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, - subrange_t{0, 4}, {n1}); - combiner.check_no_split_paths(chunk); - REQUIRE(chunk.graph->get_path_count() == 2); - } +TEST_CASE("check_no_split_paths accepts paths that differ only by locus", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("loci.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*chunk.graph, "GRCh38", "chr2", subrange_t{0, 4}, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); +} - SECTION("paths differing only by locus are separate paths") { - TestGraphCombiner::Chunk chunk = make_chunk("loci.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - add_ref_path(*chunk.graph, "GRCh38", "chr2", subrange_t{0, 4}, {n1}); - combiner.check_no_split_paths(chunk); - REQUIRE(chunk.graph->get_path_count() == 2); - } +TEST_CASE("check_no_split_paths accepts paths that differ only by phase block when phase blocks are not offsets", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("blocks.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 10, + PathMetadata::NO_SUBRANGE, {n1}); + combiner.check_no_split_paths(chunk); + REQUIRE(chunk.graph->get_path_count() == 2); +} - SECTION("phase blocks are separate paths until -P says otherwise") { - TestGraphCombiner::Chunk chunk = make_chunk("blocks.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - PathMetadata::NO_SUBRANGE, {n1}); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 10, - PathMetadata::NO_SUBRANGE, {n1}); - combiner.check_no_split_paths(chunk); - REQUIRE(chunk.graph->get_path_count() == 2); - } +TEST_CASE("check_no_split_paths rejects two pieces of one path in one graph", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("split.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{4, 7}, {n2}); + REQUIRE_THROWS_AS(combiner.check_no_split_paths(chunk), InputConflictError); + REQUIRE_THROWS_WITH(combiner.check_no_split_paths(chunk), Catch::Contains("split.pg")); +} + +TEST_CASE("check_no_split_paths rejects two phase blocks of one path when phase blocks are offsets", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, true); + TestGraphCombiner::Chunk chunk = make_chunk("blocks.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 10, + PathMetadata::NO_SUBRANGE, {n1}); + REQUIRE_THROWS_AS(combiner.check_no_split_paths(chunk), InputConflictError); + REQUIRE_THROWS_WITH(combiner.check_no_split_paths(chunk), Catch::Contains("blocks.pg")); } -TEST_CASE("describe_reference reads a chunk's position off its REFERENCE path", "[combine]") { +TEST_CASE("check_subrange_matches_length rejects a subrange end that disagrees with the sequence length", "[combine]") { + // A subrange claiming 100-999 over 11bp of sequence. + TestGraphCombiner::Chunk chunk = make_chunk("claim.pg"); + handle_t n1 = chunk.graph->create_handle("AAAAAAAAAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 999}, {n1}); - SECTION("every field is filled in from the path's metadata") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("left.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - handle_t n2 = chunk.graph->create_handle("CC"); - handle_t n3 = chunk.graph->create_handle("GGGGG"); - chunk.graph->create_edge(n1, n2); - chunk.graph->create_edge(n2, n3); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 111}, {n1, n2, n3}); - // A haplotype alongside it must not be mistaken for the reference. - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - subrange_t{100, 111}, {n1, n2, n3}); - - combiner.describe_reference(chunk); - - REQUIRE(chunk.ref_name == "GRCh38#0#chr1[100-111]"); - REQUIRE(chunk.ref_key == TestGraphCombiner::RefKey{"GRCh38", "chr1", 0, false}); - REQUIRE(chunk.ref_start_offset == 100); - // The end is the start plus the path's own length, not the subrange's end. - REQUIRE(chunk.ref_end_offset == 111); - REQUIRE(chunk.graph->get_id(chunk.ref_start_handle) == chunk.graph->get_id(n1)); - REQUIRE(chunk.graph->get_id(chunk.ref_end_handle) == chunk.graph->get_id(n3)); - REQUIRE(!chunk.graph->get_is_reverse(chunk.ref_start_handle)); - REQUIRE(!chunk.graph->get_is_reverse(chunk.ref_end_handle)); - } + REQUIRE_THROWS_AS(TestGraphCombiner::check_subrange_matches_length(chunk), + PathLengthMismatchError); + REQUIRE_THROWS_WITH(TestGraphCombiner::check_subrange_matches_length(chunk), + Catch::Contains("GRCh38#0#chr1[100-999]")); +} - SECTION("the end offset follows the sequence, not the subrange's claim") { - // A subrange claiming 100-999 over 11bp of sequence still ends at 111. - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("claim.pg"); - handle_t n1 = chunk.graph->create_handle("AAAAAAAAAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 999}, {n1}); +TEST_CASE("describe_reference fills in every field from the reference path's metadata", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("left.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CC"); + handle_t n3 = chunk.graph->create_handle("GGGGG"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n2, n3); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 111}, {n1, n2, n3}); + // A haplotype alongside it must not be mistaken for the reference. + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 111}, {n1, n2, n3}); + + chunk.ref = combiner.describe_reference(chunk); + + REQUIRE(chunk.ref.path_name == "GRCh38#0#chr1[100-111]"); + REQUIRE(chunk.ref.base_path_key == TestGraphCombiner::BasePathKey{"GRCh38", "chr1", 0}); + REQUIRE(chunk.ref.start_offset == 100); + REQUIRE(chunk.ref.end_offset == 111); + REQUIRE(chunk.graph->get_id(chunk.ref.start_handle) == chunk.graph->get_id(n1)); + REQUIRE(chunk.graph->get_id(chunk.ref.end_handle) == chunk.graph->get_id(n3)); + REQUIRE(!chunk.graph->get_is_reverse(chunk.ref.start_handle)); + REQUIRE(!chunk.graph->get_is_reverse(chunk.ref.end_handle)); +} - combiner.describe_reference(chunk); - REQUIRE(chunk.ref_start_offset == 100); - REQUIRE(chunk.ref_end_offset == 111); - } +TEST_CASE("describe_reference gives a single-node chunk the same start and end node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("solo.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + + chunk.ref = combiner.describe_reference(chunk); + REQUIRE(chunk.ref.start_handle == chunk.ref.end_handle); + REQUIRE(chunk.ref.start_offset == 0); + REQUIRE(chunk.ref.end_offset == 4); +} - SECTION("a single-node chunk starts and ends on the same node") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("solo.pg"); +TEST_CASE("describe_reference starts a reference path with no subrange at zero whether or not phase blocks are offsets", "[combine]") { + // A REFERENCE-sense path may not carry a phase block, so with no subrange there + // is no offset to read and the chunk can only be placed at offset 0. + for (bool phase_block_is_offset : {false, true}) { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, phase_block_is_offset); + TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); handle_t n1 = chunk.graph->create_handle("AAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*chunk.graph, "GRCh38", "chr1", PathMetadata::NO_SUBRANGE, {n1}); - combiner.describe_reference(chunk); - REQUIRE(chunk.ref_start_handle == chunk.ref_end_handle); - REQUIRE(chunk.ref_start_offset == 0); - REQUIRE(chunk.ref_end_offset == 4); + chunk.ref = combiner.describe_reference(chunk); + REQUIRE(chunk.ref.start_offset == 0); + REQUIRE(chunk.ref.end_offset == 4); } +} - SECTION("a reference with no subrange starts at zero, with or without -P") { - // A REFERENCE-sense path may not carry a phase block, so -P has nothing to - // read here and the chunk can only be placed at the origin. - for (bool phase_block_is_offset : {false, true}) { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, phase_block_is_offset); - TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", PathMetadata::NO_SUBRANGE, {n1}); - - combiner.describe_reference(chunk); - REQUIRE(chunk.ref_start_offset == 0); - REQUIRE(chunk.ref_end_offset == 4); - } - } +TEST_CASE("describe_reference leaves circularity out of the base path key", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("circle.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_path(*chunk.graph, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); - SECTION("a circular reference is a different reference from a linear one") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("circle.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_path(*chunk.graph, PathSense::REFERENCE, "GRCh38", "chrM", 0, - PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); + chunk.ref = combiner.describe_reference(chunk); + REQUIRE(chunk.ref.base_path_key == TestGraphCombiner::BasePathKey{"GRCh38", "chrM", 0}); + REQUIRE(chunk.ref.is_circular); +} - combiner.describe_reference(chunk); - REQUIRE(chunk.ref_key == TestGraphCombiner::RefKey{"GRCh38", "chrM", 0, true}); - } +TEST_CASE("describe_reference rejects an input with no reference path", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("noref.pg"); + add_path(*chunk.graph, PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, "plain_contig", + PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, + PathMetadata::NO_SUBRANGE, {chunk.graph->create_handle("AAAA")}); + REQUIRE_THROWS_AS(combiner.describe_reference(chunk), ReferencePathError); + REQUIRE_THROWS_WITH(combiner.describe_reference(chunk), Catch::Contains("noref.pg")); } -TEST_CASE("sort_by_reference_offset groups references and orders each by offset", "[combine]") { +TEST_CASE("describe_reference rejects an input with two reference paths", "[combine]") { TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("tworef.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*chunk.graph, "GRCh38", "chr2", subrange_t{0, 4}, {n1}); + REQUIRE_THROWS_AS(combiner.describe_reference(chunk), ReferencePathError); + REQUIRE_THROWS_WITH(combiner.describe_reference(chunk), Catch::Contains("tworef.pg")); +} - // Build chunks by hand: the sort reads only ref_key and ref_start_offset, so - // these need no graphs. - auto positioned = [](const string& name, const string& locus, offset_t start) { - TestGraphCombiner::Chunk chunk; - chunk.name = name; - chunk.ref_key = TestGraphCombiner::RefKey{"GRCh38", locus, 0, false}; - chunk.ref_start_offset = start; - return chunk; - }; - // Same, for chunks whose reference differs in something other than locus. - auto keyed = [](const string& name, const TestGraphCombiner::RefKey& ref_key, - offset_t start) { - TestGraphCombiner::Chunk chunk; - chunk.name = name; - chunk.ref_key = ref_key; - chunk.ref_start_offset = start; - return chunk; - }; - - SECTION("chunks of one reference come out in ascending offset order") { - vector chunks; - chunks.push_back(positioned("c", "chr1", 200)); - chunks.push_back(positioned("a", "chr1", 0)); - chunks.push_back(positioned("b", "chr1", 100)); - - combiner.sort_by_reference_offset(chunks); - REQUIRE(chunks[0].name == "a"); - REQUIRE(chunks[1].name == "b"); - REQUIRE(chunks[2].name == "c"); - } +TEST_CASE("describe_reference rejects a reference path that visits its first node in reverse", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("rev.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(chunk.graph->flip(n1), n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 7}, {chunk.graph->flip(n1), n2}); + REQUIRE_THROWS_AS(combiner.describe_reference(chunk), ReferencePathError); + REQUIRE_THROWS_WITH(combiner.describe_reference(chunk), Catch::Contains("GRCh38#0#chr1[0-7]")); +} - SECTION("references keep first-seen order, and their chunks stay contiguous") { - // chr2 is seen first, so it stays first even though chr1 sorts earlier. - vector chunks; - chunks.push_back(positioned("y2", "chr2", 100)); - chunks.push_back(positioned("x2", "chr1", 100)); - chunks.push_back(positioned("y1", "chr2", 0)); - chunks.push_back(positioned("x1", "chr1", 0)); - - combiner.sort_by_reference_offset(chunks); - REQUIRE(chunks[0].name == "y1"); - REQUIRE(chunks[1].name == "y2"); - REQUIRE(chunks[2].name == "x1"); - REQUIRE(chunks[3].name == "x2"); - } +TEST_CASE("sort_by_base_path_offset sorts chunks of one base path by offset", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + vector chunks; + chunks.push_back(chunk_on_locus("c", "chr1", 200)); + chunks.push_back(chunk_on_locus("a", "chr1", 0)); + chunks.push_back(chunk_on_locus("b", "chr1", 100)); + + combiner.sort_by_base_path_offset(chunks); + REQUIRE(chunks[0].name == "a"); + REQUIRE(chunks[1].name == "b"); + REQUIRE(chunks[2].name == "c"); +} - SECTION("equal offsets keep the order they arrived in") { - vector chunks; - chunks.push_back(positioned("first", "chr1", 50)); - chunks.push_back(positioned("second", "chr1", 50)); - chunks.push_back(positioned("third", "chr1", 50)); +TEST_CASE("sort_by_base_path_offset keeps base paths in first-seen order with their chunks together", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + // chr2 is seen first, so it stays first even though chr1 sorts earlier. + vector chunks; + chunks.push_back(chunk_on_locus("y2", "chr2", 100)); + chunks.push_back(chunk_on_locus("x2", "chr1", 100)); + chunks.push_back(chunk_on_locus("y1", "chr2", 0)); + chunks.push_back(chunk_on_locus("x1", "chr1", 0)); + + combiner.sort_by_base_path_offset(chunks); + REQUIRE(chunks[0].name == "y1"); + REQUIRE(chunks[1].name == "y2"); + REQUIRE(chunks[2].name == "x1"); + REQUIRE(chunks[3].name == "x2"); +} - combiner.sort_by_reference_offset(chunks); - REQUIRE(chunks[0].name == "first"); - REQUIRE(chunks[1].name == "second"); - REQUIRE(chunks[2].name == "third"); - } +TEST_CASE("sort_by_base_path_offset keeps chunks with equal offsets in input order", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + vector chunks; + chunks.push_back(chunk_on_locus("first", "chr1", 50)); + chunks.push_back(chunk_on_locus("second", "chr1", 50)); + chunks.push_back(chunk_on_locus("third", "chr1", 50)); + + combiner.sort_by_base_path_offset(chunks); + REQUIRE(chunks[0].name == "first"); + REQUIRE(chunks[1].name == "second"); + REQUIRE(chunks[2].name == "third"); +} - SECTION("haplotype and circularity make separate references of one locus") { - // All four share sample and locus, so only the other two RefKey fields can - // tell the three references apart. - vector chunks; - chunks.push_back(keyed("hap2", {"GRCh38", "chr1", 2, false}, 50)); - chunks.push_back(positioned("hap0_late", "chr1", 100)); - chunks.push_back(positioned("hap0_early", "chr1", 0)); - chunks.push_back(keyed("circular", {"GRCh38", "chr1", 0, true}, 50)); - - combiner.sort_by_reference_offset(chunks); - // References lead in first-seen order; hap0's two chunks follow in offset order. - REQUIRE(chunks[0].name == "hap2"); - REQUIRE(chunks[1].name == "hap0_early"); - REQUIRE(chunks[2].name == "hap0_late"); - REQUIRE(chunks[3].name == "circular"); - } +TEST_CASE("sort_by_base_path_offset treats haplotypes of one locus as separate base paths", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + // All three share sample and locus, so only haplotype can tell the two base + // paths apart. + vector chunks; + chunks.push_back(chunk_on_base_path("hap2", {"GRCh38", "chr1", 2}, 50)); + chunks.push_back(chunk_on_locus("hap0_late", "chr1", 100)); + chunks.push_back(chunk_on_locus("hap0_early", "chr1", 0)); + + combiner.sort_by_base_path_offset(chunks); + // Base paths lead in first-seen order; hap0's two chunks follow in offset order. + REQUIRE(chunks[0].name == "hap2"); + REQUIRE(chunks[1].name == "hap0_early"); + REQUIRE(chunks[2].name == "hap0_late"); +} + +TEST_CASE("check_overlaps_agree rejects neighbouring chunks that spell different sequences over their overlap", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + vector chunks; + chunks.push_back(make_chunk("left.pg")); + add_ref_path(*chunks[0].graph, "GRCh38", "chr1", subrange_t{0, 10}, + {chunks[0].graph->create_handle("AAAAAAAAAA")}); + chunks[0].ref = combiner.describe_reference(chunks[0]); + chunks.push_back(make_chunk("right.pg")); + handle_t r1 = chunks[1].graph->create_handle("AACCC"); + handle_t r2 = chunks[1].graph->create_handle("GGGGG"); + chunks[1].graph->create_edge(r1, r2); + add_ref_path(*chunks[1].graph, "GRCh38", "chr1", subrange_t{6, 16}, {r1, r2}); + chunks[1].ref = combiner.describe_reference(chunks[1]); + + // The overlap is [6, 10): the left chunk has AAAA there and the right chunk AACC. + REQUIRE_THROWS_AS(combiner.check_overlaps_agree(chunks), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.check_overlaps_agree(chunks), Catch::Contains("at offset 8")); } -TEST_CASE("check_all_paths_start_at and check_all_paths_end_at pass chunks with flush boundaries", "[combine]") { - // Legitimate chunks are not turned away. - // Rejection exits the process, so it is tested in test/t/51_vg_combine.t. +TEST_CASE("check_all_paths_start_at_reference_first_node and check_all_paths_end_at_reference_last_node pass chunks whose paths all start and end there", "[combine]") { TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); TestGraphCombiner::Chunk chunk = make_chunk("flush.pg"); @@ -527,16 +565,16 @@ TEST_CASE("check_all_paths_start_at and check_all_paths_end_at pass chunks with subrange_t{0, 10}, {n1, n3, n4}); add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 2, 0, subrange_t{0, 10}, {n1, n2, n4}); - chunk.ref_start_handle = n1; - chunk.ref_end_handle = n4; + chunk.ref.start_handle = n1; + chunk.ref.end_handle = n4; SECTION("all paths beginning on the boundary node are accepted") { - combiner.check_all_paths_start_at(chunk); + combiner.check_all_paths_start_at_reference_first_node(chunk); REQUIRE(chunk.graph->get_path_count() == 3); } SECTION("all paths ending on the boundary node are accepted") { - combiner.check_all_paths_end_at(chunk); + combiner.check_all_paths_end_at_reference_last_node(chunk); REQUIRE(chunk.graph->get_path_count() == 3); } @@ -548,277 +586,353 @@ TEST_CASE("check_all_paths_start_at and check_all_paths_end_at pass chunks with flipped.graph->create_edge(flipped.graph->flip(a), b); add_ref_path(*flipped.graph, "GRCh38", "chr1", subrange_t{0, 7}, {flipped.graph->flip(a), b}); - flipped.ref_start_handle = flipped.graph->flip(a); - flipped.ref_end_handle = b; + flipped.ref.start_handle = flipped.graph->flip(a); + flipped.ref.end_handle = b; - combiner.check_all_paths_start_at(flipped); - combiner.check_all_paths_end_at(flipped); + combiner.check_all_paths_start_at_reference_first_node(flipped); + combiner.check_all_paths_end_at_reference_last_node(flipped); REQUIRE(flipped.graph->get_path_count() == 1); } } -TEST_CASE("trim_and_shift cuts the overlap away and moves coordinates with it", "[combine]") { +TEST_CASE("check_all_paths_start_at_reference_first_node rejects a path running against the reference path", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("reverse.pg"); + handle_t a = chunk.graph->create_handle("AAAA"); + handle_t b = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(a, b); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 7}, {a, b}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{0, 7}, {chunk.graph->flip(b), chunk.graph->flip(a)}); + chunk.ref.start_handle = a; + chunk.ref.end_handle = b; + REQUIRE_THROWS_AS(combiner.check_all_paths_start_at_reference_first_node(chunk), ChunkOverlapError); +} - SECTION("no overlap and no fuse leaves the chunk exactly as it was") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("clean.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - handle_t n2 = chunk.graph->create_handle("CCC"); - chunk.graph->create_edge(n1, n2); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); - chunk.ref_start_handle = n1; - nid_t before = chunk.graph->get_id(n1); +TEST_CASE("check_all_paths_start_at_reference_first_node rejects a path that starts on a different node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("other_start.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + handle_t n3 = chunk.graph->create_handle("T"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n3, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 7}, {n1, n2}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{3, 7}, {n3, n2}); + chunk.ref.start_handle = n1; + chunk.ref.end_handle = n2; + REQUIRE_THROWS_AS(combiner.check_all_paths_start_at_reference_first_node(chunk), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.check_all_paths_start_at_reference_first_node(chunk), + Catch::Contains("HG002#1#chr1#0[3-7]")); +} - nid_t after = combiner.trim_and_shift(chunk, 0, 5); +TEST_CASE("check_all_paths_end_at_reference_last_node rejects a path that ends on a different node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("other_end.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + handle_t n3 = chunk.graph->create_handle("T"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n1, n3); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 7}, {n1, n2}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{0, 5}, {n1, n3}); + chunk.ref.start_handle = n1; + chunk.ref.end_handle = n2; + REQUIRE_THROWS_AS(combiner.check_all_paths_end_at_reference_last_node(chunk), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.check_all_paths_end_at_reference_last_node(chunk), + Catch::Contains("HG002#1#chr1#0[0-5]")); +} - REQUIRE(after == before); - REQUIRE(chunk.graph->get_node_count() == 2); - REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[100-107]"}); - } +TEST_CASE("trim_and_shift with no overlap and no fusing leaves the chunk unchanged", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("clean.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); + chunk.ref.start_handle = n1; + nid_t before = chunk.graph->get_id(n1); - SECTION("no overlap but fusing pulls every coordinate back by the welded node") { - // Welding buries the left chunk's last node inside this chunk's first, so - // this chunk's coordinates move back by that node's length. - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); - TestGraphCombiner::Chunk chunk = make_chunk("weld.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - handle_t n2 = chunk.graph->create_handle("CCC"); - chunk.graph->create_edge(n1, n2); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); - chunk.ref_start_handle = n1; + nid_t after = combiner.trim_and_shift(chunk, 0, 5); - combiner.trim_and_shift(chunk, 0, 5); + REQUIRE(after == before); + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[100-107]"}); +} - REQUIRE(chunk.graph->get_node_count() == 2); - REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[95-107]"}); - } +TEST_CASE("trim_and_shift with fusing and no overlap moves coordinates back by the left boundary node length", "[combine]") { + // Fusing merges the left chunk's last node into this chunk's first, so + // this chunk's coordinates move back by that node's length. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + TestGraphCombiner::Chunk chunk = make_chunk("fuse.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + handle_t n2 = chunk.graph->create_handle("CCC"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 107}, {n1, n2}); + chunk.ref.start_handle = n1; - SECTION("an overlap splits the first node, drops the front, and shifts coordinates") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("overlap.pg"); - handle_t n1 = chunk.graph->create_handle("AAAACC"); - handle_t n2 = chunk.graph->create_handle("GGG"); - chunk.graph->create_edge(n1, n2); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 109}, {n1, n2}); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - subrange_t{100, 109}, {n1, n2}); - chunk.ref_start_handle = n1; - - nid_t new_start = combiner.trim_and_shift(chunk, 4, 5); - - // The overlapping "AAAA" is gone; "CC" remains and is the new start. - REQUIRE(chunk.graph->get_node_count() == 2); - REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "CC"); - REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-109]") == "CCGGG"); - REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-109]") == "CCGGG"); - } + combiner.trim_and_shift(chunk, 0, 5); - SECTION("an overlap of one base still leaves the rest of the node behind") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("thin.pg"); - handle_t n1 = chunk.graph->create_handle("AC"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 102}, {n1}); - chunk.ref_start_handle = n1; + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[95-107]"}); +} - nid_t new_start = combiner.trim_and_shift(chunk, 1, 5); +TEST_CASE("trim_and_shift splits the first node at the overlap and shifts coordinates", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("overlap.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + handle_t n2 = chunk.graph->create_handle("GGG"); + chunk.graph->create_edge(n1, n2); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 109}, {n1, n2}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 109}, {n1, n2}); + chunk.ref.start_handle = n1; - REQUIRE(chunk.graph->get_node_count() == 1); - REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "C"); - REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[101-102]") == "C"); - } + nid_t new_start = combiner.trim_and_shift(chunk, 4, 5); - SECTION("with -P the phase block moves instead of a subrange") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, true); - TestGraphCombiner::Chunk chunk = make_chunk("pblock.pg"); - handle_t n1 = chunk.graph->create_handle("AAAACC"); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 100, - PathMetadata::NO_SUBRANGE, {n1}); - chunk.ref_start_handle = n1; + // The overlapping "AAAA" is gone; "CC" remains and is the new start. + REQUIRE(chunk.graph->get_node_count() == 2); + REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "CC"); + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-109]") == "CCGGG"); + REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-109]") == "CCGGG"); +} - combiner.trim_and_shift(chunk, 4, 5); +TEST_CASE("trim_and_shift with a one-base overlap removes only the first base of the node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("thin.pg"); + handle_t n1 = chunk.graph->create_handle("AC"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 102}, {n1}); + chunk.ref.start_handle = n1; - REQUIRE(sorted_path_names(*chunk.graph) == vector{"HG002#1#chr1#104"}); - } + nid_t new_start = combiner.trim_and_shift(chunk, 1, 5); - SECTION("a path with no coordinates at all is given a subrange to carry the shift") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); - handle_t n1 = chunk.graph->create_handle("AAAACC"); - add_path(*chunk.graph, PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, "scaffold", - PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, - PathMetadata::NO_SUBRANGE, {n1}); - chunk.ref_start_handle = n1; + REQUIRE(chunk.graph->get_node_count() == 1); + REQUIRE(chunk.graph->get_sequence(chunk.graph->get_handle(new_start)) == "C"); + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[101-102]") == "C"); +} - combiner.trim_and_shift(chunk, 4, 0); +TEST_CASE("trim_and_shift shifts the phase block instead of a subrange when phase blocks are offsets", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, true); + TestGraphCombiner::Chunk chunk = make_chunk("pblock.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 100, + PathMetadata::NO_SUBRANGE, {n1}); + chunk.ref.start_handle = n1; - REQUIRE(sorted_path_names(*chunk.graph) == vector{"scaffold[4]"}); - } + combiner.trim_and_shift(chunk, 4, 5); - SECTION("a shift that would run past zero stops at zero") { - // Fusing pulls coordinates back by 50, but the path only starts at 10. - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); - TestGraphCombiner::Chunk chunk = make_chunk("clamp.pg"); - handle_t n1 = chunk.graph->create_handle("AAAA"); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{10, 14}, {n1}); - chunk.ref_start_handle = n1; + REQUIRE(sorted_path_names(*chunk.graph) == vector{"HG002#1#chr1#104"}); +} - combiner.trim_and_shift(chunk, 0, 50); +TEST_CASE("trim_and_shift adds a subrange starting at the trimmed length to a path with no subrange", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("bare.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + add_path(*chunk.graph, PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, "scaffold", + PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, + PathMetadata::NO_SUBRANGE, {n1}); + chunk.ref.start_handle = n1; - REQUIRE(sorted_path_names(*chunk.graph) == vector{"GRCh38#0#chr1[0-14]"}); - } + combiner.trim_and_shift(chunk, 4, 0); - SECTION("trimming keeps each path on its own alleles through a bubble") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("bubble.pg"); - handle_t n1 = chunk.graph->create_handle("AAAACC"); - handle_t n2 = chunk.graph->create_handle("G"); - handle_t n3 = chunk.graph->create_handle("T"); - handle_t n4 = chunk.graph->create_handle("TTTTT"); - chunk.graph->create_edge(n1, n2); - chunk.graph->create_edge(n1, n3); - chunk.graph->create_edge(n2, n4); - chunk.graph->create_edge(n3, n4); - add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 112}, {n1, n2, n4}); - add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - subrange_t{100, 112}, {n1, n3, n4}); - chunk.ref_start_handle = n1; - - combiner.trim_and_shift(chunk, 4, 0); - - REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-112]") == "CCGTTTTT"); - REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-112]") == "CCTTTTTT"); - } + REQUIRE(sorted_path_names(*chunk.graph) == vector{"scaffold[4]"}); +} + +TEST_CASE("trim_and_shift rejects a shift that would make a start coordinate negative", "[combine]") { + // Fusing moves coordinates back by 50, but the path only starts at 10. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + TestGraphCombiner::Chunk chunk = make_chunk("negative.pg"); + handle_t n1 = chunk.graph->create_handle("AAAA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{10, 14}, {n1}); + chunk.ref.start_handle = n1; + + REQUIRE_THROWS_AS(combiner.trim_and_shift(chunk, 0, 50), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.trim_and_shift(chunk, 0, 50), + Catch::Contains("GRCh38#0#chr1[10-14]")); +} + +TEST_CASE("trim_and_shift keeps each path on its own alleles through a bubble", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("bubble.pg"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + handle_t n2 = chunk.graph->create_handle("G"); + handle_t n3 = chunk.graph->create_handle("T"); + handle_t n4 = chunk.graph->create_handle("TTTTT"); + chunk.graph->create_edge(n1, n2); + chunk.graph->create_edge(n1, n3); + chunk.graph->create_edge(n2, n4); + chunk.graph->create_edge(n3, n4); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{100, 112}, {n1, n2, n4}); + add_path(*chunk.graph, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + subrange_t{100, 112}, {n1, n3, n4}); + chunk.ref.start_handle = n1; + + combiner.trim_and_shift(chunk, 4, 0); + + REQUIRE(spell_path(*chunk.graph, "GRCh38#0#chr1[104-112]") == "CCGTTTTT"); + REQUIRE(spell_path(*chunk.graph, "HG002#1#chr1#0[104-112]") == "CCTTTTTT"); } -TEST_CASE("ingest moves a chunk into the accumulator", "[combine]") { +TEST_CASE("trim_and_shift rejects an overlap longer than the first node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("short.pg"); + handle_t n1 = chunk.graph->create_handle("AA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 2}, {n1}); + chunk.ref.start_handle = n1; + REQUIRE_THROWS_AS(combiner.trim_and_shift(chunk, 3, 0), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.trim_and_shift(chunk, 3, 0), Catch::Contains("short.pg")); +} - SECTION("the first chunk becomes the accumulator untouched") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - TestGraphCombiner::Chunk chunk = make_chunk("first.pg"); - chunk.graph->create_handle("AAAA", 5); - chunk.graph->create_handle("CCC", 9); +TEST_CASE("trim_and_shift rejects an overlap equal to the first node", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("exact.pg"); + handle_t n1 = chunk.graph->create_handle("AA"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 2}, {n1}); + chunk.ref.start_handle = n1; + REQUIRE_THROWS_AS(combiner.trim_and_shift(chunk, 2, 0), ChunkOverlapError); + REQUIRE_THROWS_WITH(combiner.trim_and_shift(chunk, 2, 0), Catch::Contains("exact.pg")); +} - int64_t shift = combiner.ingest(chunk); +TEST_CASE("trim_and_shift rejects an edge into the part of the first node it would cut off", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("dangle.pg"); + handle_t other = chunk.graph->create_handle("TT"); + handle_t n1 = chunk.graph->create_handle("AAAACC"); + chunk.graph->create_edge(other, n1); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 6}, {n1}); + chunk.ref.start_handle = n1; + REQUIRE_THROWS_AS(combiner.trim_and_shift(chunk, 4, 0), ChunkOverlapError); +} - REQUIRE(shift == 0); - REQUIRE(combiner.max_node_id == 9); - REQUIRE(combiner.dest != nullptr); - REQUIRE(combiner.dest->has_node(5)); - REQUIRE(combiner.dest->has_node(9)); - // The graph was moved out of the chunk, not copied. - REQUIRE(chunk.graph == nullptr); - } +TEST_CASE("ingest makes the first input dest unchanged", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk chunk = make_chunk("first.pg"); + chunk.graph->create_handle("AAAA", 5); + chunk.graph->create_handle("CCC", 9); + + int64_t shift = combiner.ingest(chunk); + + REQUIRE(shift == 0); + REQUIRE(combiner.max_node_id == 9); + REQUIRE(combiner.dest != nullptr); + REQUIRE(combiner.dest->has_node(5)); + REQUIRE(combiner.dest->has_node(9)); + // The graph was moved out of the chunk, not copied. + REQUIRE(chunk.graph == nullptr); +} - SECTION("renumber lifts a second chunk's IDs clear of the first") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - TestGraphCombiner::Chunk left = make_chunk("left.pg"); - handle_t l1 = left.graph->create_handle("AAAA", 1); - add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 4}, {l1}); - combiner.ingest(left); - - TestGraphCombiner::Chunk right = make_chunk("right.pg"); - handle_t r1 = right.graph->create_handle("CCC", 1); - handle_t r2 = right.graph->create_handle("GG", 2); - right.graph->create_edge(r1, r2); - add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r1, r2}); - - int64_t shift = combiner.ingest(right); - - REQUIRE(shift > 0); - // Both chunks' nodes are present and none was overwritten. - REQUIRE(combiner.dest->get_node_count() == 3); - REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1)) == "AAAA"); - REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1 + shift)) == "CCC"); - REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(1 + shift), - combiner.dest->get_handle(2 + shift))); - REQUIRE(combiner.max_node_id == 2 + shift); - REQUIRE(sorted_path_names(*combiner.dest) - == vector{"GRCh38#0#chr1[0-4]", "GRCh38#0#chr1[4-9]"}); - // The source is released rather than held until combine finishes. - REQUIRE(right.graph == nullptr); - } +TEST_CASE("ingest shifts a second input's IDs above the first when renumbering", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + TestGraphCombiner::Chunk left = make_chunk("left.pg"); + handle_t l1 = left.graph->create_handle("AAAA", 1); + add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 4}, {l1}); + combiner.ingest(left); + + TestGraphCombiner::Chunk right = make_chunk("right.pg"); + handle_t r1 = right.graph->create_handle("CCC", 1); + handle_t r2 = right.graph->create_handle("GG", 2); + right.graph->create_edge(r1, r2); + add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r1, r2}); + + int64_t shift = combiner.ingest(right); + + REQUIRE(shift > 0); + // Both chunks' nodes are present and none was overwritten. + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1)) == "AAAA"); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(1 + shift)) == "CCC"); + REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(1 + shift), + combiner.dest->get_handle(2 + shift))); + REQUIRE(combiner.max_node_id == 2 + shift); + REQUIRE(sorted_path_names(*combiner.dest) + == vector{"GRCh38#0#chr1[0-4]", "GRCh38#0#chr1[4-9]"}); + // The source is released rather than held until combine finishes. + REQUIRE(right.graph == nullptr); +} - SECTION("shared keeps IDs as they are, so nothing is shifted") { - TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); - TestGraphCombiner::Chunk left = make_chunk("left.pg"); - handle_t l1 = left.graph->create_handle("AAAA", 1); - handle_t l2 = left.graph->create_handle("CCC", 2); - left.graph->create_edge(l1, l2); - add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 7}, {l1, l2}); - combiner.ingest(left); - - TestGraphCombiner::Chunk right = make_chunk("right.pg"); - handle_t r2 = right.graph->create_handle("CCC", 2); // the shared node - handle_t r3 = right.graph->create_handle("GG", 3); - right.graph->create_edge(r2, r3); - add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r2, r3}); - - int64_t shift = combiner.ingest(right); - - REQUIRE(shift == 0); - REQUIRE(combiner.dest->get_node_count() == 3); - REQUIRE(combiner.max_node_id == 3); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[4-9]") == "CCCGG"); - } +TEST_CASE("ingest does not shift node IDs when nodes are shared by ID", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + TestGraphCombiner::Chunk left = make_chunk("left.pg"); + handle_t l1 = left.graph->create_handle("AAAA", 1); + handle_t l2 = left.graph->create_handle("CCC", 2); + left.graph->create_edge(l1, l2); + add_ref_path(*left.graph, "GRCh38", "chr1", subrange_t{0, 7}, {l1, l2}); + combiner.ingest(left); + + TestGraphCombiner::Chunk right = make_chunk("right.pg"); + handle_t r2 = right.graph->create_handle("CCC", 2); // the shared node + handle_t r3 = right.graph->create_handle("GG", 3); + right.graph->create_edge(r2, r3); + add_ref_path(*right.graph, "GRCh38", "chr1", subrange_t{4, 9}, {r2, r3}); + + int64_t shift = combiner.ingest(right); + + REQUIRE(shift == 0); + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.max_node_id == 3); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[4-9]") == "CCCGG"); } -TEST_CASE("share_nodes_by_id treats a repeated ID as the same node", "[combine]") { +TEST_CASE("share_nodes_by_id reuses an ID already in dest and creates a new one", "[combine]") { TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); combiner.dest = make_unique(); handle_t d1 = combiner.dest->create_handle("AAAA", 1); handle_t d2 = combiner.dest->create_handle("CCC", 2); combiner.dest->create_edge(d1, d2); - SECTION("an ID already present is reused, and a new one is created") { - TestGraphCombiner::Chunk chunk = make_chunk("more.pg"); - chunk.graph->create_handle("CCC", 2); // already in dest - chunk.graph->create_handle("GG", 3); // new + TestGraphCombiner::Chunk chunk = make_chunk("more.pg"); + chunk.graph->create_handle("CCC", 2); // already in dest + chunk.graph->create_handle("GG", 3); // new - combiner.share_nodes_by_id(chunk); + combiner.share_nodes_by_id(chunk); - REQUIRE(combiner.dest->get_node_count() == 3); - REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(2)) == "CCC"); - REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(3)) == "GG"); - } - - SECTION("an edge that is already there is not created twice") { - TestGraphCombiner::Chunk chunk = make_chunk("dup_edge.pg"); - handle_t c1 = chunk.graph->create_handle("AAAA", 1); - handle_t c2 = chunk.graph->create_handle("CCC", 2); - chunk.graph->create_edge(c1, c2); - - combiner.share_nodes_by_id(chunk); + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(2)) == "CCC"); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(3)) == "GG"); +} - size_t edges = 0; - combiner.dest->for_each_edge([&](const edge_t&) { edges++; }); - REQUIRE(edges == 1); - REQUIRE(combiner.dest->get_node_count() == 2); - } +TEST_CASE("share_nodes_by_id copies nodes with no edges into the resulting graph", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + TestGraphCombiner::Chunk chunk = make_chunk("island.pg"); + chunk.graph->create_handle("TTTT", 7); - SECTION("an edge's orientation survives the copy") { - TestGraphCombiner::Chunk chunk = make_chunk("flip_edge.pg"); - handle_t c2 = chunk.graph->create_handle("CCC", 2); - handle_t c3 = chunk.graph->create_handle("GG", 3); - // 2 to the reverse of 3: the copy must not quietly straighten this out. - chunk.graph->create_edge(c2, chunk.graph->flip(c3)); + combiner.share_nodes_by_id(chunk); - combiner.share_nodes_by_id(chunk); + REQUIRE(combiner.dest->has_node(7)); + REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(7)) == "TTTT"); +} - REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(2), - combiner.dest->get_handle(3, true))); - REQUIRE(!combiner.dest->has_edge(combiner.dest->get_handle(2), - combiner.dest->get_handle(3, false))); +TEST_CASE("share_nodes_by_id keeps the orientation of every kind of edge", "[combine]") { + for (bool from_rev : {false, true}) { + for (bool to_rev : {false, true}) { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + TestGraphCombiner::Chunk chunk = make_chunk("edge.pg"); + chunk.graph->create_handle("AAAA", 1); + chunk.graph->create_handle("CCC", 2); + chunk.graph->create_edge(chunk.graph->get_handle(1, from_rev), + chunk.graph->get_handle(2, to_rev)); + + combiner.share_nodes_by_id(chunk); + + REQUIRE(combiner.dest->get_edge_count() == 1); + REQUIRE(combiner.dest->has_edge(combiner.dest->get_handle(1, from_rev), + combiner.dest->get_handle(2, to_rev))); + } } +} - SECTION("nodes arriving on their own, with no edges, still land") { - TestGraphCombiner::Chunk chunk = make_chunk("island.pg"); - chunk.graph->create_handle("TTTT", 7); - - combiner.share_nodes_by_id(chunk); - - REQUIRE(combiner.dest->has_node(7)); - REQUIRE(combiner.dest->get_sequence(combiner.dest->get_handle(7)) == "TTTT"); - } +TEST_CASE("share_nodes_by_id rejects a repeated ID with a different sequence", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + combiner.dest->create_handle("CCC", 2); + TestGraphCombiner::Chunk chunk = make_chunk("clash.pg"); + chunk.graph->create_handle("GGG", 2); + REQUIRE_THROWS_AS(combiner.share_nodes_by_id(chunk), InputConflictError); + REQUIRE_THROWS_WITH(combiner.share_nodes_by_id(chunk), Catch::Contains("Node 2")); } TEST_CASE("copy_paths_checked carries a path's metadata across with it", "[combine]") { @@ -841,7 +955,7 @@ TEST_CASE("copy_paths_checked carries a path's metadata across with it", "[combi combiner.copy_paths_checked(chunk); - SECTION("every path arrives, under its own name") { + SECTION("every path is copied under its own name") { REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-7]", "HG002#1#chr1#3", "scaffold"}); } @@ -861,239 +975,294 @@ TEST_CASE("copy_paths_checked carries a path's metadata across with it", "[combi REQUIRE(combiner.dest->get_is_circular(combiner.dest->get_path_handle("scaffold"))); } - SECTION("the steps land in order, on dest's own nodes") { + SECTION("the steps are copied in order onto dest's nodes") { REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); REQUIRE(combiner.dest->get_step_count( combiner.dest->get_path_handle("GRCh38#0#chr1[0-7]")) == 2); } } -TEST_CASE("connect_seam joins the running reference to the next chunk", "[combine]") { +TEST_CASE("copy_paths_checked rejects a path name that is already in dest", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, + {combiner.dest->create_handle("AAAA")}); + TestGraphCombiner::Chunk chunk = make_chunk("dup.pg"); + add_ref_path(*chunk.graph, "GRCh38", "chr1", subrange_t{0, 4}, + {chunk.graph->create_handle("AAAA")}); + REQUIRE_THROWS_AS(combiner.copy_paths_checked(chunk), InputConflictError); + REQUIRE_THROWS_WITH(combiner.copy_paths_checked(chunk), Catch::Contains("GRCh38#0#chr1[0-4]")); +} - SECTION("without fusing, the two are joined by an edge") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - combiner.dest = make_unique(); - handle_t left = combiner.dest->create_handle("AAAA", 1); - handle_t start = combiner.dest->create_handle("CCC", 2); - handle_t end = combiner.dest->create_handle("GG", 3); - combiner.dest->create_edge(start, end); +TEST_CASE("connect_seam without fusing joins the chunk to the base path with an edge", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + handle_t end = combiner.dest->create_handle("GG", 3); + combiner.dest->create_edge(start, end); + + TestGraphCombiner::SeamState base_path_end{combiner.dest->get_id(left), false, 4, "left.pg"}; + handle_t new_end = combiner.connect_seam(base_path_end, start, end, "right.pg"); + + REQUIRE(combiner.dest->has_edge(left, start)); + REQUIRE(combiner.dest->get_node_count() == 3); + // Nothing was fused, so the chunk's own end is still the reference's end. + REQUIRE(new_end == end); +} - TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; - handle_t new_end = combiner.connect_seam(seam, "right.pg", start, end); +TEST_CASE("connect_seam with fusing merges the two boundary nodes into one", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + combiner.dest = make_unique(); + handle_t before = combiner.dest->create_handle("TT", 1); + handle_t left = combiner.dest->create_handle("AAAA", 2); + handle_t start = combiner.dest->create_handle("CCC", 3); + handle_t end = combiner.dest->create_handle("GG", 4); + combiner.dest->create_edge(before, left); + combiner.dest->create_edge(start, end); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {before, left}); + combiner.max_node_id = 4; + + TestGraphCombiner::SeamState base_path_end{combiner.dest->get_id(left), false, 6, "left.pg"}; + handle_t new_end = combiner.connect_seam(base_path_end, start, end, "right.pg"); + + REQUIRE(combiner.dest->get_node_count() == 3); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-6]") == "TTAAAACCC"); + REQUIRE(new_end == end); + REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); +} - REQUIRE(combiner.dest->has_edge(left, start)); - REQUIRE(combiner.dest->get_node_count() == 3); - // Nothing was welded, so the chunk's own end is still the reference's end. - REQUIRE(new_end == end); - } +TEST_CASE("connect_seam fusing a single-node chunk returns the handle to the fused node", "[combine]") { + // When the chunk's reference ends on the very node being fused, the old + // handle is destroyed by the fuse; the caller must get the new one back. + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + combiner.max_node_id = 2; - SECTION("an edge that is already there is not added again") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); - combiner.dest = make_unique(); - handle_t left = combiner.dest->create_handle("AAAA", 1); - handle_t start = combiner.dest->create_handle("CCC", 2); - combiner.dest->create_edge(left, start); + TestGraphCombiner::SeamState base_path_end{combiner.dest->get_id(left), false, 4, "left.pg"}; + handle_t new_end = combiner.connect_seam(base_path_end, start, start, "right.pg"); - TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; - combiner.connect_seam(seam, "right.pg", start, start); + REQUIRE(combiner.dest->get_node_count() == 1); + REQUIRE(combiner.dest->has_node(combiner.dest->get_id(new_end))); + REQUIRE(combiner.dest->get_sequence(new_end) == "AAAACCC"); + REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); +} - size_t edges = 0; - combiner.dest->for_each_edge([&](const edge_t&) { edges++; }); - REQUIRE(edges == 1); - } +TEST_CASE("connect_seam rejects fusing a boundary node that has an edge on the side being fused", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); + combiner.dest = make_unique(); + handle_t left = combiner.dest->create_handle("AAAA", 1); + handle_t start = combiner.dest->create_handle("CCC", 2); + handle_t other = combiner.dest->create_handle("GG", 3); + combiner.dest->create_edge(left, other); + combiner.max_node_id = 3; + + TestGraphCombiner::SeamState base_path_end{combiner.dest->get_id(left), false, 4, "left.pg"}; + REQUIRE_THROWS_AS(combiner.connect_seam(base_path_end, start, start, "right.pg"), FuseError); + REQUIRE_THROWS_WITH(combiner.connect_seam(base_path_end, start, start, "right.pg"), + Catch::Contains("left.pg")); +} - SECTION("fusing welds the two boundary nodes into one") { - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); - combiner.dest = make_unique(); - handle_t before = combiner.dest->create_handle("TT", 1); - handle_t left = combiner.dest->create_handle("AAAA", 2); - handle_t start = combiner.dest->create_handle("CCC", 3); - handle_t end = combiner.dest->create_handle("GG", 4); - combiner.dest->create_edge(before, left); - combiner.dest->create_edge(start, end); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {before, left}); - combiner.max_node_id = 4; - - TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 6, "left.pg"}; - handle_t new_end = combiner.connect_seam(seam, "right.pg", start, end); - - REQUIRE(combiner.dest->get_node_count() == 3); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-6]") == "TTAAAACCC"); - REQUIRE(new_end == end); - REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); - } +TEST_CASE("merge_path_fragments merges two abutting fragments into one path spanning both", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {n2}); - SECTION("fusing a single-node chunk returns the weld, not the stale handle") { - // When the chunk's reference ends on the very node being welded, the old - // handle is destroyed by the fuse; the caller must get the new one back. - TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, true, false); - combiner.dest = make_unique(); - handle_t left = combiner.dest->create_handle("AAAA", 1); - handle_t start = combiner.dest->create_handle("CCC", 2); - combiner.max_node_id = 2; - - TestGraphCombiner::SeamState seam{combiner.dest->get_id(left), false, 4, "left.pg"}; - handle_t new_end = combiner.connect_seam(seam, "right.pg", start, start); - - REQUIRE(combiner.dest->get_node_count() == 1); - REQUIRE(combiner.dest->has_node(combiner.dest->get_id(new_end))); - REQUIRE(combiner.dest->get_sequence(new_end) == "AAAACCC"); - REQUIRE(combiner.max_node_id >= combiner.dest->max_node_id()); - } -} + combiner.merge_path_fragments(); -TEST_CASE("merge_path_fragments joins pieces of one path back together", "[combine]") { + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); + // Nothing joined the two before, so the merge has to say they abut. + REQUIRE(combiner.dest->has_edge(n1, n2)); +} - SECTION("two abutting fragments become one path spanning both") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CCC"); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {n2}); +TEST_CASE("merge_path_fragments keeps the name of a lone fragment", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - combiner.merge_path_fragments(); + combiner.merge_path_fragments(); - REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-7]"}); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); - // Nothing joined the two before, so the merge has to say they abut. - REQUIRE(combiner.dest->has_edge(n1, n2)); - } + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-4]"}); +} - SECTION("a lone fragment is left alone, and keeps its name") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); +TEST_CASE("merge_path_fragments keeps the name of a lone path with no subrange", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAACCCC"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", PathMetadata::NO_SUBRANGE, {n1}); - combiner.merge_path_fragments(); + combiner.merge_path_fragments(); - REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-4]"}); - } + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1"}); +} - SECTION("fragments sharing a run of steps at the seam do not repeat it") { - TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CC"); - handle_t n3 = combiner.dest->create_handle("GGG"); - combiner.dest->create_edge(n1, n2); - combiner.dest->create_edge(n2, n3); - // Both fragments cross n2: it must appear once in the merge, not twice. - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {n1, n2}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 9}, {n2, n3}); - - combiner.merge_path_fragments(); - - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); - REQUIRE(combiner.dest->get_step_count( - combiner.dest->get_path_handle("GRCh38#0#chr1[0-9]")) == 3); - } +TEST_CASE("merge_path_fragments does not repeat steps that two fragments share", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::SHARED_IDS, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CC"); + handle_t n3 = combiner.dest->create_handle("GGG"); + combiner.dest->create_edge(n1, n2); + combiner.dest->create_edge(n2, n3); + // Both fragments cross n2: it must appear once in the merge, not twice. + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 6}, {n1, n2}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 9}, {n2, n3}); + + combiner.merge_path_fragments(); + + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); + REQUIRE(combiner.dest->get_step_count( + combiner.dest->get_path_handle("GRCh38#0#chr1[0-9]")) == 3); +} - SECTION("fragments of different paths do not bleed into each other") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t x1 = combiner.dest->create_handle("AAAA"); - handle_t x2 = combiner.dest->create_handle("CCC"); - handle_t y1 = combiner.dest->create_handle("TTTT"); - handle_t y2 = combiner.dest->create_handle("GGG"); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {x1}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {x2}); - add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{0, 4}, {y1}); - add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{4, 7}, {y2}); +TEST_CASE("merge_path_fragments does not merge fragments of different paths", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t x1 = combiner.dest->create_handle("AAAA"); + handle_t x2 = combiner.dest->create_handle("CCC"); + handle_t y1 = combiner.dest->create_handle("TTTT"); + handle_t y2 = combiner.dest->create_handle("GGG"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {x1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 7}, {x2}); + add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{0, 4}, {y1}); + add_ref_path(*combiner.dest, "GRCh38", "chr2", subrange_t{4, 7}, {y2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) + == vector{"GRCh38#0#chr1[0-7]", "GRCh38#0#chr2[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr2[0-7]") == "TTTTGGG"); + REQUIRE(!combiner.dest->has_edge(x2, y1)); +} - combiner.merge_path_fragments(); +TEST_CASE("merge_path_fragments merges three fragments given out of order in coordinate order", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n3 = combiner.dest->create_handle("GGG"); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CC"); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, {n3}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 6}, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-9]"}); + REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); + REQUIRE(combiner.dest->has_edge(n1, n2)); + REQUIRE(combiner.dest->has_edge(n2, n3)); +} - REQUIRE(sorted_path_names(*combiner.dest) - == vector{"GRCh38#0#chr1[0-7]", "GRCh38#0#chr2[0-7]"}); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-7]") == "AAAACCC"); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr2[0-7]") == "TTTTGGG"); - REQUIRE(!combiner.dest->has_edge(x2, y1)); - } +TEST_CASE("merge_path_fragments keeps the phase block of a merged haplotype path", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, + subrange_t{0, 4}, {n1}); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, + subrange_t{4, 7}, {n2}); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#2#chr22#4[0-7]"}); + path_handle_t merged = combiner.dest->get_path_handle("HG002#2#chr22#4[0-7]"); + REQUIRE(combiner.dest->get_phase_block(merged) == 4); + REQUIRE(combiner.dest->get_sense(merged) == PathSense::HAPLOTYPE); +} - SECTION("fragments arriving out of order are put back in coordinate order") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n3 = combiner.dest->create_handle("GGG"); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CC"); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, {n3}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 6}, {n2}); +TEST_CASE("merge_path_fragments positions fragments by phase block when phase blocks are offsets", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, true); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {n1}); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 4, + PathMetadata::NO_SUBRANGE, {n2}); - combiner.merge_path_fragments(); + combiner.merge_path_fragments(); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); - } + REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#1#chr1#0[0-7]"}); + REQUIRE(spell_path(*combiner.dest, "HG002#1#chr1#0[0-7]") == "AAAACCC"); +} - SECTION("a merged HAPLOTYPE path keeps the phase block it needs") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CCC"); - add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, - subrange_t{0, 4}, {n1}); - add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr22", 2, 4, - subrange_t{4, 7}, {n2}); - - combiner.merge_path_fragments(); - - REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#2#chr22#4[0-7]"}); - path_handle_t merged = combiner.dest->get_path_handle("HG002#2#chr22#4[0-7]"); - REQUIRE(combiner.dest->get_phase_block(merged) == 4); - REQUIRE(combiner.dest->get_sense(merged) == PathSense::HAPLOTYPE); - } +TEST_CASE("merge_path_fragments keeps a circular path circular", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 7}, {n2}, true); + + combiner.merge_path_fragments(); + + REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chrM[0-7]"}); + REQUIRE(combiner.dest->get_is_circular( + combiner.dest->get_path_handle("GRCh38#0#chrM[0-7]"))); +} - SECTION("with -P, phase blocks position the fragments and fold into a subrange") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, true); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CCC"); - add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, - PathMetadata::NO_SUBRANGE, {n1}); - add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 4, - PathMetadata::NO_SUBRANGE, {n2}); +TEST_CASE("merge_path_fragments rejects fragments that disagree on circularity", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + handle_t n1 = combiner.dest->create_handle("AAAA"); + handle_t n2 = combiner.dest->create_handle("CCC"); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); + add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 7}, {n2}, false); + REQUIRE_THROWS_AS(combiner.merge_path_fragments(), InputConflictError); + REQUIRE_THROWS_WITH(combiner.merge_path_fragments(), Catch::Contains("chrM")); +} - combiner.merge_path_fragments(); +TEST_CASE("merge_path_fragments rejects a gap between fragments", "[combine]") { + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); + combiner.dest = make_unique(); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, + {combiner.dest->create_handle("AAAA")}); + add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, + {combiner.dest->create_handle("CCC")}); + REQUIRE_THROWS_AS(combiner.merge_path_fragments(), FragmentMergeError); + REQUIRE_THROWS_WITH(combiner.merge_path_fragments(), Catch::Contains("Gap of 2 bp")); +} - REQUIRE(sorted_path_names(*combiner.dest) == vector{"HG002#1#chr1#0[0-7]"}); - REQUIRE(spell_path(*combiner.dest, "HG002#1#chr1#0[0-7]") == "AAAACCC"); - } +TEST_CASE("merge_path_fragments rejects phase blocks that do not line up as offsets", "[combine]") { + // Blocks 0 and 2 overlap, so this is not a gap, but 0 + 4bp should be followed by block 4. + TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, true); + combiner.dest = make_unique(); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 0, + PathMetadata::NO_SUBRANGE, {combiner.dest->create_handle("AAAA")}); + add_path(*combiner.dest, PathSense::HAPLOTYPE, "HG002", "chr1", 1, 2, + PathMetadata::NO_SUBRANGE, {combiner.dest->create_handle("CCCC")}); + REQUIRE_THROWS_AS(combiner.merge_path_fragments(), FragmentMergeError); + REQUIRE_THROWS_WITH(combiner.merge_path_fragments(), Catch::Contains("HG002#1#chr1#0")); +} - SECTION("a circular path stays circular through the merge") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CCC"); - add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, - PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, {n1}, true); - add_path(*combiner.dest, PathSense::REFERENCE, "GRCh38", "chrM", 0, - PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 7}, {n2}, true); - - combiner.merge_path_fragments(); - - REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chrM[0-7]"}); - REQUIRE(combiner.dest->get_is_circular( - combiner.dest->get_path_handle("GRCh38#0#chrM[0-7]"))); - } +TEST_CASE("combine rejects chunks of one base path that disagree on circularity", "[combine]") { + vector inputs(2); + inputs[0].name = "left.pg"; + inputs[0].graph = make_unique(); + add_path(*inputs[0].graph, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{0, 4}, + {inputs[0].graph->create_handle("AAAA")}, true); + inputs[1].name = "right.pg"; + inputs[1].graph = make_unique(); + add_path(*inputs[1].graph, PathSense::REFERENCE, "GRCh38", "chrM", 0, + PathMetadata::NO_PHASE_BLOCK, subrange_t{4, 7}, + {inputs[1].graph->create_handle("CCC")}, false); - SECTION("three fragments merge in one pass") { - TestGraphCombiner combiner(SeamPolicy::RENUMBER, false, false); - combiner.dest = make_unique(); - handle_t n1 = combiner.dest->create_handle("AAAA"); - handle_t n2 = combiner.dest->create_handle("CC"); - handle_t n3 = combiner.dest->create_handle("GGG"); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{0, 4}, {n1}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{4, 6}, {n2}); - add_ref_path(*combiner.dest, "GRCh38", "chr1", subrange_t{6, 9}, {n3}); - - combiner.merge_path_fragments(); - - REQUIRE(sorted_path_names(*combiner.dest) == vector{"GRCh38#0#chr1[0-9]"}); - REQUIRE(spell_path(*combiner.dest, "GRCh38#0#chr1[0-9]") == "AAAACCGGG"); - REQUIRE(combiner.dest->has_edge(n1, n2)); - REQUIRE(combiner.dest->has_edge(n2, n3)); - } + TestGraphCombiner combiner(SeamPolicy::COORD_TRIM, false, false); + REQUIRE_THROWS_AS(combiner.combine(std::move(inputs)), InputConflictError); } } diff --git a/src/unittest/fuse_nodes.cpp b/src/unittest/fuse_nodes.cpp index f791bf736c1..f277c153553 100644 --- a/src/unittest/fuse_nodes.cpp +++ b/src/unittest/fuse_nodes.cpp @@ -24,34 +24,51 @@ static string spell_path(const PathHandleGraph& graph, const string& name) { return sequence; } -TEST_CASE("can_fuse_nodes accepts a clean boundary and rejects the rest", "[fuse_nodes][algorithms]") { +TEST_CASE("can_fuse_nodes accepts a left node with nothing on its right and a right node with nothing on its left", "[fuse_nodes][algorithms]") { bdsg::HashGraph graph; - handle_t n1 = graph.create_handle("AAA"); - handle_t n2 = graph.create_handle("CC"); - handle_t n3 = graph.create_handle("GGGG"); - handle_t n4 = graph.create_handle("TT"); - graph.create_edge(n1, n2); - graph.create_edge(n3, n4); + handle_t a = graph.create_handle("TT"); + handle_t left = graph.create_handle("AAA"); + handle_t right = graph.create_handle("CC"); + handle_t b = graph.create_handle("GGGG"); + // Edges on the outer sides don't matter. + graph.create_edge(a, left); + graph.create_edge(right, b); + REQUIRE(algorithms::can_fuse_nodes(graph, left, right)); +} - SECTION("nothing on the inner sides of the pair") { - REQUIRE(algorithms::can_fuse_nodes(graph, n2, n3)); - } - SECTION("the left node has something on its right") { - REQUIRE(!algorithms::can_fuse_nodes(graph, n1, n3)); - } - SECTION("the right node has something on its left") { - REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n4)); - } - SECTION("a node cannot be fused with itself") { - REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n2)); - } - SECTION("an edge between the pair would be swallowed, so it is refused") { - graph.create_edge(n2, n3); - REQUIRE(!algorithms::can_fuse_nodes(graph, n2, n3)); - } +TEST_CASE("can_fuse_nodes rejects a left node with an edge on its right side", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t left = graph.create_handle("AAA"); + handle_t right = graph.create_handle("CC"); + handle_t other = graph.create_handle("GGGG"); + graph.create_edge(left, other); + REQUIRE(!algorithms::can_fuse_nodes(graph, left, right)); +} + +TEST_CASE("can_fuse_nodes rejects a right node with an edge on its left side", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t left = graph.create_handle("AAA"); + handle_t right = graph.create_handle("CC"); + handle_t other = graph.create_handle("GGGG"); + graph.create_edge(other, right); + REQUIRE(!algorithms::can_fuse_nodes(graph, left, right)); +} + +TEST_CASE("can_fuse_nodes rejects fusing a node with itself", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t node = graph.create_handle("AAA"); + REQUIRE(!algorithms::can_fuse_nodes(graph, node, node)); +} + +TEST_CASE("can_fuse_nodes rejects two nodes already joined by an edge", "[fuse_nodes][algorithms]") { + bdsg::HashGraph graph; + handle_t left = graph.create_handle("AAA"); + handle_t right = graph.create_handle("CC"); + graph.create_edge(left, right); + REQUIRE(!algorithms::can_fuse_nodes(graph, left, right)); } -TEST_CASE("fuse_nodes concatenates sequence, re-hangs edges, and keeps paths", "[fuse_nodes][algorithms]") { +TEST_CASE("fuse_nodes concatenates the sequences and keeps outer edges and paths", "[fuse_nodes][algorithms]") { bdsg::HashGraph graph; handle_t n1 = graph.create_handle("AAA"); handle_t n2 = graph.create_handle("CC"); @@ -60,8 +77,8 @@ TEST_CASE("fuse_nodes concatenates sequence, re-hangs edges, and keeps paths", " graph.create_edge(n1, n2); graph.create_edge(n3, n4); - // Three paths over the seam: one ends on it, one starts on it, and one - // crosses it backwards, an orientation the rewrite has to preserve. + // One path ends on the fused pair, one starts on it, and one visits it in reverse; + // fusing must keep all three, including the reverse one and its orientation. path_handle_t left_path = graph.create_path_handle("left"); graph.append_step(left_path, n1); graph.append_step(left_path, n2); @@ -96,8 +113,8 @@ TEST_CASE("fuse_nodes turns a cycle through the pair into a self-loop", "[fuse_n handle_t n2 = graph.create_handle("CC"); handle_t n3 = graph.create_handle("GGGG"); graph.create_edge(n1, n2); - // Back edge closing a cycle over the pair. Neither end of it sits on the - // inner side of the seam, so the pair is still fusable. + // This edge closes a cycle, but it doesn't touch the sides being fused, + // so fusing is still allowed. graph.create_edge(n3, n2); REQUIRE(algorithms::can_fuse_nodes(graph, n2, n3)); diff --git a/src/unittest/path_length.cpp b/src/unittest/path_length.cpp new file mode 100644 index 00000000000..7d48789e73f --- /dev/null +++ b/src/unittest/path_length.cpp @@ -0,0 +1,54 @@ +/// \file path_length.cpp +/// +/// Unit tests for handlegraph::algorithms::get_path_length. + +#include "catch.hpp" +#include "../handle.hpp" + +#include "bdsg/hash_graph.hpp" +#include "bdsg/overlays/path_position_overlays.hpp" + +#include "handlegraph/algorithms/path_length.hpp" + +namespace vg { +namespace unittest { + +using handlegraph::algorithms::get_path_length; + +TEST_CASE("get_path_length adds up visited node lengths", "[algorithms][path_length]") { + bdsg::HashGraph graph; + handle_t n1 = graph.create_handle("AAAA"); + handle_t n2 = graph.create_handle("CC"); + handle_t n3 = graph.create_handle("GGGGG"); + graph.create_edge(n1, n2); + graph.create_edge(n2, n3); + path_handle_t path = graph.create_path_handle("path"); + graph.append_step(path, n1); + graph.append_step(path, n2); + graph.append_step(path, n3); + + SECTION("a graph without positions is scanned step by step") { + // HashGraph is not a PathPositionHandleGraph, so this takes the scan_path branch. + REQUIRE(get_path_length(&graph, path) == 11); + } + + SECTION("a positional graph reports the same length") { + bdsg::PositionOverlay positional(&graph); + REQUIRE(get_path_length(&positional, positional.get_path_handle("path")) == 11); + } + + SECTION("an empty path has no length") { + path_handle_t empty = graph.create_path_handle("empty"); + REQUIRE(get_path_length(&graph, empty) == 0); + } + + SECTION("a step's orientation does not change how much it contributes") { + handle_t n4 = graph.create_handle("TTT"); + path_handle_t flipped = graph.create_path_handle("flipped"); + graph.append_step(flipped, graph.flip(n4)); + REQUIRE(get_path_length(&graph, flipped) == 3); + } +} + +} +} diff --git a/test/t/51_vg_combine.t b/test/t/51_vg_combine.t index 79db325a596..973164a34c1 100644 --- a/test/t/51_vg_combine.t +++ b/test/t/51_vg_combine.t @@ -5,371 +5,247 @@ BASH_TAP_ROOT=../deps/bash-tap PATH=../bin:$PATH # for vg -plan tests 115 +plan tests 45 -C=combine -mkdir -p $C +# Scratch directory for this test's files; removed at the end. +SCRATCH=combine_scratch +mkdir -p $SCRATCH -# Construct chunks holding a reference plus two haplotypes. -RS='H\tVN:Z:1.1\tRS:Z:GRCh38\n' +# GFA fixtures are written with spaces between fields; tr turns them into the tabs GFA needs. +# Construct chunks holding a reference plus two haplotypes. # Left chunk including HG002 and HG005#1 -printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-10]\t1+,3+,4+\t*\nP\tHG005#1#chr1#0[0-10]\t1+,3+,4+\t*\n' > $C/ml.gfa +tr ' ' '\t' > $SCRATCH/ml.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 1 AAAA +S 2 C +S 3 G +S 4 GGGGG +L 1 + 2 + 0M +L 1 + 3 + 0M +L 2 + 4 + 0M +L 3 + 4 + 0M +P GRCh38#0#chr1[0-10] 1+,2+,4+ * +P HG002#1#chr1#0[0-10] 1+,2+,4+ * +P HG002#2#chr1#0[0-10] 1+,3+,4+ * +P HG005#1#chr1#0[0-10] 1+,3+,4+ * +EOF # Right chunk including only HG002 -printf "$RS"'S\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[10-20]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[10-20]\t5+,6+,8+\t*\n' > $C/mr.gfa -vg convert -p $C/ml.gfa > $C/ml.pg 2>/dev/null -vg convert -p $C/mr.gfa > $C/mr.pg 2>/dev/null - -is "$(vg paths -M -v $C/ml.pg | tail -n +2 | cut -f2 | grep -c HAPLOTYPE)" "3" "the chunk carries three haplotype paths" -is "$(vg paths -M -v $C/ml.pg | tail -n +2 | cut -f2 | grep -c REFERENCE)" "1" "the chunk carries one reference path" - -vg combine $C/ml.pg $C/mr.pg > $C/o_def.vg 2>/dev/null -is "$(vg paths -Ev $C/o_def.vg | wc -l)" "4" "every path is considered" -is "$(vg paths -Ev $C/o_def.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20],HG005#1#chr1#0[0-10]" "each sample and haplotype merges within its own group" -is "$(vg paths -Ev $C/o_def.vg | cut -f2 | sort -n | paste -sd,)" "10,20,20,20" "the merged paths span both chunks and the lone one does not" -is "$(vg stats -N $C/o_def.vg)" "8" "merging keeps every node of both chunks, 4 for each" -is "$(vg stats -E $C/o_def.vg)" "9" "the three paths cross the seam on one shared edge, so the edges count increase by 1(8 -> 9)" -is "$(vg stats -s $C/o_def.vg | wc -l)" "1" "the two chunks end up in one component" - -# Only fragments of the same logical paths are merged. -is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>GRCh38#0#chr1\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTAGGGGG" "the reference only merge its own fragments" -is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG002#1#chr1#0\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTCGGGGG" "HG002#1#chr1 only merge its own fragments" -is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 only merge its own fragments" -is "$(vg paths -Fv $C/o_def.vg | grep -A1 '^>HG005#1#chr1#0\[0-10\]$' | tail -1)" "AAAAGGGGGG" "HG005#1#chr1 is the lone fragment untouched" +tr ' ' '\t' > $SCRATCH/mr.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 5 TTTT +S 6 A +S 7 C +S 8 GGGGG +L 5 + 6 + 0M +L 5 + 7 + 0M +L 6 + 8 + 0M +L 7 + 8 + 0M +P GRCh38#0#chr1[10-20] 5+,6+,8+ * +P HG002#1#chr1#0[10-20] 5+,7+,8+ * +P HG002#2#chr1#0[10-20] 5+,6+,8+ * +EOF +vg convert -p $SCRATCH/ml.gfa > $SCRATCH/ml.pg 2>/dev/null +vg convert -p $SCRATCH/mr.gfa > $SCRATCH/mr.pg 2>/dev/null + +is "$(vg paths -M -v $SCRATCH/ml.pg | tail -n +2 | cut -f2 | grep -c HAPLOTYPE)" "3" "the chunk carries three haplotype paths" +is "$(vg paths -M -v $SCRATCH/ml.pg | tail -n +2 | cut -f2 | grep -c REFERENCE)" "1" "the chunk carries one reference path" + +vg combine $SCRATCH/ml.pg $SCRATCH/mr.pg > $SCRATCH/o_def.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_def.vg | wc -l)" "4" "every path is considered" +is "$(vg paths -Ev $SCRATCH/o_def.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20],HG005#1#chr1#0[0-10]" "each sample and haplotype merges within its own group" +is "$(vg paths -Ev $SCRATCH/o_def.vg | cut -f2 | sort -n | paste -sd,)" "10,20,20,20" "the merged paths span both chunks and the lone one does not" +is "$(vg stats -N $SCRATCH/o_def.vg)" "8" "merging keeps every node of both chunks, 4 for each" +is "$(vg stats -E $SCRATCH/o_def.vg)" "9" "the three paths cross the seam on one shared edge, so the edges count increase by 1(8 -> 9)" +is "$(vg stats -s $SCRATCH/o_def.vg | wc -l)" "1" "the two chunks end up in one component" # renumber is the default, so naming it explicitly must change nothing. -vg combine -s renumber $C/ml.pg $C/mr.pg > $C/o_ren.vg 2>/dev/null -is "$(vg paths -Ev $C/o_ren.vg)" "$(vg paths -Ev $C/o_def.vg)" "--seam renumber is the default" -is "$(vg paths -Fv $C/o_ren.vg | sort)" "$(vg paths -Fv $C/o_def.vg | sort)" "and spells out the same sequences" - -# A path with no subrange is not a fragment of anything, so its name must come through untouched. -printf "$RS"'S\t1\tAAAACCCC\nP\tGRCh38#0#chr1\t1+\t*\n' > $C/solo.gfa -vg convert -p $C/solo.gfa > $C/solo.pg 2>/dev/null - -vg combine $C/solo.pg > $C/o_solo.vg 2>/dev/null -is "$(vg paths -Ev $C/o_solo.vg)" "$(printf 'GRCh38#0#chr1\t8')" "a lone path with no subrange keeps its name" - -# Fragments from different haplotype must not be merged. -printf "$RS"'S\t1\tAAAA\nS\t2\tCGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+\t*\n' > $C/h1.gfa -printf "$RS"'S\t3\tTTTT\nS\t4\tAGGGGG\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t3+,4+\t*\nP\tHG002#2#chr1#0[10-20]\t3+,4+\t*\n' > $C/h2.gfa -vg convert -p $C/h1.gfa > $C/h1.pg 2>/dev/null -vg convert -p $C/h2.gfa > $C/h2.pg 2>/dev/null - -vg combine $C/h1.pg $C/h2.pg > $C/o_hap.vg 2>/dev/null -is "$(vg paths -Ev $C/o_hap.vg | wc -l)" "3" "only the reference path has fragments to merge" -is "$(vg paths -Ev $C/o_hap.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-10],HG002#2#chr1#0[10-20]" "fragments of different haplotypes stay apart" - -# Shuffled fragments of two contigs must not splice into each other. -printf "$RS"'S\t1\tAAAA\nS\t2\tCCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/mc_chr1l.gfa -printf "$RS"'S\t1\tGGGG\nS\t2\tTTTTTT\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[10-20]\t1+,2+\t*\n' > $C/mc_chr1r.gfa -printf "$RS"'S\t1\tTTTT\nS\t2\tGGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[0-10]\t1+,2+\t*\n' > $C/mc_chr2l.gfa -printf "$RS"'S\t1\tCCCC\nS\t2\tAAAAAA\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[10-20]\t1+,2+\t*\n' > $C/mc_chr2r.gfa -for g in mc_chr1l mc_chr1r mc_chr2l mc_chr2r; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done - -vg combine $C/mc_chr2r.pg $C/mc_chr1l.pg $C/mc_chr2l.pg $C/mc_chr1r.pg > $C/o_multi.vg 2>/dev/null -is "$(vg paths -Ev $C/o_multi.vg | wc -l)" "2" "each contig yields its own merged path" -is "$(vg paths -Ev $C/o_multi.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],GRCh38#0#chr2[0-20]" "both contigs span both of their chunks" -is "$(vg paths -Ev $C/o_multi.vg | cut -f2 | sort -u)" "20" "neither contig grew by taking the other's chunks" -is "$(vg stats -s $C/o_multi.vg | wc -l)" "2" "the contigs stay in separate components" -is "$(vg stats -E $C/o_multi.vg)" "6" "only the two intra-contig seams are edged" -is "$(vg paths -Fv $C/o_multi.vg | grep -A1 '^>GRCh38#0#chr1\[0-20\]$' | tail -1)" "AAAACCCCCCGGGGTTTTTT" "chr1 spells out only its own chunks" -is "$(vg paths -Fv $C/o_multi.vg | grep -A1 '^>GRCh38#0#chr2\[0-20\]$' | tail -1)" "TTTTGGGGGGCCCCAAAAAA" "chr2 spells out only its own chunks" +vg combine -s renumber $SCRATCH/ml.pg $SCRATCH/mr.pg > $SCRATCH/o_ren.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_ren.vg)" "$(vg paths -Ev $SCRATCH/o_def.vg)" "--seam renumber is the default" +is "$(vg paths -Fv $SCRATCH/o_ren.vg | sort)" "$(vg paths -Fv $SCRATCH/o_def.vg | sort)" "and spells out the same sequences" # --seam shared: the two chunks sharing nodes 4 and 5. -printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nS\t5\tTTTT\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nL\t4\t+\t5\t+\t0M\nP\tGRCh38#0#chr1[0-14]\t1+,2+,4+,5+\t*\nP\tHG002#1#chr1#0[0-14]\t1+,2+,4+,5+\t*\nP\tHG002#2#chr1#0[0-14]\t1+,3+,4+,5+\t*\n' > $C/sl.gfa -printf "$RS"'S\t4\tGGGGG\nS\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t4\t+\t5\t+\t0M\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[5-20]\t4+,5+,6+,8+\t*\nP\tHG002#1#chr1#0[5-20]\t4+,5+,7+,8+\t*\nP\tHG002#2#chr1#0[5-20]\t4+,5+,6+,8+\t*\n' > $C/sr.gfa -vg convert -p $C/sl.gfa > $C/sl.pg 2>/dev/null -vg convert -p $C/sr.gfa > $C/sr.pg 2>/dev/null - -vg combine -s shared $C/sl.pg $C/sr.pg > $C/o_sh.vg 2>/dev/null -is "$(vg paths -Ev $C/o_sh.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20]" "--seam shared merges all three paths over their shared nodes" -is "$(vg paths -Ev $C/o_sh.vg | cut -f2 | sort -u)" "20" "no path counts the same nodes twice" -is "$(vg stats -N $C/o_sh.vg)" "8" "the shared nodes are deduplicated rather than copied" -is "$(vg stats -E $C/o_sh.vg)" "9" "the edges between them are also deduplicated" -is "$(vg paths -Fv $C/o_sh.vg | grep -A1 '^>HG002#1#chr1#0\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTCGGGGG" "HG002#1#chr1 keeps its own sequences over the shared nodes" -is "$(vg paths -Fv $C/o_sh.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 keeps its own sequences over the shared nodes" - -# --seam shared: different paths share different nodes. -printf "$RS"'S\t4\tGGGGG\nS\t5\tTTTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t4\t+\t5\t+\t0M\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[5-20]\t4+,5+,6+,8+\t*\nP\tHG002#1#chr1#0[5-20]\t4+,5+,7+,8+\t*\nP\tHG002#2#chr1#0[10-20]\t5+,6+,8+\t*\n' > $C/sr2.gfa -vg convert -p $C/sr2.gfa > $C/sr2.pg 2>/dev/null - -vg combine -s shared $C/sl.pg $C/sr2.pg > $C/o_sh2.vg 2>/dev/null -is "$(vg paths -Ev $C/o_sh2.vg | cut -f2 | sort -u)" "20" "HG002#2#chr1 has a shorter overlap but it still keeps the same total length" -is "$(vg paths -Fv $C/o_sh2.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 also keeps the same sequence" +tr ' ' '\t' > $SCRATCH/sl.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 1 AAAA +S 2 C +S 3 G +S 4 GGGGG +S 5 TTTT +L 1 + 2 + 0M +L 1 + 3 + 0M +L 2 + 4 + 0M +L 3 + 4 + 0M +L 4 + 5 + 0M +P GRCh38#0#chr1[0-14] 1+,2+,4+,5+ * +P HG002#1#chr1#0[0-14] 1+,2+,4+,5+ * +P HG002#2#chr1#0[0-14] 1+,3+,4+,5+ * +EOF +tr ' ' '\t' > $SCRATCH/sr.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 4 GGGGG +S 5 TTTT +S 6 A +S 7 C +S 8 GGGGG +L 4 + 5 + 0M +L 5 + 6 + 0M +L 5 + 7 + 0M +L 6 + 8 + 0M +L 7 + 8 + 0M +P GRCh38#0#chr1[5-20] 4+,5+,6+,8+ * +P HG002#1#chr1#0[5-20] 4+,5+,7+,8+ * +P HG002#2#chr1#0[5-20] 4+,5+,6+,8+ * +EOF +vg convert -p $SCRATCH/sl.gfa > $SCRATCH/sl.pg 2>/dev/null +vg convert -p $SCRATCH/sr.gfa > $SCRATCH/sr.pg 2>/dev/null + +vg combine -s shared $SCRATCH/sl.pg $SCRATCH/sr.pg > $SCRATCH/o_sh.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_sh.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-20],HG002#1#chr1#0[0-20],HG002#2#chr1#0[0-20]" "--seam shared merges all three paths over their shared nodes" +is "$(vg paths -Ev $SCRATCH/o_sh.vg | cut -f2 | sort -u)" "20" "no path counts the same nodes twice" +is "$(vg stats -N $SCRATCH/o_sh.vg)" "8" "the shared nodes are deduplicated rather than copied" +is "$(vg stats -E $SCRATCH/o_sh.vg)" "9" "the edges between them are also deduplicated" +is "$(vg paths -Fv $SCRATCH/o_sh.vg | grep -A1 '^>HG002#1#chr1#0\[0-20\]$' | tail -1)" "AAAACGGGGGTTTTCGGGGG" "HG002#1#chr1 keeps its own sequences over the shared nodes" +is "$(vg paths -Fv $SCRATCH/o_sh.vg | grep -A1 '^>HG002#2#chr1#0\[0-20\]$' | tail -1)" "AAAAGGGGGGTTTTAGGGGG" "HG002#2#chr1 keeps its own sequences over the shared nodes" # --seam trim: on a 2bp overlap. -printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-10]\t1+,3+,4+\t*\n' > $C/tl.gfa -printf "$RS"'S\t5\tGGTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[8-18]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[8-18]\t5+,6+,8+\t*\n' > $C/tr.gfa -vg convert -p $C/tl.gfa > $C/tl.pg 2>/dev/null -vg convert -p $C/tr.gfa > $C/tr.pg 2>/dev/null - -vg combine -s trim $C/tl.pg $C/tr.pg > $C/o_trim.vg 2>/dev/null -is "$(vg paths -Ev $C/o_trim.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "all three paths report the same trimmed span" -is "$(vg paths -Ev $C/o_trim.vg | cut -f2 | sort -u)" "18" "the overlap of three paths has been trimmed" -is "$(vg stats -N $C/o_trim.vg)" "8" "trimming splits the boundary node and drops the overlapping half, leaving the node counts unchanged" -is "$(vg stats -E $C/o_trim.vg)" "9" "seaming create one edge at the boundary" -is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>GRCh38#0#chr1\[0-18\]$' | tail -1)" "AAAACGGGGGTTAGGGGG" "the overlap of the reference path is trimmed" -is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>HG002#1#chr1#0\[0-18\]$' | tail -1)" "AAAACGGGGGTTCGGGGG" "the overlap of the HG002#1#chr1 is trimmed" -is "$(vg paths -Fv $C/o_trim.vg | grep -A1 '^>HG002#2#chr1#0\[0-18\]$' | tail -1)" "AAAAGGGGGGTTAGGGGG" "the overlap of the HG002#2#chr1 is trimmed" +tr ' ' '\t' > $SCRATCH/tl.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 1 AAAA +S 2 C +S 3 G +S 4 GGGGG +L 1 + 2 + 0M +L 1 + 3 + 0M +L 2 + 4 + 0M +L 3 + 4 + 0M +P GRCh38#0#chr1[0-10] 1+,2+,4+ * +P HG002#1#chr1#0[0-10] 1+,2+,4+ * +P HG002#2#chr1#0[0-10] 1+,3+,4+ * +EOF +tr ' ' '\t' > $SCRATCH/tr.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 5 GGTT +S 6 A +S 7 C +S 8 GGGGG +L 5 + 6 + 0M +L 5 + 7 + 0M +L 6 + 8 + 0M +L 7 + 8 + 0M +P GRCh38#0#chr1[8-18] 5+,6+,8+ * +P HG002#1#chr1#0[8-18] 5+,7+,8+ * +P HG002#2#chr1#0[8-18] 5+,6+,8+ * +EOF +vg convert -p $SCRATCH/tl.gfa > $SCRATCH/tl.pg 2>/dev/null +vg convert -p $SCRATCH/tr.gfa > $SCRATCH/tr.pg 2>/dev/null + +vg combine -s trim $SCRATCH/tl.pg $SCRATCH/tr.pg > $SCRATCH/o_trim.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_trim.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "all three paths report the same trimmed span" +is "$(vg paths -Ev $SCRATCH/o_trim.vg | cut -f2 | sort -u)" "18" "the overlap of three paths has been trimmed" +is "$(vg stats -N $SCRATCH/o_trim.vg)" "8" "trimming splits the boundary node and drops the overlapping half, leaving the node counts unchanged" +is "$(vg stats -E $SCRATCH/o_trim.vg)" "9" "seaming create one edge at the boundary" +is "$(vg paths -Fv $SCRATCH/o_trim.vg | grep -A1 '^>GRCh38#0#chr1\[0-18\]$' | tail -1)" "AAAACGGGGGTTAGGGGG" "the overlap of the reference path is trimmed" +is "$(vg paths -Fv $SCRATCH/o_trim.vg | grep -A1 '^>HG002#1#chr1#0\[0-18\]$' | tail -1)" "AAAACGGGGGTTCGGGGG" "the overlap of the HG002#1#chr1 is trimmed" +is "$(vg paths -Fv $SCRATCH/o_trim.vg | grep -A1 '^>HG002#2#chr1#0\[0-18\]$' | tail -1)" "AAAAGGGGGGTTAGGGGG" "the overlap of the HG002#2#chr1 is trimmed" # --seam trim --fuse: must rewrite the step of every path, replace the original nodes with the fused nodes. -vg combine -s trim -u $C/tl.pg $C/tr.pg > $C/o_fuse.vg 2>/dev/null -is "$(vg stats -N $C/o_fuse.vg)" "7" "the two boundary nodes become one" -is "$(vg stats -E $C/o_fuse.vg)" "8" "the edge between two boundary nodes is removed" -is "$(vg paths -Ev $C/o_fuse.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "welding does not change any path's coordinate" -is "$(vg paths -Ev $C/o_fuse.vg | cut -f2 | sort -u)" "18" "welding doesn't change any path's length" -is "$(vg paths -Fv $C/o_fuse.vg | sort)" "$(vg paths -Fv $C/o_trim.vg | sort)" "welding doesn't change any path's sequence" - -# --seam trim on two contigs interleaved: each must trim to its own span. -printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/mt_chr1l.gfa -printf "$RS"'S\t1\tCCGGG\nS\t2\tTTTTT\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t1+,2+\t*\n' > $C/mt_chr1r.gfa -printf "$RS"'S\t1\tTTTTT\nS\t2\tGGGGG\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[0-10]\t1+,2+\t*\n' > $C/mt_chr2l.gfa -printf "$RS"'S\t1\tGGAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr2[8-18]\t1+,2+\t*\n' > $C/mt_chr2r.gfa -for g in mt_chr1l mt_chr1r mt_chr2l mt_chr2r; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done - -vg combine -s trim $C/mt_chr2r.pg $C/mt_chr1l.pg $C/mt_chr2l.pg $C/mt_chr1r.pg > $C/o_mt.pg 2>/dev/null -is "$(vg paths -Ev $C/o_mt.pg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],GRCh38#0#chr2[0-18]" "each contig trims to its own span" -is "$(vg paths -Ev $C/o_mt.pg | cut -f2 | sort -u)" "18" "each contig lost only its own overlap" -is "$(vg stats -s $C/o_mt.pg | wc -l)" "2" "the contigs stay in separate components" -is "$(vg stats -E $C/o_mt.pg)" "6" "only the two intra-contig seams are edged" -is "$(vg paths -Fv $C/o_mt.pg | grep -A1 '^>GRCh38#0#chr1\[0-18\]$' | tail -1)" "AAAAACCCCCGGGTTTTT" "chr1 keeps its own sequence through the trim" -is "$(vg paths -Fv $C/o_mt.pg | grep -A1 '^>GRCh38#0#chr2\[0-18\]$' | tail -1)" "TTTTTGGGGGAAACCCCC" "chr2 keeps its own" - -# --fuse welds one seam per contig, not one overall. -vg combine -s trim -u $C/mt_chr2r.pg $C/mt_chr1l.pg $C/mt_chr2l.pg $C/mt_chr1r.pg > $C/o_mtu.pg 2>/dev/null -is "$(vg stats -N $C/o_mtu.pg)" "6" "each contig's boundary nodes weld into one" -is "$(vg stats -E $C/o_mtu.pg)" "4" "and each contig's seam edge goes with them" -is "$(vg paths -Fv $C/o_mtu.pg | sort)" "$(vg paths -Fv $C/o_mt.pg | sort)" "welding leaves both contigs spelling the same sequence" +vg combine -s trim -u $SCRATCH/tl.pg $SCRATCH/tr.pg > $SCRATCH/o_fuse.vg 2>/dev/null +is "$(vg stats -N $SCRATCH/o_fuse.vg)" "7" "the two boundary nodes become one" +is "$(vg stats -E $SCRATCH/o_fuse.vg)" "8" "the edge between two boundary nodes is removed" +is "$(vg paths -Ev $SCRATCH/o_fuse.vg | cut -f1 | sort | paste -sd,)" "GRCh38#0#chr1[0-18],HG002#1#chr1#0[0-18],HG002#2#chr1#0[0-18]" "fusing does not change any path's coordinate" +is "$(vg paths -Ev $SCRATCH/o_fuse.vg | cut -f2 | sort -u)" "18" "fusing doesn't change any path's length" +is "$(vg paths -Fv $SCRATCH/o_fuse.vg | sort)" "$(vg paths -Fv $SCRATCH/o_trim.vg | sort)" "fusing doesn't change any path's sequence" # Another single chunk. -printf "$RS"'S\t1\tAAAA\nS\t2\tCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-7]\t1+,2+\t*\n' > $C/one.gfa -vg convert -p $C/one.gfa > $C/one.pg 2>/dev/null +tr ' ' '\t' > $SCRATCH/one.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 1 AAAA +S 2 CCC +L 1 + 2 + 0M +P GRCh38#0#chr1[0-7] 1+,2+ * +EOF +vg convert -p $SCRATCH/one.gfa > $SCRATCH/one.pg 2>/dev/null # Error: nothing to combine. -vg combine -s renumber 2>$C/none.err >/dev/null +vg combine -s renumber 2>$SCRATCH/none.err >/dev/null is "$?" "1" "combining nothing is rejected" -is "$(grep -c 'At least one input graph' $C/none.err)" "1" "the error says at least one input graph is required" +is "$(grep -c 'At least one input graph' $SCRATCH/none.err)" "1" "the error says at least one input graph is required" # Error: --seam trim needs more than one chunk. -vg combine -s trim $C/one.pg 2>$C/lone.err >/dev/null +vg combine -s trim $SCRATCH/one.pg 2>$SCRATCH/lone.err >/dev/null is "$?" "1" "--seam trim on a single input is rejected" -is "$(grep -c 'at least two input graphs' $C/lone.err)" "1" "the error says trim needs at least two inputs" +is "$(grep -c 'at least two input graphs' $SCRATCH/lone.err)" "1" "the error says trim needs at least two inputs" -# Error: --fuse only makes sense when --seam trim leaves a boundary to weld. -vg combine -u $C/tl.pg $C/tr.pg 2>$C/fusearg.err >/dev/null +# Error: --fuse requires --seam trim. +vg combine -u $SCRATCH/tl.pg $SCRATCH/tr.pg 2>$SCRATCH/fusearg.err >/dev/null is "$?" "1" "--fuse without --seam trim is rejected" -is "$(grep -c 'requires --seam trim' $C/fusearg.err)" "1" "the error says --fuse needs --seam trim" +is "$(grep -c 'requires --seam trim' $SCRATCH/fusearg.err)" "1" "the error says --fuse needs --seam trim" # Error: --seam only takes the policies it knows. -vg combine -s bogus $C/tl.pg $C/tr.pg 2>$C/seam.err >/dev/null +vg combine -s bogus $SCRATCH/tl.pg $SCRATCH/tr.pg 2>$SCRATCH/seam.err >/dev/null is "$?" "1" "an unrecognized --seam policy is rejected" -is "$(grep -c 'renumber, shared, trim' $C/seam.err)" "1" "the error lists the policies that do exist" - -# Error: vg combine joins pieces from separate inputs, not pieces inside one input. -printf "$RS"'S\t1\tAAAAAAAAAA\nS\t2\tCCCCCCCCCC\nP\tGRCh38#0#chr1[0-10]\t1+\t*\nP\tGRCh38#0#chr1[10-20]\t2+\t*\n' > $C/split.gfa -# Same input, plus the edge that already joins the two pieces. -printf "$RS"'S\t1\tAAAAAAAAAA\nS\t2\tCCCCCCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+\t*\nP\tGRCh38#0#chr1[10-20]\t2+\t*\n' > $C/split_joined.gfa -vg convert -p $C/split.gfa > $C/split.pg 2>/dev/null -vg convert -p $C/split_joined.gfa > $C/split_joined.pg 2>/dev/null - -vg combine $C/split.pg 2>$C/split.err >/dev/null -is "$?" "1" "an input holding a path in pieces is rejected" -is "$(grep -c 'GRCh38#0#chr1\[0-10\].*GRCh38#0#chr1\[10-20\]' $C/split.err)" "1" "the error reports the pieces" -is "$(grep -c 'split.pg' $C/split.err)" "1" "the error reports the input holding them" - -vg combine $C/split_joined.pg 2>/dev/null >/dev/null -is "$?" "1" "and is rejected even when an edge already joins the pieces" +is "$(grep -c 'renumber, shared, trim' $SCRATCH/seam.err)" "1" "the error lists the policies that do exist" # Error: a read-only graph format cannot be combined; VPKG reports this itself. -vg gbwt -g $C/m.gbz --gbz-format -G graphs/gfa_with_reference.gfa 2>/dev/null -vg combine $C/m.gbz $C/one.pg 2>/dev/null >/dev/null +vg gbwt -g $SCRATCH/m.gbz --gbz-format -G graphs/gfa_with_reference.gfa 2>/dev/null +vg combine $SCRATCH/m.gbz $SCRATCH/one.pg 2>/dev/null >/dev/null is "$?" "1" "a GBZ input is rejected rather than crashing" # Error: --seam trim joins chunks end to end, so a gap between them is rejected. -printf "$RS"'S\t1\tAAAAAAAAAA\nP\tGRCh38#0#chr1[0-10]\t1+\t*\n' > $C/gapl.gfa -printf "$RS"'S\t2\tCCCCCCCCCC\nP\tGRCh38#0#chr1[20-30]\t2+\t*\n' > $C/gapr.gfa -vg convert -p $C/gapl.gfa > $C/gapl.pg 2>/dev/null -vg convert -p $C/gapr.gfa > $C/gapr.pg 2>/dev/null -vg combine -s trim $C/gapl.pg $C/gapr.pg 2>$C/tgap.err >/dev/null +# This also checks that an error from the combining code reaches the command line. +tr ' ' '\t' > $SCRATCH/gapl.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 1 AAAAAAAAAA +P GRCh38#0#chr1[0-10] 1+ * +EOF +tr ' ' '\t' > $SCRATCH/gapr.gfa <<'EOF' +H VN:Z:1.1 RS:Z:GRCh38 +S 2 CCCCCCCCCC +P GRCh38#0#chr1[20-30] 2+ * +EOF +vg convert -p $SCRATCH/gapl.gfa > $SCRATCH/gapl.pg 2>/dev/null +vg convert -p $SCRATCH/gapr.gfa > $SCRATCH/gapr.pg 2>/dev/null +vg combine -s trim $SCRATCH/gapl.pg $SCRATCH/gapr.pg 2>$SCRATCH/tgap.err >/dev/null is "$?" "1" "--seam trim rejects a gap between chunks" -is "$(grep -c 'REFERENCE offset gap of 10 bp' $C/tgap.err)" "1" "--seam trim also reports the size of the gap" - -# Error: the default policy refuses the same gap, caught by a different check. -vg combine $C/gapl.pg $C/gapr.pg 2>$C/gap.err >/dev/null -is "$?" "1" "the default policy rejects a gap between fragments too" -is "$(grep -c 'Gap of 10 bp' $C/gap.err)" "1" "the default policy reports the size of the gap" - -# Error: --seam trim needs at least one reference. -printf 'S\t1\tAAAA\nP\tplain_contig\t1+\t*\n' > $C/noref.gfa -vg convert -p $C/noref.gfa > $C/noref.pg 2>/dev/null -vg combine -s trim $C/noref.pg $C/one.pg 2>$C/noref.err >/dev/null -is "$?" "1" "--seam trim rejects a chunk with no REFERENCE path" -is "$(grep -c 'No REFERENCE-sense path found' $C/noref.err)" "1" "the error says a reference path is needed" - -# Error: --seam trim rejects more than one reference. -printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\nP\tGRCh38#0#chr2[0-4]\t1+\t*\n' > $C/tworef.gfa -vg convert -p $C/tworef.gfa > $C/tworef.pg 2>/dev/null -vg combine -s trim $C/tworef.pg $C/one.pg 2>$C/tworef.err >/dev/null -is "$?" "1" "--seam trim rejects a chunk with two REFERENCE paths" -is "$(grep -c '2 REFERENCE-sense paths found' $C/tworef.err)" "1" "the error counts the paths it found" - -# Error: --seam trim requires trimming on the forward strand of the boundary node. -printf "$RS"'S\t1\tAAAA\nS\t2\tCCC\nL\t1\t-\t2\t+\t0M\nP\tGRCh38#0#chr1[0-7]\t1-,2+\t*\n' > $C/rev.gfa -vg convert -p $C/rev.gfa > $C/rev.pg 2>/dev/null -vg combine -s trim $C/rev.pg $C/one.pg 2>$C/rev.err >/dev/null -is "$?" "1" "--seam trim rejects a reverse-strand boundary" -is "$(grep -c 'forward-strand boundaries' $C/rev.err)" "1" "the error says only forward boundaries work" - -# The left chunk shared by three --seam trim error cases below. -printf "$RS"'S\t1\tAAAAAAAAAA\nP\tGRCh38#0#chr1[0-10]\t1+\t*\n' > $C/bigl.gfa -vg convert -p $C/bigl.gfa > $C/bigl.pg 2>/dev/null - -# Error: --seam trim requires the left and right chunks to share the same sequence. -printf "$RS"'S\t2\tAACCC\nS\t3\tGGGGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[6-16]\t2+,3+\t*\n' > $C/mmr.gfa -vg convert -p $C/mmr.gfa > $C/mmr.pg 2>/dev/null -vg combine -s trim $C/bigl.pg $C/mmr.pg 2>$C/mm.err >/dev/null -is "$?" "1" "chunks that disagree on the overlapping sequence are rejected" -is "$(grep -c 'but disagree there' $C/mm.err)" "1" "the error says the overlap sequence does not match" -is "$(grep -c 'at offset 8' $C/mm.err)" "1" "the error reports the first position they differ at" - -# Error: --seam trim requires that all paths start from the reference path. -printf "$RS"'S\t5\tGGTT\nS\t6\tA\nS\t7\tC\nS\t8\tGGGGG\nL\t5\t+\t6\t+\t0M\nL\t5\t+\t7\t+\t0M\nL\t6\t+\t8\t+\t0M\nL\t7\t+\t8\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t5+,6+,8+\t*\nP\tHG002#1#chr1#0[8-18]\t5+,7+,8+\t*\nP\tHG002#2#chr1#0[12-18]\t6+,8+\t*\n' > $C/trb.gfa -printf "$RS"'S\t1\tAAAA\nS\t2\tC\nS\t3\tG\nS\t4\tGGGGG\nL\t1\t+\t2\t+\t0M\nL\t1\t+\t3\t+\t0M\nL\t2\t+\t4\t+\t0M\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+,4+\t*\nP\tHG002#1#chr1#0[0-10]\t1+,2+,4+\t*\nP\tHG002#2#chr1#0[0-5]\t1+,3+\t*\n' > $C/tlb.gfa -vg convert -p $C/trb.gfa > $C/trb.pg 2>/dev/null -vg convert -p $C/tlb.gfa > $C/tlb.pg 2>/dev/null - -vg combine -s trim $C/tl.pg $C/trb.pg >/dev/null 2>$C/start.err -is "$?" "1" "a path that does not start at the REFERENCE start node is rejected" -is "$(grep -c 'does not start at the REFERENCE start node (id 5)' $C/start.err)" "1" "the error reports the path and the REFERENCE start node it should have started on" - -# Error: --seam trim requires that all paths end at the reference path. -vg combine -s trim $C/tlb.pg $C/tr.pg >/dev/null 2>$C/end.err -is "$?" "1" "a path that does not end at the REFERENCE end node is rejected" -is "$(grep -c 'does not end at the REFERENCE end node (id 4)' $C/end.err)" "1" "the error reports the path and the REFERENCE end node it should have ended on" - -# Error: --seam trim does not allow the overlap to exceed the length of the first node of the right chunk. -printf "$RS"'S\t2\tAA\nS\t3\tAAAAGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[4-12]\t2+,3+\t*\n' > $C/bigr.gfa -vg convert -p $C/bigr.gfa > $C/bigr.pg 2>/dev/null -vg combine -s trim $C/bigl.pg $C/bigr.pg 2>$C/big.err >/dev/null -is "$?" "1" "an overlap longer than the first node is rejected" -is "$(grep -c 'exceeds the length of its first node' $C/big.err)" "1" "the error says the overlap does not fit" - -# Error: --seam trim does not allow the overlap to equal the length of the first node of the right chunk. -printf "$RS"'S\t2\tAAA\nS\t3\tGGGGGG\nL\t2\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[7-16]\t2+,3+\t*\n' > $C/eqr.gfa -vg convert -p $C/eqr.gfa > $C/eqr.pg 2>/dev/null -vg combine -s trim $C/bigl.pg $C/eqr.pg 2>$C/eq.err >/dev/null -is "$?" "1" "an overlap that would empty the first node is rejected" -is "$(grep -c 'would leave a zero-length' $C/eq.err)" "1" "the error says the node would be left empty" - -# Error: --seam trim needs a clean chunk boundary. -printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nL\t1\t+\t2\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/bl.gfa -printf "$RS"'S\t3\tCCGGG\nS\t4\tTTTTT\nL\t3\t+\t4\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t3+,4+\t*\n' > $C/br.gfa -# Same right chunk, plus an edge back into the node the trim want to remove. -printf "$RS"'S\t3\tCCGGG\nS\t4\tTTTTT\nL\t3\t+\t4\t+\t0M\nL\t4\t+\t3\t+\t0M\nP\tGRCh38#0#chr1[8-18]\t3+,4+\t*\n' > $C/br_back.gfa -# Same left chunk, plus a node hanging off the right of its boundary node. -printf "$RS"'S\t1\tAAAAA\nS\t2\tCCCCC\nS\t5\tAC\nL\t1\t+\t2\t+\t0M\nL\t2\t+\t5\t+\t0M\nP\tGRCh38#0#chr1[0-10]\t1+,2+\t*\n' > $C/bl_dangle.gfa -for g in bl br br_back bl_dangle; do vg convert -p $C/$g.gfa > $C/$g.pg 2>/dev/null; done - -vg combine -s trim $C/bl.pg $C/br.pg > $C/o_b.pg 2>/dev/null -is "$(vg paths -Ev $C/o_b.pg)" "$(printf 'GRCh38#0#chr1[0-18]\t18')" "the two chunks are joined into one path" -is "$(vg paths -Fv $C/o_b.pg | tail -1)" "AAAAACCCCCGGGTTTTT" "the overlapped CC is trimmed away and the two chunks are joined into one sequence" - -vg combine -s trim $C/bl.pg $C/br_back.pg >/dev/null 2>$C/back.err -is "$?" "1" "an edge into the node that would be trimmed is rejected" -is "$(grep -c 'would be lost' $C/back.err)" "1" "the error says the edge would be lost" - -# An edge off the left boundary is fine when seaming, but not for fusing. -vg combine -s trim $C/bl_dangle.pg $C/br.pg > $C/o_bd.pg 2>/dev/null -is "$(vg paths -Ev $C/o_bd.pg)" "$(printf 'GRCh38#0#chr1[0-18]\t18')" "an edge off the left boundary is fine for seaming" - -vg combine -s trim -u $C/bl_dangle.pg $C/br.pg >/dev/null 2>$C/fuse.err -is "$?" "1" "--fuse rejects a boundary node with an edge on the welded side" -is "$(grep -c 'weld would have to drop' $C/fuse.err)" "1" "the error says what the weld would drop" - -vg combine -s trim -u $C/bl.pg $C/br.pg > $C/o_bu.pg 2>/dev/null -is "$(vg stats -N $C/o_bu.pg)" "3" "a clean boundary is required for fusing into one node" - -# Error: --seam shared requires that all shared nodes have the same sequence. -printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\n' > $C/c1.gfa -printf "$RS"'S\t1\tCCCC\nP\tGRCh38#0#chr1[4-8]\t1+\t*\n' > $C/c2.gfa -vg convert -p $C/c1.gfa > $C/c1.pg 2>/dev/null -vg convert -p $C/c2.gfa > $C/c2.pg 2>/dev/null -vg combine -s shared $C/c1.pg $C/c2.pg 2>$C/conflict.err >/dev/null -is "$?" "1" "--seam shared rejects same ID carrying different sequences" -is "$(grep -c 'has different sequences' $C/conflict.err)" "1" "the error reports the node that disagrees" - -# Error: pieces of one path are told apart by name, so the same name cannot arrive twice. -printf "$RS"'S\t2\tGGGG\nP\tGRCh38#0#chr1[0-4]\t2+\t*\n' > $C/dup.gfa -vg convert -p $C/dup.gfa > $C/dup.pg 2>/dev/null -vg combine $C/c1.pg $C/dup.pg 2>$C/dup.err >/dev/null -is "$?" "1" "a path with the same name in both chunks is rejected" -is "$(grep -c 'in an earlier input' $C/dup.err)" "1" "the error reports the path that was duplicated" - -# Phase-block fragments: blocks 0 and 10 read as offsets, block 1 does not. -printf 'H\tVN:Z:1.0\nS\t1\tACGTA\nS\t2\tCCGGT\nP\tHG002#1#x#0\t1+,2+\t*\nL\t1\t+\t2\t+\t0M\n' > $C/pb1.gfa -printf 'H\tVN:Z:1.0\nS\t3\tTTACG\nS\t4\tGGCAT\nP\tHG002#1#x#10\t3+,4+\t*\nL\t3\t+\t4\t+\t0M\n' > $C/pb2.gfa -printf 'H\tVN:Z:1.0\nS\t3\tTTACG\nS\t4\tGGCAT\nP\tHG002#1#x#1\t3+,4+\t*\nL\t3\t+\t4\t+\t0M\n' > $C/pbnum.gfa -vg convert -p $C/pb1.gfa > $C/pb1.pg 2>/dev/null -vg convert -p $C/pb2.gfa > $C/pb2.pg 2>/dev/null -vg convert -p $C/pbnum.gfa > $C/pbnum.pg 2>/dev/null - -# By default a phase block is just a phase block, not an offset, so these fragments do not merge. -vg combine $C/pb1.pg $C/pb2.pg > $C/o_pb.vg 2>/dev/null -is "$(vg paths -Ev $C/o_pb.vg | wc -l)" "2" "phase blocks is not merged by default" -is "$(vg paths -Ev $C/o_pb.vg | cut -f1 | sort | paste -sd,)" "HG002#1#x#0,HG002#1#x#10" "the untouched fragments keep their names" - -# -P reads the phase block as a start offset instead. -vg combine -P $C/pb1.pg $C/pb2.pg > $C/o_pbP.vg 2>/dev/null -is "$(vg paths -Ev $C/o_pbP.vg | cut -f1)" "HG002#1#x#0[0-20]" "-P merges by phase block into one spanning path" -is "$(vg paths -Ev $C/o_pbP.vg | cut -f2)" "20" "the merged length matches the span the blocks imply" - -# The phase block as a start offset is not continuous. -vg combine -P $C/pb1.pg $C/pbnum.pg > $C/o_pbnum.vg 2>$C/pb.err -is "$?" "1" "-P reads the phase block as a start offset, but they are not continuous" -is "$(grep -c 'rather than offsets' $C/pb.err)" "1" "the error asks whether these are numbers, not offsets" - -# A phase block that is not 0 has to survive the merge intact. -printf "$RS"'S\t1\tACGTA\nS\t2\tCCGGT\nL\t1\t+\t2\t+\t0M\nP\tHG002#2#chr22#4[0-10]\t1+,2+\t*\n' > $C/hb1.gfa -printf "$RS"'S\t3\tTTACG\nS\t4\tGGCAT\nL\t3\t+\t4\t+\t0M\nP\tHG002#2#chr22#4[10-20]\t3+,4+\t*\n' > $C/hb2.gfa -vg convert -p $C/hb1.gfa > $C/hb1.pg 2>/dev/null -vg convert -p $C/hb2.gfa > $C/hb2.pg 2>/dev/null - -is "$(vg paths -M -v $C/hb1.pg | tail -1 | cut -f2,6)" "$(printf 'HAPLOTYPE\t4')" "the fixture really is a HAPLOTYPE path with phase block 4" -vg combine $C/hb1.pg $C/hb2.pg > $C/o_hb.pg 2>/dev/null -is "$(vg paths -Ev $C/o_hb.pg)" "$(printf 'HG002#2#chr22#4[0-20]\t20')" "merging keeps the phase block and spans both fragments" - -# Error: a merged name that is already taken is rejected. -printf "$RS"'S\t1\tAAAACCCC\nP\tGRCh38#0#chr1[0-8]\t1+\t*\n' > $C/whole.gfa -vg convert -v $C/whole.gfa > $C/whole_lin.vg 2>/dev/null -vg circularize -p 'GRCh38#0#chr1[0-8]' $C/whole_lin.vg > $C/whole.vg 2>/dev/null -printf "$RS"'S\t1\tAAAA\nP\tGRCh38#0#chr1[0-4]\t1+\t*\n' > $C/fa.gfa -printf "$RS"'S\t1\tCCCC\nP\tGRCh38#0#chr1[4-8]\t1+\t*\n' > $C/fb.gfa -vg convert -p $C/fa.gfa > $C/fa.pg 2>/dev/null -vg convert -p $C/fb.gfa > $C/fb.pg 2>/dev/null -vg combine $C/whole.vg $C/fa.pg $C/fb.pg 2>$C/collide.err >/dev/null -is "$?" "1" "a merged name that is already taken is rejected" -is "$(grep -c 'already an unrelated path' $C/collide.err)" "1" "the error names the path standing in the way" - -# vg combine has to work on graphs vg itself produces, not just hand-built GFA. -sed 's/^>x/>SAMPLE#0#x/' small/x.fa > $C/pansn.fa -vg construct -r $C/pansn.fa > $C/base.vg -vg index -x $C/base.xg $C/base.vg -is "$(vg paths -M -v $C/base.vg | tail -1 | cut -f2)" "REFERENCE" "PanSN naming yields a REFERENCE-sense path" +is "$(grep -c 'offset gap of 10 bp' $SCRATCH/tgap.err)" "1" "--seam trim also reports the size of the gap" + +# Two fragments of one haplotype whose phase blocks, 0 and 10, are their start offsets. +tr ' ' '\t' > $SCRATCH/pb1.gfa <<'EOF' +H VN:Z:1.0 +S 1 ACGTA +S 2 CCGGT +P HG002#1#x#0 1+,2+ * +L 1 + 2 + 0M +EOF +tr ' ' '\t' > $SCRATCH/pb2.gfa <<'EOF' +H VN:Z:1.0 +S 3 TTACG +S 4 GGCAT +P HG002#1#x#10 3+,4+ * +L 3 + 4 + 0M +EOF +vg convert -p $SCRATCH/pb1.gfa > $SCRATCH/pb1.pg 2>/dev/null +vg convert -p $SCRATCH/pb2.gfa > $SCRATCH/pb2.pg 2>/dev/null + +# -P reads the phase block as a start offset. +vg combine -P $SCRATCH/pb1.pg $SCRATCH/pb2.pg > $SCRATCH/o_pbP.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_pbP.vg | cut -f1)" "HG002#1#x#0[0-20]" "-P merges by phase block into one spanning path" +is "$(vg paths -Ev $SCRATCH/o_pbP.vg | cut -f2)" "20" "the merged length matches the span the blocks imply" + +# Combine chunks made by vg construct and vg chunk, not just hand-written GFA. +sed 's/^>x/>SAMPLE#0#x/' small/x.fa > $SCRATCH/pansn.fa +vg construct -r $SCRATCH/pansn.fa > $SCRATCH/base.vg +vg index -x $SCRATCH/base.xg $SCRATCH/base.vg +is "$(vg paths -M -v $SCRATCH/base.vg | tail -1 | cut -f2)" "REFERENCE" "PanSN naming yields a REFERENCE-sense path" # Chunking by node range gives two chunks that meet end to end and share no node IDs, # which is what the default policy expects. -vg chunk -x $C/base.xg -r 1:16 -c 0 > $C/adj1.vg -vg chunk -x $C/base.xg -r 17:32 -c 0 > $C/adj2.vg -vg combine $C/adj1.vg $C/adj2.vg > $C/o_real.vg 2>/dev/null -is "$(vg paths -Ev $C/o_real.vg)" "$(printf 'SAMPLE#0#x[0-1001]\t1001')" "two node-range chunks merge into one contig-spanning path" -is "$(vg stats -N $C/o_real.vg)" "32" "the merged graph keeps every input node" -is "$(vg stats -E $C/o_real.vg)" "31" "merging adds exactly one inter-fragment edge" - -# Whatever VPKG can load, combine can merge. -vg convert -p $C/adj1.vg > $C/adj1.pg -vg convert -a $C/adj2.vg > $C/adj2.hg -vg combine $C/adj1.pg $C/adj2.hg > $C/o_fmt.vg 2>/dev/null -is "$(vg paths -Ev $C/o_fmt.vg)" "$(vg paths -Ev $C/o_real.vg)" "PackedGraph and HashGraph inputs merge the same way" -is "$(vg stats -N $C/o_fmt.vg)" "32" "and keep the same nodes" - -rm -rf $C +vg chunk -x $SCRATCH/base.xg -r 1:16 -c 0 > $SCRATCH/adj1.vg +vg chunk -x $SCRATCH/base.xg -r 17:32 -c 0 > $SCRATCH/adj2.vg +vg combine $SCRATCH/adj1.vg $SCRATCH/adj2.vg > $SCRATCH/o_real.vg 2>/dev/null +is "$(vg paths -Ev $SCRATCH/o_real.vg)" "$(printf 'SAMPLE#0#x[0-1001]\t1001')" "two node-range chunks merge into one contig-spanning path" +is "$(vg stats -N $SCRATCH/o_real.vg)" "32" "the merged graph keeps every input node" +is "$(vg stats -E $SCRATCH/o_real.vg)" "31" "merging adds exactly one inter-fragment edge" + +rm -rf $SCRATCH