Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions artifacts/tee_alignment_emitter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* \file tee_alignment_emitter.cpp
* Implementation for TeeAlignmentEmitter
*/

#include "tee_alignment_emitter.hpp"

#include <stdexcept>

namespace vg {

TeeAlignmentEmitter::TeeAlignmentEmitter(std::unique_ptr<vg::io::AlignmentEmitter>&& primary,
std::unique_ptr<vg::io::AlignmentEmitter>&& copy) :
primary(std::move(primary)), copy(std::move(copy)) {
if (!this->primary || !this->copy) {
throw std::invalid_argument("TeeAlignmentEmitter needs two emitters");
}
}

void TeeAlignmentEmitter::emit_singles(std::vector<Alignment>&& aln_batch) {
primary->emit_singles(std::vector<Alignment>(aln_batch));
copy->emit_singles(std::move(aln_batch));
}

void TeeAlignmentEmitter::emit_mapped_singles(std::vector<std::vector<Alignment>>&& alns_batch) {
primary->emit_mapped_singles(std::vector<std::vector<Alignment>>(alns_batch));
copy->emit_mapped_singles(std::move(alns_batch));
}

void TeeAlignmentEmitter::emit_pairs(std::vector<Alignment>&& aln1_batch, std::vector<Alignment>&& aln2_batch,
std::vector<int64_t>&& tlen_limit_batch) {
primary->emit_pairs(std::vector<Alignment>(aln1_batch), std::vector<Alignment>(aln2_batch),
std::vector<int64_t>(tlen_limit_batch));
copy->emit_pairs(std::move(aln1_batch), std::move(aln2_batch), std::move(tlen_limit_batch));
}

void TeeAlignmentEmitter::emit_mapped_pairs(std::vector<std::vector<Alignment>>&& alns1_batch,
std::vector<std::vector<Alignment>>&& alns2_batch,
std::vector<int64_t>&& tlen_limit_batch) {
primary->emit_mapped_pairs(std::vector<std::vector<Alignment>>(alns1_batch),
std::vector<std::vector<Alignment>>(alns2_batch),
std::vector<int64_t>(tlen_limit_batch));
copy->emit_mapped_pairs(std::move(alns1_batch), std::move(alns2_batch), std::move(tlen_limit_batch));
}

void TeeAlignmentEmitter::emit_extra_message(const std::string& tag, std::string&& data) {
primary->emit_extra_message(tag, std::string(data));
copy->emit_extra_message(tag, std::move(data));
}

}
66 changes: 66 additions & 0 deletions artifacts/tee_alignment_emitter.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#ifndef VG_TEE_ALIGNMENT_EMITTER_HPP_INCLUDED
#define VG_TEE_ALIGNMENT_EMITTER_HPP_INCLUDED

/** \file
*
* Holds an AlignmentEmitter that writes every alignment to two outputs.
*/

#include "vg/io/alignment_emitter.hpp"

#include <memory>
#include <string>
#include <vector>

namespace vg {

/**
* A TeeAlignmentEmitter is an AlignmentEmitter that hands every batch of
* alignments to two emitters it owns: a primary one and a copy one.
*
* Both emitters receive the same alignments in the same batches, so each one
* writes what it would have written if it had been the only emitter. This lets
* one mapping run write, for example, surjected SAM/BAM/CRAM through the
* primary emitter and the graph alignments as GAM or GAF through the copy
* emitter, instead of mapping the reads twice.
*
* The primary emitter gets a copy of each batch, and the copy emitter gets the
* batch itself, since an emitter may move from or change what it is given.
* Like any AlignmentEmitter, a TeeAlignmentEmitter may be called from several
* threads at once, and relies on the two emitters it owns to be safe for that.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This sounds a lot like unrefined Anthropic Claude output:

  • It restates invariants of the base class only to tell us that of course they continue to apply.
  • It defines the class in terms of the user-facing feature it is going to enable instead of as an abstraction to be understood in isolation.
  • It doesn't put the cover on the abstraction, so the distinct concepts of "primary emitter" and "copy emitter" leak out to take up slots in the class user's brain when they don't need to.
  • It has a lot of unnecessarily specific qualifiers (we hand "every batch" of alignments to the other emitters, as if a class that decided not to handle some batches was a live possibility).
  • It's 3 paragraphs long when An AlignmentEmitter that emits alignments with two other AlignmentEmitters. would suffice.

Now, this is in artifacts/, where unrefined Anthropic Claude output indeed belongs. But it's such a simple job for the class that I don't know why we wouldn't just fix it up and promote it.

*/
class TeeAlignmentEmitter : public vg::io::AlignmentEmitter {
public:

/// Make an emitter that sends every batch to both of the given emitters,
/// and takes ownership of them. Neither may be null.
TeeAlignmentEmitter(std::unique_ptr<vg::io::AlignmentEmitter>&& primary,
std::unique_ptr<vg::io::AlignmentEmitter>&& copy);

/// Emit a batch of Alignments to both emitters.
void emit_singles(std::vector<Alignment>&& aln_batch) override;
/// Emit a batch of Alignments with secondaries to both emitters.
void emit_mapped_singles(std::vector<std::vector<Alignment>>&& alns_batch) override;
/// Emit a batch of pairs of Alignments, with the maximum pairing distance
/// of each pair, to both emitters.
void emit_pairs(std::vector<Alignment>&& aln1_batch, std::vector<Alignment>&& aln2_batch,
std::vector<int64_t>&& tlen_limit_batch) override;
/// Emit the mappings of a batch of pairs of Alignments, with the maximum
/// pairing distance of each pair, to both emitters.
void emit_mapped_pairs(std::vector<std::vector<Alignment>>&& alns1_batch,
std::vector<std::vector<Alignment>>&& alns2_batch,
std::vector<int64_t>&& tlen_limit_batch) override;
/// Emit some extra type-tagged data to both emitters, each of which keeps
/// it if its format supports it.
void emit_extra_message(const std::string& tag, std::string&& data) override;

private:
/// The emitter for the main output, which gets a copy of each batch.
std::unique_ptr<vg::io::AlignmentEmitter> primary;
/// The emitter for the second output, which gets each batch itself.
std::unique_ptr<vg::io::AlignmentEmitter> copy;
};

}

#endif
111 changes: 111 additions & 0 deletions artifacts/unittest/tee_alignment_emitter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/// \file tee_alignment_emitter.cpp
///
/// Unit tests for TeeAlignmentEmitter.

#include <memory>
#include <string>
#include <vector>

#include "../../src/unittest/catch.hpp"
#include "../tee_alignment_emitter.hpp"

namespace vg {
namespace unittest {

using namespace std;

namespace {

/// A log of what an emitter was given: one line per alignment, naming the
/// method that received it, and one line per extra message.
using EmitterLog = vector<string>;

/// An emitter that writes what it is given into a log it shares with the test,
/// and then clears the alignments it was given, as an emitter that moves from
/// its batches would leave them.
class LoggingEmitter : public vg::io::AlignmentEmitter {
public:
explicit LoggingEmitter(shared_ptr<EmitterLog> log) : log(std::move(log)) {}

void emit_singles(vector<Alignment>&& aln_batch) override {
for (auto& aln : aln_batch) {
log->push_back("single " + aln.name());
aln.Clear();
}
}
void emit_mapped_singles(vector<vector<Alignment>>&& alns_batch) override {
for (auto& alns : alns_batch) {
for (auto& aln : alns) {
log->push_back("mapped single " + aln.name());
aln.Clear();
}
}
}
void emit_pairs(vector<Alignment>&& aln1_batch, vector<Alignment>&& aln2_batch,
vector<int64_t>&& tlen_limit_batch) override {
for (size_t i = 0; i < aln1_batch.size(); i++) {
log->push_back("pair " + aln1_batch[i].name() + " " + aln2_batch[i].name() +
" " + to_string(tlen_limit_batch.at(i)));
aln1_batch[i].Clear();
aln2_batch[i].Clear();
}
}
void emit_mapped_pairs(vector<vector<Alignment>>&& alns1_batch, vector<vector<Alignment>>&& alns2_batch,
vector<int64_t>&& tlen_limit_batch) override {
for (size_t i = 0; i < alns1_batch.size(); i++) {
for (size_t j = 0; j < alns1_batch[i].size(); j++) {
log->push_back("mapped pair " + alns1_batch[i][j].name() + " " + alns2_batch[i][j].name() +
" " + to_string(tlen_limit_batch.at(i)));
alns1_batch[i][j].Clear();
alns2_batch[i][j].Clear();
}
}
}
void emit_extra_message(const string& tag, string&& data) override {
log->push_back("extra " + tag + " " + data);
data.clear();
}

private:
shared_ptr<EmitterLog> log;
};

/// Make an alignment with the given read name.
Alignment named(const string& name) {
Alignment aln;
aln.set_name(name);
return aln;
}

}

TEST_CASE("TeeAlignmentEmitter gives both emitters the same alignments", "[alignment][emitter]") {
auto primary_log = make_shared<EmitterLog>();
auto copy_log = make_shared<EmitterLog>();
TeeAlignmentEmitter tee(make_unique<LoggingEmitter>(primary_log), make_unique<LoggingEmitter>(copy_log));

tee.emit_extra_message("PARAMS_JSON", "{}");
tee.emit_singles({named("a"), named("b")});
tee.emit_mapped_singles({{named("c"), named("c2")}});
tee.emit_pairs({named("d1")}, {named("d2")}, {500});
tee.emit_mapped_pairs({{named("e1"), named("f1")}}, {{named("e2"), named("f2")}}, {600});

EmitterLog expected {
"extra PARAMS_JSON {}",
"single a", "single b",
"mapped single c", "mapped single c2",
"pair d1 d2 500",
"mapped pair e1 e2 600", "mapped pair f1 f2 600"
};
REQUIRE(*primary_log == expected);
REQUIRE(*copy_log == expected);
}

TEST_CASE("TeeAlignmentEmitter needs two emitters", "[alignment][emitter]") {
auto log = make_shared<EmitterLog>();
REQUIRE_THROWS_AS(TeeAlignmentEmitter(make_unique<LoggingEmitter>(log), nullptr), std::invalid_argument);
REQUIRE_THROWS_AS(TeeAlignmentEmitter(nullptr, make_unique<LoggingEmitter>(log)), std::invalid_argument);
}

}
}
64 changes: 58 additions & 6 deletions src/subcommand/giraffe_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <vg/io/vpkg.hpp>
#include <vg/io/stream.hpp>
#include "../hts_alignment_emitter.hpp"
#include "../../artifacts/tee_alignment_emitter.hpp"
#include "../minimizer_mapper.hpp"
#include "../index_registry.hpp"
#include "../utility.hpp"
Expand Down Expand Up @@ -738,7 +739,10 @@ void help_giraffe(char** argv, const BaseOptionGroup& parser, const std::map<std
<< " --ref-paths FILE ordered list of paths in the graph, one per line" << endl
<< " or HTSlib .dict, for HTSLib @SQ headers" << endl
<< " --ref-name NAME name of reference in the graph for HTSlib output" << endl
<< " --named-coordinates make GAM/GAF output in named-segment (GFA) space" << endl;
<< " --named-coordinates make GAM/GAF output in named-segment (GFA) space" << endl
<< " --graph-output FILE also write the graph alignments to FILE, as" << endl
<< " -o GAM/GAF would, before any surjection" << endl
<< " --graph-output-fmt NAME format for --graph-output {gam / gaf} [gam]" << endl;
if (full_help) {
cerr << " --add-graph-aln annotate linear formats with graph alignment" << endl
<< " --left-align attempt to left-align indels in linear formats" << endl
Expand Down Expand Up @@ -810,6 +814,8 @@ int main_giraffe(int argc, char** argv) {
constexpr int OPT_OFF_REF_POSITION = 1014;
constexpr int OPT_LEFT_ALIGN = 1015;
constexpr int OPT_NO_REC_MODE = 1016;
constexpr int OPT_GRAPH_OUTPUT = 1017;
constexpr int OPT_GRAPH_OUTPUT_FORMAT = 1018;
constexpr int OPT_HAPLOTYPE_NAME = 1100;

constexpr int OPT_KFF_NAME = 1101;
Expand Down Expand Up @@ -847,6 +853,9 @@ int main_giraffe(int argc, char** argv) {

string output_basename;
string report_name;
// File for a second copy of the graph alignments, and its format.
string graph_output_name;
string graph_output_format = "GAM";
bool show_progress = false;

// Main Giraffe program options struct
Expand Down Expand Up @@ -1185,6 +1194,8 @@ int main_giraffe(int argc, char** argv) {
{"off-ref-position", no_argument, 0, OPT_OFF_REF_POSITION},
{"left-align", no_argument, 0, OPT_LEFT_ALIGN},
{"named-coordinates", no_argument, 0, OPT_NAMED_COORDINATES},
{"graph-output", required_argument, 0, OPT_GRAPH_OUTPUT},
{"graph-output-fmt", required_argument, 0, OPT_GRAPH_OUTPUT_FORMAT},
{"discard", no_argument, 0, 'n'},
{"output-basename", required_argument, 0, OPT_OUTPUT_BASENAME},
{"report-name", required_argument, 0, OPT_REPORT_NAME},
Expand Down Expand Up @@ -1370,6 +1381,20 @@ int main_giraffe(int argc, char** argv) {
case OPT_NAMED_COORDINATES:
named_coordinates = true;
break;

case OPT_GRAPH_OUTPUT:
graph_output_name = ensure_writable(logger, optarg);
break;

case OPT_GRAPH_OUTPUT_FORMAT:
graph_output_format = optarg;
for (char& c : graph_output_format) {
c = std::toupper(c);
}
if (graph_output_format != "GAM" && graph_output_format != "GAF") {
logger.error() << "Invalid graph output format: " << graph_output_format << std::endl;
}
break;

case OPT_ADD_GRAPH_ALIGNMENT:
add_graph_alignment = true;
Expand Down Expand Up @@ -1571,6 +1596,15 @@ int main_giraffe(int argc, char** argv) {
logger.error() << "Using an output basename (--output-basename) "
<< "only makes sense for GAM format (-o GAM)." << std::endl;
}

if (!graph_output_name.empty() && !output_basename.empty()) {
logger.error() << "A graph output file (--graph-output) cannot be used "
<< "with an output basename (--output-basename)." << std::endl;
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why shouldn't this work? It makes sense in the one-setting-combination case.

if (!graph_output_name.empty() && discard_alignments) {
logger.error() << "A graph output file (--graph-output) cannot be used "
<< "when discarding alignments (-n)." << std::endl;
}

if (interleaved && !fastq_filename_2.empty()) {
logger.error() << "Cannot designate both interleaved paired ends (-i) "
Expand Down Expand Up @@ -2241,11 +2275,29 @@ int main_giraffe(int argc, char** argv) {
// Emit GAF header lines now, if applicable.
// TODO: We should have a function that can take any graph and any alignment emitter
// and determine if there is any header information it can emit.
io::GafAlignmentEmitter* gaf_emitter = dynamic_cast<io::GafAlignmentEmitter*>(alignment_emitter.get());
if (gaf_emitter != nullptr) {
gbwtgraph::GraphName graph_name = gbz->graph_name();
std::vector<std::string> header_lines = graph_name.gaf_header_lines();
gaf_emitter->emit_header_lines(header_lines);
auto emit_gaf_header = [&](vg::io::AlignmentEmitter* emitter) {
io::GafAlignmentEmitter* gaf_emitter = dynamic_cast<io::GafAlignmentEmitter*>(emitter);
if (gaf_emitter != nullptr) {
gbwtgraph::GraphName graph_name = gbz->graph_name();
std::vector<std::string> header_lines = graph_name.gaf_header_lines();
gaf_emitter->emit_header_lines(header_lines);
}
};
emit_gaf_header(alignment_emitter.get());

if (!graph_output_name.empty()) {
// Write a second copy of every alignment, before any
// surjection, to the file the user named. This emitter is
// the one `-o GAM` or `-o GAF` would make, so the file
// holds the same records a separate run with that output
// format would write.
unique_ptr<AlignmentEmitter> graph_emitter = get_alignment_emitter(
graph_output_name, graph_output_format,
SequenceDictionary(), thread_count, emitter_graph, flags
);
emit_gaf_header(graph_emitter.get());
alignment_emitter = make_unique<TeeAlignmentEmitter>(std::move(alignment_emitter),
std::move(graph_emitter));
}
}

Expand Down
20 changes: 19 additions & 1 deletion test/t/50_vg_giraffe.t
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ BASH_TAP_ROOT=../deps/bash-tap

PATH=../bin:$PATH # for vg

plan tests 90
plan tests 96

vg construct -a -r small/x.fa -v small/x.vcf.gz >x.vg
vg index -x x.xg x.vg
Expand Down Expand Up @@ -206,6 +206,24 @@ samtools view surject.bam | cut -f2 | sort | uniq -c > surject.flags
diff direct.flags surject.flags
is $? 0 "SAM flags are the same between giraffe and surject"

# One mapping run can write SAM/BAM/CRAM and also the graph alignments, as a
# separate run with -o GAM or -o GAF would write them. Every GAM record holds
# the time it took to map, which differs from run to run, so that is ignored.
GIRAFFE_PAIRED="vg giraffe x.fa x.vcf.gz -f small/x.fa_1.fastq -f small/x.fa_2.fastq --fragment-mean 300 --fragment-stdev 100 -t 1"
${GIRAFFE_PAIRED} -o SAM > plain.sam
${GIRAFFE_PAIRED} -o SAM --graph-output both.gam > both-gam.sam
${GIRAFFE_PAIRED} -o SAM --graph-output both.gaf --graph-output-fmt gaf > both-gaf.sam
${GIRAFFE_PAIRED} -o GAM > alone.gam
${GIRAFFE_PAIRED} -o GAF > alone.gaf
is "$(md5sum < both-gam.sam)" "$(md5sum < plain.sam)" "giraffe SAM output is unchanged by also writing GAM"
is "$(md5sum < both-gaf.sam)" "$(md5sum < plain.sam)" "giraffe SAM output is unchanged by also writing GAF"
is "$(vg view -aj both.gam | jq -c '.time_used = null' | md5sum)" "$(vg view -aj alone.gam | jq -c '.time_used = null' | md5sum)" "giraffe can write the GAM of a separate run beside SAM"
is "$(vg view --extract-tag PARAMS_JSON both.gam)" "$(vg view --extract-tag PARAMS_JSON alone.gam)" "GAM written beside SAM embeds the same parameters"
is "$(md5sum < both.gaf)" "$(md5sum < alone.gaf)" "giraffe can write the GAF of a separate run beside SAM"
${GIRAFFE_PAIRED} -o GAM --graph-output both.gam --output-basename x 2>/dev/null >/dev/null
is "$?" "1" "giraffe refuses a graph output file with an output basename"
rm -f plain.sam both-gam.sam both-gaf.sam both.gam both.gaf alone.gam alone.gaf

# Test paired surjected mapping
vg giraffe x.fa x.vcf.gz --show-work -iG <(vg view -a small/x-s13241-n1-p500-v300.gam | sed 's%_1%/1%' | sed 's%_2%/2%' | vg view -JaG - ) --output-format SAM >surjected.sam
is "$(cat surjected.sam | grep -v '^@' | sort -k4 | cut -f 4)" "$(printf '321\n762')" "surjection of paired reads to SAM yields correct positions"
Expand Down