<commit_before>/** \file call_main.cpp
 *
 * Defines the "vg call" subcommand, which calls variation from a graph and a pileup.
 */

#include <omp.h>
#include <unistd.h>
#include <getopt.h>

#include <list>
#include <fstream>

#include "subcommand.hpp"

#include "../vg.hpp"
#include "../caller.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_call(char** argv) {
    cerr << "usage: " << argv[0] << " call [options] <graph.vg> <pileup.vgpu> > output.vcf" << endl
         << "Output variant calls in VCF format given a graph and pileup" << endl
         << endl
         << "options:" << endl
         << "    -d, --min_depth INT        minimum depth of pileup [" << Caller::Default_min_depth <<"]" << endl
         << "    -e, --max_depth INT        maximum depth of pileup [" << Caller::Default_max_depth <<"]" << endl
         << "    -s, --min_support INT      minimum number of reads required to support snp [" << Caller::Default_min_support <<"]" << endl
         << "    -f, --min_frac FLOAT       minimum percentage of reads required to support snp[" << Caller::Default_min_frac <<"]" << endl
         << "    -q, --default_read_qual N  phred quality score to use if none found in the pileup ["
         << (int)Caller::Default_default_quality << "]" << endl
         << "    -b, --max_strand_bias FLOAT limit to absolute difference between 0.5 and proportion of supporting reads on reverse strand. [" << Caller::Default_max_strand_bias << "]" << endl
         << "    -a, --link-alts            add all possible edges between adjacent alts" << endl
         << "    -A, --aug-graph FILE       write out the agumented graph in vg format" << endl
         << "    -r, --ref PATH             use the given path name as the reference path" << endl
         << "    -c, --contig NAME          use the given name as the VCF contig name" << endl
         << "    -S, --sample NAME          name the sample in the VCF with the given name [SAMPLE]" << endl
         << "    -o, --offset INT           offset variant positions by this amount in VCF [0]" << endl
         << "    -l, --length INT           override total sequence length in VCF" << endl
         << "    -U, --subgraph             expect a subgraph and ignore extra pileup entries outside it" << endl
         << "    -P, --pileup               write pileup under VCF lines (for debugging, output not valid VCF)" << endl
         << "    -D, --depth INT            maximum depth for path search [default 10 nodes]" << endl
         << "    -F, --min_cov_frac FLOAT   min fraction of average coverage at which to call [0.0]" << endl
         << "    -H, --max_het_bias FLOAT   max imbalance factor between alts to call heterozygous [3]" << endl
         << "    -R, --max_ref_bias FLOAT   max imbalance factor between ref and alts to call heterozygous ref [4]" << endl
         << "    -M, --bias_mult FLOAT      multiplier for bias limits for indels as opposed to substitutions [1]" << endl
         << "    -n, --min_count INT        min total supporting read count to call a variant [1]" << endl
         << "    -B, --bin_size  INT        bin size used for counting coverage [250]" << endl
         << "    -C, --exp_coverage INT     specify expected coverage (instead of computing on reference)" << endl
         << "    -O, --no_overlap           don't emit new variants that overlap old ones" << endl
         << "    -u, --use_avg_support      use average instead of minimum support" << endl
         << "    -I, --singleallelic        disable support for multiallelic sites" << endl
         << "    -E, --min_mad              min. minimum allele depth required to PASS filter [5]" << endl
         << "    -h, --help                 print this help message" << endl
         << "    -p, --progress             show progress" << endl
         << "    -v, --verbose              print information and warnings about vcf generation" << endl
         << "    -t, --threads N            number of threads to use" << endl;
}

int main_call(int argc, char** argv) {

    if (argc <= 3) {
        help_call(argv);
        return 1;
    }

    double het_prior = Caller::Default_het_prior;
    int min_depth = Caller::Default_min_depth;
    int max_depth = Caller::Default_max_depth;
    int min_support = Caller::Default_min_support;
    double min_frac = Caller::Default_min_frac;
    int default_read_qual = Caller::Default_default_quality;
    double max_strand_bias = Caller::Default_max_strand_bias;
    string aug_file;
    bool bridge_alts = false;

    // Should we expect a subgraph and ignore pileups for missing nodes/edges?
    bool expectSubgraph = false;

    // Should we annotate the VCF with pileup info?
    bool pileupAnnotate = false;

    // This manages conversion from an augmented graph to a VCF, and makes the
    // actual calls.
    Call2Vcf call2vcf;

    bool show_progress = false;
    int thread_count = 1;

    int c;
    optind = 2; // force optind past command positional arguments
    while (true) {
        static struct option long_options[] =
            {
                {"min_depth", required_argument, 0, 'd'},
                {"max_depth", required_argument, 0, 'e'},
                {"min_support", required_argument, 0, 's'},
                {"min_frac", required_argument, 0, 'f'},
                {"default_read_qual", required_argument, 0, 'q'},
                {"max_strand_bias", required_argument, 0, 'b'},
                {"aug_graph", required_argument, 0, 'A'},
                {"link-alts", no_argument, 0, 'a'},
                {"progress", no_argument, 0, 'p'},
                {"verbose", no_argument, 0, 'v'},
                {"threads", required_argument, 0, 't'},
                {"ref", required_argument, 0, 'r'},
                {"contig", required_argument, 0, 'c'},
                {"sample", required_argument, 0, 'S'},
                {"offset", required_argument, 0, 'o'},
                {"depth", required_argument, 0, 'D'},
                {"length", required_argument, 0, 'l'},
                {"subgraph", no_argument, 0, 'U'},
                {"pileup", no_argument, 0, 'P'},
                {"min_cov_frac", required_argument, 0, 'F'},
                {"max_het_bias", required_argument, 0, 'H'},
                {"max_ref_bias", required_argument, 0, 'R'},
                {"bias_mult", required_argument, 0, 'M'},
                {"min_count", required_argument, 0, 'n'},
                {"bin_size", required_argument, 0, 'B'},
                {"avg_coverage", required_argument, 0, 'C'},
                {"no_overlap", no_argument, 0, 'O'},
                {"use_avg_support", no_argument, 0, 'u'},
                {"min_mad", required_argument, 0, 'E'},
                {"help", no_argument, 0, 'h'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "d:e:s:f:q:b:A:apvt:r:c:S:o:D:l:UPF:H:R:M:n:B:C:OuE:h",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'd':
            min_depth = atoi(optarg);
            break;
        case 'e':
            max_depth = atoi(optarg);
            break;
        case 's':
            min_support = atoi(optarg);
            break;
        case 'f':
            min_frac = atof(optarg);
            break;
        case 'q':
            default_read_qual = atoi(optarg);
            break;
        case 'b':
            max_strand_bias = atof(optarg);
            break;
        case 'A':
            aug_file = optarg;
            break;
        case 'a':
            bridge_alts = true;
            break;
        // old glenn2vcf opts start here
        case 'r':
            // Set the reference path name
            call2vcf.refPathName = optarg;
            break;
        case 'c':
            // Set the contig name
            call2vcf.contigName = optarg;
            break;
        case 'S':
            // Set the sample name
            call2vcf.sampleName = optarg;
            break;
        case 'o':
            // Offset variants
            call2vcf.variantOffset = std::stoll(optarg);
            break;
        case 'D':
            // Limit max depth for pathing to primary path
            call2vcf.maxDepth = std::stoll(optarg);
            break;
        case 'l':
            // Set a length override
            call2vcf.lengthOverride = std::stoll(optarg);
            break;
        case 'U':
            expectSubgraph = true;
            break;
        case 'P':
            pileupAnnotate = true;
            break;
        case 'F':
            // Set min fraction of average coverage for a call
            call2vcf.minFractionForCall = std::stod(optarg);
            break;
        case 'H':
            // Set max factor between reads on one alt and reads on the other
            // alt for calling a het.
            call2vcf.maxHetBias = std::stod(optarg);
            break;
        case 'R':
            // Set max factor between reads on ref and reads on the other
            // alt for calling a homo ref.
            call2vcf.maxRefHetBias = std::stod(optarg);
            break;
        case 'M':
            // Set multiplier for bias limits for indels
            call2vcf.indelBiasMultiple = std::stod(optarg);
            break;
        case 'n':
            // How many reads need to touch an allele before we are willing to
            // call it?
            call2vcf.minTotalSupportForCall = std::stoll(optarg);
            break;
        case 'B':
            // Set the reference bin size
            call2vcf.refBinSize = std::stoll(optarg);
            break;
        case 'C':
            // Override expected coverage
            call2vcf.expCoverage = std::stoll(optarg);
            break;
        case 'O':
            // Suppress variants that overlap others
            call2vcf.suppress_overlaps = true;
            break;
        case 'u':
            // Average (isntead of min) support
            call2vcf.useAverageSupport = true;
            break;
        case 'E':
            // Minimum min-allele-depth required to give Filter column a PASS
            call2vcf.min_mad_for_filter = std::stoi(optarg);
            break;
        case 'p':
            show_progress = true;
            break;
        case 'v':
            call2vcf.verbose = true;
            break;
        case 't':
            thread_count = atoi(optarg);
            break;
        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_call(argv);
            exit(1);
            break;
        default:
          abort ();
        }
    }
    omp_set_num_threads(thread_count);
    thread_count = get_thread_count();

    // Parse the arguments
    if (optind >= argc) {
        help_call(argv);
        return 1;
    }
    string graph_file_name = get_input_file_name(optind, argc, argv);
    if (optind >= argc) {
        help_call(argv);
        return 1;
    }
    string pileup_file_name = get_input_file_name(optind, argc, argv);

    if (pileup_file_name == "-" && graph_file_name == "-") {
        cerr << "error: graph and pileup can't both be from stdin." << endl;
        exit(1);
    }

    // read the graph
    if (show_progress) {
        cerr << "Reading input graph" << endl;
    }
    VG* graph;
    get_input_file(graph_file_name, [&](istream& in) {
        graph = new VG(in);
    });

    if (show_progress) {
        cerr << "Computing augmented graph" << endl;
    }
    Caller caller(graph,
                  het_prior, min_depth, max_depth, min_support,
                  min_frac, Caller::Default_min_log_likelihood,
                  default_read_qual, max_strand_bias, bridge_alts);

    // setup pileup stream
    get_input_file(pileup_file_name, [&](istream& pileup_stream) {
        // compute the augmented graph
        function<void(Pileup&)> lambda = [&](Pileup& pileup) {
            for (int i = 0; i < pileup.node_pileups_size(); ++i) {
                if (!graph->has_node(pileup.node_pileups(i).node_id())) {
                    // This pileup doesn't belong in this graph
                    if(!expectSubgraph) {
                        throw runtime_error("Found pileup for nonexistent node " + to_string(pileup.node_pileups(i).node_id()));
                    }
                    // If that's expected, just skip it
                    continue;
                }
                // Send approved pileups to the caller
                caller.call_node_pileup(pileup.node_pileups(i));
            }
            for (int i = 0; i < pileup.edge_pileups_size(); ++i) {
                if (!graph->has_edge(pileup.edge_pileups(i).edge())) {
                    // This pileup doesn't belong in this graph
                    if(!expectSubgraph) {
                        throw runtime_error("Found pileup for nonexistent edge " + pb2json(pileup.edge_pileups(i).edge()));
                    }
                    // If that's expected, just skip it
                    continue;
                }
                // Send approved pileups to the caller
                caller.call_edge_pileup(pileup.edge_pileups(i));
            }
        };
        stream::for_each(pileup_stream, lambda);
    });

    // map the edges from original graph
    if (show_progress) {
        cerr << "Mapping edges into augmented graph" << endl;
    }
    caller.update_augmented_graph();

    // map the paths from the original graph
    if (show_progress) {
        cerr << "Mapping paths into augmented graph" << endl;
    }
    caller.map_paths();

    if (!aug_file.empty()) {
        // write the augmented graph
        if (show_progress) {
            cerr << "Writing augmented graph" << endl;
        }
        ofstream aug_stream(aug_file.c_str());
        caller.write_augmented_graph(aug_stream, false);
    }

    if (show_progress) {
        cerr << "Calling variants" << endl;
    }

    // project the augmented graph to a reference path
    // in order to create a VCF of calls.
    call2vcf.call(caller._augmented_graph,
        pileupAnnotate ? pileup_file_name : string());

    return 0;
}

// Register subcommand
static Subcommand vg_call("call", "call variants on a graph from a pileup", main_call);

<commit_msg>Change underscore options to dashes like the other subcommands use<commit_after>/** \file call_main.cpp
 *
 * Defines the "vg call" subcommand, which calls variation from a graph and a pileup.
 */

#include <omp.h>
#include <unistd.h>
#include <getopt.h>

#include <list>
#include <fstream>

#include "subcommand.hpp"

#include "../vg.hpp"
#include "../caller.hpp"

using namespace std;
using namespace vg;
using namespace vg::subcommand;

void help_call(char** argv) {
    cerr << "usage: " << argv[0] << " call [options] <graph.vg> <pileup.vgpu> > output.vcf" << endl
         << "Output variant calls in VCF format given a graph and pileup" << endl
         << endl
         << "options:" << endl
         << "    -d, --min-depth INT        minimum depth of pileup [" << Caller::Default_min_depth <<"]" << endl
         << "    -e, --max-depth INT        maximum depth of pileup [" << Caller::Default_max_depth <<"]" << endl
         << "    -s, --min-support INT      minimum number of reads required to support snp [" << Caller::Default_min_support <<"]" << endl
         << "    -f, --min-frac FLOAT       minimum percentage of reads required to support snp[" << Caller::Default_min_frac <<"]" << endl
         << "    -q, --default-read-qual N  phred quality score to use if none found in the pileup ["
         << (int)Caller::Default_default_quality << "]" << endl
         << "    -b, --max-strand-bias FLOAT limit to absolute difference between 0.5 and proportion of supporting reads on reverse strand. [" << Caller::Default_max_strand_bias << "]" << endl
         << "    -a, --link-alts            add all possible edges between adjacent alts" << endl
         << "    -A, --aug-graph FILE       write out the agumented graph in vg format" << endl
         << "    -r, --ref PATH             use the given path name as the reference path" << endl
         << "    -c, --contig NAME          use the given name as the VCF contig name" << endl
         << "    -S, --sample NAME          name the sample in the VCF with the given name [SAMPLE]" << endl
         << "    -o, --offset INT           offset variant positions by this amount in VCF [0]" << endl
         << "    -l, --length INT           override total sequence length in VCF" << endl
         << "    -U, --subgraph             expect a subgraph and ignore extra pileup entries outside it" << endl
         << "    -P, --pileup               write pileup under VCF lines (for debugging, output not valid VCF)" << endl
         << "    -D, --depth INT            maximum depth for path search [default 10 nodes]" << endl
         << "    -F, --min-cov-frac FLOAT   min fraction of average coverage at which to call [0.0]" << endl
         << "    -H, --max-het-bias FLOAT   max imbalance factor between alts to call heterozygous [3]" << endl
         << "    -R, --max-ref-bias FLOAT   max imbalance factor between ref and alts to call heterozygous ref [4]" << endl
         << "    -M, --bias-mult FLOAT      multiplier for bias limits for indels as opposed to substitutions [1]" << endl
         << "    -n, --min-count INT        min total supporting read count to call a variant [1]" << endl
         << "    -B, --bin-size  INT        bin size used for counting coverage [250]" << endl
         << "    -C, --exp-coverage INT     specify expected coverage (instead of computing on reference)" << endl
         << "    -O, --no-overlap           don't emit new variants that overlap old ones" << endl
         << "    -u, --use-avg-support      use average instead of minimum support" << endl
         << "    -I, --singleallelic        disable support for multiallelic sites" << endl
         << "    -E, --min_mad              min. minimum allele depth required to PASS filter [5]" << endl
         << "    -h, --help                 print this help message" << endl
         << "    -p, --progress             show progress" << endl
         << "    -v, --verbose              print information and warnings about vcf generation" << endl
         << "    -t, --threads N            number of threads to use" << endl;
}

int main_call(int argc, char** argv) {

    if (argc <= 3) {
        help_call(argv);
        return 1;
    }

    double het_prior = Caller::Default_het_prior;
    int min_depth = Caller::Default_min_depth;
    int max_depth = Caller::Default_max_depth;
    int min_support = Caller::Default_min_support;
    double min_frac = Caller::Default_min_frac;
    int default_read_qual = Caller::Default_default_quality;
    double max_strand_bias = Caller::Default_max_strand_bias;
    string aug_file;
    bool bridge_alts = false;

    // Should we expect a subgraph and ignore pileups for missing nodes/edges?
    bool expectSubgraph = false;

    // Should we annotate the VCF with pileup info?
    bool pileupAnnotate = false;

    // This manages conversion from an augmented graph to a VCF, and makes the
    // actual calls.
    Call2Vcf call2vcf;

    bool show_progress = false;
    int thread_count = 1;

    int c;
    optind = 2; // force optind past command positional arguments
    while (true) {
        static struct option long_options[] =
            {
                {"min-depth", required_argument, 0, 'd'},
                {"max-depth", required_argument, 0, 'e'},
                {"min-support", required_argument, 0, 's'},
                {"min-frac", required_argument, 0, 'f'},
                {"default-read-qual", required_argument, 0, 'q'},
                {"max-strand-bias", required_argument, 0, 'b'},
                {"aug-graph", required_argument, 0, 'A'},
                {"link-alts", no_argument, 0, 'a'},
                {"progress", no_argument, 0, 'p'},
                {"verbose", no_argument, 0, 'v'},
                {"threads", required_argument, 0, 't'},
                {"ref", required_argument, 0, 'r'},
                {"contig", required_argument, 0, 'c'},
                {"sample", required_argument, 0, 'S'},
                {"offset", required_argument, 0, 'o'},
                {"depth", required_argument, 0, 'D'},
                {"length", required_argument, 0, 'l'},
                {"subgraph", no_argument, 0, 'U'},
                {"pileup", no_argument, 0, 'P'},
                {"min-cov-frac", required_argument, 0, 'F'},
                {"max-het-bias", required_argument, 0, 'H'},
                {"max-ref-bias", required_argument, 0, 'R'},
                {"bias-mult", required_argument, 0, 'M'},
                {"min-count", required_argument, 0, 'n'},
                {"bin-size", required_argument, 0, 'B'},
                {"avg-coverage", required_argument, 0, 'C'},
                {"no-overlap", no_argument, 0, 'O'},
                {"use-avg-support", no_argument, 0, 'u'},
                {"min-mad", required_argument, 0, 'E'},
                {"help", no_argument, 0, 'h'},
                {0, 0, 0, 0}
            };

        int option_index = 0;
        c = getopt_long (argc, argv, "d:e:s:f:q:b:A:apvt:r:c:S:o:D:l:UPF:H:R:M:n:B:C:OuE:h",
                         long_options, &option_index);

        /* Detect the end of the options. */
        if (c == -1)
            break;

        switch (c)
        {
        case 'd':
            min_depth = atoi(optarg);
            break;
        case 'e':
            max_depth = atoi(optarg);
            break;
        case 's':
            min_support = atoi(optarg);
            break;
        case 'f':
            min_frac = atof(optarg);
            break;
        case 'q':
            default_read_qual = atoi(optarg);
            break;
        case 'b':
            max_strand_bias = atof(optarg);
            break;
        case 'A':
            aug_file = optarg;
            break;
        case 'a':
            bridge_alts = true;
            break;
        // old glenn2vcf opts start here
        case 'r':
            // Set the reference path name
            call2vcf.refPathName = optarg;
            break;
        case 'c':
            // Set the contig name
            call2vcf.contigName = optarg;
            break;
        case 'S':
            // Set the sample name
            call2vcf.sampleName = optarg;
            break;
        case 'o':
            // Offset variants
            call2vcf.variantOffset = std::stoll(optarg);
            break;
        case 'D':
            // Limit max depth for pathing to primary path
            call2vcf.maxDepth = std::stoll(optarg);
            break;
        case 'l':
            // Set a length override
            call2vcf.lengthOverride = std::stoll(optarg);
            break;
        case 'U':
            expectSubgraph = true;
            break;
        case 'P':
            pileupAnnotate = true;
            break;
        case 'F':
            // Set min fraction of average coverage for a call
            call2vcf.minFractionForCall = std::stod(optarg);
            break;
        case 'H':
            // Set max factor between reads on one alt and reads on the other
            // alt for calling a het.
            call2vcf.maxHetBias = std::stod(optarg);
            break;
        case 'R':
            // Set max factor between reads on ref and reads on the other
            // alt for calling a homo ref.
            call2vcf.maxRefHetBias = std::stod(optarg);
            break;
        case 'M':
            // Set multiplier for bias limits for indels
            call2vcf.indelBiasMultiple = std::stod(optarg);
            break;
        case 'n':
            // How many reads need to touch an allele before we are willing to
            // call it?
            call2vcf.minTotalSupportForCall = std::stoll(optarg);
            break;
        case 'B':
            // Set the reference bin size
            call2vcf.refBinSize = std::stoll(optarg);
            break;
        case 'C':
            // Override expected coverage
            call2vcf.expCoverage = std::stoll(optarg);
            break;
        case 'O':
            // Suppress variants that overlap others
            call2vcf.suppress_overlaps = true;
            break;
        case 'u':
            // Average (isntead of min) support
            call2vcf.useAverageSupport = true;
            break;
        case 'E':
            // Minimum min-allele-depth required to give Filter column a PASS
            call2vcf.min_mad_for_filter = std::stoi(optarg);
            break;
        case 'p':
            show_progress = true;
            break;
        case 'v':
            call2vcf.verbose = true;
            break;
        case 't':
            thread_count = atoi(optarg);
            break;
        case 'h':
        case '?':
            /* getopt_long already printed an error message. */
            help_call(argv);
            exit(1);
            break;
        default:
          abort ();
        }
    }
    omp_set_num_threads(thread_count);
    thread_count = get_thread_count();

    // Parse the arguments
    if (optind >= argc) {
        help_call(argv);
        return 1;
    }
    string graph_file_name = get_input_file_name(optind, argc, argv);
    if (optind >= argc) {
        help_call(argv);
        return 1;
    }
    string pileup_file_name = get_input_file_name(optind, argc, argv);

    if (pileup_file_name == "-" && graph_file_name == "-") {
        cerr << "error: graph and pileup can't both be from stdin." << endl;
        exit(1);
    }

    // read the graph
    if (show_progress) {
        cerr << "Reading input graph" << endl;
    }
    VG* graph;
    get_input_file(graph_file_name, [&](istream& in) {
        graph = new VG(in);
    });

    if (show_progress) {
        cerr << "Computing augmented graph" << endl;
    }
    Caller caller(graph,
                  het_prior, min_depth, max_depth, min_support,
                  min_frac, Caller::Default_min_log_likelihood,
                  default_read_qual, max_strand_bias, bridge_alts);

    // setup pileup stream
    get_input_file(pileup_file_name, [&](istream& pileup_stream) {
        // compute the augmented graph
        function<void(Pileup&)> lambda = [&](Pileup& pileup) {
            for (int i = 0; i < pileup.node_pileups_size(); ++i) {
                if (!graph->has_node(pileup.node_pileups(i).node_id())) {
                    // This pileup doesn't belong in this graph
                    if(!expectSubgraph) {
                        throw runtime_error("Found pileup for nonexistent node " + to_string(pileup.node_pileups(i).node_id()));
                    }
                    // If that's expected, just skip it
                    continue;
                }
                // Send approved pileups to the caller
                caller.call_node_pileup(pileup.node_pileups(i));
            }
            for (int i = 0; i < pileup.edge_pileups_size(); ++i) {
                if (!graph->has_edge(pileup.edge_pileups(i).edge())) {
                    // This pileup doesn't belong in this graph
                    if(!expectSubgraph) {
                        throw runtime_error("Found pileup for nonexistent edge " + pb2json(pileup.edge_pileups(i).edge()));
                    }
                    // If that's expected, just skip it
                    continue;
                }
                // Send approved pileups to the caller
                caller.call_edge_pileup(pileup.edge_pileups(i));
            }
        };
        stream::for_each(pileup_stream, lambda);
    });

    // map the edges from original graph
    if (show_progress) {
        cerr << "Mapping edges into augmented graph" << endl;
    }
    caller.update_augmented_graph();

    // map the paths from the original graph
    if (show_progress) {
        cerr << "Mapping paths into augmented graph" << endl;
    }
    caller.map_paths();

    if (!aug_file.empty()) {
        // write the augmented graph
        if (show_progress) {
            cerr << "Writing augmented graph" << endl;
        }
        ofstream aug_stream(aug_file.c_str());
        caller.write_augmented_graph(aug_stream, false);
    }

    if (show_progress) {
        cerr << "Calling variants" << endl;
    }

    // project the augmented graph to a reference path
    // in order to create a VCF of calls.
    call2vcf.call(caller._augmented_graph,
        pileupAnnotate ? pileup_file_name : string());

    return 0;
}

// Register subcommand
static Subcommand vg_call("call", "call variants on a graph from a pileup", main_call);

<|endoftext|>


<commit_before>#ifndef ROCKET_SYSTEM_HPP_
#define ROCKET_SYSTEM_HPP_

#include "system/vehicle_system.hpp"
#include "util/optional.hpp"

// Communication
#include "communication/communicator.hpp"
#include "communication/message_listener.hpp"

// Control
#include "controller/position_controller.hpp"
#include "controller/controller_pipeline.hpp"
#include "controller/setpoint_types.hpp"
#include "controller/zero_controller.hpp"
#include "input/input_source.hpp"
#include "motor/motor_mapper.hpp"
#include "motor/pwm_device_group.hpp"

// Filesystem
#include "filesystem/logger.hpp"

// Platform
#include "variant/digital_platform.hpp"
#include "variant/pwm_platform.hpp"
#include "variant/platform.hpp"

// World estimation
#include "estimator/world_estimator.hpp"

// Sensors
#include "sensor/sensor_measurements.hpp"

enum class RocketState {
  DISARMED,
  PRE_ARM,
  ARMED,
  FLIGHT,
  APOGEE,
  MICROGRAVITY,   // Dummy state to keep state numbers the same as payload. Firmware doesn't care, but this makes the current ground station simpler. It all needs to be rethought anyway.
  DESCENT,
  RECOVERY
};

class RocketSystem : public VehicleSystem, public MessageListener {
public:
  RocketSystem(
      Accelerometer& accel,
      optional<Accelerometer *> accelH,
      optional<Barometer *> bar,
      optional<Geiger *> ggr,
      optional<GPS *> gps,
      Gyroscope& gyr,
      optional<Magnetometer *> mag,
      WorldEstimator& estimator, InputSource& inputSource,
      MotorMapper& motorMapper, Communicator& communicator, Logger& logger,
      Platform& platform);

  void update() override;
  bool healthy();

  void on(const protocol::message::set_arm_state_message_t& m) override;

private:
  Accelerometer& accel;
  optional<Accelerometer *> accelH;
  optional<Barometer *> bar;
  optional<Geiger *> ggr;
  optional<GPS *> gps;
  Gyroscope& gyr;
  optional<Magnetometer *> mag;

  WorldEstimator& estimator;
  InputSource& inputSource;

  PositionController posController;
  ControllerPipeline<ActuatorSetpoint> pipeline;

  ZeroController<ActuatorSetpoint> zeroController;

  MotorMapper& motorMapper;
  Platform& platform;

  RateLimitedStream systemStream;
  Logger& logger;

  void updateStreams(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Pin config
   */
  // ADC
  const uint8_t PIN_EXT_TEMP_THERM_CH = 0;   // PC0
  // Digital
  const uint8_t PIN_DROGUE_CH = 2;   // PC4
  const uint8_t PIN_MAIN_CH   = 3;   // PC5

  /**
   * For now, we proceed directly to PRE_ARM.
   */
  RocketState DisarmedState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Full data stream to ground station begins here.
   * Sensor calibration should be performed, hopefully with physical access to
   * the board.
   *
   * Proceed to ARMED on meeting all of the following conditions:
   *
   *   1. Sensor health checks passed
   *   2. GPS lock
   *   3. Software arm signal received from GS
   */
  RocketState PreArmState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Sensor calibration should be finished.
   *
   * Proceed to FLIGHT if X accel exceeds 1.1g.
   */
  RocketState ArmedState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Begin onboard data logging.
   * NOTE(yoos): Unfortunately, we do not have onboard logging yet.
   *
   * Detect apogee based mainly on four measurements:
   *
   *   1. Pressure rate of change
   *   2. Net acceleration magnitude
   *   3. Non-roll rotation rate magnitude
   *   4. Orientation
   *
   * We first determine possible apogee based on the altitude rate of change,
   * which is estimated from the barometer and GPS. (Currently, only the
   * barometer is used for altitude.)
   *
   * We check for the motor cutoff acceleration drop to negative. If we sense
   * we are falling faster than -40 m/s, we transition to APOGEE. Otherwise, we
   * check for the ideal case of zero altitude change and that we are not
   * observing a false apogee during a subsonic transition.
   *
   * We do not track powered and coasting portions of flight in separate states
   * because a mach transition may happen at an unknowable time, and it's
   * probably easier to track state variables this way.
   */
  RocketState FlightState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Deploy drogue chute. If magnitude of net proper acceleration does not
   * change within 5 seconds, deploy main.
   */
  RocketState ApogeeState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Deploy main chute at 1500' AGL.
   */
  RocketState DescentState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Turn off all telemetry (maybe?) except GPS.
   *
   * Try to conserve power.
   */
  RocketState RecoveryState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * RGB LED stuff.
   */
  void SetLED(float r, float g, float b);
  void BlinkLED(float r, float g, float b, float freq);
  void PulseLED(float r, float g, float b, float freq);
  void RGBLED(float freq);

  /**
   * In-class global vars
   */
  RocketState state;
  float motorDC;

  /**
   * Per-launch calibration
   */
  float groundAltitude;
  float velocity;
};

#endif
<commit_msg>Comment out drogue pin definition in rocket.<commit_after>#ifndef ROCKET_SYSTEM_HPP_
#define ROCKET_SYSTEM_HPP_

#include "system/vehicle_system.hpp"
#include "util/optional.hpp"

// Communication
#include "communication/communicator.hpp"
#include "communication/message_listener.hpp"

// Control
#include "controller/position_controller.hpp"
#include "controller/controller_pipeline.hpp"
#include "controller/setpoint_types.hpp"
#include "controller/zero_controller.hpp"
#include "input/input_source.hpp"
#include "motor/motor_mapper.hpp"
#include "motor/pwm_device_group.hpp"

// Filesystem
#include "filesystem/logger.hpp"

// Platform
#include "variant/digital_platform.hpp"
#include "variant/pwm_platform.hpp"
#include "variant/platform.hpp"

// World estimation
#include "estimator/world_estimator.hpp"

// Sensors
#include "sensor/sensor_measurements.hpp"

enum class RocketState {
  DISARMED,
  PRE_ARM,
  ARMED,
  FLIGHT,
  APOGEE,
  MICROGRAVITY,   // Dummy state to keep state numbers the same as payload. Firmware doesn't care, but this makes the current ground station simpler. It all needs to be rethought anyway.
  DESCENT,
  RECOVERY
};

class RocketSystem : public VehicleSystem, public MessageListener {
public:
  RocketSystem(
      Accelerometer& accel,
      optional<Accelerometer *> accelH,
      optional<Barometer *> bar,
      optional<Geiger *> ggr,
      optional<GPS *> gps,
      Gyroscope& gyr,
      optional<Magnetometer *> mag,
      WorldEstimator& estimator, InputSource& inputSource,
      MotorMapper& motorMapper, Communicator& communicator, Logger& logger,
      Platform& platform);

  void update() override;
  bool healthy();

  void on(const protocol::message::set_arm_state_message_t& m) override;

private:
  Accelerometer& accel;
  optional<Accelerometer *> accelH;
  optional<Barometer *> bar;
  optional<Geiger *> ggr;
  optional<GPS *> gps;
  Gyroscope& gyr;
  optional<Magnetometer *> mag;

  WorldEstimator& estimator;
  InputSource& inputSource;

  PositionController posController;
  ControllerPipeline<ActuatorSetpoint> pipeline;

  ZeroController<ActuatorSetpoint> zeroController;

  MotorMapper& motorMapper;
  Platform& platform;

  RateLimitedStream systemStream;
  Logger& logger;

  void updateStreams(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Pin config
   */
  // ADC
  const uint8_t PIN_EXT_TEMP_THERM_CH = 0;   // PC0
  // Digital
  //const uint8_t PIN_DROGUE_CH = 2;   // PC4
  const uint8_t PIN_MAIN_CH   = 3;   // PC5

  /**
   * For now, we proceed directly to PRE_ARM.
   */
  RocketState DisarmedState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Full data stream to ground station begins here.
   * Sensor calibration should be performed, hopefully with physical access to
   * the board.
   *
   * Proceed to ARMED on meeting all of the following conditions:
   *
   *   1. Sensor health checks passed
   *   2. GPS lock
   *   3. Software arm signal received from GS
   */
  RocketState PreArmState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Sensor calibration should be finished.
   *
   * Proceed to FLIGHT if X accel exceeds 1.1g.
   */
  RocketState ArmedState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Begin onboard data logging.
   * NOTE(yoos): Unfortunately, we do not have onboard logging yet.
   *
   * Detect apogee based mainly on four measurements:
   *
   *   1. Pressure rate of change
   *   2. Net acceleration magnitude
   *   3. Non-roll rotation rate magnitude
   *   4. Orientation
   *
   * We first determine possible apogee based on the altitude rate of change,
   * which is estimated from the barometer and GPS. (Currently, only the
   * barometer is used for altitude.)
   *
   * We check for the motor cutoff acceleration drop to negative. If we sense
   * we are falling faster than -40 m/s, we transition to APOGEE. Otherwise, we
   * check for the ideal case of zero altitude change and that we are not
   * observing a false apogee during a subsonic transition.
   *
   * We do not track powered and coasting portions of flight in separate states
   * because a mach transition may happen at an unknowable time, and it's
   * probably easier to track state variables this way.
   */
  RocketState FlightState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Deploy drogue chute. If magnitude of net proper acceleration does not
   * change within 5 seconds, deploy main.
   */
  RocketState ApogeeState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Deploy main chute at 1500' AGL.
   */
  RocketState DescentState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * Turn off all telemetry (maybe?) except GPS.
   *
   * Try to conserve power.
   */
  RocketState RecoveryState(SensorMeasurements meas, WorldEstimate est, ActuatorSetpoint& sp);

  /**
   * RGB LED stuff.
   */
  void SetLED(float r, float g, float b);
  void BlinkLED(float r, float g, float b, float freq);
  void PulseLED(float r, float g, float b, float freq);
  void RGBLED(float freq);

  /**
   * In-class global vars
   */
  RocketState state;
  float motorDC;

  /**
   * Per-launch calibration
   */
  float groundAltitude;
  float velocity;
};

#endif
<|endoftext|>


<commit_before>#include <cstdlib>
#include <cstdio> // debug
#include <algorithm>
#include <iostream>

#include <tightdb/utilities.hpp>
#include <tightdb/column.hpp>
#include <tightdb/array_string.hpp>

using namespace std;

namespace {

const int max_width = 64;

// When len = 0 returns 0
// When len = 1 returns 4
// When 2 <= len < 256, returns 2**ceil(log2(len+1)).
// Thus, 0 < len < 256 implies that len < round_up(len).
size_t round_up(size_t len)
{
    if (len < 2) return len << 2;
    len |= len >> 1;
    len |= len >> 2;
    len |= len >> 4;
    ++len;
    return len;
}

} // anonymous namespace

namespace tightdb {

void ArrayString::set(size_t ndx, StringData value)
{
    TIGHTDB_ASSERT(ndx < m_len);
    TIGHTDB_ASSERT(value.size() < size_t(max_width)); // otherwise we have to use another column type

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // Make room for the new value plus a zero-termination
    if (m_width <= value.size()) {
        if (value.size() == 0 && m_width == 0) return;

        TIGHTDB_ASSERT(0 < value.size());

        // Calc min column width
        size_t new_width = ::round_up(value.size());

        TIGHTDB_ASSERT(value.size() < new_width);

        // FIXME: Should we try to avoid double copying when realloc fails to preserve the address?
        Alloc(m_len, new_width); // Throws

        char* base = m_data;
        char* new_end = base + m_len*new_width;

        // Expand the old values in reverse order
        if (0 < m_width) {
            const char* old_end = base + m_len*m_width;
            while (new_end != base) {
                *--new_end = char(*--old_end + (new_width-m_width));
                {
                    char* new_begin = new_end - (new_width-m_width);
                    fill(new_begin, new_end, 0); // Extend padding with zero bytes
                    new_end = new_begin;
                }
                {
                    const char* old_begin = old_end - (m_width-1);
                    new_end = copy_backward(old_begin, old_end, new_end);
                    old_end = old_begin;
                }
            }
        }
        else {
            while (new_end != base) {
                *--new_end = char(new_width-1);
                {
                    char* new_begin = new_end - (new_width-1);
                    fill(new_begin, new_end, 0); // Fill with zero bytes
                    new_end = new_begin;
                }
            }
        }

        m_width = new_width;
    }

    TIGHTDB_ASSERT(0 < m_width);

    // Set the value
    char* begin = m_data + (ndx * m_width);
    char* end   = begin + (m_width-1);
    begin = copy(value.data(), value.data()+value.size(), begin);
    fill(begin, end, 0); // Pad with zero bytes
    TIGHTDB_STATIC_ASSERT(max_width <= 128, "Padding size must fit in 7-bits");
    TIGHTDB_ASSERT(end - begin < max_width);
    int pad_size = int(end - begin);
    *end = char(pad_size);
}

void ArrayString::insert(size_t ndx, StringData value)
{
    TIGHTDB_ASSERT(ndx <= m_len);
    TIGHTDB_ASSERT(value.size() < size_t(max_width)); // otherwise we have to use another column type

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // Calc min column width (incl trailing zero-byte)
    size_t new_width = max(m_width, ::round_up(value.size()));

    // Make room for the new value
    Alloc(m_len+1, new_width); // Throws

    if (0 < value.size() || 0 < m_width) {
        char* base = m_data;
        const char* old_end = base + m_len*m_width;
        char*       new_end = base + m_len*new_width + new_width;

        // Move values after insertion point (may expand)
        if (ndx != m_len) {
            if (TIGHTDB_UNLIKELY(m_width < new_width)) {
                char* const new_begin = base + ndx*new_width + new_width;
                if (0 < m_width) {
                    // Expand the old values
                    do {
                        *--new_end = char(*--old_end + (new_width-m_width));
                        {
                            char* new_begin2 = new_end - (new_width-m_width);
                            fill(new_begin2, new_end, 0); // Extend padding with zero bytes
                            new_end = new_begin2;
                        }
                        {
                            const char* old_begin = old_end - (m_width-1);
                            new_end = copy_backward(old_begin, old_end, new_end);
                            old_end = old_begin;
                        }
                    }
                    while (new_end != new_begin);
                }
                else {
                    do {
                        *--new_end = char(new_width-1);
                        {
                            char* new_begin2 = new_end - (new_width-1);
                            fill(new_begin2, new_end, 0); // Fill with zero bytes
                            new_end = new_begin2;
                        }
                    }
                    while (new_end != new_begin);
                }
            }
            else {
                // when no expansion just move the following entries forward
                const char* old_begin = base + ndx*m_width;
                new_end = copy_backward(old_begin, old_end, new_end);
                old_end = old_begin;
            }
        }

        // Set the value
        {
            char* new_begin = new_end - new_width;
            char* pad_begin = copy(value.data(), value.data()+value.size(), new_begin);
            --new_end;
            fill(pad_begin, new_end, 0); // Pad with zero bytes
            TIGHTDB_STATIC_ASSERT(max_width <= 128, "Padding size must fit in 7-bits");
            TIGHTDB_ASSERT(new_end - pad_begin < max_width);
            int pad_size = int(new_end - pad_begin);
            *new_end = char(pad_size);
            new_end = new_begin;
        }

        // Expand values before insertion point
        if (TIGHTDB_UNLIKELY(m_width < new_width)) {
            if (0 < m_width) {
                while (new_end != base) {
                    *--new_end = char(*--old_end + (new_width-m_width));
                    {
                        char* new_begin = new_end - (new_width-m_width);
                        fill(new_begin, new_end, 0); // Extend padding with zero bytes
                        new_end = new_begin;
                    }
                    {
                        const char* old_begin = old_end - (m_width-1);
                        new_end = copy_backward(old_begin, old_end, new_end);
                        old_end = old_begin;
                    }
                }
            }
            else {
                while (new_end != base) {
                    *--new_end = char(new_width-1);
                    {
                        char* new_begin = new_end - (new_width-1);
                        fill(new_begin, new_end, 0); // Fill with zero bytes
                        new_end = new_begin;
                    }
                }
            }
            m_width = new_width;
        }
    }

    ++m_len;
}

void ArrayString::erase(size_t ndx)
{
    TIGHTDB_ASSERT(ndx < m_len);

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // move data backwards after deletion
    if (ndx < m_len-1) {
        char* const new_begin = m_data + ndx*m_width;
        char* const old_begin = new_begin + m_width;
        char* const old_end   = m_data + m_len*m_width;
        copy(old_begin, old_end, new_begin);
    }

    --m_len;

    // Update length in header
    set_header_len(m_len);
}

size_t ArrayString::CalcByteLen(size_t count, size_t width) const
{
    // FIXME: This arithemtic could overflow. Consider using <tightdb/overflow.hpp>
    return 8 + (count * width);
}

size_t ArrayString::CalcItemCount(size_t bytes, size_t width) const TIGHTDB_NOEXCEPT
{
    if (width == 0) return size_t(-1); // zero-width gives infinite space

    const size_t bytes_without_header = bytes - 8;
    return bytes_without_header / width;
}

size_t ArrayString::count(StringData value, size_t begin, size_t end) const
{
    size_t count = 0;

    size_t lastmatch = begin - 1;
    for (;;) {
        lastmatch = find_first(value, lastmatch+1, end);
        if (lastmatch != not_found)
            ++count;
        else break;
    }

    return count;
}

size_t ArrayString::find_first(StringData value, size_t begin, size_t end) const
{
    if (end == size_t(-1)) end = m_len;
    TIGHTDB_ASSERT(begin <= m_len && end <= m_len && begin <= end);

    // A string can never be wider than the column width
    if (m_width <= value.size()) return size_t(-1);

    if (m_width == 0) {
        return value.size() == 0 && begin < end ? begin : -1;
    }

    for (size_t i=begin; i<end; ++i) {
        const char* data = m_data + (i * m_width);
        std::size_t size = (m_width-1) - data[m_width-1];
        if (StringData(data, size) == value) return i;
    }

    return size_t(-1); // not found
}

void ArrayString::find_all(Array& result, StringData value, size_t add_offset,
                           size_t begin, size_t end)
{
    size_t first = begin - 1;
    for (;;) {
        first = find_first(value, first + 1, end);
        if (first != size_t(-1))
            result.add(first + add_offset);
        else break;
    }
}

bool ArrayString::Compare(const ArrayString& c) const
{
    if (c.size() != size()) return false;

    for (size_t i = 0; i < size(); ++i) {
        if (get(i) != c.get(i)) return false;
    }

    return true;
}

#ifdef TIGHTDB_DEBUG

void ArrayString::StringStats() const
{
    size_t total = 0;
    size_t longest = 0;

    for (size_t i = 0; i < m_len; ++i) {
        StringData str = get(i);
        const size_t len = str.size() + 1;
        total += len;
        if (len > longest) longest = len;
    }

    const size_t size = m_len * m_width;
    const size_t zeroes = size - total;
    const size_t zavg = zeroes / (m_len ? m_len : 1); // avoid possible div by zero

    cout << "Count: " << m_len << "\n";
    cout << "Width: " << m_width << "\n";
    cout << "Total: " << size << "\n";
    cout << "Capacity: " << m_capacity << "\n\n";
    cout << "Bytes string: " << total << "\n";
    cout << "     longest: " << longest << "\n";
    cout << "Bytes zeroes: " << zeroes << "\n";
    cout << "         avg: " << zavg << "\n";
}

/*
void ArrayString::ToDot(FILE* f) const
{
    const size_t ref = getRef();

    fprintf(f, "n%zx [label=\"", ref);

    for (size_t i = 0; i < m_len; ++i) {
        if (i > 0) fprintf(f, " | ");

        fprintf(f, "%s", get_c_str(i));
    }

    fprintf(f, "\"];\n");
}
*/

void ArrayString::ToDot(ostream& out, StringData title) const
{
    const size_t ref = GetRef();

    if (0 < title.size()) {
        out << "subgraph cluster_" << ref << " {" << endl;
        out << " label = \"" << title << "\";" << endl;
        out << " color = white;" << endl;
    }

    out << "n" << hex << ref << dec << "[shape=none,label=<";
    out << "

" << endl;

    // Header
    out << "";
    out << "0x" << hex << ref << dec << "	" << endl;

    for (size_t i = 0; i < m_len; ++i) {
        out << "\"" << get(i) << "\"	" << endl;
    }

    out << "

>];" << endl;
    if (0 < title.size()) out << "}" << endl;
}

#endif // TIGHTDB_DEBUG

}
<commit_msg>Fixed performance issue when searching in short string array<commit_after>#include <cstdlib>
#include <cstdio> // debug
#include <algorithm>
#include <iostream>

#include <tightdb/utilities.hpp>
#include <tightdb/column.hpp>
#include <tightdb/array_string.hpp>

using namespace std;

namespace {

const int max_width = 64;

// When len = 0 returns 0
// When len = 1 returns 4
// When 2 <= len < 256, returns 2**ceil(log2(len+1)).
// Thus, 0 < len < 256 implies that len < round_up(len).
size_t round_up(size_t len)
{
    if (len < 2) return len << 2;
    len |= len >> 1;
    len |= len >> 2;
    len |= len >> 4;
    ++len;
    return len;
}

} // anonymous namespace

namespace tightdb {

void ArrayString::set(size_t ndx, StringData value)
{
    TIGHTDB_ASSERT(ndx < m_len);
    TIGHTDB_ASSERT(value.size() < size_t(max_width)); // otherwise we have to use another column type

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // Make room for the new value plus a zero-termination
    if (m_width <= value.size()) {
        if (value.size() == 0 && m_width == 0) return;

        TIGHTDB_ASSERT(0 < value.size());

        // Calc min column width
        size_t new_width = ::round_up(value.size());

        TIGHTDB_ASSERT(value.size() < new_width);

        // FIXME: Should we try to avoid double copying when realloc fails to preserve the address?
        Alloc(m_len, new_width); // Throws

        char* base = m_data;
        char* new_end = base + m_len*new_width;

        // Expand the old values in reverse order
        if (0 < m_width) {
            const char* old_end = base + m_len*m_width;
            while (new_end != base) {
                *--new_end = char(*--old_end + (new_width-m_width));
                {
                    char* new_begin = new_end - (new_width-m_width);
                    fill(new_begin, new_end, 0); // Extend padding with zero bytes
                    new_end = new_begin;
                }
                {
                    const char* old_begin = old_end - (m_width-1);
                    new_end = copy_backward(old_begin, old_end, new_end);
                    old_end = old_begin;
                }
            }
        }
        else {
            while (new_end != base) {
                *--new_end = char(new_width-1);
                {
                    char* new_begin = new_end - (new_width-1);
                    fill(new_begin, new_end, 0); // Fill with zero bytes
                    new_end = new_begin;
                }
            }
        }

        m_width = new_width;
    }

    TIGHTDB_ASSERT(0 < m_width);

    // Set the value
    char* begin = m_data + (ndx * m_width);
    char* end   = begin + (m_width-1);
    begin = copy(value.data(), value.data()+value.size(), begin);
    fill(begin, end, 0); // Pad with zero bytes
    TIGHTDB_STATIC_ASSERT(max_width <= 128, "Padding size must fit in 7-bits");
    TIGHTDB_ASSERT(end - begin < max_width);
    int pad_size = int(end - begin);
    *end = char(pad_size);
}

void ArrayString::insert(size_t ndx, StringData value)
{
    TIGHTDB_ASSERT(ndx <= m_len);
    TIGHTDB_ASSERT(value.size() < size_t(max_width)); // otherwise we have to use another column type

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // Calc min column width (incl trailing zero-byte)
    size_t new_width = max(m_width, ::round_up(value.size()));

    // Make room for the new value
    Alloc(m_len+1, new_width); // Throws

    if (0 < value.size() || 0 < m_width) {
        char* base = m_data;
        const char* old_end = base + m_len*m_width;
        char*       new_end = base + m_len*new_width + new_width;

        // Move values after insertion point (may expand)
        if (ndx != m_len) {
            if (TIGHTDB_UNLIKELY(m_width < new_width)) {
                char* const new_begin = base + ndx*new_width + new_width;
                if (0 < m_width) {
                    // Expand the old values
                    do {
                        *--new_end = char(*--old_end + (new_width-m_width));
                        {
                            char* new_begin2 = new_end - (new_width-m_width);
                            fill(new_begin2, new_end, 0); // Extend padding with zero bytes
                            new_end = new_begin2;
                        }
                        {
                            const char* old_begin = old_end - (m_width-1);
                            new_end = copy_backward(old_begin, old_end, new_end);
                            old_end = old_begin;
                        }
                    }
                    while (new_end != new_begin);
                }
                else {
                    do {
                        *--new_end = char(new_width-1);
                        {
                            char* new_begin2 = new_end - (new_width-1);
                            fill(new_begin2, new_end, 0); // Fill with zero bytes
                            new_end = new_begin2;
                        }
                    }
                    while (new_end != new_begin);
                }
            }
            else {
                // when no expansion just move the following entries forward
                const char* old_begin = base + ndx*m_width;
                new_end = copy_backward(old_begin, old_end, new_end);
                old_end = old_begin;
            }
        }

        // Set the value
        {
            char* new_begin = new_end - new_width;
            char* pad_begin = copy(value.data(), value.data()+value.size(), new_begin);
            --new_end;
            fill(pad_begin, new_end, 0); // Pad with zero bytes
            TIGHTDB_STATIC_ASSERT(max_width <= 128, "Padding size must fit in 7-bits");
            TIGHTDB_ASSERT(new_end - pad_begin < max_width);
            int pad_size = int(new_end - pad_begin);
            *new_end = char(pad_size);
            new_end = new_begin;
        }

        // Expand values before insertion point
        if (TIGHTDB_UNLIKELY(m_width < new_width)) {
            if (0 < m_width) {
                while (new_end != base) {
                    *--new_end = char(*--old_end + (new_width-m_width));
                    {
                        char* new_begin = new_end - (new_width-m_width);
                        fill(new_begin, new_end, 0); // Extend padding with zero bytes
                        new_end = new_begin;
                    }
                    {
                        const char* old_begin = old_end - (m_width-1);
                        new_end = copy_backward(old_begin, old_end, new_end);
                        old_end = old_begin;
                    }
                }
            }
            else {
                while (new_end != base) {
                    *--new_end = char(new_width-1);
                    {
                        char* new_begin = new_end - (new_width-1);
                        fill(new_begin, new_end, 0); // Fill with zero bytes
                        new_end = new_begin;
                    }
                }
            }
            m_width = new_width;
        }
    }

    ++m_len;
}

void ArrayString::erase(size_t ndx)
{
    TIGHTDB_ASSERT(ndx < m_len);

    // Check if we need to copy before modifying
    CopyOnWrite(); // Throws

    // move data backwards after deletion
    if (ndx < m_len-1) {
        char* const new_begin = m_data + ndx*m_width;
        char* const old_begin = new_begin + m_width;
        char* const old_end   = m_data + m_len*m_width;
        copy(old_begin, old_end, new_begin);
    }

    --m_len;

    // Update length in header
    set_header_len(m_len);
}

size_t ArrayString::CalcByteLen(size_t count, size_t width) const
{
    // FIXME: This arithemtic could overflow. Consider using <tightdb/overflow.hpp>
    return 8 + (count * width);
}

size_t ArrayString::CalcItemCount(size_t bytes, size_t width) const TIGHTDB_NOEXCEPT
{
    if (width == 0) return size_t(-1); // zero-width gives infinite space

    const size_t bytes_without_header = bytes - 8;
    return bytes_without_header / width;
}

size_t ArrayString::count(StringData value, size_t begin, size_t end) const
{
    size_t count = 0;

    size_t lastmatch = begin - 1;
    for (;;) {
        lastmatch = find_first(value, lastmatch+1, end);
        if (lastmatch != not_found)
            ++count;
        else break;
    }

    return count;
}

size_t ArrayString::find_first(StringData value, size_t begin, size_t end) const
{
    if (end == size_t(-1)) end = m_len;
    TIGHTDB_ASSERT(begin <= m_len && end <= m_len && begin <= end);

    // A string can never be wider than the column width
    if (m_width <= value.size()) return size_t(-1);

    if (m_width == 0) {
        return value.size() == 0 && begin < end ? begin : -1;
    }

    if (value.size() == 0) {
        const char* data = m_data + (m_width-1);
        for (size_t i = begin; i != end; ++i) {
            size_t size = (m_width-1) - data[i * m_width];
            if (size == 0) return i;
        }
    }
    else {
        for (size_t i = begin; i != end; ++i) {
            const char* data = m_data + (i * m_width);
            size_t j = 0;
            for (;;) {
                if (TIGHTDB_LIKELY(data[j] != value[j])) break;
                ++j;
                if (TIGHTDB_UNLIKELY(j == value.size())) {
                    size_t size = (m_width-1) - data[m_width-1];
                    if (TIGHTDB_LIKELY(size == value.size())) return i;
                    break;
                }
            }
        }
    }

    return size_t(-1); // not found
}

void ArrayString::find_all(Array& result, StringData value, size_t add_offset,
                           size_t begin, size_t end)
{
    size_t first = begin - 1;
    for (;;) {
        first = find_first(value, first + 1, end);
        if (first != size_t(-1))
            result.add(first + add_offset);
        else break;
    }
}

bool ArrayString::Compare(const ArrayString& c) const
{
    if (c.size() != size()) return false;

    for (size_t i = 0; i < size(); ++i) {
        if (get(i) != c.get(i)) return false;
    }

    return true;
}

#ifdef TIGHTDB_DEBUG

void ArrayString::StringStats() const
{
    size_t total = 0;
    size_t longest = 0;

    for (size_t i = 0; i < m_len; ++i) {
        StringData str = get(i);
        const size_t len = str.size() + 1;
        total += len;
        if (len > longest) longest = len;
    }

    const size_t size = m_len * m_width;
    const size_t zeroes = size - total;
    const size_t zavg = zeroes / (m_len ? m_len : 1); // avoid possible div by zero

    cout << "Count: " << m_len << "\n";
    cout << "Width: " << m_width << "\n";
    cout << "Total: " << size << "\n";
    cout << "Capacity: " << m_capacity << "\n\n";
    cout << "Bytes string: " << total << "\n";
    cout << "     longest: " << longest << "\n";
    cout << "Bytes zeroes: " << zeroes << "\n";
    cout << "         avg: " << zavg << "\n";
}

/*
void ArrayString::ToDot(FILE* f) const
{
    const size_t ref = getRef();

    fprintf(f, "n%zx [label=\"", ref);

    for (size_t i = 0; i < m_len; ++i) {
        if (i > 0) fprintf(f, " | ");

        fprintf(f, "%s", get_c_str(i));
    }

    fprintf(f, "\"];\n");
}
*/

void ArrayString::ToDot(ostream& out, StringData title) const
{
    const size_t ref = GetRef();

    if (0 < title.size()) {
        out << "subgraph cluster_" << ref << " {" << endl;
        out << " label = \"" << title << "\";" << endl;
        out << " color = white;" << endl;
    }

    out << "n" << hex << ref << dec << "[shape=none,label=<";
    out << "

" << endl;

    // Header
    out << "";
    out << "0x" << hex << ref << dec << "	" << endl;

    for (size_t i = 0; i < m_len; ++i) {
        out << "\"" << get(i) << "\"	" << endl;
    }

    out << "

>];" << endl;
    if (0 < title.size()) out << "}" << endl;
}

#endif // TIGHTDB_DEBUG

}
<|endoftext|>


<commit_before>//------------------------------------------------------------------------------
// Copyright (c) 2018 by contributors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//------------------------------------------------------------------------------

/*
This file is the implementation of Reader class.
*/

#include "src/reader/reader.h"

#include <string.h>
#include <algorithm> // for random_shuffle

#include "src/base/file_util.h"
#include "src/base/split_string.h"
#include "src/base/format_print.h"

namespace xLearn {

//------------------------------------------------------------------------------
// Class register
//------------------------------------------------------------------------------
CLASS_REGISTER_IMPLEMENT_REGISTRY(xLearn_reader_registry, Reader);
REGISTER_READER("memory", InmemReader);
REGISTER_READER("disk", OndiskReader);

// Check current file format and
// return 'libsvm', 'libffm', or 'csv'.
// This function will also check if current
// data has the label y.
std::string Reader::check_file_format() {
  FILE* file = OpenFileOrDie(filename_.c_str(), "r");
  // get the first line of data
  std::string data_line;
  GetLine(file, data_line);
  Close(file);
  // Find the split string
  int space_count = 0;
  int table_count = 0;
  int comma_count = 0;
  for (size_t i = 0; i < data_line.size(); ++i) {
    if (data_line[i] == ' ') {
      space_count++;
    } else if (data_line[i] == '\t') {
      table_count++;
    } else if (data_line[i] == ',') {
      comma_count++;
    }
  }
  if (space_count > table_count &&
      space_count > comma_count) {
    splitor_ = " ";
  } else if (table_count > space_count &&
             table_count > comma_count) {
    splitor_ = "\t";
  } else if (comma_count > space_count &&
             comma_count > table_count) {
    splitor_ = ",";
  } else {
    LOG(FATAL) << "File format error!";
  }
  // Split the first line of data
  std::vector<std::string> str_list;
  SplitStringUsing(data_line, splitor_.c_str(), &str_list);
  // has y?
  size_t found = str_list[0].find(":");
  if (found != std::string::npos) {  // find ":", no label
    has_label_ = false;
  } else {
    has_label_ = true;
  }
  // check file format
  int count = 0;
  for (int i = 0; i < str_list[1].size(); ++i) {
    if (str_list[1][i] == ':') {
      count++;
    }
  }
  if (count == 1) {
    return "libsvm";
  } else if (count == 2) {
    return "libffm";
  } else if (count == 0){
    return "csv";
  }
  Color::print_error("Unknow file format");
  exit(0);
}

// Find the last '\n' in block, and shrink back file pointer
void Reader::shrink_block(char* block, size_t* ret, FILE* file) {
  // Find the last '\n'
  size_t index = *ret-1;
  while (block[index] != '\n') { index--; }
  // Shrink back file pointer
  fseek(file, index-*ret+1, SEEK_CUR);
  // The real size of block
  *ret = index + 1;
}

//------------------------------------------------------------------------------
// Implementation of InmemReader
//------------------------------------------------------------------------------

// Pre-load all the data into memory buffer (data_buf_).
// Note that this funtion will first check whether we
// can use the existing binary file. If not, reader will
// generate one automatically.
void InmemReader::Initialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true)
  filename_ = filename;
  Color::print_info("First check if the text file has been already "
                    "converted to binary format.");
  // HashBinary() will read the first two hash value
  // and then check it whether equal to the hash value generated
  // by HashFile() function from current txt file.
  if (hash_binary(filename_)) {
    Color::print_info(
      StringPrintf("Binary file (%s.bin) found. "
                   "Skip converting text to binary.",
                   filename_.c_str())
    );
    filename_ += ".bin";
    init_from_binary();
  } else {
    Color::print_info(
      StringPrintf("Binary file (%s.bin) NOT found. Convert text "
                   "file to binary file.",
                   filename_.c_str())
    );
    // Allocate memory for block
    try {
      this->block_ = (char*)malloc(block_size_*1024*1024);
    } catch (std::bad_alloc&) {
      LOG(FATAL) << "Cannot allocate enough memory for data  \
                     block. Block size: "
                 << block_size_ << "MB. "
                 << "You set change the block size via configuration.";
    }
    init_from_txt();
  }
}

// Check wheter current path has a binary file.
// We use double check here, that is, we first check
// the hash value of a small data block, then check the whole file.
bool InmemReader::hash_binary(const std::string& filename) {
  std::string bin_file = filename + ".bin";
  // If the ".bin" file does not exists, return false.
  if (!FileExist(bin_file.c_str())) { return false; }
  FILE* file = OpenFileOrDie(bin_file.c_str(), "r");
  // Check the first hash value
  uint64 hash_1 = 0;
  ReadDataFromDisk(file, (char*)&hash_1, sizeof(hash_1));
  if (hash_1 != HashFile(filename, true)) {
    Close(file);
    return false;
  }
  // Check the second hash value
  uint64 hash_2 = 0;
  ReadDataFromDisk(file, (char*)&hash_2, sizeof(hash_2));
  if (hash_2 != HashFile(filename, false)) {
    Close(file);
    return false;
  }
  Close(file);
  return true;
}

// In-memory Reader can be initialized from binary file.
void InmemReader::init_from_binary() {
  // Init data_buf_
  data_buf_.Deserialize(filename_);
  has_label_ = data_buf_.has_label;
  // Init data_samples_
  num_samples_ = data_buf_.row_length;
  data_samples_.ReAlloc(num_samples_);
  // for shuffle
  order_.resize(num_samples_);
  for (int i = 0; i < order_.size(); ++i) {
    order_[i] = i;
  }
}

// Pre-load all the data to memory buffer from txt file.
void InmemReader::init_from_txt() {
  // Init parser_
  parser_ = CreateParser(check_file_format().c_str());
  if (has_label_) parser_->setLabel(true);
  else parser_->setLabel(false);
  // Set splitor
  parser_->setSplitor(this->splitor_);
  // Convert MB to Byte
  uint64 read_byte = block_size_ * 1024 * 1024;
  // Open file
  FILE* file = OpenFileOrDie(filename_.c_str(), "r");
  // Read until the end of file
  for (;;) {
    // Read a block of data from disk file
    size_t ret = ReadDataFromDisk(file, block_, read_byte);
    if (ret == 0) {
      break;
    } else if (ret == read_byte) {
      // Find the last '\n', and shrink back file pointer
      this->shrink_block(block_, &ret, file);
    } // else ret < read_byte: we don't need shrink_block()
    parser_->Parse(block_, ret, data_buf_, false);
  }
  data_buf_.SetHash(HashFile(filename_, true),
                    HashFile(filename_, false));
  data_buf_.has_label = has_label_;
  // Init data_samples_
  num_samples_ = data_buf_.row_length;
  data_samples_.ReAlloc(num_samples_, has_label_);
  // for shuffle
  order_.resize(num_samples_);
  for (int i = 0; i < order_.size(); ++i) {
    order_[i] = i;
  }
  // Deserialize in-memory buffer to disk file.
  std::string bin_file = filename_ + ".bin";
  data_buf_.Serialize(bin_file);
  delete [] block_;
  Close(file);
}

// Smaple data from memory buffer.
index_t InmemReader::Samples(DMatrix* &matrix) {
  for (int i = 0; i < num_samples_; ++i) {
    if (pos_ >= data_buf_.row_length) {
      // End of the data buffer
      if (i == 0) {
        if (shuffle_) {
          random_shuffle(order_.begin(), order_.end());
        }
        matrix = nullptr;
        return 0;
      }
      break;
    }
    // Copy data between different DMatrix.
    data_samples_.row[i] = data_buf_.row[order_[pos_]];
    data_samples_.Y[i] = data_buf_.Y[order_[pos_]];
    data_samples_.norm[i] = data_buf_.norm[order_[pos_]];
    pos_++;
  }
  matrix = &data_samples_;
  return num_samples_;
}

// Return to the begining of the data buffer.
void InmemReader::Reset() { pos_ = 0; }

//------------------------------------------------------------------------------
// Implementation of OndiskReader.
//------------------------------------------------------------------------------

// Create parser and open file
void OndiskReader::Initialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  this->filename_ = filename;
  // Init parser_
  parser_ = CreateParser(check_file_format().c_str());
  if (has_label_) parser_->setLabel(true);
  else parser_->setLabel(false);
  // Set splitor
  parser_->setSplitor(this->splitor_);
  // Allocate memory for block
  try {
    this->block_ = (char*)malloc(block_size_*1024*1024);
  } catch (std::bad_alloc&) {
    LOG(FATAL) << "Cannot allocate enough memory for data  \
                   block. Block size: "
               << block_size_ << "MB. "
               << "You set change the block size via configuration.";
  }
  // Open file
  file_ptr_ = OpenFileOrDie(filename_.c_str(), "r");
}

// Return to the begining of the file
void OndiskReader::Reset() {
  int ret = fseek(file_ptr_, 0, SEEK_SET);
  if (ret != 0) {
    LOG(FATAL) << "Fail to return to the head of file.";
  }
}

// Sample data from disk file.
index_t OndiskReader::Samples(DMatrix* &matrix) {
  // Convert MB to Byte
  uint64 read_byte = block_size_ * 1024 * 1024;
  // Read a block of data from disk file
  size_t ret = ReadDataFromDisk(file_ptr_, block_, read_byte);
  if (ret == 0) {
    matrix = nullptr;
    return 0;
  } else if (ret == read_byte) {
    // Find the last '\n', and shrink back file pointer
    shrink_block(block_, &ret, file_ptr_);
  } // else ret < read_byte: we don't need shrink_block()
  // Parse block to data_sample_
  parser_->Parse(block_, ret, data_samples_, true);
  matrix = &data_samples_;
  return data_samples_.row_length;
}

}  // namespace xLearn
<commit_msg>small fix<commit_after>//------------------------------------------------------------------------------
// Copyright (c) 2018 by contributors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//------------------------------------------------------------------------------

/*
This file is the implementation of Reader class.
*/

#include "src/reader/reader.h"

#include <string.h>
#include <algorithm> // for random_shuffle

#include "src/base/file_util.h"
#include "src/base/split_string.h"
#include "src/base/format_print.h"

namespace xLearn {

//------------------------------------------------------------------------------
// Class register
//------------------------------------------------------------------------------
CLASS_REGISTER_IMPLEMENT_REGISTRY(xLearn_reader_registry, Reader);
REGISTER_READER("memory", InmemReader);
REGISTER_READER("disk", OndiskReader);

// Check current file format and
// return 'libsvm', 'libffm', or 'csv'.
// This function will also check if current
// data has the label y.
std::string Reader::check_file_format() {
  FILE* file = OpenFileOrDie(filename_.c_str(), "r");
  // get the first line of data
  std::string data_line;
  GetLine(file, data_line);
  Close(file);
  // Find the split string
  int space_count = 0;
  int table_count = 0;
  int comma_count = 0;
  for (size_t i = 0; i < data_line.size(); ++i) {
    if (data_line[i] == ' ') {
      space_count++;
    } else if (data_line[i] == '\t') {
      table_count++;
    } else if (data_line[i] == ',') {
      comma_count++;
    }
  }
  if (space_count > table_count &&
      space_count > comma_count) {
    splitor_ = " ";
  } else if (table_count > space_count &&
             table_count > comma_count) {
    splitor_ = "\t";
  } else if (comma_count > space_count &&
             comma_count > table_count) {
    splitor_ = ",";
  } else {
    LOG(FATAL) << "File format error!";
  }
  // Split the first line of data
  std::vector<std::string> str_list;
  SplitStringUsing(data_line, splitor_.c_str(), &str_list);
  // has y?
  size_t found = str_list[0].find(":");
  if (found != std::string::npos) {  // find ":", no label
    has_label_ = false;
  } else {
    has_label_ = true;
  }
  // check file format
  int count = 0;
  for (int i = 0; i < str_list[1].size(); ++i) {
    if (str_list[1][i] == ':') {
      count++;
    }
  }
  if (count == 1) {
    return "libsvm";
  } else if (count == 2) {
    return "libffm";
  } else if (count == 0){
    return "csv";
  }
  Color::print_error("Unknow file format");
  exit(0);
}

// Find the last '\n' in block, and shrink back file pointer
void Reader::shrink_block(char* block, size_t* ret, FILE* file) {
  // Find the last '\n'
  size_t index = *ret-1;
  while (block[index] != '\n') { index--; }
  // Shrink back file pointer
  fseek(file, index-*ret+1, SEEK_CUR);
  // The real size of block
  *ret = index + 1;
}

//------------------------------------------------------------------------------
// Implementation of InmemReader
//------------------------------------------------------------------------------

// Pre-load all the data into memory buffer (data_buf_).
// Note that this funtion will first check whether we
// can use the existing binary file. If not, reader will
// generate one automatically.
void InmemReader::Initialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true)
  filename_ = filename;
  Color::print_info("First check if the text file has been already "
                    "converted to binary format.");
  // HashBinary() will read the first two hash value
  // and then check it whether equal to the hash value generated
  // by HashFile() function from current txt file.
  if (hash_binary(filename_)) {
    Color::print_info(
      StringPrintf("Binary file (%s.bin) found. "
                   "Skip converting text to binary.",
                   filename_.c_str())
    );
    filename_ += ".bin";
    init_from_binary();
  } else {
    Color::print_info(
      StringPrintf("Binary file (%s.bin) NOT found. Convert text "
                   "file to binary file.",
                   filename_.c_str())
    );
    // Allocate memory for block
    try {
      this->block_ = (char*)malloc(block_size_*1024*1024);
    } catch (std::bad_alloc&) {
      LOG(FATAL) << "Cannot allocate enough memory for data  \
                     block. Block size: "
                 << block_size_ << "MB. "
                 << "You set change the block size via configuration.";
    }
    init_from_txt();
  }
}

// Check wheter current path has a binary file.
// We use double check here, that is, we first check
// the hash value of a small data block, then check the whole file.
bool InmemReader::hash_binary(const std::string& filename) {
  std::string bin_file = filename + ".bin";
  // If the ".bin" file does not exists, return false.
  if (!FileExist(bin_file.c_str())) { return false; }
  FILE* file = OpenFileOrDie(bin_file.c_str(), "r");
  // Check the first hash value
  uint64 hash_1 = 0;
  ReadDataFromDisk(file, (char*)&hash_1, sizeof(hash_1));
  if (hash_1 != HashFile(filename, true)) {
    Close(file);
    return false;
  }
  // Check the second hash value
  uint64 hash_2 = 0;
  ReadDataFromDisk(file, (char*)&hash_2, sizeof(hash_2));
  if (hash_2 != HashFile(filename, false)) {
    Close(file);
    return false;
  }
  Close(file);
  return true;
}

// In-memory Reader can be initialized from binary file.
void InmemReader::init_from_binary() {
  // Init data_buf_
  data_buf_.Deserialize(filename_);
  has_label_ = data_buf_.has_label;
  // Init data_samples_
  num_samples_ = data_buf_.row_length;
  data_samples_.ReAlloc(num_samples_);
  // for shuffle
  order_.resize(num_samples_);
  for (int i = 0; i < order_.size(); ++i) {
    order_[i] = i;
  }
}

// Pre-load all the data to memory buffer from txt file.
void InmemReader::init_from_txt() {
  // Init parser_
  parser_ = CreateParser(check_file_format().c_str());
  if (has_label_) parser_->setLabel(true);
  else parser_->setLabel(false);
  // Set splitor
  parser_->setSplitor(this->splitor_);
  // Convert MB to Byte
  uint64 read_byte = block_size_ * 1024 * 1024;
  // Open file
  FILE* file = OpenFileOrDie(filename_.c_str(), "r");
  // Read until the end of file
  for (;;) {
    // Read a block of data from disk file
    size_t ret = ReadDataFromDisk(file, block_, read_byte);
    if (ret == 0) {
      break;
    } else if (ret == read_byte) {
      // Find the last '\n', and shrink back file pointer
      this->shrink_block(block_, &ret, file);
    } // else ret < read_byte: we don't need shrink_block()
    parser_->Parse(block_, ret, data_buf_, false);
  }
  data_buf_.SetHash(HashFile(filename_, true),
                    HashFile(filename_, false));
  data_buf_.has_label = has_label_;
  // Init data_samples_
  num_samples_ = data_buf_.row_length;
  data_samples_.ReAlloc(num_samples_, has_label_);
  // for shuffle
  order_.resize(num_samples_);
  for (int i = 0; i < order_.size(); ++i) {
    order_[i] = i;
  }
  // Deserialize in-memory buffer to disk file.
  std::string bin_file = filename_ + ".bin";
  data_buf_.Serialize(bin_file);
  delete [] block_;
  Close(file);
}

// Smaple data from memory buffer.
index_t InmemReader::Samples(DMatrix* &matrix) {
  for (int i = 0; i < num_samples_; ++i) {
    if (pos_ >= data_buf_.row_length) {
      // End of the data buffer
      if (i == 0) {
        if (shuffle_) {
          random_shuffle(order_.begin(), order_.end());
        }
        matrix = nullptr;
        return 0;
      }
      break;
    }
    // Copy data between different DMatrix.
    data_samples_.row[i] = data_buf_.row[order_[pos_]];
    data_samples_.Y[i] = data_buf_.Y[order_[pos_]];
    data_samples_.norm[i] = data_buf_.norm[order_[pos_]];
    pos_++;
  }
  matrix = &data_samples_;
  return num_samples_;
}

// Return to the begining of the data buffer.
void InmemReader::Reset() { pos_ = 0; }

//------------------------------------------------------------------------------
// Implementation of OndiskReader.
//------------------------------------------------------------------------------

// Create parser and open file
void OndiskReader::Initialize(const std::string& filename) {
  CHECK_NE(filename.empty(), true);
  this->filename_ = filename;
  // Init parser_
  parser_ = CreateParser(check_file_format().c_str());
  if (has_label_) parser_->setLabel(true);
  else parser_->setLabel(false);
  // Set splitor
  parser_->setSplitor(this->splitor_);
  // Allocate memory for block
  try {
    this->block_ = (char*)malloc(block_size_*1024*1024);
  } catch (std::bad_alloc&) {
    LOG(FATAL) << "Cannot allocate enough memory for data  \
                   block. Block size: "
               << block_size_ << "MB. "
               << "You set change the block size via configuration.";
  }
  // Open file
  file_ptr_ = OpenFileOrDie(filename_.c_str(), "r");
}

// Return to the begining of the file
void OndiskReader::Reset() {
  int ret = fseek(file_ptr_, 0, SEEK_SET);
  if (ret != 0) {
    LOG(FATAL) << "Fail to return to the head of file.";
  }
}

// Sample data from disk file.
index_t OndiskReader::Samples(DMatrix* &matrix) {
  // Convert MB to Byte
  uint64 read_byte = block_size_ * 1024 * 1024;
  // Read a block of data from disk file
  size_t ret = ReadDataFromDisk(file_ptr_, block_, read_byte);
  if (ret == 0) {
    matrix = nullptr;
    return 0;
  } else if (ret == read_byte) {
    // Find the last '\n', and shrink back file pointer
    shrink_block(block_, &ret, file_ptr_);
  } // else ret < read_byte: we don't need shrink_block()
  // Parse block to data_sample_
  parser_->Parse(block_, ret, data_samples_, true);
  matrix = &data_samples_;
  return data_samples_.row_length;
}

}  // namespace xLearn
<|endoftext|>


<commit_before>/* Copyright (c) 2010, Antonie Jovanoski
 *
 * All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 * Contact e-mail: Antonie Jovanoski <minimoog77_at_gmail.com>
 */

#include <QDateTime>
#include "qtweetuser.h"

QTweetUser::QTweetUser()
{
}

void QTweetUser::setId(qint64 id)
{
    m_userInfo.insert(QTweetUser::Id, id);
}

qint64 QTweetUser::id() const
{
    return m_userInfo.value(QTweetUser::Id).toLongLong();
}

void QTweetUser::setName(const QString &name)
{
    m_userInfo.insert(QTweetUser::Name, name);
}

QString QTweetUser::name() const
{
    return m_userInfo.value(QTweetUser::Name).toString();
}

void QTweetUser::setScreenName(const QString &screenName)
{
    m_userInfo.insert(QTweetUser::ScreenName, screenName);
}

QString QTweetUser::screenName() const
{
    return m_userInfo.value(QTweetUser::ScreenName).toString();
}

void QTweetUser::setLocation(const QString &location)
{
    m_userInfo.insert(QTweetUser::Location, location);
}

QString QTweetUser::location() const
{
    return m_userInfo.value(QTweetUser::Location).toString();
}

void QTweetUser::setDescription(const QString &desc)
{
    m_userInfo.insert(QTweetUser::Description, desc);
}

QString QTweetUser::description() const
{
    return m_userInfo.value(QTweetUser::Description).toString();
}

void QTweetUser::setprofileImageUrl(const QString &url)
{
    m_userInfo.insert(QTweetUser::ProfileImageUrl, url);
}

QString QTweetUser::profileImageUrl() const
{
    return m_userInfo.value(QTweetUser::ProfileImageUrl).toString();
}

void QTweetUser::setUrl(const QString &url)
{
    m_userInfo.insert(QTweetUser::Url, url);
}

QString QTweetUser::url() const
{
    return m_userInfo.value(QTweetUser::Url).toString();
}

void QTweetUser::setProtected(bool protected_)
{
    m_userInfo.insert(QTweetUser::Protected, protected_);
}

bool QTweetUser::isProtected() const
{
    return m_userInfo.value(QTweetUser::Protected).toBool();
}

void QTweetUser::setFollowersCount(int count)
{
    m_userInfo.insert(QTweetUser::FollowersCount, count);
}

int QTweetUser::followersCount() const
{
    return m_userInfo.value(QTweetUser::FollowersCount).toInt();
}

void QTweetUser::setFriendsCount(int count)
{
    m_userInfo.insert(QTweetUser::FriendsCount, count);
}

int QTweetUser::friendsCount() const
{
    return m_userInfo.value(QTweetUser::FriendsCount).toInt();
}

void QTweetUser::setCreatedAt(const QString &twitterDate)
{
    QDateTime datetime = twitterDateToQDateTime(twitterDate);

    m_userInfo.insert(QTweetUser::CreatedAt, datetime);
}

QDateTime QTweetUser::createdAt() const
{
    return m_userInfo.value(QTweetUser::CreatedAt).toDateTime();
}

void QTweetUser::setFavouritesCount(int count)
{
    m_userInfo.insert(QTweetUser::FavouritesCount, count);
}

int QTweetUser::favouritesCount() const
{
    return m_userInfo.value(QTweetUser::FavouritesCount).toInt();
}

void QTweetUser::setUtcOffset(int sec)
{
    m_userInfo.insert(QTweetUser::UtcOffset, sec);
}

int QTweetUser::utcOffset() const
{
    return m_userInfo.value(QTweetUser::UtcOffset).toInt();
}

void QTweetUser::setTimezone(const QString &timezone)
{
    m_userInfo.insert(QTweetUser::TimeZone, timezone);
}

QString QTweetUser::timezone() const
{
    return m_userInfo.value(QTweetUser::TimeZone).toString();
}

void QTweetUser::setGeoEnabled(bool isGeoEnabled)
{
    m_userInfo.insert(QTweetUser::GeoEnabled, isGeoEnabled);
}

bool QTweetUser::isGeoEnabled() const
{
    return m_userInfo.value(QTweetUser::GeoEnabled, false).toBool();
}

void QTweetUser::setVerified(bool verified)
{
    m_userInfo.insert(QTweetUser::Verified, verified);
}

bool QTweetUser::isVerified() const
{
    return m_userInfo.value(QTweetUser::Verified, false).toBool();
}

void QTweetUser::setFollowing(bool following)
{
    m_userInfo.insert(QTweetUser::Following, following);
}

bool QTweetUser::isFollowing() const
{
    return m_userInfo.value(QTweetUser::Following, false).toBool();
}

void QTweetUser::setStatusesCount(int count)
{
    m_userInfo.insert(QTweetUser::StatusesCount, count);
}

int QTweetUser::statusesCount() const
{
    return m_userInfo.value(QTweetUser::StatusesCount).toInt();
}

QDateTime QTweetUser::twitterDateToQDateTime(const QString &twitterDate)
{
    //Twitter Date Format: 'Wed Sep 01 11:27:25 +0000 2010'  UTC
    QString dateString = twitterDate.left(10) + twitterDate.right(4);
    QString timeString = twitterDate.mid(11, 8);

    QDate date = QDate::fromString(dateString);
    QTime time = QTime::fromString(timeString);

    if (date.isValid() && time.isValid())
        return QDateTime(date, time, Qt::UTC);
    else
        return QDateTime();
}
<commit_msg>Fixed parsing twitter date (fixed by Rui Zhang).<commit_after>/* Copyright (c) 2010, Antonie Jovanoski
 *
 * All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 * Contact e-mail: Antonie Jovanoski <minimoog77_at_gmail.com>
 */

#include <QDateTime>
#include "qtweetuser.h"

QTweetUser::QTweetUser()
{
}

void QTweetUser::setId(qint64 id)
{
    m_userInfo.insert(QTweetUser::Id, id);
}

qint64 QTweetUser::id() const
{
    return m_userInfo.value(QTweetUser::Id).toLongLong();
}

void QTweetUser::setName(const QString &name)
{
    m_userInfo.insert(QTweetUser::Name, name);
}

QString QTweetUser::name() const
{
    return m_userInfo.value(QTweetUser::Name).toString();
}

void QTweetUser::setScreenName(const QString &screenName)
{
    m_userInfo.insert(QTweetUser::ScreenName, screenName);
}

QString QTweetUser::screenName() const
{
    return m_userInfo.value(QTweetUser::ScreenName).toString();
}

void QTweetUser::setLocation(const QString &location)
{
    m_userInfo.insert(QTweetUser::Location, location);
}

QString QTweetUser::location() const
{
    return m_userInfo.value(QTweetUser::Location).toString();
}

void QTweetUser::setDescription(const QString &desc)
{
    m_userInfo.insert(QTweetUser::Description, desc);
}

QString QTweetUser::description() const
{
    return m_userInfo.value(QTweetUser::Description).toString();
}

void QTweetUser::setprofileImageUrl(const QString &url)
{
    m_userInfo.insert(QTweetUser::ProfileImageUrl, url);
}

QString QTweetUser::profileImageUrl() const
{
    return m_userInfo.value(QTweetUser::ProfileImageUrl).toString();
}

void QTweetUser::setUrl(const QString &url)
{
    m_userInfo.insert(QTweetUser::Url, url);
}

QString QTweetUser::url() const
{
    return m_userInfo.value(QTweetUser::Url).toString();
}

void QTweetUser::setProtected(bool protected_)
{
    m_userInfo.insert(QTweetUser::Protected, protected_);
}

bool QTweetUser::isProtected() const
{
    return m_userInfo.value(QTweetUser::Protected).toBool();
}

void QTweetUser::setFollowersCount(int count)
{
    m_userInfo.insert(QTweetUser::FollowersCount, count);
}

int QTweetUser::followersCount() const
{
    return m_userInfo.value(QTweetUser::FollowersCount).toInt();
}

void QTweetUser::setFriendsCount(int count)
{
    m_userInfo.insert(QTweetUser::FriendsCount, count);
}

int QTweetUser::friendsCount() const
{
    return m_userInfo.value(QTweetUser::FriendsCount).toInt();
}

void QTweetUser::setCreatedAt(const QString &twitterDate)
{
    QDateTime datetime = twitterDateToQDateTime(twitterDate);

    m_userInfo.insert(QTweetUser::CreatedAt, datetime);
}

QDateTime QTweetUser::createdAt() const
{
    return m_userInfo.value(QTweetUser::CreatedAt).toDateTime();
}

void QTweetUser::setFavouritesCount(int count)
{
    m_userInfo.insert(QTweetUser::FavouritesCount, count);
}

int QTweetUser::favouritesCount() const
{
    return m_userInfo.value(QTweetUser::FavouritesCount).toInt();
}

void QTweetUser::setUtcOffset(int sec)
{
    m_userInfo.insert(QTweetUser::UtcOffset, sec);
}

int QTweetUser::utcOffset() const
{
    return m_userInfo.value(QTweetUser::UtcOffset).toInt();
}

void QTweetUser::setTimezone(const QString &timezone)
{
    m_userInfo.insert(QTweetUser::TimeZone, timezone);
}

QString QTweetUser::timezone() const
{
    return m_userInfo.value(QTweetUser::TimeZone).toString();
}

void QTweetUser::setGeoEnabled(bool isGeoEnabled)
{
    m_userInfo.insert(QTweetUser::GeoEnabled, isGeoEnabled);
}

bool QTweetUser::isGeoEnabled() const
{
    return m_userInfo.value(QTweetUser::GeoEnabled, false).toBool();
}

void QTweetUser::setVerified(bool verified)
{
    m_userInfo.insert(QTweetUser::Verified, verified);
}

bool QTweetUser::isVerified() const
{
    return m_userInfo.value(QTweetUser::Verified, false).toBool();
}

void QTweetUser::setFollowing(bool following)
{
    m_userInfo.insert(QTweetUser::Following, following);
}

bool QTweetUser::isFollowing() const
{
    return m_userInfo.value(QTweetUser::Following, false).toBool();
}

void QTweetUser::setStatusesCount(int count)
{
    m_userInfo.insert(QTweetUser::StatusesCount, count);
}

int QTweetUser::statusesCount() const
{
    return m_userInfo.value(QTweetUser::StatusesCount).toInt();
}

QDateTime QTweetUser::twitterDateToQDateTime(const QString &twitterDate)
{
    //Twitter Date Format: 'Wed Sep 01 11:27:25 +0000 2010'  UTC
    QString dateString = twitterDate.left(11) + twitterDate.right(4);
    QString timeString = twitterDate.mid(11, 8);

    QDate date = QDate::fromString(dateString);
    QTime time = QTime::fromString(timeString);

    if (date.isValid() && time.isValid())
        return QDateTime(date, time, Qt::UTC);
    else
        return QDateTime();
}
<|endoftext|>