<commit_before>#include <iostream>
#include <seqan/arg_parse.h>
#include <seqan/seq_io.h>

using namespace seqan;
using namespace std;

const char *PROGRAM_NAME = "biotool";
const char *HEADER = "FILENAME\tNUMSEQ\tTOTAL\tMIN\tAVG\tMAX";
const unsigned DEFAULT_MIN_LEN = 0;

typedef enum {Error_command_line=1, Error_open_file=2, Error_parse_file=3} exit_status;

struct BiotoolOptions
{
    unsigned minlen;
    bool verbose;
    bool version;
    vector<string> fasta_files;

    BiotoolOptions() :
    minlen(DEFAULT_MIN_LEN), verbose(false), version(false), fasta_files({})
    {}
};

void print_error(const char *message)
{
    cerr << PROGRAM_NAME << " ERROR: " << message << endl;
}

BiotoolOptions parse_options(int argc, char const **argv)
{
    ArgumentParser parser(PROGRAM_NAME);
    addArgument(parser, ArgParseArgument(
        ArgParseArgument::STRING, "FASTA_FILE", true));
    addOption(parser, ArgParseOption(
        "m", "minlen", "Minimum length sequence to include in stats (default " + to_string(DEFAULT_MIN_LEN) + ")",
        ArgParseArgument::INTEGER, "INT"));
    addOption(parser, ArgParseOption(
        "v", "version", "Display program version and exit"));
    addOption(parser, ArgParseOption(
        "b", "verbose", "Print more stuff about what's happening"));
    ArgumentParser::ParseResult res = parse(parser, argc, argv);

    if (res == ArgumentParser::PARSE_ERROR)
    {
        exit(Error_command_line);
    }

    BiotoolOptions options;
    getOptionValue(options.minlen, parser, "minlen");
    options.verbose = isSet(parser, "verbose");
    options.version = isSet(parser, "version");
    options.fasta_files = getArgumentValues(parser, 0);
    return options;
}

void process_files(BiotoolOptions options)
{
    CharString id;
    CharString seq;

    cout << HEADER << endl;

    for (string filename : options.fasta_files) {
	SeqFileIn seqFileIn;

        if (!open(seqFileIn, toCString(filename)))
        {
	    print_error("Could not open the file.");
            exit(Error_open_file);
        }

        int max_len = 0;
        int min_len = -1;
	int this_len;
        unsigned total_len = 0;
        unsigned num_seqs = 0;
	unsigned int average;

        while(!atEnd(seqFileIn))
        {
	    try
	    {
	        readRecord(id, seq, seqFileIn);
	    }
            catch (Exception const & e)
            {
	        print_error(e.what());
                exit(Error_parse_file);
            }
	    this_len = length(seq);
	    if (this_len > options.minlen)
            {
	        num_seqs++;
	        total_len += this_len;
	        max_len = max(max_len, this_len);
	        if (min_len < 0 || this_len < min_len)
                {
                    min_len = this_len;
                }
	    }
        }
	if (num_seqs > 0)
        {
            average = (unsigned int) floor((double) total_len / (double) num_seqs);
	    cout << filename << '\t' << num_seqs << '\t' \
		 << min_len << '\t' << average << '\t' << max_len << endl;
        }
	else
        {
	    cout << filename << "\t0\t0\t-\t-\t-" << endl;
        }
    }
}

int main(int argc, char const **argv)
{
    BiotoolOptions options = parse_options(argc, argv);
    process_files(options);

    return 0;
}
<commit_msg>Make CPP version exit cleanly if --help is supplied on command line<commit_after>#include <iostream>
#include <seqan/arg_parse.h>
#include <seqan/seq_io.h>

using namespace seqan;
using namespace std;

const char *PROGRAM_NAME = "biotool";
const char *HEADER = "FILENAME\tNUMSEQ\tTOTAL\tMIN\tAVG\tMAX";
const unsigned DEFAULT_MIN_LEN = 0;

typedef enum {Success=0, Error_command_line=1, Error_open_file=2, Error_parse_file=3} exit_status;

struct BiotoolOptions
{
    unsigned minlen;
    bool verbose;
    bool version;
    vector<string> fasta_files;

    BiotoolOptions() :
    minlen(DEFAULT_MIN_LEN), verbose(false), version(false), fasta_files({})
    {}
};

void print_error(const char *message)
{
    cerr << PROGRAM_NAME << " ERROR: " << message << endl;
}

BiotoolOptions parse_options(int argc, char const **argv)
{
    ArgumentParser parser(PROGRAM_NAME);
    addArgument(parser, ArgParseArgument(
        ArgParseArgument::STRING, "FASTA_FILE", true));
    addOption(parser, ArgParseOption(
        "m", "minlen", "Minimum length sequence to include in stats (default " + to_string(DEFAULT_MIN_LEN) + ")",
        ArgParseArgument::INTEGER, "INT"));
    addOption(parser, ArgParseOption(
        "v", "version", "Display program version and exit"));
    addOption(parser, ArgParseOption(
        "b", "verbose", "Print more stuff about what's happening"));
    ArgumentParser::ParseResult res = parse(parser, argc, argv);

    //if (res == ArgumentParser::PARSE_ERROR)
    if (res != ArgumentParser::PARSE_OK)
    {
	if (res == ArgumentParser::PARSE_ERROR) {
           exit(Error_command_line);
	}
	else {
           exit(Success);
        }
    }

    BiotoolOptions options;
    getOptionValue(options.minlen, parser, "minlen");
    options.verbose = isSet(parser, "verbose");
    options.version = isSet(parser, "version");
    options.fasta_files = getArgumentValues(parser, 0);
    return options;
}

void process_files(BiotoolOptions options)
{
    CharString id;
    CharString seq;

    cout << HEADER << endl;

    for (string filename : options.fasta_files) {
	SeqFileIn seqFileIn;

        if (!open(seqFileIn, toCString(filename)))
        {
	    print_error("Could not open the file.");
            exit(Error_open_file);
        }

        int max_len = 0;
        int min_len = -1;
	int this_len;
        unsigned total_len = 0;
        unsigned num_seqs = 0;
	unsigned int average;

        while(!atEnd(seqFileIn))
        {
	    try
	    {
	        readRecord(id, seq, seqFileIn);
	    }
            catch (Exception const & e)
            {
	        print_error(e.what());
                exit(Error_parse_file);
            }
	    this_len = length(seq);
	    if (this_len > options.minlen)
            {
	        num_seqs++;
	        total_len += this_len;
	        max_len = max(max_len, this_len);
	        if (min_len < 0 || this_len < min_len)
                {
                    min_len = this_len;
                }
	    }
        }
	if (num_seqs > 0)
        {
            average = (unsigned int) floor((double) total_len / (double) num_seqs);
	    cout << filename << '\t' << num_seqs << '\t' \
		 << min_len << '\t' << average << '\t' << max_len << endl;
        }
	else
        {
	    cout << filename << "\t0\t0\t-\t-\t-" << endl;
        }
    }
}

int main(int argc, char const **argv)
{
    BiotoolOptions options = parse_options(argc, argv);
    process_files(options);

    return 0;
}
<|endoftext|>


<commit_before>#include "Item.h"

namespace Analytics {

uint qHash(const AssociationRule & r) {
    QString s;
    foreach (const ItemID & id, r.antecedent)
        s += QString::number(id) + ':';
    s += "=>";
    foreach (const ItemID & id, r.consequent)
        s += QString::number(id) + ':';
    return qHash(s);
}

#ifdef DEBUG
    QDebug operator<<(QDebug dbg, const Item & i) {
        dbg.nospace() << i.IDNameHash->value(i.id).toStdString().c_str()
                << "("
                << i.id
                << ")="
                << i.supportCount;

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const ItemIDList & pattern) {
        dbg.nospace() << "{";

        for (int i = 0; i < pattern.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            dbg.nospace() << pattern[i];
        }
        dbg.nospace() << "}";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const Transaction & transaction) {
        QString itemOutput;

        dbg.nospace() << "[size=" << transaction.size() << "] {";

        for (int i = 0; i < transaction.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            // Generate output for item.
            itemOutput.clear();
            QDebug(&itemOutput) << transaction[i];

            dbg.nospace() << itemOutput.toStdString().c_str();
        }
        dbg.nospace() << "}";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const FrequentItemset & frequentItemset) {
        QString itemOutput;

        dbg.nospace() << "({";
        itemIDHelper(dbg, frequentItemset.itemset, frequentItemset.IDNameHash);
        dbg.nospace() << "}, sup: "
                      << frequentItemset.support
                      << ")";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const AssociationRule & associationRule) {
        dbg.nospace() << "{";
        itemIDHelper(dbg, associationRule.antecedent, associationRule.IDNameHash);
        dbg.nospace() << "} => {";
        itemIDHelper(dbg, associationRule.consequent, associationRule.IDNameHash);
        dbg.nospace() << "}";

        dbg.nospace() << " ("
                      << "sup=" << associationRule.support
                      << ", conf=" << associationRule.confidence
                      << ")";

        return dbg.nospace();
    }

    QDebug itemIDHelper(QDebug dbg, const ItemIDList & itemset, ItemIDNameHash const * const IDNameHash) {
        for (int i = 0; i < itemset.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            if (IDNameHash != NULL) {
                dbg.nospace() << IDNameHash->value(itemset[i]).toStdString().c_str()
                        << "("
                        << itemset[i]
                        << ")";
            }
            else
                dbg.nospace() << itemset[i];
        }

        return dbg.nospace();
    }

#endif

}
<commit_msg>Minor formatting fix in Item.cpp.<commit_after>#include "Item.h"

namespace Analytics {

    uint qHash(const AssociationRule & r) {
        QString s;
        foreach (const ItemID & id, r.antecedent)
            s += QString::number(id) + ':';
        s += "=>";
        foreach (const ItemID & id, r.consequent)
            s += QString::number(id) + ':';
        return qHash(s);
    }

#ifdef DEBUG
    QDebug operator<<(QDebug dbg, const Item & i) {
        dbg.nospace() << i.IDNameHash->value(i.id).toStdString().c_str()
                << "("
                << i.id
                << ")="
                << i.supportCount;

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const ItemIDList & pattern) {
        dbg.nospace() << "{";

        for (int i = 0; i < pattern.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            dbg.nospace() << pattern[i];
        }
        dbg.nospace() << "}";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const Transaction & transaction) {
        QString itemOutput;

        dbg.nospace() << "[size=" << transaction.size() << "] {";

        for (int i = 0; i < transaction.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            // Generate output for item.
            itemOutput.clear();
            QDebug(&itemOutput) << transaction[i];

            dbg.nospace() << itemOutput.toStdString().c_str();
        }
        dbg.nospace() << "}";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const FrequentItemset & frequentItemset) {
        QString itemOutput;

        dbg.nospace() << "({";
        itemIDHelper(dbg, frequentItemset.itemset, frequentItemset.IDNameHash);
        dbg.nospace() << "}, sup: "
                      << frequentItemset.support
                      << ")";

        return dbg.nospace();
    }

    QDebug operator<<(QDebug dbg, const AssociationRule & associationRule) {
        dbg.nospace() << "{";
        itemIDHelper(dbg, associationRule.antecedent, associationRule.IDNameHash);
        dbg.nospace() << "} => {";
        itemIDHelper(dbg, associationRule.consequent, associationRule.IDNameHash);
        dbg.nospace() << "}";

        dbg.nospace() << " ("
                      << "sup=" << associationRule.support
                      << ", conf=" << associationRule.confidence
                      << ")";

        return dbg.nospace();
    }

    QDebug itemIDHelper(QDebug dbg, const ItemIDList & itemset, ItemIDNameHash const * const IDNameHash) {
        for (int i = 0; i < itemset.size(); i++) {
            if (i > 0)
                dbg.nospace() << ", ";

            if (IDNameHash != NULL) {
                dbg.nospace() << IDNameHash->value(itemset[i]).toStdString().c_str()
                        << "("
                        << itemset[i]
                        << ")";
            }
            else
                dbg.nospace() << itemset[i];
        }

        return dbg.nospace();
    }

#endif

}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "JoinedQuery.h"
#include "CTRCounter.h"

using namespace fnord;

typedef Tuple<
    String,
    String,
    uint64_t,
    uint64_t,
    uint64_t,
    double,
    double,
    double,
    double,
    double,
    double,
    double,
    double> OutputRow;

typedef HashMap<String, cm::CTRCounter> CounterMap;

/* read all input sstables */
void importInputTables(const Vector<String> sstables, CounterMap* counters) {
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    const auto& sstable = sstables[tbl_idx];
    fnord::logInfo("cm.ctrstats", "Importing sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.ctrstats", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();
    int row_idx = 0;

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      fnord::logInfo(
          "cm.ctrstats",
          "[$1/$2] [$0%] Reading sstable... rows=$3",
          (size_t) ((cursor->position() / (double) body_size) * 100),
          tbl_idx + 1, sstables.size(), row_idx);
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();
      sstable::SSTableColumnReader cols(&schema, val);

      auto& counter = (*counters)[key];
      auto num_views = cols.getUInt64Column(schema.columnID("num_views"));
      auto num_clicks = cols.getUInt64Column(schema.columnID("num_clicks"));
      auto num_clicked = cols.getUInt64Column(schema.columnID("num_clicked"));
      counter.num_views += num_views;
      counter.num_clicks += num_clicks;
      counter.num_clicked += num_clicked;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }
}

/* write output table */
void writeOutputTable(const String& filename, const Vector<OutputRow>& rows) {
  /* prepare output sstable schema */
  sstable::SSTableColumnSchema schema;
  schema.addColumn("dim1", 1, sstable::SSTableColumnType::STRING);
  schema.addColumn("num_views", 2, sstable::SSTableColumnType::UINT64);
  schema.addColumn("num_clicks", 3, sstable::SSTableColumnType::UINT64);
  schema.addColumn("num_clicked", 4, sstable::SSTableColumnType::UINT64);
  schema.addColumn("ctr", 5, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("cpq", 6, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_view_base", 7, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_click_base", 8, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_clicked_base", 9, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_view_dim1", 10, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_click_dim1", 11, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_clicked_dim1", 12, sstable::SSTableColumnType::FLOAT);

  /* open output sstable */
  fnord::logInfo("cm.ctrstats", "Writing results to: $0", filename);
  auto sstable_writer = sstable::SSTableWriter::create(
      filename,
      sstable::IndexProvider{},
      nullptr,
      0);

  /* write output sstable */
  for (const auto& r : rows) {
    sstable::SSTableColumnWriter cols(&schema);

    cols.addStringColumn(1, std::get<1>(r));
    cols.addUInt64Column(2, std::get<2>(r));
    cols.addUInt64Column(3, std::get<3>(r));
    cols.addUInt64Column(4, std::get<4>(r));
    cols.addFloatColumn(5, std::get<5>(r));
    cols.addFloatColumn(6, std::get<6>(r));
    cols.addFloatColumn(7, std::get<7>(r));
    cols.addFloatColumn(8, std::get<8>(r));
    cols.addFloatColumn(9, std::get<9>(r));
    cols.addFloatColumn(10, std::get<10>(r));
    cols.addFloatColumn(11, std::get<11>(r));
    cols.addFloatColumn(12, std::get<12>(r));

    sstable_writer->appendRow(std::get<0>(r), cols);
  }

  schema.writeIndex(sstable_writer.get());
  sstable_writer->finalize();
}

/* aggregate ctr counters (flat) */
void aggregateCounters(CounterMap* counters, Vector<OutputRow>* rows) {
  const auto& global_counter_iter = counters->find("__GLOBAL");
  if (global_counter_iter == counters->end()) {
    fnord::logCritical("cm.ctrstatsexport", "missing global counter");
    exit(1);
  }
  const auto& global_counter = global_counter_iter->second;

  for (const auto& row : *counters) {
    const auto& c = row.second;

    auto sep = row.first.find("~");
    if (sep == std::string::npos) {
      continue;
    }
    auto dim1 = row.first.substr(0, sep);
    auto dim2 = row.first.substr(sep + 1);

    if (dim1.length() == 0 || dim2.length() == 0) {
      continue;
    }

    const auto& group_counter_iter = counters->find(dim1);
    if (group_counter_iter == counters->end()) {
      fnord::logWarning("cm.ctrstatsexport", "missing row: $0", dim1);
      continue;
    }
    const auto& group_counter = group_counter_iter->second;

    double ctr = c.num_clicked / (double) c.num_views;
    double cpq = c.num_clicks / (double) c.num_views;
    double p_view_base = c.num_views / (double) global_counter.num_views;
    double p_click_base = c.num_clicks / (double) global_counter.num_clicks;
    double p_clicked_base = c.num_clicked / (double) global_counter.num_clicked;
    double p_view_dim1 = c.num_views / (double) group_counter.num_views;
    double p_click_dim1 = c.num_clicks / (double) group_counter.num_clicks;
    double p_clicked_dim1 = c.num_clicked / (double) group_counter.num_clicked;

    rows->emplace_back(
        dim1,
        dim2,
        row.second.num_views,
        row.second.num_clicks,
        row.second.num_clicked,
        ctr,
        cpq,
        p_view_base,
        p_click_base,
        p_clicked_base,
        p_view_dim1,
        p_click_dim1,
        p_clicked_dim1);
  }
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_file",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* read input tables */
  HashMap<String, cm::CTRCounter> counters;
  importInputTables(flags.getArgv(), &counters);

  /* aggregate counters */
  Vector<OutputRow> rows;
  aggregateCounters(&counters, &rows);

  /* write output table */
  writeOutputTable(flags.getString("output_file"), rows);

  return 0;
}

<commit_msg>calculate perf_{base,dim1} score in rollup2d<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "JoinedQuery.h"
#include "CTRCounter.h"

using namespace fnord;

typedef Tuple<
    String,
    String,
    uint64_t,
    uint64_t,
    uint64_t,
    double,
    double,
    double,
    double,
    double,
    double,
    double,
    double,
    double,
    double> OutputRow;

typedef HashMap<String, cm::CTRCounter> CounterMap;

/* read all input sstables */
void importInputTables(const Vector<String> sstables, CounterMap* counters) {
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    const auto& sstable = sstables[tbl_idx];
    fnord::logInfo("cm.ctrstats", "Importing sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.ctrstats", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();
    int row_idx = 0;

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      fnord::logInfo(
          "cm.ctrstats",
          "[$1/$2] [$0%] Reading sstable... rows=$3",
          (size_t) ((cursor->position() / (double) body_size) * 100),
          tbl_idx + 1, sstables.size(), row_idx);
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();
      sstable::SSTableColumnReader cols(&schema, val);

      auto& counter = (*counters)[key];
      auto num_views = cols.getUInt64Column(schema.columnID("num_views"));
      auto num_clicks = cols.getUInt64Column(schema.columnID("num_clicks"));
      auto num_clicked = cols.getUInt64Column(schema.columnID("num_clicked"));
      counter.num_views += num_views;
      counter.num_clicks += num_clicks;
      counter.num_clicked += num_clicked;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }
}

/* write output table */
void writeOutputTable(const String& filename, const Vector<OutputRow>& rows) {
  /* prepare output sstable schema */
  sstable::SSTableColumnSchema schema;
  schema.addColumn("dim1", 1, sstable::SSTableColumnType::STRING);
  schema.addColumn("num_views", 2, sstable::SSTableColumnType::UINT64);
  schema.addColumn("num_clicks", 3, sstable::SSTableColumnType::UINT64);
  schema.addColumn("num_clicked", 4, sstable::SSTableColumnType::UINT64);
  schema.addColumn("ctr", 5, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("cpq", 6, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_view_base", 7, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_click_base", 8, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_clicked_base", 9, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_view_dim1", 10, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_click_dim1", 11, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("p_clicked_dim1", 12, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("perf_base", 13, sstable::SSTableColumnType::FLOAT);
  schema.addColumn("perf_dim1", 14, sstable::SSTableColumnType::FLOAT);

  /* open output sstable */
  fnord::logInfo("cm.ctrstats", "Writing results to: $0", filename);
  auto sstable_writer = sstable::SSTableWriter::create(
      filename,
      sstable::IndexProvider{},
      nullptr,
      0);

  /* write output sstable */
  for (const auto& r : rows) {
    sstable::SSTableColumnWriter cols(&schema);

    cols.addStringColumn(1, std::get<1>(r));
    cols.addUInt64Column(2, std::get<2>(r));
    cols.addUInt64Column(3, std::get<3>(r));
    cols.addUInt64Column(4, std::get<4>(r));
    cols.addFloatColumn(5, std::get<5>(r));
    cols.addFloatColumn(6, std::get<6>(r));
    cols.addFloatColumn(7, std::get<7>(r));
    cols.addFloatColumn(8, std::get<8>(r));
    cols.addFloatColumn(9, std::get<9>(r));
    cols.addFloatColumn(10, std::get<10>(r));
    cols.addFloatColumn(11, std::get<11>(r));
    cols.addFloatColumn(12, std::get<12>(r));
    cols.addFloatColumn(13, std::get<13>(r));
    cols.addFloatColumn(14, std::get<14>(r));

    sstable_writer->appendRow(std::get<0>(r), cols);
  }

  schema.writeIndex(sstable_writer.get());
  sstable_writer->finalize();
}

/* aggregate ctr counters (flat) */
void aggregateCounters(CounterMap* counters, Vector<OutputRow>* rows) {
  const auto& global_counter_iter = counters->find("__GLOBAL");
  if (global_counter_iter == counters->end()) {
    fnord::logCritical("cm.ctrstatsexport", "missing global counter");
    exit(1);
  }
  const auto& global_counter = global_counter_iter->second;

  for (const auto& row : *counters) {
    const auto& c = row.second;

    auto sep = row.first.find("~");
    if (sep == std::string::npos) {
      continue;
    }
    auto dim1 = row.first.substr(0, sep);
    auto dim2 = row.first.substr(sep + 1);

    if (dim1.length() == 0 || dim2.length() == 0) {
      continue;
    }

    const auto& group_counter_iter = counters->find(dim1);
    if (group_counter_iter == counters->end()) {
      fnord::logWarning("cm.ctrstatsexport", "missing row: $0", dim1);
      continue;
    }
    const auto& group_counter = group_counter_iter->second;

    double ctr = c.num_clicked / (double) c.num_views;
    double cpq = c.num_clicks / (double) c.num_views;
    double p_view_base = c.num_views / (double) global_counter.num_views;
    double p_click_base = c.num_clicks / (double) global_counter.num_clicks;
    double p_clicked_base = c.num_clicked / (double) global_counter.num_clicked;
    double perf_base = p_click_base / p_view_base;
    double p_view_dim1 = c.num_views / (double) group_counter.num_views;
    double p_click_dim1 = c.num_clicks / (double) group_counter.num_clicks;
    double p_clicked_dim1 = c.num_clicked / (double) group_counter.num_clicked;
    double perf_dim1 = p_click_dim1 / p_view_dim1;

    rows->emplace_back(
        dim1,
        dim2,
        row.second.num_views,
        row.second.num_clicks,
        row.second.num_clicked,
        ctr,
        cpq,
        p_view_base,
        p_click_base,
        p_clicked_base,
        p_view_dim1,
        p_click_dim1,
        p_clicked_dim1,
        perf_base,
        perf_dim1);
  }
}

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_file",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "output file path",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  /* read input tables */
  HashMap<String, cm::CTRCounter> counters;
  importInputTables(flags.getArgv(), &counters);

  /* aggregate counters */
  Vector<OutputRow> rows;
  aggregateCounters(&counters, &rows);

  /* write output table */
  writeOutputTable(flags.getString("output_file"), rows);

  return 0;
}

<|endoftext|>


<commit_before>// FRAGMENT(includes)
#include <iostream>
#include <seqan/align.h>
#include <seqan/index.h>
#include <seqan/misc/misc_random.h>

using namespace seqan;

// FRAGMENT(matrix_init)
template <typename TStringSet, typename TIndexSpec>
void kmerCounting(TStringSet &set, TIndexSpec)
{
	typedef Index<TStringSet, TIndexSpec> TIndex;
	typedef typename Fibre<TIndex, QGram_Counts>::Type TCounts;
	typedef typename Fibre<TIndex, QGram_CountsDir>::Type TCountsDir;
	typedef typename Value<TCountsDir>::Type TDirValue;
	typedef typename Iterator<TCounts, Standard>::Type TIterCounts;
	typedef typename Iterator<TCountsDir, Standard>::Type TIterCountsDir;

	TIndex index(set);
	indexRequire(index, QGram_Counts());

	// initialize distance matrix
	int seqNum = countSequences(index);
	Matrix<int, 2> distMat;
	setLength(distMat, 0, seqNum);
	setLength(distMat, 1, seqNum);
	fill(distMat, 0);

	std::cout << std::endl << "Length of the CountsDir fibre: " << length(indexCountsDir(index)) << std::endl;
	TIterCountsDir itCountsDir = begin(indexCountsDir(index), Standard());
	TIterCountsDir itCountsDirEnd = end(indexCountsDir(index), Standard());
	TIterCounts itCountsBegin = begin(indexCounts(index), Standard());

// FRAGMENT(matrix_calculation)
	// for each bucket count common q-grams for each sequence pair
	TDirValue bucketBegin = *itCountsDir;
	for(++itCountsDir; itCountsDir != itCountsDirEnd; ++itCountsDir)
	{
		TDirValue bucketEnd = *itCountsDir;

		// q-gram must occur in at least 2 different sequences
		if (bucketBegin != bucketEnd)
		{
			TIterCounts itA = itCountsBegin + bucketBegin;
			TIterCounts itEnd = itCountsBegin + bucketEnd;
			for(; itA != itEnd; ++itA)
				for(TIterCounts itB = itA; itB != itEnd; ++itB)
					distMat((*itA).i1, (*itB).i1) += _min((*itA).i2, (*itB).i2);
		}
		bucketBegin = bucketEnd;
	}

	std::cout << std::endl << "Common 5-mers for Seq_i, Seq_j" << std::endl;
	std::cout << distMat;
}

// FRAGMENT(initialization)
int main ()
{
	mtRandInit();

	// create StringSet of 3 random sequences
	StringSet<DnaString> stringSet;
	reserve(stringSet, 3);
	for (int seqNo = 0; seqNo < 3; ++seqNo)
	{
		DnaString tmp;
		int len = mtRand() % 100 + 10;
		for (int i = 0; i < len; ++i)
			appendValue(tmp, Dna(mtRand() % 4));
		appendValue(stringSet, tmp);
		std::cout << ">Seq" << seqNo << std::endl << tmp << std::endl;
	}

	kmerCounting(stringSet, Index_QGram<UngappedShape<5> >());
	kmerCounting(stringSet, Index_QGram<UngappedShape<5>, OpenAddressing>());
	return 0;
}

<commit_msg>added tut<commit_after>// FRAGMENT(includes)
#include <iostream>
#include <seqan/align.h>
#include <seqan/index.h>
#include <seqan/misc/misc_random.h>

using namespace seqan;

// FRAGMENT(matrix_init)
template <typename TStringSet, typename TIndexSpec>
void kmerCounting(TStringSet &set, TIndexSpec)
{
	typedef Index<TStringSet, TIndexSpec> TIndex;
	typedef typename Fibre<TIndex, QGram_Counts>::Type TCounts;
	typedef typename Fibre<TIndex, QGram_CountsDir>::Type TCountsDir;
	typedef typename Value<TCountsDir>::Type TDirValue;
	typedef typename Iterator<TCounts, Standard>::Type TIterCounts;
	typedef typename Iterator<TCountsDir, Standard>::Type TIterCountsDir;

	TIndex index(set);
	indexRequire(index, QGram_Counts());

	// initialize distance matrix
	int seqNum = countSequences(index);
	Matrix<int, 2> distMat;
	setLength(distMat, 0, seqNum);
	setLength(distMat, 1, seqNum);
	fill(distMat, 0);

	std::cout << std::endl << "Length of the CountsDir fibre: " << length(indexCountsDir(index)) << std::endl;
	TIterCountsDir itCountsDir = begin(indexCountsDir(index), Standard());
	TIterCountsDir itCountsDirEnd = end(indexCountsDir(index), Standard());
	TIterCounts itCountsBegin = begin(indexCounts(index), Standard());

// FRAGMENT(matrix_calculation)
	// for each bucket count common q-grams for each sequence pair
	TDirValue bucketBegin = *itCountsDir;
	for(++itCountsDir; itCountsDir != itCountsDirEnd; ++itCountsDir)
	{
		TDirValue bucketEnd = *itCountsDir;

		// q-gram must occur in at least 2 different sequences
		if (bucketBegin != bucketEnd)
		{
			TIterCounts itA = itCountsBegin + bucketBegin;
			TIterCounts itEnd = itCountsBegin + bucketEnd;
			for(; itA != itEnd; ++itA)
				for(TIterCounts itB = itA; itB != itEnd; ++itB)
					distMat((*itA).i1, (*itB).i1) += _min((*itA).i2, (*itB).i2);
		}
		bucketBegin = bucketEnd;
	}

	std::cout << std::endl << "Common 5-mers for Seq_i, Seq_j" << std::endl;
	std::cout << distMat;
}

// FRAGMENT(initialization)
int main ()
{
	mtRandInit();

	// create StringSet of 3 random sequences
	StringSet<DnaString> stringSet;
	reserve(stringSet, 3);
	for (int seqNo = 0; seqNo < 3; ++seqNo)
	{
		DnaString tmp;
		int len = mtRand() % 100 + 10;
		for (int i = 0; i < len; ++i)
			appendValue(tmp, Dna(mtRand() % 4));
		appendValue(stringSet, tmp);
		std::cout << ">Seq" << seqNo << std::endl << tmp << std::endl;
	}

	kmerCounting(stringSet, Index_QGram<UngappedShape<5> >());
	kmerCounting(stringSet, Index_QGram<UngappedShape<5>, OpenAddressing>());
	return 0;
}

<|endoftext|>


<commit_before>#include <stdio.h>
#include "Halide.h"

using namespace Halide;
using namespace Halide::Internal;

class FindPowerHvxOn : public IRVisitor {
public:
    bool found = false;

    void visit(const Call *op) {
        if (op->name == "halide_hexagon_power_hvx_on") {
            found = true;
        }
        IRVisitor::visit(op);
    }
};

class ExpectPowerHvxOn : public IRMutator {
    bool expected;

public:
    ExpectPowerHvxOn(bool expected) : expected(expected) {}
    using IRMutator::mutate;

    Stmt mutate(Stmt s) {
        FindPowerHvxOn v;
        s.accept(&v);

        if (v.found && !expected) {
            printf("Found halide_hexagon_power_hvx_on call when it was not expected.\n");
            exit(-1);
        } else if (!v.found && expected) {
            printf("Did not find halide_hexagon_power_hvx_on call when it was expected.\n");
            exit(-1);
        }

        return s;
    }
};

int main(int argc, char **argv) {

    //int W = 64*3, H = 64*3;
    const int W = 128, H = 48;

    Buffer<uint16_t> in(W, H);
    for (int y = 0; y < H; y++) {
        for (int x = 0; x < W; x++) {
            in(x, y) = rand() & 0xff;
        }
    }

    Var x("x"), y("y");

    Buffer<uint16_t> tent(3, 3);
    tent(0, 0) = 1;
    tent(0, 1) = 2;
    tent(0, 2) = 1;
    tent(1, 0) = 2;
    tent(1, 1) = 4;
    tent(1, 2) = 2;
    tent(2, 0) = 1;
    tent(2, 1) = 2;
    tent(2, 2) = 1;

    Func input("input");
    input(x, y) = in(clamp(x, 0, W-1), clamp(y, 0, H-1));
    input.compute_root();

    RDom r(tent);

    /* This first zeros blur1, and then accumulates into it. I.e. the for loops look like:
     * for y:
     *   for x:
     *     blur1(x, y) = 0
     * for y:
     *   for x:
     *     for r.y:
     *       for r.x:
     *         blur1(x, y) += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1)
     *
     * In general, reductions iterate over the reduction domain outermost.
     */
    Func blur1("blur1");
    blur1(x, y) += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1);

    /* This uses an inline reduction, and is the more traditional way
     * of scheduling a convolution. "sum" creates an anonymous
     * reduction function that is computed within the for loop over x
     * in blur2. blur2 isn't actually a reduction. The loop nest looks like:
     * for y:
     *   for x:
     *     tmp = 0
     *     for r.y:
     *       for r.x:
     *         tmp += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1)
     *     blur(x, y) = tmp
     */
    Func blur2("blur2");
    blur2(x, y) = sum(tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1));

    Target target = get_jit_target_from_environment();
    if (target.has_gpu_feature()) {
        Var xi("xi"), yi("yi");

        // Initialization (basically memset) done in a GPU kernel
        blur1.gpu_tile(x, y, xi, yi, 16, 16);

        // Summation is done as an outermost loop on the cpu
        blur1.update().reorder(x, y, r.x, r.y).gpu_tile(x, y, xi, yi, 16, 16);

        // Summation is done as a sequential loop within each gpu thread
        blur2.gpu_tile(x, y, xi, yi, 16, 16);
    } else if (target.has_feature(Target::HVX_64) || target.has_feature(Target::HVX_128)) {
        int hvx_vector_width = target.has_feature(Target::HVX_128) ? 64 : 32;
        // Take this opportunity to test scheduling the pure dimensions in a reduction
        Var xi("xi"), yi("yi");
        blur1.hexagon().tile(x, y, xi, yi, 6, 6);
        // TODO: Add parallel to the schedule.
        blur1.update().hexagon().tile(x, y, xi, yi, hvx_vector_width, 4).vectorize(xi);

        // TODO: Add parallel to the schedule.
        blur2.hexagon().vectorize(x, hvx_vector_width);

        // blur1 has two hexagon kernels, so it should have a halide_hexagon_power_hvx_on call.
        // blur2 has one hexagon kernel, so it should not.
        blur1.add_custom_lowering_pass(new ExpectPowerHvxOn(true));
        blur2.add_custom_lowering_pass(new ExpectPowerHvxOn(false));
    } else {
        // Take this opportunity to test scheduling the pure dimensions in a reduction
        Var xi("xi"), yi("yi");
        blur1.tile(x, y, xi, yi, 6, 6);
        blur1.update().tile(x, y, xi, yi, 4, 4).vectorize(xi).parallel(y);

        blur2.vectorize(x, 4).parallel(y);
    }

    Buffer<uint16_t> out1 = blur1.realize(W, H, target);
    Buffer<uint16_t> out2 = blur2.realize(W, H, target);

    for (int y = 1; y < H-1; y++) {
        for (int x = 1; x < W-1; x++) {
            uint16_t correct = (1*in(x-1, y-1) + 2*in(x, y-1) + 1*in(x+1, y-1) +
                                2*in(x-1, y)   + 4*in(x, y) +   2*in(x+1, y) +
                                1*in(x-1, y+1) + 2*in(x, y+1) + 1*in(x+1, y+1));
            if (out1(x, y) != correct) {
                printf("out1(%d, %d) = %d instead of %d\n", x, y, out1(x, y), correct);
                return -1;
            }
            if (out2(x, y) != correct) {
                printf("out2(%d, %d) = %d instead of %d\n", x, y, out2(x, y), correct);
                return -1;
            }
        }
    }

    printf("Success!\n");

    return 0;

}
<commit_msg>Fix missing visitors.<commit_after>#include <stdio.h>
#include "Halide.h"

using namespace Halide;
using namespace Halide::Internal;

class FindPowerHvxOn : public IRVisitor {
public:
    bool found = false;

    void visit(const Call *op) {
        if (op->name == "halide_hexagon_power_hvx_on") {
            found = true;
        }
        IRVisitor::visit(op);
    }

    using IRVisitor::visit;
};

class ExpectPowerHvxOn : public IRMutator {
    bool expected;

public:
    ExpectPowerHvxOn(bool expected) : expected(expected) {}
    using IRMutator::mutate;

    Stmt mutate(Stmt s) {
        FindPowerHvxOn v;
        s.accept(&v);

        if (v.found && !expected) {
            printf("Found halide_hexagon_power_hvx_on call when it was not expected.\n");
            exit(-1);
        } else if (!v.found && expected) {
            printf("Did not find halide_hexagon_power_hvx_on call when it was expected.\n");
            exit(-1);
        }

        return s;
    }

    using IRMutator::mutate;
};

int main(int argc, char **argv) {

    //int W = 64*3, H = 64*3;
    const int W = 128, H = 48;

    Buffer<uint16_t> in(W, H);
    for (int y = 0; y < H; y++) {
        for (int x = 0; x < W; x++) {
            in(x, y) = rand() & 0xff;
        }
    }

    Var x("x"), y("y");

    Buffer<uint16_t> tent(3, 3);
    tent(0, 0) = 1;
    tent(0, 1) = 2;
    tent(0, 2) = 1;
    tent(1, 0) = 2;
    tent(1, 1) = 4;
    tent(1, 2) = 2;
    tent(2, 0) = 1;
    tent(2, 1) = 2;
    tent(2, 2) = 1;

    Func input("input");
    input(x, y) = in(clamp(x, 0, W-1), clamp(y, 0, H-1));
    input.compute_root();

    RDom r(tent);

    /* This first zeros blur1, and then accumulates into it. I.e. the for loops look like:
     * for y:
     *   for x:
     *     blur1(x, y) = 0
     * for y:
     *   for x:
     *     for r.y:
     *       for r.x:
     *         blur1(x, y) += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1)
     *
     * In general, reductions iterate over the reduction domain outermost.
     */
    Func blur1("blur1");
    blur1(x, y) += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1);

    /* This uses an inline reduction, and is the more traditional way
     * of scheduling a convolution. "sum" creates an anonymous
     * reduction function that is computed within the for loop over x
     * in blur2. blur2 isn't actually a reduction. The loop nest looks like:
     * for y:
     *   for x:
     *     tmp = 0
     *     for r.y:
     *       for r.x:
     *         tmp += tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1)
     *     blur(x, y) = tmp
     */
    Func blur2("blur2");
    blur2(x, y) = sum(tent(r.x, r.y) * input(x + r.x - 1, y + r.y - 1));

    Target target = get_jit_target_from_environment();
    if (target.has_gpu_feature()) {
        Var xi("xi"), yi("yi");

        // Initialization (basically memset) done in a GPU kernel
        blur1.gpu_tile(x, y, xi, yi, 16, 16);

        // Summation is done as an outermost loop on the cpu
        blur1.update().reorder(x, y, r.x, r.y).gpu_tile(x, y, xi, yi, 16, 16);

        // Summation is done as a sequential loop within each gpu thread
        blur2.gpu_tile(x, y, xi, yi, 16, 16);
    } else if (target.has_feature(Target::HVX_64) || target.has_feature(Target::HVX_128)) {
        int hvx_vector_width = target.has_feature(Target::HVX_128) ? 64 : 32;
        // Take this opportunity to test scheduling the pure dimensions in a reduction
        Var xi("xi"), yi("yi");
        blur1.hexagon().tile(x, y, xi, yi, 6, 6);
        // TODO: Add parallel to the schedule.
        blur1.update().hexagon().tile(x, y, xi, yi, hvx_vector_width, 4).vectorize(xi);

        // TODO: Add parallel to the schedule.
        blur2.hexagon().vectorize(x, hvx_vector_width);

        // blur1 has two hexagon kernels, so it should have a halide_hexagon_power_hvx_on call.
        // blur2 has one hexagon kernel, so it should not.
        blur1.add_custom_lowering_pass(new ExpectPowerHvxOn(true));
        blur2.add_custom_lowering_pass(new ExpectPowerHvxOn(false));
    } else {
        // Take this opportunity to test scheduling the pure dimensions in a reduction
        Var xi("xi"), yi("yi");
        blur1.tile(x, y, xi, yi, 6, 6);
        blur1.update().tile(x, y, xi, yi, 4, 4).vectorize(xi).parallel(y);

        blur2.vectorize(x, 4).parallel(y);
    }

    Buffer<uint16_t> out1 = blur1.realize(W, H, target);
    Buffer<uint16_t> out2 = blur2.realize(W, H, target);

    for (int y = 1; y < H-1; y++) {
        for (int x = 1; x < W-1; x++) {
            uint16_t correct = (1*in(x-1, y-1) + 2*in(x, y-1) + 1*in(x+1, y-1) +
                                2*in(x-1, y)   + 4*in(x, y) +   2*in(x+1, y) +
                                1*in(x-1, y+1) + 2*in(x, y+1) + 1*in(x+1, y+1));
            if (out1(x, y) != correct) {
                printf("out1(%d, %d) = %d instead of %d\n", x, y, out1(x, y), correct);
                return -1;
            }
            if (out2(x, y) != correct) {
                printf("out2(%d, %d) = %d instead of %d\n", x, y, out2(x, y), correct);
                return -1;
            }
        }
    }

    printf("Success!\n");

    return 0;

}
<|endoftext|>


<commit_before>#include <stdio.h> /* printf */
#include <stdlib.h> /* rand */
#include <time.h> /* clock */

#include <set> /* std::set<int> */
#include "scapegoat.h"

#define INSERT 0
#define SEARCH 1
#define DELETE 2
#define MAX_OP 3

struct bench_node {
    int key;
    unsigned char op : 2;
};

inline double time_elapsed(time_t begin, time_t end) {
	return (double)(end - begin) / CLOCKS_PER_SEC;
}

inline void report(unsigned int n, time_t begin) {
	double t = time_elapsed(begin, clock());
	printf("%.3f s, %.3f Mop/s\n", t, n / (t * 1000 * 1000));
}

void bench_sg(const char* type, t_sg_tree* sg_tree, unsigned int n, struct bench_node** bench_nodes) {
	unsigned int i;
	t_sg_node *sg_node;
	time_t begin;
	begin = clock();
	for (i = 0; i < n; ++i) {
		if (bench_nodes[i]->op == INSERT) {
			sg_insert(sg_tree, bench_nodes[i]->key);
		}
		else if (bench_nodes[i]->op == SEARCH) {
			sg_node = sg_search(sg_tree, bench_nodes[i]->key);
		}
		else {
			sg_delete(sg_tree, bench_nodes[i]->key);
		}
	}
	printf("SG-%.2f %s: ", sg_tree->alpha, type);
	report(n, begin);
}

void bench_rb(const char* type, std::set<int>* rb_tree, unsigned int n, struct bench_node** bench_nodes) {
	unsigned int i;
	std::set<int>::iterator it;
	time_t begin;
	begin = clock();
	for (i = 0; i < n; ++i) {
		if (bench_nodes[i]->op == INSERT) {
			rb_tree->insert(bench_nodes[i]->key);
		}
		else if (bench_nodes[i]->op == SEARCH) {
			it = rb_tree->find(bench_nodes[i]->key);
		}
		else {
			rb_tree->erase(bench_nodes[i]->key);
		}
	}
	printf("RB %s: ", type);
	report(n, begin);
}

int main(void) {
	double alphas[] = {0.55, 0.6, 0.65, 0.7, 0.75};
    unsigned int i, j, k, r, M = 10, n = M * 1000 * 1000, n_alphas = sizeof(alphas) / sizeof(alphas[0]), rand_max = 10 * 1000;
    struct bench_node **bench_nodes, *temp;
    struct sg_tree* sg_tree[sizeof(alphas) / sizeof(alphas[0])];
    std::set<int> rb_tree;

    printf("=== BENCHMARK - START ===\n");

    bench_nodes = (bench_node**)malloc(sizeof(struct bench_node*) * n);
    for (i = 0; i < n; ++i) {
		bench_nodes[i] = (bench_node*)malloc(sizeof(struct bench_node));
	}

	for (j = 0; j < n_alphas; j++) {
		sg_tree[j] = sg_create_tree(alphas[j]);
	}

	for (k = 0; k < 2; k++) {
		if (k == 0) {
			printf("BENCHMARK - Sequential test %dM INSERT -> %dM SEARCH -> %dM DELETE\n", M, M, M);
			for (i = 0; i < n; ++i) {
				bench_nodes[i]->key = i;
			}
		}
		else {
			printf("BENCHMARK - Shuffled test %dM INSERT -> %dM SEARCH -> %dM DELETE\n", M, M, M);
			for (i = 0; i < n; ++i) {
				// swap key_nodes[i] <-> key_nodes[r]
				r = rand() % n;
				temp = bench_nodes[r];
				bench_nodes[r] = bench_nodes[i];
				bench_nodes[i] = temp;
			}
		}

		// RB
		rb_tree.clear();

		printf("BENCHMARK - RB\n");
		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = INSERT;
		}
		bench_rb("INSERT", &rb_tree, n, bench_nodes);

		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = SEARCH;
		}
		bench_rb("SEARCH", &rb_tree, n, bench_nodes);

		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = DELETE;
		}
		bench_rb("DELETE", &rb_tree, n, bench_nodes);

		// SG
		for (j = 0; j < n_alphas; j++) {
			sg_clear_tree(sg_tree[j]);

			printf("BENCHMARK - SG-%.2f\n", alphas[j]);
			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = INSERT;
			}
			bench_sg("INSERT", sg_tree[j], n, bench_nodes);

			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = SEARCH;
			}
			bench_sg("SEARCH", sg_tree[j], n, bench_nodes);

			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = DELETE;
			}
			bench_sg("DELETE", sg_tree[j], n, bench_nodes);
		}
	}

	// Random
	printf("BENCHMARK - Random test %dMops, rand_max key = %d\n", M, rand_max);
	for (i = 0; i < n; ++i) {
		bench_nodes[i]->op = rand() % MAX_OP;
		bench_nodes[i]->key = rand() % rand_max;
	}

	// RB
	rb_tree.clear();
	printf("BENCHMARK - RB\n");
	bench_rb("RANDOM", &rb_tree, n, bench_nodes);
	rb_tree.clear();

	// SG
	for (j = 0; j < n_alphas; j++) {
		sg_clear_tree(sg_tree[j]);
		printf("BENCHMARK - SG-%.2f\n", alphas[j]);
		bench_sg("RANDOM", sg_tree[j], n, bench_nodes);
		sg_clear_tree(sg_tree[j]);
	}

	for (j = 0; j < n_alphas; j++) {
		sg_delete_tree(sg_tree[j]);
	}

	for (i = 0; i < n; ++i) {
		free(bench_nodes[i]);
	}
	free(bench_nodes);

	printf("=== BENCHMARK - END ===\n");

    return 0;
}
<commit_msg>Fixed indentation.<commit_after>#include <stdio.h> /* printf */
#include <stdlib.h> /* rand */
#include <time.h> /* clock */

#include <set> /* std::set<int> */
#include "scapegoat.h"

#define INSERT 0
#define SEARCH 1
#define DELETE 2
#define MAX_OP 3

struct bench_node {
	int key;
	unsigned char op : 2;
};

inline double time_elapsed(time_t begin, time_t end) {
	return (double)(end - begin) / CLOCKS_PER_SEC;
}

inline void report(unsigned int n, time_t begin) {
	double t = time_elapsed(begin, clock());
	printf("%.3f s, %.3f Mop/s\n", t, n / (t * 1000 * 1000));
}

void bench_sg(const char* type, t_sg_tree* sg_tree, unsigned int n, struct bench_node** bench_nodes) {
	unsigned int i;
	t_sg_node *sg_node;
	time_t begin;
	begin = clock();
	for (i = 0; i < n; ++i) {
		if (bench_nodes[i]->op == INSERT) {
			sg_insert(sg_tree, bench_nodes[i]->key);
		}
		else if (bench_nodes[i]->op == SEARCH) {
			sg_node = sg_search(sg_tree, bench_nodes[i]->key);
		}
		else {
			sg_delete(sg_tree, bench_nodes[i]->key);
		}
	}
	printf("SG-%.2f %s: ", sg_tree->alpha, type);
	report(n, begin);
}

void bench_rb(const char* type, std::set<int>* rb_tree, unsigned int n, struct bench_node** bench_nodes) {
	unsigned int i;
	std::set<int>::iterator it;
	time_t begin;
	begin = clock();
	for (i = 0; i < n; ++i) {
		if (bench_nodes[i]->op == INSERT) {
			rb_tree->insert(bench_nodes[i]->key);
		}
		else if (bench_nodes[i]->op == SEARCH) {
			it = rb_tree->find(bench_nodes[i]->key);
		}
		else {
			rb_tree->erase(bench_nodes[i]->key);
		}
	}
	printf("RB %s: ", type);
	report(n, begin);
}

int main(void) {
	double alphas[] = {0.55, 0.6, 0.65, 0.7, 0.75};
	unsigned int i, j, k, r, M = 10, n = M * 1000 * 1000, n_alphas = sizeof(alphas) / sizeof(alphas[0]), rand_max = 10 * 1000;
	struct bench_node **bench_nodes, *temp;
	struct sg_tree* sg_tree[sizeof(alphas) / sizeof(alphas[0])];
	std::set<int> rb_tree;

	printf("=== BENCHMARK - START ===\n");

	bench_nodes = (bench_node**)malloc(sizeof(struct bench_node*) * n);
	for (i = 0; i < n; ++i) {
		bench_nodes[i] = (bench_node*)malloc(sizeof(struct bench_node));
	}

	for (j = 0; j < n_alphas; j++) {
		sg_tree[j] = sg_create_tree(alphas[j]);
	}

	for (k = 0; k < 2; k++) {
		if (k == 0) {
			printf("BENCHMARK - Sequential test %dM INSERT -> %dM SEARCH -> %dM DELETE\n", M, M, M);
			for (i = 0; i < n; ++i) {
				bench_nodes[i]->key = i;
			}
		}
		else {
			printf("BENCHMARK - Shuffled test %dM INSERT -> %dM SEARCH -> %dM DELETE\n", M, M, M);
			for (i = 0; i < n; ++i) {
				// swap key_nodes[i] <-> key_nodes[r]
				r = rand() % n;
				temp = bench_nodes[r];
				bench_nodes[r] = bench_nodes[i];
				bench_nodes[i] = temp;
			}
		}

		// RB
		rb_tree.clear();

		printf("BENCHMARK - RB\n");
		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = INSERT;
		}
		bench_rb("INSERT", &rb_tree, n, bench_nodes);

		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = SEARCH;
		}
		bench_rb("SEARCH", &rb_tree, n, bench_nodes);

		for (i = 0; i < n; ++i) {
			bench_nodes[i]->op = DELETE;
		}
		bench_rb("DELETE", &rb_tree, n, bench_nodes);

		// SG
		for (j = 0; j < n_alphas; j++) {
			sg_clear_tree(sg_tree[j]);

			printf("BENCHMARK - SG-%.2f\n", alphas[j]);
			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = INSERT;
			}
			bench_sg("INSERT", sg_tree[j], n, bench_nodes);

			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = SEARCH;
			}
			bench_sg("SEARCH", sg_tree[j], n, bench_nodes);

			for (i = 0; i < n; ++i) {
				bench_nodes[i]->op = DELETE;
			}
			bench_sg("DELETE", sg_tree[j], n, bench_nodes);
		}
	}

	// Random
	printf("BENCHMARK - Random test %dMops, rand_max key = %d\n", M, rand_max);
	for (i = 0; i < n; ++i) {
		bench_nodes[i]->op = rand() % MAX_OP;
		bench_nodes[i]->key = rand() % rand_max;
	}

	// RB
	rb_tree.clear();
	printf("BENCHMARK - RB\n");
	bench_rb("RANDOM", &rb_tree, n, bench_nodes);
	rb_tree.clear();

	// SG
	for (j = 0; j < n_alphas; j++) {
		sg_clear_tree(sg_tree[j]);
		printf("BENCHMARK - SG-%.2f\n", alphas[j]);
		bench_sg("RANDOM", sg_tree[j], n, bench_nodes);
		sg_clear_tree(sg_tree[j]);
	}

	for (j = 0; j < n_alphas; j++) {
		sg_delete_tree(sg_tree[j]);
	}

	for (i = 0; i < n; ++i) {
		free(bench_nodes[i]);
	}
	free(bench_nodes);

	printf("=== BENCHMARK - END ===\n");

	return 0;
}
<|endoftext|>


<commit_before>// File:      Message_PrinterOStream.cxx
// Created:   Sat Jan  6 20:01:38 2001
// Author:    OCC Team
// Copyright: Open CASCADE S.A. 2005

#include <Message_PrinterOStream.ixx>

#include <Message_Gravity.hxx>
#include <TCollection_AsciiString.hxx>
#include <TCollection_ExtendedString.hxx>
#include <Standard_Mutex.hxx>
#include <Standard_Stream.hxx>

//=======================================================================
//function : Constructor
//purpose  : Empty constructor, defaulting to cerr
//=======================================================================

Message_PrinterOStream::Message_PrinterOStream (const Message_Gravity theTraceLevel)
: myTraceLevel(theTraceLevel), myStream(&cout),
  myIsFile(Standard_False), myUseUtf8(Standard_False)
{
}

//=======================================================================
//function : Constructor
//purpose  : Opening a file as an ostream
//           for specific file names standard streams are created
//=======================================================================
Message_PrinterOStream::Message_PrinterOStream (const Standard_CString theFileName,
						const Standard_Boolean doAppend,
						const Message_Gravity theTraceLevel)
: myTraceLevel(theTraceLevel), myStream(&cout), myIsFile(Standard_False)
{
  if ( strcasecmp(theFileName, "cout") == 0 )
    myStream = &cerr;
  else if ( strcasecmp(theFileName, "cerr") == 0 )
    myStream = &cout;
  else
  {
    TCollection_AsciiString aFileName (theFileName);
#ifdef WNT
    aFileName.ChangeAll ('/', '\\');
#endif

    ofstream *ofile = new ofstream (aFileName.ToCString(),
#ifdef USE_STL_STREAMS
				 (doAppend ? (std::ios_base::app | std::ios_base::out) : std::ios_base::out ) );
#else
				 (doAppend ? ios::app : ios::out ) );
#endif
    if ( ofile ) {
      myStream = (Standard_OStream*)ofile;
      myIsFile = Standard_True;
    }
    else {
      myStream = &cout;
      cerr << "Error opening " << theFileName << endl << flush;
    }
  }
}

//=======================================================================
//function : Close
//purpose  :
//=======================================================================

void Message_PrinterOStream::Close ()
{
  if ( ! myStream ) return;
  Standard_OStream* ostr = (Standard_OStream*)myStream;
  myStream = 0;

  ostr->flush();
  if ( myIsFile )
  {
    ofstream* ofile = (ofstream*)ostr;
    ofile->close();
    delete ofile;
    myIsFile = Standard_False;
  }
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const Standard_CString theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  if ( theGravity < myTraceLevel || ! myStream ) return;
  Standard_OStream* ostr = (Standard_OStream*)myStream;
  (*ostr) << theString;
  if ( putEndl ) (*ostr) << endl;
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const TCollection_AsciiString &theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  Send ( theString.ToCString(), theGravity, putEndl );
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const TCollection_ExtendedString &theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  // Note: the string might need to be converted to Ascii
  if ( myUseUtf8 ) {
    char* astr = new char[theString.LengthOfCString()+1];
    theString.ToUTF8CString (astr);
    Send ( astr, theGravity, putEndl );
    delete astr;
  }
  else {
    TCollection_AsciiString aStr ( theString, '?' );
    Send ( aStr.ToCString(), theGravity, putEndl );
  }
}
<commit_msg>[cppcheck-error-fix][mismatch-allocation-deallocation]<commit_after>// File:      Message_PrinterOStream.cxx
// Created:   Sat Jan  6 20:01:38 2001
// Author:    OCC Team
// Copyright: Open CASCADE S.A. 2005

#include <Message_PrinterOStream.ixx>

#include <Message_Gravity.hxx>
#include <TCollection_AsciiString.hxx>
#include <TCollection_ExtendedString.hxx>
#include <Standard_Mutex.hxx>
#include <Standard_Stream.hxx>

//=======================================================================
//function : Constructor
//purpose  : Empty constructor, defaulting to cerr
//=======================================================================

Message_PrinterOStream::Message_PrinterOStream (const Message_Gravity theTraceLevel)
: myTraceLevel(theTraceLevel), myStream(&cout),
  myIsFile(Standard_False), myUseUtf8(Standard_False)
{
}

//=======================================================================
//function : Constructor
//purpose  : Opening a file as an ostream
//           for specific file names standard streams are created
//=======================================================================
Message_PrinterOStream::Message_PrinterOStream (const Standard_CString theFileName,
						const Standard_Boolean doAppend,
						const Message_Gravity theTraceLevel)
: myTraceLevel(theTraceLevel), myStream(&cout), myIsFile(Standard_False)
{
  if ( strcasecmp(theFileName, "cout") == 0 )
    myStream = &cerr;
  else if ( strcasecmp(theFileName, "cerr") == 0 )
    myStream = &cout;
  else
  {
    TCollection_AsciiString aFileName (theFileName);
#ifdef WNT
    aFileName.ChangeAll ('/', '\\');
#endif

    ofstream *ofile = new ofstream (aFileName.ToCString(),
#ifdef USE_STL_STREAMS
				 (doAppend ? (std::ios_base::app | std::ios_base::out) : std::ios_base::out ) );
#else
				 (doAppend ? ios::app : ios::out ) );
#endif
    if ( ofile ) {
      myStream = (Standard_OStream*)ofile;
      myIsFile = Standard_True;
    }
    else {
      myStream = &cout;
      cerr << "Error opening " << theFileName << endl << flush;
    }
  }
}

//=======================================================================
//function : Close
//purpose  :
//=======================================================================

void Message_PrinterOStream::Close ()
{
  if ( ! myStream ) return;
  Standard_OStream* ostr = (Standard_OStream*)myStream;
  myStream = 0;

  ostr->flush();
  if ( myIsFile )
  {
    ofstream* ofile = (ofstream*)ostr;
    ofile->close();
    delete ofile;
    myIsFile = Standard_False;
  }
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const Standard_CString theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  if ( theGravity < myTraceLevel || ! myStream ) return;
  Standard_OStream* ostr = (Standard_OStream*)myStream;
  (*ostr) << theString;
  if ( putEndl ) (*ostr) << endl;
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const TCollection_AsciiString &theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  Send ( theString.ToCString(), theGravity, putEndl );
}

//=======================================================================
//function : Send
//purpose  :
//=======================================================================

void Message_PrinterOStream::Send (const TCollection_ExtendedString &theString,
				   const Message_Gravity theGravity,
				   const Standard_Boolean putEndl) const
{
  // Note: the string might need to be converted to Ascii
  if ( myUseUtf8 ) {
    char* astr = new char[theString.LengthOfCString()+1];
    theString.ToUTF8CString (astr);
    Send ( astr, theGravity, putEndl );
    delete [] astr;
  }
  else {
    TCollection_AsciiString aStr ( theString, '?' );
    Send ( aStr.ToCString(), theGravity, putEndl );
  }
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <math.h>
#include <stdlib.h>

#include "bloom.h"
#include "main.h"
#include "script.h"

#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455
#define LN2 0.6931471805599453094172321214581765680755001343602552

using namespace std;

static const unsigned char bit_mask[8] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80};

CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) :
// The ideal size for a bloom filter with a given number of elements and false positive rate is:
// - nElements * log(fp rate) / ln(2)^2
// We ignore filter parameters which will create a bloom filter larger than the protocol limits
vData(min((unsigned int)(-1  / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8),
// The ideal number of hash functions is filter size * ln(2) / number of elements
// Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits
// See http://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas
nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)),
nTweak(nTweakIn),
nFlags(nFlagsIn)
{
}

inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const
{
    // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values.
    return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8);
}

void CBloomFilter::insert(const vector<unsigned char>& vKey)
{
    for (unsigned int i = 0; i < nHashFuncs; i++)
    {
        unsigned int nIndex = Hash(i, vKey);
        // Sets bit nIndex of vData
        vData[nIndex >> 3] |= bit_mask[7 & nIndex];
    }
}

void CBloomFilter::insert(const COutPoint& outpoint)
{
    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
    stream << outpoint;
    vector<unsigned char> data(stream.begin(), stream.end());
    insert(data);
}

void CBloomFilter::insert(const uint256& hash)
{
    vector<unsigned char> data(hash.begin(), hash.end());
    insert(data);
}

bool CBloomFilter::contains(const vector<unsigned char>& vKey) const
{
    for (unsigned int i = 0; i < nHashFuncs; i++)
    {
        unsigned int nIndex = Hash(i, vKey);
        // Checks bit nIndex of vData
        if (!(vData[nIndex >> 3] & bit_mask[7 & nIndex]))
            return false;
    }
    return true;
}

bool CBloomFilter::contains(const COutPoint& outpoint) const
{
    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
    stream << outpoint;
    vector<unsigned char> data(stream.begin(), stream.end());
    return contains(data);
}

bool CBloomFilter::contains(const uint256& hash) const
{
    vector<unsigned char> data(hash.begin(), hash.end());
    return contains(data);
}

bool CBloomFilter::IsWithinSizeConstraints() const
{
    return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS;
}

bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx, const uint256& hash)
{
    bool fFound = false;
    // Match if the filter contains the hash of tx
    //  for finding tx when they appear in a block
    if (contains(hash))
        fFound = true;

    for (unsigned int i = 0; i < tx.vout.size(); i++)
    {
        const CTxOut& txout = tx.vout[i];
        // Match if the filter contains any arbitrary script data element in any scriptPubKey in tx
        // If this matches, also add the specific output that was matched.
        // This means clients don't have to update the filter themselves when a new relevant tx
        // is discovered in order to find spending transactions, which avoids round-tripping and race conditions.
        CScript::const_iterator pc = txout.scriptPubKey.begin();
        vector<unsigned char> data;
        while (pc < txout.scriptPubKey.end())
        {
            opcodetype opcode;
            if (!txout.scriptPubKey.GetOp(pc, opcode, data))
                break;
            if (data.size() != 0 && contains(data))
            {
                fFound = true;
                if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_ALL)
                    insert(COutPoint(hash, i));
                else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY)
                {
                    txnouttype type;
                    vector<vector<unsigned char> > vSolutions;
                    if (Solver(txout.scriptPubKey, type, vSolutions) &&
                            (type == TX_PUBKEY || type == TX_MULTISIG))
                        insert(COutPoint(hash, i));
                }
                break;
            }
        }
    }

    if (fFound)
        return true;

    BOOST_FOREACH(const CTxIn& txin, tx.vin)
    {
        // Match if the filter contains an outpoint tx spends
        if (contains(txin.prevout))
            return true;

        // Match if the filter contains any arbitrary script data element in any scriptSig in tx
        CScript::const_iterator pc = txin.scriptSig.begin();
        vector<unsigned char> data;
        while (pc < txin.scriptSig.end())
        {
            opcodetype opcode;
            if (!txin.scriptSig.GetOp(pc, opcode, data))
                break;
            if (data.size() != 0 && contains(data))
                return true;
        }
    }

    return false;
}
<commit_msg>Short-circuit bloom checking if we will always return true.<commit_after>// Copyright (c) 2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <math.h>
#include <stdlib.h>

#include "bloom.h"
#include "main.h"
#include "script.h"

#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455
#define LN2 0.6931471805599453094172321214581765680755001343602552

using namespace std;

static const unsigned char bit_mask[8] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80};

CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) :
// The ideal size for a bloom filter with a given number of elements and false positive rate is:
// - nElements * log(fp rate) / ln(2)^2
// We ignore filter parameters which will create a bloom filter larger than the protocol limits
vData(min((unsigned int)(-1  / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8),
// The ideal number of hash functions is filter size * ln(2) / number of elements
// Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits
// See http://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas
nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)),
nTweak(nTweakIn),
nFlags(nFlagsIn)
{
}

inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const
{
    // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values.
    return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8);
}

void CBloomFilter::insert(const vector<unsigned char>& vKey)
{
    if (vData.size() == 1 && vData[0] == 0xff)
        return;
    for (unsigned int i = 0; i < nHashFuncs; i++)
    {
        unsigned int nIndex = Hash(i, vKey);
        // Sets bit nIndex of vData
        vData[nIndex >> 3] |= bit_mask[7 & nIndex];
    }
}

void CBloomFilter::insert(const COutPoint& outpoint)
{
    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
    stream << outpoint;
    vector<unsigned char> data(stream.begin(), stream.end());
    insert(data);
}

void CBloomFilter::insert(const uint256& hash)
{
    vector<unsigned char> data(hash.begin(), hash.end());
    insert(data);
}

bool CBloomFilter::contains(const vector<unsigned char>& vKey) const
{
    if (vData.size() == 1 && vData[0] == 0xff)
        return true;
    for (unsigned int i = 0; i < nHashFuncs; i++)
    {
        unsigned int nIndex = Hash(i, vKey);
        // Checks bit nIndex of vData
        if (!(vData[nIndex >> 3] & bit_mask[7 & nIndex]))
            return false;
    }
    return true;
}

bool CBloomFilter::contains(const COutPoint& outpoint) const
{
    CDataStream stream(SER_NETWORK, PROTOCOL_VERSION);
    stream << outpoint;
    vector<unsigned char> data(stream.begin(), stream.end());
    return contains(data);
}

bool CBloomFilter::contains(const uint256& hash) const
{
    vector<unsigned char> data(hash.begin(), hash.end());
    return contains(data);
}

bool CBloomFilter::IsWithinSizeConstraints() const
{
    return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS;
}

bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx, const uint256& hash)
{
    bool fFound = false;
    // Match if the filter contains the hash of tx
    //  for finding tx when they appear in a block
    if (contains(hash))
        fFound = true;

    for (unsigned int i = 0; i < tx.vout.size(); i++)
    {
        const CTxOut& txout = tx.vout[i];
        // Match if the filter contains any arbitrary script data element in any scriptPubKey in tx
        // If this matches, also add the specific output that was matched.
        // This means clients don't have to update the filter themselves when a new relevant tx
        // is discovered in order to find spending transactions, which avoids round-tripping and race conditions.
        CScript::const_iterator pc = txout.scriptPubKey.begin();
        vector<unsigned char> data;
        while (pc < txout.scriptPubKey.end())
        {
            opcodetype opcode;
            if (!txout.scriptPubKey.GetOp(pc, opcode, data))
                break;
            if (data.size() != 0 && contains(data))
            {
                fFound = true;
                if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_ALL)
                    insert(COutPoint(hash, i));
                else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY)
                {
                    txnouttype type;
                    vector<vector<unsigned char> > vSolutions;
                    if (Solver(txout.scriptPubKey, type, vSolutions) &&
                            (type == TX_PUBKEY || type == TX_MULTISIG))
                        insert(COutPoint(hash, i));
                }
                break;
            }
        }
    }

    if (fFound)
        return true;

    BOOST_FOREACH(const CTxIn& txin, tx.vin)
    {
        // Match if the filter contains an outpoint tx spends
        if (contains(txin.prevout))
            return true;

        // Match if the filter contains any arbitrary script data element in any scriptSig in tx
        CScript::const_iterator pc = txin.scriptSig.begin();
        vector<unsigned char> data;
        while (pc < txin.scriptSig.end())
        {
            opcodetype opcode;
            if (!txin.scriptSig.GetOp(pc, opcode, data))
                break;
            if (data.size() != 0 && contains(data))
                return true;
        }
    }

    return false;
}
<|endoftext|>


<commit_before>#include "board.hpp"

#include <cstring>

#include <boost/lexical_cast.hpp>

#include "exception.hpp"
#include "walk_move.hpp"

namespace Quoridor {

Board::Board(int row_num, int col_num) : row_num_(), col_num_(),
        occ_fields_(), pawn_pos_(), sides_(), pawn_sides_(), walls_()
{
    set_size(row_num, col_num);
    sides_.push_back(std::pair<int, int>(0, 0));
    sides_.push_back(std::pair<int, int>(2, 0));
    sides_.push_back(std::pair<int, int>(1, 0));
    sides_.push_back(std::pair<int, int>(3, 0));
}

Board::~Board()
{
}

void Board::set_size(int row_num, int col_num)
{
    if (row_num <= 0) {
        throw Exception("illegal row number: "
                + boost::lexical_cast<std::string>(row_num));
    }
    if (col_num <= 0) {
        throw Exception("illegal column number: "
                + boost::lexical_cast<std::string>(col_num));
    }
    row_num_ = row_num;
    col_num_ = col_num;
}

int Board::next_side() const
{
    for (auto &side : sides_) {
        if (side.second == 0) {
            side.second = 1;
            return side.first;
        }
    }
    return -1;
}

int Board::add_pawn(std::shared_ptr<Pawn> pawn)
{
    pawn_sides_[pawn] = next_side();
    pos_t pos;

    switch (pawn_sides_[pawn]) {
    case 0:
        pos.row = 0;
        pos.col = 4;
        break;
    case 1:
        pos.row = 4;
        pos.col = 0;
        break;
    case 2:
        pos.row = 8;
        pos.col = 4;
        break;
    case 3:
        pos.row = 4;
        pos.col = 8;
        break;
    default:
        throw Exception("invalid pawn side: "
                + boost::lexical_cast<std::string>(pawn_sides_[pawn]));
    }

    occ_fields_[pos] = pawn;
    pawn_pos_[pawn] = pos;

    return 0;
}

void Board::add_occupied(const pos_t &pos, std::shared_ptr<Pawn> pawn)
{
    if (occ_fields_.count(pos) > 0) {
        throw Exception("cell (" + boost::lexical_cast<std::string>(pos.row)
                + ":" + boost::lexical_cast<std::string>(pos.col)
                + ") is already occupied");
    }
    occ_fields_[pos] = pawn;
    pawn_pos_[pawn] = pos;
}

void Board::rm_occupied(const pos_t &pos)
{
    occ_fields_.erase(pos);
}

pos_t Board::pawn_pos(std::shared_ptr<Pawn> pawn) const
{
    return pawn_pos_.at(pawn);
}

bool Board::is_at_opposite_side(std::shared_ptr<Pawn> pawn) const
{
    switch (pawn_sides_.at(pawn)) {
    case 0:
        return pawn_pos_.at(pawn).row == row_num() - 1;
    case 1:
        return pawn_pos_.at(pawn).col == col_num() - 1;
    case 2:
        return pawn_pos_.at(pawn).row == 0;
    case 3:
        return pawn_pos_.at(pawn).col == 0;
    default:
        throw Exception("invalid board side: "
                + boost::lexical_cast<std::string>(pawn_sides_.at(pawn)));
    }
}

int Board::recalc_dir(int dir, std::shared_ptr<Pawn> pawn)
{
    int m = (dir + pawn_sides_[pawn]) % 4;
    return m;
}

int Board::make_walking_move(int dir, std::shared_ptr<Pawn> pawn)
{
    dir = recalc_dir(dir, pawn);
    pos_t pos = pawn_pos_[pawn];
    pos_t inc_pos;
    pos_t lim_pos = pos;

    switch (dir) {
        case WalkMove::Direction::kForward:
        lim_pos.row = row_num() - 1;
        inc_pos.row = 1;
        break;
    case WalkMove::Direction::kRight:
        lim_pos.col = col_num() - 1;
        inc_pos.col = 1;
        break;
    case WalkMove::Direction::kBackward:
        lim_pos.row = 0;
        inc_pos.row = -1;
        break;
    case WalkMove::Direction::kLeft:
        lim_pos.col = 0;
        inc_pos.col = -1;
        break;
    case WalkMove::Direction::kEnd:
    default:
        return -1;
    }

    // error, pawn is already at the opposite side
    if (pos == lim_pos) {
        return -1;
    }

    if (is_possible_move(pos, inc_pos)) {
        pos_t possible_pos = pos + inc_pos;
        if (occ_fields_.count(possible_pos) == 0) {
            pos += inc_pos;
        }
        else {
            if (is_possible_move(possible_pos, inc_pos)) {
                pos += inc_pos;
                pos += inc_pos;
            }
            else {
                return -2;
            }
        }
    }
    else {
        return -1;
    }

    // pawn cannot make specified move
    if (is_outside_board(pos)) {
        return -1;
    }

    // update pawn's position
    occ_fields_.erase(pawn_pos_[pawn]);
    pawn_pos_[pawn] = pos;
    occ_fields_[pos] = pawn;

    return 0;
}

int Board::add_wall(const Wall &wall)
{
    int line_lim = wall.orientation() ? col_num() : row_num() - 1;
    int start_pos_lim = (wall.orientation() ? row_num() : col_num()) - 1;
    if ((wall.line() >= line_lim)
            || (wall.start_pos() + wall.cnt() >= start_pos_lim)) {
        return -1;
    }

    if (wall_intersects(wall)) {
        return -2;
    }

    walls_[wall.orientation()][wall.line()].insert(std::map<int, Wall>::value_type(wall.start_pos(), Wall(wall)));

    return 0;
}

bool Board::wall_intersects(const Wall &wall) const
{
    // check intersections
    if (walls_.count(1 - wall.orientation()) > 0) {
        for (int i = 0; i < wall.cnt() - 1; ++i) {
            // there are walls on the intersected line
            if (walls_.at(1 - wall.orientation()).count(wall.start_pos()) != 0) {
                auto line_walls = walls_.at(1 - wall.orientation()).at(wall.start_pos() + i);
                auto it = line_walls.upper_bound(wall.line());

                // all walls on the line are settled before new wall
                if (it == line_walls.end()) {
                    auto rit = line_walls.rbegin();
                    if (rit->second.start_pos() + rit->second.cnt() >= wall.line()) {
                        return true;
                    }
                }
                else if (it != line_walls.begin()) {
                    --it;
                    if (it->second.start_pos() + it->second.cnt() >= wall.line()) {
                        return true;
                    }
                }
            }
        }
    }

    // check overlaps
    if (walls_.count(wall.orientation()) > 0) {
        if (walls_.at(wall.orientation()).count(wall.line()) != 0) {
            auto line_walls = walls_.at(wall.orientation()).at(wall.line());
            auto it = line_walls.upper_bound(wall.start_pos());

            // all walls on the line are settled before new wall
            if (it == line_walls.end()) {
                auto rit = line_walls.rbegin();
                if (rit->second.start_pos() + rit->second.cnt() >= wall.start_pos()) {
                    return true;
                }
            }
            else {
                // check if new wall overlaps over next wall on the line
                if (wall.start_pos() + wall.cnt() >= it->second.start_pos()) {
                    return true;
                }

                if (it != line_walls.begin()) {
                    --it;
                    if (it->second.start_pos() + it->second.cnt() >= wall.start_pos()) {
                        return true;
                    }
                }
            }
        }
    }

    return false;
}

bool Board::is_outside_board(const pos_t &pos) const
{
    if ((pos.row >= row_num()) || (pos.row < 0)
            || (pos.col >= col_num()) || (pos.col < 0))
        return true;
    return false;
}

bool Board::is_possible_move(const pos_t &pos, const pos_t &inc_pos) const
{
    int orientation;
    int st;
    int crossed_line;

    if (inc_pos.row != 0) {
        orientation = 0;
        st = pos.col;
        crossed_line = std::min(pos.row, pos.row + inc_pos.row);
    }
    else if (inc_pos.col != 0) {
        orientation = 1;
        st = pos.row;
        crossed_line = std::min(pos.col, pos.col + inc_pos.col);
    }
    else {
        throw Exception("zero incrementation move");
    }

    if (walls_.count(orientation) == 0) {
        return true;
    }

    if (walls_.at(orientation).count(crossed_line) != 0) {
        auto line_walls = walls_.at(orientation).at(crossed_line);
        auto it = line_walls.upper_bound(st);

        // all walls on the line are settled before specified position
        if (it == line_walls.end()) {
            auto rit = line_walls.rbegin();
            if (rit->second.start_pos() + rit->second.cnt() >= st) {
                return false;
            }
        }
        else if (it != line_walls.begin()) {
            --it;
            if (it->second.start_pos() + it->second.cnt() >= st) {
                return false;
            }
        }
    }

    return true;
}

}  /* namespace Quoridor */
<commit_msg>Fix wall end positions.<commit_after>#include "board.hpp"

#include <cstring>

#include <boost/lexical_cast.hpp>

#include "exception.hpp"
#include "walk_move.hpp"

namespace Quoridor {

Board::Board(int row_num, int col_num) : row_num_(), col_num_(),
        occ_fields_(), pawn_pos_(), sides_(), pawn_sides_(), walls_()
{
    set_size(row_num, col_num);
    sides_.push_back(std::pair<int, int>(0, 0));
    sides_.push_back(std::pair<int, int>(2, 0));
    sides_.push_back(std::pair<int, int>(1, 0));
    sides_.push_back(std::pair<int, int>(3, 0));
}

Board::~Board()
{
}

void Board::set_size(int row_num, int col_num)
{
    if (row_num <= 0) {
        throw Exception("illegal row number: "
                + boost::lexical_cast<std::string>(row_num));
    }
    if (col_num <= 0) {
        throw Exception("illegal column number: "
                + boost::lexical_cast<std::string>(col_num));
    }
    row_num_ = row_num;
    col_num_ = col_num;
}

int Board::next_side() const
{
    for (auto &side : sides_) {
        if (side.second == 0) {
            side.second = 1;
            return side.first;
        }
    }
    return -1;
}

int Board::add_pawn(std::shared_ptr<Pawn> pawn)
{
    pawn_sides_[pawn] = next_side();
    pos_t pos;

    switch (pawn_sides_[pawn]) {
    case 0:
        pos.row = 0;
        pos.col = 4;
        break;
    case 1:
        pos.row = 4;
        pos.col = 0;
        break;
    case 2:
        pos.row = 8;
        pos.col = 4;
        break;
    case 3:
        pos.row = 4;
        pos.col = 8;
        break;
    default:
        throw Exception("invalid pawn side: "
                + boost::lexical_cast<std::string>(pawn_sides_[pawn]));
    }

    occ_fields_[pos] = pawn;
    pawn_pos_[pawn] = pos;

    return 0;
}

void Board::add_occupied(const pos_t &pos, std::shared_ptr<Pawn> pawn)
{
    if (occ_fields_.count(pos) > 0) {
        throw Exception("cell (" + boost::lexical_cast<std::string>(pos.row)
                + ":" + boost::lexical_cast<std::string>(pos.col)
                + ") is already occupied");
    }
    occ_fields_[pos] = pawn;
    pawn_pos_[pawn] = pos;
}

void Board::rm_occupied(const pos_t &pos)
{
    occ_fields_.erase(pos);
}

pos_t Board::pawn_pos(std::shared_ptr<Pawn> pawn) const
{
    return pawn_pos_.at(pawn);
}

bool Board::is_at_opposite_side(std::shared_ptr<Pawn> pawn) const
{
    switch (pawn_sides_.at(pawn)) {
    case 0:
        return pawn_pos_.at(pawn).row == row_num() - 1;
    case 1:
        return pawn_pos_.at(pawn).col == col_num() - 1;
    case 2:
        return pawn_pos_.at(pawn).row == 0;
    case 3:
        return pawn_pos_.at(pawn).col == 0;
    default:
        throw Exception("invalid board side: "
                + boost::lexical_cast<std::string>(pawn_sides_.at(pawn)));
    }
}

int Board::recalc_dir(int dir, std::shared_ptr<Pawn> pawn)
{
    int m = (dir + pawn_sides_[pawn]) % 4;
    return m;
}

int Board::make_walking_move(int dir, std::shared_ptr<Pawn> pawn)
{
    dir = recalc_dir(dir, pawn);
    pos_t pos = pawn_pos_[pawn];
    pos_t inc_pos;
    pos_t lim_pos = pos;

    switch (dir) {
        case WalkMove::Direction::kForward:
        lim_pos.row = row_num() - 1;
        inc_pos.row = 1;
        break;
    case WalkMove::Direction::kRight:
        lim_pos.col = col_num() - 1;
        inc_pos.col = 1;
        break;
    case WalkMove::Direction::kBackward:
        lim_pos.row = 0;
        inc_pos.row = -1;
        break;
    case WalkMove::Direction::kLeft:
        lim_pos.col = 0;
        inc_pos.col = -1;
        break;
    case WalkMove::Direction::kEnd:
    default:
        return -1;
    }

    // error, pawn is already at the opposite side
    if (pos == lim_pos) {
        return -1;
    }

    if (is_possible_move(pos, inc_pos)) {
        pos_t possible_pos = pos + inc_pos;
        if (occ_fields_.count(possible_pos) == 0) {
            pos += inc_pos;
        }
        else {
            if (is_possible_move(possible_pos, inc_pos)) {
                pos += inc_pos;
                pos += inc_pos;
            }
            else {
                return -2;
            }
        }
    }
    else {
        return -1;
    }

    // pawn cannot make specified move
    if (is_outside_board(pos)) {
        return -1;
    }

    // update pawn's position
    occ_fields_.erase(pawn_pos_[pawn]);
    pawn_pos_[pawn] = pos;
    occ_fields_[pos] = pawn;

    return 0;
}

int Board::add_wall(const Wall &wall)
{
    int line_lim = wall.orientation() ? col_num() : row_num() - 1;
    int start_pos_lim = (wall.orientation() ? row_num() : col_num()) - 1;
    if ((wall.line() >= line_lim)
            || (wall.end_pos() >= start_pos_lim)) {
        return -1;
    }

    if (wall_intersects(wall)) {
        return -2;
    }

    walls_[wall.orientation()][wall.line()].insert(std::map<int, Wall>::value_type(wall.start_pos(), Wall(wall)));

    return 0;
}

bool Board::wall_intersects(const Wall &wall) const
{
    // check intersections
    if (walls_.count(1 - wall.orientation()) > 0) {
        for (int i = 0; i < wall.cnt() - 1; ++i) {
            // there are walls on the intersected line
            if (walls_.at(1 - wall.orientation()).count(wall.start_pos()) != 0) {
                auto line_walls = walls_.at(1 - wall.orientation()).at(wall.start_pos() + i);
                auto it = line_walls.upper_bound(wall.line());

                // all walls on the line are settled before new wall
                if (it == line_walls.end()) {
                    auto rit = line_walls.rbegin();
                    if (rit->second.end_pos() >= wall.line()) {
                        return true;
                    }
                }
                else if (it != line_walls.begin()) {
                    --it;
                    if (it->second.end_pos() >= wall.line()) {
                        return true;
                    }
                }
            }
        }
    }

    // check overlaps
    if (walls_.count(wall.orientation()) > 0) {
        if (walls_.at(wall.orientation()).count(wall.line()) != 0) {
            auto line_walls = walls_.at(wall.orientation()).at(wall.line());
            auto it = line_walls.upper_bound(wall.start_pos());

            // all walls on the line are settled before new wall
            if (it == line_walls.end()) {
                auto rit = line_walls.rbegin();
                if (rit->second.end_pos() >= wall.start_pos()) {
                    return true;
                }
            }
            else {
                // check if new wall overlaps over next wall on the line
                if (wall.end_pos() >= it->second.start_pos()) {
                    return true;
                }

                if (it != line_walls.begin()) {
                    --it;
                    if (it->second.end_pos() >= wall.start_pos()) {
                        return true;
                    }
                }
            }
        }
    }

    return false;
}

bool Board::is_outside_board(const pos_t &pos) const
{
    if ((pos.row >= row_num()) || (pos.row < 0)
            || (pos.col >= col_num()) || (pos.col < 0))
        return true;
    return false;
}

bool Board::is_possible_move(const pos_t &pos, const pos_t &inc_pos) const
{
    int orientation;
    int st;
    int crossed_line;

    if (inc_pos.row != 0) {
        orientation = 0;
        st = pos.col;
        crossed_line = std::min(pos.row, pos.row + inc_pos.row);
    }
    else if (inc_pos.col != 0) {
        orientation = 1;
        st = pos.row;
        crossed_line = std::min(pos.col, pos.col + inc_pos.col);
    }
    else {
        throw Exception("zero incrementation move");
    }

    if (walls_.count(orientation) == 0) {
        return true;
    }

    if (walls_.at(orientation).count(crossed_line) != 0) {
        auto line_walls = walls_.at(orientation).at(crossed_line);
        auto it = line_walls.upper_bound(st);

        // all walls on the line are settled before specified position
        if (it == line_walls.end()) {
            auto rit = line_walls.rbegin();
            if (rit->second.end_pos() >= st) {
                return false;
            }
        }
        else if (it != line_walls.begin()) {
            --it;
            if (it->second.end_pos() >= st) {
                return false;
            }
        }
    }

    return true;
}

}  /* namespace Quoridor */
<|endoftext|>