<commit_before>#include "config.h"
#include "configuration.hpp"
#include "immix_marker.hpp"
#include "metrics.hpp"
#include "object_memory.hpp"
#include "state.hpp"
#include "thread_phase.hpp"
#include "vm.hpp"

#include "builtin/class.hpp"
#include "builtin/thread.hpp"

#include "gc/gc.hpp"
#include "gc/immix.hpp"

#include "dtrace/dtrace.h"
#include "instruments/timing.hpp"

namespace rubinius {

  ImmixMarker::ImmixMarker(STATE, ImmixGC* immix)
    : InternalThread(state, "rbx.immix")
    , immix_(immix)
    , data_(NULL)
  {
    state->memory()->set_immix_marker(this);

    initialize(state);
    start_thread(state);
  }

  ImmixMarker::~ImmixMarker() {
    cleanup();
  }

  void ImmixMarker::initialize(STATE) {
    InternalThread::initialize(state);

    Thread::create(state, vm());

    run_lock_.init();
    run_cond_.init();
  }

  void ImmixMarker::wakeup(STATE) {
    utilities::thread::Mutex::LockGuard lg(run_lock_);

    InternalThread::wakeup(state);

    run_cond_.signal();
  }

  void ImmixMarker::after_fork_child(STATE) {
    cleanup();

    InternalThread::after_fork_child(state);
  }

  void ImmixMarker::cleanup() {
    if(data_) {
      delete data_;
      data_ = NULL;
    }
  }

  void ImmixMarker::stop(STATE) {
    InternalThread::stop(state);
  }

  void ImmixMarker::concurrent_mark(GCData* data) {
    utilities::thread::Mutex::LockGuard lg(run_lock_);
    data_ = data;
    run_cond_.signal();
  }

  void ImmixMarker::run(STATE) {
    state->vm()->become_managed();

    while(!thread_exit_) {
      if(data_) {
        {
          timer::StopWatch<timer::milliseconds> timer(
              state->vm()->metrics().gc.immix_concurrent_ms);

          state->shared().thread_nexus()->blocking(state->vm());

          while(immix_->process_mark_stack(immix_->memory()->collect_mature_now)) {
            if(state->shared().thread_nexus()->stop_p()) {
              state->shared().thread_nexus()->yielding(state->vm());
            }
            state->shared().thread_nexus()->blocking(state->vm());
          }
        }

        state->vm()->become_managed();

        if(thread_exit_) break;

        {
          timer::StopWatch<timer::milliseconds> timer(
              state->vm()->metrics().gc.immix_stop_ms);

          LockPhase locked(state);

          state->memory()->clear_mature_mark_in_progress();
          state->memory()->collect_mature_finish(state, data_);
        }
      }

      {
        utilities::thread::Mutex::LockGuard guard(run_lock_);

        cleanup();
        if(thread_exit_) break;

        {
          UnmanagedPhase unmanaged(state);
          run_cond_.wait(run_lock_);
        }
      }
    }

    state->memory()->clear_mature_mark_in_progress();
  }
}
<commit_msg>Let pending young collection interrupt concurrent marker.<commit_after>#include "config.h"
#include "configuration.hpp"
#include "immix_marker.hpp"
#include "metrics.hpp"
#include "object_memory.hpp"
#include "state.hpp"
#include "thread_phase.hpp"
#include "vm.hpp"

#include "builtin/class.hpp"
#include "builtin/thread.hpp"

#include "gc/gc.hpp"
#include "gc/immix.hpp"

#include "dtrace/dtrace.h"
#include "instruments/timing.hpp"

namespace rubinius {

  ImmixMarker::ImmixMarker(STATE, ImmixGC* immix)
    : InternalThread(state, "rbx.immix")
    , immix_(immix)
    , data_(NULL)
  {
    state->memory()->set_immix_marker(this);

    initialize(state);
    start_thread(state);
  }

  ImmixMarker::~ImmixMarker() {
    cleanup();
  }

  void ImmixMarker::initialize(STATE) {
    InternalThread::initialize(state);

    Thread::create(state, vm());

    run_lock_.init();
    run_cond_.init();
  }

  void ImmixMarker::wakeup(STATE) {
    utilities::thread::Mutex::LockGuard lg(run_lock_);

    InternalThread::wakeup(state);

    run_cond_.signal();
  }

  void ImmixMarker::after_fork_child(STATE) {
    cleanup();

    InternalThread::after_fork_child(state);
  }

  void ImmixMarker::cleanup() {
    if(data_) {
      delete data_;
      data_ = NULL;
    }
  }

  void ImmixMarker::stop(STATE) {
    InternalThread::stop(state);
  }

  void ImmixMarker::concurrent_mark(GCData* data) {
    utilities::thread::Mutex::LockGuard lg(run_lock_);
    data_ = data;
    run_cond_.signal();
  }

  void ImmixMarker::run(STATE) {
    state->vm()->become_managed();

    while(!thread_exit_) {
      if(data_) {
        {
          timer::StopWatch<timer::milliseconds> timer(
              state->vm()->metrics().gc.immix_concurrent_ms);

          state->shared().thread_nexus()->blocking(state->vm());

          while(immix_->process_mark_stack(immix_->memory()->collect_young_now)) {
            if(state->shared().thread_nexus()->stop_p()) {
              state->shared().thread_nexus()->yielding(state->vm());
            }
            state->shared().thread_nexus()->blocking(state->vm());
          }
        }

        state->vm()->become_managed();

        if(thread_exit_) break;

        {
          timer::StopWatch<timer::milliseconds> timer(
              state->vm()->metrics().gc.immix_stop_ms);

          LockPhase locked(state);

          state->memory()->clear_mature_mark_in_progress();
          state->memory()->collect_mature_finish(state, data_);
        }
      }

      {
        utilities::thread::Mutex::LockGuard guard(run_lock_);

        cleanup();
        if(thread_exit_) break;

        {
          UnmanagedPhase unmanaged(state);
          run_cond_.wait(run_lock_);
        }
      }
    }

    state->memory()->clear_mature_mark_in_progress();
  }
}
<|endoftext|>


<commit_before>// Copyright © 2011, Université catholique de Louvain
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// *  Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// *  Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

#ifndef MOZART_SMALLINT_H
#define MOZART_SMALLINT_H

#include "mozartcore.hh"

#include <string>
#include <limits>

#ifndef MOZART_GENERATOR

namespace mozart {

//////////////
// SmallInt //
//////////////

#include "SmallInt-implem.hh"

void SmallInt::create(nativeint& self, VM vm, GR gr, SmallInt from) {
  self = from.value();
}

bool SmallInt::equals(VM vm, RichNode right) {
  return value() == right.as<SmallInt>().value();
}

int SmallInt::compareFeatures(VM vm, RichNode right) {
  auto rhs = right.as<SmallInt>().value();

  if (value() == rhs)
    return 0;
  else if (value() < rhs)
    return -1;
  else
    return 1;
}

// Comparable ------------------------------------------------------------------

int SmallInt::compare(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return (value() == smallInt) ? 0 : (value() < smallInt) ? -1 : 1;
  } else if (right.is<BigInt>()) {
    UnstableNode self = SmallInt::build(vm, value());
    return -Comparable(right).compare(vm, self);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

// Numeric ---------------------------------------------------------------------

UnstableNode SmallInt::opposite(VM vm) {
  // Detecting overflow - platform dependent (2's complement)
  if (value() != std::numeric_limits<nativeint>::min()) {
    // No overflow
    return SmallInt::build(vm, -value());
  } else {
    UnstableNode big = vm->newBigInt(std::numeric_limits<nativeint>::min());
    return Numeric(big).opposite(vm);
  }
}

UnstableNode SmallInt::add(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return add(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).add(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::add(VM vm, nativeint b) {
  nativeint a = value();
  nativeint c = a + b;

  // Detecting overflow - platform dependent (2's complement)
  if ((((a ^ c) & (b ^ c)) >> std::numeric_limits<nativeint>::digits) == 0) {
    // No overflow
    return SmallInt::build(vm, c);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).add(vm, right);
  }
}

UnstableNode SmallInt::subtract(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return subtractValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).subtract(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::subtractValue(VM vm, nativeint b) {
  nativeint a = value();
  nativeint c = a - b;

  // Detecting overflow - platform dependent (2's complement)
  if ((((a ^ c) & (-b ^ c)) >> std::numeric_limits<nativeint>::digits) == 0) {
    // No overflow
    return SmallInt::build(vm, c);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).subtract(vm, right);
  }
}

UnstableNode SmallInt::multiply(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return multiplyValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).multiply(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

bool SmallInt::testMultiplyOverflow(nativeint a, nativeint b) {
  // This is platform dependent (2's complement)

  nativeint absa = a < 0 ? -a : a;
  nativeint absb = b < 0 ? -b : b;

  // Fast test first
  // If both absa and absb < sqrt(max()), then obviously there is no overflow
  const int bits = std::numeric_limits<nativeint>::digits / 2;
  if (((absa | absb) >> bits) == 0)
    return false;

  // Slow test (because of the division)
  return (b != 0) && (absa >= std::numeric_limits<nativeint>::max() / absb);
}

UnstableNode SmallInt::multiplyValue(VM vm, nativeint b) {
  nativeint a = value();

  // Detecting overflow
  if (!testMultiplyOverflow(a, b)) {
    // No overflow
    return SmallInt::build(vm, a * b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).multiply(vm, right);
  }
}

UnstableNode SmallInt::div(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return divValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).div(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::divValue(VM vm, nativeint b) {
  nativeint a = value();
  if (b == 0) {
    raiseKernelError(vm, "Integer division: Division by zero");
  }

  // Detecting overflow
  if ((a != std::numeric_limits<nativeint>::min()) || (b != -1)) {
    // No overflow
    return SmallInt::build(vm, a / b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).div(vm, right);
  }
}

UnstableNode SmallInt::mod(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return modValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).mod(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::modValue(VM vm, nativeint b) {
  nativeint a = value();

  // Detecting overflow
  if ((a != std::numeric_limits<nativeint>::min()) || (b != -1)) {
    // No overflow
    return SmallInt::build(vm, a % b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).mod(vm, right);
  }
}

UnstableNode SmallInt::abs(VM vm) {
  nativeint a = value();
  // Detecting overflow - platform dependent (2's complement)
  if (a != std::numeric_limits<nativeint>::min()) {
    // No overflow
    return SmallInt::build(vm, a >= 0 ? a : -a);
  } else {
    UnstableNode big = vm->newBigInt(std::numeric_limits<nativeint>::min());
    return Numeric(big).opposite(vm);
  }
}

}

#endif // MOZART_GENERATOR

#endif // MOZART_SMALLINT_H
<commit_msg>Fix some overflow conditions on SmallInt<commit_after>// Copyright © 2011, Université catholique de Louvain
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// *  Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimer.
// *  Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

#ifndef MOZART_SMALLINT_H
#define MOZART_SMALLINT_H

#include "mozartcore.hh"

#include <string>
#include <limits>

#ifndef MOZART_GENERATOR

namespace mozart {

//////////////
// SmallInt //
//////////////

#include "SmallInt-implem.hh"

void SmallInt::create(nativeint& self, VM vm, GR gr, SmallInt from) {
  self = from.value();
}

bool SmallInt::equals(VM vm, RichNode right) {
  return value() == right.as<SmallInt>().value();
}

int SmallInt::compareFeatures(VM vm, RichNode right) {
  auto rhs = right.as<SmallInt>().value();

  if (value() == rhs)
    return 0;
  else if (value() < rhs)
    return -1;
  else
    return 1;
}

// Comparable ------------------------------------------------------------------

int SmallInt::compare(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return (value() == smallInt) ? 0 : (value() < smallInt) ? -1 : 1;
  } else if (right.is<BigInt>()) {
    UnstableNode self = SmallInt::build(vm, value());
    return -Comparable(right).compare(vm, self);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

// Numeric ---------------------------------------------------------------------

UnstableNode SmallInt::opposite(VM vm) {
  // Detecting overflow - platform dependent (2's complement)
  if (value() != std::numeric_limits<nativeint>::min()) {
    // No overflow
    return SmallInt::build(vm, -value());
  } else {
    UnstableNode big = vm->newBigInt(std::numeric_limits<nativeint>::min());
    return Numeric(big).opposite(vm);
  }
}

UnstableNode SmallInt::add(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return add(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).add(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::add(VM vm, nativeint b) {
  nativeint a = value();
  nativeint c = a + b;

  // Detecting overflow - platform dependent (2's complement)
  if ((((a ^ c) & (b ^ c)) >> std::numeric_limits<nativeint>::digits) == 0) {
    // No overflow
    return SmallInt::build(vm, c);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).add(vm, right);
  }
}

UnstableNode SmallInt::subtract(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return subtractValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).subtract(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::subtractValue(VM vm, nativeint b) {
  nativeint a = value();
  nativeint c = a - b;

  // Detecting overflow - platform dependent (2's complement)
  if (((~(a ^ ~b) & (a ^ c)) >> std::numeric_limits<nativeint>::digits) == 0) {
    // No overflow
    return SmallInt::build(vm, c);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).subtract(vm, right);
  }
}

UnstableNode SmallInt::multiply(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return multiplyValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).multiply(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

bool SmallInt::testMultiplyOverflow(nativeint a, nativeint b) {
  // This is platform dependent (2's complement)

  nativeint absa = a < 0 ? -a : a;
  nativeint absb = b < 0 ? -b : b;

  // Fast test first
  // If both absa and absb < sqrt(max()), then obviously there is no overflow
  const int bits = std::numeric_limits<nativeint>::digits / 2;
  if (((absa | absb) >> bits) == 0)
    return false;

  // Slow test (because of the division)
  return (a == std::numeric_limits<nativeint>::min()) ||
         ((b != 0) && (absa >= std::numeric_limits<nativeint>::max() / absb));
}

UnstableNode SmallInt::multiplyValue(VM vm, nativeint b) {
  nativeint a = value();

  // Detecting overflow
  if (!testMultiplyOverflow(a, b)) {
    // No overflow
    return SmallInt::build(vm, a * b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).multiply(vm, right);
  }
}

UnstableNode SmallInt::div(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return divValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).div(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::divValue(VM vm, nativeint b) {
  nativeint a = value();
  if (b == 0) {
    raiseKernelError(vm, "Integer division: Division by zero");
  }

  // Detecting overflow
  if ((a != std::numeric_limits<nativeint>::min()) || (b != -1)) {
    // No overflow
    return SmallInt::build(vm, a / b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).div(vm, right);
  }
}

UnstableNode SmallInt::mod(VM vm, RichNode right) {
  using namespace mozart::patternmatching;

  nativeint smallInt;
  if (matches(vm, right, capture(smallInt))) {
    return modValue(vm, smallInt);
  } else if (right.is<BigInt>()) {
    UnstableNode big = vm->newBigInt(value());
    return Numeric(big).mod(vm, right);
  } else {
    raiseTypeError(vm, "Integer", right);
  }
}

UnstableNode SmallInt::modValue(VM vm, nativeint b) {
  nativeint a = value();

  // Detecting overflow
  if ((a != std::numeric_limits<nativeint>::min()) || (b != -1)) {
    // No overflow
    return SmallInt::build(vm, a % b);
  } else {
    UnstableNode left = vm->newBigInt(a);
    UnstableNode right = SmallInt::build(vm, b);
    return Numeric(left).mod(vm, right);
  }
}

UnstableNode SmallInt::abs(VM vm) {
  nativeint a = value();
  // Detecting overflow - platform dependent (2's complement)
  if (a != std::numeric_limits<nativeint>::min()) {
    // No overflow
    return SmallInt::build(vm, a >= 0 ? a : -a);
  } else {
    UnstableNode big = vm->newBigInt(std::numeric_limits<nativeint>::min());
    return Numeric(big).opposite(vm);
  }
}

}

#endif // MOZART_GENERATOR

#endif // MOZART_SMALLINT_H
<|endoftext|>


<commit_before>
#include "tokenizer.hpp"

namespace mustache {

void Tokenizer::setStartSequence(const std::string& start) {
  _startSequence.assign(start);
}

void Tokenizer::setStartSequence(const char * start, long len) {
  if( len <= 0 ) {
    _startSequence.assign(start);
  } else {
    _startSequence.assign(start, (size_t) len);
  }
}

void Tokenizer::setStopSequence(const std::string& stop) {
  _stopSequence.assign(stop);
}

void Tokenizer::setStopSequence(const char * stop, long len) {
  if( len <= 0 ) {
    _stopSequence.assign(stop);
  } else {
    _stopSequence.assign(stop, (size_t) len);
  }
}

void Tokenizer::setEscapeByDefault(bool flag) {
  _escapeByDefault = flag;
}

const std::string& Tokenizer::getStartSequence() {
  return _startSequence;
}

const std::string& Tokenizer::getStopSequence() {
  return _stopSequence;
}

bool Tokenizer::getEscapeByDefault() {
  return _escapeByDefault;
}

void Tokenizer::tokenize(std::string * tmpl, Node * root)
{
  unsigned int tmplL = tmpl->length();

  std::string start(_startSequence);
  char startC = start.at(0);
  long startL = start.length();

  std::string stop(_stopSequence);
  char stopC = stop.at(0);
  long stopL = stop.length();
  long tmpStopL = stopL;

  long pos = 0;
  const char * chr;
  long skipUntil = -1;
  long lineNo = 1;
  long charNo = 0;

  int inTag = 0;
  int inTripleTag = 0;
  int skip = 0;
  int startCharNo = 0;
  int startLineNo = 0;
  int currentFlags = Node::FlagNone;
  std::string buffer;
  buffer.reserve(100); // Reserver 100 chars

  int depth = 0;
  Node::Stack nodeStack;
  Node * node;

  // Initialize root node and stack[0]
  root->type = Node::TypeRoot;
  root->flags = Node::FlagNone;
  root->data = NULL;

  nodeStack.push(root);

  // Scan loop
  chr = tmpl->c_str();
  for( pos = 0; pos < tmplL; pos++ ) {

    // Track line numbers
    if( *chr == '\n' ) {
      lineNo++;
      charNo = 0;
    } else {
      charNo++;
    }

    // Skip until position
    if( skipUntil > -1 ) {
      if( pos <= skipUntil ) {
        chr++;
        continue;
      } else {
        skipUntil = -1;
      }
    }

    // Main
    if( !inTag ) {
      if( *chr == startC && tmpl->compare(pos, startL, start) == 0 ) {
        // Close previous buffer
        if( buffer.length() > 0 ) {
          node = new Node();
          node->type = Node::TypeOutput;
          node->data = new std::string(buffer);
          nodeStack.top()->children.push_back(node);
          buffer.clear();
        }
        // Open new buffer
        inTag = true;
        skipUntil = pos + startL - 1;
        startLineNo = lineNo;
        startCharNo = charNo; // Could be inaccurate
        // Triple mustache
        if( start.compare("{{") == 0 && tmpl->compare(pos+2, 1, "{") == 0 ) {
          inTripleTag = true;
          skipUntil++;
          startCharNo++;
        }
      }
    } else {
      if( *chr == stopC && tmpl->compare(pos, stopL, stop) == 0 ) {
        // Trim buffer
        trim(buffer);
        if( buffer.length() <= 0 ) {
          std::ostringstream oss;
          oss << "Empty tag"
              << " at "
              << startLineNo << ":" << startCharNo;
          throw Exception(oss.str());
        }
        // Close and process previous buffer
        skip = false;
        tmpStopL = stopL;
        currentFlags = Node::FlagNone;
        switch( buffer.at(0) ) {
          case '&':
            currentFlags = Node::FlagEscape;
            break;
          case '^':
            currentFlags = Node::FlagNegate;
            break;
          case '#':
            currentFlags = Node::FlagSection;
            break;
          case '/':
            currentFlags = Node::FlagStop;
            break;
          case '!':
            currentFlags = Node::FlagComment;
            break;
          case '>':
            currentFlags = Node::FlagPartial;
            break;
          case '<':
            currentFlags = Node::FlagInlinePartial;
            break;
          case '=':
            if( buffer.at(buffer.length()-1) != '=' ) {
              std::ostringstream oss;
              oss << "Missing closing delimiter (=)"
                  << " in tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }
            trim(buffer, " \f\n\r\t\v=");

            std::vector<std::string> delims;
            stringTok(buffer, whiteSpaces, &delims);
            if( delims.size() != 2 || delims[0].size() < 1 || delims[1].size() < 1 ) {
              std::ostringstream oss;
              oss << "Invalid delimiter format"
                  << " in tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }

            // Assign new start/stop
            start.assign(delims.at(0));
            startC = start.at(0);
            startL = start.length();
            stop.assign(delims.at(1));
            stopC = stop.at(0);
            stopL = stop.length();
            skip = 1;
            break;
        }
        if( !skip ) {
          if( currentFlags > 0 ) {
            buffer.erase(0, 1);
            trim(buffer);
          }
          if( inTripleTag ) {
            currentFlags = currentFlags | Node::FlagEscape;
          }
          // Create node
          if( currentFlags & Node::FlagInlinePartial ) {
            root->partials.insert(std::make_pair(buffer, mustache::Node()));
            node = &(root->partials[buffer]);
            node->type = Node::TypeRoot; // Kind of hackish
            node->data = new std::string(buffer);
            node->flags = currentFlags;
          } else {
            node = new Node();
            node->type = Node::TypeTag;
            node->data = new std::string(buffer);
            node->flags = currentFlags;
            nodeStack.top()->children.push_back(node);
          }
          // Push/pop stack
          if( currentFlags & Node::FlagHasChildren ) {
            depth++;
            nodeStack.push(node);
          } else if( currentFlags & Node::FlagStop ) {
            nodeStack.pop();
            depth--;
            if( depth < 0 ) {
              std::ostringstream oss;
              oss << "Extra closing section or missing opening section"
                  << " detected after tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }
          }
        }
        // Clear buffer
        buffer.clear();
        // Open new buffer
        inTag = false;
        skipUntil = pos + tmpStopL - 1;
        // Triple mustache
        if( !skip && inTripleTag && stop.compare("}}") == 0 ) {
          if( tmpl->compare(pos+2, 1, "}") != 0 ) {
            std::ostringstream oss;
            oss << "Missing closing triple mustache delimiter at "
                << lineNo << ":" << charNo
                << " in tag starting at "
                << startLineNo << ":" << startCharNo;
            throw Exception(oss.str());
          }
          skipUntil++;
          startCharNo++;
        }
        inTripleTag = false;
        startLineNo = lineNo;
        startCharNo = charNo; // Could be inaccurate
      }
    }

    // Append to buffer
    if( skipUntil == -1 ) {
      buffer.append(1, *chr);
    }

    // Increment character pointer
    chr++;
  }

  if( inTag ) {
    std::ostringstream oss;
    oss << "Unclosed tag at end of template, "
        << "starting at "
        << startLineNo << ":" << startCharNo;
    throw Exception(oss.str());
  } else if( buffer.length() > 0 ) {
    node = new Node();
    node->type = Node::TypeOutput;
    node->data = new std::string(buffer);
    nodeStack.top()->children.push_back(node);
    buffer.clear();
  }
}

} // namespace Mustache
<commit_msg>Adding better parse error messages<commit_after>
#include "tokenizer.hpp"

namespace mustache {

void Tokenizer::setStartSequence(const std::string& start) {
  _startSequence.assign(start);
}

void Tokenizer::setStartSequence(const char * start, long len) {
  if( len <= 0 ) {
    _startSequence.assign(start);
  } else {
    _startSequence.assign(start, (size_t) len);
  }
}

void Tokenizer::setStopSequence(const std::string& stop) {
  _stopSequence.assign(stop);
}

void Tokenizer::setStopSequence(const char * stop, long len) {
  if( len <= 0 ) {
    _stopSequence.assign(stop);
  } else {
    _stopSequence.assign(stop, (size_t) len);
  }
}

void Tokenizer::setEscapeByDefault(bool flag) {
  _escapeByDefault = flag;
}

const std::string& Tokenizer::getStartSequence() {
  return _startSequence;
}

const std::string& Tokenizer::getStopSequence() {
  return _stopSequence;
}

bool Tokenizer::getEscapeByDefault() {
  return _escapeByDefault;
}

void Tokenizer::tokenize(std::string * tmpl, Node * root)
{
  unsigned int tmplL = tmpl->length();

  std::string start(_startSequence);
  char startC = start.at(0);
  long startL = start.length();

  std::string stop(_stopSequence);
  char stopC = stop.at(0);
  long stopL = stop.length();
  long tmpStopL = stopL;

  long pos = 0;
  const char * chr;
  long skipUntil = -1;
  long lineNo = 1;
  long charNo = 0;

  int inTag = 0;
  int inTripleTag = 0;
  int skip = 0;
  int startCharNo = 0;
  int startLineNo = 0;
  int currentFlags = Node::FlagNone;
  std::string buffer;
  buffer.reserve(100); // Reserver 100 chars

  int depth = 0;
  Node::Stack nodeStack;
  Node * node;

  // Initialize root node and stack[0]
  root->type = Node::TypeRoot;
  root->flags = Node::FlagNone;
  root->data = NULL;

  nodeStack.push(root);

  // Scan loop
  chr = tmpl->c_str();
  for( pos = 0; pos < tmplL; pos++ ) {

    // Track line numbers
    if( *chr == '\n' ) {
      lineNo++;
      charNo = 0;
    } else {
      charNo++;
    }

    // Skip until position
    if( skipUntil > -1 ) {
      if( pos <= skipUntil ) {
        chr++;
        continue;
      } else {
        skipUntil = -1;
      }
    }

    // Main
    if( !inTag ) {
      if( *chr == startC && tmpl->compare(pos, startL, start) == 0 ) {
        // Close previous buffer
        if( buffer.length() > 0 ) {
          node = new Node();
          node->type = Node::TypeOutput;
          node->data = new std::string(buffer);
          nodeStack.top()->children.push_back(node);
          buffer.clear();
        }
        // Open new buffer
        inTag = true;
        skipUntil = pos + startL - 1;
        startLineNo = lineNo;
        startCharNo = charNo; // Could be inaccurate
        // Triple mustache
        if( start.compare("{{") == 0 && tmpl->compare(pos+2, 1, "{") == 0 ) {
          inTripleTag = true;
          skipUntil++;
        }
      }
    } else {
      if( *chr == stopC && tmpl->compare(pos, stopL, stop) == 0 ) {
        // Trim buffer
        trim(buffer);
        if( buffer.length() <= 0 ) {
          std::ostringstream oss;
          oss << "Empty tag"
              << " at "
              << startLineNo << ":" << startCharNo;
          throw Exception(oss.str());
        }
        // Close and process previous buffer
        skip = false;
        tmpStopL = stopL;
        currentFlags = Node::FlagNone;
        switch( buffer.at(0) ) {
          case '&':
            currentFlags = Node::FlagEscape;
            break;
          case '^':
            currentFlags = Node::FlagNegate;
            break;
          case '#':
            currentFlags = Node::FlagSection;
            break;
          case '/':
            currentFlags = Node::FlagStop;
            break;
          case '!':
            currentFlags = Node::FlagComment;
            break;
          case '>':
            currentFlags = Node::FlagPartial;
            break;
          case '<':
            currentFlags = Node::FlagInlinePartial;
            break;
          case '=':
            if( buffer.at(buffer.length()-1) != '=' ) {
              std::ostringstream oss;
              oss << "Missing closing delimiter (=)"
                  << " in tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }
            trim(buffer, " \f\n\r\t\v=");

            std::vector<std::string> delims;
            stringTok(buffer, whiteSpaces, &delims);
            if( delims.size() != 2 || delims[0].size() < 1 || delims[1].size() < 1 ) {
              std::ostringstream oss;
              oss << "Invalid delimiter format"
                  << " in tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }

            // Assign new start/stop
            start.assign(delims.at(0));
            startC = start.at(0);
            startL = start.length();
            stop.assign(delims.at(1));
            stopC = stop.at(0);
            stopL = stop.length();
            skip = 1;
            break;
        }
        if( !skip ) {
          if( currentFlags > 0 ) {
            buffer.erase(0, 1);
            trim(buffer);
          }
          if( inTripleTag ) {
            currentFlags = currentFlags | Node::FlagEscape;
          }
          // Create node
          if( currentFlags & Node::FlagInlinePartial ) {
            root->partials.insert(std::make_pair(buffer, mustache::Node()));
            node = &(root->partials[buffer]);
            node->type = Node::TypeRoot; // Kind of hackish
            node->data = new std::string(buffer);
            node->flags = currentFlags;
          } else {
            node = new Node();
            node->type = Node::TypeTag;
            node->data = new std::string(buffer);
            node->flags = currentFlags;
            nodeStack.top()->children.push_back(node);
          }
          // Push/pop stack
          if( currentFlags & Node::FlagHasChildren ) {
            depth++;
            nodeStack.push(node);
          } else if( currentFlags & Node::FlagStop ) {
            nodeStack.pop();
            depth--;
            if( depth < 0 ) {
              std::ostringstream oss;
              oss << "Extra closing section or missing opening section"
                  << " detected after tag starting at "
                  << startLineNo << ":" << startCharNo;
              throw Exception(oss.str());
            }
          }
        }
        // Clear buffer
        buffer.clear();
        // Open new buffer
        inTag = false;
        skipUntil = pos + tmpStopL - 1;
        // Triple mustache
        if( !skip && inTripleTag && stop.compare("}}") == 0 ) {
          if( tmpl->compare(pos+2, 1, "}") != 0 ) {
            std::ostringstream oss;
            oss << "Missing closing triple mustache delimiter at "
                << lineNo << ":" << charNo
                << " in tag starting at "
                << startLineNo << ":" << startCharNo;
            throw Exception(oss.str());
          }
          skipUntil++;
        }
        inTripleTag = false;
        startLineNo = lineNo;
        startCharNo = charNo; // Could be inaccurate
      }
    }

    // Append to buffer
    if( skipUntil == -1 ) {
      buffer.append(1, *chr);
    }

    // Increment character pointer
    chr++;
  }

  if( inTag ) {
    std::ostringstream oss;
    oss << "Unclosed tag at end of template, "
        << "starting at "
        << startLineNo << ":" << startCharNo;
    throw Exception(oss.str());
  } else if( depth > 0 ) {
    std::ostringstream oss;
    oss << "Unclosed section at end of template";
    throw Exception(oss.str());
  } else if( buffer.length() > 0 ) {
    node = new Node();
    node->type = Node::TypeOutput;
    node->data = new std::string(buffer);
    nodeStack.top()->children.push_back(node);
    buffer.clear();
  }
}

} // namespace Mustache
<|endoftext|>


<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdint.h>
#include <assert.h>

#include "mailbox.h"
#include "v3d.h"

// I/O access
volatile unsigned *v3d;
int mbox;

// Execute a nop control list to prove that we have contol.
void testControlLists() {
// First we allocate and map some videocore memory to build our control list in.

  // ask the blob to allocate us 256 bytes with 4k alignment, zero it.
  unsigned int handle = mem_alloc(mbox, 0x100, 0x1000, MEM_FLAG_COHERENT | MEM_FLAG_ZERO);
  if (!handle) {
    printf("Error: Unable to allocate memory");
    return;
  }
  // ask the blob to lock our memory at a single location at give is that address.
  uint32_t bus_addr = mem_lock(mbox, handle);
  // map that address into our local address space.
  uint8_t *list = (uint8_t*) mapmem(bus_addr, 0x100);

// Now we construct our control list.
  // 255 nops, with a halt somewhere in the middle
  for(int i = 0; i < 0xff; i++) {
    list[i] = 1; // NOP
  }
  list[0xbb] = 0; // Halt.

// And then we setup the v3d pipeline to execute the control list.
  printf("V3D_CT0CS: 0x%08x\n", v3d[V3D_CT0CS]);
  printf("Start Address: 0x%08x\n", bus_addr);
  // Current Address = start of our list (bus address)
  v3d[V3D_CT0CA] = bus_addr;
  // End Address = just after the end of our list (bus address)
  // This also starts execution.
  v3d[V3D_CT0EA] = bus_addr + 0x100;

  // print status while running
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

  // Wait a second to be sure the contorl list execution has finished
  while(v3d[V3D_CT0CS] & 0x20);

  // print the status again.
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

// Release memory;
  unmapmem((void *) list, 0x100);
  mem_unlock(mbox, handle);
  mem_free(mbox, handle);
}

void addbyte(uint8_t **list, uint8_t d) {
  *((*list)++) = d;
}

void addshort(uint8_t **list, uint16_t d) {
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
}

void addword(uint8_t **list, uint32_t d) {
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
  *((*list)++) = (d >> 16) & 0xff;
  *((*list)++) = (d >> 24) & 0xff;
}

void addfloat(uint8_t **list, float f) {
  uint32_t d = *((uint32_t *)&f);
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
  *((*list)++) = (d >> 16) & 0xff;
  *((*list)++) = (d >> 24) & 0xff;
}

void testBinner() {
// Like above, we allocate/lock/map some videocore memory
  // I'm just shoving everything in a single buffer because I'm lazy
  // 8Mb, 4k alignment
  unsigned int handle = mem_alloc(mbox, 0x800000, 0x1000, MEM_FLAG_COHERENT | MEM_FLAG_ZERO);
  if (!handle) {
    printf("Error: Unable to allocate memory");
    return;
  }
  uint32_t bus_addr = mem_lock(mbox, handle);
  uint8_t *list = (uint8_t*) mapmem(bus_addr, 0x800000);

  uint8_t *p = list;

// Configuration stuff
  // Tile Binning Configuration.
  //   Tile state data is 48 bytes per tile, I think it can be thrown away
  //   as soon as binning is finished.
  addbyte(&p, 112);
  addword(&p, bus_addr + 0x6200); // tile allocation memory address
  addword(&p, 0x8000); // tile allocation memory size
  addword(&p, bus_addr + 0x100); // Tile state data address
  addbyte(&p, 30); // 1920/64
  addbyte(&p, 17); // 1080/64 (16.875)
  addbyte(&p, 0x04); // config

  // Start tile binning.
  addbyte(&p, 6);

  // Primitive type
  addbyte(&p, 56);
  addbyte(&p, 0x32); // 16 bit triangle

  // Clip Window
  addbyte(&p, 102);
  addshort(&p, 0);
  addshort(&p, 0);
  addshort(&p, 1920); // width
  addshort(&p, 1080); // height

  // State
  addbyte(&p, 96);
  addbyte(&p, 0x03); // enable both foward and back facing polygons
  addbyte(&p, 0x00); // depth testing disabled
  addbyte(&p, 0x02); // enable early depth write

  // Viewport offset
  addbyte(&p, 103);
  addshort(&p, 0);
  addshort(&p, 0);

// The triangle
  // No Vertex Shader state (takes pre-transformed vertexes,
  // so we don't have to supply a working coordinate shader to test the binner.
  addbyte(&p, 65);
  addword(&p, bus_addr + 0x80); // Shader Record

  // primitive index list
  addbyte(&p, 32);
  addbyte(&p, 0x04); // 8bit index, trinagles
  addword(&p, 3); // Length
  addword(&p, bus_addr + 0x70); // address
  addword(&p, 2); // Maximum index

// End of bin list
  // Flush
  addbyte(&p, 5);
  // Nop
  addbyte(&p, 1);
  // Halt
  addbyte(&p, 0);

  int length = p - list;
  assert(length < 0x80);

// Shader Record
  p = list + 0x80;
  addbyte(&p, 0x01); // flags
  addbyte(&p, 6*4); // stride
  addbyte(&p, 0xcc); // num uniforms (not used)
  addbyte(&p, 0); // num varyings
  addword(&p, bus_addr + 0xfe00); // Fragment shader code
  addword(&p, bus_addr + 0xff00); // Fragment shader uniforms
  addword(&p, bus_addr + 0xa0); // Vertex Data

// Vertex Data
  p = list + 0xa0;
  // Vertex: Top, red
  addshort(&p, 200 << 4); // Y in 12.4 fixed point
  addshort(&p, (1920/2) << 4); // X in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 1.0f); // Varying 0 (Red)
  addfloat(&p, 0.0f); // Varying 1 (Green)
  addfloat(&p, 0.0f); // Varying 2 (Blue)

  // Vertex: bottom left, Green
  addshort(&p, 800 << 4); // Y in 12.4 fixed point
  addshort(&p, 560 << 4); // X in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 0.0f); // Varying 0 (Red)
  addfloat(&p, 1.0f); // Varying 1 (Green)
  addfloat(&p, 0.0f); // Varying 2 (Blue)

  // Vertex: bottom right, Blue
  addshort(&p, 800 << 4); // Y in 12.4 fixed point
  addshort(&p, 1460 << 4); // X in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 0.0f); // Varying 0 (Red)
  addfloat(&p, 0.0f); // Varying 1 (Green)
  addfloat(&p, 1.0f); // Varying 2 (Blue)

// Vertex list
  p = list + 0x70;
  addbyte(&p, 0); // top
  addbyte(&p, 1); // bottom left
  addbyte(&p, 2); // bottom right

// fragment shader
  p = list + 0xfe00;
  addword(&p, 0xffffffff);
  addword(&p, 0xe0020ba7); /* ldi tlbc, 0xffffffff */
  addword(&p, 0x009e7000);
  addword(&p, 0x500009e7); /* nop; nop; sbdone */
  addword(&p, 0x009e7000);
  addword(&p, 0x300009e7); /* nop; nop; thrend */
  addword(&p, 0x009e7000);
  addword(&p, 0x100009e7); /* nop; nop; nop */
  addword(&p, 0x009e7000);
  addword(&p, 0x100009e7); /* nop; nop; nop */

// Render control list
  p = list + 0xe200;

  // Clear color
  addbyte(&p, 114);
  addword(&p, 0);
  addword(&p, 0);
  addword(&p, 0);
  addbyte(&p, 0);

  // Tile Rendering Mode Configuration
  addbyte(&p, 113);
  addword(&p, bus_addr + 0x10000); // framebuffer addresss
  addshort(&p, 1920); // width
  addshort(&p, 1080); // height
  addbyte(&p, 0x04); // framebuffer mode (linear rgba8888)
  addbyte(&p, 0x00);

  // Link all binned lists together
  for(int x = 0; x < 30; x++) {
    for(int y = 0; y < 17; y++) {

      // Tile Coordinates
      addbyte(&p, 115);
      addbyte(&p, x);
      addbyte(&p, y);

      // Call Tile sublist
      addbyte(&p, 17);
      addword(&p, bus_addr + 0x6200 + (y * 30 + x) * 32);

      // Last tile needs a special store instruction
      if(x == 29 && y == 16) {
        // Store resolved tile color buffer and signal end of frame
        addbyte(&p, 25);
      } else {
        // Store resolved tile color buffer
        addbyte(&p, 24);
      }
    }
  }

  int render_length = p - (list + 0xe200);

// Run our control list
  printf("Binner control list constructed\n");
  printf("Start Address: 0x%08x, length: 0x%x\n", bus_addr, length);

  v3d[V3D_CT0CA] = bus_addr;
  v3d[V3D_CT0EA] = bus_addr + length;
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

  // Wait for control list to execute
  while(v3d[V3D_CT0CS] & 0x20);

  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);
  printf("V3D_CT1CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT1CS], v3d[V3D_CT1CA]);

  v3d[V3D_CT1CA] = bus_addr + 0xe200;
  v3d[V3D_CT1EA] = bus_addr + 0xe200 + render_length;

  sleep(1);
  //while(v3d[V3D_CT1CS] & 0x20);

  printf("V3D_CT1CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT1CS], v3d[V3D_CT1CA]);
  v3d[V3D_CT1CS] = 0x20;

  // just dump the buffer to a file
  FILE *f = fopen("binner_dump.bin", "w");
  for(int i = 0; i < 0x800000; i++) {
    fputc(list[i], f);
  }
  fclose(f);
  printf("Buffer containing binned tile lists dumpped to binner_dump.bin\n");

// Release resources
  unmapmem((void *) list, 0x800000);
  mem_unlock(mbox, handle);
  mem_free(mbox, handle);
}

int main(int argc, char **argv) {
  mbox = mbox_open();

  // The blob now has this nice handy call which powers up the v3d pipeline.
  qpu_enable(mbox, 1);

  // map v3d's registers into our address space.
  v3d = (unsigned *) mapmem(0x20c00000, 0x1000);

  if(v3d[V3D_IDENT0] != 0x02443356) { // Magic number.
    printf("Error: V3D pipeline isn't powered up and accessable.\n");
    exit(-1);
  }

  // We now have access to the v3d registers, we should do something.
  testBinner();

  return 0;
}

<commit_msg>Swap vertex x and y coordinates to correct location.<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <stdint.h>
#include <assert.h>

#include "mailbox.h"
#include "v3d.h"

// I/O access
volatile unsigned *v3d;
int mbox;

// Execute a nop control list to prove that we have contol.
void testControlLists() {
// First we allocate and map some videocore memory to build our control list in.

  // ask the blob to allocate us 256 bytes with 4k alignment, zero it.
  unsigned int handle = mem_alloc(mbox, 0x100, 0x1000, MEM_FLAG_COHERENT | MEM_FLAG_ZERO);
  if (!handle) {
    printf("Error: Unable to allocate memory");
    return;
  }
  // ask the blob to lock our memory at a single location at give is that address.
  uint32_t bus_addr = mem_lock(mbox, handle);
  // map that address into our local address space.
  uint8_t *list = (uint8_t*) mapmem(bus_addr, 0x100);

// Now we construct our control list.
  // 255 nops, with a halt somewhere in the middle
  for(int i = 0; i < 0xff; i++) {
    list[i] = 1; // NOP
  }
  list[0xbb] = 0; // Halt.

// And then we setup the v3d pipeline to execute the control list.
  printf("V3D_CT0CS: 0x%08x\n", v3d[V3D_CT0CS]);
  printf("Start Address: 0x%08x\n", bus_addr);
  // Current Address = start of our list (bus address)
  v3d[V3D_CT0CA] = bus_addr;
  // End Address = just after the end of our list (bus address)
  // This also starts execution.
  v3d[V3D_CT0EA] = bus_addr + 0x100;

  // print status while running
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

  // Wait a second to be sure the contorl list execution has finished
  while(v3d[V3D_CT0CS] & 0x20);

  // print the status again.
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

// Release memory;
  unmapmem((void *) list, 0x100);
  mem_unlock(mbox, handle);
  mem_free(mbox, handle);
}

void addbyte(uint8_t **list, uint8_t d) {
  *((*list)++) = d;
}

void addshort(uint8_t **list, uint16_t d) {
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
}

void addword(uint8_t **list, uint32_t d) {
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
  *((*list)++) = (d >> 16) & 0xff;
  *((*list)++) = (d >> 24) & 0xff;
}

void addfloat(uint8_t **list, float f) {
  uint32_t d = *((uint32_t *)&f);
  *((*list)++) = (d) & 0xff;
  *((*list)++) = (d >> 8)  & 0xff;
  *((*list)++) = (d >> 16) & 0xff;
  *((*list)++) = (d >> 24) & 0xff;
}

void testBinner() {
// Like above, we allocate/lock/map some videocore memory
  // I'm just shoving everything in a single buffer because I'm lazy
  // 8Mb, 4k alignment
  unsigned int handle = mem_alloc(mbox, 0x800000, 0x1000, MEM_FLAG_COHERENT | MEM_FLAG_ZERO);
  if (!handle) {
    printf("Error: Unable to allocate memory");
    return;
  }
  uint32_t bus_addr = mem_lock(mbox, handle);
  uint8_t *list = (uint8_t*) mapmem(bus_addr, 0x800000);

  uint8_t *p = list;

// Configuration stuff
  // Tile Binning Configuration.
  //   Tile state data is 48 bytes per tile, I think it can be thrown away
  //   as soon as binning is finished.
  addbyte(&p, 112);
  addword(&p, bus_addr + 0x6200); // tile allocation memory address
  addword(&p, 0x8000); // tile allocation memory size
  addword(&p, bus_addr + 0x100); // Tile state data address
  addbyte(&p, 30); // 1920/64
  addbyte(&p, 17); // 1080/64 (16.875)
  addbyte(&p, 0x04); // config

  // Start tile binning.
  addbyte(&p, 6);

  // Primitive type
  addbyte(&p, 56);
  addbyte(&p, 0x32); // 16 bit triangle

  // Clip Window
  addbyte(&p, 102);
  addshort(&p, 0);
  addshort(&p, 0);
  addshort(&p, 1920); // width
  addshort(&p, 1080); // height

  // State
  addbyte(&p, 96);
  addbyte(&p, 0x03); // enable both foward and back facing polygons
  addbyte(&p, 0x00); // depth testing disabled
  addbyte(&p, 0x02); // enable early depth write

  // Viewport offset
  addbyte(&p, 103);
  addshort(&p, 0);
  addshort(&p, 0);

// The triangle
  // No Vertex Shader state (takes pre-transformed vertexes,
  // so we don't have to supply a working coordinate shader to test the binner.
  addbyte(&p, 65);
  addword(&p, bus_addr + 0x80); // Shader Record

  // primitive index list
  addbyte(&p, 32);
  addbyte(&p, 0x04); // 8bit index, trinagles
  addword(&p, 3); // Length
  addword(&p, bus_addr + 0x70); // address
  addword(&p, 2); // Maximum index

// End of bin list
  // Flush
  addbyte(&p, 5);
  // Nop
  addbyte(&p, 1);
  // Halt
  addbyte(&p, 0);

  int length = p - list;
  assert(length < 0x80);

// Shader Record
  p = list + 0x80;
  addbyte(&p, 0x01); // flags
  addbyte(&p, 6*4); // stride
  addbyte(&p, 0xcc); // num uniforms (not used)
  addbyte(&p, 0); // num varyings
  addword(&p, bus_addr + 0xfe00); // Fragment shader code
  addword(&p, bus_addr + 0xff00); // Fragment shader uniforms
  addword(&p, bus_addr + 0xa0); // Vertex Data

// Vertex Data
  p = list + 0xa0;
  // Vertex: Top, red
  addshort(&p, (1920/2) << 4); // X in 12.4 fixed point
  addshort(&p, 200 << 4); // Y in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 1.0f); // Varying 0 (Red)
  addfloat(&p, 0.0f); // Varying 1 (Green)
  addfloat(&p, 0.0f); // Varying 2 (Blue)

  // Vertex: bottom left, Green
  addshort(&p, 560 << 4); // X in 12.4 fixed point
  addshort(&p, 800 << 4); // Y in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 0.0f); // Varying 0 (Red)
  addfloat(&p, 1.0f); // Varying 1 (Green)
  addfloat(&p, 0.0f); // Varying 2 (Blue)

  // Vertex: bottom right, Blue
  addshort(&p, 1460 << 4); // X in 12.4 fixed point
  addshort(&p, 800 << 4); // Y in 12.4 fixed point
  addfloat(&p, 1.0f); // Z
  addfloat(&p, 1.0f); // 1/W
  addfloat(&p, 0.0f); // Varying 0 (Red)
  addfloat(&p, 0.0f); // Varying 1 (Green)
  addfloat(&p, 1.0f); // Varying 2 (Blue)

// Vertex list
  p = list + 0x70;
  addbyte(&p, 0); // top
  addbyte(&p, 1); // bottom left
  addbyte(&p, 2); // bottom right

// fragment shader
  p = list + 0xfe00;
  addword(&p, 0xffffffff);
  addword(&p, 0xe0020ba7); /* ldi tlbc, 0xffffffff */
  addword(&p, 0x009e7000);
  addword(&p, 0x500009e7); /* nop; nop; sbdone */
  addword(&p, 0x009e7000);
  addword(&p, 0x300009e7); /* nop; nop; thrend */
  addword(&p, 0x009e7000);
  addword(&p, 0x100009e7); /* nop; nop; nop */
  addword(&p, 0x009e7000);
  addword(&p, 0x100009e7); /* nop; nop; nop */

// Render control list
  p = list + 0xe200;

  // Clear color
  addbyte(&p, 114);
  addword(&p, 0);
  addword(&p, 0);
  addword(&p, 0);
  addbyte(&p, 0);

  // Tile Rendering Mode Configuration
  addbyte(&p, 113);
  addword(&p, bus_addr + 0x10000); // framebuffer addresss
  addshort(&p, 1920); // width
  addshort(&p, 1080); // height
  addbyte(&p, 0x04); // framebuffer mode (linear rgba8888)
  addbyte(&p, 0x00);

  // Link all binned lists together
  for(int x = 0; x < 30; x++) {
    for(int y = 0; y < 17; y++) {

      // Tile Coordinates
      addbyte(&p, 115);
      addbyte(&p, x);
      addbyte(&p, y);

      // Call Tile sublist
      addbyte(&p, 17);
      addword(&p, bus_addr + 0x6200 + (y * 30 + x) * 32);

      // Last tile needs a special store instruction
      if(x == 29 && y == 16) {
        // Store resolved tile color buffer and signal end of frame
        addbyte(&p, 25);
      } else {
        // Store resolved tile color buffer
        addbyte(&p, 24);
      }
    }
  }

  int render_length = p - (list + 0xe200);

// Run our control list
  printf("Binner control list constructed\n");
  printf("Start Address: 0x%08x, length: 0x%x\n", bus_addr, length);

  v3d[V3D_CT0CA] = bus_addr;
  v3d[V3D_CT0EA] = bus_addr + length;
  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);

  // Wait for control list to execute
  while(v3d[V3D_CT0CS] & 0x20);

  printf("V3D_CT0CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT0CS], v3d[V3D_CT0CA]);
  printf("V3D_CT1CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT1CS], v3d[V3D_CT1CA]);

  v3d[V3D_CT1CA] = bus_addr + 0xe200;
  v3d[V3D_CT1EA] = bus_addr + 0xe200 + render_length;

  sleep(1);
  //while(v3d[V3D_CT1CS] & 0x20);

  printf("V3D_CT1CS: 0x%08x, Address: 0x%08x\n", v3d[V3D_CT1CS], v3d[V3D_CT1CA]);
  v3d[V3D_CT1CS] = 0x20;

  // just dump the buffer to a file
  FILE *f = fopen("binner_dump.bin", "w");
  for(int i = 0; i < 0x800000; i++) {
    fputc(list[i], f);
  }
  fclose(f);
  printf("Buffer containing binned tile lists dumpped to binner_dump.bin\n");

// Release resources
  unmapmem((void *) list, 0x800000);
  mem_unlock(mbox, handle);
  mem_free(mbox, handle);
}

int main(int argc, char **argv) {
  mbox = mbox_open();

  // The blob now has this nice handy call which powers up the v3d pipeline.
  qpu_enable(mbox, 1);

  // map v3d's registers into our address space.
  v3d = (unsigned *) mapmem(0x20c00000, 0x1000);

  if(v3d[V3D_IDENT0] != 0x02443356) { // Magic number.
    printf("Error: V3D pipeline isn't powered up and accessable.\n");
    exit(-1);
  }

  // We now have access to the v3d registers, we should do something.
  testBinner();

  return 0;
}

<|endoftext|>


<commit_before>#include "toolchain.h"

Toolchain::Toolchain(QObject *parent) : QObject(parent) {
	this->process = new QProcess(this);

	Toolchain::connect(this->process, &QProcess::started, this, &Toolchain::process_started);
	Toolchain::connect(this->process, &QProcess::readyReadStandardOutput, this, &Toolchain::process_readyReadStandardOutput);
	Toolchain::connect(this->process, SIGNAL(error(QProcess::ProcessError)), this, SLOT(process_error(QProcess::ProcessError)));
	Toolchain::connect(this->process, SIGNAL(finished(int , QProcess::ExitStatus )), this, SLOT(process_finished(int)));

	this->state = Toolchain::StateIdle;
	this->terminated = false;

	this->version();
}

Toolchain::~Toolchain() {

	if(this->process->state() != QProcess::NotRunning) {
		this->process->kill();
		this->process->waitForFinished();
	}

	delete this->process;
}

bool Toolchain::isReady() {
	return (this->process->state() == QProcess::NotRunning && this->state == Toolchain::StateIdle);
}

void Toolchain::version() {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateVersion;
	this->run(QStringList() << "version");
}

void Toolchain::init(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateInit;
	this->run(QStringList() << "init" << "-source" << project->path());
}

void Toolchain::serve(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateServe;
	this->run(QStringList() << "serve" << "-source" << project->path());
}

void Toolchain::build(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateBuild;
	this->run(QStringList() << "build" << "-source" << project->path());
}

void Toolchain::stop() {
	this->terminated = true;
	this->process->kill();
}

QByteArray Toolchain::readOutput() {
	return this->process->readAllStandardOutput();
}

QString Toolchain::versionString() {
	return this->toolchainVersion;
}

void Toolchain::run(QStringList arguments) {
	if(this->process->state() == QProcess::NotRunning) {
		this->process->start("tiberius",arguments,QProcess::ReadOnly);
	}
}

void Toolchain::process_finished(int exitCode) {
	Toolchain::State oldState = this->state;

	this->state = Toolchain::StateIdle;

	if(oldState == Toolchain::StateVersion) {

		if(exitCode != 0) {
			emit this->error();
		}

		this->toolchainVersion = this->process->readAllStandardOutput();

		if(this->toolchainVersion.startsWith("Tiberius ")) {
			emit this->available(this->toolchainVersion);
			return;
		} else {
			this->toolchainVersion = QString();
			emit this->error();
			return;
		}
	}

	emit this->finished((!this->terminated && exitCode != 0), oldState);

	this->terminated = false;

}

void Toolchain::process_started() {
	if(this->state != Toolchain::StateVersion) {
		emit this->started();
	}
}

void Toolchain::process_readyReadStandardOutput() {
	if(this->state != Toolchain::StateVersion) {
		emit this->hasOutput();
	}
}

void Toolchain::process_error(QProcess::ProcessError error) {
	emit this->error();
}
<commit_msg>Fix error emitted when the process crashed due to our kill command.<commit_after>#include "toolchain.h"

Toolchain::Toolchain(QObject *parent) : QObject(parent) {
	this->process = new QProcess(this);

	Toolchain::connect(this->process, &QProcess::started, this, &Toolchain::process_started);
	Toolchain::connect(this->process, &QProcess::readyReadStandardOutput, this, &Toolchain::process_readyReadStandardOutput);
	Toolchain::connect(this->process, SIGNAL(error(QProcess::ProcessError)), this, SLOT(process_error(QProcess::ProcessError)));
	Toolchain::connect(this->process, SIGNAL(finished(int , QProcess::ExitStatus )), this, SLOT(process_finished(int)));

	this->state = Toolchain::StateIdle;
	this->terminated = false;

	this->version();
}

Toolchain::~Toolchain() {

	if(this->process->state() != QProcess::NotRunning) {
		this->process->kill();
		this->process->waitForFinished();
	}

	delete this->process;
}

bool Toolchain::isReady() {
	return (this->process->state() == QProcess::NotRunning && this->state == Toolchain::StateIdle);
}

void Toolchain::version() {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateVersion;
	this->run(QStringList() << "version");
}

void Toolchain::init(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateInit;
	this->run(QStringList() << "init" << "-source" << project->path());
}

void Toolchain::serve(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateServe;
	this->run(QStringList() << "serve" << "-source" << project->path());
}

void Toolchain::build(Project *project) {

	if(this->state != Toolchain::StateIdle) {
		return;
	}

	this->state = Toolchain::StateBuild;
	this->run(QStringList() << "build" << "-source" << project->path());
}

void Toolchain::stop() {
	this->terminated = true;
	this->process->kill();
}

QByteArray Toolchain::readOutput() {
	return this->process->readAllStandardOutput();
}

QString Toolchain::versionString() {
	return this->toolchainVersion;
}

void Toolchain::run(QStringList arguments) {
	if(this->process->state() == QProcess::NotRunning) {
		this->process->start("tiberius",arguments,QProcess::ReadOnly);
	}
}

void Toolchain::process_finished(int exitCode) {
	Toolchain::State oldState = this->state;

	this->state = Toolchain::StateIdle;

	if(oldState == Toolchain::StateVersion) {

		if(exitCode != 0) {
			emit this->error();
		}

		this->toolchainVersion = this->process->readAllStandardOutput();

		if(this->toolchainVersion.startsWith("Tiberius ")) {
			emit this->available(this->toolchainVersion);
			return;
		} else {
			this->toolchainVersion = QString();
			emit this->error();
			return;
		}
	}

	emit this->finished((!this->terminated && exitCode != 0), oldState);

	this->terminated = false;

}

void Toolchain::process_started() {
	if(this->state != Toolchain::StateVersion) {
		emit this->started();
	}
}

void Toolchain::process_readyReadStandardOutput() {
	if(this->state != Toolchain::StateVersion) {
		emit this->hasOutput();
	}
}

void Toolchain::process_error(QProcess::ProcessError error) {

	Toolchain::State oldState = this->state;

	this->state = Toolchain::StateIdle;

	if(error == QProcess::FailedToStart) {
		emit this->error();
	} else {

		// This happens when we kill the process (serve, ..)
		emit this->finished(false, oldState);
	}

}
<|endoftext|>


<commit_before>#include "vocab.h"

using namespace std;

////////////////////////////////////////////////////////
inline std::vector<std::string> Tokenize(const std::string& str,
    const std::string& delimiters = " \t")
{
  std::vector<std::string> tokens;
  // Skip delimiters at beginning.
  std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
  // Find first "non-delimiter".
  std::string::size_type pos = str.find_first_of(delimiters, lastPos);

  while (std::string::npos != pos || std::string::npos != lastPos) {
    // Found a token, add it to the vector.
    tokens.push_back(str.substr(lastPos, pos - lastPos));
    // Skip delimiters.  Note the "not_of"
    lastPos = str.find_first_not_of(delimiters, pos);
    // Find next "non-delimiter"
    pos = str.find_first_of(delimiters, lastPos);
  }

  return tokens;
}
////////////////////////////////////////////////////////
size_t Vocab::GetUNK() const
{
	return std::numeric_limits<size_t>::max();
}

size_t Vocab::GetPad() const
{
	return std::numeric_limits<size_t>::max() - 1;
}

size_t Vocab::GetEOS() const
{
	return std::numeric_limits<size_t>::max() - 2;
}

size_t Vocab::GetOrCreate(const std::string &word)
{
	size_t id;
	Coll::const_iterator iter = coll_.find(word);
	if (iter == coll_.end()) {
		id = coll_.size();
		coll_[word] = id;
	}
	else {
		id = iter->second;
	}
	return id;
}

std::vector<size_t> Vocab::ProcessSentence(const std::string &sentence)
{
	vector<string> toks = Tokenize(sentence);
	vector<size_t> ret(toks.size());

	for (size_t i = 0; i < toks.size(); ++i) {
		size_t id = GetOrCreate(toks[i]);
		ret[i] = id;
	}

	return ret;
}
<commit_msg>#include<commit_after>#include <limits>
#include "vocab.h"

using namespace std;

////////////////////////////////////////////////////////
inline std::vector<std::string> Tokenize(const std::string& str,
    const std::string& delimiters = " \t")
{
  std::vector<std::string> tokens;
  // Skip delimiters at beginning.
  std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
  // Find first "non-delimiter".
  std::string::size_type pos = str.find_first_of(delimiters, lastPos);

  while (std::string::npos != pos || std::string::npos != lastPos) {
    // Found a token, add it to the vector.
    tokens.push_back(str.substr(lastPos, pos - lastPos));
    // Skip delimiters.  Note the "not_of"
    lastPos = str.find_first_not_of(delimiters, pos);
    // Find next "non-delimiter"
    pos = str.find_first_of(delimiters, lastPos);
  }

  return tokens;
}
////////////////////////////////////////////////////////
size_t Vocab::GetUNK() const
{
	return std::numeric_limits<size_t>::max();
}

size_t Vocab::GetPad() const
{
	return std::numeric_limits<size_t>::max() - 1;
}

size_t Vocab::GetEOS() const
{
	return std::numeric_limits<size_t>::max() - 2;
}

size_t Vocab::GetOrCreate(const std::string &word)
{
	size_t id;
	Coll::const_iterator iter = coll_.find(word);
	if (iter == coll_.end()) {
		id = coll_.size();
		coll_[word] = id;
	}
	else {
		id = iter->second;
	}
	return id;
}

std::vector<size_t> Vocab::ProcessSentence(const std::string &sentence)
{
	vector<string> toks = Tokenize(sentence);
	vector<size_t> ret(toks.size());

	for (size_t i = 0; i < toks.size(); ++i) {
		size_t id = GetOrCreate(toks[i]);
		ret[i] = id;
	}

	return ret;
}
<|endoftext|>


<commit_before>#include <iostream>
#include <sqlite3.h>
#include <string>
#include <cstring>
#include <map>
#include <sstream>
#include "ReadGzip.h"
using namespace std;
#include "MBTileReader.h"
#include "vector_tile20/vector_tile.pb.h"

int main(int argc, char **argv)
{
	class MBTileReader mbTileReader("cairo_egypt.mbtiles");

	cout << "name:" << mbTileReader.GetMetadata("name") << endl;
	cout << "type:" << mbTileReader.GetMetadata("type") << endl;
	string version = mbTileReader.GetMetadata("version");
	cout << "version:" << version << endl;
	cout << "description:" << mbTileReader.GetMetadata("description") << endl;
	string format = mbTileReader.GetMetadata("format");
	cout << "format:" << format << endl;
	cout << "bounds:" << mbTileReader.GetMetadata("bounds") << endl;

	if(0) //Get metadata fields
	{
		std::vector<std::string> fieldNames;
		mbTileReader.GetMetadataFields(fieldNames);
		for(unsigned i=0;i<fieldNames.size();i++) cout << fieldNames[i] << endl;
	}

	if(0) //Get list of tiles
	{
		TileInfoRows tileInfoRows;
		mbTileReader.ListTiles(tileInfoRows);
		for(unsigned i=0;i < tileInfoRows.size(); i++)
		{
			for(size_t j=0; j < tileInfoRows[i].size(); j++)
				cout << tileInfoRows[i][j] << ",";
			cout << endl;
		}
	}

	string blob;
	mbTileReader.GetTile(14,9618,9611,blob);
	cout << "blob size: "<< blob.size() << endl;

	if(format == "pbf" && version == "2.0")
	{
		std::stringbuf buff;
		buff.sputn(blob.c_str(), blob.size());
		DecodeGzip dec(buff);

		string tileData;
		char tmp[1024];
		while(dec.in_avail())
		{
			streamsize bytes = dec.sgetn(tmp, 1024);
			tileData.insert(0, tmp, bytes);
		}

		vector_tile::Tile tile;
		cout << tile.ParseFromString(tileData) << endl;
		cout << tile.layers_size() << endl;
	}
}

<commit_msg>Successfully parsing the pbf<commit_after>#include <iostream>
#include <sqlite3.h>
#include <string>
#include <cstring>
#include <map>
#include <sstream>
#include "ReadGzip.h"
using namespace std;
#include "MBTileReader.h"
#include "vector_tile20/vector_tile.pb.h"

int main(int argc, char **argv)
{
	class MBTileReader mbTileReader("cairo_egypt.mbtiles");

	cout << "name:" << mbTileReader.GetMetadata("name") << endl;
	cout << "type:" << mbTileReader.GetMetadata("type") << endl;
	string version = mbTileReader.GetMetadata("version");
	cout << "version:" << version << endl;
	cout << "description:" << mbTileReader.GetMetadata("description") << endl;
	string format = mbTileReader.GetMetadata("format");
	cout << "format:" << format << endl;
	cout << "bounds:" << mbTileReader.GetMetadata("bounds") << endl;

	if(0) //Get metadata fields
	{
		std::vector<std::string> fieldNames;
		mbTileReader.GetMetadataFields(fieldNames);
		for(unsigned i=0;i<fieldNames.size();i++) cout << fieldNames[i] << endl;
	}

	if(0) //Get list of tiles
	{
		TileInfoRows tileInfoRows;
		mbTileReader.ListTiles(tileInfoRows);
		for(unsigned i=0;i < tileInfoRows.size(); i++)
		{
			for(size_t j=0; j < tileInfoRows[i].size(); j++)
				cout << tileInfoRows[i][j] << ",";
			cout << endl;
		}
	}

	string blob;
	mbTileReader.GetTile(14,9618,9611,blob);

	if(format == "pbf" && version == "2.0")
	{
		std::stringbuf buff;
		buff.sputn(blob.c_str(), blob.size());
		DecodeGzip dec(buff);

		string tileData;

		char tmp[1024];
		while(dec.in_avail())
		{
			streamsize bytes = dec.sgetn(tmp, 1024);
			tileData.append(tmp, bytes);
		}

		vector_tile::Tile tile;
		cout << "ParseFromString: " << tile.ParseFromString(tileData) << endl;
		cout << "Num layers: " << tile.layers_size() << endl;
	}
}

<|endoftext|>


<commit_before>/*
 * Daniel Alves Paladim 2015
 */

#include <iostream>
#include "callByName.h"

FUNCTION_CALL_BY_NAME(AAAA) {
    std::cout << "Called AAAA\n";
}

FUNCTION_CALL_BY_NAME(BBBB) {
    std::cout << "Called BBBB\n";
}

int main(int argc, char* argv[]) {
    callFunction(argv[1]);
    return 0;
}<commit_msg>Improved the test<commit_after>/*
 * Daniel Alves Paladim 2015
 */

#include <iostream>
#include "callByName.h"

FUNCTION_CALL_BY_NAME(first_function) {
    std::cout << "Called first_function\n";
}

FUNCTION_CALL_BY_NAME(second_function) {
    std::cout << "Called second_function\n";
}

int main(int argc, char* argv[]) {
    if(argc > 1) {
        try {
        callFunction(argv[1]);
        } catch(std::out_of_range& ex) {
            std::cerr << "The function " << argv[1] << " does not exist." << '\n';
        }
    }
    else {
        std::cout << "No function to call provided. Type: " << argv[0] << " FUNCTION_NAME \n"
                     "where FUNCTION_NAME is any of the following\n";
        for(const auto& name_function : functionContainer) {
            std::cout << '\t' << name_function.first << '\n';
        }
    }
    return 0;
}<|endoftext|>


<commit_before>#ifndef CHIG_HAS_UUID
#define CHIG_HAS_UUID

#include <boost/uuid/uuid.hpp>

#include <utility>

namespace std {

template <>
struct hash<boost::uuids::uuid> {
	size_t operator()(const boost::uuids::uuid& toHash) const {
		return boost::uuids::hash_value(toHash);
	}
};
}

#endif  // CHIG_HAS_UUID
<commit_msg>Fix include in hash UUID<commit_after>#ifndef CHIG_HAS_UUID
#define CHIG_HAS_UUID

#include <boost/uuid/uuid.hpp>

#include <functional>

namespace std {

template <>
struct hash<boost::uuids::uuid> {
	size_t operator()(const boost::uuids::uuid& toHash) const {
		return boost::uuids::hash_value(toHash);
	}
};
}

#endif  // CHIG_HAS_UUID
<|endoftext|>


<commit_before>//===- BreakCriticalEdges.cpp - Critical Edge Elimination Pass ------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// BreakCriticalEdges pass - Break all of the critical edges in the CFG by
// inserting a dummy basic block.  This pass may be "required" by passes that
// cannot deal with critical edges.  For this usage, the structure type is
// forward declared.  This pass obviously invalidates the CFG, but can update
// forward dominator (set, immediate dominators, tree, and frontier)
// information.
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Analysis/Dominators.h"
#include "llvm/Function.h"
#include "llvm/Instructions.h"
#include "llvm/Type.h"
#include "llvm/Support/CFG.h"
#include "llvm/ADT/Statistic.h"
using namespace llvm;

namespace {
  Statistic<> NumBroken("break-crit-edges", "Number of blocks inserted");

  struct BreakCriticalEdges : public FunctionPass {
    virtual bool runOnFunction(Function &F);

    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
      AU.addPreserved<DominatorSet>();
      AU.addPreserved<ImmediateDominators>();
      AU.addPreserved<DominatorTree>();
      AU.addPreserved<DominanceFrontier>();

      // No loop canonicalization guarantees are broken by this pass.
      AU.addPreservedID(LoopSimplifyID);
    }
  };

  RegisterOpt<BreakCriticalEdges> X("break-crit-edges",
                                    "Break critical edges in CFG");
}

// Publically exposed interface to pass...
const PassInfo *llvm::BreakCriticalEdgesID = X.getPassInfo();
FunctionPass *llvm::createBreakCriticalEdgesPass() {
  return new BreakCriticalEdges();
}

// runOnFunction - Loop over all of the edges in the CFG, breaking critical
// edges as they are found.
//
bool BreakCriticalEdges::runOnFunction(Function &F) {
  bool Changed = false;
  for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
    TerminatorInst *TI = I->getTerminator();
    if (TI->getNumSuccessors() > 1)
      for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
        if (SplitCriticalEdge(TI, i, this)) {
          ++NumBroken;
          Changed = true;
        }
  }

  return Changed;
}

//===----------------------------------------------------------------------===//
//    Implementation of the external critical edge manipulation functions
//===----------------------------------------------------------------------===//

// isCriticalEdge - Return true if the specified edge is a critical edge.
// Critical edges are edges from a block with multiple successors to a block
// with multiple predecessors.
//
bool llvm::isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum) {
  assert(SuccNum < TI->getNumSuccessors() && "Illegal edge specification!");
  if (TI->getNumSuccessors() == 1) return false;

  const BasicBlock *Dest = TI->getSuccessor(SuccNum);
  pred_const_iterator I = pred_begin(Dest), E = pred_end(Dest);

  // If there is more than one predecessor, this is a critical edge...
  assert(I != E && "No preds, but we have an edge to the block?");
  ++I;        // Skip one edge due to the incoming arc from TI.
  return I != E;
}

// SplitCriticalEdge - If this edge is a critical edge, insert a new node to
// split the critical edge.  This will update DominatorSet, ImmediateDominator,
// DominatorTree, and DominatorFrontier information if it is available, thus
// calling this pass will not invalidate either of them.  This returns true if
// the edge was split, false otherwise.
//
bool llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, Pass *P) {
  if (!isCriticalEdge(TI, SuccNum)) return false;
  BasicBlock *TIBB = TI->getParent();
  BasicBlock *DestBB = TI->getSuccessor(SuccNum);

  // Create a new basic block, linking it into the CFG.
  BasicBlock *NewBB = new BasicBlock(TIBB->getName() + "." +
                                     DestBB->getName() + "_crit_edge");
  // Create our unconditional branch...
  new BranchInst(DestBB, NewBB);

  // Branch to the new block, breaking the edge...
  TI->setSuccessor(SuccNum, NewBB);

  // Insert the block into the function... right after the block TI lives in.
  Function &F = *TIBB->getParent();
  F.getBasicBlockList().insert(TIBB->getNext(), NewBB);

  // If there are any PHI nodes in DestBB, we need to update them so that they
  // merge incoming values from NewBB instead of from TIBB.
  //
  for (BasicBlock::iterator I = DestBB->begin(); isa<PHINode>(I); ++I) {
    PHINode *PN = cast<PHINode>(I);
    // We no longer enter through TIBB, now we come in through NewBB.  Revector
    // exactly one entry in the PHI node that used to come from TIBB to come
    // from NewBB.
    Value *InVal = PN->removeIncomingValue(TIBB, false);
    PN->addIncoming(InVal, NewBB);
  }

  // If we don't have a pass object, we can't update anything...
  if (P == 0) return true;

  // Now update analysis information.  These are the analyses that we are
  // currently capable of updating...
  //

  // Should we update DominatorSet information?
  if (DominatorSet *DS = P->getAnalysisToUpdate<DominatorSet>()) {
    // The blocks that dominate the new one are the blocks that dominate TIBB
    // plus the new block itself.
    DominatorSet::DomSetType DomSet = DS->getDominators(TIBB);
    DomSet.insert(NewBB);  // A block always dominates itself.
    DS->addBasicBlock(NewBB, DomSet);
  }

  // Should we update ImmediateDominator information?
  if (ImmediateDominators *ID = P->getAnalysisToUpdate<ImmediateDominators>()) {
    // TIBB is the new immediate dominator for NewBB.  NewBB doesn't dominate
    // anything.
    ID->addNewBlock(NewBB, TIBB);
  }

  // Should we update DominatorTree information?
  if (DominatorTree *DT = P->getAnalysisToUpdate<DominatorTree>()) {
    DominatorTree::Node *TINode = DT->getNode(TIBB);

    // The new block is not the immediate dominator for any other nodes, but
    // TINode is the immediate dominator for the new node.
    //
    if (TINode)        // Don't break unreachable code!
      DT->createNewNode(NewBB, TINode);
  }

  // Should we update DominanceFrontier information?
  if (DominanceFrontier *DF = P->getAnalysisToUpdate<DominanceFrontier>()) {
    // Since the new block is dominated by its only predecessor TIBB,
    // it cannot be in any block's dominance frontier.  Its dominance
    // frontier is {DestBB}.
    DominanceFrontier::DomSetType NewDFSet;
    NewDFSet.insert(DestBB);
    DF->addBasicBlock(NewBB, NewDFSet);
  }
  return true;
}
<commit_msg>Change break critical edges to not remove, then insert, PHI node entries. Instead, just update the BB in-place.  This is both faster, and it prevents split-critical-edges from shuffling the PHI argument list unneccesarily.<commit_after>//===- BreakCriticalEdges.cpp - Critical Edge Elimination Pass ------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// BreakCriticalEdges pass - Break all of the critical edges in the CFG by
// inserting a dummy basic block.  This pass may be "required" by passes that
// cannot deal with critical edges.  For this usage, the structure type is
// forward declared.  This pass obviously invalidates the CFG, but can update
// forward dominator (set, immediate dominators, tree, and frontier)
// information.
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Analysis/Dominators.h"
#include "llvm/Function.h"
#include "llvm/Instructions.h"
#include "llvm/Type.h"
#include "llvm/Support/CFG.h"
#include "llvm/ADT/Statistic.h"
using namespace llvm;

namespace {
  Statistic<> NumBroken("break-crit-edges", "Number of blocks inserted");

  struct BreakCriticalEdges : public FunctionPass {
    virtual bool runOnFunction(Function &F);

    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
      AU.addPreserved<DominatorSet>();
      AU.addPreserved<ImmediateDominators>();
      AU.addPreserved<DominatorTree>();
      AU.addPreserved<DominanceFrontier>();

      // No loop canonicalization guarantees are broken by this pass.
      AU.addPreservedID(LoopSimplifyID);
    }
  };

  RegisterOpt<BreakCriticalEdges> X("break-crit-edges",
                                    "Break critical edges in CFG");
}

// Publically exposed interface to pass...
const PassInfo *llvm::BreakCriticalEdgesID = X.getPassInfo();
FunctionPass *llvm::createBreakCriticalEdgesPass() {
  return new BreakCriticalEdges();
}

// runOnFunction - Loop over all of the edges in the CFG, breaking critical
// edges as they are found.
//
bool BreakCriticalEdges::runOnFunction(Function &F) {
  bool Changed = false;
  for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
    TerminatorInst *TI = I->getTerminator();
    if (TI->getNumSuccessors() > 1)
      for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
        if (SplitCriticalEdge(TI, i, this)) {
          ++NumBroken;
          Changed = true;
        }
  }

  return Changed;
}

//===----------------------------------------------------------------------===//
//    Implementation of the external critical edge manipulation functions
//===----------------------------------------------------------------------===//

// isCriticalEdge - Return true if the specified edge is a critical edge.
// Critical edges are edges from a block with multiple successors to a block
// with multiple predecessors.
//
bool llvm::isCriticalEdge(const TerminatorInst *TI, unsigned SuccNum) {
  assert(SuccNum < TI->getNumSuccessors() && "Illegal edge specification!");
  if (TI->getNumSuccessors() == 1) return false;

  const BasicBlock *Dest = TI->getSuccessor(SuccNum);
  pred_const_iterator I = pred_begin(Dest), E = pred_end(Dest);

  // If there is more than one predecessor, this is a critical edge...
  assert(I != E && "No preds, but we have an edge to the block?");
  ++I;        // Skip one edge due to the incoming arc from TI.
  return I != E;
}

// SplitCriticalEdge - If this edge is a critical edge, insert a new node to
// split the critical edge.  This will update DominatorSet, ImmediateDominator,
// DominatorTree, and DominatorFrontier information if it is available, thus
// calling this pass will not invalidate either of them.  This returns true if
// the edge was split, false otherwise.
//
bool llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, Pass *P) {
  if (!isCriticalEdge(TI, SuccNum)) return false;
  BasicBlock *TIBB = TI->getParent();
  BasicBlock *DestBB = TI->getSuccessor(SuccNum);

  // Create a new basic block, linking it into the CFG.
  BasicBlock *NewBB = new BasicBlock(TIBB->getName() + "." +
                                     DestBB->getName() + "_crit_edge");
  // Create our unconditional branch...
  new BranchInst(DestBB, NewBB);

  // Branch to the new block, breaking the edge...
  TI->setSuccessor(SuccNum, NewBB);

  // Insert the block into the function... right after the block TI lives in.
  Function &F = *TIBB->getParent();
  F.getBasicBlockList().insert(TIBB->getNext(), NewBB);

  // If there are any PHI nodes in DestBB, we need to update them so that they
  // merge incoming values from NewBB instead of from TIBB.
  //
  for (BasicBlock::iterator I = DestBB->begin(); isa<PHINode>(I); ++I) {
    PHINode *PN = cast<PHINode>(I);
    // We no longer enter through TIBB, now we come in through NewBB.  Revector
    // exactly one entry in the PHI node that used to come from TIBB to come
    // from NewBB.
    int BBIdx = PN->getBasicBlockIndex(TIBB);
    PN->setIncomingBlock(BBIdx, NewBB);
  }

  // If we don't have a pass object, we can't update anything...
  if (P == 0) return true;

  // Now update analysis information.  These are the analyses that we are
  // currently capable of updating...
  //

  // Should we update DominatorSet information?
  if (DominatorSet *DS = P->getAnalysisToUpdate<DominatorSet>()) {
    // The blocks that dominate the new one are the blocks that dominate TIBB
    // plus the new block itself.
    DominatorSet::DomSetType DomSet = DS->getDominators(TIBB);
    DomSet.insert(NewBB);  // A block always dominates itself.
    DS->addBasicBlock(NewBB, DomSet);
  }

  // Should we update ImmediateDominator information?
  if (ImmediateDominators *ID = P->getAnalysisToUpdate<ImmediateDominators>()) {
    // TIBB is the new immediate dominator for NewBB.  NewBB doesn't dominate
    // anything.
    ID->addNewBlock(NewBB, TIBB);
  }

  // Should we update DominatorTree information?
  if (DominatorTree *DT = P->getAnalysisToUpdate<DominatorTree>()) {
    DominatorTree::Node *TINode = DT->getNode(TIBB);

    // The new block is not the immediate dominator for any other nodes, but
    // TINode is the immediate dominator for the new node.
    //
    if (TINode)        // Don't break unreachable code!
      DT->createNewNode(NewBB, TINode);
  }

  // Should we update DominanceFrontier information?
  if (DominanceFrontier *DF = P->getAnalysisToUpdate<DominanceFrontier>()) {
    // Since the new block is dominated by its only predecessor TIBB,
    // it cannot be in any block's dominance frontier.  Its dominance
    // frontier is {DestBB}.
    DominanceFrontier::DomSetType NewDFSet;
    NewDFSet.insert(DestBB);
    DF->addBasicBlock(NewBB, NewDFSet);
  }
  return true;
}
<|endoftext|>


<commit_before>#include "ms/isocalc.hpp"
#include "utils/metrics.hpp"

#include "cxxopts.hpp"

#include <iostream>
#include <vector>
#include <algorithm>
#include <string>
#include <stdexcept>
#include <fstream>
#include <functional>
#include <random>
#include <ctime>
#include <iomanip>

typedef utils::Metrics Metrics;

std::vector<Metrics> sortBy(const std::vector<Metrics>& metrics, std::function<double(const Metrics&)> f)
{
  std::vector<double> values(metrics.size());
  std::vector<size_t> indices(values.size());
  for (size_t i = 0; i < metrics.size(); i++) {
    values[i] = f(metrics[i]);
    indices[i] = i;
  }
  std::sort(indices.begin(), indices.end(),
            [&](size_t n, size_t m) { return values[n] < values[m]; });

  std::vector<Metrics> result(metrics.size());
  size_t k = 0;
  for (auto i : indices)
    result[k++] = metrics[i];
  return result;
}

std::vector<Metrics> readResults(const std::string& filename) {
  std::vector<Metrics> metrics;
  std::ifstream in(filename);
  Metrics m;
  std::string line;

  std::getline(in, line); // skip header
  if (line != Metrics::header())
    throw std::runtime_error("CSV header is incompatible with ims-fdr tool");

  while (Metrics::read(in, m))
    metrics.push_back(m);

  return sortBy(metrics, [](const Metrics& a) { return -a.msm();});
}

static std::mt19937 rng;

std::vector<float> estimateFDR(const std::vector<Metrics>& target_metrics,
                               const std::vector<Metrics>& decoy_metrics,
                               size_t n_top=std::numeric_limits<size_t>::max(),
                               bool verbose=false)
{
  std::vector<float> fdr;
  size_t n = target_metrics.size();

  std::vector<size_t> decoy_indices(n);
  std::iota(decoy_indices.begin(), decoy_indices.end(), 0UL);
  // reservoir sampling
  for (size_t i = n; i < decoy_metrics.size(); i++) {
    std::uniform_int_distribution<size_t> gen(0, i);
    auto k = gen(rng);
    if (k < n)
      decoy_indices[k] = i;
  }

  std::sort(decoy_indices.begin(), decoy_indices.end(),
            [&](size_t n, size_t m) {
              return decoy_metrics[n].msm() > decoy_metrics[m].msm();
            });

  size_t n_decoy_hits = 0;
  size_t n_target_hits = 0;
  auto it = decoy_indices.begin();
  for (auto& m: target_metrics) {
    if (n_target_hits >= n_top)
      break;
    while (it != decoy_indices.end() && decoy_metrics[*it].msm() > m.msm()) {
      if (verbose)
        std::cout << "\x1b[31mDECOY  \x1b[39m" << decoy_metrics[*it] << std::endl;
      ++it;
      ++n_decoy_hits;
    }
    ++n_target_hits;
    fdr.push_back(float(n_decoy_hits) / float(n_target_hits));

    if (verbose)
      std::cout << "\x1b[32mTARGET \x1b[39m" << m
                << "\t" << fdr.back() << "\t" << std::endl;
  }
  return fdr;
}

std::vector<float> estimateAverageFDR(const std::vector<Metrics>& target_metrics,
                                      const std::vector<Metrics>& decoy_metrics,
                                      size_t n_repetitions)
{
  // FIXME: use median instead of mean
  std::vector<float> average_fdr(target_metrics.size());
  for (size_t j = 0; j < n_repetitions; j++) {
    auto fdr = estimateFDR(target_metrics, decoy_metrics);
    for (size_t i = 0; i < target_metrics.size(); ++i)
      average_fdr[i] += fdr[i] / float(n_repetitions);
  }
  return average_fdr;
}

int fdr_main(int argc, char** argv) {
  std::string target_csv_fn, decoy_csv_fn, adduct, output_filename;
  unsigned n_repeats;

  cxxopts::Options options("ims fdr", " <target_results.csv> <decoy_results.csv>");
  options.add_options()
    ("out", "Output filename",
     cxxopts::value<std::string>(output_filename)->default_value("/dev/stdout"))
    ("repeats", "Number of random subsampling runs used for FDR estimation",
     cxxopts::value<unsigned>(n_repeats)->default_value("100"))
    ("adduct", "If specified, only use target results for a specific adduct",
     cxxopts::value<std::string>(adduct)->default_value(""))
    ("help", "Print help");

  options.add_options("hidden")
    ("target_csv", "", cxxopts::value<std::string>(target_csv_fn))
    ("decoy_csv", "", cxxopts::value<std::string>(decoy_csv_fn));

  options.parse_positional(std::vector<std::string>{"target_csv", "decoy_csv"});

  options.parse(argc, argv);

  if (options.count("help") || target_csv_fn.empty() || decoy_csv_fn.empty()) {
    std::cout << options.help({""}) << std::endl;
    return 0;
  }

  auto target_metrics = readResults(target_csv_fn);
  auto decoy_metrics = readResults(decoy_csv_fn);

  if (!adduct.empty()) {
    auto it = std::remove_if(target_metrics.begin(), target_metrics.end(),
                             [&](const Metrics& m) { return m.adduct != adduct; });
    target_metrics.erase(it, target_metrics.end());
  }

  rng.seed(std::random_device{}());

  auto fdr = estimateAverageFDR(target_metrics, decoy_metrics, n_repeats);
  std::ofstream out(output_filename);
  out << Metrics::header() << ",fdr" << std::endl;
  for (size_t j = 0; j < target_metrics.size(); j++)
    out << target_metrics[j] << "," << fdr[j] << std::endl;

  return 0;
}
<commit_msg>added --groundtruth option to ims-fdr<commit_after>#include "ms/isocalc.hpp"
#include "utils/metrics.hpp"

#include "cxxopts.hpp"

#include <iostream>
#include <vector>
#include <set>
#include <algorithm>
#include <string>
#include <sstream>
#include <stdexcept>
#include <fstream>
#include <functional>
#include <random>
#include <ctime>
#include <iomanip>

typedef utils::Metrics Metrics;

std::vector<Metrics> sortBy(const std::vector<Metrics>& metrics, std::function<double(const Metrics&)> f)
{
  std::vector<double> values(metrics.size());
  std::vector<size_t> indices(values.size());
  for (size_t i = 0; i < metrics.size(); i++) {
    values[i] = f(metrics[i]);
    indices[i] = i;
  }
  std::sort(indices.begin(), indices.end(),
            [&](size_t n, size_t m) { return values[n] < values[m]; });

  std::vector<Metrics> result(metrics.size());
  size_t k = 0;
  for (auto i : indices)
    result[k++] = metrics[i];
  return result;
}

std::vector<Metrics> readResults(const std::string& filename) {
  std::vector<Metrics> metrics;
  std::ifstream in(filename);
  Metrics m;
  std::string line;

  std::getline(in, line); // skip header
  if (line != Metrics::header())
    throw std::runtime_error("CSV header is incompatible with ims-fdr tool");

  while (Metrics::read(in, m))
    metrics.push_back(m);

  return sortBy(metrics, [](const Metrics& a) { return -a.msm();});
}

static std::mt19937 rng;

std::vector<float> estimateFDR(const std::vector<Metrics>& target_metrics,
                               const std::vector<Metrics>& decoy_metrics,
                               size_t n_top=std::numeric_limits<size_t>::max(),
                               bool verbose=false)
{
  std::vector<float> fdr;
  size_t n = target_metrics.size();

  std::vector<size_t> decoy_indices(n);
  std::iota(decoy_indices.begin(), decoy_indices.end(), 0UL);
  // reservoir sampling
  for (size_t i = n; i < decoy_metrics.size(); i++) {
    std::uniform_int_distribution<size_t> gen(0, i);
    auto k = gen(rng);
    if (k < n)
      decoy_indices[k] = i;
  }

  std::sort(decoy_indices.begin(), decoy_indices.end(),
            [&](size_t n, size_t m) {
              return decoy_metrics[n].msm() > decoy_metrics[m].msm();
            });

  size_t n_decoy_hits = 0;
  size_t n_target_hits = 0;
  auto it = decoy_indices.begin();
  for (auto& m: target_metrics) {
    if (n_target_hits >= n_top)
      break;
    while (it != decoy_indices.end() && decoy_metrics[*it].msm() > m.msm()) {
      if (verbose)
        std::cout << "\x1b[31mDECOY  \x1b[39m" << decoy_metrics[*it] << std::endl;
      ++it;
      ++n_decoy_hits;
    }
    ++n_target_hits;
    fdr.push_back(float(n_decoy_hits) / float(n_target_hits));

    if (verbose)
      std::cout << "\x1b[32mTARGET \x1b[39m" << m
                << "\t" << fdr.back() << "\t" << std::endl;
  }
  return fdr;
}

std::vector<float> estimateAverageFDR(const std::vector<Metrics>& target_metrics,
                                      const std::vector<Metrics>& decoy_metrics,
                                      size_t n_repetitions)
{
  // FIXME: use median instead of mean
  std::vector<float> average_fdr(target_metrics.size());
  for (size_t j = 0; j < n_repetitions; j++) {
    auto fdr = estimateFDR(target_metrics, decoy_metrics);
    for (size_t i = 0; i < target_metrics.size(); ++i)
      average_fdr[i] += fdr[i] / float(n_repetitions);
  }
  return average_fdr;
}

int fdr_main(int argc, char** argv) {
  std::string target_csv_fn, decoy_csv_fn, adduct, output_filename;
  unsigned n_repeats;
  std::string groundtruth_filename;

  cxxopts::Options options("ims fdr", " <target_results.csv> <decoy_results.csv>");
  options.add_options()
    ("out", "Output filename",
     cxxopts::value<std::string>(output_filename)->default_value("/dev/stdout"))
    ("repeats", "Number of random subsampling runs used for FDR estimation",
     cxxopts::value<unsigned>(n_repeats)->default_value("100"))
    ("adduct", "Only use target results for a specific adduct",
     cxxopts::value<std::string>(adduct)->default_value(""))
    ("groundtruth", "CSV file with ground-truth formula/adduct pairs",
     cxxopts::value<std::string>(groundtruth_filename)->default_value(""))
    ("help", "Print help");

  options.add_options("hidden")
    ("target_csv", "", cxxopts::value<std::string>(target_csv_fn))
    ("decoy_csv", "", cxxopts::value<std::string>(decoy_csv_fn));

  options.parse_positional(std::vector<std::string>{"target_csv", "decoy_csv"});

  options.parse(argc, argv);

  if (options.count("help") || target_csv_fn.empty() || decoy_csv_fn.empty()) {
    std::cout << options.help({""}) << std::endl;
    return 0;
  }

  auto target_metrics = readResults(target_csv_fn);
  auto decoy_metrics = readResults(decoy_csv_fn);

  if (!adduct.empty()) {
    auto it = std::remove_if(target_metrics.begin(), target_metrics.end(),
                             [&](const Metrics& m) { return m.adduct != adduct; });
    target_metrics.erase(it, target_metrics.end());
  }

  rng.seed(std::random_device{}());

  auto fdr = estimateAverageFDR(target_metrics, decoy_metrics, n_repeats);
  std::ofstream out(output_filename);

  if (groundtruth_filename.empty()) {
    out << Metrics::header() << ",fdr" << std::endl;
    for (size_t j = 0; j < target_metrics.size(); j++)
      out << target_metrics[j] << "," << fdr[j] << std::endl;
  } else {
    std::set<std::pair<std::string, std::string>> groundtruth;

    auto is_correct = [&](const Metrics& metrics) -> bool {
      auto key = std::make_pair(metrics.sf, metrics.adduct);
      return groundtruth.find(key) != groundtruth.end();
    };

    std::ifstream gt(groundtruth_filename);
    std::string line;
    while (std::getline(gt, line)) {
      std::istringstream is(line);
      std::string sf, adduct;
      std::getline(is, sf, ',');
      std::getline(is, adduct, ',');
      groundtruth.insert(std::make_pair(sf, adduct));
    }

    out << Metrics::header() << ",est_fdr,true_fdr" << std::endl;
    size_t true_hits = 0;

    for (size_t j = 0; j < target_metrics.size(); j++) {
      true_hits += is_correct(target_metrics[j]);
      double true_fdr = double(j + 1 - true_hits) / double(j + 1);
      out << target_metrics[j] << "," << fdr[j] << "," << true_fdr << std::endl;
    }
  }

  return 0;
}
<|endoftext|>