<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stx/io/fileutil.h>
#include <stx/io/mmappedfile.h>
#include <stx/util/binarymessagereader.h>
#include <stx/util/binarymessagewriter.h>
#include <sstable/SSTableEditor.h>
#include <sstable/sstablereader.h>
#include <zbase/core/RecordSet.h>

using namespace stx;

namespace zbase {

RecordRef::RecordRef(
    const SHA1Hash& _record_id,
    const Buffer& _record) :
    record_id(_record_id),
    record(_record) {}

RecordSet::RecordSet(
    const String& datadir,
    const String& filename_prefix,
    RecordSetState state /* = RecordSetState{} */) :
    datadir_(datadir),
    filename_prefix_(filename_prefix),
    state_(state),
    max_datafile_size_(kDefaultMaxDatafileSize) {
  auto id_index_fn = [this] (const SHA1Hash& id, const void* data, size_t size) {
    commitlog_ids_.emplace(id);
  };

  for (const auto& o : state_.old_commitlogs) {
    loadCommitlog(o, id_index_fn);
  }

  if (!state.commitlog.isEmpty()) {
    loadCommitlog(state.commitlog.get(), id_index_fn);
  }
}

RecordSet::RecordSetState RecordSet::getState() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_;
}

size_t RecordSet::version() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_.version;
}

SHA1Hash RecordSet::checksum() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_.checksum;
}

size_t RecordSet::commitlogSize() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return commitlog_ids_.size();
}

void RecordSet::addRecord(const SHA1Hash& record_id, const Buffer& message) {

  util::BinaryMessageWriter buf;
  buf.append(record_id.data(), record_id.size());
  buf.appendVarUInt(message.size());
  buf.append(message.data(), message.size());

  std::unique_lock<std::mutex> lk(mutex_);

  if (commitlog_ids_.count(record_id) > 0) {
    return;
  }

  addRecords(buf);

  commitlog_ids_.emplace(record_id);
}

void RecordSet::addRecords(const Vector<RecordRef>& records) {
  std::unique_lock<std::mutex> lk(mutex_);

  util::BinaryMessageWriter buf;
  for (const auto& rec : records) {
    if (commitlog_ids_.count(rec.record_id) > 0) {
      continue;
    }

    buf.append(rec.record_id.data(), rec.record_id.size());
    buf.appendVarUInt(rec.record.size());
    buf.append(rec.record.data(), rec.record.size());
  }

  if (buf.size() == 0) {
    return;
  }

  addRecords(buf);

  for (const auto& rec : records) {
    commitlog_ids_.emplace(rec.record_id);
  }
}

// precondition: must hold lock
void RecordSet::addRecords(const util::BinaryMessageWriter& buf) {
  String commitlog;
  uint64_t commitlog_size;
  if (state_.commitlog.isEmpty()) {
    commitlog = filename_prefix_ + rnd_.hex64() + ".log";
    commitlog_size = 0;
    ++state_.version;
  } else {
    commitlog = state_.commitlog.get();
    commitlog_size = state_.commitlog_size;
  }

  auto commitlog_path = FileUtil::joinPaths(datadir_, commitlog);
  auto file = File::openFile(
      commitlog_path,
      File::O_WRITE | File::O_CREATEOROPEN);

  file.truncate(sizeof(uint64_t) + commitlog_size + buf.size());
  file.seekTo(sizeof(uint64_t) + commitlog_size);
  file.write(buf.data(), buf.size());

  // FIXPAUL fsync here for non order preserving storage?

  commitlog_size += buf.size();
  file.seekTo(0);
  file.write(&commitlog_size, sizeof(commitlog_size));

  state_.commitlog = Some(commitlog);
  state_.commitlog_size = commitlog_size;
}

void RecordSet::rollCommitlog() {
  std::unique_lock<std::mutex> lk(mutex_);
  rollCommitlogWithLock();
}

// precondition: must hold lock
void RecordSet::rollCommitlogWithLock() {
  if (state_.commitlog.isEmpty()) {
    return;
  }

  auto old_log = state_.commitlog.get();
  auto old_log_path = FileUtil::joinPaths(datadir_, old_log);
  FileUtil::truncate(old_log_path, state_.commitlog_size + sizeof(uint64_t));

  state_.old_commitlogs.emplace(old_log);
  state_.commitlog = None<String>();
  state_.commitlog_size = 0;
  ++state_.version;
}

void RecordSet::compact() {
  Set<String> deleted_files;
  compact(&deleted_files);
}

void RecordSet::compact(Set<String>* deleted_files) {
  std::unique_lock<std::mutex> compact_lk(compact_mutex_, std::defer_lock);
  if (!compact_lk.try_lock()) {
    return; // compaction is already running
  }

  std::unique_lock<std::mutex> lk(mutex_);
  rollCommitlogWithLock();
  auto snap = state_;
  lk.unlock();

  if (snap.old_commitlogs.size() == 0) {
    return;
  }

  auto outfile_name = filename_prefix_ + rnd_.hex64() + ".sst";
  auto outfile_path = FileUtil::joinPaths(datadir_, outfile_name);
  auto outfile = sstable::SSTableEditor::create(
      outfile_path + "~",
      sstable::IndexProvider{},
      nullptr,
      0);

  size_t outfile_nrecords = 0;
  size_t outfile_offset = 0;

  Set<SHA1Hash> id_set;
  Set<SHA1Hash> new_id_set;

  bool rewrite_last = false;
  if (snap.datafiles.size() > 0) {
    auto cur_datafile_size = FileUtil::size(
        FileUtil::joinPaths(datadir_, snap.datafiles.back().filename));

    if (cur_datafile_size < max_datafile_size_) {
      rewrite_last = true;
    }
  }

  if (rewrite_last) {
    outfile_offset = snap.datafiles.back().offset;
  } else {
    outfile_offset = snap.datafiles.size() > 0 ?
        snap.datafiles.back().offset + snap.datafiles.back().num_records :
        0;
  }

  for (int j = 0; j < snap.datafiles.size(); ++j) {
    auto sstable_path = FileUtil::joinPaths(
        datadir_,
        snap.datafiles[j].filename);

    sstable::SSTableReader reader(sstable_path);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      void* key;
      size_t key_size;
      cursor->getKey(&key, &key_size);

      SHA1Hash msgid(key, key_size);
      id_set.emplace(msgid);

      if (rewrite_last && j + 1 == snap.datafiles.size()) {
        void* data;
        size_t data_size;
        cursor->getData(&data, &data_size);

        outfile->appendRow(key, key_size, data, data_size);
        ++outfile_nrecords;
      }

      if (!cursor->next()) {
        break;
      }
    }
  }

  for (const auto& cl : snap.old_commitlogs) {
    loadCommitlog(cl, [this, &outfile, &id_set, &new_id_set, &outfile_nrecords] (
        const SHA1Hash& id,
        const void* data,
        size_t size) {
      if (new_id_set.count(id) > 0) {
        return;
      }

      new_id_set.emplace(id);

      if (id_set.count(id) > 0) {
        return;
      }

      outfile->appendRow(id.data(), id.size(), data, size);
      ++outfile_nrecords;
    });
  }

  for (const auto& new_id : new_id_set) {
    id_set.emplace(new_id);
  }

  auto new_checksum = calculateChecksum(id_set);

  if (outfile_nrecords == 0) {
    FileUtil::rm(outfile_path + "~");
  } else {
    outfile->finalize();
    FileUtil::mv(outfile_path + "~", outfile_path);
  }

  lk.lock();

  if (rewrite_last) {
    deleted_files->emplace(state_.datafiles.back().filename);
    state_.datafiles.pop_back();
  }

  if (outfile_nrecords > 0) {
    state_.datafiles.emplace_back(DatafileRef {
      .filename = outfile_name,
      .num_records = outfile_nrecords,
      .offset = outfile_offset
    });
  }

  for (const auto& cl : snap.old_commitlogs) {
    state_.old_commitlogs.erase(cl);
    deleted_files->emplace(cl);
  }

  for (const auto& id : new_id_set) {
    commitlog_ids_.erase(id);
  }

  state_.checksum = new_checksum;
  ++state_.version;
}

void RecordSet::loadCommitlog(
    const String& filename,
    Function<void (const SHA1Hash&, const void*, size_t)> fn) {
  auto filepath = FileUtil::joinPaths(datadir_, filename);
  io::MmappedFile mmap(File::openFile(filepath, File::O_READ));
  util::BinaryMessageReader reader(mmap.data(), mmap.size());
  auto limit = *reader.readUInt64() + sizeof(uint64_t);

  while (reader.position() < limit) {
    auto id_data = reader.read(SHA1Hash::kSize);
    SHA1Hash id(id_data, SHA1Hash::kSize);
    auto len = reader.readVarUInt();
    auto data = reader.read(len);

    fn(id, data, len);
  }
}

uint64_t RecordSet::numRecords() const {
  uint64_t res = 0;

  std::unique_lock<std::mutex> lk(mutex_);
  for (const auto& df : state_.datafiles) {
    res += df.num_records;
  }

  return res;
}

uint64_t RecordSet::firstOffset() const {
  std::unique_lock<std::mutex> lk(mutex_);

  if (state_.datafiles.size() == 0) {
    return 0;
  } else {
    return state_.datafiles.front().offset;
  }
}

uint64_t RecordSet::lastOffset() const {
  std::unique_lock<std::mutex> lk(mutex_);

  if (state_.datafiles.size() == 0) {
    return 0;
  } else {
    return state_.datafiles.back().offset + state_.datafiles.back().num_records;
  }
}

void RecordSet::fetchRecords(
      uint64_t offset,
      uint64_t limit,
      Function<void (
          const SHA1Hash& record_id,
          const void* record_data,
          size_t record_size)> fn) {
  std::unique_lock<std::mutex> lk(mutex_);
  auto datafiles = state_.datafiles;
  lk.unlock();

  if (datafiles.size() == 0) {
    return;
  }

  size_t o = datafiles.front().offset;
  if (o > offset) {
    RAISEF(kRuntimeError, "offset was garbage collected: $0", offset);
  }

  size_t l = 0;
  for (const auto& datafile : datafiles) {
    if (o + datafile.num_records <= offset) {
      o += datafile.num_records;
      continue;
    }

    auto filepath = FileUtil::joinPaths(datadir_, datafile.filename);
    sstable::SSTableReader reader(filepath);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      if (++o > offset) {
        void* key;
        size_t key_size;
        cursor->getKey(&key, &key_size);
        SHA1Hash msgid(key, key_size);

        void* data;
        size_t data_size;
        cursor->getData(&data, &data_size);
        fn(msgid, data, data_size);

        if (++l == limit) {
          return;
        }
      }

      if (!cursor->next()) {
        break;
      }
    }
  }
}

Set<SHA1Hash> RecordSet::listRecords() const {
  Set<SHA1Hash> res;

  std::unique_lock<std::mutex> lk(mutex_);
  if (state_.datafiles.empty()) {
    return res;
  }

  auto datafiles = state_.datafiles;
  lk.unlock();

  for (const auto& datafile : datafiles) {
    auto filepath = FileUtil::joinPaths(datadir_, datafile.filename);
    sstable::SSTableReader reader(filepath);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      void* key;
      size_t key_size;
      cursor->getKey(&key, &key_size);

      res.emplace(key, key_size);

      if (!cursor->next()) {
        break;
      }
    }
  }

  return res;
}

Vector<String> RecordSet::listDatafiles() const {
  std::unique_lock<std::mutex> lk(mutex_);
  Vector<String> datafiles;

  for (const auto& df : state_.datafiles) {
    datafiles.emplace_back(df.filename);
  }

  return datafiles;
}

void RecordSet::setMaxDatafileSize(size_t size) {
  max_datafile_size_ = size;
}

RecordSet::RecordSetState::RecordSetState() :
    commitlog_size(0),
    version(0) {}

void RecordSet::RecordSetState::encode(
    util::BinaryMessageWriter* writer) const {
  writer->appendVarUInt(0x01);

  Set<String> all_commitlogs = old_commitlogs;
  if (!commitlog.isEmpty()) {
    all_commitlogs.emplace(commitlog.get());
  }

  writer->appendVarUInt(datafiles.size());
  for (const auto& d : datafiles) {
    writer->appendVarUInt(d.num_records);
    writer->appendVarUInt(d.offset);
    writer->appendLenencString(d.filename);
  }

  writer->appendVarUInt(all_commitlogs.size());
  for (const auto& cl : all_commitlogs) {
    writer->appendLenencString(cl);
  }

  writer->append(checksum.data(), checksum.size());
}

void RecordSet::RecordSetState::decode(util::BinaryMessageReader* reader) {
  auto v = reader->readVarUInt();

  auto num_datafiles = reader->readVarUInt();
  for (size_t i = 0; i < num_datafiles; ++i) {
    auto num_records = reader->readVarUInt();
    auto offset = reader->readVarUInt();
    auto fname = reader->readLenencString();
    datafiles.emplace_back(DatafileRef {
      .filename = fname,
      .num_records = num_records,
      .offset = offset
    });
  }

  auto num_commitlogs = reader->readVarUInt();
  for (size_t i = 0; i < num_commitlogs; ++i) {
    auto fname = reader->readLenencString();
    old_commitlogs.emplace(fname);
  }

  checksum = SHA1Hash(reader->read(SHA1Hash::kSize), SHA1Hash::kSize);
}

// FIXPAUL this should use an incremental algo and should be O(1) in space
// instead of O(N)
SHA1Hash RecordSet::calculateChecksum(const Set<SHA1Hash>& id_set) const {
  Buffer id_set_concat;

  for (const auto& id : id_set) {
    id_set_concat.append(id.data(), id.size());
  }

  return SHA1::compute(id_set_concat);
}

} // namespace zbase

<commit_msg>fix warning<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 Paul Asmuth
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <stx/io/fileutil.h>
#include <stx/io/mmappedfile.h>
#include <stx/util/binarymessagereader.h>
#include <stx/util/binarymessagewriter.h>
#include <sstable/SSTableEditor.h>
#include <sstable/sstablereader.h>
#include <zbase/core/RecordSet.h>

using namespace stx;

namespace zbase {

RecordRef::RecordRef(
    const SHA1Hash& _record_id,
    const Buffer& _record) :
    record_id(_record_id),
    record(_record) {}

RecordSet::RecordSet(
    const String& datadir,
    const String& filename_prefix,
    RecordSetState state /* = RecordSetState{} */) :
    datadir_(datadir),
    filename_prefix_(filename_prefix),
    state_(state),
    max_datafile_size_(kDefaultMaxDatafileSize) {
  auto id_index_fn = [this] (const SHA1Hash& id, const void* data, size_t size) {
    commitlog_ids_.emplace(id);
  };

  for (const auto& o : state_.old_commitlogs) {
    loadCommitlog(o, id_index_fn);
  }

  if (!state.commitlog.isEmpty()) {
    loadCommitlog(state.commitlog.get(), id_index_fn);
  }
}

RecordSet::RecordSetState RecordSet::getState() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_;
}

size_t RecordSet::version() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_.version;
}

SHA1Hash RecordSet::checksum() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return state_.checksum;
}

size_t RecordSet::commitlogSize() const {
  std::unique_lock<std::mutex> lk(mutex_);
  return commitlog_ids_.size();
}

void RecordSet::addRecord(const SHA1Hash& record_id, const Buffer& message) {

  util::BinaryMessageWriter buf;
  buf.append(record_id.data(), record_id.size());
  buf.appendVarUInt(message.size());
  buf.append(message.data(), message.size());

  std::unique_lock<std::mutex> lk(mutex_);

  if (commitlog_ids_.count(record_id) > 0) {
    return;
  }

  addRecords(buf);

  commitlog_ids_.emplace(record_id);
}

void RecordSet::addRecords(const Vector<RecordRef>& records) {
  std::unique_lock<std::mutex> lk(mutex_);

  util::BinaryMessageWriter buf;
  for (const auto& rec : records) {
    if (commitlog_ids_.count(rec.record_id) > 0) {
      continue;
    }

    buf.append(rec.record_id.data(), rec.record_id.size());
    buf.appendVarUInt(rec.record.size());
    buf.append(rec.record.data(), rec.record.size());
  }

  if (buf.size() == 0) {
    return;
  }

  addRecords(buf);

  for (const auto& rec : records) {
    commitlog_ids_.emplace(rec.record_id);
  }
}

// precondition: must hold lock
void RecordSet::addRecords(const util::BinaryMessageWriter& buf) {
  String commitlog;
  uint64_t commitlog_size;
  if (state_.commitlog.isEmpty()) {
    commitlog = filename_prefix_ + rnd_.hex64() + ".log";
    commitlog_size = 0;
    ++state_.version;
  } else {
    commitlog = state_.commitlog.get();
    commitlog_size = state_.commitlog_size;
  }

  auto commitlog_path = FileUtil::joinPaths(datadir_, commitlog);
  auto file = File::openFile(
      commitlog_path,
      File::O_WRITE | File::O_CREATEOROPEN);

  file.truncate(sizeof(uint64_t) + commitlog_size + buf.size());
  file.seekTo(sizeof(uint64_t) + commitlog_size);
  file.write(buf.data(), buf.size());

  // FIXPAUL fsync here for non order preserving storage?

  commitlog_size += buf.size();
  file.seekTo(0);
  file.write(&commitlog_size, sizeof(commitlog_size));

  state_.commitlog = Some(commitlog);
  state_.commitlog_size = commitlog_size;
}

void RecordSet::rollCommitlog() {
  std::unique_lock<std::mutex> lk(mutex_);
  rollCommitlogWithLock();
}

// precondition: must hold lock
void RecordSet::rollCommitlogWithLock() {
  if (state_.commitlog.isEmpty()) {
    return;
  }

  auto old_log = state_.commitlog.get();
  auto old_log_path = FileUtil::joinPaths(datadir_, old_log);
  FileUtil::truncate(old_log_path, state_.commitlog_size + sizeof(uint64_t));

  state_.old_commitlogs.emplace(old_log);
  state_.commitlog = None<String>();
  state_.commitlog_size = 0;
  ++state_.version;
}

void RecordSet::compact() {
  Set<String> deleted_files;
  compact(&deleted_files);
}

void RecordSet::compact(Set<String>* deleted_files) {
  std::unique_lock<std::mutex> compact_lk(compact_mutex_, std::defer_lock);
  if (!compact_lk.try_lock()) {
    return; // compaction is already running
  }

  std::unique_lock<std::mutex> lk(mutex_);
  rollCommitlogWithLock();
  auto snap = state_;
  lk.unlock();

  if (snap.old_commitlogs.size() == 0) {
    return;
  }

  auto outfile_name = filename_prefix_ + rnd_.hex64() + ".sst";
  auto outfile_path = FileUtil::joinPaths(datadir_, outfile_name);
  auto outfile = sstable::SSTableEditor::create(
      outfile_path + "~",
      sstable::IndexProvider{},
      nullptr,
      0);

  size_t outfile_nrecords = 0;
  size_t outfile_offset = 0;

  Set<SHA1Hash> id_set;
  Set<SHA1Hash> new_id_set;

  bool rewrite_last = false;
  if (snap.datafiles.size() > 0) {
    auto cur_datafile_size = FileUtil::size(
        FileUtil::joinPaths(datadir_, snap.datafiles.back().filename));

    if (cur_datafile_size < max_datafile_size_) {
      rewrite_last = true;
    }
  }

  if (rewrite_last) {
    outfile_offset = snap.datafiles.back().offset;
  } else {
    outfile_offset = snap.datafiles.size() > 0 ?
        snap.datafiles.back().offset + snap.datafiles.back().num_records :
        0;
  }

  for (int j = 0; j < snap.datafiles.size(); ++j) {
    auto sstable_path = FileUtil::joinPaths(
        datadir_,
        snap.datafiles[j].filename);

    sstable::SSTableReader reader(sstable_path);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      void* key;
      size_t key_size;
      cursor->getKey(&key, &key_size);

      SHA1Hash msgid(key, key_size);
      id_set.emplace(msgid);

      if (rewrite_last && j + 1 == snap.datafiles.size()) {
        void* data;
        size_t data_size;
        cursor->getData(&data, &data_size);

        outfile->appendRow(key, key_size, data, data_size);
        ++outfile_nrecords;
      }

      if (!cursor->next()) {
        break;
      }
    }
  }

  for (const auto& cl : snap.old_commitlogs) {
    loadCommitlog(cl, [this, &outfile, &id_set, &new_id_set, &outfile_nrecords] (
        const SHA1Hash& id,
        const void* data,
        size_t size) {
      if (new_id_set.count(id) > 0) {
        return;
      }

      new_id_set.emplace(id);

      if (id_set.count(id) > 0) {
        return;
      }

      outfile->appendRow(id.data(), id.size(), data, size);
      ++outfile_nrecords;
    });
  }

  for (const auto& new_id : new_id_set) {
    id_set.emplace(new_id);
  }

  auto new_checksum = calculateChecksum(id_set);

  if (outfile_nrecords == 0) {
    FileUtil::rm(outfile_path + "~");
  } else {
    outfile->finalize();
    FileUtil::mv(outfile_path + "~", outfile_path);
  }

  lk.lock();

  if (rewrite_last) {
    deleted_files->emplace(state_.datafiles.back().filename);
    state_.datafiles.pop_back();
  }

  if (outfile_nrecords > 0) {
    state_.datafiles.emplace_back(DatafileRef {
      .filename = outfile_name,
      .num_records = outfile_nrecords,
      .offset = outfile_offset
    });
  }

  for (const auto& cl : snap.old_commitlogs) {
    state_.old_commitlogs.erase(cl);
    deleted_files->emplace(cl);
  }

  for (const auto& id : new_id_set) {
    commitlog_ids_.erase(id);
  }

  state_.checksum = new_checksum;
  ++state_.version;
}

void RecordSet::loadCommitlog(
    const String& filename,
    Function<void (const SHA1Hash&, const void*, size_t)> fn) {
  auto filepath = FileUtil::joinPaths(datadir_, filename);
  io::MmappedFile mmap(File::openFile(filepath, File::O_READ));
  util::BinaryMessageReader reader(mmap.data(), mmap.size());
  auto limit = *reader.readUInt64() + sizeof(uint64_t);

  while (reader.position() < limit) {
    auto id_data = reader.read(SHA1Hash::kSize);
    SHA1Hash id(id_data, SHA1Hash::kSize);
    auto len = reader.readVarUInt();
    auto data = reader.read(len);

    fn(id, data, len);
  }
}

uint64_t RecordSet::numRecords() const {
  uint64_t res = 0;

  std::unique_lock<std::mutex> lk(mutex_);
  for (const auto& df : state_.datafiles) {
    res += df.num_records;
  }

  return res;
}

uint64_t RecordSet::firstOffset() const {
  std::unique_lock<std::mutex> lk(mutex_);

  if (state_.datafiles.size() == 0) {
    return 0;
  } else {
    return state_.datafiles.front().offset;
  }
}

uint64_t RecordSet::lastOffset() const {
  std::unique_lock<std::mutex> lk(mutex_);

  if (state_.datafiles.size() == 0) {
    return 0;
  } else {
    return state_.datafiles.back().offset + state_.datafiles.back().num_records;
  }
}

void RecordSet::fetchRecords(
      uint64_t offset,
      uint64_t limit,
      Function<void (
          const SHA1Hash& record_id,
          const void* record_data,
          size_t record_size)> fn) {
  std::unique_lock<std::mutex> lk(mutex_);
  auto datafiles = state_.datafiles;
  lk.unlock();

  if (datafiles.size() == 0) {
    return;
  }

  size_t o = datafiles.front().offset;
  if (o > offset) {
    RAISEF(kRuntimeError, "offset was garbage collected: $0", offset);
  }

  size_t l = 0;
  for (const auto& datafile : datafiles) {
    if (o + datafile.num_records <= offset) {
      o += datafile.num_records;
      continue;
    }

    auto filepath = FileUtil::joinPaths(datadir_, datafile.filename);
    sstable::SSTableReader reader(filepath);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      if (++o > offset) {
        void* key;
        size_t key_size;
        cursor->getKey(&key, &key_size);
        SHA1Hash msgid(key, key_size);

        void* data;
        size_t data_size;
        cursor->getData(&data, &data_size);
        fn(msgid, data, data_size);

        if (++l == limit) {
          return;
        }
      }

      if (!cursor->next()) {
        break;
      }
    }
  }
}

Set<SHA1Hash> RecordSet::listRecords() const {
  Set<SHA1Hash> res;

  std::unique_lock<std::mutex> lk(mutex_);
  if (state_.datafiles.empty()) {
    return res;
  }

  auto datafiles = state_.datafiles;
  lk.unlock();

  for (const auto& datafile : datafiles) {
    auto filepath = FileUtil::joinPaths(datadir_, datafile.filename);
    sstable::SSTableReader reader(filepath);
    auto cursor = reader.getCursor();

    while (cursor->valid()) {
      void* key;
      size_t key_size;
      cursor->getKey(&key, &key_size);

      res.emplace(key, key_size);

      if (!cursor->next()) {
        break;
      }
    }
  }

  return res;
}

Vector<String> RecordSet::listDatafiles() const {
  std::unique_lock<std::mutex> lk(mutex_);
  Vector<String> datafiles;

  for (const auto& df : state_.datafiles) {
    datafiles.emplace_back(df.filename);
  }

  return datafiles;
}

void RecordSet::setMaxDatafileSize(size_t size) {
  max_datafile_size_ = size;
}

RecordSet::RecordSetState::RecordSetState() :
    commitlog_size(0),
    version(0) {}

void RecordSet::RecordSetState::encode(
    util::BinaryMessageWriter* writer) const {
  writer->appendVarUInt(0x01);

  Set<String> all_commitlogs = old_commitlogs;
  if (!commitlog.isEmpty()) {
    all_commitlogs.emplace(commitlog.get());
  }

  writer->appendVarUInt(datafiles.size());
  for (const auto& d : datafiles) {
    writer->appendVarUInt(d.num_records);
    writer->appendVarUInt(d.offset);
    writer->appendLenencString(d.filename);
  }

  writer->appendVarUInt(all_commitlogs.size());
  for (const auto& cl : all_commitlogs) {
    writer->appendLenencString(cl);
  }

  writer->append(checksum.data(), checksum.size());
}

void RecordSet::RecordSetState::decode(util::BinaryMessageReader* reader) {
  auto v = reader->readVarUInt();
  (void) v;

  auto num_datafiles = reader->readVarUInt();
  for (size_t i = 0; i < num_datafiles; ++i) {
    auto num_records = reader->readVarUInt();
    auto offset = reader->readVarUInt();
    auto fname = reader->readLenencString();
    datafiles.emplace_back(DatafileRef {
      .filename = fname,
      .num_records = num_records,
      .offset = offset
    });
  }

  auto num_commitlogs = reader->readVarUInt();
  for (size_t i = 0; i < num_commitlogs; ++i) {
    auto fname = reader->readLenencString();
    old_commitlogs.emplace(fname);
  }

  checksum = SHA1Hash(reader->read(SHA1Hash::kSize), SHA1Hash::kSize);
}

// FIXPAUL this should use an incremental algo and should be O(1) in space
// instead of O(N)
SHA1Hash RecordSet::calculateChecksum(const Set<SHA1Hash>& id_set) const {
  Buffer id_set_concat;

  for (const auto& id : id_set) {
    id_set_concat.append(id.data(), id.size());
  }

  return SHA1::compute(id_set_concat);
}

} // namespace zbase

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#pragma once

#include <cassert>
#include <cstdint>
#include <string>

#include <fc/container/flat.hpp>
#include <fc/reflect/reflect.hpp>

namespace graphene { namespace chain {

/**
 * @brief An ID for some votable object
 *
 * This class stores an ID for a votable object. The ID is comprised of two fields: a type, and an instance. The
 * type field stores which kind of object is being voted on, and the instance stores which specific object of that
 * type is being referenced by this ID.
 *
 * A value of vote_id_type is implicitly convertible to an unsigned 32-bit integer containing only the instance. It
 * may also be implicitly assigned from a uint32_t, which will update the instance. It may not, however, be
 * implicitly constructed from a uint32_t, as in this case, the type would be unknown.
 *
 * On the wire, a vote_id_type is represented as a 32-bit integer with the type in the lower 8 bits and the instance
 * in the upper 24 bits. This means that types may never exceed 8 bits, and instances may never exceed 24 bits.
 *
 * In JSON, a vote_id_type is represented as a string "type:instance", i.e. "1:5" would be type 1 and instance 5.
 *
 * @note In the Graphene protocol, vote_id_type instances are unique across types; that is to say, if an object of
 * type 1 has instance 4, an object of type 0 may not also have instance 4. In other words, the type is not a
 * namespace for instances; it is only an informational field.
 */
struct vote_id_type
{
   /// Lower 8 bits are type; upper 24 bits are instance
   uint32_t content;

   friend size_t hash_value( vote_id_type v ) { return std::hash<uint32_t>()(v.content); }
   enum vote_type
   {
      committee,
      witness,
      worker,
      VOTE_TYPE_COUNT
   };

   /// Default constructor. Sets type and instance to 0
   vote_id_type():content(0){}
   /// Construct this vote_id_type with provided type and instance
   vote_id_type(vote_type type, uint32_t instance = 0)
      : content(instance<<8 | type)
   {}
   /// Construct this vote_id_type from a serial string in the form "type:instance"
   explicit vote_id_type(const std::string& serial)
   {
      auto colon = serial.find(':');
      if( colon != std::string::npos )
         *this = vote_id_type(vote_type(std::stoul(serial.substr(0, colon))), std::stoul(serial.substr(colon+1)));
   }

   /// Set the type of this vote_id_type
   void set_type(vote_type type)
   {
      content &= 0xffffff00;
      content |= type & 0xff;
   }
   /// Get the type of this vote_id_type
   vote_type type()const
   {
      return vote_type(content & 0xff);
   }

   /// Set the instance of this vote_id_type
   void set_instance(uint32_t instance)
   {
      assert(instance < 0x01000000);
      content &= 0xff;
      content |= instance << 8;
   }
   /// Get the instance of this vote_id_type
   uint32_t instance()const
   {
      return content >> 8;
   }

   vote_id_type& operator =(vote_id_type other)
   {
      content = other.content;
      return *this;
   }
   /// Set the instance of this vote_id_type
   vote_id_type& operator =(uint32_t instance)
   {
      set_instance(instance);
      return *this;
   }
   /// Get the instance of this vote_id_type
   operator uint32_t()const
   {
      return instance();
   }

   /// Convert this vote_id_type to a serial string in the form "type:instance"
   explicit operator std::string()const
   {
      return std::to_string(type()) + ":" + std::to_string(instance());
   }
};

class global_property_object;

vote_id_type get_next_vote_id( global_property_object& gpo, vote_id_type::vote_type type );

} } // graphene::chain

namespace fc
{

class variant;

void to_variant( const graphene::chain::vote_id_type& var, fc::variant& vo );
void from_variant( const fc::variant& var, graphene::chain::vote_id_type& vo );

} // fc

FC_REFLECT_TYPENAME( fc::flat_set<graphene::chain::vote_id_type> )

FC_REFLECT_ENUM( graphene::chain::vote_id_type::vote_type, (witness)(committee)(worker)(VOTE_TYPE_COUNT) )
FC_REFLECT( graphene::chain::vote_id_type, (content) )
<commit_msg>fix vote_id_type might be uninitialized #25<commit_after>/*
 * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#pragma once

#include <cassert>
#include <cstdint>
#include <string>

#include <fc/container/flat.hpp>
#include <fc/reflect/reflect.hpp>

namespace graphene { namespace chain {

/**
 * @brief An ID for some votable object
 *
 * This class stores an ID for a votable object. The ID is comprised of two fields: a type, and an instance. The
 * type field stores which kind of object is being voted on, and the instance stores which specific object of that
 * type is being referenced by this ID.
 *
 * A value of vote_id_type is implicitly convertible to an unsigned 32-bit integer containing only the instance. It
 * may also be implicitly assigned from a uint32_t, which will update the instance. It may not, however, be
 * implicitly constructed from a uint32_t, as in this case, the type would be unknown.
 *
 * On the wire, a vote_id_type is represented as a 32-bit integer with the type in the lower 8 bits and the instance
 * in the upper 24 bits. This means that types may never exceed 8 bits, and instances may never exceed 24 bits.
 *
 * In JSON, a vote_id_type is represented as a string "type:instance", i.e. "1:5" would be type 1 and instance 5.
 *
 * @note In the Graphene protocol, vote_id_type instances are unique across types; that is to say, if an object of
 * type 1 has instance 4, an object of type 0 may not also have instance 4. In other words, the type is not a
 * namespace for instances; it is only an informational field.
 */
struct vote_id_type
{
   /// Lower 8 bits are type; upper 24 bits are instance
   uint32_t content;

   friend size_t hash_value( vote_id_type v ) { return std::hash<uint32_t>()(v.content); }
   enum vote_type
   {
      committee,
      witness,
      worker,
      VOTE_TYPE_COUNT
   };

   /// Default constructor. Sets type and instance to 0
   vote_id_type():content(0){}
   /// Construct this vote_id_type with provided type and instance
   vote_id_type(vote_type type, uint32_t instance = 0)
      : content(instance<<8 | type)
   {}
   /// Construct this vote_id_type from a serial string in the form "type:instance"
   explicit vote_id_type(const std::string& serial)
   {
      auto colon = serial.find(':');
      FC_ASSERT( colon != std::string::npos );
      *this = vote_id_type(vote_type(std::stoul(serial.substr(0, colon))), std::stoul(serial.substr(colon+1)));
   }

   /// Set the type of this vote_id_type
   void set_type(vote_type type)
   {
      content &= 0xffffff00;
      content |= type & 0xff;
   }
   /// Get the type of this vote_id_type
   vote_type type()const
   {
      return vote_type(content & 0xff);
   }

   /// Set the instance of this vote_id_type
   void set_instance(uint32_t instance)
   {
      assert(instance < 0x01000000);
      content &= 0xff;
      content |= instance << 8;
   }
   /// Get the instance of this vote_id_type
   uint32_t instance()const
   {
      return content >> 8;
   }

   vote_id_type& operator =(vote_id_type other)
   {
      content = other.content;
      return *this;
   }
   /// Set the instance of this vote_id_type
   vote_id_type& operator =(uint32_t instance)
   {
      set_instance(instance);
      return *this;
   }
   /// Get the instance of this vote_id_type
   operator uint32_t()const
   {
      return instance();
   }

   /// Convert this vote_id_type to a serial string in the form "type:instance"
   explicit operator std::string()const
   {
      return std::to_string(type()) + ":" + std::to_string(instance());
   }
};

class global_property_object;

vote_id_type get_next_vote_id( global_property_object& gpo, vote_id_type::vote_type type );

} } // graphene::chain

namespace fc
{

class variant;

void to_variant( const graphene::chain::vote_id_type& var, fc::variant& vo );
void from_variant( const fc::variant& var, graphene::chain::vote_id_type& vo );

} // fc

FC_REFLECT_TYPENAME( fc::flat_set<graphene::chain::vote_id_type> )

FC_REFLECT_ENUM( graphene::chain::vote_id_type::vote_type, (witness)(committee)(worker)(VOTE_TYPE_COUNT) )
FC_REFLECT( graphene::chain::vote_id_type, (content) )
<|endoftext|>


<commit_before>// Copyright 2008 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <stdlib.h>

#include "d8.h"
#include "debug.h"
#include "api.h"
#include "natives.h"

namespace v8 {

const char* Shell::kHistoryFileName = ".d8_history";
const char* Shell::kPrompt = "d8> ";

LineEditor *LineEditor::first_ = NULL;

LineEditor::LineEditor(Type type, const char* name)
    : type_(type),
      name_(name),
      next_(first_) {
  first_ = this;
}

LineEditor* LineEditor::Get() {
  LineEditor* current = first_;
  LineEditor* best = current;
  while (current != NULL) {
    if (current->type_ > best->type_)
      best = current;
    current = current->next_;
  }
  return best;
}

class DumbLineEditor: public LineEditor {
 public:
  DumbLineEditor() : LineEditor(LineEditor::DUMB, "dumb") { }
  virtual i::SmartPointer<char> Prompt(const char* prompt);
};

static DumbLineEditor dumb_line_editor;

i::SmartPointer<char> DumbLineEditor::Prompt(const char* prompt) {
  static const int kBufferSize = 256;
  char buffer[kBufferSize];
  printf("%s", prompt);
  char* str = fgets(buffer, kBufferSize, stdin);
  return i::SmartPointer<char>(str ? i::StrDup(str) : str);
}

Shell::CounterMap Shell::counter_map_;
i::OS::MemoryMappedFile* Shell::counters_file_ = NULL;
CounterCollection Shell::local_counters_;
CounterCollection* Shell::counters_ = &local_counters_;
Persistent<Context> Shell::utility_context_;
Persistent<Context> Shell::evaluation_context_;

// Executes a string within the current v8 context.
bool Shell::ExecuteString(Handle<String> source,
                          Handle<Value> name,
                          bool print_result,
                          bool report_exceptions) {
  HandleScope handle_scope;
  TryCatch try_catch;
  Handle<Script> script = Script::Compile(source, name);
  if (script.IsEmpty()) {
    // Print errors that happened during compilation.
    if (report_exceptions)
      ReportException(&try_catch);
    return false;
  } else {
    Handle<Value> result = script->Run();
    if (result.IsEmpty()) {
      // Print errors that happened during execution.
      if (report_exceptions)
        ReportException(&try_catch);
      return false;
    } else {
      if (print_result && !result->IsUndefined()) {
        // If all went well and the result wasn't undefined then print
        // the returned value.
        String::Utf8Value str(result);
        printf("%s\n", *str);
      }
      return true;
    }
  }
}

Handle<Value> Shell::Print(const Arguments& args) {
  bool first = true;
  for (int i = 0; i < args.Length(); i++) {
    HandleScope handle_scope;
    if (first) {
      first = false;
    } else {
      printf(" ");
    }
    String::Utf8Value str(args[i]);
    printf("%s", *str);
  }
  printf("\n");
  return Undefined();
}

Handle<Value> Shell::Load(const Arguments& args) {
  for (int i = 0; i < args.Length(); i++) {
    HandleScope handle_scope;
    String::Utf8Value file(args[i]);
    Handle<String> source = ReadFile(*file);
    if (source.IsEmpty()) {
      return ThrowException(String::New("Error loading file"));
    }
    if (!ExecuteString(source, String::New(*file), false, false)) {
      return ThrowException(String::New("Error executing  file"));
    }
  }
  return Undefined();
}

Handle<Value> Shell::Quit(const Arguments& args) {
  int exit_code = args[0]->Int32Value();
  OnExit();
  exit(exit_code);
  return Undefined();
}

Handle<Value> Shell::Version(const Arguments& args) {
  return String::New(V8::GetVersion());
}

void Shell::ReportException(v8::TryCatch* try_catch) {
  HandleScope handle_scope;
  String::Utf8Value exception(try_catch->Exception());
  Handle<Message> message = try_catch->Message();
  if (message.IsEmpty()) {
    // V8 didn't provide any extra information about this error; just
    // print the exception.
    printf("%s\n", *exception);
  } else {
    // Print (filename):(line number): (message).
    String::Utf8Value filename(message->GetScriptResourceName());
    int linenum = message->GetLineNumber();
    printf("%s:%i: %s\n", *filename, linenum, *exception);
    // Print line of source code.
    String::Utf8Value sourceline(message->GetSourceLine());
    printf("%s\n", *sourceline);
    // Print wavy underline (GetUnderline is deprecated).
    int start = message->GetStartColumn();
    for (int i = 0; i < start; i++) {
      printf(" ");
    }
    int end = message->GetEndColumn();
    for (int i = start; i < end; i++) {
      printf("^");
    }
    printf("\n");
  }
}

Handle<Array> Shell::GetCompletions(Handle<String> text, Handle<String> full) {
  HandleScope handle_scope;
  Context::Scope context_scope(utility_context_);
  Handle<Object> global = utility_context_->Global();
  Handle<Value> fun = global->Get(String::New("GetCompletions"));
  static const int kArgc = 3;
  Handle<Value> argv[kArgc] = { evaluation_context_->Global(), text, full };
  Handle<Value> val = Handle<Function>::Cast(fun)->Call(global, kArgc, argv);
  return handle_scope.Close(Handle<Array>::Cast(val));
}

int32_t* Counter::Bind(const char* name) {
  int i;
  for (i = 0; i < kMaxNameSize - 1 && name[i]; i++)
    name_[i] = static_cast<char>(name[i]);
  name_[i] = '\0';
  return &counter_;
}

CounterCollection::CounterCollection() {
  magic_number_ = 0xDEADFACE;
  max_counters_ = kMaxCounters;
  max_name_size_ = Counter::kMaxNameSize;
  counters_in_use_ = 0;
}

Counter* CounterCollection::GetNextCounter() {
  if (counters_in_use_ == kMaxCounters) return NULL;
  return &counters_[counters_in_use_++];
}

void Shell::MapCounters(const char* name) {
  counters_file_ = i::OS::MemoryMappedFile::create(name,
    sizeof(CounterCollection), &local_counters_);
  void* memory = (counters_file_ == NULL) ?
      NULL : counters_file_->memory();
  if (memory == NULL) {
    printf("Could not map counters file %s\n", name);
    exit(1);
  }
  counters_ = static_cast<CounterCollection*>(memory);
  V8::SetCounterFunction(LookupCounter);
}

int* Shell::LookupCounter(const char* name) {
  CounterMap::iterator item = counter_map_.find(name);
  if (item != counter_map_.end()) {
    Counter* result = (*item).second;
    return result->ptr();
  }
  Counter* result = counters_->GetNextCounter();
  if (result == NULL) return NULL;
  return result->Bind(name);
}

void Shell::Initialize() {
  // Set up counters
  if (i::FLAG_map_counters != NULL)
    MapCounters(i::FLAG_map_counters);
  if (i::FLAG_dump_counters)
    V8::SetCounterFunction(LookupCounter);
  // Initialize the global objects
  HandleScope scope;
  Handle<ObjectTemplate> global_template = ObjectTemplate::New();
  global_template->Set(String::New("print"), FunctionTemplate::New(Print));
  global_template->Set(String::New("load"), FunctionTemplate::New(Load));
  global_template->Set(String::New("quit"), FunctionTemplate::New(Quit));
  global_template->Set(String::New("version"), FunctionTemplate::New(Version));

  utility_context_ = Context::New(NULL, global_template);
  utility_context_->SetSecurityToken(Undefined());
  Context::Scope utility_scope(utility_context_);

  i::JSArguments js_args = i::FLAG_js_arguments;
  i::Handle<i::FixedArray> arguments_array =
      i::Factory::NewFixedArray(js_args.argc());
  for (int j = 0; j < js_args.argc(); j++) {
    i::Handle<i::String> arg =
        i::Factory::NewStringFromUtf8(i::CStrVector(js_args[j]));
    arguments_array->set(j, *arg);
  }
  i::Handle<i::JSArray> arguments_jsarray =
      i::Factory::NewJSArrayWithElements(arguments_array);
  global_template->Set(String::New("arguments"),
                       Utils::ToLocal(arguments_jsarray));

  // Install the debugger object in the utility scope
  i::Debug::Load();
  i::JSObject* debug = i::Debug::debug_context()->global();
  utility_context_->Global()->Set(String::New("$debug"),
                                  Utils::ToLocal(&debug));

  // Run the d8 shell utility script in the utility context
  int source_index = i::NativesCollection<i::D8>::GetIndex("d8");
  i::Vector<const char> shell_source
      = i::NativesCollection<i::D8>::GetScriptSource(source_index);
  i::Vector<const char> shell_source_name
      = i::NativesCollection<i::D8>::GetScriptName(source_index);
  Handle<String> source = String::New(shell_source.start(),
                                      shell_source.length());
  Handle<String> name = String::New(shell_source_name.start(),
                                    shell_source_name.length());
  Script::Compile(source, name)->Run();

  // Create the evaluation context
  evaluation_context_ = Context::New(NULL, global_template);
  evaluation_context_->SetSecurityToken(Undefined());

  // Set the security token of the debug context to allow access.
  i::Debug::debug_context()->set_security_token(i::Heap::undefined_value());
}

void Shell::OnExit() {
  if (i::FLAG_dump_counters) {
    ::printf("+----------------------------------------+----------+\n");
    ::printf("| Name                                   | Value    |\n");
    ::printf("+----------------------------------------+----------+\n");
    for (CounterMap::iterator i = counter_map_.begin();
         i != counter_map_.end();
         i++) {
      Counter* counter = (*i).second;
      ::printf("| %-38s | %8i |\n", (*i).first, counter->value());
    }
    ::printf("+----------------------------------------+----------+\n");
  }
  if (counters_file_ != NULL)
    delete counters_file_;
}

// Reads a file into a v8 string.
Handle<String> Shell::ReadFile(const char* name) {
  FILE* file = i::OS::FOpen(name, "rb");
  if (file == NULL) return Handle<String>();

  fseek(file, 0, SEEK_END);
  int size = ftell(file);
  rewind(file);

  char* chars = new char[size + 1];
  chars[size] = '\0';
  for (int i = 0; i < size;) {
    int read = fread(&chars[i], 1, size - i, file);
    i += read;
  }
  fclose(file);
  Handle<String> result = String::New(chars, size);
  delete[] chars;
  return result;
}

void Shell::RunShell() {
  LineEditor* editor = LineEditor::Get();
  printf("V8 version %s [console: %s]\n", V8::GetVersion(), editor->name());
  editor->Open();
  while (true) {
    HandleScope handle_scope;
    i::SmartPointer<char> input = editor->Prompt(Shell::kPrompt);
    if (input.is_empty())
      break;
    editor->AddHistory(*input);
    Handle<String> name = String::New("(d8)");
    ExecuteString(String::New(*input), name, true, true);
  }
  editor->Close();
  printf("\n");
}

int Shell::Main(int argc, char* argv[]) {
  i::FlagList::SetFlagsFromCommandLine(&argc, argv, true);
  if (i::FLAG_help) {
    return 1;
  }
  Initialize();
  bool run_shell = (argc == 1);
  Context::Scope context_scope(evaluation_context_);
  for (int i = 1; i < argc; i++) {
    char* str = argv[i];
    if (strcmp(str, "-f") == 0) {
      // Ignore any -f flags for compatibility with other stand-alone
      // JavaScript engines.
      continue;
    } else if (strncmp(str, "--", 2) == 0) {
      printf("Warning: unknown flag %s.\nTry --help for options\n", str);
    } else if (strcmp(str, "-e") == 0 && i + 1 < argc) {
      // Execute argument given to -e option directly.
      v8::HandleScope handle_scope;
      v8::Handle<v8::String> file_name = v8::String::New("unnamed");
      v8::Handle<v8::String> source = v8::String::New(argv[i + 1]);
      if (!ExecuteString(source, file_name, false, true))
        return 1;
      i++;
    } else {
      // Use all other arguments as names of files to load and run.
      HandleScope handle_scope;
      Handle<String> file_name = v8::String::New(str);
      Handle<String> source = ReadFile(str);
      if (source.IsEmpty()) {
        printf("Error reading '%s'\n", str);
        return 1;
      }
      if (!ExecuteString(source, file_name, false, true))
        return 1;
    }
  }
  if (run_shell)
    RunShell();
  OnExit();
  return 0;
}

}  // namespace v8

int main(int argc, char* argv[]) {
  return v8::Shell::Main(argc, argv);
}
<commit_msg>Fixed the caching of counters. This also fixes the --dump-counters which reads the counters from the cache when dumping. Review URL: http://codereview.chromium.org/13171<commit_after>// Copyright 2008 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <stdlib.h>

#include "d8.h"
#include "debug.h"
#include "api.h"
#include "natives.h"

namespace v8 {

const char* Shell::kHistoryFileName = ".d8_history";
const char* Shell::kPrompt = "d8> ";

LineEditor *LineEditor::first_ = NULL;

LineEditor::LineEditor(Type type, const char* name)
    : type_(type),
      name_(name),
      next_(first_) {
  first_ = this;
}

LineEditor* LineEditor::Get() {
  LineEditor* current = first_;
  LineEditor* best = current;
  while (current != NULL) {
    if (current->type_ > best->type_)
      best = current;
    current = current->next_;
  }
  return best;
}

class DumbLineEditor: public LineEditor {
 public:
  DumbLineEditor() : LineEditor(LineEditor::DUMB, "dumb") { }
  virtual i::SmartPointer<char> Prompt(const char* prompt);
};

static DumbLineEditor dumb_line_editor;

i::SmartPointer<char> DumbLineEditor::Prompt(const char* prompt) {
  static const int kBufferSize = 256;
  char buffer[kBufferSize];
  printf("%s", prompt);
  char* str = fgets(buffer, kBufferSize, stdin);
  return i::SmartPointer<char>(str ? i::StrDup(str) : str);
}

Shell::CounterMap Shell::counter_map_;
i::OS::MemoryMappedFile* Shell::counters_file_ = NULL;
CounterCollection Shell::local_counters_;
CounterCollection* Shell::counters_ = &local_counters_;
Persistent<Context> Shell::utility_context_;
Persistent<Context> Shell::evaluation_context_;

// Executes a string within the current v8 context.
bool Shell::ExecuteString(Handle<String> source,
                          Handle<Value> name,
                          bool print_result,
                          bool report_exceptions) {
  HandleScope handle_scope;
  TryCatch try_catch;
  Handle<Script> script = Script::Compile(source, name);
  if (script.IsEmpty()) {
    // Print errors that happened during compilation.
    if (report_exceptions)
      ReportException(&try_catch);
    return false;
  } else {
    Handle<Value> result = script->Run();
    if (result.IsEmpty()) {
      // Print errors that happened during execution.
      if (report_exceptions)
        ReportException(&try_catch);
      return false;
    } else {
      if (print_result && !result->IsUndefined()) {
        // If all went well and the result wasn't undefined then print
        // the returned value.
        String::Utf8Value str(result);
        printf("%s\n", *str);
      }
      return true;
    }
  }
}

Handle<Value> Shell::Print(const Arguments& args) {
  bool first = true;
  for (int i = 0; i < args.Length(); i++) {
    HandleScope handle_scope;
    if (first) {
      first = false;
    } else {
      printf(" ");
    }
    String::Utf8Value str(args[i]);
    printf("%s", *str);
  }
  printf("\n");
  return Undefined();
}

Handle<Value> Shell::Load(const Arguments& args) {
  for (int i = 0; i < args.Length(); i++) {
    HandleScope handle_scope;
    String::Utf8Value file(args[i]);
    Handle<String> source = ReadFile(*file);
    if (source.IsEmpty()) {
      return ThrowException(String::New("Error loading file"));
    }
    if (!ExecuteString(source, String::New(*file), false, false)) {
      return ThrowException(String::New("Error executing  file"));
    }
  }
  return Undefined();
}

Handle<Value> Shell::Quit(const Arguments& args) {
  int exit_code = args[0]->Int32Value();
  OnExit();
  exit(exit_code);
  return Undefined();
}

Handle<Value> Shell::Version(const Arguments& args) {
  return String::New(V8::GetVersion());
}

void Shell::ReportException(v8::TryCatch* try_catch) {
  HandleScope handle_scope;
  String::Utf8Value exception(try_catch->Exception());
  Handle<Message> message = try_catch->Message();
  if (message.IsEmpty()) {
    // V8 didn't provide any extra information about this error; just
    // print the exception.
    printf("%s\n", *exception);
  } else {
    // Print (filename):(line number): (message).
    String::Utf8Value filename(message->GetScriptResourceName());
    int linenum = message->GetLineNumber();
    printf("%s:%i: %s\n", *filename, linenum, *exception);
    // Print line of source code.
    String::Utf8Value sourceline(message->GetSourceLine());
    printf("%s\n", *sourceline);
    // Print wavy underline (GetUnderline is deprecated).
    int start = message->GetStartColumn();
    for (int i = 0; i < start; i++) {
      printf(" ");
    }
    int end = message->GetEndColumn();
    for (int i = start; i < end; i++) {
      printf("^");
    }
    printf("\n");
  }
}

Handle<Array> Shell::GetCompletions(Handle<String> text, Handle<String> full) {
  HandleScope handle_scope;
  Context::Scope context_scope(utility_context_);
  Handle<Object> global = utility_context_->Global();
  Handle<Value> fun = global->Get(String::New("GetCompletions"));
  static const int kArgc = 3;
  Handle<Value> argv[kArgc] = { evaluation_context_->Global(), text, full };
  Handle<Value> val = Handle<Function>::Cast(fun)->Call(global, kArgc, argv);
  return handle_scope.Close(Handle<Array>::Cast(val));
}

int32_t* Counter::Bind(const char* name) {
  int i;
  for (i = 0; i < kMaxNameSize - 1 && name[i]; i++)
    name_[i] = static_cast<char>(name[i]);
  name_[i] = '\0';
  return &counter_;
}

CounterCollection::CounterCollection() {
  magic_number_ = 0xDEADFACE;
  max_counters_ = kMaxCounters;
  max_name_size_ = Counter::kMaxNameSize;
  counters_in_use_ = 0;
}

Counter* CounterCollection::GetNextCounter() {
  if (counters_in_use_ == kMaxCounters) return NULL;
  return &counters_[counters_in_use_++];
}

void Shell::MapCounters(const char* name) {
  counters_file_ = i::OS::MemoryMappedFile::create(name,
    sizeof(CounterCollection), &local_counters_);
  void* memory = (counters_file_ == NULL) ?
      NULL : counters_file_->memory();
  if (memory == NULL) {
    printf("Could not map counters file %s\n", name);
    exit(1);
  }
  counters_ = static_cast<CounterCollection*>(memory);
  V8::SetCounterFunction(LookupCounter);
}

int* Shell::LookupCounter(const char* name) {
  CounterMap::iterator item = counter_map_.find(name);
  if (item != counter_map_.end()) {
    Counter* result = (*item).second;
    return result->ptr();
  }
  Counter* result = counters_->GetNextCounter();
  if (result == NULL) return NULL;
  counter_map_[name] = result;
  return result->Bind(name);
}

void Shell::Initialize() {
  // Set up counters
  if (i::FLAG_map_counters != NULL)
    MapCounters(i::FLAG_map_counters);
  if (i::FLAG_dump_counters)
    V8::SetCounterFunction(LookupCounter);
  // Initialize the global objects
  HandleScope scope;
  Handle<ObjectTemplate> global_template = ObjectTemplate::New();
  global_template->Set(String::New("print"), FunctionTemplate::New(Print));
  global_template->Set(String::New("load"), FunctionTemplate::New(Load));
  global_template->Set(String::New("quit"), FunctionTemplate::New(Quit));
  global_template->Set(String::New("version"), FunctionTemplate::New(Version));

  utility_context_ = Context::New(NULL, global_template);
  utility_context_->SetSecurityToken(Undefined());
  Context::Scope utility_scope(utility_context_);

  i::JSArguments js_args = i::FLAG_js_arguments;
  i::Handle<i::FixedArray> arguments_array =
      i::Factory::NewFixedArray(js_args.argc());
  for (int j = 0; j < js_args.argc(); j++) {
    i::Handle<i::String> arg =
        i::Factory::NewStringFromUtf8(i::CStrVector(js_args[j]));
    arguments_array->set(j, *arg);
  }
  i::Handle<i::JSArray> arguments_jsarray =
      i::Factory::NewJSArrayWithElements(arguments_array);
  global_template->Set(String::New("arguments"),
                       Utils::ToLocal(arguments_jsarray));

  // Install the debugger object in the utility scope
  i::Debug::Load();
  i::JSObject* debug = i::Debug::debug_context()->global();
  utility_context_->Global()->Set(String::New("$debug"),
                                  Utils::ToLocal(&debug));

  // Run the d8 shell utility script in the utility context
  int source_index = i::NativesCollection<i::D8>::GetIndex("d8");
  i::Vector<const char> shell_source
      = i::NativesCollection<i::D8>::GetScriptSource(source_index);
  i::Vector<const char> shell_source_name
      = i::NativesCollection<i::D8>::GetScriptName(source_index);
  Handle<String> source = String::New(shell_source.start(),
                                      shell_source.length());
  Handle<String> name = String::New(shell_source_name.start(),
                                    shell_source_name.length());
  Script::Compile(source, name)->Run();

  // Create the evaluation context
  evaluation_context_ = Context::New(NULL, global_template);
  evaluation_context_->SetSecurityToken(Undefined());

  // Set the security token of the debug context to allow access.
  i::Debug::debug_context()->set_security_token(i::Heap::undefined_value());
}

void Shell::OnExit() {
  if (i::FLAG_dump_counters) {
    ::printf("+----------------------------------------+----------+\n");
    ::printf("| Name                                   | Value    |\n");
    ::printf("+----------------------------------------+----------+\n");
    for (CounterMap::iterator i = counter_map_.begin();
         i != counter_map_.end();
         i++) {
      Counter* counter = (*i).second;
      ::printf("| %-38s | %8i |\n", (*i).first, counter->value());
    }
    ::printf("+----------------------------------------+----------+\n");
  }
  if (counters_file_ != NULL)
    delete counters_file_;
}

// Reads a file into a v8 string.
Handle<String> Shell::ReadFile(const char* name) {
  FILE* file = i::OS::FOpen(name, "rb");
  if (file == NULL) return Handle<String>();

  fseek(file, 0, SEEK_END);
  int size = ftell(file);
  rewind(file);

  char* chars = new char[size + 1];
  chars[size] = '\0';
  for (int i = 0; i < size;) {
    int read = fread(&chars[i], 1, size - i, file);
    i += read;
  }
  fclose(file);
  Handle<String> result = String::New(chars, size);
  delete[] chars;
  return result;
}

void Shell::RunShell() {
  LineEditor* editor = LineEditor::Get();
  printf("V8 version %s [console: %s]\n", V8::GetVersion(), editor->name());
  editor->Open();
  while (true) {
    HandleScope handle_scope;
    i::SmartPointer<char> input = editor->Prompt(Shell::kPrompt);
    if (input.is_empty())
      break;
    editor->AddHistory(*input);
    Handle<String> name = String::New("(d8)");
    ExecuteString(String::New(*input), name, true, true);
  }
  editor->Close();
  printf("\n");
}

int Shell::Main(int argc, char* argv[]) {
  i::FlagList::SetFlagsFromCommandLine(&argc, argv, true);
  if (i::FLAG_help) {
    return 1;
  }
  Initialize();
  bool run_shell = (argc == 1);
  Context::Scope context_scope(evaluation_context_);
  for (int i = 1; i < argc; i++) {
    char* str = argv[i];
    if (strcmp(str, "-f") == 0) {
      // Ignore any -f flags for compatibility with other stand-alone
      // JavaScript engines.
      continue;
    } else if (strncmp(str, "--", 2) == 0) {
      printf("Warning: unknown flag %s.\nTry --help for options\n", str);
    } else if (strcmp(str, "-e") == 0 && i + 1 < argc) {
      // Execute argument given to -e option directly.
      v8::HandleScope handle_scope;
      v8::Handle<v8::String> file_name = v8::String::New("unnamed");
      v8::Handle<v8::String> source = v8::String::New(argv[i + 1]);
      if (!ExecuteString(source, file_name, false, true))
        return 1;
      i++;
    } else {
      // Use all other arguments as names of files to load and run.
      HandleScope handle_scope;
      Handle<String> file_name = v8::String::New(str);
      Handle<String> source = ReadFile(str);
      if (source.IsEmpty()) {
        printf("Error reading '%s'\n", str);
        return 1;
      }
      if (!ExecuteString(source, file_name, false, true))
        return 1;
    }
  }
  if (run_shell)
    RunShell();
  OnExit();
  return 0;
}

}  // namespace v8

int main(int argc, char* argv[]) {
  return v8::Shell::Main(argc, argv);
}
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2016 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

namespace dll {

template <typename Augment>
struct augmenter;

template <std::size_t C>
struct augmenter <copy<C>> {
    template <typename Input, typename Output>
    static void apply(Output& result, const Input& input) {
        for(std::size_t c = 0; c < C; ++c){
            // Simply create a copy
            result.push_back(input);
        }
    }

    static void concat_name(std::string& name) {
        name += " copy<" + std::to_string(C) + ">";
    }
};

template <std::size_t C, std::size_t K>
struct augmenter <elastic<C, K>> {
    static_assert(K % 2 == 1, "The kernel size must be odd");

    template<typename W>
    static void gaussian_blur(const etl::dyn_matrix<W>& d, etl::dyn_matrix<W>& d_blur){
        auto kernel_size = K;
        auto mid = kernel_size / 2;

        double sigma = 0.8 + 0.3 * ((kernel_size - 1) * 0.5 - 1);

        const std::size_t width = d.dim(0);
        const std::size_t height = d.dim(1);

        etl::dyn_matrix<W> kernel(kernel_size, kernel_size);

        auto gaussian = [](double x, double y, double sigma) {
            auto Z = 2.0 * M_PI * sigma * sigma;
            return (1.0 / Z) * std::exp(-((x * x + y * y) / (2.0 * sigma * sigma)));
        };

        for (std::size_t i = 0; i < kernel_size; ++i) {
            for (std::size_t j = 0; j < kernel_size; ++j) {
                kernel(i, j) = gaussian(double(i) - mid, double(j) - mid, sigma);;
            }
        }

        for (std::size_t j = 0; j < width; ++j) {
            for (std::size_t k = 0; k < height; ++k) {
                W sum(0.0);

                for (std::size_t p = 0; p < kernel_size; ++p) {
                    if (long(j) + p - mid >= 0 && long(j) + p - mid < width) {
                        for (std::size_t q = 0; q < kernel_size; ++q) {
                            if (long(k) + q - mid >= 0 && long(k) + q - mid < height) {
                                sum += kernel(p, q) * d(j + p - mid, k + q - mid);
                            }
                        }
                    }
                }

                d_blur(j, k) = d(j, k) - (sum / (kernel_size * kernel_size));
            }
        }
    }

    template <typename Input, typename Output>
    static void apply(Output& output, const Input& input) {
        for(std::size_t c = 0; c < C; ++c){
            // Create a copy of the same type
            auto result = input;

            using weight = etl::value_t<Input>;

            const std::size_t width = input.dim(1);
            const std::size_t height = input.dim(2);

            // 0. Generate random displacement fields

            etl::dyn_matrix<weight> d_x(width, height);
            etl::dyn_matrix<weight> d_y(width, height);

            d_x = etl::uniform_generator(-1.0, 1.0);
            d_y = etl::uniform_generator(-1.0, 1.0);

            // 1. Gaussian blur the displacement fields

            etl::dyn_matrix<weight> d_x_blur(width, height);
            etl::dyn_matrix<weight> d_y_blur(width, height);

            gaussian_blur(d_x, d_x_blur);
            gaussian_blur(d_y, d_y_blur);

            // 2. Normalize the displacement field

            d_x_blur /= sum(d_x_blur);
            d_y_blur /= sum(d_y_blur);

            // 3. Scale the displacement field

            weight alpha(8);

            d_x_blur *= alpha;
            d_y_blur *= alpha;

            // 4. Apply the displacement field (using bilinear interpolation)

            auto safe = [&](std::size_t channel, weight x, weight y) {
                if (x < 0 || y < 0 || x > width - 1 || y > height - 1) {
                    return input(channel, 0, 0);
                } else {
                    return input(channel, x, y);
                }
            };

            for (std::size_t channel = 0; channel < etl::dim<0>(input); ++channel) {
                for (int x = 0; x < int(width); ++x) {
                    for (int y = 0; y < int(height); ++y) {
                        auto dx = d_x_blur(y, x);
                        auto dy = d_y_blur(y, x);

                        weight px = x + dx;
                        weight py = y + dy;

                        weight a = safe(channel, std::floor(px), std::floor(py));
                        weight b = safe(channel, std::ceil(px), std::floor(py));
                        weight c = safe(channel, std::ceil(px), std::ceil(py));
                        weight d = safe(channel, std::floor(px), std::ceil(py));

                        auto e = a * (1.0 - (px - std::floor(px))) + d * (px - std::floor(px));
                        auto f = b * (1.0 - (px - std::floor(px))) + c * (px - std::floor(px));

                        auto value = e * (1.0 - (py - std::floor(py))) + f * (py - std::floor(py));

                        result(channel, x, y) = value;;
                    }
                }
            }

            output.push_back(result);
        }
    }

    static void concat_name(std::string& name) {
        name += " elastic<" + std::to_string(C) + ", " + std::to_string(K) + ">";
    }
};

} //end of dll namespace
<commit_msg>Fix bad OPENCV habits<commit_after>//=======================================================================
// Copyright (c) 2014-2016 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

namespace dll {

template <typename Augment>
struct augmenter;

template <std::size_t C>
struct augmenter <copy<C>> {
    template <typename Input, typename Output>
    static void apply(Output& result, const Input& input) {
        for(std::size_t c = 0; c < C; ++c){
            // Simply create a copy
            result.push_back(input);
        }
    }

    static void concat_name(std::string& name) {
        name += " copy<" + std::to_string(C) + ">";
    }
};

template <std::size_t C, std::size_t K>
struct augmenter <elastic<C, K>> {
    static_assert(K % 2 == 1, "The kernel size must be odd");

    template<typename W>
    static void gaussian_blur(const etl::dyn_matrix<W>& d, etl::dyn_matrix<W>& d_blur){
        auto kernel_size = K;
        auto mid = kernel_size / 2;

        double sigma = 0.8 + 0.3 * ((kernel_size - 1) * 0.5 - 1);

        const std::size_t width = d.dim(0);
        const std::size_t height = d.dim(1);

        etl::dyn_matrix<W> kernel(kernel_size, kernel_size);

        auto gaussian = [](double x, double y, double sigma) {
            auto Z = 2.0 * M_PI * sigma * sigma;
            return (1.0 / Z) * std::exp(-((x * x + y * y) / (2.0 * sigma * sigma)));
        };

        for (std::size_t i = 0; i < kernel_size; ++i) {
            for (std::size_t j = 0; j < kernel_size; ++j) {
                kernel(i, j) = gaussian(double(i) - mid, double(j) - mid, sigma);;
            }
        }

        for (std::size_t j = 0; j < width; ++j) {
            for (std::size_t k = 0; k < height; ++k) {
                W sum(0.0);

                for (std::size_t p = 0; p < kernel_size; ++p) {
                    if (long(j) + p - mid >= 0 && long(j) + p - mid < width) {
                        for (std::size_t q = 0; q < kernel_size; ++q) {
                            if (long(k) + q - mid >= 0 && long(k) + q - mid < height) {
                                sum += kernel(p, q) * d(j + p - mid, k + q - mid);
                            }
                        }
                    }
                }

                d_blur(j, k) = d(j, k) - (sum / (kernel_size * kernel_size));
            }
        }
    }

    template <typename Input, typename Output>
    static void apply(Output& output, const Input& input) {
        for(std::size_t c = 0; c < C; ++c){
            // Create a copy of the same type
            auto result = input;

            using weight = etl::value_t<Input>;

            const std::size_t width = input.dim(1);
            const std::size_t height = input.dim(2);

            // 0. Generate random displacement fields

            etl::dyn_matrix<weight> d_x(width, height);
            etl::dyn_matrix<weight> d_y(width, height);

            d_x = etl::uniform_generator(-1.0, 1.0);
            d_y = etl::uniform_generator(-1.0, 1.0);

            // 1. Gaussian blur the displacement fields

            etl::dyn_matrix<weight> d_x_blur(width, height);
            etl::dyn_matrix<weight> d_y_blur(width, height);

            gaussian_blur(d_x, d_x_blur);
            gaussian_blur(d_y, d_y_blur);

            // 2. Normalize the displacement field

            d_x_blur /= sum(d_x_blur);
            d_y_blur /= sum(d_y_blur);

            // 3. Scale the displacement field

            weight alpha(8);

            d_x_blur *= alpha;
            d_y_blur *= alpha;

            // 4. Apply the displacement field (using bilinear interpolation)

            auto safe = [&](std::size_t channel, weight x, weight y) {
                if (x < 0 || y < 0 || x > width - 1 || y > height - 1) {
                    return input(channel, 0, 0);
                } else {
                    return input(channel, x, y);
                }
            };

            for (std::size_t channel = 0; channel < etl::dim<0>(input); ++channel) {
                for (int x = 0; x < int(width); ++x) {
                    for (int y = 0; y < int(height); ++y) {
                        auto dx = d_x_blur(x, y);
                        auto dy = d_y_blur(x, y);

                        weight px = x + dx;
                        weight py = y + dy;

                        weight a = safe(channel, std::floor(px), std::floor(py));
                        weight b = safe(channel, std::ceil(px), std::floor(py));
                        weight c = safe(channel, std::ceil(px), std::ceil(py));
                        weight d = safe(channel, std::floor(px), std::ceil(py));

                        auto e = a * (1.0 - (px - std::floor(px))) + d * (px - std::floor(px));
                        auto f = b * (1.0 - (px - std::floor(px))) + c * (px - std::floor(px));

                        auto value = e * (1.0 - (py - std::floor(py))) + f * (py - std::floor(py));

                        result(channel, x, y) = value;;
                    }
                }
            }

            output.push_back(result);
        }
    }

    static void concat_name(std::string& name) {
        name += " elastic<" + std::to_string(C) + ", " + std::to_string(K) + ">";
    }
};

} //end of dll namespace
<|endoftext|>


<commit_before>#ifndef GZILT_DELEGATE_HPP
#define GZILT_DELEGATE_HPP

#include <utility>

namespace gzilt {

template< typename FunctionSignature >
class Delegate;

template< typename ReturnType, typename... Arguments >
class Delegate< ReturnType(Arguments...) > {
    using FunctionPtrType = ReturnType(*)(const void *, Arguments...);

    FunctionPtrType delegated_function_;
    const void * this_ptr_;

    Delegate() = delete;
public:
    Delegate(FunctionPtrType delegated_function, const void* this_ptr) :
        delegated_function_(delegated_function),
        this_ptr_(this_ptr) {}
    template< typename... ForwardArguments >
    ReturnType invoke(ForwardArguments&&... arguments) const {
        return (*delegated_function_)(this_ptr_, std::forward<ForwardArguments>(arguments)...);
    }
    template< typename... ForwardArguments >
    ReturnType operator()(ForwardArguments&&... arguments) const {
        return invoke(std::forward<ForwardArguments>(arguments)...);
    }
};

namespace detail {

template< typename ReturnType, class T, typename... Arguments >
class DelegateMemberFunction {
    DelegateMemberFunction() = delete;
    template< ReturnType(T::*MemberFunction)(Arguments...) >
    static ReturnType caller(const void * this_ptr, Arguments... arguments) {
        auto callee = const_cast<T *> (static_cast<const T *>(this_ptr));
        return (callee->*MemberFunction)(arguments ...);
    }
public:
    template< ReturnType(T::*MemberFunction)(Arguments...) >
    static Delegate< ReturnType(Arguments...) > bind(T * this_ptr, bool /*unused*/ = false) {
        auto static_caller = &DelegateMemberFunction::caller<MemberFunction>;
        return Delegate< ReturnType(Arguments...) >(static_caller, static_cast<const void *>(this_ptr));
    }
};

template< typename ReturnType, class T, typename... Arguments >
class DelegateConstMemberFunction {
	DelegateConstMemberFunction() = delete;
	template< ReturnType(T::*ConstMemberFunction)(Arguments...) const >
	static ReturnType caller(const void * this_ptr, Arguments... arguments) {
		auto callee = static_cast<const T*>(this_ptr);
		return (callee->*ConstMemberFunction)(arguments...);
	}
public:
	template< ReturnType(T::*ConstMemberFunction)(Arguments...) const >
	static Delegate< ReturnType(Arguments...) > bind(T * this_ptr, bool /*unused*/ = false) {
		auto static_caller = &DelegateConstMemberFunction::caller<ConstMemberFunction>;
		return Delegate< ReturnType(Arguments...) >(static_caller, static_cast<const void*>(this_ptr));
	}
};

template< typename ReturnType, typename... Arguments >
class DelegateFreeFunction {
    DelegateFreeFunction() = delete;
    template< ReturnType(*FreeFunction)(Arguments... arguments) >
    static ReturnType caller(const void * /*unused*/, Arguments... arguments) {
        return (*FreeFunction)(arguments...);
    }
public:
    template< ReturnType(*FreeFunction)(Arguments... args) >
    static Delegate< ReturnType(Arguments...) > bind(bool /*unused*/ = false) {
        auto static_caller = &DelegateFreeFunction::caller<FreeFunction>;
        return Delegate< ReturnType(Arguments...) >(static_caller, nullptr);
    }
};

template< typename FunctionSignature >
struct DelegateMatcher;

template< typename ReturnType, class Class, typename... Arguments >
struct DelegateMatcher< ReturnType(Class::*)(Arguments...) > {
    using DelegateType = DelegateMemberFunction< ReturnType, Class, Arguments...>;
    DelegateMatcher() = delete;
};
template< typename ReturnType, class Class, typename... Arguments >
struct DelegateMatcher< ReturnType(Class::*)(Arguments...) const > {
	using DelegateType = DelegateConstMemberFunction< ReturnType, Class, Arguments...>;
	DelegateMatcher() = delete;
};
template< typename ReturnType, typename... Arguments >
struct DelegateMatcher< ReturnType(*)(Arguments...) > {
    using DelegateType = DelegateFreeFunction< ReturnType, Arguments...>;
    DelegateMatcher() = delete;
};

} // namespace:detail

} // namespace: gzilt

#define CREATE_DELEGATE_( target, ... ) \
    (gzilt::detail::DelegateMatcher< decltype(target) >::DelegateType::bind<target>(__VA_ARGS__))

#ifdef _MSC_VER
#define CREATE_DELEGATE CREATE_DELEGATE_
#else
#define CREATE_DELEGATE(...) CREATE_DELEGATE_( __VA_ARGS__, false )
#endif

#endif
<commit_msg>More consistent parameter naming<commit_after>#ifndef GZILT_DELEGATE_HPP
#define GZILT_DELEGATE_HPP

#include <utility>

namespace gzilt {

template< typename FunctionSignature >
class Delegate;

template< typename ReturnType, typename... Arguments >
class Delegate< ReturnType(Arguments...) > {
    using FunctionPtrType = ReturnType(*)(const void *, Arguments...);

    FunctionPtrType delegated_function_;
    const void * this_ptr_;

    Delegate() = delete;
public:
    Delegate(FunctionPtrType delegated_function, const void* this_ptr) :
        delegated_function_(delegated_function),
        this_ptr_(this_ptr) {}
    template< typename... ForwardArguments >
    ReturnType invoke(ForwardArguments&&... arguments) const {
        return (*delegated_function_)(this_ptr_, std::forward<ForwardArguments>(arguments)...);
    }
    template< typename... ForwardArguments >
    ReturnType operator()(ForwardArguments&&... arguments) const {
        return invoke(std::forward<ForwardArguments>(arguments)...);
    }
};

namespace detail {

template< typename ReturnType, class Class, typename... Arguments >
class DelegateMemberFunction {
    DelegateMemberFunction() = delete;
    template< ReturnType(Class::*MemberFunction)(Arguments...) >
    static ReturnType caller(const void * this_ptr, Arguments... arguments) {
        auto callee = const_cast<Class *> (static_cast<const Class *>(this_ptr));
        return (callee->*MemberFunction)(arguments ...);
    }
public:
    template< ReturnType(Class::*MemberFunction)(Arguments...) >
    static Delegate< ReturnType(Arguments...) > bind(Class * this_ptr, bool /*unused*/ = false) {
        auto static_caller = &DelegateMemberFunction::caller<MemberFunction>;
        return Delegate< ReturnType(Arguments...) >(static_caller, static_cast<const void *>(this_ptr));
    }
};

template< typename ReturnType, class Class, typename... Arguments >
class DelegateConstMemberFunction {
	DelegateConstMemberFunction() = delete;
	template< ReturnType(Class::*ConstMemberFunction)(Arguments...) const >
	static ReturnType caller(const void * this_ptr, Arguments... arguments) {
		auto callee = static_cast<const Class*>(this_ptr);
		return (callee->*ConstMemberFunction)(arguments...);
	}
public:
	template< ReturnType(Class::*ConstMemberFunction)(Arguments...) const >
	static Delegate< ReturnType(Arguments...) > bind(Class * this_ptr, bool /*unused*/ = false) {
		auto static_caller = &DelegateConstMemberFunction::caller<ConstMemberFunction>;
		return Delegate< ReturnType(Arguments...) >(static_caller, static_cast<const void*>(this_ptr));
	}
};

template< typename ReturnType, typename... Arguments >
class DelegateFreeFunction {
    DelegateFreeFunction() = delete;
    template< ReturnType(*FreeFunction)(Arguments... arguments) >
    static ReturnType caller(const void * /*unused*/, Arguments... arguments) {
        return (*FreeFunction)(arguments...);
    }
public:
    template< ReturnType(*FreeFunction)(Arguments... arguments) >
    static Delegate< ReturnType(Arguments...) > bind(bool /*unused*/ = false) {
        auto static_caller = &DelegateFreeFunction::caller<FreeFunction>;
        return Delegate< ReturnType(Arguments...) >(static_caller, nullptr);
    }
};

template< typename FunctionSignature >
struct DelegateMatcher;

template< typename ReturnType, class Class, typename... Arguments >
struct DelegateMatcher< ReturnType(Class::*)(Arguments...) > {
    using DelegateType = DelegateMemberFunction< ReturnType, Class, Arguments...>;
    DelegateMatcher() = delete;
};
template< typename ReturnType, class Class, typename... Arguments >
struct DelegateMatcher< ReturnType(Class::*)(Arguments...) const > {
	using DelegateType = DelegateConstMemberFunction< ReturnType, Class, Arguments...>;
	DelegateMatcher() = delete;
};
template< typename ReturnType, typename... Arguments >
struct DelegateMatcher< ReturnType(*)(Arguments...) > {
    using DelegateType = DelegateFreeFunction< ReturnType, Arguments...>;
    DelegateMatcher() = delete;
};

} // namespace:detail

} // namespace: gzilt

#define CREATE_DELEGATE_( target, ... ) \
    (gzilt::detail::DelegateMatcher< decltype(target) >::DelegateType::bind<target>(__VA_ARGS__))

#ifdef _MSC_VER
#define CREATE_DELEGATE CREATE_DELEGATE_
#else
#define CREATE_DELEGATE(...) CREATE_DELEGATE_( __VA_ARGS__, false )
#endif

#endif
<|endoftext|>


<commit_before>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

#ifndef NIX_GROUP_H
#define NIX_GROUP_H

#include <string>

#include <nix/hdf5/hdf5include.hpp>
#include <nix/hdf5/DataSet.hpp>

namespace nix {
namespace hdf5 {

/**
 * TODO documentation
 */
class Group {

private:

    H5::Group h5group;

public:

    Group();

    Group(H5::Group h5group);

    Group(const Group &group);

    Group& operator=(const Group &group);

    bool hasAttr(const std::string &name) const;
    void removeAttr(const std::string &name) const;

    template <typename T>
    void setAttr(const std::string &name, const T &value) const;

    template <typename T>
    bool getAttr(const std::string &name, T &value) const;

    bool hasObject(const std::string &path) const;
    size_t objectCount() const;
    std::string objectName(size_t index) const;

    bool hasData(const std::string &name) const;
    DataSet openData(const std::string &name) const;
    void removeData(const std::string &name);

    bool hasGroup(const std::string &name) const;
    Group openGroup(const std::string &name, bool create = true) const;
    void removeGroup(const std::string &name);
    void renameGroup(const std::string &old_name, const std::string &new_name);

    bool operator==(const Group &group) const;
    bool operator!=(const Group &group) const;

    H5::Group h5Group() const;
    virtual ~Group();
}; // group Group

//template functions

template<typename T> void Group::setAttr(const std::string &name, const T &value) const
{
    const Charon<const T> charon(value);
    H5::Attribute attr;

    if (hasAttr(name)) {
        attr = h5group.openAttribute(name);
    } else {
        H5::DataType fileType = charon.getFileType();
        H5::DataSpace fileSpace = charon.createDataSpace(false);
        attr = h5group.createAttribute(name, fileType, fileSpace);
    }

    typedef typename Charon<const T>::dbox_type dbox_type;
    const dbox_type data = charon.get_data();
    attr.write(charon.getMemType(), *data);
    data.finish();
}

template<typename T> bool Group::getAttr(const std::string &name, T &value) const
{

    if (!hasAttr(name)) {
        return false;
    }

    Charon<T> charon(value);
    H5::Attribute attr = h5group.openAttribute(name);

    H5::DataSpace space = attr.getSpace();
    int rank = space.getSimpleExtentNdims();
    NDSize dims(static_cast<size_t>(rank));
    space.getSimpleExtentDims (dims.data(), nullptr);
    charon.resize(dims);

    typedef typename Charon<T>::dbox_type dbox_type;
    dbox_type data = charon.get_data();
    attr.read(charon.getMemType(), *data);
    data.finish(&space);

    return true;
}

} // namespace hdf5
} // namespace nix

#endif /* NIX_GROUP_H */
<commit_msg>hdf5::Group: refactor to use Hydra instead of Charon<commit_after>// Copyright (c) 2013, German Neuroinformatics Node (G-Node)
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted under the terms of the BSD License. See
// LICENSE file in the root of the Project.

#ifndef NIX_GROUP_H
#define NIX_GROUP_H

#include <string>

#include <nix/hdf5/hdf5include.hpp>
#include <nix/hdf5/DataSet.hpp>

#include <nix/Hydra.hpp>
#include <nix/hdf5/DataSpace.hpp>

namespace nix {
namespace hdf5 {

/**
 * TODO documentation
 */
class Group {

private:

    H5::Group h5group;

public:

    Group();

    Group(H5::Group h5group);

    Group(const Group &group);

    Group& operator=(const Group &group);

    bool hasAttr(const std::string &name) const;
    void removeAttr(const std::string &name) const;

    template <typename T>
    void setAttr(const std::string &name, const T &value) const;

    template <typename T>
    bool getAttr(const std::string &name, T &value) const;

    bool hasObject(const std::string &path) const;
    size_t objectCount() const;
    std::string objectName(size_t index) const;

    bool hasData(const std::string &name) const;
    DataSet openData(const std::string &name) const;
    void removeData(const std::string &name);

    bool hasGroup(const std::string &name) const;
    Group openGroup(const std::string &name, bool create = true) const;
    void removeGroup(const std::string &name);
    void renameGroup(const std::string &old_name, const std::string &new_name);

    bool operator==(const Group &group) const;
    bool operator!=(const Group &group) const;

    H5::Group h5Group() const;
    virtual ~Group();
}; // group Group

//template functions

template<typename T> void Group::setAttr(const std::string &name, const T &value) const
{
    typedef Hydra<const T> hydra_t;
    typedef typename hydra_t::reader_t reader_t;

    const hydra_t hydra(value);
    DataType dtype = hydra.element_data_type();
    NDSize shape = hydra.shape();

    H5::Attribute attr;

    if (hasAttr(name)) {
        attr = h5group.openAttribute(name);
    } else {
        H5::DataType fileType = data_type_to_h5_filetype(dtype);
        H5::DataSpace fileSpace = DataSpace::create(shape, false);
        attr = h5group.createAttribute(name, fileType, fileSpace);
    }

    reader_t reader = hydra.reader();
    attr.write(data_type_to_h5_memtype(dtype), reader.begin());
    reader.finish();
}

template<typename T> bool Group::getAttr(const std::string &name, T &value) const
{
    typedef Hydra<T> hydra_t;
    typedef typename hydra_t::writer_t writer_t;

    if (!hasAttr(name)) {
        return false;
    }

    hydra_t hydra(value);

    //determine attr's size and resize value accordingly
    H5::Attribute attr = h5group.openAttribute(name);
    H5::DataSpace space = attr.getSpace();
    int rank = space.getSimpleExtentNdims();
    NDSize dims(static_cast<size_t>(rank));
    space.getSimpleExtentDims (dims.data(), nullptr);
    hydra.resize(dims);

    writer_t writer = hydra.writer();
    DataType dtype = hydra.element_data_type();

    attr.read(data_type_to_h5_memtype(dtype), writer.begin());
    writer.finish();

    if (dtype == DataType::String) {
        H5::DataSet::vlenReclaim(writer.begin(), data_type_to_h5_memtype(dtype), space);
    }

    return true;
}

} // namespace hdf5
} // namespace nix

#endif /* NIX_GROUP_H */
<|endoftext|>