<commit_before>/**
 * Copyright (C) 2013 Regents of the University of California.
 * @author: Jeff Thompson <jefft0@remap.ucla.edu>
 * See COPYING for copyright and distribution information.
 */

#ifndef NDN_SHA256_WITH_RSA_SIGNATURE_HPP
#define	NDN_SHA256_WITH_RSA_SIGNATURE_HPP

#include "data.hpp"
#include "publisher-public-key-digest.hpp"

namespace ndn {

/**
 * A Sha256WithRsaSignature extends Signature and holds the signature bits and other info representing a
 * SHA256-with-RSA signature in a data packet.
 */
class Sha256WithRsaSignature : public Signature {
public:
  /**
   * Return a pointer to a new Sha256WithRsaSignature which is a copy of this signature.
   */
  virtual ptr_lib::shared_ptr<Signature>
  clone() const;

  /**
   * Set the signatureStruct to point to the values in this signature object, without copying any memory.
   * WARNING: The resulting pointers in signatureStruct are invalid after a further use of this object which could reallocate memory.
   * @param signatureStruct a C ndn_Signature struct where the name components array is already allocated.
   */
  virtual void
  get(struct ndn_Signature& signatureStruct) const;

  /**
   * Clear this signature, and set the values by copying from the ndn_Signature struct.
   * @param signatureStruct a C ndn_Signature struct
   */
  virtual void
  set(const struct ndn_Signature& signatureStruct);

  const Blob&
  getDigestAlgorithm() const { return digestAlgorithm_; }

  const Blob&
  getWitness() const { return witness_; }

  const Blob&
  getSignature() const { return signature_; }

  const PublisherPublicKeyDigest&
  getPublisherPublicKeyDigest() const { return publisherPublicKeyDigest_; }

  PublisherPublicKeyDigest&
  getPublisherPublicKeyDigest() { return publisherPublicKeyDigest_; }

  const KeyLocator&
  getKeyLocator() const { return keyLocator_; }

  KeyLocator&
  getKeyLocator() { return keyLocator_; }

  void
  setDigestAlgorithm(const std::vector<unsigned char>& digestAlgorithm) { digestAlgorithm_ = digestAlgorithm; }

  void
  setDigestAlgorithm(const unsigned char *digestAlgorithm, unsigned int digestAlgorithmLength)
  {
    digestAlgorithm_ = Blob(digestAlgorithm, digestAlgorithmLength);
  }

  void
  setWitness(const std::vector<unsigned char>& witness) { witness_ = witness; }

  void
  setWitness(const unsigned char *witness, unsigned int witnessLength)
  {
    witness_ = Blob(witness, witnessLength);
  }

  void
  setSignature(const std::vector<unsigned char>& signature) { signature_ = signature; }

  void
  setSignature(const unsigned char *signature, unsigned int signatureLength)
  {
    signature_ = Blob(signature, signatureLength);
  }

  void
  setPublisherPublicKeyDigest(const PublisherPublicKeyDigest& publisherPublicKeyDigest) { publisherPublicKeyDigest_ = publisherPublicKeyDigest; }

  void
  setKeyLocator(const KeyLocator& keyLocator) { keyLocator_ = keyLocator; }

  /**
   * Clear all the fields.
   */
  void
  clear()
  {
    digestAlgorithm_.reset();
    witness_.reset();
    signature_.reset();
    publisherPublicKeyDigest_.clear();
    keyLocator_.clear();
  }

private:
  Blob digestAlgorithm_; /**< if empty, the default is 2.16.840.1.101.3.4.2.1 (sha-256) */
  Blob witness_;
  Blob signature_;
  PublisherPublicKeyDigest publisherPublicKeyDigest_;
  KeyLocator keyLocator_;
};

}

#endif
<commit_msg>security: Added setSignature for Blob.<commit_after>/**
 * Copyright (C) 2013 Regents of the University of California.
 * @author: Jeff Thompson <jefft0@remap.ucla.edu>
 * See COPYING for copyright and distribution information.
 */

#ifndef NDN_SHA256_WITH_RSA_SIGNATURE_HPP
#define	NDN_SHA256_WITH_RSA_SIGNATURE_HPP

#include "data.hpp"
#include "key.hpp"
#include "publisher-public-key-digest.hpp"

namespace ndn {

/**
 * A Sha256WithRsaSignature extends Signature and holds the signature bits and other info representing a
 * SHA256-with-RSA signature in a data packet.
 */
class Sha256WithRsaSignature : public Signature {
public:
  /**
   * Return a pointer to a new Sha256WithRsaSignature which is a copy of this signature.
   */
  virtual ptr_lib::shared_ptr<Signature>
  clone() const;

  /**
   * Set the signatureStruct to point to the values in this signature object, without copying any memory.
   * WARNING: The resulting pointers in signatureStruct are invalid after a further use of this object which could reallocate memory.
   * @param signatureStruct a C ndn_Signature struct where the name components array is already allocated.
   */
  virtual void
  get(struct ndn_Signature& signatureStruct) const;

  /**
   * Clear this signature, and set the values by copying from the ndn_Signature struct.
   * @param signatureStruct a C ndn_Signature struct
   */
  virtual void
  set(const struct ndn_Signature& signatureStruct);

  const Blob&
  getDigestAlgorithm() const { return digestAlgorithm_; }

  const Blob&
  getWitness() const { return witness_; }

  const Blob&
  getSignature() const { return signature_; }

  const PublisherPublicKeyDigest&
  getPublisherPublicKeyDigest() const { return publisherPublicKeyDigest_; }

  PublisherPublicKeyDigest&
  getPublisherPublicKeyDigest() { return publisherPublicKeyDigest_; }

  const KeyLocator&
  getKeyLocator() const { return keyLocator_; }

  KeyLocator&
  getKeyLocator() { return keyLocator_; }

  void
  setDigestAlgorithm(const std::vector<unsigned char>& digestAlgorithm) { digestAlgorithm_ = digestAlgorithm; }

  void
  setDigestAlgorithm(const unsigned char *digestAlgorithm, unsigned int digestAlgorithmLength)
  {
    digestAlgorithm_ = Blob(digestAlgorithm, digestAlgorithmLength);
  }

  void
  setWitness(const std::vector<unsigned char>& witness) { witness_ = witness; }

  void
  setWitness(const unsigned char *witness, unsigned int witnessLength)
  {
    witness_ = Blob(witness, witnessLength);
  }

  void
  setSignature(const std::vector<unsigned char>& signature) { signature_ = signature; }

  void
  setSignature(const unsigned char *signature, unsigned int signatureLength)
  {
    signature_ = Blob(signature, signatureLength);
  }

  /**
   * Set signature to point to an existing byte array.  IMPORTANT: After calling this,
   * if you keep a pointer to the array then you must treat the array as immutable and promise not to change it.
   * @param signature A pointer to a vector with the byte array.  This takes another reference and does not copy the bytes.
   */
  void
  setSignature(const ptr_lib::shared_ptr<std::vector<unsigned char> > &signature) { signature_ = signature; }

  void
  setSignature(const ptr_lib::shared_ptr<const std::vector<unsigned char> > &signature) { signature_ = signature; }

  void
  setPublisherPublicKeyDigest(const PublisherPublicKeyDigest& publisherPublicKeyDigest) { publisherPublicKeyDigest_ = publisherPublicKeyDigest; }

  void
  setKeyLocator(const KeyLocator& keyLocator) { keyLocator_ = keyLocator; }

  /**
   * Clear all the fields.
   */
  void
  clear()
  {
    digestAlgorithm_.reset();
    witness_.reset();
    signature_.reset();
    publisherPublicKeyDigest_.clear();
    keyLocator_.clear();
  }

private:
  Blob digestAlgorithm_; /**< if empty, the default is 2.16.840.1.101.3.4.2.1 (sha-256) */
  Blob witness_;
  Blob signature_;
  PublisherPublicKeyDigest publisherPublicKeyDigest_;
  KeyLocator keyLocator_;
};

}

#endif
<|endoftext|>


<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/perftimer.h"
#include "base/scoped_handle.h"
#include "base/timer.h"
#include "net/base/net_errors.h"
#include "net/disk_cache/block_files.h"
#include "net/disk_cache/disk_cache.h"
#include "net/disk_cache/disk_cache_test_base.h"
#include "net/disk_cache/disk_cache_test_util.h"
#include "net/disk_cache/hash.h"
#include "testing/gtest/include/gtest/gtest.h"

extern int g_cache_tests_max_id;
extern volatile int g_cache_tests_received;
extern volatile bool g_cache_tests_error;

namespace {

bool EvictFileFromSystemCache(const wchar_t* name) {
  // Overwrite it with no buffering.
  ScopedHandle file(CreateFile(name, GENERIC_READ | GENERIC_WRITE,
                               FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
                               OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, NULL));
  if (!file.IsValid())
    return false;

  // Execute in chunks. It could be optimized. We want to do few of these since
  // these opterations will be slow without the cache.
  char buffer[128 * 1024];
  int total_bytes = 0;
  DWORD bytes_read;
  for (;;) {
    if (!ReadFile(file, buffer, sizeof(buffer), &bytes_read, NULL))
      return false;
    if (bytes_read == 0)
      break;

    bool final = false;
    if (bytes_read < sizeof(buffer))
      final = true;

    DWORD to_write = final ? sizeof(buffer) : bytes_read;

    DWORD actual;
    SetFilePointer(file, total_bytes, 0, FILE_BEGIN);
    if (!WriteFile(file, buffer, to_write, &actual, NULL))
      return false;
    total_bytes += bytes_read;

    if (final) {
      SetFilePointer(file, total_bytes, 0, FILE_BEGIN);
      SetEndOfFile(file);
      break;
    }
  }
  return true;
}

struct TestEntry {
  std::string key;
  int data_len;
};
typedef std::vector<TestEntry> TestEntries;

const int kMaxSize = 16 * 1024 - 1;

// Creates num_entries on the cache, and writes 200 bytes of metadata and up
// to kMaxSize of data to each entry.
int TimeWrite(int num_entries, disk_cache::Backend* cache,
              TestEntries* entries) {
  char buffer1[200];
  char buffer2[kMaxSize];

  CacheTestFillBuffer(buffer1, sizeof(buffer1), false);
  CacheTestFillBuffer(buffer2, sizeof(buffer2), false);

  CallbackTest callback(1);
  g_cache_tests_error = false;
  g_cache_tests_max_id = 1;
  g_cache_tests_received = 0;
  int expected = 0;

  MessageLoopHelper helper;

  PerfTimeLogger timer("Write disk cache entries");

  for (int i = 0; i < num_entries; i++) {
    TestEntry entry;
    entry.key = GenerateKey(true);
    entry.data_len = rand() % sizeof(buffer2);
    entries->push_back(entry);

    disk_cache::Entry* cache_entry;
    if (!cache->CreateEntry(entry.key, &cache_entry))
      break;
    int ret = cache_entry->WriteData(0, 0, buffer1, sizeof(buffer1), &callback,
                                     false);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (sizeof(buffer1) != ret)
      break;

    ret = cache_entry->WriteData(1, 0, buffer2, entry.data_len, &callback,
                                 false);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (entry.data_len != ret)
      break;
    cache_entry->Close();
  }

  helper.WaitUntilCacheIoFinished(expected);
  timer.Done();

  return expected;
}

// Reads the data and metadata from each entry listed on |entries|.
int TimeRead(int num_entries, disk_cache::Backend* cache,
             const TestEntries& entries, bool cold) {
  char buffer1[200];
  char buffer2[kMaxSize];

  CacheTestFillBuffer(buffer1, sizeof(buffer1), false);
  CacheTestFillBuffer(buffer2, sizeof(buffer2), false);

  CallbackTest callback(1);
  g_cache_tests_error = false;
  g_cache_tests_max_id = 1;
  g_cache_tests_received = 0;
  int expected = 0;

  MessageLoopHelper helper;

  const char* message = cold ? "Read disk cache entries (cold)" :
                        "Read disk cache entries (warm)";
  PerfTimeLogger timer(message);

  for (int i = 0; i < num_entries; i++) {
    disk_cache::Entry* cache_entry;
    if (!cache->OpenEntry(entries[i].key, &cache_entry))
      break;
    int ret = cache_entry->ReadData(0, 0, buffer1, sizeof(buffer1), &callback);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (sizeof(buffer1) != ret)
      break;

    ret = cache_entry->ReadData(1, 0, buffer2, entries[i].data_len, &callback);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (entries[i].data_len != ret)
      break;
    cache_entry->Close();
  }

  helper.WaitUntilCacheIoFinished(expected);
  timer.Done();

  return expected;
}

int BlockSize() {
  // We can use form 1 to 4 blocks.
  return (rand() & 0x3) + 1;
}

}  // namespace

TEST_F(DiskCacheTest, Hash) {
  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  PerfTimeLogger timer("Hash disk cache keys");
  for (int i = 0; i < 300000; i++) {
    std::string key = GenerateKey(true);
    uint32 hash = disk_cache::Hash(key);
  }
  timer.Done();
}

TEST_F(DiskCacheTest, CacheBackendPerformance) {
  MessageLoopForIO message_loop;

  std::wstring path = GetCachePath();
  ASSERT_TRUE(DeleteCache(path.c_str()));
  disk_cache::Backend* cache = disk_cache::CreateCacheBackend(path, false, 0);
  ASSERT_TRUE(NULL != cache);

  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  TestEntries entries;
  int num_entries = 1000;

  int ret = TimeWrite(num_entries, cache, &entries);
  EXPECT_EQ(ret, g_cache_tests_received);

  delete cache;

  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\index").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_0").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_1").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_2").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_3").c_str()));

  cache = disk_cache::CreateCacheBackend(path, false, 0);
  ASSERT_TRUE(NULL != cache);

  ret = TimeRead(num_entries, cache, entries, true);
  EXPECT_EQ(ret, g_cache_tests_received);

  ret = TimeRead(num_entries, cache, entries, false);
  EXPECT_EQ(ret, g_cache_tests_received);

  delete cache;
}

TEST_F(DiskCacheTest, BlockFilesPerformance) {
  std::wstring path = GetCachePath();
  ASSERT_TRUE(DeleteCache(path.c_str()));

  disk_cache::BlockFiles files(path);
  ASSERT_TRUE(files.Init(true));

  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  const int kNumEntries = 60000;
  int32 buffer[kNumEntries];
  memset(buffer, 0, sizeof(buffer));
  disk_cache::Addr* address = reinterpret_cast<disk_cache::Addr*>(buffer);
  ASSERT_EQ(sizeof(*address), sizeof(*buffer));

  PerfTimeLogger timer1("Fill three block-files");

  // Fill up the 32-byte block file (use three files).
  for (int i = 0; i < kNumEntries; i++) {
    EXPECT_TRUE(files.CreateBlock(disk_cache::RANKINGS, BlockSize(),
                                  &address[i]));
  }

  timer1.Done();
  PerfTimeLogger timer2("Create and delete blocks");

  for (int i = 0; i < 200000; i++) {
    int entry = rand() * (kNumEntries / RAND_MAX + 1);
    if (entry >= kNumEntries)
      entry = 0;

    files.DeleteBlock(address[entry], false);
    EXPECT_TRUE(files.CreateBlock(disk_cache::RANKINGS, BlockSize(),
                                  &address[entry]));
  }

  timer2.Done();
}
<commit_msg>Fixing the comment that was not included with the last cl.<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/perftimer.h"
#include "base/scoped_handle.h"
#include "base/timer.h"
#include "net/base/net_errors.h"
#include "net/disk_cache/block_files.h"
#include "net/disk_cache/disk_cache.h"
#include "net/disk_cache/disk_cache_test_base.h"
#include "net/disk_cache/disk_cache_test_util.h"
#include "net/disk_cache/hash.h"
#include "testing/gtest/include/gtest/gtest.h"

extern int g_cache_tests_max_id;
extern volatile int g_cache_tests_received;
extern volatile bool g_cache_tests_error;

namespace {

bool EvictFileFromSystemCache(const wchar_t* name) {
  // Overwrite it with no buffering.
  ScopedHandle file(CreateFile(name, GENERIC_READ | GENERIC_WRITE,
                               FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
                               OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, NULL));
  if (!file.IsValid())
    return false;

  // Execute in chunks. It could be optimized. We want to do few of these since
  // these opterations will be slow without the cache.
  char buffer[128 * 1024];
  int total_bytes = 0;
  DWORD bytes_read;
  for (;;) {
    if (!ReadFile(file, buffer, sizeof(buffer), &bytes_read, NULL))
      return false;
    if (bytes_read == 0)
      break;

    bool final = false;
    if (bytes_read < sizeof(buffer))
      final = true;

    DWORD to_write = final ? sizeof(buffer) : bytes_read;

    DWORD actual;
    SetFilePointer(file, total_bytes, 0, FILE_BEGIN);
    if (!WriteFile(file, buffer, to_write, &actual, NULL))
      return false;
    total_bytes += bytes_read;

    if (final) {
      SetFilePointer(file, total_bytes, 0, FILE_BEGIN);
      SetEndOfFile(file);
      break;
    }
  }
  return true;
}

struct TestEntry {
  std::string key;
  int data_len;
};
typedef std::vector<TestEntry> TestEntries;

const int kMaxSize = 16 * 1024 - 1;

// Creates num_entries on the cache, and writes 200 bytes of metadata and up
// to kMaxSize of data to each entry.
int TimeWrite(int num_entries, disk_cache::Backend* cache,
              TestEntries* entries) {
  char buffer1[200];
  char buffer2[kMaxSize];

  CacheTestFillBuffer(buffer1, sizeof(buffer1), false);
  CacheTestFillBuffer(buffer2, sizeof(buffer2), false);

  CallbackTest callback(1);
  g_cache_tests_error = false;
  g_cache_tests_max_id = 1;
  g_cache_tests_received = 0;
  int expected = 0;

  MessageLoopHelper helper;

  PerfTimeLogger timer("Write disk cache entries");

  for (int i = 0; i < num_entries; i++) {
    TestEntry entry;
    entry.key = GenerateKey(true);
    entry.data_len = rand() % sizeof(buffer2);
    entries->push_back(entry);

    disk_cache::Entry* cache_entry;
    if (!cache->CreateEntry(entry.key, &cache_entry))
      break;
    int ret = cache_entry->WriteData(0, 0, buffer1, sizeof(buffer1), &callback,
                                     false);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (sizeof(buffer1) != ret)
      break;

    ret = cache_entry->WriteData(1, 0, buffer2, entry.data_len, &callback,
                                 false);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (entry.data_len != ret)
      break;
    cache_entry->Close();
  }

  helper.WaitUntilCacheIoFinished(expected);
  timer.Done();

  return expected;
}

// Reads the data and metadata from each entry listed on |entries|.
int TimeRead(int num_entries, disk_cache::Backend* cache,
             const TestEntries& entries, bool cold) {
  char buffer1[200];
  char buffer2[kMaxSize];

  CacheTestFillBuffer(buffer1, sizeof(buffer1), false);
  CacheTestFillBuffer(buffer2, sizeof(buffer2), false);

  CallbackTest callback(1);
  g_cache_tests_error = false;
  g_cache_tests_max_id = 1;
  g_cache_tests_received = 0;
  int expected = 0;

  MessageLoopHelper helper;

  const char* message = cold ? "Read disk cache entries (cold)" :
                        "Read disk cache entries (warm)";
  PerfTimeLogger timer(message);

  for (int i = 0; i < num_entries; i++) {
    disk_cache::Entry* cache_entry;
    if (!cache->OpenEntry(entries[i].key, &cache_entry))
      break;
    int ret = cache_entry->ReadData(0, 0, buffer1, sizeof(buffer1), &callback);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (sizeof(buffer1) != ret)
      break;

    ret = cache_entry->ReadData(1, 0, buffer2, entries[i].data_len, &callback);
    if (net::ERR_IO_PENDING == ret)
      expected++;
    else if (entries[i].data_len != ret)
      break;
    cache_entry->Close();
  }

  helper.WaitUntilCacheIoFinished(expected);
  timer.Done();

  return expected;
}

int BlockSize() {
  // We can use form 1 to 4 blocks.
  return (rand() & 0x3) + 1;
}

}  // namespace

TEST_F(DiskCacheTest, Hash) {
  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  PerfTimeLogger timer("Hash disk cache keys");
  for (int i = 0; i < 300000; i++) {
    std::string key = GenerateKey(true);
    uint32 hash = disk_cache::Hash(key);
  }
  timer.Done();
}

TEST_F(DiskCacheTest, CacheBackendPerformance) {
  MessageLoopForIO message_loop;

  std::wstring path = GetCachePath();
  ASSERT_TRUE(DeleteCache(path.c_str()));
  disk_cache::Backend* cache = disk_cache::CreateCacheBackend(path, false, 0);
  ASSERT_TRUE(NULL != cache);

  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  TestEntries entries;
  int num_entries = 1000;

  int ret = TimeWrite(num_entries, cache, &entries);
  EXPECT_EQ(ret, g_cache_tests_received);

  delete cache;

  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\index").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_0").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_1").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_2").c_str()));
  ASSERT_TRUE(EvictFileFromSystemCache((path + L"\\data_3").c_str()));

  cache = disk_cache::CreateCacheBackend(path, false, 0);
  ASSERT_TRUE(NULL != cache);

  ret = TimeRead(num_entries, cache, entries, true);
  EXPECT_EQ(ret, g_cache_tests_received);

  ret = TimeRead(num_entries, cache, entries, false);
  EXPECT_EQ(ret, g_cache_tests_received);

  delete cache;
}

// Creating and deleting "entries" on a block-file is something quite frequent
// (after all, almost everything is stored on block files). The operation is
// almost free when the file is empty, but can be expensive if the file gets
// fragmented, or if we have multiple files. This test measures that scenario,
// by using multiple, highly fragmented files.
TEST_F(DiskCacheTest, BlockFilesPerformance) {
  std::wstring path = GetCachePath();
  ASSERT_TRUE(DeleteCache(path.c_str()));

  disk_cache::BlockFiles files(path);
  ASSERT_TRUE(files.Init(true));

  int seed = static_cast<int>(Time::Now().ToInternalValue());
  srand(seed);

  const int kNumEntries = 60000;
  int32 buffer[kNumEntries];
  memset(buffer, 0, sizeof(buffer));
  disk_cache::Addr* address = reinterpret_cast<disk_cache::Addr*>(buffer);
  ASSERT_EQ(sizeof(*address), sizeof(*buffer));

  PerfTimeLogger timer1("Fill three block-files");

  // Fill up the 32-byte block file (use three files).
  for (int i = 0; i < kNumEntries; i++) {
    EXPECT_TRUE(files.CreateBlock(disk_cache::RANKINGS, BlockSize(),
                                  &address[i]));
  }

  timer1.Done();
  PerfTimeLogger timer2("Create and delete blocks");

  for (int i = 0; i < 200000; i++) {
    int entry = rand() * (kNumEntries / RAND_MAX + 1);
    if (entry >= kNumEntries)
      entry = 0;

    files.DeleteBlock(address[entry], false);
    EXPECT_TRUE(files.CreateBlock(disk_cache::RANKINGS, BlockSize(),
                                  &address[entry]));
  }

  timer2.Done();
}
<|endoftext|>


<commit_before>#include "streams.h"
#include <algorithm>
#include <core/json.h>
#include <core/memory.h>
#include <core/stream.h>
#include <fstream>
#include <limits>
#include <pcg/pcg_random.hpp>
#include <random>
#include <string>

#ifdef BUILD_estream
#include <streams/estream/estream_stream.h>
#endif

#ifdef BUILD_sha3
#include <streams/sha3/sha3_stream.h>
#endif

#ifdef BUILD_block
#include <streams/block/block_stream.h>
#endif

namespace _impl {

    template <std::uint8_t value> struct const_stream : stream {
        const_stream(const std::size_t osize)
            : stream(osize)
            , _data(osize) {
            std::fill_n(_data.begin(), osize, value);
        }

        vec_view next() override { return make_cview(_data); }

    private:
        std::vector<value_type> _data;
    };

    template <typename Generator> struct rng_stream : stream {
        template <typename Seeder>
        rng_stream(Seeder&& seeder, const std::size_t osize)
            : stream(osize)
            , _rng(std::forward<Seeder>(seeder))
            , _data(osize) { }

        vec_view next() override {
            std::generate_n(_data.data(), osize(), [this]() {
                return std::uniform_int_distribution<std::uint8_t>()(_rng);
            });
            return make_cview(_data);
        }

    private:
        Generator _rng;
        std::vector<value_type> _data;
    };

} // namespace _impl

/**
 * \brief Stream of true bits
 */
using true_stream = _impl::const_stream<std::numeric_limits<std::uint8_t>::max()>;

/**
 * \brief Stream of false bits
 */
using false_stream = _impl::const_stream<std::numeric_limits<std::uint8_t>::min()>;

/**
 * \brief Stream of data produced by Merseine Twister
 */
using mt19937_stream = _impl::rng_stream<std::mt19937>;

/**
 * \brief Stream of data produced by PCG (Permutation Congruential Generator)
 */
using pcg32_stream = _impl::rng_stream<pcg32>;

/**
 * @brief Stream of data read from a file
 */
struct file_stream : stream {
    file_stream(const json& config, const std::size_t osize)
        : stream(osize)
        , _path(config.at("path").get<std::string>())
        , _istream(_path, std::ios::binary)
        , _data(osize) {}

    vec_view next() override {
        _istream.read(_data.data(), osize());

        if (_istream.fail())
            throw std::runtime_error("I/O error while reading a file " + _path);
        if (_istream.eof())
            throw std::runtime_error("end of file " + _path + " reached, not enough data!");

        return make_cview(_data);
    }

private:
    const std::string _path;
    std::basic_ifstream<std::uint8_t> _istream;
    std::vector<value_type> _data;
};

/**
 * @brief Stream of counter
 */
struct counter : stream {
    counter(const std::size_t osize)
        : stream(osize)
        , _data(osize) {
        std::fill(_data.begin(), _data.end(), std::numeric_limits<value_type>::min());
    }

    vec_view next() override {
        for (value_type& value : _data) {
            if (value != std::numeric_limits<value_type>::max()) {
                ++value;
                break;
            }
            value = std::numeric_limits<value_type>::min();
        }
        return make_cview(_data);
    }

private:
    std::vector<value_type> _data;
};

std::unique_ptr<stream>
make_stream(const json& config, default_seed_source& seeder, std::size_t osize = 0) {
    const std::string type = config.at("type");

    if (osize == 0)
        throw std::runtime_error("Stream's osize " + type + " is not set in parent stream.");

    if (type == "file-stream")
        return std::make_unique<file_stream>(config, osize);
    else if (type == "true-stream")
        return std::make_unique<true_stream>(osize);
    else if (type == "false-stream")
        return std::make_unique<false_stream>(osize);
    else if (type == "counter")
        return std::make_unique<counter>(osize);
    else if (type == "mt19937-stream")
        return std::make_unique<mt19937_stream>(seeder, osize);
    else if (type == "pcg32-stream")
        return std::make_unique<pcg32_stream>(seeder, osize);
#ifdef BUILD_estream
    else if (type == "estream")
        return std::make_unique<estream_stream>(config, seeder, osize);
#endif
#ifdef BUILD_sha3
    else if (type == "sha3")
        return std::make_unique<sha3_stream>(config, seeder, osize);
#endif
#ifdef BUILD_block
    else if (type == "block")
        return std::make_unique<block::block_stream>(config, seeder, osize);
#endif
    throw std::runtime_error("requested stream named \"" + type + "\" does not exist");
}

void stream_to_dataset(dataset& set, std::unique_ptr<stream>& source) {

    auto beg = set.rawdata();
    auto end = set.rawdata() + set.rawsize();

    for (; beg != end;) {
        vec_view n = source->next();
        beg = std::copy(n.begin(), n.end(), beg);
    }
}
<commit_msg>Streams: strict avalanche criterion<commit_after>#include "streams.h"
#include <algorithm>
#include <core/json.h>
#include <core/memory.h>
#include <core/stream.h>
#include <fstream>
#include <limits>
#include <pcg/pcg_random.hpp>
#include <random>
#include <string>

#ifdef BUILD_estream
#include <streams/estream/estream_stream.h>
#endif

#ifdef BUILD_sha3
#include <streams/sha3/sha3_stream.h>
#endif

#ifdef BUILD_block
#include <streams/block/block_stream.h>
#endif

namespace _impl {

    template <std::uint8_t value> struct const_stream : stream {
        const_stream(const std::size_t osize)
            : stream(osize)
            , _data(osize) {
            std::fill_n(_data.begin(), osize, value);
        }

        vec_view next() override { return make_cview(_data); }

    private:
        std::vector<value_type> _data;
    };

    template <typename Generator> struct rng_stream : stream {
        template <typename Seeder>
        rng_stream(Seeder&& seeder, const std::size_t osize)
            : stream(osize)
            , _rng(std::forward<Seeder>(seeder))
            , _data(osize) {}

        vec_view next() override {
            std::generate_n(_data.data(), osize(), [this]() {
                return std::uniform_int_distribution<std::uint8_t>()(_rng);
            });
            return make_cview(_data);
        }

    private:
        Generator _rng;
        std::vector<value_type> _data;
    };

} // namespace _impl

/**
 * \brief Stream of true bits
 */
using true_stream = _impl::const_stream<std::numeric_limits<std::uint8_t>::max()>;

/**
 * \brief Stream of false bits
 */
using false_stream = _impl::const_stream<std::numeric_limits<std::uint8_t>::min()>;

/**
 * \brief Stream of data produced by Merseine Twister
 */
using mt19937_stream = _impl::rng_stream<std::mt19937>;

/**
 * \brief Stream of data produced by PCG (Permutation Congruential Generator)
 */
using pcg32_stream = _impl::rng_stream<pcg32>;

/**
 * @brief Stream of data read from a file
 */
struct file_stream : stream {
    file_stream(const json& config, const std::size_t osize)
        : stream(osize)
        , _path(config.at("path").get<std::string>())
        , _istream(_path, std::ios::binary)
        , _data(osize) {}

    vec_view next() override {
        _istream.read(_data.data(), osize());

        if (_istream.fail())
            throw std::runtime_error("I/O error while reading a file " + _path);
        if (_istream.eof())
            throw std::runtime_error("end of file " + _path + " reached, not enough data!");

        return make_cview(_data);
    }

private:
    const std::string _path;
    std::basic_ifstream<std::uint8_t> _istream;
    std::vector<value_type> _data;
};

/**
 * @brief Stream of counter
 */
struct counter : stream {
    counter(const std::size_t osize)
        : stream(osize)
        , _data(osize) {
        std::fill(_data.begin(), _data.end(), std::numeric_limits<value_type>::min());
    }

    vec_view next() override {
        for (value_type& value : _data) {
            if (value != std::numeric_limits<value_type>::max()) {
                ++value;
                break;
            }
            value = std::numeric_limits<value_type>::min();
        }
        return make_cview(_data);
    }

private:
    std::vector<value_type> _data;
};

/**
 * @brief Stream for testing strict avalanche criterion
 *
 * The vector consists 2 parts of same length. The first part is random,
 * the second is copy of the first with one flipped bit
 */
template <typename Generator> struct rng_stream : stream {
    template <typename Seeder>
    rng_stream(Seeder&& seeder, const std::size_t osize)
        : stream(osize)
        , _rng(std::forward<Seeder>(seeder))
        , _data(osize) {
        if (osize % 2 == 1)
            throw std::runtime_error(
                    "Stream's osize has to be even (so it contains 2 vectors of same legth).");
    }

    vec_view next() override {
        std::generate_n(_data.data(), osize() / 2, [this]() {
            return std::uniform_int_distribution<std::uint8_t>()(_rng);
        });

        std::copy_n(_data.begin(), osize() / 2, _data.begin() + osize() / 2);

        std::uniform_int_distribution<std::size_t> dist{0, osize() / 2 * 8};
        std::size_t pos = dist(_rng) + osize() / 2 * 8;

        _data[pos / 8] ^= (1 << (pos % 8));
        return make_cview(_data);
    }

private:
    Generator _rng;
    std::vector<value_type> _data;
};

std::unique_ptr<stream>
make_stream(const json& config, default_seed_source& seeder, std::size_t osize = 0) {
    const std::string type = config.at("type");

    if (osize == 0)
        throw std::runtime_error("Stream's osize " + type + " is not set in parent stream.");

    if (type == "file-stream")
        return std::make_unique<file_stream>(config, osize);
    else if (type == "true-stream")
        return std::make_unique<true_stream>(osize);
    else if (type == "false-stream")
        return std::make_unique<false_stream>(osize);
    else if (type == "counter")
        return std::make_unique<counter>(osize);
    else if (type == "mt19937-stream")
        return std::make_unique<mt19937_stream>(seeder, osize);
    else if (type == "pcg32-stream")
        return std::make_unique<pcg32_stream>(seeder, osize);
    else if (type == "sac-pcg32")
        return std::make_unique<pcg32_stream>(seeder, osize);
#ifdef BUILD_estream
    else if (type == "estream")
        return std::make_unique<estream_stream>(config, seeder, osize);
#endif
#ifdef BUILD_sha3
    else if (type == "sha3")
        return std::make_unique<sha3_stream>(config, seeder, osize);
#endif
#ifdef BUILD_block
    else if (type == "block")
        return std::make_unique<block::block_stream>(config, seeder, osize);
#endif
    throw std::runtime_error("requested stream named \"" + type + "\" does not exist");
}

void stream_to_dataset(dataset& set, std::unique_ptr<stream>& source) {

    auto beg = set.rawdata();
    auto end = set.rawdata() + set.rawsize();

    for (; beg != end;) {
        vec_view n = source->next();
        beg = std::copy(n.begin(), n.end(), beg);
    }
}
<|endoftext|>


<commit_before>// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include "output_format.h"
#include "utils/parse_int.h"
#include "utils/split.h"
#include "utils/xml_encoded.h"

namespace ufal {
namespace udpipe {

// CoNLL-U output format
class output_format_conllu : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;

 private:
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
};

const string output_format_conllu::underscore = "_";

void output_format_conllu::write_sentence(const sentence& s, ostream& os) {
  // Comments
  for (auto&& comment : s.comments)
    os << comment << '\n';

  // Words and multiword tokens
  size_t multiword_token = 0, empty_node = 0;
  for (int i = 0; i < int(s.words.size()); i++) {
    // Write non-root nodes
    if (i > 0) {
      // Multiword token if present
      if (multiword_token < s.multiword_tokens.size() &&
          i == s.multiword_tokens[multiword_token].id_first) {
        os << s.multiword_tokens[multiword_token].id_first << '-'
           << s.multiword_tokens[multiword_token].id_last << '\t'
           << s.multiword_tokens[multiword_token].form << "\t_\t_\t_\t_\t_\t_\t_\t"
           << underscore_on_empty(s.multiword_tokens[multiword_token].misc) << '\n';
        multiword_token++;
      }

      // Write the word
      os << i << '\t'
         << s.words[i].form << '\t'
         << underscore_on_empty(s.words[i].lemma) << '\t'
         << underscore_on_empty(s.words[i].upostag) << '\t'
         << underscore_on_empty(s.words[i].xpostag) << '\t'
         << underscore_on_empty(s.words[i].feats) << '\t';
      if (s.words[i].head < 0) os << '_'; else os << s.words[i].head; os << '\t'
         << underscore_on_empty(s.words[i].deprel) << '\t'
         << underscore_on_empty(s.words[i].deps) << '\t'
         << underscore_on_empty(s.words[i].misc) << '\n';
    }

    // Empty nodes
    for (; empty_node < s.empty_nodes.size() && i == s.empty_nodes[empty_node].id; empty_node++) {
      os << i << '.' << s.empty_nodes[empty_node].index << '\t'
         << s.empty_nodes[empty_node].form << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].lemma) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].upostag) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].xpostag) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].feats) << '\t'
         << "_\t"
         << "_\t"
         << underscore_on_empty(s.empty_nodes[empty_node].deps) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].misc) << '\n';
    }
  }
  os << endl;
}

// Matxin output format
class output_format_matxin : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  void write_node(const sentence& s, int node, string& pad, ostream& os);

  int sentences = 0;
};

void output_format_matxin::write_sentence(const sentence& s, ostream& os) {
  if (!sentences) {
    os << "<corpus>";
  }
  os << "\n<SENTENCE ord=\"" << ++sentences << "\" alloc=\"" << 0 << "\">\n";

  string pad;
  for (auto&& node : s.words[0].children)
    write_node(s, node, pad, os);

  os << "</SENTENCE>" << endl;
}

void output_format_matxin::finish_document(ostream& os) {
  os << "</corpus>\n";

  sentences = 0;
}

void output_format_matxin::write_node(const sentence& s, int node, string& pad, ostream& os) {
  // <NODE ord="%d" alloc="%d" form="%s" lem="%s" mi="%s" si="%s">
  pad.push_back(' ');

  os << pad << "<NODE ord=\"" << node << "\" alloc=\"" << 0
     << "\" form=\"" << xml_encoded(s.words[node].form, true)
     << "\" lem=\"" << xml_encoded(s.words[node].lemma, true)
     << "\" mi=\"" << xml_encoded(s.words[node].feats, true)
     << "\" si=\"" << xml_encoded(s.words[node].deprel, true) << '"';

  if (s.words[node].children.empty()) {
    os << "/>\n";
  } else {
    os << ">\n";
    for (auto&& child : s.words[node].children)
      write_node(s, child, pad, os);
    os << pad << "</NODE>\n";
  }

  pad.pop_back();
}

// Horizontal output format
class output_format_horizontal : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
};

void output_format_horizontal::write_sentence(const sentence& s, ostream& os) {
  string line;

  for (size_t i = 1; i < s.words.size(); i++) {
    // Append word, but replace spaces by  s
    for (auto&& chr : s.words[i].form)
      if (chr == ' ')
        line.append("\302\240");
      else
        line.push_back(chr);

    if (i+1 < s.words.size())
      line.push_back(' ');
  }
  os << line << endl;
}

// Plaintext output format
class output_format_plaintext : public output_format {
 public:
  output_format_plaintext(bool normalized): normalized(normalized) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
 private:
  bool normalized;
};

void output_format_plaintext::write_sentence(const sentence& s, ostream& os) {
  if (normalized) {
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      if (i > 1 && tok.get_space_after()) os << ' ';
      os << tok.form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << endl;
  } else {
    string spaces;
    for (size_t i = 1; i < s.words.size(); i++) {
      s.words[i].get_spaces_before(spaces); os << spaces;
      s.words[i].get_spaces_in_token(spaces); os << (!spaces.empty() ? spaces : s.words[i].form);
      s.words[i].get_spaces_after(spaces); os << spaces;
    }
    os << flush;
  }
}

// Vertical output format
class output_format_vertical : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
};

void output_format_vertical::write_sentence(const sentence& s, ostream& os) {
  for (size_t i = 1; i < s.words.size(); i++)
    os << s.words[i].form << '\n';
  os << endl;
}

// Static factory methods
output_format* output_format::new_conllu_output_format() {
  return new output_format_conllu();
}

output_format* output_format::new_matxin_output_format() {
  return new output_format_matxin();
}

output_format* output_format::new_horizontal_output_format() {
  return new output_format_horizontal();
}

output_format* output_format::new_plaintext_output_format() {
  return new output_format_plaintext(false);
}

output_format* output_format::new_plaintext_normalized_output_format() {
  return new output_format_plaintext(true);
}

output_format* output_format::new_vertical_output_format() {
  return new output_format_vertical();
}

output_format* output_format::new_output_format(const string& name) {
  if (name == "conllu") return new_conllu_output_format();
  if (name == "matxin") return new_matxin_output_format();
  if (name == "horizontal") return new_horizontal_output_format();
  if (name == "plaintext") return new_plaintext_output_format();
  if (name == "plaintext_normalized") return new_plaintext_normalized_output_format();
  if (name == "vertical") return new_vertical_output_format();
  return nullptr;
}

} // namespace udpipe
} // namespace ufal
<commit_msg>Utilize newdoc and newpar comments in plaintext_normalized output format.<commit_after>// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include "output_format.h"
#include "utils/parse_int.h"
#include "utils/split.h"
#include "utils/xml_encoded.h"

namespace ufal {
namespace udpipe {

// CoNLL-U output format
class output_format_conllu : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;

 private:
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
};

const string output_format_conllu::underscore = "_";

void output_format_conllu::write_sentence(const sentence& s, ostream& os) {
  // Comments
  for (auto&& comment : s.comments)
    os << comment << '\n';

  // Words and multiword tokens
  size_t multiword_token = 0, empty_node = 0;
  for (int i = 0; i < int(s.words.size()); i++) {
    // Write non-root nodes
    if (i > 0) {
      // Multiword token if present
      if (multiword_token < s.multiword_tokens.size() &&
          i == s.multiword_tokens[multiword_token].id_first) {
        os << s.multiword_tokens[multiword_token].id_first << '-'
           << s.multiword_tokens[multiword_token].id_last << '\t'
           << s.multiword_tokens[multiword_token].form << "\t_\t_\t_\t_\t_\t_\t_\t"
           << underscore_on_empty(s.multiword_tokens[multiword_token].misc) << '\n';
        multiword_token++;
      }

      // Write the word
      os << i << '\t'
         << s.words[i].form << '\t'
         << underscore_on_empty(s.words[i].lemma) << '\t'
         << underscore_on_empty(s.words[i].upostag) << '\t'
         << underscore_on_empty(s.words[i].xpostag) << '\t'
         << underscore_on_empty(s.words[i].feats) << '\t';
      if (s.words[i].head < 0) os << '_'; else os << s.words[i].head; os << '\t'
         << underscore_on_empty(s.words[i].deprel) << '\t'
         << underscore_on_empty(s.words[i].deps) << '\t'
         << underscore_on_empty(s.words[i].misc) << '\n';
    }

    // Empty nodes
    for (; empty_node < s.empty_nodes.size() && i == s.empty_nodes[empty_node].id; empty_node++) {
      os << i << '.' << s.empty_nodes[empty_node].index << '\t'
         << s.empty_nodes[empty_node].form << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].lemma) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].upostag) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].xpostag) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].feats) << '\t'
         << "_\t"
         << "_\t"
         << underscore_on_empty(s.empty_nodes[empty_node].deps) << '\t'
         << underscore_on_empty(s.empty_nodes[empty_node].misc) << '\n';
    }
  }
  os << endl;
}

// Matxin output format
class output_format_matxin : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  void write_node(const sentence& s, int node, string& pad, ostream& os);

  int sentences = 0;
};

void output_format_matxin::write_sentence(const sentence& s, ostream& os) {
  if (!sentences) {
    os << "<corpus>";
  }
  os << "\n<SENTENCE ord=\"" << ++sentences << "\" alloc=\"" << 0 << "\">\n";

  string pad;
  for (auto&& node : s.words[0].children)
    write_node(s, node, pad, os);

  os << "</SENTENCE>" << endl;
}

void output_format_matxin::finish_document(ostream& os) {
  os << "</corpus>\n";

  sentences = 0;
}

void output_format_matxin::write_node(const sentence& s, int node, string& pad, ostream& os) {
  // <NODE ord="%d" alloc="%d" form="%s" lem="%s" mi="%s" si="%s">
  pad.push_back(' ');

  os << pad << "<NODE ord=\"" << node << "\" alloc=\"" << 0
     << "\" form=\"" << xml_encoded(s.words[node].form, true)
     << "\" lem=\"" << xml_encoded(s.words[node].lemma, true)
     << "\" mi=\"" << xml_encoded(s.words[node].feats, true)
     << "\" si=\"" << xml_encoded(s.words[node].deprel, true) << '"';

  if (s.words[node].children.empty()) {
    os << "/>\n";
  } else {
    os << ">\n";
    for (auto&& child : s.words[node].children)
      write_node(s, child, pad, os);
    os << pad << "</NODE>\n";
  }

  pad.pop_back();
}

// Horizontal output format
class output_format_horizontal : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
};

void output_format_horizontal::write_sentence(const sentence& s, ostream& os) {
  string line;

  for (size_t i = 1; i < s.words.size(); i++) {
    // Append word, but replace spaces by  s
    for (auto&& chr : s.words[i].form)
      if (chr == ' ')
        line.append("\302\240");
      else
        line.push_back(chr);

    if (i+1 < s.words.size())
      line.push_back(' ');
  }
  os << line << endl;
}

// Plaintext output format
class output_format_plaintext : public output_format {
 public:
  output_format_plaintext(bool normalized): normalized(normalized), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }
 private:
  bool normalized;
  bool empty;
};

void output_format_plaintext::write_sentence(const sentence& s, ostream& os) {
  if (normalized) {
    string doc_par_id;
    if (!empty && (s.get_new_doc(doc_par_id) || s.get_new_par(doc_par_id)))
      os << '\n';
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      if (i > 1 && tok.get_space_after()) os << ' ';
      os << tok.form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << endl;
  } else {
    string spaces;
    for (size_t i = 1; i < s.words.size(); i++) {
      s.words[i].get_spaces_before(spaces); os << spaces;
      s.words[i].get_spaces_in_token(spaces); os << (!spaces.empty() ? spaces : s.words[i].form);
      s.words[i].get_spaces_after(spaces); os << spaces;
    }
    os << flush;
  }
  empty = false;
}

// Vertical output format
class output_format_vertical : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
};

void output_format_vertical::write_sentence(const sentence& s, ostream& os) {
  for (size_t i = 1; i < s.words.size(); i++)
    os << s.words[i].form << '\n';
  os << endl;
}

// Static factory methods
output_format* output_format::new_conllu_output_format() {
  return new output_format_conllu();
}

output_format* output_format::new_matxin_output_format() {
  return new output_format_matxin();
}

output_format* output_format::new_horizontal_output_format() {
  return new output_format_horizontal();
}

output_format* output_format::new_plaintext_output_format() {
  return new output_format_plaintext(false);
}

output_format* output_format::new_plaintext_normalized_output_format() {
  return new output_format_plaintext(true);
}

output_format* output_format::new_vertical_output_format() {
  return new output_format_vertical();
}

output_format* output_format::new_output_format(const string& name) {
  if (name == "conllu") return new_conllu_output_format();
  if (name == "matxin") return new_matxin_output_format();
  if (name == "horizontal") return new_horizontal_output_format();
  if (name == "plaintext") return new_plaintext_output_format();
  if (name == "plaintext_normalized") return new_plaintext_normalized_output_format();
  if (name == "vertical") return new_vertical_output_format();
  return nullptr;
}

} // namespace udpipe
} // namespace ufal
<|endoftext|>


<commit_before>#ifndef __LIBEVENT_SSL_SOCKET_HPP__
#define __LIBEVENT_SSL_SOCKET_HPP__

#include <event2/buffer.h>
#include <event2/bufferevent_ssl.h>
#include <event2/event.h>
#include <event2/listener.h>
#include <event2/util.h>

#include <atomic>
#include <memory>

#include <process/queue.hpp>
#include <process/socket.hpp>

namespace process {
namespace network {

class LibeventSSLSocketImpl : public Socket::Impl
{
public:
  // See 'Socket::create()'.
  static Try<std::shared_ptr<Socket::Impl>> create(int s);

  LibeventSSLSocketImpl(int _s);

  virtual ~LibeventSSLSocketImpl();

  // Implement 'Socket::Impl' interface.
  virtual Future<Nothing> connect(const Address& address);
  virtual Future<size_t> recv(char* data, size_t size);
  // Send does not currently support discard. See implementation.
  virtual Future<size_t> send(const char* data, size_t size);
  virtual Future<size_t> sendfile(int fd, off_t offset, size_t size);
  virtual Try<Nothing> listen(int backlog);
  virtual Future<Socket> accept();
  virtual Socket::Kind kind() const { return Socket::SSL; }

  // This call is used to do the equivalent of shutting down the read
  // end. This means finishing the future of any outstanding read
  // request.
  virtual void shutdown();

  // We need a post-initializer because 'shared_from_this()' is not
  // valid until the constructor has finished.
  void initialize();

private:
  // A set of helper functions that transitions an accepted socket to
  // an SSL connected socket. With the libevent-openssl library, once
  // we return from the 'accept_callback()' which is scheduled by
  // 'listen' then we still need to wait for the 'BEV_EVENT_CONNECTED'
  // state before we know the SSL connection has been established.
  struct AcceptRequest
  {
    AcceptRequest(
        int _socket,
        evconnlistener* _listener,
        const Option<net::IP>& _ip)
      : listener(_listener),
        socket(_socket),
        ip(_ip) {}
    Promise<Socket> promise;
    evconnlistener* listener;
    int socket;
    Option<net::IP> ip;
  };

  struct RecvRequest
  {
    RecvRequest(char* _data, size_t _size)
      : data(_data), size(_size) {}
    Promise<size_t> promise;
    char* data;
    size_t size;
  };

  struct SendRequest
  {
    SendRequest(size_t _size)
      : size(_size) {}
    Promise<size_t> promise;
    size_t size;
  };

  struct ConnectRequest
  {
    Promise<Nothing> promise;
  };

  // This is a private constructor used by the accept helper
  // functions.
  LibeventSSLSocketImpl(
      int _s,
      bufferevent* bev,
      Option<std::string>&& peer_hostname);

  // This is called when the equivalent of 'accept' returns. The role
  // of this function is to set up the SSL object and bev.
  void accept_callback(AcceptRequest* request);

  // The following are function pairs of static functions to member
  // functions. The static functions test and hold the weak pointer to
  // the socket before calling the member functions. This protects
  // against the socket being destroyed before the event loop calls
  // the callbacks.
  static void recv_callback(bufferevent* bev, void* arg);
  void recv_callback();

  static void send_callback(bufferevent* bev, void* arg);
  void send_callback();

  static void event_callback(bufferevent* bev, short events, void* arg);
  void event_callback(short events);

  bufferevent* bev;

  evconnlistener* listener;

  // Protects the following instance variables.
  std::atomic_flag lock = ATOMIC_LOCK_INIT;
  Owned<RecvRequest> recv_request;
  Owned<SendRequest> send_request;
  Owned<ConnectRequest> connect_request;

  // This is a weak pointer to 'this', i.e., ourselves, this class
  // instance. We need this for our event loop callbacks because it's
  // possible that we'll actually want to cleanup this socket impl
  // before the event loop callback gets executed ... and we'll check
  // in each event loop callback whether or not this weak_ptr is valid
  // by attempting to upgrade it to shared_ptr. It is the
  // responsibility of the event loop through the deferred lambda in
  // the destructor to clean up this pointer.
  // 1) It is a 'weak_ptr' as opposed to a 'shared_ptr' because we
  // want to test whether the object is still around from within the
  // event loop. If it was a 'shared_ptr' then we would be
  // contributing to the lifetime of the object and would no longer be
  // able to test the lifetime correctly.
  // 2) This is a pointer to a 'weak_ptr' so that we can pass this
  // through to the event loop through the C-interface. We need access
  // to the 'weak_ptr' from outside the object (in the event loop) to
  // test if the object is still alive. By maintaining this 'weak_ptr'
  // on the heap we can be sure it is safe to access from the
  // event loop until it is destroyed.
  std::weak_ptr<LibeventSSLSocketImpl>* event_loop_handle;

  // This queue stores buffered accepted sockets. 'Queue' is a thread
  // safe queue implementation, and the event loop pushes connected
  // sockets onto it, the 'accept()' call pops them off. We wrap these
  // sockets with futures so that we can pass errors through and chain
  // futures as well.
  Queue<Future<Socket>> accept_queue;

  // This bool represents whether this socket was created through an
  // 'accept' flow. We use this in the destructor to change
  // the clean-up behavior for the SSL context object.
  bool accepted;

  Option<std::string> peer_hostname;
};

} // namespace network {
} // namespace process {

#endif // __LIBEVENT_SSL_SOCKET_HPP__
<commit_msg>Fix ATOMIC_FLAG_INIT typo.<commit_after>#ifndef __LIBEVENT_SSL_SOCKET_HPP__
#define __LIBEVENT_SSL_SOCKET_HPP__

#include <event2/buffer.h>
#include <event2/bufferevent_ssl.h>
#include <event2/event.h>
#include <event2/listener.h>
#include <event2/util.h>

#include <atomic>
#include <memory>

#include <process/queue.hpp>
#include <process/socket.hpp>

namespace process {
namespace network {

class LibeventSSLSocketImpl : public Socket::Impl
{
public:
  // See 'Socket::create()'.
  static Try<std::shared_ptr<Socket::Impl>> create(int s);

  LibeventSSLSocketImpl(int _s);

  virtual ~LibeventSSLSocketImpl();

  // Implement 'Socket::Impl' interface.
  virtual Future<Nothing> connect(const Address& address);
  virtual Future<size_t> recv(char* data, size_t size);
  // Send does not currently support discard. See implementation.
  virtual Future<size_t> send(const char* data, size_t size);
  virtual Future<size_t> sendfile(int fd, off_t offset, size_t size);
  virtual Try<Nothing> listen(int backlog);
  virtual Future<Socket> accept();
  virtual Socket::Kind kind() const { return Socket::SSL; }

  // This call is used to do the equivalent of shutting down the read
  // end. This means finishing the future of any outstanding read
  // request.
  virtual void shutdown();

  // We need a post-initializer because 'shared_from_this()' is not
  // valid until the constructor has finished.
  void initialize();

private:
  // A set of helper functions that transitions an accepted socket to
  // an SSL connected socket. With the libevent-openssl library, once
  // we return from the 'accept_callback()' which is scheduled by
  // 'listen' then we still need to wait for the 'BEV_EVENT_CONNECTED'
  // state before we know the SSL connection has been established.
  struct AcceptRequest
  {
    AcceptRequest(
        int _socket,
        evconnlistener* _listener,
        const Option<net::IP>& _ip)
      : listener(_listener),
        socket(_socket),
        ip(_ip) {}
    Promise<Socket> promise;
    evconnlistener* listener;
    int socket;
    Option<net::IP> ip;
  };

  struct RecvRequest
  {
    RecvRequest(char* _data, size_t _size)
      : data(_data), size(_size) {}
    Promise<size_t> promise;
    char* data;
    size_t size;
  };

  struct SendRequest
  {
    SendRequest(size_t _size)
      : size(_size) {}
    Promise<size_t> promise;
    size_t size;
  };

  struct ConnectRequest
  {
    Promise<Nothing> promise;
  };

  // This is a private constructor used by the accept helper
  // functions.
  LibeventSSLSocketImpl(
      int _s,
      bufferevent* bev,
      Option<std::string>&& peer_hostname);

  // This is called when the equivalent of 'accept' returns. The role
  // of this function is to set up the SSL object and bev.
  void accept_callback(AcceptRequest* request);

  // The following are function pairs of static functions to member
  // functions. The static functions test and hold the weak pointer to
  // the socket before calling the member functions. This protects
  // against the socket being destroyed before the event loop calls
  // the callbacks.
  static void recv_callback(bufferevent* bev, void* arg);
  void recv_callback();

  static void send_callback(bufferevent* bev, void* arg);
  void send_callback();

  static void event_callback(bufferevent* bev, short events, void* arg);
  void event_callback(short events);

  bufferevent* bev;

  evconnlistener* listener;

  // Protects the following instance variables.
  std::atomic_flag lock = ATOMIC_FLAG_INIT;
  Owned<RecvRequest> recv_request;
  Owned<SendRequest> send_request;
  Owned<ConnectRequest> connect_request;

  // This is a weak pointer to 'this', i.e., ourselves, this class
  // instance. We need this for our event loop callbacks because it's
  // possible that we'll actually want to cleanup this socket impl
  // before the event loop callback gets executed ... and we'll check
  // in each event loop callback whether or not this weak_ptr is valid
  // by attempting to upgrade it to shared_ptr. It is the
  // responsibility of the event loop through the deferred lambda in
  // the destructor to clean up this pointer.
  // 1) It is a 'weak_ptr' as opposed to a 'shared_ptr' because we
  // want to test whether the object is still around from within the
  // event loop. If it was a 'shared_ptr' then we would be
  // contributing to the lifetime of the object and would no longer be
  // able to test the lifetime correctly.
  // 2) This is a pointer to a 'weak_ptr' so that we can pass this
  // through to the event loop through the C-interface. We need access
  // to the 'weak_ptr' from outside the object (in the event loop) to
  // test if the object is still alive. By maintaining this 'weak_ptr'
  // on the heap we can be sure it is safe to access from the
  // event loop until it is destroyed.
  std::weak_ptr<LibeventSSLSocketImpl>* event_loop_handle;

  // This queue stores buffered accepted sockets. 'Queue' is a thread
  // safe queue implementation, and the event loop pushes connected
  // sockets onto it, the 'accept()' call pops them off. We wrap these
  // sockets with futures so that we can pass errors through and chain
  // futures as well.
  Queue<Future<Socket>> accept_queue;

  // This bool represents whether this socket was created through an
  // 'accept' flow. We use this in the destructor to change
  // the clean-up behavior for the SSL context object.
  bool accepted;

  Option<std::string> peer_hostname;
};

} // namespace network {
} // namespace process {

#endif // __LIBEVENT_SSL_SOCKET_HPP__
<|endoftext|>


<commit_before>#include <znc/Client.h>
#include <znc/Modules.h>
#include <znc/IRCNetwork.h>
#include <znc/IRCSock.h>

class CPrioritySend : public CModule {
public:
	MODCONSTRUCTOR(CPrioritySend) {}

	virtual EModRet OnUserNotice(CString& sTarget, CString& sMessage) override {
		return ProcessMessage("NOTICE", sTarget, sMessage);
	}

	virtual EModRet OnUserMsg(CString& sTarget, CString& sMessage) override {
		return ProcessMessage("PRIVMSG", sTarget, sMessage);
	}

	EModRet ProcessMessage(const CString& sFunction, CString& sTarget, CString& sMessage) {
		if (!sMessage.TrimPrefix("(p!)")) // TODO: This could be a config variable
			return CONTINUE;

		CIRCSock* pIRCSock = GetNetwork()->GetIRCSock();

		if (!pIRCSock)
			return CONTINUE;

		pIRCSock->PutIRCQuick(sFunction + " " + sTarget + " :" + sMessage);

		return HALT;
	}
};

NETWORKMODULEDEFS(CPrioritySend, "Allows certain messages to be placed in front of the send queue")
<commit_msg>Make sure GetNetwork doesn't return null<commit_after>#include <znc/Client.h>
#include <znc/Modules.h>
#include <znc/IRCNetwork.h>
#include <znc/IRCSock.h>

class CPrioritySend : public CModule {
public:
	MODCONSTRUCTOR(CPrioritySend) {}

	virtual EModRet OnUserNotice(CString& sTarget, CString& sMessage) override {
		return ProcessMessage("NOTICE", sTarget, sMessage);
	}

	virtual EModRet OnUserMsg(CString& sTarget, CString& sMessage) override {
		return ProcessMessage("PRIVMSG", sTarget, sMessage);
	}

	EModRet ProcessMessage(const CString& sFunction, CString& sTarget, CString& sMessage) {
		if (!sMessage.TrimPrefix("(p!)")) // TODO: This could be a config variable
			return CONTINUE;

		CIRCNetwork* pNetwork = GetNetwork();

		if (!pNetwork)
			return CONTINUE;

		CIRCSock* pIRCSock = pNetwork->GetIRCSock();

		if (!pIRCSock)
			return CONTINUE;

		pIRCSock->PutIRCQuick(sFunction + " " + sTarget + " :" + sMessage);

		return HALT;
	}
};

NETWORKMODULEDEFS(CPrioritySend, "Allows certain messages to be placed in front of the send queue")
<|endoftext|>


<commit_before>/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*/

#include <gmock/gmock.h>

#include <deque>
#include <string>

#include <process/socket.hpp>

#include <stout/gtest.hpp>

#include "decoder.hpp"

using namespace process;
using namespace process::http;

using std::deque;
using std::string;

using process::network::Socket;

TEST(DecoderTest, Request)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json?key1=value1&key2=value2#fragment HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "Connection: close\r\n"
    "Accept-Encoding: compress, gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_EQ("GET", request->method);

  EXPECT_EQ("/path/file.json", request->url.path);
  EXPECT_SOME_EQ("fragment", request->url.fragment);
  EXPECT_EQ(2u, request->url.query.size());
  EXPECT_SOME_EQ("value1", request->url.query.get("key1"));
  EXPECT_SOME_EQ("value2", request->url.query.get("key2"));

  EXPECT_TRUE(request->body.empty());
  EXPECT_FALSE(request->keepAlive);

  EXPECT_EQ(3u, request->headers.size());
  EXPECT_SOME_EQ("localhost", request->headers.get("Host"));
  EXPECT_SOME_EQ("close", request->headers.get("Connection"));
  EXPECT_SOME_EQ("compress, gzip", request->headers.get("Accept-Encoding"));

  delete request;
}

TEST(DecoderTest, RequestHeaderContinuation)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "Connection: close\r\n"
    "Accept-Encoding: compress,"
    "                 gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_SOME_EQ("compress,                 gzip",
                 request->headers.get("Accept-Encoding"));
  delete request;
}

// This is expected to fail for now, see my TODO(bmahler) on http::Request.
TEST(DecoderTest, DISABLED_RequestHeaderCaseInsensitive)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "cOnnECtioN: close\r\n"
    "accept-ENCODING: compress, gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_FALSE(request->keepAlive);

  EXPECT_SOME_EQ("compress, gzip", request->headers.get("Accept-Encoding"));

  delete request;
}

TEST(DecoderTest, Response)
{
  ResponseDecoder decoder;

  const string data =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n"
    "hi";

  deque<Response*> responses = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(Response::BODY, response->type);
  EXPECT_EQ("hi", response->body);

  EXPECT_EQ(3, response->headers.size());

  delete response;
}

TEST(DecoderTest, StreamingResponse)
{
  StreamingResponseDecoder decoder;

  const string headers =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n";

  const string body = "hi";

  deque<Response*> responses = decoder.decode(headers.data(), headers.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(3, response->headers.size());

  ASSERT_EQ(Response::PIPE, response->type);
  ASSERT_SOME(response->reader);

  http::Pipe::Reader reader = response->reader.get();
  Future<string> read = reader.read();
  EXPECT_TRUE(read.isPending());

  decoder.decode(body.data(), body.length());

  // Feeding EOF to the decoder should be ok.
  decoder.decode("", 0);

  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("hi", read.get());

  // Response should be complete.
  read = reader.read();
  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("", read.get());
}

TEST(DecoderTest, StreamingResponseFailure)
{
  StreamingResponseDecoder decoder;

  const string headers =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n";

  // The body is shorter than the content length!
  const string body = "1";

  deque<Response*> responses = decoder.decode(headers.data(), headers.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(3, response->headers.size());

  ASSERT_EQ(Response::PIPE, response->type);
  ASSERT_SOME(response->reader);

  http::Pipe::Reader reader = response->reader.get();
  Future<string> read = reader.read();
  EXPECT_TRUE(read.isPending());

  decoder.decode(body.data(), body.length());

  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("1", read.get());

  // Body is not yet complete.
  read = reader.read();
  EXPECT_TRUE(read.isPending());

  // Feeding EOF to the decoder should trigger a failure!
  decoder.decode("", 0);

  EXPECT_TRUE(read.isFailed());
  EXPECT_EQ("failed to decode body", read.failure());
}
<commit_msg>Re-enabled decoder test for case insensitive headers.<commit_after>/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*/

#include <gmock/gmock.h>

#include <deque>
#include <string>

#include <process/socket.hpp>

#include <stout/gtest.hpp>

#include "decoder.hpp"

using namespace process;
using namespace process::http;

using std::deque;
using std::string;

using process::network::Socket;

TEST(DecoderTest, Request)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json?key1=value1&key2=value2#fragment HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "Connection: close\r\n"
    "Accept-Encoding: compress, gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_EQ("GET", request->method);

  EXPECT_EQ("/path/file.json", request->url.path);
  EXPECT_SOME_EQ("fragment", request->url.fragment);
  EXPECT_EQ(2u, request->url.query.size());
  EXPECT_SOME_EQ("value1", request->url.query.get("key1"));
  EXPECT_SOME_EQ("value2", request->url.query.get("key2"));

  EXPECT_TRUE(request->body.empty());
  EXPECT_FALSE(request->keepAlive);

  EXPECT_EQ(3u, request->headers.size());
  EXPECT_SOME_EQ("localhost", request->headers.get("Host"));
  EXPECT_SOME_EQ("close", request->headers.get("Connection"));
  EXPECT_SOME_EQ("compress, gzip", request->headers.get("Accept-Encoding"));

  delete request;
}

TEST(DecoderTest, RequestHeaderContinuation)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "Connection: close\r\n"
    "Accept-Encoding: compress,"
    "                 gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_SOME_EQ("compress,                 gzip",
                 request->headers.get("Accept-Encoding"));
  delete request;
}

TEST(DecoderTest, RequestHeaderCaseInsensitive)
{
  Try<Socket> socket = Socket::create();
  ASSERT_SOME(socket);
  DataDecoder decoder = DataDecoder(socket.get());

  const string data =
    "GET /path/file.json HTTP/1.1\r\n"
    "Host: localhost\r\n"
    "cOnnECtioN: close\r\n"
    "accept-ENCODING: compress, gzip\r\n"
    "\r\n";

  deque<Request*> requests = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, requests.size());

  Request* request = requests[0];
  EXPECT_FALSE(request->keepAlive);

  EXPECT_SOME_EQ("compress, gzip", request->headers.get("Accept-Encoding"));

  delete request;
}

TEST(DecoderTest, Response)
{
  ResponseDecoder decoder;

  const string data =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n"
    "hi";

  deque<Response*> responses = decoder.decode(data.data(), data.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(Response::BODY, response->type);
  EXPECT_EQ("hi", response->body);

  EXPECT_EQ(3, response->headers.size());

  delete response;
}

TEST(DecoderTest, StreamingResponse)
{
  StreamingResponseDecoder decoder;

  const string headers =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n";

  const string body = "hi";

  deque<Response*> responses = decoder.decode(headers.data(), headers.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(3, response->headers.size());

  ASSERT_EQ(Response::PIPE, response->type);
  ASSERT_SOME(response->reader);

  http::Pipe::Reader reader = response->reader.get();
  Future<string> read = reader.read();
  EXPECT_TRUE(read.isPending());

  decoder.decode(body.data(), body.length());

  // Feeding EOF to the decoder should be ok.
  decoder.decode("", 0);

  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("hi", read.get());

  // Response should be complete.
  read = reader.read();
  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("", read.get());
}

TEST(DecoderTest, StreamingResponseFailure)
{
  StreamingResponseDecoder decoder;

  const string headers =
    "HTTP/1.1 200 OK\r\n"
    "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 2\r\n"
    "\r\n";

  // The body is shorter than the content length!
  const string body = "1";

  deque<Response*> responses = decoder.decode(headers.data(), headers.length());
  ASSERT_FALSE(decoder.failed());
  ASSERT_EQ(1, responses.size());

  Response* response = responses[0];

  EXPECT_EQ("200 OK", response->status);
  EXPECT_EQ(3, response->headers.size());

  ASSERT_EQ(Response::PIPE, response->type);
  ASSERT_SOME(response->reader);

  http::Pipe::Reader reader = response->reader.get();
  Future<string> read = reader.read();
  EXPECT_TRUE(read.isPending());

  decoder.decode(body.data(), body.length());

  EXPECT_TRUE(read.isReady());
  EXPECT_EQ("1", read.get());

  // Body is not yet complete.
  read = reader.read();
  EXPECT_TRUE(read.isPending());

  // Feeding EOF to the decoder should trigger a failure!
  decoder.decode("", 0);

  EXPECT_TRUE(read.isFailed());
  EXPECT_EQ("failed to decode body", read.failure());
}
<|endoftext|>


<commit_before><commit_msg>11581 - Grid Successors<commit_after><|endoftext|>


<commit_before>/*
 * GoForward.cpp
 *
 *  Created on: Mar 25, 2014
 *      Author: user
 */

#include "GoBackward.h"

GoBackward::GoBackward(Robot* robot):Behavior(robot) {
	// TODO Auto-generated constructor stub

}
bool GoBackward::startCondition()
{

	return false;
}
void GoBackward::action()
{

}
bool GoBackward::stopCondition()
{
	return false;
}
GoBackward::~GoBackward() {
	// TODO Auto-generated destructor stub
}
<commit_msg>Add logic to this behavior<commit_after>/*
 * GoForward.cpp
 *
 *  Created on: Mar 25, 2014
 *      Author: user
 */

#include "GoBackward.h"

GoBackward::GoBackward(Robot* robot):Behavior(robot), _steps_count(0)
{
}

bool GoBackward::startCondition()
{
	return true;
}

void GoBackward::action()
{
	++_steps_count;
	_robot->setSpeed(-0.1,0.0);
}

bool GoBackward::stopCondition()
{
	if (_steps_count < 10)
	{
		return false;
	}
	else
	{
		_steps_count = 0;
		_robot->setSpeed(0.0, 0.0);
		return true;
	}

}

GoBackward::~GoBackward()
{
}
<|endoftext|>


<commit_before>#include "ofMain.h"

#include "Fisheye.h"

#include "ofxTiming.h"
#include "ofxOculusDK2.h"
#include "ofxOsc.h"
#include "ofxSyphon.h"

class ofApp : public ofBaseApp {
public:
    ofxSyphonClient cam;
    Fisheye fisheye;
	RateTimer cameraTimer, renderTimer;
    ofCamera camera;
    ofxOculusDK2 oculusRift;
    ofxOscReceiver osc;
    float targetLookAngle = 0;
    float lookAngle = 0;
    bool debug = false;

	void setup() {
        ofBackground(0);
        ofHideCursor();

        ofSetWindowPosition(1920, 0);
        ofToggleFullscreen();
        ofViewport(ofGetNativeViewport());

        cam.setup();
        cam.set("","Black Syphon");
        fisheye.setup();

        ofXml config;
        config.load("config.xml");
        osc.setup(config.getIntValue("osc/port"));

        oculusRift.baseCamera = &camera;
        oculusRift.setup();
	}
    void update() {
        renderTimer.tick();
        while(osc.hasWaitingMessages()) {
            ofxOscMessage msg;
            osc.getNextMessage(&msg);
            if(msg.getAddress() == "/lookAngle") {
                targetLookAngle = msg.getArgAsFloat(0);
            }
        }
        lookAngle = ofLerp(lookAngle, targetLookAngle, .1);
	}
    void draw() {
        ofEnableDepthTest();

        oculusRift.beginLeftEye();
        drawScene();
        oculusRift.endLeftEye();

        oculusRift.beginRightEye();
        drawScene();
        oculusRift.endRightEye();

        oculusRift.draw();

        // this is essential for setting up the texture
        cam.bind();
        cam.unbind();
    }
    void drawScene() {
        ofPushMatrix();
        ofScale(100, 100, 100); // avoid clipping
        ofRotateX(-90);
        ofRotateZ(-lookAngle);
        fisheye.draw(cam.getTexture());
        ofPopMatrix();

        if(debug) {
            ofDisableDepthTest();
            ofTranslate(-50, 0, -100);
            ofScale(1, -1, 1);
            ofSetDrawBitmapMode(OF_BITMAPMODE_MODEL);
            ofDrawBitmapStringHighlight("Camera: " + ofToString((int) cameraTimer.getFramerate()), 0, 0);
            ofSetDrawBitmapMode(OF_BITMAPMODE_MODEL);
            ofDrawBitmapStringHighlight("Render: " + ofToString((int) renderTimer.getFramerate()), 0, 40);
        }
	}
	void keyPressed(int key) {
        if(key == 'd') {
            debug = !debug;
        }
		if(key == 'f') {
			ofToggleFullscreen();
		}
        if(key == ' ') {
            string path = ofGetTimestampString() + ".jpg";
//            ofSaveImage(cam.getColorPixels(), path);
        }
	}
};

int main() {
    ofGLWindowSettings settings;
    settings.width = 1280;
    settings.height = 800;
    settings.setGLVersion(4, 1);
    ofCreateWindow(settings);
	ofRunApp(new ofApp());
}
<commit_msg>correct orientation and rotation<commit_after>#include "ofMain.h"

#include "Fisheye.h"

#include "ofxTiming.h"
#include "ofxOculusDK2.h"
#include "ofxOsc.h"
#include "ofxSyphon.h"

class ofApp : public ofBaseApp {
public:
    ofxSyphonClient cam;
    Fisheye fisheye;
	RateTimer cameraTimer, renderTimer;
    ofCamera camera;
    ofxOculusDK2 oculusRift;
    ofxOscReceiver osc;
    float targetLookAngle = 0;
    float lookAngle = 0;
    bool debug = false;

	void setup() {
        ofBackground(0);
        ofHideCursor();

        ofSetWindowPosition(1920, 0);
        ofToggleFullscreen();
        ofViewport(ofGetNativeViewport());

        cam.setup();
        cam.set("","Black Syphon");
        fisheye.setup();

        ofXml config;
        config.load("config.xml");
        osc.setup(config.getIntValue("osc/port"));

        oculusRift.baseCamera = &camera;
        oculusRift.setup();
	}
    void update() {
        renderTimer.tick();
        while(osc.hasWaitingMessages()) {
            ofxOscMessage msg;
            osc.getNextMessage(&msg);
            if(msg.getAddress() == "/lookAngle") {
                targetLookAngle = msg.getArgAsFloat(0);
            }
        }
        lookAngle = ofLerp(lookAngle, targetLookAngle, .1);
	}
    void draw() {
        ofEnableDepthTest();

        oculusRift.beginLeftEye();
        drawScene();
        oculusRift.endLeftEye();

        oculusRift.beginRightEye();
        drawScene();
        oculusRift.endRightEye();

        oculusRift.draw();

        // this is essential for setting up the texture
        cam.bind();
        cam.unbind();
    }
    void drawScene() {
        ofPushMatrix();
        ofScale(100, -100, 100); // avoid clipping
        ofRotateX(+90);
        ofRotateZ(+lookAngle);
        fisheye.draw(cam.getTexture());
        ofPopMatrix();

        if(debug) {
            ofDisableDepthTest();
            ofTranslate(-50, 0, -100);
            ofScale(1, -1, 1);
            ofSetDrawBitmapMode(OF_BITMAPMODE_MODEL);
            ofDrawBitmapStringHighlight("Camera: " + ofToString((int) cameraTimer.getFramerate()), 0, 0);
            ofSetDrawBitmapMode(OF_BITMAPMODE_MODEL);
            ofDrawBitmapStringHighlight("Render: " + ofToString((int) renderTimer.getFramerate()), 0, 40);
        }
	}
	void keyPressed(int key) {
        if(key == 'd') {
            debug = !debug;
        }
		if(key == 'f') {
			ofToggleFullscreen();
		}
        if(key == ' ') {
            string path = ofGetTimestampString() + ".jpg";
//            ofSaveImage(cam.getColorPixels(), path);
        }
	}
};

int main() {
    ofGLWindowSettings settings;
    settings.width = 1280;
    settings.height = 800;
    settings.setGLVersion(4, 1);
    ofCreateWindow(settings);
	ofRunApp(new ofApp());
}
<|endoftext|>


<commit_before>// __BEGIN_LICENSE__
// Copyright (C) 2006-2011 United States Government as represented by
// the Administrator of the National Aeronautics and Space Administration.
// All Rights Reserved.
// __END_LICENSE__

/// \file disparitydebug.cc
///

#ifdef _MSC_VER
#pragma warning(disable:4244)
#pragma warning(disable:4267)
#pragma warning(disable:4996)
#endif

#ifdef NDEBUG
#undef NDEBUG
#endif

#include <stdlib.h>

#include <vw/FileIO.h>
#include <vw/Image.h>
#include <vw/Stereo/DisparityMap.h>
#include <vw/tools/Common.h>
#include <asp/Core/Macros.h>
#include <asp/Core/Common.h>
using namespace vw;
using namespace vw::stereo;

namespace po = boost::program_options;
namespace fs = boost::filesystem;

struct Options : asp::BaseOptions {
  // Input
  std::string input_file_name;

  // Output
  std::string output_prefix, output_file_type;
};

void handle_arguments( int argc, char *argv[], Options& opt ) {
  po::options_description general_options("");
  general_options.add_options()
    ("output-prefix,o", po::value(&opt.output_prefix), "Specify the output prefix")
    ("output-filetype,t", po::value(&opt.output_file_type)->default_value("tif"), "Specify the output file");
  general_options.add( asp::BaseOptionsDescription(opt) );

  po::options_description positional("");
  positional.add_options()
    ("input-file", po::value(&opt.input_file_name), "Input disparity map");

  po::positional_options_description positional_desc;
  positional_desc.add("input-file", 1);

  std::ostringstream usage;
  usage << "Usage: " << argv[0] << " [options] <input disparity map> \n";

  po::variables_map vm =
    asp::check_command_line( argc, argv, opt, general_options,
                             positional, positional_desc, usage.str() );

  if ( opt.input_file_name.empty() )
    vw_throw( ArgumentErr() << "Missing input file!\n"
              << usage.str() << general_options );
  if ( opt.output_prefix.empty() )
    opt.output_prefix = fs::path(opt.input_file_name).stem();
}

template <class PixelT>
void do_disparity_visualization(Options& opt) {
  DiskImageView<PixelT > disk_disparity_map(opt.input_file_name);

  vw_out() << "\t--> Computing disparity range \n";
  BBox2 disp_range = get_disparity_range(disk_disparity_map);
  vw_out() << "\t    Horizontal - [" << disp_range.min().x()
           << " " << disp_range.max().x() << "]    Vertical: ["
           << disp_range.min().y() << " " << disp_range.max().y() << "]\n";

  typedef typename PixelChannelType<PixelT>::type ChannelT;
  ImageViewRef<ChannelT> horizontal = apply_mask(copy_mask(clamp(normalize(select_channel(disk_disparity_map,0), disp_range.min().x(), disp_range.max().x())),disk_disparity_map));
  ImageViewRef<ChannelT> vertical = apply_mask(copy_mask(clamp(normalize(select_channel(disk_disparity_map,1), disp_range.min().y(), disp_range.max().y())),disk_disparity_map));

  vw_out() << "\t--> Saving disparity debug images\n";
  block_write_gdal_image( opt.output_prefix+"-H."+opt.output_file_type,
                          channel_cast_rescale<uint8>(horizontal),
                          opt, TerminalProgressCallback("asp","\t    Left  : "));
  block_write_gdal_image( opt.output_prefix + "-V." + opt.output_file_type,
                          channel_cast_rescale<uint8>(vertical),
                          opt, TerminalProgressCallback("asp","\t    Right : "));
}

int main( int argc, char *argv[] ) {

  Options opt;
  try {
    handle_arguments( argc, argv, opt );

    vw_out() << "Opening " << opt.input_file_name << "\n";
    ImageFormat fmt = tools::taste_image(opt.input_file_name);

    switch(fmt.pixel_format) {
    case VW_PIXEL_GENERIC_2_CHANNEL:
      switch (fmt.channel_type) {
      case VW_CHANNEL_INT32:
        do_disparity_visualization<Vector2i>(opt); break;
      default:
        do_disparity_visualization<Vector2f>(opt); break;
      } break;
    case VW_PIXEL_RGB:
    case VW_PIXEL_GENERIC_3_CHANNEL:
      switch (fmt.channel_type) {
      case VW_CHANNEL_INT32:
        do_disparity_visualization<PixelMask<Vector2i> >(opt); break;
      default:
        do_disparity_visualization<PixelMask<Vector2f> >(opt); break;
      } break;
    default:
      vw_throw( ArgumentErr() << "Unsupported pixel format. Expected GENERIC 2 or 3 CHANNEL image." );
    }

  } ASP_STANDARD_CATCHES;

  return 0;
}
<commit_msg>Correct the normalization in disparity debug<commit_after>// __BEGIN_LICENSE__
// Copyright (C) 2006-2011 United States Government as represented by
// the Administrator of the National Aeronautics and Space Administration.
// All Rights Reserved.
// __END_LICENSE__

/// \file disparitydebug.cc
///

#ifdef _MSC_VER
#pragma warning(disable:4244)
#pragma warning(disable:4267)
#pragma warning(disable:4996)
#endif

#ifdef NDEBUG
#undef NDEBUG
#endif

#include <stdlib.h>

#include <vw/FileIO.h>
#include <vw/Image.h>
#include <vw/Stereo/DisparityMap.h>
#include <vw/tools/Common.h>
#include <asp/Core/Macros.h>
#include <asp/Core/Common.h>
using namespace vw;
using namespace vw::stereo;

namespace po = boost::program_options;
namespace fs = boost::filesystem;

struct Options : asp::BaseOptions {
  // Input
  std::string input_file_name;

  // Output
  std::string output_prefix, output_file_type;
};

void handle_arguments( int argc, char *argv[], Options& opt ) {
  po::options_description general_options("");
  general_options.add_options()
    ("output-prefix,o", po::value(&opt.output_prefix), "Specify the output prefix")
    ("output-filetype,t", po::value(&opt.output_file_type)->default_value("tif"), "Specify the output file");
  general_options.add( asp::BaseOptionsDescription(opt) );

  po::options_description positional("");
  positional.add_options()
    ("input-file", po::value(&opt.input_file_name), "Input disparity map");

  po::positional_options_description positional_desc;
  positional_desc.add("input-file", 1);

  std::ostringstream usage;
  usage << "Usage: " << argv[0] << " [options] <input disparity map> \n";

  po::variables_map vm =
    asp::check_command_line( argc, argv, opt, general_options,
                             positional, positional_desc, usage.str() );

  if ( opt.input_file_name.empty() )
    vw_throw( ArgumentErr() << "Missing input file!\n"
              << usage.str() << general_options );
  if ( opt.output_prefix.empty() )
    opt.output_prefix = fs::path(opt.input_file_name).stem();
}

template <class PixelT>
void do_disparity_visualization(Options& opt) {
  DiskImageView<PixelT > disk_disparity_map(opt.input_file_name);

  vw_out() << "\t--> Computing disparity range \n";
  BBox2 disp_range = get_disparity_range(disk_disparity_map);
  vw_out() << "\t    Horizontal - [" << disp_range.min().x()
           << " " << disp_range.max().x() << "]    Vertical: ["
           << disp_range.min().y() << " " << disp_range.max().y() << "]\n";

  typedef typename PixelChannelType<PixelT>::type ChannelT;
  ImageViewRef<ChannelT> horizontal =
    apply_mask(copy_mask(clamp(normalize(select_channel(disk_disparity_map,0),
                                         disp_range.min().x(), disp_range.max().x(),
                                         ChannelRange<ChannelT>::min(),ChannelRange<ChannelT>::max())),
                         disk_disparity_map));
  ImageViewRef<ChannelT> vertical =
    apply_mask(copy_mask(clamp(normalize(select_channel(disk_disparity_map,1),
                                         disp_range.min().y(), disp_range.max().y(),
                                         ChannelRange<ChannelT>::min(),ChannelRange<ChannelT>::max())),
                         disk_disparity_map));

  vw_out() << "\t--> Saving disparity debug images\n";
  block_write_gdal_image( opt.output_prefix+"-H."+opt.output_file_type,
                          channel_cast_rescale<uint8>(horizontal),
                          opt, TerminalProgressCallback("asp","\t    Left  : "));
  block_write_gdal_image( opt.output_prefix + "-V." + opt.output_file_type,
                          channel_cast_rescale<uint8>(vertical),
                          opt, TerminalProgressCallback("asp","\t    Right : "));
}

int main( int argc, char *argv[] ) {

  Options opt;
  try {
    handle_arguments( argc, argv, opt );

    vw_out() << "Opening " << opt.input_file_name << "\n";
    ImageFormat fmt = tools::taste_image(opt.input_file_name);

    switch(fmt.pixel_format) {
    case VW_PIXEL_GENERIC_2_CHANNEL:
      switch (fmt.channel_type) {
      case VW_CHANNEL_INT32:
        do_disparity_visualization<Vector2i>(opt); break;
      default:
        do_disparity_visualization<Vector2f>(opt); break;
      } break;
    case VW_PIXEL_RGB:
    case VW_PIXEL_GENERIC_3_CHANNEL:
      switch (fmt.channel_type) {
      case VW_CHANNEL_INT32:
        do_disparity_visualization<PixelMask<Vector2i> >(opt); break;
      default:
        do_disparity_visualization<PixelMask<Vector2f> >(opt); break;
      } break;
    default:
      vw_throw( ArgumentErr() << "Unsupported pixel format. Expected GENERIC 2 or 3 CHANNEL image." );
    }

  } ASP_STANDARD_CATCHES;

  return 0;
}
<|endoftext|>