<commit_before>#include <polarssl/error.h>
#include <polarssl/entropy.h>
#include <polarssl/ctr_drbg.h>

#include "random.h"

void PolarSSL::Random::Init(v8::Handle<v8::Object> target) {
  target->Set(v8::String::NewSymbol("randomBytes"), v8::FunctionTemplate::New(RandomBytes)->GetFunction());
}

NAN_METHOD(PolarSSL::Random::RandomBytes) {
  NanScope();

  char err[1024];
  int rc;

  ctr_drbg_context ctr_drbg;
  entropy_context entropy;

  entropy_init(&entropy);

  rc = ctr_drbg_init(&ctr_drbg, entropy_func, &entropy, (const unsigned char *)"RANDOM_GEN", 10);
  if (rc != 0) {
    polarssl_strerror(rc, err, sizeof(err));
    NanThrowError(err);
    NanReturnUndefined();
  }
  ctr_drbg_set_prediction_resistance(&ctr_drbg, CTR_DRBG_PR_OFF);

  int bytes = args[0]->Uint32Value();

  unsigned char* data = new unsigned char[bytes];

  rc = ctr_drbg_random(&ctr_drbg, data, bytes);
  if (rc != 0) {
    polarssl_strerror(rc, err, sizeof(err));
    NanThrowError(err);
    NanReturnUndefined();
  }

  NanReturnValue(NanNewBufferHandle(reinterpret_cast<char*>(data), bytes));
}
<commit_msg>only pull 768 bytes from random data generator at a time<commit_after>#include <polarssl/error.h>
#include <polarssl/entropy.h>
#include <polarssl/ctr_drbg.h>

#include "random.h"

void PolarSSL::Random::Init(v8::Handle<v8::Object> target) {
  target->Set(v8::String::NewSymbol("randomBytes"), v8::FunctionTemplate::New(RandomBytes)->GetFunction());
}

NAN_METHOD(PolarSSL::Random::RandomBytes) {
  NanScope();

  char err[1024];
  int rc;

  ctr_drbg_context ctr_drbg;
  entropy_context entropy;

  entropy_init(&entropy);

  rc = ctr_drbg_init(&ctr_drbg, entropy_func, &entropy, (const unsigned char *)"RANDOM_GEN", 10);
  if (rc != 0) {
    polarssl_strerror(rc, err, sizeof(err));
    NanThrowError(err);
    NanReturnUndefined();
  }
  ctr_drbg_set_prediction_resistance(&ctr_drbg, CTR_DRBG_PR_OFF);

  int bytes = args[0]->Uint32Value();

  unsigned char* data = new unsigned char[bytes];

  int chunkSize = 768;

  for (int i=0;i<bytes;i+=chunkSize) {
    rc = ctr_drbg_random(&ctr_drbg, data + i, (chunkSize < (bytes - i)) ? chunkSize : (bytes - i));

    if (rc != 0) {
      polarssl_strerror(rc, err, sizeof(err));
      NanThrowError(err);
      NanReturnUndefined();
    }
  }

  NanReturnValue(NanNewBufferHandle(reinterpret_cast<char*>(data), bytes));
}
<|endoftext|>


<commit_before>/*
* 2013+ Copyright (c) Andrey Kashin <kashin.andrej@gmail.com>
* All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*/

#define REACT_CPP

#include "react/react.hpp"

#include <stdexcept>
#include <iostream>
#include <mutex>

using namespace react;

static actions_set_t *actions_set = new actions_set_t();

int react_define_new_action(const char *action_name) {
	try {
		return actions_set->define_new_action(action_name);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -ENOMEM;
	}
}

struct react_context_t {
	react_context_t(react::aggregator_t *aggregator):
		call_tree(*actions_set), updater(call_tree), aggregator(aggregator) {}

	concurrent_call_tree_t call_tree;
	call_tree_updater_t updater;
	react::aggregator_t *aggregator;
};

static __thread react_context_t *thread_react_context = NULL;

int react_is_active() {
	return thread_react_context != NULL;
}

react_context_t *react_activate(void *react_aggregator) {
	try {
		if (react_is_active()) {
			std::string error_message("Can't activate react: React is already active");
			throw std::runtime_error(error_message);
		}

		thread_react_context = new react_context_t(
					static_cast<react::aggregator_t*>(react_aggregator)
		);

		react::add_stat("complete", false);

		return thread_react_context;
	} catch (std::exception &e) {
		std::cerr << e.what() << std::endl;
		return NULL;
	}
}

int react_deactivate(react_context_t *react_context) {
	try {
		if (!react_is_active()) {
			std::string error_message("Can't deactivate react: React is not active");
			throw std::runtime_error(error_message);
		}

		if (react_context != thread_react_context) {
			std::string error_message("Can't deactivate react: passed context doesn't match \
									  thread_local context");
			throw std::invalid_argument(error_message);
		}

		react::add_stat("complete", true);

		if (thread_react_context->aggregator) {
			thread_react_context->aggregator->aggregate(thread_react_context->call_tree.get_call_tree());
		}
		delete thread_react_context;
		thread_react_context = NULL;
	} catch (std::exception &e) {
		std::cerr << e.what() << std::endl;
		return -EFAULT;
	}
	return 0;
}

int react_start_action(int action_code) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		thread_react_context->updater.start(action_code);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_stop_action(int action_code) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		thread_react_context->updater.stop(action_code);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_add_stat_bool(const char *key, bool value) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		react::add_stat(std::string(key), value);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_add_stat_int(const char *key, int value) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		react::add_stat(std::string(key), value);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_add_stat_double(const char *key, double value) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		react::add_stat(std::string(key), value);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_add_stat_string(const char *key, const char *value) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		react::add_stat(std::string(key), value);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_submit_progress() {
	try {
		if (!react_is_active()) {
			return 0;
		}

		if (thread_react_context->aggregator) {
			thread_react_context->aggregator->aggregate(thread_react_context->call_tree.get_call_tree());
		}
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

namespace react {

action_guard::action_guard(int action_code) {
	if (react_is_active()) {
		m_action_guard.reset(
					new action_guard_t(&thread_react_context->updater, action_code)
		);
	}
}

action_guard::~action_guard() {}

void react::action_guard::stop() {
	if (m_action_guard) {
		m_action_guard->stop();
	}
}

const actions_set_t &get_actions_set() {
	return *actions_set;
}

call_tree_t get_react_context_call_tree(react_context_t *react_context) {
	if (!react_context) {
		std::string error_message("Can't get react context call tree: passed context is null");
		throw std::invalid_argument(error_message);
	}

	return react_context->call_tree.copy_call_tree();
}

void add_stat(const std::string &key, const char *value) {
	add_stat_impl(key, react::stat_value_t(std::string(value)));
}

void add_stat_impl(const std::string &key, const react::stat_value_t &value) {
	thread_react_context->call_tree.get_call_tree().add_stat(key, value);
}

} // namespace react
<commit_msg>api: add_stat functions are now generated using define<commit_after>/*
* 2013+ Copyright (c) Andrey Kashin <kashin.andrej@gmail.com>
* All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*/

#define REACT_CPP

#include "react/react.hpp"

#include <stdexcept>
#include <iostream>
#include <mutex>

using namespace react;

static actions_set_t *actions_set = new actions_set_t();

int react_define_new_action(const char *action_name) {
	try {
		return actions_set->define_new_action(action_name);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -ENOMEM;
	}
}

struct react_context_t {
	react_context_t(react::aggregator_t *aggregator):
		call_tree(*actions_set), updater(call_tree), aggregator(aggregator) {}

	concurrent_call_tree_t call_tree;
	call_tree_updater_t updater;
	react::aggregator_t *aggregator;
};

static __thread react_context_t *thread_react_context = NULL;

int react_is_active() {
	return thread_react_context != NULL;
}

react_context_t *react_activate(void *react_aggregator) {
	try {
		if (react_is_active()) {
			std::string error_message("Can't activate react: React is already active");
			throw std::runtime_error(error_message);
		}

		thread_react_context = new react_context_t(
					static_cast<react::aggregator_t*>(react_aggregator)
		);

		react::add_stat("complete", false);

		return thread_react_context;
	} catch (std::exception &e) {
		std::cerr << e.what() << std::endl;
		return NULL;
	}
}

int react_deactivate(react_context_t *react_context) {
	try {
		if (!react_is_active()) {
			std::string error_message("Can't deactivate react: React is not active");
			throw std::runtime_error(error_message);
		}

		if (react_context != thread_react_context) {
			std::string error_message("Can't deactivate react: passed context doesn't match \
									  thread_local context");
			throw std::invalid_argument(error_message);
		}

		react::add_stat("complete", true);

		if (thread_react_context->aggregator) {
			thread_react_context->aggregator->aggregate(thread_react_context->call_tree.get_call_tree());
		}
		delete thread_react_context;
		thread_react_context = NULL;
	} catch (std::exception &e) {
		std::cerr << e.what() << std::endl;
		return -EFAULT;
	}
	return 0;
}

int react_start_action(int action_code) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		thread_react_context->updater.start(action_code);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

int react_stop_action(int action_code) {
	try {
		if (!react_is_active()) {
			return 0;
		}

		thread_react_context->updater.stop(action_code);
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

#define DEFINE_STAT_TYPE(name, type)                     \
int react_add_stat_##name(const char *key, type value) { \
	try {                                                \
		if (!react_is_active()) {                        \
			return 0;                                    \
		}                                                \
		react::add_stat(std::string(key), value);        \
	} catch (std::exception& e) {                        \
		std::cerr << e.what() << std::endl;              \
		return -EINVAL;                                  \
	}                                                    \
	return 0;                                            \
}

DEFINE_STAT_TYPE(bool,   bool)
DEFINE_STAT_TYPE(int,    int)
DEFINE_STAT_TYPE(double, double)
DEFINE_STAT_TYPE(string, const char *)

int react_submit_progress() {
	try {
		if (!react_is_active()) {
			return 0;
		}

		if (thread_react_context->aggregator) {
			thread_react_context->aggregator->aggregate(thread_react_context->call_tree.get_call_tree());
		}
	} catch (std::exception& e) {
		std::cerr << e.what() << std::endl;
		return -EINVAL;
	}
	return 0;
}

namespace react {

action_guard::action_guard(int action_code) {
	if (react_is_active()) {
		m_action_guard.reset(
					new action_guard_t(&thread_react_context->updater, action_code)
		);
	}
}

action_guard::~action_guard() {}

void react::action_guard::stop() {
	if (m_action_guard) {
		m_action_guard->stop();
	}
}

const actions_set_t &get_actions_set() {
	return *actions_set;
}

call_tree_t get_react_context_call_tree(react_context_t *react_context) {
	if (!react_context) {
		std::string error_message("Can't get react context call tree: passed context is null");
		throw std::invalid_argument(error_message);
	}

	return react_context->call_tree.copy_call_tree();
}

void add_stat(const std::string &key, const char *value) {
	add_stat_impl(key, react::stat_value_t(std::string(value)));
}

void add_stat_impl(const std::string &key, const react::stat_value_t &value) {
	thread_react_context->call_tree.get_call_tree().add_stat(key, value);
}

} // namespace react
<|endoftext|>


<commit_before>#ifndef _SCENE_HPP_
#define _SCENE_HPP_

#include "object.hpp"

#include <vector>

#include <glm/glm.hpp>

struct Scene
{
  inline Scene &operator <<(const Object *object);

  inline void draw(const glm::mat4 &mvp) const;

private:
  std::vector<const Object*> _objects;
};

Scene &Scene::operator <<(const Object *object)
{
  _objects.push_back(object);

  return *this;
}

void Scene::draw(const glm::mat4 &mvp) const
{
  for (auto object : _objects)
    object->draw(mvp);
}

#endif // _SCENE_HPP_
<commit_msg>Add sun to scene<commit_after>#ifndef _SCENE_HPP_
#define _SCENE_HPP_

#include "object.hpp"
#include "light/sun.hpp"

#include <vector>

#include <glm/glm.hpp>

struct Scene
{
  inline Scene &operator<<(const Object *object);
  inline Scene &operator<<(const Lights::Sun *sun);

  inline const Lights::Sun *sun() const;

  inline void draw(const glm::mat4 &mvp) const;

private:
  std::vector<const Object*> _objects;
  const Lights::Sun *_sun = nullptr;
};

Scene &Scene::operator<<(const Object *object)
{
  _objects.push_back(object);
  return *this;
}

Scene &Scene::operator<<(const Lights::Sun *sun)
{
  _sun = sun;
  return *this;
}

const Lights::Sun *Scene::sun() const
{
  return _sun;
};

void Scene::draw(const glm::mat4 &mvp) const
{
  for (auto object : _objects)
    object->draw(mvp);
}

#endif // _SCENE_HPP_
<|endoftext|>


<commit_before>/*
	This file is part of cpp-ethereum.

	cpp-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	cpp-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file Ethasher.cpp
 * @author Gav Wood <i@gavwood.com>
 * @date 2014
 */

#include <boost/detail/endian.hpp>
#include <boost/filesystem.hpp>
#include <chrono>
#include <array>
#include <random>
#include <thread>
#include <libdevcore/Common.h>
#include <libdevcore/Guards.h>
#include <libdevcore/Log.h>
#include <libdevcrypto/CryptoPP.h>
#include <libdevcrypto/SHA3.h>
#include <libdevcrypto/FileSystem.h>
#include <libethcore/Params.h>
#include "BlockInfo.h"
#include "Ethasher.h"
using namespace std;
using namespace chrono;
using namespace dev;
using namespace eth;

Ethasher* dev::eth::Ethasher::s_this = nullptr;

bytes const& Ethasher::cache(BlockInfo const& _header)
{
	RecursiveGuard l(x_this);
	if (_header.number > c_ethashEpochLength * 2048)
	{
		std::ostringstream error;
		error << "block number is too high; max is " << c_ethashEpochLength * 2048 << "(was " << _header.number << ")";
		throw std::invalid_argument( error.str() );
 	}

	if (!m_caches.count(_header.seedHash()))
	{
		ethash_params p = params((unsigned)_header.number);
		m_caches[_header.seedHash()].resize(p.cache_size);
		ethash_prep_light(m_caches[_header.seedHash()].data(), &p, _header.seedHash().data());
	}
	return m_caches[_header.seedHash()];
}

bytesConstRef Ethasher::full(BlockInfo const& _header)
{
	RecursiveGuard l(x_this);
	if (!m_fulls.count(_header.seedHash()))
	{
		if (!m_fulls.empty())
		{
			delete [] m_fulls.begin()->second.data();
			m_fulls.erase(m_fulls.begin());
		}
		try {
			boost::filesystem::create_directories(getDataDir("ethash"));
		} catch (...) {}

		std::string memoFile = getDataDir("ethash") + "/full";
		auto info = rlpList(c_ethashRevision, _header.seedHash());
		if (boost::filesystem::exists(memoFile) && contents(memoFile + ".info") != info)
			boost::filesystem::remove(memoFile);
		m_fulls[_header.seedHash()] = contentsNew(memoFile);
		if (!m_fulls[_header.seedHash()])
		{
			ethash_params p = params((unsigned)_header.number);
			m_fulls[_header.seedHash()] = bytesRef(new byte[p.full_size], p.full_size);
			auto c = cache(_header);
			ethash_prep_full(m_fulls[_header.seedHash()].data(), &p, c.data());
			writeFile(memoFile, m_fulls[_header.seedHash()]);
			writeFile(memoFile + ".info", info);
		}
	}
	return m_fulls[_header.seedHash()];
}

ethash_params Ethasher::params(BlockInfo const& _header)
{
	return params((unsigned)_header.number);
}

ethash_params Ethasher::params(unsigned _n)
{
	ethash_params p;
	p.cache_size = ethash_get_cachesize(_n);
	p.full_size = ethash_get_datasize(_n);
	return p;
}

bool Ethasher::verify(BlockInfo const& _header)
{
	if (_header.number >= c_ethashEpochLength * 2048)
		return false;

	h256 boundary = u256((bigint(1) << 256) / _header.difficulty);

	bool ret = ethash_quick_check_difficulty(
		_header.headerHash(WithoutNonce).data(),
		(uint64_t)(u64)_header.nonce,
		_header.mixHash.data(),
		boundary.data());

#if ETH_DEBUG
	// should be equivalent to:
	auto result = eval(_header);
	assert((result.mixHash == _header.mixHash && result.value <= boundary) == ret);
#endif

	return ret;
}

Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce)
{
	auto p = Ethasher::params(_header);
	ethash_return_value r;
	ethash_compute_light(&r, Ethasher::get()->cache(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce);
//	cdebug << "Ethasher::eval sha3(cache):" << sha3(Ethasher::get()->cache(_header)) << "hh:" << _header.headerHash(WithoutNonce) << "nonce:" << _nonce << " => " << h256(r.result, h256::ConstructFromPointer);
	return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)};
}
<commit_msg>Extra debug information for #1438.<commit_after>/*
	This file is part of cpp-ethereum.

	cpp-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	cpp-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file Ethasher.cpp
 * @author Gav Wood <i@gavwood.com>
 * @date 2014
 */

#include <boost/detail/endian.hpp>
#include <boost/filesystem.hpp>
#include <chrono>
#include <array>
#include <random>
#include <thread>
#include <libdevcore/Common.h>
#include <libdevcore/Guards.h>
#include <libdevcore/Log.h>
#include <libdevcrypto/CryptoPP.h>
#include <libdevcrypto/SHA3.h>
#include <libdevcrypto/FileSystem.h>
#include <libethcore/Params.h>
#include "BlockInfo.h"
#include "Ethasher.h"
using namespace std;
using namespace chrono;
using namespace dev;
using namespace eth;

Ethasher* dev::eth::Ethasher::s_this = nullptr;

bytes const& Ethasher::cache(BlockInfo const& _header)
{
	RecursiveGuard l(x_this);
	if (_header.number > c_ethashEpochLength * 2048)
	{
		std::ostringstream error;
		error << "block number is too high; max is " << c_ethashEpochLength * 2048 << "(was " << _header.number << ")";
		throw std::invalid_argument( error.str() );
 	}

	if (!m_caches.count(_header.seedHash()))
	{
		ethash_params p = params((unsigned)_header.number);
		m_caches[_header.seedHash()].resize(p.cache_size);
		ethash_prep_light(m_caches[_header.seedHash()].data(), &p, _header.seedHash().data());
	}
	return m_caches[_header.seedHash()];
}

bytesConstRef Ethasher::full(BlockInfo const& _header)
{
	RecursiveGuard l(x_this);
	if (!m_fulls.count(_header.seedHash()))
	{
		if (!m_fulls.empty())
		{
			delete [] m_fulls.begin()->second.data();
			m_fulls.erase(m_fulls.begin());
		}
		try {
			boost::filesystem::create_directories(getDataDir("ethash"));
		} catch (...) {}

		std::string memoFile = getDataDir("ethash") + "/full";
		auto info = rlpList(c_ethashRevision, _header.seedHash());
		if (boost::filesystem::exists(memoFile) && contents(memoFile + ".info") != info)
			boost::filesystem::remove(memoFile);
		m_fulls[_header.seedHash()] = contentsNew(memoFile);
		if (!m_fulls[_header.seedHash()])
		{
			ethash_params p = params((unsigned)_header.number);
			m_fulls[_header.seedHash()] = bytesRef(new byte[p.full_size], p.full_size);
			auto c = cache(_header);
			ethash_prep_full(m_fulls[_header.seedHash()].data(), &p, c.data());
			writeFile(memoFile, m_fulls[_header.seedHash()]);
			writeFile(memoFile + ".info", info);
		}
	}
	return m_fulls[_header.seedHash()];
}

ethash_params Ethasher::params(BlockInfo const& _header)
{
	return params((unsigned)_header.number);
}

ethash_params Ethasher::params(unsigned _n)
{
	ethash_params p;
	p.cache_size = ethash_get_cachesize(_n);
	p.full_size = ethash_get_datasize(_n);
	return p;
}

bool Ethasher::verify(BlockInfo const& _header)
{
	if (_header.number >= c_ethashEpochLength * 2048)
		return false;

	h256 boundary = u256((bigint(1) << 256) / _header.difficulty);

	bool ret = ethash_quick_check_difficulty(
		_header.headerHash(WithoutNonce).data(),
		(uint64_t)(u64)_header.nonce,
		_header.mixHash.data(),
		boundary.data());

#if ETH_DEBUG
	auto result = eval(_header);
	if ((result.value <= boundary && result.mixHash == _header.mixHash) != ret)
	{
		cwarn << "Assertion failure coming: evaluated result gives different outcome to ethash_quick_check_difficulty";
		cwarn << "headerHash:" << _header.headerHash(WithoutNonce);
		cwarn << "nonce:" << _header.nonce;
		cwarn << "mixHash:" << _header.mixHash;
		cwarn << "difficulty:" << _header.difficulty;
		cwarn << "boundary:" << boundary;
		cwarn << "result.value:" << result.value;
		cwarn << "result.mixHash:" << result.mixHash;
	}
	assert((result.value <= boundary) == ret);
	if (result.value <= boundary)
		assert(result.mixHash == _header.mixHash);
#endif

	return ret;
}

Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce)
{
	auto p = Ethasher::params(_header);
	ethash_return_value r;
	ethash_compute_light(&r, Ethasher::get()->cache(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce);
//	cdebug << "Ethasher::eval sha3(cache):" << sha3(Ethasher::get()->cache(_header)) << "hh:" << _header.headerHash(WithoutNonce) << "nonce:" << _nonce << " => " << h256(r.result, h256::ConstructFromPointer);
	return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)};
}
<|endoftext|>


<commit_before>#ifndef PONG_DEFS_HPP_
#define PONG_DEFS_HPP_

#include <string>

namespace pong {

constexpr const wchar_t* WINDOW_TITLE = L"Pong";
constexpr const int WINDOW_WIDTH = 640;
constexpr const int WINDOW_HEIGHT = 480;
constexpr const float WINDOW_HALF_WIDTH = WINDOW_WIDTH / 2.0f;
constexpr const float WINDOW_HALF_HEIGHT = WINDOW_HEIGHT / 2.0f;

enum Player {
    PLAYER_1, PLAYER_2,
    PLAYER_COUNT
};

constexpr const int PIXELS_PER_METER = 64;

constexpr const int MATCH_POINT = 10;

constexpr const int GAME_FRAMES_PER_SECOND = 60;
constexpr const float GAME_TIME_STEP = 1.0f / GAME_FRAMES_PER_SECOND;
constexpr const int GAME_VELOCITY_ITERATIONS = 8;
constexpr const int GAME_POSITION_ITERATIONS = 3;

constexpr const float WALL_WIDTH = WINDOW_WIDTH;
constexpr const float WALL_HALF_WIDTH = WALL_WIDTH / 2.0f;
constexpr const float WALL_HEIGHT = 10.0f;
constexpr const float WALL_HALF_HEIGHT = WALL_HEIGHT / 2.0f;

constexpr const float BALL_WIDTH = 10.0f;
constexpr const float BALL_HALF_WIDTH = BALL_WIDTH / 2.0f;
constexpr const float BALL_HEIGHT = 10.0f;
constexpr const float BALL_HALF_HEIGHT = BALL_HEIGHT / 2.0f;
constexpr const float BALL_MIN_SPEED = 2.0f;
constexpr const float BALL_MAX_SPEED = 15.0f;
constexpr const float BALL_VELOCITY_STEP = 0.001;
constexpr const float BALL_MIN_ROTATION_SPEED = 0.1f;
constexpr const float BALL_MAX_ROTATION_SPEED = 15.0f;

constexpr const float RAQUET_MARGIN = 15.0f;
constexpr const float RAQUET_WIDTH = 20.0f;
constexpr const float RAQUET_HALF_WIDTH = RAQUET_WIDTH / 2.0f;
constexpr const float RAQUET_HEIGHT = 80.0f;
constexpr const float RAQUET_HALF_HEIGHT = RAQUET_HEIGHT / 2.0f;
constexpr const float RAQUET_BASE_SPEED = 5.0f;

const std::string ASSETS_PATH = "assets/";
constexpr const int DEFAULT_FONT_SIZE = 54;

} /* namespace pong */
#endif /* PONG_DEFS_HPP_ */
<commit_msg>Adjust ball speed<commit_after>#ifndef PONG_DEFS_HPP_
#define PONG_DEFS_HPP_

#include <string>

namespace pong {

constexpr const wchar_t* WINDOW_TITLE = L"Pong";
constexpr const int WINDOW_WIDTH = 640;
constexpr const int WINDOW_HEIGHT = 480;
constexpr const float WINDOW_HALF_WIDTH = WINDOW_WIDTH / 2.0f;
constexpr const float WINDOW_HALF_HEIGHT = WINDOW_HEIGHT / 2.0f;

enum Player {
    PLAYER_1, PLAYER_2,
    PLAYER_COUNT
};

constexpr const int PIXELS_PER_METER = 64;

constexpr const int MATCH_POINT = 10;

constexpr const int GAME_FRAMES_PER_SECOND = 60;
constexpr const float GAME_TIME_STEP = 1.0f / GAME_FRAMES_PER_SECOND;
constexpr const int GAME_VELOCITY_ITERATIONS = 8;
constexpr const int GAME_POSITION_ITERATIONS = 3;

constexpr const float WALL_WIDTH = WINDOW_WIDTH;
constexpr const float WALL_HALF_WIDTH = WALL_WIDTH / 2.0f;
constexpr const float WALL_HEIGHT = 10.0f;
constexpr const float WALL_HALF_HEIGHT = WALL_HEIGHT / 2.0f;

constexpr const float BALL_WIDTH = 10.0f;
constexpr const float BALL_HALF_WIDTH = BALL_WIDTH / 2.0f;
constexpr const float BALL_HEIGHT = 10.0f;
constexpr const float BALL_HALF_HEIGHT = BALL_HEIGHT / 2.0f;
constexpr const float BALL_MIN_SPEED = 2.0f;
constexpr const float BALL_MAX_SPEED = 15.0f;
constexpr const float BALL_VELOCITY_STEP = 0.003;
constexpr const float BALL_MIN_ROTATION_SPEED = 0.1f;
constexpr const float BALL_MAX_ROTATION_SPEED = 10.0f;

constexpr const float RAQUET_MARGIN = 15.0f;
constexpr const float RAQUET_WIDTH = 20.0f;
constexpr const float RAQUET_HALF_WIDTH = RAQUET_WIDTH / 2.0f;
constexpr const float RAQUET_HEIGHT = 80.0f;
constexpr const float RAQUET_HALF_HEIGHT = RAQUET_HEIGHT / 2.0f;
constexpr const float RAQUET_BASE_SPEED = 5.0f;

const std::string ASSETS_PATH = "assets/";
constexpr const int DEFAULT_FONT_SIZE = 54;

} /* namespace pong */
#endif /* PONG_DEFS_HPP_ */
<|endoftext|>


<commit_before>#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include "base64.h"
#include "util.h"
#include "Profiling.h"

namespace i2p
{
namespace data
{
	RouterProfile::RouterProfile (const IdentHash& identHash):
		m_IdentHash (identHash), m_LastUpdateTime (boost::posix_time::second_clock::local_time()),
		m_NumTunnelsAgreed (0), m_NumTunnelsDeclined (0),
		m_NumTunnelsNonReplied (0)
	{
	}

	boost::posix_time::ptime RouterProfile::GetTime () const
	{
		return boost::posix_time::second_clock::local_time();
	}

	void RouterProfile::UpdateTime ()
	{
		m_LastUpdateTime = GetTime ();
	}

	void RouterProfile::Save ()
	{
		// fill sections
		boost::property_tree::ptree participation;
		participation.put (PEER_PROFILE_PARTICIPATION_AGREED, m_NumTunnelsAgreed);
		participation.put (PEER_PROFILE_PARTICIPATION_DECLINED, m_NumTunnelsDeclined);
		participation.put (PEER_PROFILE_PARTICIPATION_NON_REPLIED, m_NumTunnelsNonReplied);
		// fill property tree
		boost::property_tree::ptree pt;
		pt.put (PEER_PROFILE_LAST_UPDATE_TIME, boost::posix_time::to_simple_string (m_LastUpdateTime));
		pt.put_child (PEER_PROFILE_SECTION_PARTICIPATION, participation);

		// save to file
		auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY;
		if (!boost::filesystem::exists (path))
		{
			// Create directory is necessary
			if (!boost::filesystem::create_directory (path))
			{
				LogPrint (eLogError, "Failed to create directory ", path);
				return;
			}
			const char * chars = GetBase64SubstitutionTable (); // 64 bytes
			for (int i = 0; i < 64; i++)
			{
				auto path1 = path / (std::string ("p") + chars[i]);
				if (!boost::filesystem::create_directory (path1))
				{
					LogPrint (eLogError, "Failed to create directory ", path1);
					return;
				}
			}
		}
		std::string base64 = m_IdentHash.ToBase64 ();
		path = path / (std::string ("p") + base64[0]);
		auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt");
		try
		{
			boost::property_tree::write_ini (filename.string (), pt);
		}
		catch (std::exception& ex)
		{
			LogPrint (eLogError, "Can't write ", filename, ": ", ex.what ());
		}
	}

	void RouterProfile::Load ()
	{
		std::string base64 = m_IdentHash.ToBase64 ();
		auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY;
		path /= std::string ("p") + base64[0];
		auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt");
		if (boost::filesystem::exists (filename))
		{
			boost::property_tree::ptree pt;
			try
			{
				boost::property_tree::read_ini (filename.string (), pt);
			}
			catch (std::exception& ex)
			{
				LogPrint (eLogError, "Can't read ", filename, ": ", ex.what ());
				return;
			}
			try
			{
				auto t = pt.get (PEER_PROFILE_LAST_UPDATE_TIME, "");
				if (t.length () > 0)
					m_LastUpdateTime = boost::posix_time::time_from_string (t);
				if ((GetTime () - m_LastUpdateTime).hours () < PEER_PROFILE_EXPIRATION_TIMEOUT)
				{
					// read participations
					auto participations = pt.get_child (PEER_PROFILE_SECTION_PARTICIPATION);
					m_NumTunnelsAgreed = participations.get (PEER_PROFILE_PARTICIPATION_AGREED, 0);
					m_NumTunnelsDeclined = participations.get (PEER_PROFILE_PARTICIPATION_DECLINED, 0);
					m_NumTunnelsNonReplied = participations.get (PEER_PROFILE_PARTICIPATION_NON_REPLIED, 0);
				}
				else
					*this = RouterProfile (m_IdentHash);
			}
			catch (std::exception& ex)
			{
				LogPrint (eLogError, "Can't read profile ", base64, " :", ex.what ());
			}
		}
	}

	void RouterProfile::TunnelBuildResponse (uint8_t ret)
	{
		if (ret > 0)
			m_NumTunnelsDeclined++;
		else
			m_NumTunnelsAgreed++;
		UpdateTime ();
	}

	void RouterProfile::TunnelNonReplied ()
	{
		m_NumTunnelsNonReplied++;
		UpdateTime ();
	}

	bool RouterProfile::IsLowPartcipationRate (int elapsedTime) const
	{
		if (elapsedTime < 900) // if less than 15 minutes
			return m_NumTunnelsAgreed < m_NumTunnelsDeclined; // 50% rate
		else
			return 3*m_NumTunnelsAgreed < m_NumTunnelsDeclined; // 25% rate
	}

	bool RouterProfile::IsLowReplyRate (int elapsedTime) const
	{
		auto total = m_NumTunnelsAgreed + m_NumTunnelsDeclined;
		if (elapsedTime < 300) // if less than 5 minutes
			return m_NumTunnelsNonReplied > 10*total;
		else
			return !total && m_NumTunnelsNonReplied > 20;
	}

	bool RouterProfile::IsBad () const
	{
		auto elapsedTime = (GetTime () - m_LastUpdateTime).total_seconds ();
		return IsAlwaysDeclining () || IsLowPartcipationRate (elapsedTime) || IsLowReplyRate (elapsedTime);
	}

	std::shared_ptr<RouterProfile> GetRouterProfile (const IdentHash& identHash)
	{
		auto profile = std::make_shared<RouterProfile> (identHash);
		profile->Load (); // if possible
		return profile;
	}

	void DeleteObsoleteProfiles ()
	{
		int num = 0;
		auto ts = boost::posix_time::second_clock::local_time();
		boost::filesystem::path p (i2p::util::filesystem::GetDataDir()/PEER_PROFILES_DIRECTORY);
		if (boost::filesystem::exists (p))
		{
			boost::filesystem::directory_iterator end;
			for (boost::filesystem::directory_iterator it (p); it != end; ++it)
			{
				if (boost::filesystem::is_directory (it->status()))
				{
					for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1)
					{
						auto lastModified = boost::posix_time::from_time_t (boost::filesystem::last_write_time (it1->path ()));
						if ((ts - lastModified).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT)
						{
							boost::filesystem::remove (it1->path ());
							num++;
						}
					}
				}
			}
		}
		LogPrint (eLogInfo, num, " obsolete profiles deleted");
	}
}
}	<commit_msg>changed some profiling parameters<commit_after>#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include "base64.h"
#include "util.h"
#include "Profiling.h"

namespace i2p
{
namespace data
{
	RouterProfile::RouterProfile (const IdentHash& identHash):
		m_IdentHash (identHash), m_LastUpdateTime (boost::posix_time::second_clock::local_time()),
		m_NumTunnelsAgreed (0), m_NumTunnelsDeclined (0),
		m_NumTunnelsNonReplied (0)
	{
	}

	boost::posix_time::ptime RouterProfile::GetTime () const
	{
		return boost::posix_time::second_clock::local_time();
	}

	void RouterProfile::UpdateTime ()
	{
		m_LastUpdateTime = GetTime ();
	}

	void RouterProfile::Save ()
	{
		// fill sections
		boost::property_tree::ptree participation;
		participation.put (PEER_PROFILE_PARTICIPATION_AGREED, m_NumTunnelsAgreed);
		participation.put (PEER_PROFILE_PARTICIPATION_DECLINED, m_NumTunnelsDeclined);
		participation.put (PEER_PROFILE_PARTICIPATION_NON_REPLIED, m_NumTunnelsNonReplied);
		// fill property tree
		boost::property_tree::ptree pt;
		pt.put (PEER_PROFILE_LAST_UPDATE_TIME, boost::posix_time::to_simple_string (m_LastUpdateTime));
		pt.put_child (PEER_PROFILE_SECTION_PARTICIPATION, participation);

		// save to file
		auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY;
		if (!boost::filesystem::exists (path))
		{
			// Create directory is necessary
			if (!boost::filesystem::create_directory (path))
			{
				LogPrint (eLogError, "Failed to create directory ", path);
				return;
			}
			const char * chars = GetBase64SubstitutionTable (); // 64 bytes
			for (int i = 0; i < 64; i++)
			{
				auto path1 = path / (std::string ("p") + chars[i]);
				if (!boost::filesystem::create_directory (path1))
				{
					LogPrint (eLogError, "Failed to create directory ", path1);
					return;
				}
			}
		}
		std::string base64 = m_IdentHash.ToBase64 ();
		path = path / (std::string ("p") + base64[0]);
		auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt");
		try
		{
			boost::property_tree::write_ini (filename.string (), pt);
		}
		catch (std::exception& ex)
		{
			LogPrint (eLogError, "Can't write ", filename, ": ", ex.what ());
		}
	}

	void RouterProfile::Load ()
	{
		std::string base64 = m_IdentHash.ToBase64 ();
		auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY;
		path /= std::string ("p") + base64[0];
		auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt");
		if (boost::filesystem::exists (filename))
		{
			boost::property_tree::ptree pt;
			try
			{
				boost::property_tree::read_ini (filename.string (), pt);
			}
			catch (std::exception& ex)
			{
				LogPrint (eLogError, "Can't read ", filename, ": ", ex.what ());
				return;
			}
			try
			{
				auto t = pt.get (PEER_PROFILE_LAST_UPDATE_TIME, "");
				if (t.length () > 0)
					m_LastUpdateTime = boost::posix_time::time_from_string (t);
				if ((GetTime () - m_LastUpdateTime).hours () < PEER_PROFILE_EXPIRATION_TIMEOUT)
				{
					// read participations
					auto participations = pt.get_child (PEER_PROFILE_SECTION_PARTICIPATION);
					m_NumTunnelsAgreed = participations.get (PEER_PROFILE_PARTICIPATION_AGREED, 0);
					m_NumTunnelsDeclined = participations.get (PEER_PROFILE_PARTICIPATION_DECLINED, 0);
					m_NumTunnelsNonReplied = participations.get (PEER_PROFILE_PARTICIPATION_NON_REPLIED, 0);
				}
				else
					*this = RouterProfile (m_IdentHash);
			}
			catch (std::exception& ex)
			{
				LogPrint (eLogError, "Can't read profile ", base64, " :", ex.what ());
			}
		}
	}

	void RouterProfile::TunnelBuildResponse (uint8_t ret)
	{
		if (ret > 0)
			m_NumTunnelsDeclined++;
		else
			m_NumTunnelsAgreed++;
		UpdateTime ();
	}

	void RouterProfile::TunnelNonReplied ()
	{
		m_NumTunnelsNonReplied++;
		UpdateTime ();
	}

	bool RouterProfile::IsLowPartcipationRate (int elapsedTime) const
	{
		if (elapsedTime < 900) // if less than 15 minutes
			return m_NumTunnelsAgreed < m_NumTunnelsDeclined; // 50% rate
		else
			return 3*m_NumTunnelsAgreed < m_NumTunnelsDeclined; // 25% rate
	}

	bool RouterProfile::IsLowReplyRate (int elapsedTime) const
	{
		auto total = m_NumTunnelsAgreed + m_NumTunnelsDeclined;
		if (elapsedTime < 300) // if less than 5 minutes
			return m_NumTunnelsNonReplied > 5*(total + 1);
		else
			return !total && m_NumTunnelsNonReplied*15 > elapsedTime;
	}

	bool RouterProfile::IsBad () const
	{
		auto elapsedTime = (GetTime () - m_LastUpdateTime).total_seconds ();
		return IsAlwaysDeclining () || IsLowPartcipationRate (elapsedTime) || IsLowReplyRate (elapsedTime);
	}

	std::shared_ptr<RouterProfile> GetRouterProfile (const IdentHash& identHash)
	{
		auto profile = std::make_shared<RouterProfile> (identHash);
		profile->Load (); // if possible
		return profile;
	}

	void DeleteObsoleteProfiles ()
	{
		int num = 0;
		auto ts = boost::posix_time::second_clock::local_time();
		boost::filesystem::path p (i2p::util::filesystem::GetDataDir()/PEER_PROFILES_DIRECTORY);
		if (boost::filesystem::exists (p))
		{
			boost::filesystem::directory_iterator end;
			for (boost::filesystem::directory_iterator it (p); it != end; ++it)
			{
				if (boost::filesystem::is_directory (it->status()))
				{
					for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1)
					{
						auto lastModified = boost::posix_time::from_time_t (boost::filesystem::last_write_time (it1->path ()));
						if ((ts - lastModified).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT)
						{
							boost::filesystem::remove (it1->path ());
							num++;
						}
					}
				}
			}
		}
		LogPrint (eLogInfo, num, " obsolete profiles deleted");
	}
}
}
<|endoftext|>


<commit_before>#include <fcntl.h>

#include <netinet/in.h>
#include <sys/socket.h>

#include "server.h"
#include "net/length.h"

#include "xapian.h"

const int MSECS_IDLE_TIMEOUT_DEFAULT = 60000;
const int MSECS_ACTIVE_TIMEOUT_DEFAULT = 15000;

void print_string(const std::string &string) {
	const char *p = string.c_str();
	const char *p_end = p + string.size();
	printf("'");
	while (p != p_end) {
		if (*p >= ' ' && *p <= '~') {
			printf("%c", *p++);
		} else {
			printf("\\x%02x", *p++ & 0xff);
		}
	}
	printf("'\n");
}

class XapianWorker : public Task {
private:
	XapiandClient *client;

public:
	XapianWorker(XapiandClient *client_) : Task(), client(client_) {}

	~XapianWorker() {}

	virtual void run() {
		client->run();
	}
};

//
//   Buffer class - allow for output buffering such that it can be written out
//                                 into async pieces
//
struct Buffer {
	char type;
	char *data;
	ssize_t len;
	ssize_t pos;

	Buffer(char type_, const char *bytes, ssize_t nbytes) {
		pos = 0;
		len = nbytes;
		type = type_;
		data = new char[nbytes];
		memcpy(data, bytes, nbytes);
	}

	virtual ~Buffer() {
		delete [] data;
	}

	char *dpos() {
		return data + pos;
	}

	ssize_t nbytes() {
		return len - pos;
	}
};

void XapiandClient::callback(ev::io &watcher, int revents)
{
	if (EV_ERROR & revents) {
		perror("got invalid event");
		return;
	}

	if (revents & EV_READ)
		read_cb(watcher);

	if (revents & EV_WRITE)
		write_cb(watcher);

	pthread_mutex_lock(&qmtx);
	if (write_queue.empty()) {
		io.set(ev::READ);
	} else {
		io.set(ev::READ|ev::WRITE);
	}
	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::async_cb(ev::async &watcher, int revents)
{
	pthread_mutex_lock(&qmtx);
	if (!write_queue.empty()) {
		io.set(ev::READ|ev::WRITE);
	}
	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::write_cb(ev::io &watcher)
{
	pthread_mutex_lock(&qmtx);

	if (write_queue.empty()) {
		io.set(ev::READ);
	} else {
		Buffer* buffer = write_queue.front();

		// printf("sent:");
		// print_string(std::string(buffer->dpos(), buffer->nbytes()));

		ssize_t written = write(watcher.fd, buffer->dpos(), buffer->nbytes());
		if (written < 0) {
			perror("read error");
		} else {
			buffer->pos += written;
			if (buffer->nbytes() == 0) {
				write_queue.pop_front();
				delete buffer;
			}
		}
	}

	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::read_cb(ev::io &watcher)
{
	char buf[1024];

	ssize_t received = recv(watcher.fd, buf, sizeof(buf), 0);

	if (received < 0) {
		perror("read error");
		return;
	}

	if (received == 0) {
		// Gack - we're deleting ourself inside of ourself!
		delete this;
	} else {
		buffer.append(buf, received);
		if (buffer.length() >= 2) {
			const char *o = buffer.data();
			const char *p = o;
			const char *p_end = p + buffer.size();

			message_type type = static_cast<message_type>(*p++);
			size_t len;
			try {
				len = decode_length(&p, p_end, true);
			} catch (const Xapian::NetworkError & e) {
				return;
			}
			std::string data = std::string(p, len);
			buffer.erase(0, p - o + len);

			// printf("received:");
			// print_string(data);

			Buffer *msg = new Buffer(type, data.c_str(), data.size());

			messages_queue.push(msg);

			if (type != MSG_GETMSET && type != MSG_SHUTDOWN) {
				thread_pool->addTask(new XapianWorker(this));
			}

		}
	}
}

void XapiandClient::signal_cb(ev::sig &signal, int revents)
{
	delete this;
}

message_type XapiandClient::get_message(double timeout, std::string & result, message_type required_type)
{
	Buffer* msg;
	if (!messages_queue.pop(msg)) {
		throw Xapian::NetworkError("No message available");
	}

	std::string buf(&msg->type, 1);
	buf += encode_length(msg->nbytes());
	buf += std::string(msg->dpos(), msg->nbytes());
	printf("get_message:");
	print_string(buf);

	message_type type = static_cast<message_type>(msg->type);
	result.assign(msg->dpos(), msg->nbytes());

	delete msg;
	return type;
}

void XapiandClient::send_message(reply_type type, const std::string &message) {
	char type_as_char = static_cast<char>(type);
	std::string buf(&type_as_char, 1);
	buf += encode_length(message.size());
	buf += message;

	printf("send_message:");
	print_string(buf);

	pthread_mutex_lock(&qmtx);
	write_queue.push_back(new Buffer(type, buf.c_str(), buf.size()));
	pthread_mutex_unlock(&qmtx);

	async.send();
}

void XapiandClient::send_message(reply_type type, const std::string &message, double end_time)
{
	send_message(type, message);
}

Xapian::Database * XapiandClient::get_db(bool writable_)
{
	if (dbpaths.empty()) {
		return NULL;
	}
	Xapian::Database *db_;
	if (writable_) {
		db_ = new Xapian::WritableDatabase(dbpaths[0], Xapian::DB_CREATE_OR_OPEN);
	} else {
		db_ = new Xapian::Database(dbpaths[0], Xapian::DB_CREATE_OR_OPEN);
		if (!writable) {
			std::vector<std::string>::const_iterator i(dbpaths.begin());
			for (++i; i != dbpaths.end(); ++i) {
				db_->add_database(Xapian::Database(*i));
			}
		} else if (dbpaths.size() != 1) {
			printf("ERROR: Expecting exactly one database.");
		}

	}
	printf("NEW DB!\n");
	return db_;
}

void XapiandClient::release_db(Xapian::Database *db_)
{
	printf("RELEASED DB!\n");
	delete db_;
}

void XapiandClient::select_db(const std::vector<std::string> &dbpaths_, bool writable_)
{
	dbpaths = dbpaths_;
}

void XapiandClient::run()
{
	try {
		run_one();
	} catch (const Xapian::NetworkError &e) {
		printf("ERROR: %s\n", e.get_msg().c_str());
	} catch (...) {
		printf("ERROR!\n");
	}
}

XapiandClient::XapiandClient(int sock_, ThreadPool *thread_pool_, double active_timeout_, double idle_timeout_)
	: RemoteProtocol(std::vector<std::string>(), active_timeout_, idle_timeout_, true),
	  sock(sock_),
	  thread_pool(thread_pool_)
{
	pthread_mutex_init(&qmtx, 0);

	fcntl(sock, F_SETFL, fcntl(sock, F_GETFL, 0) | O_NONBLOCK);

	printf("Got connection, %d client(s) connected.\n", ++total_clients);

	io.set<XapiandClient, &XapiandClient::callback>(this);
	io.start(sock, ev::READ);

	sig.set<XapiandClient, &XapiandClient::signal_cb>(this);
	sig.start(SIGINT);

	async.set<XapiandClient, &XapiandClient::async_cb>(this);
	async.start();

	try {
		msg_update(std::string());
	} catch (const Xapian::NetworkError &e) {
		printf("ERROR: %s\n", e.get_msg().c_str());
	} catch (...) {
		printf("ERROR!\n");
	}
}

XapiandClient::~XapiandClient()
{
	shutdown(sock, SHUT_RDWR);

	// Stop and free watcher if client socket is closing
	io.stop();
	sig.stop();
	async.stop();

	close(sock);

	printf("Lost connection, %d client(s) connected.\n", --total_clients);

	pthread_mutex_destroy(&qmtx);
}

void XapiandServer::io_accept(ev::io &watcher, int revents)
{
	if (EV_ERROR & revents) {
		perror("got invalid event");
		return;
	}

	struct sockaddr_in client_addr;
	socklen_t client_len = sizeof(client_addr);

	int client_sock = accept(watcher.fd, (struct sockaddr *)&client_addr, &client_len);

	if (client_sock < 0) {
		perror("accept error");
		return;
	}

	double active_timeout = MSECS_ACTIVE_TIMEOUT_DEFAULT * 1e-3;
	double idle_timeout = MSECS_IDLE_TIMEOUT_DEFAULT * 1e-3;
	new XapiandClient(client_sock, this->thread_pool, active_timeout, idle_timeout);
}

void XapiandServer::signal_cb(ev::sig &signal, int revents)
{
	signal.loop.break_loop();
}

XapiandServer::XapiandServer(int port, ThreadPool *thread_pool_)
	: thread_pool(thread_pool_)
{
	int optval = 1;
	struct sockaddr_in addr;

	sock = socket(PF_INET, SOCK_STREAM, 0);
	setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&optval, sizeof(optval));

	addr.sin_family = AF_INET;
	addr.sin_port = htons(port);
	addr.sin_addr.s_addr = INADDR_ANY;

	if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) != 0) {
		perror("bind");
		close(sock);
		sock = 0;
	} else {
		printf("Listening on port %d\n", port);
		fcntl(sock, F_SETFL, fcntl(sock, F_GETFL, 0) | O_NONBLOCK);

		listen(sock, 5);

		io.set<XapiandServer, &XapiandServer::io_accept>(this);
		io.start(sock, ev::READ);

		sig.set<&XapiandServer::signal_cb>();
		sig.start(SIGINT);
	}
}

XapiandServer::~XapiandServer()
{
	shutdown(sock, SHUT_RDWR);

	io.stop();
	sig.stop();

	close(sock);

	printf("Done with all work!\n");
}

int XapiandClient::total_clients = 0;
<commit_msg>Added endpoint class<commit_after>#include <fcntl.h>

#include <netinet/in.h>
#include <sys/socket.h>

#include "server.h"
#include "net/length.h"

#include "xapian.h"

const int MSECS_IDLE_TIMEOUT_DEFAULT = 60000;
const int MSECS_ACTIVE_TIMEOUT_DEFAULT = 15000;

void print_string(const std::string &string) {
	const char *p = string.c_str();
	const char *p_end = p + string.size();
	printf("'");
	while (p != p_end) {
		if (*p >= ' ' && *p <= '~') {
			printf("%c", *p++);
		} else {
			printf("\\x%02x", *p++ & 0xff);
		}
	}
	printf("'\n");
}

class XapianWorker : public Task {
private:
	XapiandClient *client;

public:
	XapianWorker(XapiandClient *client_) : Task(), client(client_) {}

	~XapianWorker() {}

	virtual void run() {
		client->run();
	}
};

//
//   Buffer class - allow for output buffering such that it can be written out
//                                 into async pieces
//
struct Buffer {
	char type;
	char *data;
	ssize_t len;
	ssize_t pos;

	Buffer(char type_, const char *bytes, ssize_t nbytes) {
		pos = 0;
		len = nbytes;
		type = type_;
		data = new char[nbytes];
		memcpy(data, bytes, nbytes);
	}

	virtual ~Buffer() {
		delete [] data;
	}

	char *dpos() {
		return data + pos;
	}

	ssize_t nbytes() {
		return len - pos;
	}
};

void XapiandClient::callback(ev::io &watcher, int revents)
{
	if (EV_ERROR & revents) {
		perror("got invalid event");
		return;
	}

	if (revents & EV_READ)
		read_cb(watcher);

	if (revents & EV_WRITE)
		write_cb(watcher);

	pthread_mutex_lock(&qmtx);
	if (write_queue.empty()) {
		io.set(ev::READ);
	} else {
		io.set(ev::READ|ev::WRITE);
	}
	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::async_cb(ev::async &watcher, int revents)
{
	pthread_mutex_lock(&qmtx);
	if (!write_queue.empty()) {
		io.set(ev::READ|ev::WRITE);
	}
	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::write_cb(ev::io &watcher)
{
	pthread_mutex_lock(&qmtx);

	if (write_queue.empty()) {
		io.set(ev::READ);
	} else {
		Buffer* buffer = write_queue.front();

		// printf("sent:");
		// print_string(std::string(buffer->dpos(), buffer->nbytes()));

		ssize_t written = write(watcher.fd, buffer->dpos(), buffer->nbytes());
		if (written < 0) {
			perror("read error");
		} else {
			buffer->pos += written;
			if (buffer->nbytes() == 0) {
				write_queue.pop_front();
				delete buffer;
			}
		}
	}

	pthread_mutex_unlock(&qmtx);
}

void XapiandClient::read_cb(ev::io &watcher)
{
	char buf[1024];

	ssize_t received = recv(watcher.fd, buf, sizeof(buf), 0);

	if (received < 0) {
		perror("read error");
		return;
	}

	if (received == 0) {
		// Gack - we're deleting ourself inside of ourself!
		delete this;
	} else {
		buffer.append(buf, received);
		if (buffer.length() >= 2) {
			const char *o = buffer.data();
			const char *p = o;
			const char *p_end = p + buffer.size();

			message_type type = static_cast<message_type>(*p++);
			size_t len;
			try {
				len = decode_length(&p, p_end, true);
			} catch (const Xapian::NetworkError & e) {
				return;
			}
			std::string data = std::string(p, len);
			buffer.erase(0, p - o + len);

			// printf("received:");
			// print_string(data);

			Buffer *msg = new Buffer(type, data.c_str(), data.size());

			messages_queue.push(msg);

			if (type != MSG_GETMSET && type != MSG_SHUTDOWN) {
				thread_pool->addTask(new XapianWorker(this));
			}

		}
	}
}

void XapiandClient::signal_cb(ev::sig &signal, int revents)
{
	delete this;
}

message_type XapiandClient::get_message(double timeout, std::string & result, message_type required_type)
{
	Buffer* msg;
	if (!messages_queue.pop(msg)) {
		throw Xapian::NetworkError("No message available");
	}

	std::string buf(&msg->type, 1);
	buf += encode_length(msg->nbytes());
	buf += std::string(msg->dpos(), msg->nbytes());
	printf("get_message:");
	print_string(buf);

	message_type type = static_cast<message_type>(msg->type);
	result.assign(msg->dpos(), msg->nbytes());

	delete msg;
	return type;
}

void XapiandClient::send_message(reply_type type, const std::string &message) {
	char type_as_char = static_cast<char>(type);
	std::string buf(&type_as_char, 1);
	buf += encode_length(message.size());
	buf += message;

	printf("send_message:");
	print_string(buf);

	pthread_mutex_lock(&qmtx);
	write_queue.push_back(new Buffer(type, buf.c_str(), buf.size()));
	pthread_mutex_unlock(&qmtx);

	async.send();
}

void XapiandClient::send_message(reply_type type, const std::string &message, double end_time)
{
	send_message(type, message);
}

Xapian::Database * XapiandClient::get_db(bool writable_)
{
	if (dbpaths.empty()) {
		return NULL;
	}
	Xapian::Database *db_;
	if (writable_) {
		db_ = new Xapian::WritableDatabase(dbpaths[0], Xapian::DB_CREATE_OR_OPEN);
	} else {
		db_ = new Xapian::Database(dbpaths[0], Xapian::DB_CREATE_OR_OPEN);
		if (!writable) {
			std::vector<std::string>::const_iterator i(dbpaths.begin());
			for (++i; i != dbpaths.end(); ++i) {
				db_->add_database(Xapian::Database(*i));
			}
		} else if (dbpaths.size() != 1) {
			printf("ERROR: Expecting exactly one database.");
		}

	}
	printf("NEW DB!\n");
	return db_;
}

void XapiandClient::release_db(Xapian::Database *db_)
{
	printf("RELEASED DB!\n");
	delete db_;
}

void XapiandClient::select_db(const std::vector<std::string> &dbpaths_, bool writable_)
{
	std::vector<std::string>::const_iterator i(dbpaths_.begin());
	for (; i != dbpaths_.end(); i++) {
		std::string dbpath(*i);
		Endpoint endpoint = Endpoint(dbpath, std::string(), 8890);
		endpoints.push_back(endpoint);
	}
	std::sort(endpoints.begin(), endpoints.end());

	std::string es = std::string();
	std::vector<Endpoint>::const_iterator j(endpoints.begin());
	for (; j != endpoints.end(); j++) {
		es += ":";
		es += (*j).as_string().c_str();
	}

	std::hash<std::string> hash_fn;
	size_t hash = hash_fn(es);
	printf("> %s -> %lx\n", es.c_str(), hash);

	dbpaths = dbpaths_;
}

void XapiandClient::run()
{
	try {
		run_one();
	} catch (const Xapian::NetworkError &e) {
		printf("ERROR: %s\n", e.get_msg().c_str());
	} catch (...) {
		printf("ERROR!\n");
	}
}

XapiandClient::XapiandClient(int sock_, ThreadPool *thread_pool_, double active_timeout_, double idle_timeout_)
	: RemoteProtocol(std::vector<std::string>(), active_timeout_, idle_timeout_, true),
	  sock(sock_),
	  thread_pool(thread_pool_)
{
	pthread_mutex_init(&qmtx, 0);

	fcntl(sock, F_SETFL, fcntl(sock, F_GETFL, 0) | O_NONBLOCK);

	printf("Got connection, %d client(s) connected.\n", ++total_clients);

	io.set<XapiandClient, &XapiandClient::callback>(this);
	io.start(sock, ev::READ);

	sig.set<XapiandClient, &XapiandClient::signal_cb>(this);
	sig.start(SIGINT);

	async.set<XapiandClient, &XapiandClient::async_cb>(this);
	async.start();

	try {
		msg_update(std::string());
	} catch (const Xapian::NetworkError &e) {
		printf("ERROR: %s\n", e.get_msg().c_str());
	} catch (...) {
		printf("ERROR!\n");
	}
}

XapiandClient::~XapiandClient()
{
	shutdown(sock, SHUT_RDWR);

	// Stop and free watcher if client socket is closing
	io.stop();
	sig.stop();
	async.stop();

	close(sock);

	printf("Lost connection, %d client(s) connected.\n", --total_clients);

	pthread_mutex_destroy(&qmtx);
}

void XapiandServer::io_accept(ev::io &watcher, int revents)
{
	if (EV_ERROR & revents) {
		perror("got invalid event");
		return;
	}

	struct sockaddr_in client_addr;
	socklen_t client_len = sizeof(client_addr);

	int client_sock = accept(watcher.fd, (struct sockaddr *)&client_addr, &client_len);

	if (client_sock < 0) {
		perror("accept error");
		return;
	}

	double active_timeout = MSECS_ACTIVE_TIMEOUT_DEFAULT * 1e-3;
	double idle_timeout = MSECS_IDLE_TIMEOUT_DEFAULT * 1e-3;
	new XapiandClient(client_sock, this->thread_pool, active_timeout, idle_timeout);
}

void XapiandServer::signal_cb(ev::sig &signal, int revents)
{
	signal.loop.break_loop();
}

XapiandServer::XapiandServer(int port, ThreadPool *thread_pool_)
	: thread_pool(thread_pool_)
{
	int optval = 1;
	struct sockaddr_in addr;

	sock = socket(PF_INET, SOCK_STREAM, 0);
	setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&optval, sizeof(optval));

	addr.sin_family = AF_INET;
	addr.sin_port = htons(port);
	addr.sin_addr.s_addr = INADDR_ANY;

	if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) != 0) {
		perror("bind");
		close(sock);
		sock = 0;
	} else {
		printf("Listening on port %d\n", port);
		fcntl(sock, F_SETFL, fcntl(sock, F_GETFL, 0) | O_NONBLOCK);

		listen(sock, 5);

		io.set<XapiandServer, &XapiandServer::io_accept>(this);
		io.start(sock, ev::READ);

		sig.set<&XapiandServer::signal_cb>();
		sig.start(SIGINT);
	}
}

XapiandServer::~XapiandServer()
{
	shutdown(sock, SHUT_RDWR);

	io.stop();
	sig.stop();

	close(sock);

	printf("Done with all work!\n");
}

int XapiandClient::total_clients = 0;
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifdef INTEL_MKL

#include <algorithm>
#include <vector>
#include "tensorflow/core/framework/numeric_op.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/kernels/ops_util.h"
#include "tensorflow/core/platform/byte_order.h"
#include "tensorflow/core/platform/cpu_info.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/util/tensor_format.h"

#include "mkldnn.hpp"
#include "tensorflow/core/kernels/mkl_tfconv_op.h"
#include "tensorflow/core/util/mkl_util.h"
using mkldnn::stream;

namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;

///////////////////////////////////////////////////////////
//               Op kernel
// Checks and ensures that the 2 inputs are compatible for mkl binary ops.
// Here's the basic logic:
//
// if both inputs are in TF format:
//   pass the inputs through to the output
// else if both inputs are in mkl format:
//   if both have the same shape:
//     pass the inputs through to the output
//   else:
//     convert both to TF
// else if one is TF and one is MKL:
//   if broadcast is needed:
//     convert the MKL format input to TF format
//   else:
//     convert the TF format input to MKL format
///////////////////////////////////////////////////////////

template <typename Device, typename T>
class MklInputConversionOp : public OpKernel {
 public:
  explicit MklInputConversionOp(OpKernelConstruction* context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format_str));
    OP_REQUIRES_OK(context, context->GetAttr("T", &op_data_type));
    has_avx512f_ = port::TestCPUFeature(port::CPUFeature::AVX512F);
  }

 private:
  void Compute(OpKernelContext* context) override {
    const int kInputIndex_0 = 0, kInputIndex_1 = 1;
    const Tensor& input_tensor_0 = MklGetInput(context, kInputIndex_0);
    MklDnnShape input_shape_0;
    GetMklShape(context, kInputIndex_0, &input_shape_0);

    const Tensor& input_tensor_1 = MklGetInput(context, kInputIndex_1);
    MklDnnShape input_shape_1;
    GetMklShape(context, kInputIndex_1, &input_shape_1);

    VLOG(1) << "MklInputConversionOp: Input shapes are: "
            << context->input(kInputIndex_0).shape().DebugString() << " and "
            << context->input(kInputIndex_1).shape().DebugString();

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // if both inputs are in TF format, just copy input tensors to output.
    if (!input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) {
      VLOG(1) << "MklInputConversionOp: No conversion needed, "
              << "copying TF inputs to output";

      ForwardTfTensorInToOut(context, kInputIndex_0, kInputIndex_0);
      ForwardTfTensorInToOut(context, kInputIndex_1, kInputIndex_1);
      return;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // If both inputs are in MKL format
    if (input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
      // It is safer to compare the original TensorFlow shapes than to compare
      // Mkl shapes since element wise ops are forwarded to Eigen
      // implementation.
      TensorShape tf_shape0 = input_shape_0.GetTfShape();
      TensorShape tf_shape1 = input_shape_1.GetTfShape();
      TensorShape tensor_shape0 = input_tensor_0.shape();
      TensorShape tensor_shape1 = input_tensor_1.shape();
      if (tf_shape0 == tf_shape1 && tensor_shape0 == tensor_shape1) {
        auto input0_md = input_shape_0.GetMklLayout();
        auto input1_md = input_shape_1.GetMklLayout();

        // If both have the same shape and same format, pass them through
        if (input0_md.data.format == input1_md.data.format) {
          VLOG(1) << "MklInputConversionOp: No conversion needed, "
                  << "copying MKL inputs with identical shapes to output";

          ForwardMklTensorInToOut(context, kInputIndex_0, kInputIndex_0);
          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
          return;
        } else {
          VLOG(1) << "MklInputConversionOp: Shape is same, but format is "
                     "different, "
                  << "need to convert to same format";
          // TODO: For now, input0 is converted and input1 is unchanged
          //       we should choose the optimal MKL format to convert to.
          Tensor* tensor_out;
          MklDnnShape mkl_output_mkl_shape;
          mkl_output_mkl_shape.SetMklTensor(true);
          mkl_output_mkl_shape.SetElemType(MklDnnType<T>());
          mkl_output_mkl_shape.SetTfLayout(input_shape_0.GetDimension(),
                                           input_shape_0.GetSizesAsMklDnnDims(),
                                           input_shape_0.GetTfDataFormat());

          // Get MKL layout from input1 as destination layout
          mkl_output_mkl_shape.SetMklLayout(&input1_md);

          // Create output Mkl tensor for index 0
          AllocateOutputSetMklShape(context, kInputIndex_0, &tensor_out,
                                    input_tensor_0.shape(),
                                    mkl_output_mkl_shape);

          // Create MklDnnData object for input0 tesnsor
          auto cpu_engine = engine(engine::cpu, 0);
          MklDnnData<T> input(&cpu_engine);
          input.SetUsrMem(input0_md, &input_tensor_0);

          // Create reorder from input0's layout to input1's layout
          std::vector<primitive> net;
          CHECK_EQ(input.CheckReorderToOpMem(
                       memory::primitive_desc(input1_md, cpu_engine),
                       tensor_out, &net),
                   true);
          stream(stream::kind::eager).submit(net).wait();

          // Input1 will be passed through
          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
          return;
        }
      }

      // Sanity check
      bool mkl_shapes_are_same = ((input_shape_0 == input_shape_1) &&
                                  (tensor_shape0 == tensor_shape1));
      if (mkl_shapes_are_same) {
        CHECK(false) << "MklInputConversionOp: Unexpected: TF shapes are "
                        "different but MKL shapes are same";
      }

      // Both have different shapes, so broadcast will be necessary.
      // Convert to TF and pass both tensors through (we can't do broadcast
      // with MKL tensors)
      VLOG(1) << "MklInputConversionOp: Broadcast needed, "
              << "converted MKL inputs to TF format";
      // TODO: Cleanup op_data_type and has_avx512f_ after these two parameters
      //       are removed from ConvertMklToTf
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           kInputIndex_0);
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           kInputIndex_1);
      SetDummyMklDnnShapeOutput(context, kInputIndex_0);
      SetDummyMklDnnShapeOutput(context, kInputIndex_1);
      return;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // One input is MKL and one is TF. If no broadcast is needed, convert
    // the TF tensor to MKL, otherwise convert the MKL tensor to TF format
    VLOG(1) << "MklInputConversionOp: Inputs in different formats (MKL/TF)";

    const Tensor* mkl_tensor;
    const MklDnnShape* mkl_shape;
    const Tensor* tf_tensor;
    uint mkl_tensor_index;
    uint tf_tensor_index;
    if (input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) {
      mkl_tensor = &input_tensor_0;
      mkl_shape = &input_shape_0;
      mkl_tensor_index = 0;
      tf_tensor = &input_tensor_1;
      tf_tensor_index = 1;
    } else if (!input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
      mkl_tensor = &input_tensor_1;
      mkl_shape = &input_shape_1;
      mkl_tensor_index = 1;
      tf_tensor = &input_tensor_0;
      tf_tensor_index = 0;
    } else {
      CHECK(false) << "MklInputConversionOp: Unexpected combination of input "
                      "shapes for MKL "
                   << "element-wise op";
    }

    // Broadcast is needed if the shapes are not the same
    if (mkl_shape->GetTfShape().num_elements() ==
        tf_tensor->shape().num_elements()) {
      // Both shapes are same, convert the TF input to MKL
      VLOG(1) << "MklInputConversionOp: No broadcast needed.";
      VLOG(1) << "MklInputConversionOp: Converting input " << tf_tensor_index
              << " to MKL format";

      // Create MklDnnShape for output Mkl tensor.
      Tensor* tensor_out;
      MklDnnShape mkl_output_mkl_shape;
      mkl_output_mkl_shape.SetMklTensor(true);
      mkl_output_mkl_shape.SetElemType(MklDnnType<T>());
      mkl_output_mkl_shape.SetTfLayout(mkl_shape->GetDimension(),
                                       mkl_shape->GetSizesAsMklDnnDims(),
                                       mkl_shape->GetTfDataFormat());
      // ** Temporarily borrow the layout from the MKL input **
      auto output_mkl_md = mkl_shape->GetMklLayout();
      mkl_output_mkl_shape.SetMklLayout(&output_mkl_md);

      // Create output Mkl tensor
      AllocateOutputSetMklShape(context, tf_tensor_index, &tensor_out,
                                mkl_tensor->shape(), mkl_output_mkl_shape);

      // Create MklDnnData object for input tensor. Input tensor is in
      // Tensorflow layout.
      auto cpu_engine = engine(engine::cpu, 0);
      MklDnnData<T> tf_input(&cpu_engine);
      auto input_tf_md = mkl_output_mkl_shape.GetTfLayout();
      tf_input.SetUsrMem(input_tf_md, tf_tensor);

      // Create reorder between tensorflow layout and Mkl layout if necessary
      std::vector<primitive> net;
      bool reordered = tf_input.CheckReorderToOpMem(
          memory::primitive_desc(output_mkl_md, cpu_engine), tensor_out, &net);

      if (!reordered) {
        // This is the case that the TF tensor has the same shape and format of
        // mkl tensor. However, tf_tensor can not be simply forwarded to the
        // output tensor since mkl data tensor is always one dimensional tensor.
        // Tensor::CopyFrom shares the buffer of the other tensor while set its
        // shape to the other tensor.
        CHECK(tensor_out->CopyFrom(*tf_tensor, tensor_out->shape()));
      } else {
        stream(stream::kind::eager).submit(net).wait();
      }

      // -- The tensor in MKL format passes through --
      ForwardMklTensorInToOut(context, mkl_tensor_index, mkl_tensor_index);
    } else {
      // Broadcast is needed, so convert the MKL input to TF
      VLOG(1) << "MklInputConversionOp: Broadcast needed.";
      VLOG(1) << "MklInputConversionOp: Converting input " << mkl_tensor_index
              << " to TF format";
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           mkl_tensor_index);
      SetDummyMklDnnShapeOutput(context, mkl_tensor_index);

      // The tensor in TF format passes through
      ForwardTfTensorInToOut(context, tf_tensor_index, tf_tensor_index);
    }

    VLOG(1) << "MklInputConversionOp: Shapes (output): "
            << context->mutable_output(kInputIndex_0)->shape().DebugString()
            << " and "
            << context->mutable_output(kInputIndex_1)->shape().DebugString();

    VLOG(1) << "MklInputConversion completed successfully.";
  }

 private:
  /// Data format of the operation
  string data_format_str;

  /// Data type of the operation
  DataType op_data_type;

  /// CPUIDInfo
  bool has_avx512f_ = false;
};

///////////////////////////////////////////////////////////
//               Register kernel
///////////////////////////////////////////////////////////

#define REGISTER_CPU(T)                                             \
  REGISTER_KERNEL_BUILDER(Name("_MklInputConversion")               \
                              .Device(DEVICE_CPU)                   \
                              .TypeConstraint<T>("T")               \
                              .Label(mkl_op_registry::kMklOpLabel), \
                          MklInputConversionOp<CPUDevice, T>);

// TODO(nhasabni): We cannot support all number types since MklDnn does
// not support types.
// TF_CALL_NUMBER_TYPES(REGISTER_CPU);
TF_CALL_float(REGISTER_CPU);
TF_CALL_bfloat16(REGISTER_CPU);
#undef REGISTER_CPU
}  // namespace tensorflow
#endif  // INTEL_MKL
<commit_msg>[Intel MKL] Removing duplicate registration of MklInputConversionOp for BFloat16<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#ifdef INTEL_MKL

#include <algorithm>
#include <vector>
#include "tensorflow/core/framework/numeric_op.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/kernels/ops_util.h"
#include "tensorflow/core/platform/byte_order.h"
#include "tensorflow/core/platform/cpu_info.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/util/tensor_format.h"

#include "mkldnn.hpp"
#include "tensorflow/core/kernels/mkl_tfconv_op.h"
#include "tensorflow/core/util/mkl_util.h"
using mkldnn::stream;

namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;

///////////////////////////////////////////////////////////
//               Op kernel
// Checks and ensures that the 2 inputs are compatible for mkl binary ops.
// Here's the basic logic:
//
// if both inputs are in TF format:
//   pass the inputs through to the output
// else if both inputs are in mkl format:
//   if both have the same shape:
//     pass the inputs through to the output
//   else:
//     convert both to TF
// else if one is TF and one is MKL:
//   if broadcast is needed:
//     convert the MKL format input to TF format
//   else:
//     convert the TF format input to MKL format
///////////////////////////////////////////////////////////

template <typename Device, typename T>
class MklInputConversionOp : public OpKernel {
 public:
  explicit MklInputConversionOp(OpKernelConstruction* context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format_str));
    OP_REQUIRES_OK(context, context->GetAttr("T", &op_data_type));
    has_avx512f_ = port::TestCPUFeature(port::CPUFeature::AVX512F);
  }

 private:
  void Compute(OpKernelContext* context) override {
    const int kInputIndex_0 = 0, kInputIndex_1 = 1;
    const Tensor& input_tensor_0 = MklGetInput(context, kInputIndex_0);
    MklDnnShape input_shape_0;
    GetMklShape(context, kInputIndex_0, &input_shape_0);

    const Tensor& input_tensor_1 = MklGetInput(context, kInputIndex_1);
    MklDnnShape input_shape_1;
    GetMklShape(context, kInputIndex_1, &input_shape_1);

    VLOG(1) << "MklInputConversionOp: Input shapes are: "
            << context->input(kInputIndex_0).shape().DebugString() << " and "
            << context->input(kInputIndex_1).shape().DebugString();

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // if both inputs are in TF format, just copy input tensors to output.
    if (!input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) {
      VLOG(1) << "MklInputConversionOp: No conversion needed, "
              << "copying TF inputs to output";

      ForwardTfTensorInToOut(context, kInputIndex_0, kInputIndex_0);
      ForwardTfTensorInToOut(context, kInputIndex_1, kInputIndex_1);
      return;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // If both inputs are in MKL format
    if (input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
      // It is safer to compare the original TensorFlow shapes than to compare
      // Mkl shapes since element wise ops are forwarded to Eigen
      // implementation.
      TensorShape tf_shape0 = input_shape_0.GetTfShape();
      TensorShape tf_shape1 = input_shape_1.GetTfShape();
      TensorShape tensor_shape0 = input_tensor_0.shape();
      TensorShape tensor_shape1 = input_tensor_1.shape();
      if (tf_shape0 == tf_shape1 && tensor_shape0 == tensor_shape1) {
        auto input0_md = input_shape_0.GetMklLayout();
        auto input1_md = input_shape_1.GetMklLayout();

        // If both have the same shape and same format, pass them through
        if (input0_md.data.format == input1_md.data.format) {
          VLOG(1) << "MklInputConversionOp: No conversion needed, "
                  << "copying MKL inputs with identical shapes to output";

          ForwardMklTensorInToOut(context, kInputIndex_0, kInputIndex_0);
          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
          return;
        } else {
          VLOG(1) << "MklInputConversionOp: Shape is same, but format is "
                     "different, "
                  << "need to convert to same format";
          // TODO: For now, input0 is converted and input1 is unchanged
          //       we should choose the optimal MKL format to convert to.
          Tensor* tensor_out;
          MklDnnShape mkl_output_mkl_shape;
          mkl_output_mkl_shape.SetMklTensor(true);
          mkl_output_mkl_shape.SetElemType(MklDnnType<T>());
          mkl_output_mkl_shape.SetTfLayout(input_shape_0.GetDimension(),
                                           input_shape_0.GetSizesAsMklDnnDims(),
                                           input_shape_0.GetTfDataFormat());

          // Get MKL layout from input1 as destination layout
          mkl_output_mkl_shape.SetMklLayout(&input1_md);

          // Create output Mkl tensor for index 0
          AllocateOutputSetMklShape(context, kInputIndex_0, &tensor_out,
                                    input_tensor_0.shape(),
                                    mkl_output_mkl_shape);

          // Create MklDnnData object for input0 tesnsor
          auto cpu_engine = engine(engine::cpu, 0);
          MklDnnData<T> input(&cpu_engine);
          input.SetUsrMem(input0_md, &input_tensor_0);

          // Create reorder from input0's layout to input1's layout
          std::vector<primitive> net;
          CHECK_EQ(input.CheckReorderToOpMem(
                       memory::primitive_desc(input1_md, cpu_engine),
                       tensor_out, &net),
                   true);
          stream(stream::kind::eager).submit(net).wait();

          // Input1 will be passed through
          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
          return;
        }
      }

      // Sanity check
      bool mkl_shapes_are_same = ((input_shape_0 == input_shape_1) &&
                                  (tensor_shape0 == tensor_shape1));
      if (mkl_shapes_are_same) {
        CHECK(false) << "MklInputConversionOp: Unexpected: TF shapes are "
                        "different but MKL shapes are same";
      }

      // Both have different shapes, so broadcast will be necessary.
      // Convert to TF and pass both tensors through (we can't do broadcast
      // with MKL tensors)
      VLOG(1) << "MklInputConversionOp: Broadcast needed, "
              << "converted MKL inputs to TF format";
      // TODO: Cleanup op_data_type and has_avx512f_ after these two parameters
      //       are removed from ConvertMklToTf
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           kInputIndex_0);
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           kInputIndex_1);
      SetDummyMklDnnShapeOutput(context, kInputIndex_0);
      SetDummyMklDnnShapeOutput(context, kInputIndex_1);
      return;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    // One input is MKL and one is TF. If no broadcast is needed, convert
    // the TF tensor to MKL, otherwise convert the MKL tensor to TF format
    VLOG(1) << "MklInputConversionOp: Inputs in different formats (MKL/TF)";

    const Tensor* mkl_tensor;
    const MklDnnShape* mkl_shape;
    const Tensor* tf_tensor;
    uint mkl_tensor_index;
    uint tf_tensor_index;
    if (input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) {
      mkl_tensor = &input_tensor_0;
      mkl_shape = &input_shape_0;
      mkl_tensor_index = 0;
      tf_tensor = &input_tensor_1;
      tf_tensor_index = 1;
    } else if (!input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
      mkl_tensor = &input_tensor_1;
      mkl_shape = &input_shape_1;
      mkl_tensor_index = 1;
      tf_tensor = &input_tensor_0;
      tf_tensor_index = 0;
    } else {
      CHECK(false) << "MklInputConversionOp: Unexpected combination of input "
                      "shapes for MKL "
                   << "element-wise op";
    }

    // Broadcast is needed if the shapes are not the same
    if (mkl_shape->GetTfShape().num_elements() ==
        tf_tensor->shape().num_elements()) {
      // Both shapes are same, convert the TF input to MKL
      VLOG(1) << "MklInputConversionOp: No broadcast needed.";
      VLOG(1) << "MklInputConversionOp: Converting input " << tf_tensor_index
              << " to MKL format";

      // Create MklDnnShape for output Mkl tensor.
      Tensor* tensor_out;
      MklDnnShape mkl_output_mkl_shape;
      mkl_output_mkl_shape.SetMklTensor(true);
      mkl_output_mkl_shape.SetElemType(MklDnnType<T>());
      mkl_output_mkl_shape.SetTfLayout(mkl_shape->GetDimension(),
                                       mkl_shape->GetSizesAsMklDnnDims(),
                                       mkl_shape->GetTfDataFormat());
      // ** Temporarily borrow the layout from the MKL input **
      auto output_mkl_md = mkl_shape->GetMklLayout();
      mkl_output_mkl_shape.SetMklLayout(&output_mkl_md);

      // Create output Mkl tensor
      AllocateOutputSetMklShape(context, tf_tensor_index, &tensor_out,
                                mkl_tensor->shape(), mkl_output_mkl_shape);

      // Create MklDnnData object for input tensor. Input tensor is in
      // Tensorflow layout.
      auto cpu_engine = engine(engine::cpu, 0);
      MklDnnData<T> tf_input(&cpu_engine);
      auto input_tf_md = mkl_output_mkl_shape.GetTfLayout();
      tf_input.SetUsrMem(input_tf_md, tf_tensor);

      // Create reorder between tensorflow layout and Mkl layout if necessary
      std::vector<primitive> net;
      bool reordered = tf_input.CheckReorderToOpMem(
          memory::primitive_desc(output_mkl_md, cpu_engine), tensor_out, &net);

      if (!reordered) {
        // This is the case that the TF tensor has the same shape and format of
        // mkl tensor. However, tf_tensor can not be simply forwarded to the
        // output tensor since mkl data tensor is always one dimensional tensor.
        // Tensor::CopyFrom shares the buffer of the other tensor while set its
        // shape to the other tensor.
        CHECK(tensor_out->CopyFrom(*tf_tensor, tensor_out->shape()));
      } else {
        stream(stream::kind::eager).submit(net).wait();
      }

      // -- The tensor in MKL format passes through --
      ForwardMklTensorInToOut(context, mkl_tensor_index, mkl_tensor_index);
    } else {
      // Broadcast is needed, so convert the MKL input to TF
      VLOG(1) << "MklInputConversionOp: Broadcast needed.";
      VLOG(1) << "MklInputConversionOp: Converting input " << mkl_tensor_index
              << " to TF format";
      MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
                                           op_data_type, has_avx512f_,
                                           mkl_tensor_index);
      SetDummyMklDnnShapeOutput(context, mkl_tensor_index);

      // The tensor in TF format passes through
      ForwardTfTensorInToOut(context, tf_tensor_index, tf_tensor_index);
    }

    VLOG(1) << "MklInputConversionOp: Shapes (output): "
            << context->mutable_output(kInputIndex_0)->shape().DebugString()
            << " and "
            << context->mutable_output(kInputIndex_1)->shape().DebugString();

    VLOG(1) << "MklInputConversion completed successfully.";
  }

 private:
  /// Data format of the operation
  string data_format_str;

  /// Data type of the operation
  DataType op_data_type;

  /// CPUIDInfo
  bool has_avx512f_ = false;
};

///////////////////////////////////////////////////////////
//               Register kernel
///////////////////////////////////////////////////////////

#define REGISTER_CPU(T)                                             \
  REGISTER_KERNEL_BUILDER(Name("_MklInputConversion")               \
                              .Device(DEVICE_CPU)                   \
                              .TypeConstraint<T>("T")               \
                              .Label(mkl_op_registry::kMklOpLabel), \
                          MklInputConversionOp<CPUDevice, T>);

// TODO(nhasabni): We cannot support all number types since MklDnn does
// not support types.
// TF_CALL_NUMBER_TYPES(REGISTER_CPU);
TF_CALL_float(REGISTER_CPU);
#undef REGISTER_CPU
}  // namespace tensorflow
#endif  // INTEL_MKL
<|endoftext|>


<commit_before>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <cstdint>
#include <functional>
#include <memory>
#include <random>

#include <gtest/gtest.h>
#include <pthreadpool.h>
#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"

namespace tflite {
namespace xnnpack {

TEST(Delegate, CreateWithoutParams) {
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(nullptr),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, CreateWithDefaultParams) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, CreateWithNumThreadsParam) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  delegate_options.num_threads = 2;
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, GetThreadPool) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  delegate_options.num_threads = 2;
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);

  pthreadpool_t threadpool = static_cast<pthreadpool_t>(
      TfLiteXNNPackDelegateGetThreadPool(xnnpack_delegate.get()));
  ASSERT_TRUE(threadpool);
  ASSERT_EQ(2, pthreadpool_get_threads_count(threadpool));
}

}  // namespace xnnpack
}  // namespace tflite
<commit_msg>Minor change to pthreadpool includes.<commit_after>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <cstdint>
#include <functional>
#include <memory>
#include <random>

#include <gtest/gtest.h>
#include "pthreadpool.h"  // from @pthreadpool
#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"

namespace tflite {
namespace xnnpack {

TEST(Delegate, CreateWithoutParams) {
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(nullptr),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, CreateWithDefaultParams) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, CreateWithNumThreadsParam) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  delegate_options.num_threads = 2;
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);
}

TEST(Delegate, GetThreadPool) {
  TfLiteXNNPackDelegateOptions delegate_options =
      TfLiteXNNPackDelegateOptionsDefault();
  delegate_options.num_threads = 2;
  std::unique_ptr<TfLiteDelegate, decltype(&TfLiteXNNPackDelegateDelete)>
      xnnpack_delegate(TfLiteXNNPackDelegateCreate(&delegate_options),
                       TfLiteXNNPackDelegateDelete);

  pthreadpool_t threadpool = static_cast<pthreadpool_t>(
      TfLiteXNNPackDelegateGetThreadPool(xnnpack_delegate.get()));
  ASSERT_TRUE(threadpool);
  ASSERT_EQ(2, pthreadpool_get_threads_count(threadpool));
}

}  // namespace xnnpack
}  // namespace tflite
<|endoftext|>