<commit_before>// Copyright (c) 2009-2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <boost/assign/list_of.hpp> // for 'map_list_of()'
#include <boost/foreach.hpp>

#include "checkpoints.h"

#include "txdb.h"
#include "main.h"
#include "uint256.h"

static const int nCheckpointSpan = 500;

namespace Checkpoints
{
    typedef std::map<int, uint256> MapCheckpoints;

    //
    // What makes a good checkpoint block?
    // + Is surrounded by blocks with reasonable timestamps
    //   (no blocks before with a timestamp after, none after with
    //    timestamp before)
    // + Contains no strange transactions
    //

    static MapCheckpoints mapCheckpoints =
        boost::assign::map_list_of
        ( 15000, uint256("0xca3548f27e851f563008cfd79b3f97b555002ade6d6d1abf82af5161b3e5b0df") )
        ( 55000, uint256("0xb12293afc8d69f285cf0f7c1767b55d27c571c499ac91bdb47a5bccfa76b63b8") )
        ( 110000, uint256("0x92f5e226fb7bcc58f9933fa1c4836b5e2bf029bdb8e9eae366d80b14017841bc") )
        ( 300500, uint256("0x09d9bd6b9002bf66527e098f0e620a6f59c6a3b244bc0bc8f4c4879e19b8fbfa") )
    ;

    // TestNet has no checkpoints
    static MapCheckpoints mapCheckpointsTestnet;

    bool CheckHardened(int nHeight, const uint256& hash)
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        MapCheckpoints::const_iterator i = checkpoints.find(nHeight);
        if (i == checkpoints.end()) return true;
        return hash == i->second;
    }

    int GetTotalBlocksEstimate()
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        if (checkpoints.empty())
            return 0;
        return checkpoints.rbegin()->first;
    }

    CBlockIndex* GetLastCheckpoint(const std::map<uint256, CBlockIndex*>& mapBlockIndex)
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints)
        {
            const uint256& hash = i.second;
            std::map<uint256, CBlockIndex*>::const_iterator t = mapBlockIndex.find(hash);
            if (t != mapBlockIndex.end())
                return t->second;
        }
        return NULL;
    }

    // Automatically select a suitable sync-checkpoint
    const CBlockIndex* AutoSelectSyncCheckpoint()
    {
        const CBlockIndex *pindex = pindexBest;
        // Search backward for a block within max span and maturity window
        while (pindex->pprev && pindex->nHeight + nCheckpointSpan > pindexBest->nHeight)
            pindex = pindex->pprev;
        return pindex;
    }

    // Check against synchronized checkpoint
    bool CheckSync(int nHeight)
    {
        const CBlockIndex* pindexSync = AutoSelectSyncCheckpoint();

        if (nHeight <= pindexSync->nHeight)
            return false;
        return true;
    }
}
<commit_msg>Added checkpoint at block 345000<commit_after>// Copyright (c) 2009-2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <boost/assign/list_of.hpp> // for 'map_list_of()'
#include <boost/foreach.hpp>

#include "checkpoints.h"

#include "txdb.h"
#include "main.h"
#include "uint256.h"

static const int nCheckpointSpan = 500;

namespace Checkpoints
{
    typedef std::map<int, uint256> MapCheckpoints;

    //
    // What makes a good checkpoint block?
    // + Is surrounded by blocks with reasonable timestamps
    //   (no blocks before with a timestamp after, none after with
    //    timestamp before)
    // + Contains no strange transactions
    //

    static MapCheckpoints mapCheckpoints =
        boost::assign::map_list_of
        ( 15000, uint256("0xca3548f27e851f563008cfd79b3f97b555002ade6d6d1abf82af5161b3e5b0df") )
        ( 55000, uint256("0xb12293afc8d69f285cf0f7c1767b55d27c571c499ac91bdb47a5bccfa76b63b8") )
        ( 110000, uint256("0x92f5e226fb7bcc58f9933fa1c4836b5e2bf029bdb8e9eae366d80b14017841bc") )
        ( 300500, uint256("0x09d9bd6b9002bf66527e098f0e620a6f59c6a3b244bc0bc8f4c4879e19b8fbfa") )
        ( 345000, uint256("0x251f44f94e369ebebc51304ce1a7b367ecad8025e822a5a28407085915542a47") )
    ;

    // TestNet has no checkpoints
    static MapCheckpoints mapCheckpointsTestnet;

    bool CheckHardened(int nHeight, const uint256& hash)
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        MapCheckpoints::const_iterator i = checkpoints.find(nHeight);
        if (i == checkpoints.end()) return true;
        return hash == i->second;
    }

    int GetTotalBlocksEstimate()
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        if (checkpoints.empty())
            return 0;
        return checkpoints.rbegin()->first;
    }

    CBlockIndex* GetLastCheckpoint(const std::map<uint256, CBlockIndex*>& mapBlockIndex)
    {
        MapCheckpoints& checkpoints = (TestNet() ? mapCheckpointsTestnet : mapCheckpoints);

        BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints)
        {
            const uint256& hash = i.second;
            std::map<uint256, CBlockIndex*>::const_iterator t = mapBlockIndex.find(hash);
            if (t != mapBlockIndex.end())
                return t->second;
        }
        return NULL;
    }

    // Automatically select a suitable sync-checkpoint
    const CBlockIndex* AutoSelectSyncCheckpoint()
    {
        const CBlockIndex *pindex = pindexBest;
        // Search backward for a block within max span and maturity window
        while (pindex->pprev && pindex->nHeight + nCheckpointSpan > pindexBest->nHeight)
            pindex = pindex->pprev;
        return pindex;
    }

    // Check against synchronized checkpoint
    bool CheckSync(int nHeight)
    {
        const CBlockIndex* pindexSync = AutoSelectSyncCheckpoint();

        if (nHeight <= pindexSync->nHeight)
            return false;
        return true;
    }
}
<|endoftext|>


<commit_before>#include <sys/types.h>
#include <sys/endian.h>
#include <stdio.h>
#include <iostream>
#include <map>
#include <string>

#include "uvxbridge.h"

static int
cmdmap_get_num(cmdmap_t &map, const char *key, uint64_t &val)
{
		auto it = map.find(string(key));

		if (it == map.end())
				return EINVAL;
		if (it->second.tag != TAG_NUMERIC)
				return EINVAL;
		val = it->second.value.numeric;

		return 0;
}

static int
cmdmap_get_str(cmdmap_t &map, const char *key, char **val)
{
		auto it = map.find(string(key));

		if (it == map.end())
				return EINVAL;
		if (it->second.tag != TAG_STRING)
				return EINVAL;
		*val = it->second.value.text;

		return 0;
}

static string
default_result(uint64_t seqno)
{
		char buf[32];
		snprintf(buf, 32, "(result:0x%lX)", seqno);
		return string(buf);
}
parse_value::parse_value(uint64_t numeric)
{
		this->tag = TAG_NUMERIC;
		this->value.numeric = numeric;
}

parse_value::parse_value(char *text)
{
		this->tag = TAG_STRING;
		this->value.text = strdup(text);
}

parse_value::~parse_value()
{
		switch(this->tag) {
		case TAG_STRING:
				free(this->value.text);
				break;
		case TAG_NUMERIC:
				break;
		}
}

typedef int (*cmdhandler_t)(cmdmap_t &map, uint64_t seqno, vxstate_t&, string&);

static int
fte_update_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		uint64_t mac, expire;
		char *ip;
		int gen;

		if (cmdmap_get_num(map, "mac", mac))
				return EINVAL;
		if (cmdmap_get_num(map, "expire", expire))
				return EINVAL;
		if (cmdmap_get_str(map, "raddr", &ip))
				return EINVAL;
		auto fe = vxlan_ftable_entry(ip, expire);
		auto it = state.vs_ftable.find(mac);
		if (it == state.vs_ftable.end())
				gen = 0;
		else
				gen = it->second.vfe_gen + 1;
		state.vs_ftable.insert(fwdent(mac, fe));

		return 0;
}

static int
fte_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		result = default_result(seqno);
		return 0;
}

static int
fte_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
fte_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_set_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_del_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_set_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_del_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_update_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
vm_vni_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
route_update_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
route_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

typedef struct _ent {
		const char *ent;
		cmdhandler_t handler;
		enum verb action;
		int pad0;
} ent_t;

#define KEYENT(action, handler) {#action, handler, action, 0}

static ent_t ent_list[] = {
		KEYENT(VERB_UPDATE_FTE, fte_update_handler),
		KEYENT(VERB_REMOVE_FTE, fte_remove_handler),
		KEYENT(VERB_GET_FTE, fte_get_handler),
		KEYENT(VERB_GET_ALL_FTE, fte_get_all_handler),

		KEYENT(VERB_GET_PHYS_ND, nd_phys_get_handler),
		KEYENT(VERB_SET_PHYS_ND, nd_phys_set_handler),
		KEYENT(VERB_DEL_PHYS_ND, nd_phys_del_handler),
		KEYENT(VERB_GET_ALL_PHYS_ND, nd_phys_get_all_handler),

		KEYENT(VERB_UPDATE_VM_VNI, vm_vni_update_handler),
		KEYENT(VERB_REMOVE_VM_VNI, vm_vni_remove_handler),
		KEYENT(VERB_GET_VM_VNI, vm_vni_get_handler),
		KEYENT(VERB_GET_ALL_VM_VNI, vm_vni_get_all_handler),

		KEYENT(VERB_GET_VX_ND, nd_vx_get_handler),
		KEYENT(VERB_SET_VX_ND, nd_vx_set_handler),
		KEYENT(VERB_DEL_VX_ND, nd_vx_del_handler),
		KEYENT(VERB_GET_ALL_VX_ND, nd_vx_get_all_handler),

		KEYENT(VERB_UPDATE_DEFAULT_ROUTE, route_update_handler),
		KEYENT(VERB_REMOVE_DEFAULT_ROUTE, route_remove_handler),

		KEYENT(VERB_BAD, NULL) /* must be last */
};

static void
gather_args(cmdmap_t &map, char *input)
{
		char *indexp, *k, *tmp;
		const char *delim = " ";
		const char *kvdelim = ":";

		while (input != NULL) {
				indexp = strsep(&input, delim);
				k = strsep(&indexp, kvdelim);
				/* badly formed K:V pair */
				if (indexp == NULL)
						continue;
				/* STRING */
				if (indexp[0] == '"') {
						indexp++;
						if ((tmp = index(indexp, '"')) == NULL)
								continue;
						*tmp = '\0';
						map.insert(cmdent(string(k), parse_value(indexp)));
				}
				/* NUMBER */
				else if (indexp[0] == '0' && indexp[1] == 'x') {
						uint64_t v;
						indexp += 2;
						v = static_cast<uint64_t>(strtoll(indexp, NULL, 16));
						map.insert(cmdent(string(k), parse_value(v)));

				}
				/* UNKNOWN */
				else
						continue;
		}
}

int
parse_input(char *input, struct vxlan_state &state, string &result)
{
		cmdmap_t map;
		const char *delim = " ";
		const char *kvdelim = ":";
		char *indexp, *verbstr;
		ent_t *verbent = ent_list;
		enum verb verb;
		uint64_t seqno;

		indexp = strsep(&input, delim);
		verbstr = strsep(&indexp, kvdelim);
		if (indexp == NULL || indexp[0] != '0' || indexp[1] != 'x')
				return EINVAL;
		indexp += 2;
		seqno = static_cast<uint64_t>(strtoll(indexp, NULL, 16));

		for (verbent = ent_list; verbent->action != VERB_BAD; verbent++) {
				if (!strcmp(verbent->ent, verbstr))
						break;
		}
		verb = verbent->action;
		if (verb == VERB_BAD)
				return EINVAL;
		gather_args(map, input);
		return verbent->handler(map, seqno, state, result);
}

<commit_msg>finish update<commit_after>#include <sys/types.h>
#include <sys/endian.h>
#include <stdio.h>
#include <iostream>
#include <map>
#include <string>

#include <sys/socket.h>
#include <arpa/inet.h>
#include "uvxbridge.h"

static int
cmdmap_get_num(cmdmap_t &map, const char *key, uint64_t &val)
{
		auto it = map.find(string(key));

		if (it == map.end())
				return EINVAL;
		if (it->second.tag != TAG_NUMERIC)
				return EINVAL;
		val = it->second.value.numeric;

		return 0;
}

static int
cmdmap_get_str(cmdmap_t &map, const char *key, char **val)
{
		auto it = map.find(string(key));

		if (it == map.end())
				return EINVAL;
		if (it->second.tag != TAG_STRING)
				return EINVAL;
		*val = it->second.value.text;

		return 0;
}

static string
default_result(uint64_t seqno)
{
		char buf[32];
		snprintf(buf, 32, "(result:0x%lX)", seqno);
		return string(buf);
}
parse_value::parse_value(uint64_t numeric)
{
		this->tag = TAG_NUMERIC;
		this->value.numeric = numeric;
}

parse_value::parse_value(char *text)
{
		this->tag = TAG_STRING;
		this->value.text = strdup(text);
}

parse_value::~parse_value()
{
		switch(this->tag) {
		case TAG_STRING:
				free(this->value.text);
				break;
		case TAG_NUMERIC:
				break;
		}
}

typedef int (*cmdhandler_t)(cmdmap_t &map, uint64_t seqno, vxstate_t&, string&);

static int
fte_update_handler(cmdmap_t &map, uint64_t seqno, vxstate_t &state, string &result)
{
		uint64_t mac, expire;
		char *ip;
		int gen;

		if (cmdmap_get_num(map, "mac", mac))
				return EINVAL;
		if (cmdmap_get_num(map, "expire", expire))
				return EINVAL;
		if (cmdmap_get_str(map, "raddr", &ip))
				return EINVAL;
		auto it = state.vs_ftable.find(mac);
		if (it == state.vs_ftable.end()) {
				auto fe = vxlan_ftable_entry(ip, expire);
				state.vs_ftable.insert(fwdent(mac, fe));
		} else {
				auto fe = &it->second;
				int is_v6 = (index(ip, ':') != NULL);
				fe->vfe_gen++;
				fe->vfe_expire = expire;
				if (is_v6) {
						fe->vfe_v6 = 1;
						if (inet_pton(AF_INET6, ip, &fe->vfe_raddr.in6)) {
								state.vs_ftable.erase(mac);
								return EINVAL;
						}
				} else {
						fe->vfe_v6 = 0;
						if (inet_aton(ip, &fe->vfe_raddr.in4)) {
								state.vs_ftable.erase(mac);
								return EINVAL;
						}
				}
		}
		return 0;
}

static int
fte_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		result = default_result(seqno);
		return 0;
}

static int
fte_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
fte_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_set_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_del_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_phys_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_set_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_del_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
nd_vx_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_update_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{

		return 0;
}

static int
vm_vni_get_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
vm_vni_get_all_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
route_update_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

static int
route_remove_handler(cmdmap_t &map __unused, uint64_t seqno, vxstate_t &state, string &result)
{
		return 0;
}

typedef struct _ent {
		const char *ent;
		cmdhandler_t handler;
		enum verb action;
		int pad0;
} ent_t;

#define KEYENT(action, handler) {#action, handler, action, 0}

static ent_t ent_list[] = {
		KEYENT(VERB_UPDATE_FTE, fte_update_handler),
		KEYENT(VERB_REMOVE_FTE, fte_remove_handler),
		KEYENT(VERB_GET_FTE, fte_get_handler),
		KEYENT(VERB_GET_ALL_FTE, fte_get_all_handler),

		KEYENT(VERB_GET_PHYS_ND, nd_phys_get_handler),
		KEYENT(VERB_SET_PHYS_ND, nd_phys_set_handler),
		KEYENT(VERB_DEL_PHYS_ND, nd_phys_del_handler),
		KEYENT(VERB_GET_ALL_PHYS_ND, nd_phys_get_all_handler),

		KEYENT(VERB_UPDATE_VM_VNI, vm_vni_update_handler),
		KEYENT(VERB_REMOVE_VM_VNI, vm_vni_remove_handler),
		KEYENT(VERB_GET_VM_VNI, vm_vni_get_handler),
		KEYENT(VERB_GET_ALL_VM_VNI, vm_vni_get_all_handler),

		KEYENT(VERB_GET_VX_ND, nd_vx_get_handler),
		KEYENT(VERB_SET_VX_ND, nd_vx_set_handler),
		KEYENT(VERB_DEL_VX_ND, nd_vx_del_handler),
		KEYENT(VERB_GET_ALL_VX_ND, nd_vx_get_all_handler),

		KEYENT(VERB_UPDATE_DEFAULT_ROUTE, route_update_handler),
		KEYENT(VERB_REMOVE_DEFAULT_ROUTE, route_remove_handler),

		KEYENT(VERB_BAD, NULL) /* must be last */
};

static void
gather_args(cmdmap_t &map, char *input)
{
		char *indexp, *k, *tmp;
		const char *delim = " ";
		const char *kvdelim = ":";

		while (input != NULL) {
				indexp = strsep(&input, delim);
				k = strsep(&indexp, kvdelim);
				/* badly formed K:V pair */
				if (indexp == NULL)
						continue;
				/* STRING */
				if (indexp[0] == '"') {
						indexp++;
						if ((tmp = index(indexp, '"')) == NULL)
								continue;
						*tmp = '\0';
						map.insert(cmdent(string(k), parse_value(indexp)));
				}
				/* NUMBER */
				else if (indexp[0] == '0' && indexp[1] == 'x') {
						uint64_t v;
						indexp += 2;
						v = static_cast<uint64_t>(strtoll(indexp, NULL, 16));
						map.insert(cmdent(string(k), parse_value(v)));

				}
				/* UNKNOWN */
				else
						continue;
		}
}

int
parse_input(char *input, struct vxlan_state &state, string &result)
{
		cmdmap_t map;
		const char *delim = " ";
		const char *kvdelim = ":";
		char *indexp, *verbstr;
		ent_t *verbent = ent_list;
		enum verb verb;
		uint64_t seqno;

		indexp = strsep(&input, delim);
		verbstr = strsep(&indexp, kvdelim);
		if (indexp == NULL || indexp[0] != '0' || indexp[1] != 'x')
				return EINVAL;
		indexp += 2;
		seqno = static_cast<uint64_t>(strtoll(indexp, NULL, 16));

		for (verbent = ent_list; verbent->action != VERB_BAD; verbent++) {
				if (!strcmp(verbent->ent, verbstr))
						break;
		}
		verb = verbent->action;
		if (verb == VERB_BAD)
				return EINVAL;
		gather_args(map, input);
		return verbent->handler(map, seqno, state, result);
}

<|endoftext|>


<commit_before>#include <karl/storage.hpp>

#include <karl/util/guard.hpp>

namespace supermarx
{

storage::storage(std::string const& embedded_dir)
{
	const char *server_options[] = \
	{"test", "--innodb=OFF", "-h", embedded_dir.c_str(), NULL};
	int num_options = (sizeof(server_options)/sizeof(char*)) - 1;
	mysql_library_init(num_options, const_cast<char**>(server_options), NULL);

	{
		auto db = std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo());

		auto result = db->select_query("show databases");
		bool found_karl = false;
		while(result){
			if(result.get<std::string>("Database") == "karl"){
				found_karl = true;
			}
			result.next();
		}
		if(!found_karl){
			db->execute("create database karl");
		}
	}

	database = std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo("karl"));

	update_database_schema();
	prepare_statements();
}

storage::storage(const std::string &host, const std::string &user, const std::string &password, const std::string& db)
	: database(std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo(host, 3306, user, password, db)))
	, prepared_statements()
{
	update_database_schema();
	prepare_statements();
}

storage::~storage()
{
	prepared_statements.clear();
	database.reset();
	mysql_library_end();
}

void storage::lock_products_read()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("lock tables product read");
}

void
storage::lock_products_write()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("lock tables product write");
}

void storage::unlock_all()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("unlock tables");
}

boost::optional<id_t> storage::find_product_unsafe(std::string const& identifier, id_t supermarket_id)
{
	auto& qselect = *prepared_statements.at(statement::get_product_by_identifier);
	qselect.execute(identifier, supermarket_id);

	id_t product_id;
	qselect.bind_results(product_id);

	if(qselect.fetch())
		return product_id;

	return boost::none;
}

boost::optional<std::pair<id_t, product>> storage::fetch_last_productdetails_unsafe(id_t product_id)
{
	auto& qolder = *prepared_statements.at(statement::get_last_productdetails_by_product);
	qolder.execute(product_id);

	id_t id;
	product p;
	std::string raw_discount_condition, raw_valid_on, raw_retrieved_on;

	qolder.bind_results(id, p.identifier, p.name, p.orig_price, p.price, raw_discount_condition, raw_valid_on, raw_retrieved_on);
	if(!qolder.fetch())
		return boost::none;

	p.discount_condition = to_condition(raw_discount_condition);
	p.valid_on = to_date(raw_valid_on);
	p.retrieved_on = to_datetime(raw_retrieved_on);

	return std::make_pair(id, p);
}

id_t storage::find_add_product(std::string const& identifier, id_t supermarket_id)
{
	{
		lock_products_read();
		guard unlocker([&]() {
			unlock_all();
		});

		auto product_id = find_product_unsafe(identifier, supermarket_id);
		if(product_id)
			return product_id.get();
	}

	{
		lock_products_write();
		guard unlocker([&]() {
			unlock_all();
		});

		auto product_id = find_product_unsafe(identifier, supermarket_id);
		if(product_id)
			return product_id.get();

		auto& qinsert = *prepared_statements.at(statement::add_product);
		qinsert.execute(identifier, supermarket_id);
		return qinsert.insert_id();
	}
}

void storage::register_productdetailsrecord(id_t productdetails_id, datetime retrieved_on)
{
	auto& q = *prepared_statements.at(statement::add_productdetailsrecord);
	q.execute(productdetails_id, to_string(retrieved_on));
}

void storage::add_product(product const& p, id_t supermarket_id)
{
	id_t product_id = find_add_product(p.identifier, supermarket_id);

	// TODO Does not need locks, but does require a transaction

	// Check if an older version of the product exactly matches what we've got
	auto p_old_opt_kvp = fetch_last_productdetails_unsafe(product_id);
	if(p_old_opt_kvp)
	{
		product const& p_old = p_old_opt_kvp->second;
		bool similar = (
			p.discount_condition == p_old.discount_condition &&
			p.name == p_old.name &&
			p.orig_price == p_old.orig_price &&
			p.price == p_old.price
		);

		if(similar)
		{
			register_productdetailsrecord(p_old_opt_kvp->first, p.retrieved_on);
			return;
		}
		else
		{
			auto& qinvalidate = *prepared_statements.at(statement::invalidate_productdetails);
			qinvalidate.execute(to_string(p.valid_on), product_id);
		}
	}

	//TODO check if a newer entry is already entered. Perhaps invalidate that entry in such a case.

	auto& qadd = *prepared_statements.at(statement::add_productdetails);
	qadd.execute(product_id, p.name, p.orig_price, p.price, to_string(p.discount_condition), to_string(p.valid_on), to_string(p.retrieved_on));
	id_t productdetails_id = qadd.insert_id();
	std::cerr << "Inserted new productdetails " << productdetails_id << " for product " << p.identifier << " [" << product_id << ']' << std::endl;

	register_productdetailsrecord(productdetails_id, p.retrieved_on);
}

std::vector<product> storage::get_products_by_name(std::string const& name, id_t supermarket_id)
{
	lock_products_read();
	guard unlocker([&]() {
		unlock_all();
	});

	rusql::PreparedStatement& query = *prepared_statements.at(statement::get_last_productdetails_by_name);
	query.execute(std::string("%") + name + "%", supermarket_id);

	std::vector<product> products;
	product row;
	std::string raw_discount_condition, raw_valid_on, raw_retrieved_on;

	query.bind_results(row.identifier, row.name, row.orig_price, row.price, raw_discount_condition, raw_valid_on, raw_retrieved_on);
	while(query.fetch())
	{
		row.discount_condition = to_condition(raw_discount_condition);
		row.valid_on = to_date(raw_valid_on);
		row.retrieved_on = to_datetime(raw_retrieved_on);

		products.push_back(row);
	}

	return products;
}

void storage::update_database_schema()
{
	bool product_found = false;
	bool productdetails_found = false;
	bool productdetailsrecord_found = false;
	bool supermarket_found = false;

	auto result = database->select_query("show tables");
	while(result)
	{
		std::string tablename = result.get<std::string>(0);
		if(tablename == "product")
			product_found = true;
		else if(tablename == "productdetails")
			productdetails_found = true;
		else if(tablename == "productdetailsrecord")
			productdetailsrecord_found = true;
		else if(tablename == "supermarket")
			supermarket_found = true;

		result.next();
	}

	if(!product_found)
		database->execute(std::string() +
			"create table `product` ("+
			"`id` int(11) unsigned not null auto_increment,"+
			"`identifier` varchar(1024) not null,"+
			"`supermarket_id` int(11) unsigned null,"+
			"primary key (`id`),"+
			"key `supermarket_id` (`supermarket_id`)"+
			") character set utf8 collate utf8_bin");

	if(!productdetails_found)
		database->execute(std::string() +
			"create table `productdetails` (" +
			"`id` int(10) unsigned not null auto_increment," +
			"`product_id` int(10) unsigned not null," +
			"`name` varchar(1024) character set utf8 not null," +
			"`orig_price` int not null," +
			"`price` int not null," +
			"`discount_condition` enum('ALWAYS','AT_TWO','AT_THREE') not null," +
			"`valid_on` date not null," +
			"`valid_until` date," +
			"`retrieved_on` datetime not null," +
			"primary key (`id`)," +
			"key `product_id` (`product_id`)" +
			") character set utf8 collate utf8_bin");

	if(!productdetailsrecord_found)
		database->execute(std::string() +
			"create table `productdetailsrecord` (" +
			"`id` int(10) unsigned not null auto_increment," +
			"`productdetails_id` int(10) unsigned not null," +
			"`retrieved_on` datetime not null," +
			"primary key (`id`)," +
			"key `productdetails_id` (`productdetails_id`)" +
			") character set utf8 collate utf8_bin");

	if(!supermarket_found)
		database->execute(std::string() +
			"create table `supermarket` (" +
			"`id` int(11) unsigned not null auto_increment," +
			"`name` varchar(1024) not null," +
			"primary key (`id`)" +
			") character set utf8 collate utf8_bin");
}

void storage::prepare_statements()
{
	auto create_statement =
			[&](statement const& s, std::string const& query)
	{
		prepared_statements[s] = std::make_shared<rusql::PreparedStatement>(database->prepare(query));
	};

	database->query("SET NAMES 'utf8'");
	database->query("SET CHARACTER SET utf8");

	create_statement(statement::add_product, "insert into product (identifier, supermarket_id) values (?, ?)");
	create_statement(statement::get_product_by_identifier, "select product.id from product where product.identifier = ? and product.supermarket_id = ?");

	create_statement(statement::add_productdetails, "insert into productdetails (product_id, name, orig_price, price, discount_condition, valid_on, valid_until, retrieved_on) values(?, ?, ?, ?, ?, ?, NULL, ?)");
	create_statement(statement::add_productdetailsrecord, "insert into productdetailsrecord (productdetails_id, retrieved_on) values(?, ?)");

	create_statement(statement::get_last_productdetails_by_product, "select productdetails.id, product.identifier, productdetails.name, productdetails.orig_price, productdetails.price, productdetails.discount_condition, productdetails.valid_on, productdetails.retrieved_on from product inner join productdetails on (product.id = productdetails.product_id) where productdetails.product_id = ? AND productdetails.valid_until is NULL");
	create_statement(statement::get_last_productdetails_by_name, "select product.identifier, productdetails.name, productdetails.orig_price, productdetails.price, productdetails.discount_condition, productdetails.valid_on, productdetails.retrieved_on from product inner join productdetails on (product.id = productdetails.product_id) where productdetails.name LIKE ? AND productdetails.valid_until is NULL AND product.supermarket_id = ?");
	create_statement(statement::invalidate_productdetails, "update productdetails set productdetails.valid_until = ? where productdetails.valid_until is null and productdetails.product_id = ?");
}

}
<commit_msg>Added ci collation<commit_after>#include <karl/storage.hpp>

#include <karl/util/guard.hpp>

namespace supermarx
{

storage::storage(std::string const& embedded_dir)
{
	const char *server_options[] = \
	{"test", "--innodb=OFF", "-h", embedded_dir.c_str(), NULL};
	int num_options = (sizeof(server_options)/sizeof(char*)) - 1;
	mysql_library_init(num_options, const_cast<char**>(server_options), NULL);

	{
		auto db = std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo());

		auto result = db->select_query("show databases");
		bool found_karl = false;
		while(result){
			if(result.get<std::string>("Database") == "karl"){
				found_karl = true;
			}
			result.next();
		}
		if(!found_karl){
			db->execute("create database karl");
		}
	}

	database = std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo("karl"));

	update_database_schema();
	prepare_statements();
}

storage::storage(const std::string &host, const std::string &user, const std::string &password, const std::string& db)
	: database(std::make_shared<rusql::Database>(rusql::Database::ConstructionInfo(host, 3306, user, password, db)))
	, prepared_statements()
{
	update_database_schema();
	prepare_statements();
}

storage::~storage()
{
	prepared_statements.clear();
	database.reset();
	mysql_library_end();
}

void storage::lock_products_read()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("lock tables product read");
}

void
storage::lock_products_write()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("lock tables product write");
}

void storage::unlock_all()
{
	// LibRUSQL does not support grabbing a specific connection (yet), do not lock for now
	//database->query("unlock tables");
}

boost::optional<id_t> storage::find_product_unsafe(std::string const& identifier, id_t supermarket_id)
{
	auto& qselect = *prepared_statements.at(statement::get_product_by_identifier);
	qselect.execute(identifier, supermarket_id);

	id_t product_id;
	qselect.bind_results(product_id);

	if(qselect.fetch())
		return product_id;

	return boost::none;
}

boost::optional<std::pair<id_t, product>> storage::fetch_last_productdetails_unsafe(id_t product_id)
{
	auto& qolder = *prepared_statements.at(statement::get_last_productdetails_by_product);
	qolder.execute(product_id);

	id_t id;
	product p;
	std::string raw_discount_condition, raw_valid_on, raw_retrieved_on;

	qolder.bind_results(id, p.identifier, p.name, p.orig_price, p.price, raw_discount_condition, raw_valid_on, raw_retrieved_on);
	if(!qolder.fetch())
		return boost::none;

	p.discount_condition = to_condition(raw_discount_condition);
	p.valid_on = to_date(raw_valid_on);
	p.retrieved_on = to_datetime(raw_retrieved_on);

	return std::make_pair(id, p);
}

id_t storage::find_add_product(std::string const& identifier, id_t supermarket_id)
{
	{
		lock_products_read();
		guard unlocker([&]() {
			unlock_all();
		});

		auto product_id = find_product_unsafe(identifier, supermarket_id);
		if(product_id)
			return product_id.get();
	}

	{
		lock_products_write();
		guard unlocker([&]() {
			unlock_all();
		});

		auto product_id = find_product_unsafe(identifier, supermarket_id);
		if(product_id)
			return product_id.get();

		auto& qinsert = *prepared_statements.at(statement::add_product);
		qinsert.execute(identifier, supermarket_id);
		return qinsert.insert_id();
	}
}

void storage::register_productdetailsrecord(id_t productdetails_id, datetime retrieved_on)
{
	auto& q = *prepared_statements.at(statement::add_productdetailsrecord);
	q.execute(productdetails_id, to_string(retrieved_on));
}

void storage::add_product(product const& p, id_t supermarket_id)
{
	id_t product_id = find_add_product(p.identifier, supermarket_id);

	// TODO Does not need locks, but does require a transaction

	// Check if an older version of the product exactly matches what we've got
	auto p_old_opt_kvp = fetch_last_productdetails_unsafe(product_id);
	if(p_old_opt_kvp)
	{
		product const& p_old = p_old_opt_kvp->second;
		bool similar = (
			p.discount_condition == p_old.discount_condition &&
			p.name == p_old.name &&
			p.orig_price == p_old.orig_price &&
			p.price == p_old.price
		);

		if(similar)
		{
			register_productdetailsrecord(p_old_opt_kvp->first, p.retrieved_on);
			return;
		}
		else
		{
			auto& qinvalidate = *prepared_statements.at(statement::invalidate_productdetails);
			qinvalidate.execute(to_string(p.valid_on), product_id);
		}
	}

	//TODO check if a newer entry is already entered. Perhaps invalidate that entry in such a case.

	auto& qadd = *prepared_statements.at(statement::add_productdetails);
	qadd.execute(product_id, p.name, p.orig_price, p.price, to_string(p.discount_condition), to_string(p.valid_on), to_string(p.retrieved_on));
	id_t productdetails_id = qadd.insert_id();
	std::cerr << "Inserted new productdetails " << productdetails_id << " for product " << p.identifier << " [" << product_id << ']' << std::endl;

	register_productdetailsrecord(productdetails_id, p.retrieved_on);
}

std::vector<product> storage::get_products_by_name(std::string const& name, id_t supermarket_id)
{
	lock_products_read();
	guard unlocker([&]() {
		unlock_all();
	});

	rusql::PreparedStatement& query = *prepared_statements.at(statement::get_last_productdetails_by_name);
	query.execute(std::string("%") + name + "%", supermarket_id);

	std::vector<product> products;
	product row;
	std::string raw_discount_condition, raw_valid_on, raw_retrieved_on;

	query.bind_results(row.identifier, row.name, row.orig_price, row.price, raw_discount_condition, raw_valid_on, raw_retrieved_on);
	while(query.fetch())
	{
		row.discount_condition = to_condition(raw_discount_condition);
		row.valid_on = to_date(raw_valid_on);
		row.retrieved_on = to_datetime(raw_retrieved_on);

		products.push_back(row);
	}

	return products;
}

void storage::update_database_schema()
{
	bool product_found = false;
	bool productdetails_found = false;
	bool productdetailsrecord_found = false;
	bool supermarket_found = false;

	auto result = database->select_query("show tables");
	while(result)
	{
		std::string tablename = result.get<std::string>(0);
		if(tablename == "product")
			product_found = true;
		else if(tablename == "productdetails")
			productdetails_found = true;
		else if(tablename == "productdetailsrecord")
			productdetailsrecord_found = true;
		else if(tablename == "supermarket")
			supermarket_found = true;

		result.next();
	}

	if(!product_found)
		database->execute(std::string() +
			"create table `product` ("+
			"`id` int(11) unsigned not null auto_increment,"+
			"`identifier` varchar(1024) not null,"+
			"`supermarket_id` int(11) unsigned null,"+
			"primary key (`id`),"+
			"key `supermarket_id` (`supermarket_id`)"+
			") character set utf8 collate utf8_bin");

	if(!productdetails_found)
		database->execute(std::string() +
			"create table `productdetails` (" +
			"`id` int(10) unsigned not null auto_increment," +
			"`product_id` int(10) unsigned not null," +
			"`name` varchar(1024) character set utf8 not null," +
			"`orig_price` int not null," +
			"`price` int not null," +
			"`discount_condition` enum('ALWAYS','AT_TWO','AT_THREE') not null," +
			"`valid_on` date not null," +
			"`valid_until` date," +
			"`retrieved_on` datetime not null," +
			"primary key (`id`)," +
			"key `product_id` (`product_id`)" +
			") character set utf8 collate utf8_bin");

	if(!productdetailsrecord_found)
		database->execute(std::string() +
			"create table `productdetailsrecord` (" +
			"`id` int(10) unsigned not null auto_increment," +
			"`productdetails_id` int(10) unsigned not null," +
			"`retrieved_on` datetime not null," +
			"primary key (`id`)," +
			"key `productdetails_id` (`productdetails_id`)" +
			") character set utf8 collate utf8_bin");

	if(!supermarket_found)
		database->execute(std::string() +
			"create table `supermarket` (" +
			"`id` int(11) unsigned not null auto_increment," +
			"`name` varchar(1024) not null," +
			"primary key (`id`)" +
			") character set utf8 collate utf8_bin");
}

void storage::prepare_statements()
{
	auto create_statement =
			[&](statement const& s, std::string const& query)
	{
		prepared_statements[s] = std::make_shared<rusql::PreparedStatement>(database->prepare(query));
	};

	database->query("SET NAMES 'utf8'");
	database->query("SET CHARACTER SET utf8");

	create_statement(statement::add_product, "insert into product (identifier, supermarket_id) values (?, ?)");
	create_statement(statement::get_product_by_identifier, "select product.id from product where product.identifier = ? and product.supermarket_id = ?");

	create_statement(statement::add_productdetails, "insert into productdetails (product_id, name, orig_price, price, discount_condition, valid_on, valid_until, retrieved_on) values(?, ?, ?, ?, ?, ?, NULL, ?)");
	create_statement(statement::add_productdetailsrecord, "insert into productdetailsrecord (productdetails_id, retrieved_on) values(?, ?)");

	create_statement(statement::get_last_productdetails_by_product, "select productdetails.id, product.identifier, productdetails.name, productdetails.orig_price, productdetails.price, productdetails.discount_condition, productdetails.valid_on, productdetails.retrieved_on from product inner join productdetails on (product.id = productdetails.product_id) where productdetails.product_id = ? AND productdetails.valid_until is NULL");
	create_statement(statement::get_last_productdetails_by_name, "select product.identifier, productdetails.name, productdetails.orig_price, productdetails.price, productdetails.discount_condition, productdetails.valid_on, productdetails.retrieved_on from product inner join productdetails on (product.id = productdetails.product_id) where productdetails.name like ? collate utf8_general_ci AND productdetails.valid_until is NULL AND product.supermarket_id = ?");
	create_statement(statement::invalidate_productdetails, "update productdetails set productdetails.valid_until = ? where productdetails.valid_until is null and productdetails.product_id = ?");
}

}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "timedata.h"

#include "netbase.h"
#include "sync.h"
#include "ui_interface.h"
#include "util.h"
#include "utilstrencodings.h"

#include <boost/foreach.hpp>

using namespace std;

static CCriticalSection cs_nTimeOffset;
static int64_t nTimeOffset = 0;

/**
 * "Never go to sea with two chronometers; take one or three."
 * Our three time sources are:
 *  - System clock
 *  - Median of other nodes clocks
 *  - The user (asking the user to fix the system clock if the first two disagree)
 */
int64_t GetTimeOffset()
{
    LOCK(cs_nTimeOffset);
    return nTimeOffset;
}

int64_t GetAdjustedTime()
{
    return GetTime() + GetTimeOffset();
}

static int64_t abs64(int64_t n)
{
    return (n >= 0 ? n : -n);
}

void AddTimeData(const CNetAddr& ip, int64_t nTime)
{
    int64_t nOffsetSample = nTime - GetTime();

    LOCK(cs_nTimeOffset);
    // Ignore duplicates
    static set<CNetAddr> setKnown;
    if (!setKnown.insert(ip).second)
        return;

    // Add data
    static CMedianFilter<int64_t> vTimeOffsets(200,0);
    vTimeOffsets.input(nOffsetSample);
    LogPrintf("Added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60);

    // There is a known issue here (see issue #4521):
    //
    // - The structure vTimeOffsets contains up to 200 elements, after which
    // any new element added to it will not increase its size, replacing the
    // oldest element.
    //
    // - The condition to update nTimeOffset includes checking whether the
    // number of elements in vTimeOffsets is odd, which will never happen after
    // there are 200 elements.
    //
    // But in this case the 'bug' is protective against some attacks, and may
    // actually explain why we've never seen attacks which manipulate the
    // clock offset.
    //
    // So we should hold off on fixing this and clean it up as part of
    // a timing cleanup that strengthens it in a number of other ways.
    //
    if (vTimeOffsets.size() >= 5 && vTimeOffsets.size() % 2 == 1)
    {
        int64_t nMedian = vTimeOffsets.median();
        std::vector<int64_t> vSorted = vTimeOffsets.sorted();
        // Only let other nodes change our time by so much
        if (abs64(nMedian) < 70 * 60)
        {
            nTimeOffset = nMedian;
        }
        else
        {
            nTimeOffset = 0;

            static bool fDone;
            if (!fDone)
            {
                // If nobody has a time different than ours but within 5 minutes of ours, give a warning
                bool fMatch = false;
                BOOST_FOREACH(int64_t nOffset, vSorted)
                    if (nOffset != 0 && abs64(nOffset) < 5 * 60)
                        fMatch = true;

                if (!fMatch)
                {
                    fDone = true;
                    string strMessage = _("Warning: Please check that your computer's date and time are correct! If your clock is wrong Bitcoin Core will not work properly.");
                    strMiscWarning = strMessage;
                    LogPrintf("*** %s\n", strMessage);
                    uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING);
                }
            }
        }
        if (fDebug) {
            BOOST_FOREACH(int64_t n, vSorted)
                LogPrintf("%+d  ", n);
            LogPrintf("|  ");
        }
        LogPrintf("nTimeOffset = %+d  (%+d minutes)\n", nTimeOffset, nTimeOffset/60);
    }
}
<commit_msg>Do not store more than 200 timedata samples.<commit_after>// Copyright (c) 2014 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "timedata.h"

#include "netbase.h"
#include "sync.h"
#include "ui_interface.h"
#include "util.h"
#include "utilstrencodings.h"

#include <boost/foreach.hpp>

using namespace std;

static CCriticalSection cs_nTimeOffset;
static int64_t nTimeOffset = 0;

/**
 * "Never go to sea with two chronometers; take one or three."
 * Our three time sources are:
 *  - System clock
 *  - Median of other nodes clocks
 *  - The user (asking the user to fix the system clock if the first two disagree)
 */
int64_t GetTimeOffset()
{
    LOCK(cs_nTimeOffset);
    return nTimeOffset;
}

int64_t GetAdjustedTime()
{
    return GetTime() + GetTimeOffset();
}

static int64_t abs64(int64_t n)
{
    return (n >= 0 ? n : -n);
}

#define BITCOIN_TIMEDATA_MAX_SAMPLES 200

void AddTimeData(const CNetAddr& ip, int64_t nTime)
{
    int64_t nOffsetSample = nTime - GetTime();

    LOCK(cs_nTimeOffset);
    // Ignore duplicates
    static set<CNetAddr> setKnown;
    if (setKnown.size() == BITCOIN_TIMEDATA_MAX_SAMPLES)
        return;
    if (!setKnown.insert(ip).second)
        return;

    // Add data
    static CMedianFilter<int64_t> vTimeOffsets(BITCOIN_TIMEDATA_MAX_SAMPLES, 0);
    vTimeOffsets.input(nOffsetSample);
    LogPrintf("Added time data, samples %d, offset %+d (%+d minutes)\n", vTimeOffsets.size(), nOffsetSample, nOffsetSample/60);

    // There is a known issue here (see issue #4521):
    //
    // - The structure vTimeOffsets contains up to 200 elements, after which
    // any new element added to it will not increase its size, replacing the
    // oldest element.
    //
    // - The condition to update nTimeOffset includes checking whether the
    // number of elements in vTimeOffsets is odd, which will never happen after
    // there are 200 elements.
    //
    // But in this case the 'bug' is protective against some attacks, and may
    // actually explain why we've never seen attacks which manipulate the
    // clock offset.
    //
    // So we should hold off on fixing this and clean it up as part of
    // a timing cleanup that strengthens it in a number of other ways.
    //
    if (vTimeOffsets.size() >= 5 && vTimeOffsets.size() % 2 == 1)
    {
        int64_t nMedian = vTimeOffsets.median();
        std::vector<int64_t> vSorted = vTimeOffsets.sorted();
        // Only let other nodes change our time by so much
        if (abs64(nMedian) < 70 * 60)
        {
            nTimeOffset = nMedian;
        }
        else
        {
            nTimeOffset = 0;

            static bool fDone;
            if (!fDone)
            {
                // If nobody has a time different than ours but within 5 minutes of ours, give a warning
                bool fMatch = false;
                BOOST_FOREACH(int64_t nOffset, vSorted)
                    if (nOffset != 0 && abs64(nOffset) < 5 * 60)
                        fMatch = true;

                if (!fMatch)
                {
                    fDone = true;
                    string strMessage = _("Warning: Please check that your computer's date and time are correct! If your clock is wrong Bitcoin Core will not work properly.");
                    strMiscWarning = strMessage;
                    LogPrintf("*** %s\n", strMessage);
                    uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING);
                }
            }
        }
        if (fDebug) {
            BOOST_FOREACH(int64_t n, vSorted)
                LogPrintf("%+d  ", n);
            LogPrintf("|  ");
        }
        LogPrintf("nTimeOffset = %+d  (%+d minutes)\n", nTimeOffset, nTimeOffset/60);
    }
}
<|endoftext|>


<commit_before>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Log$
 * Revision 1.7  2003/12/17 00:18:41  cargilld
 * Update to memory management so that the static memory manager (one used to call Initialize) is only for static data.
 *
 * Revision 1.6  2003/12/16 18:41:15  knoaman
 * Make IC_Field stateless
 *
 * Revision 1.5  2003/05/22 02:10:52  knoaman
 * Default the memory manager.
 *
 * Revision 1.4  2003/05/15 18:59:34  knoaman
 * Partial implementation of the configurable memory manager.
 *
 * Revision 1.3  2002/11/04 14:47:41  tng
 * C++ Namespace Support.
 *
 * Revision 1.2  2002/08/27 05:56:19  knoaman
 * Identity Constraint: handle case of recursive elements.
 *
 * Revision 1.1.1.1  2002/02/01 22:22:50  peiyongz
 * sane_include
 *
 * Revision 1.1  2001/11/02 14:08:40  knoaman
 * Add support for identity constraints.
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/schema/identity/FieldActivator.hpp>
#include <xercesc/validators/schema/identity/ValueStore.hpp>
#include <xercesc/validators/schema/identity/ValueStoreCache.hpp>
#include <xercesc/validators/schema/identity/XPathMatcherStack.hpp>
#include <xercesc/util/HashPtr.hpp>

XERCES_CPP_NAMESPACE_BEGIN

// ---------------------------------------------------------------------------
//  FieldActivator: Constructors and Destructor
// ---------------------------------------------------------------------------
FieldActivator::FieldActivator(ValueStoreCache* const valueStoreCache,
                               XPathMatcherStack* const matcherStack,
                               MemoryManager* const manager)
    : fValueStoreCache(valueStoreCache)
    , fMatcherStack(matcherStack)
    , fMayMatch(0)
    , fMemoryManager(manager)
{
    fMayMatch = new (manager) ValueHashTableOf<bool>(29, new (manager) HashPtr(), manager);
}

FieldActivator::FieldActivator(const FieldActivator& other)
    : fValueStoreCache(other.fValueStoreCache)
    , fMatcherStack(other.fMatcherStack)
    , fMayMatch(0)
    , fMemoryManager(other.fMemoryManager)
{
    fMayMatch = new (fMemoryManager) ValueHashTableOf<bool>(29, new (fMemoryManager) HashPtr(), fMemoryManager);
    ValueHashTableOfEnumerator<bool> mayMatchEnum(other.fMayMatch, false, fMemoryManager);

    // Build key set
    while (mayMatchEnum.hasMoreElements())
    {
        IC_Field* field = (IC_Field*) mayMatchEnum.nextElementKey();
        fMayMatch->put(field, other.fMayMatch->get(field));
    }
}

FieldActivator::~FieldActivator()
{
}

// ---------------------------------------------------------------------------
//  FieldActivator: Operator methods
// ---------------------------------------------------------------------------
FieldActivator& FieldActivator::operator =(const FieldActivator& other) {

    if (this == &other) {
        return *this;
    }

    fValueStoreCache = other.fValueStoreCache;
    fMatcherStack = other.fMatcherStack;
    return *this;
}

// ---------------------------------------------------------------------------
//  FieldActivator: Operator methods
// ---------------------------------------------------------------------------
XPathMatcher* FieldActivator::activateField(IC_Field* const field, const int initialDepth) {

    ValueStore* valueStore = fValueStoreCache->getValueStoreFor(field, initialDepth);
    XPathMatcher* matcher = field->createMatcher(this, valueStore, fMemoryManager);

    setMayMatch(field, true);
    fMatcherStack->addMatcher(matcher);
    matcher->startDocumentFragment();

    return matcher;
}

void FieldActivator::startValueScopeFor(const IdentityConstraint* const ic,
                                        const int initialDepth) {

    unsigned int fieldCount = ic->getFieldCount();

    for(unsigned int i=0; i<fieldCount; i++) {

        const IC_Field* field = ic->getFieldAt(i);
        ValueStore* valueStore = fValueStoreCache->getValueStoreFor(field, initialDepth);

        valueStore->startValueScope();
    }
}

void FieldActivator::endValueScopeFor(const IdentityConstraint* const ic, const int initialDepth) {

    ValueStore* valueStore = fValueStoreCache->getValueStoreFor(ic, initialDepth);

    valueStore->endValueScope();
}

XERCES_CPP_NAMESPACE_END

/**
  * End of file FieldActivator.cpp
  */

<commit_msg>Fix memhandlertest failure (memory not deleted).<commit_after>/*
 * The Apache Software License, Version 1.1
 *
 * Copyright (c) 2001 The Apache Software Foundation.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The end-user documentation included with the redistribution,
 *    if any, must include the following acknowledgment:
 *       "This product includes software developed by the
 *        Apache Software Foundation (http://www.apache.org/)."
 *    Alternately, this acknowledgment may appear in the software itself,
 *    if and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Xerces" and "Apache Software Foundation" must
 *    not be used to endorse or promote products derived from this
 *    software without prior written permission. For written
 *    permission, please contact apache\@apache.org.
 *
 * 5. Products derived from this software may not be called "Apache",
 *    nor may "Apache" appear in their name, without prior written
 *    permission of the Apache Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation, and was
 * originally based on software copyright (c) 1999, International
 * Business Machines, Inc., http://www.ibm.com .  For more information
 * on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 */

/*
 * $Log$
 * Revision 1.8  2003/12/17 01:13:10  cargilld
 * Fix memhandlertest failure (memory not deleted).
 *
 * Revision 1.7  2003/12/17 00:18:41  cargilld
 * Update to memory management so that the static memory manager (one used to call Initialize) is only for static data.
 *
 * Revision 1.6  2003/12/16 18:41:15  knoaman
 * Make IC_Field stateless
 *
 * Revision 1.5  2003/05/22 02:10:52  knoaman
 * Default the memory manager.
 *
 * Revision 1.4  2003/05/15 18:59:34  knoaman
 * Partial implementation of the configurable memory manager.
 *
 * Revision 1.3  2002/11/04 14:47:41  tng
 * C++ Namespace Support.
 *
 * Revision 1.2  2002/08/27 05:56:19  knoaman
 * Identity Constraint: handle case of recursive elements.
 *
 * Revision 1.1.1.1  2002/02/01 22:22:50  peiyongz
 * sane_include
 *
 * Revision 1.1  2001/11/02 14:08:40  knoaman
 * Add support for identity constraints.
 *
 */

// ---------------------------------------------------------------------------
//  Includes
// ---------------------------------------------------------------------------
#include <xercesc/validators/schema/identity/FieldActivator.hpp>
#include <xercesc/validators/schema/identity/ValueStore.hpp>
#include <xercesc/validators/schema/identity/ValueStoreCache.hpp>
#include <xercesc/validators/schema/identity/XPathMatcherStack.hpp>
#include <xercesc/util/HashPtr.hpp>

XERCES_CPP_NAMESPACE_BEGIN

// ---------------------------------------------------------------------------
//  FieldActivator: Constructors and Destructor
// ---------------------------------------------------------------------------
FieldActivator::FieldActivator(ValueStoreCache* const valueStoreCache,
                               XPathMatcherStack* const matcherStack,
                               MemoryManager* const manager)
    : fValueStoreCache(valueStoreCache)
    , fMatcherStack(matcherStack)
    , fMayMatch(0)
    , fMemoryManager(manager)
{
    fMayMatch = new (manager) ValueHashTableOf<bool>(29, new (manager) HashPtr(), manager);
}

FieldActivator::FieldActivator(const FieldActivator& other)
    : fValueStoreCache(other.fValueStoreCache)
    , fMatcherStack(other.fMatcherStack)
    , fMayMatch(0)
    , fMemoryManager(other.fMemoryManager)
{
    fMayMatch = new (fMemoryManager) ValueHashTableOf<bool>(29, new (fMemoryManager) HashPtr(), fMemoryManager);
    ValueHashTableOfEnumerator<bool> mayMatchEnum(other.fMayMatch, false, fMemoryManager);

    // Build key set
    while (mayMatchEnum.hasMoreElements())
    {
        IC_Field* field = (IC_Field*) mayMatchEnum.nextElementKey();
        fMayMatch->put(field, other.fMayMatch->get(field));
    }
}

FieldActivator::~FieldActivator()
{
    delete fMayMatch;
}

// ---------------------------------------------------------------------------
//  FieldActivator: Operator methods
// ---------------------------------------------------------------------------
FieldActivator& FieldActivator::operator =(const FieldActivator& other) {

    if (this == &other) {
        return *this;
    }

    fValueStoreCache = other.fValueStoreCache;
    fMatcherStack = other.fMatcherStack;
    return *this;
}

// ---------------------------------------------------------------------------
//  FieldActivator: Operator methods
// ---------------------------------------------------------------------------
XPathMatcher* FieldActivator::activateField(IC_Field* const field, const int initialDepth) {

    ValueStore* valueStore = fValueStoreCache->getValueStoreFor(field, initialDepth);
    XPathMatcher* matcher = field->createMatcher(this, valueStore, fMemoryManager);

    setMayMatch(field, true);
    fMatcherStack->addMatcher(matcher);
    matcher->startDocumentFragment();

    return matcher;
}

void FieldActivator::startValueScopeFor(const IdentityConstraint* const ic,
                                        const int initialDepth) {

    unsigned int fieldCount = ic->getFieldCount();

    for(unsigned int i=0; i<fieldCount; i++) {

        const IC_Field* field = ic->getFieldAt(i);
        ValueStore* valueStore = fValueStoreCache->getValueStoreFor(field, initialDepth);

        valueStore->startValueScope();
    }
}

void FieldActivator::endValueScopeFor(const IdentityConstraint* const ic, const int initialDepth) {

    ValueStore* valueStore = fValueStoreCache->getValueStoreFor(ic, initialDepth);

    valueStore->endValueScope();
}

XERCES_CPP_NAMESPACE_END

/**
  * End of file FieldActivator.cpp
  */

<|endoftext|>


<commit_before>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef CONFIG_ARGS_HPP_
#define CONFIG_ARGS_HPP_

#define KILOBYTE 1024LL
#define MEGABYTE (KILOBYTE * 1024LL)
#define GIGABYTE (MEGABYTE * 1024LL)
#define TERABYTE (GIGABYTE * 1024LL)

#define THOUSAND 1000LL
#define MILLION (THOUSAND * THOUSAND)
#define BILLION (THOUSAND * MILLION)

/*!
 * Version strings
 */

#define SOFTWARE_NAME_STRING "RethinkDB"
#define SERIALIZER_VERSION_STRING "1.6"

/**
 * Basic configuration parameters.
 */

// Defines the maximum size of the batch of IO events to process on
// each loop iteration. A larger number will increase throughput but
// decrease concurrency
#define MAX_IO_EVENT_PROCESSING_BATCH_SIZE        50

// The io batch factor ensures a minimum number of i/o operations
// which are picked from any specific i/o account consecutively.
// A higher value might be advantageous for throughput if seek times
// matter on the underlying i/o system. A low value improves latency.
// The advantage only holds as long as each account has a tendentially
// sequential set of i/o operations though. If access patterns are random
// for all accounts, a low io batch factor does just as well (as bad) when it comes
// to the number of random seeks, but might still provide a lower latency
// than a high batch factor. Our serializer writes usually generate a sequential
// access pattern and therefore take considerable advantage from a high io batch
// factor.
#define DEFAULT_IO_BATCH_FACTOR                   8

// Currently, each cache uses two IO accounts:
// one account for writes, and one account for reads.
// By adjusting the priorities of these accounts, reads
// can be prioritized over writes or the other way around.
//
// This is a one-per-serializer/file priority.
// The per-cache priorities are dynamically derived by dividing these priorities
// by the number of slices on a specific file.
#define CACHE_READS_IO_PRIORITY                   512
#define CACHE_WRITES_IO_PRIORITY                  64

// Garbage Colletion uses its own two IO accounts.
// There is one low-priority account that is meant to guarantee
// (performance-wise) unintrusive garbage collection.
// If the garbage ratio keeps growing,
// GC starts using the high priority account instead, which
// might have a negative influence on database performance
// under i/o heavy workloads but guarantees that the database
// doesn't grow indefinitely.
//
// This is a one-per-serializer/file priority.
#define GC_IO_PRIORITY_NICE                       8
#define GC_IO_PRIORITY_HIGH                       (4 * CACHE_WRITES_IO_PRIORITY)

// Size of the buffer used to perform IO operations (in bytes).
#define IO_BUFFER_SIZE                            (4 * KILOBYTE)

// Size of the device block size (in bytes)
#define DEVICE_BLOCK_SIZE                         (4 * KILOBYTE)

// Size of each btree node (in bytes) on disk
#define DEFAULT_BTREE_BLOCK_SIZE                  (4 * KILOBYTE)

// Maximum number of data blocks
#define MAX_DATA_EXTENTS                          (TERABYTE / (16 * KILOBYTE))

// Size of each extent (in bytes)
#define DEFAULT_EXTENT_SIZE                       (512 * KILOBYTE)

// Max number of blocks which can be read ahead in one i/o transaction (if enabled)

// Ratio of free ram to use for the cache by default
// TODO: DEFAULT_MAX_CACHE_RATIO is unused. Should it be deleted?
#define DEFAULT_MAX_CACHE_RATIO                   0.5

// Maximum number of threads we support
// TODO: make this dynamic where possible
#define MAX_THREADS                               128

// Ticks (in milliseconds) the internal timed tasks are performed at
#define TIMER_TICKS_IN_MS                         5

// How many milliseconds to allow changes to sit in memory before flushing to disk
#define DEFAULT_FLUSH_TIMER_MS                    1000

// If non-null disk_ack_signals are present, concurrent flushing can be used to reduce the
// latency of each single flush. max_concurrent_flushes controls how many flushes can be active
// on a specific slice at any given time.
#define DEFAULT_MAX_CONCURRENT_FLUSHES            1

// If more than this many bytes of dirty data accumulate in the cache, then write
// transactions will be throttled.  A value of 0 means that it will automatically be
// set to MAX_UNSAVED_DATA_LIMIT_FRACTION times the max cache size
#define DEFAULT_UNSAVED_DATA_LIMIT                (4096 * MEGABYTE)

// The unsaved data limit cannot exceed this fraction of the max cache size
#define MAX_UNSAVED_DATA_LIMIT_FRACTION           0.9

// We start flushing dirty pages as soon as we hit this fraction of the unsaved data limit
#define FLUSH_AT_FRACTION_OF_UNSAVED_DATA_LIMIT   0.2

// How many times the page replacement algorithm tries to find an eligible page before giving up.
// Note that (MAX_UNSAVED_DATA_LIMIT_FRACTION ** PAGE_REPL_NUM_TRIES) is the probability that the
// page replacement algorithm will succeed on a given try, and if that probability is less than 1/2
// then the page replacement algorithm will on average be unable to evict pages from the cache.
#define PAGE_REPL_NUM_TRIES                       10

// How large can the key be, in bytes?  This value needs to fit in a byte.
#define MAX_KEY_SIZE                              250

// Any values of this size or less will be directly stored in btree leaf nodes.
// Values greater than this size will be stored in overflow blocks. This value
// needs to fit in a byte.
#define MAX_IN_NODE_VALUE_SIZE                    250

// memcached specifies the maximum value size to be 1MB, but customers asked this to be much higher
#define MAX_VALUE_SIZE                            (10 * MEGABYTE)

// Values larger than this will be streamed in a get operation
#define MAX_BUFFERED_GET_SIZE                     MAX_VALUE_SIZE // streaming is too slow for now, so we disable it completely

// If a single connection sends this many 'noreply' commands, the next command will
// have to wait until the first one finishes
#define MAX_CONCURRENT_QUERIES_PER_CONNECTION     500

// The number of concurrent queries when loading memcached operations from a file.
#define MAX_CONCURRENT_QUEURIES_ON_IMPORT         1000

// How many timestamps we store in a leaf node.  We store the
// NUM_LEAF_NODE_EARLIER_TIMES+1 most-recent timestamps.
#define NUM_LEAF_NODE_EARLIER_TIMES               4

// We assume there will never be more than this many blocks. The value
// is computed by dividing 1 TB by the smallest reasonable block size.
// This value currently fits in 32 bits, and so block_id_t is a uint32_t.
#define MAX_BLOCK_ID                              (TERABYTE / KILOBYTE)

// We assume that there will never be more than this many blocks held in memory by the cache at
// any one time. The value is computed by dividing 50 GB by the smallest reasonable block size.
#define MAX_BLOCKS_IN_MEMORY                      (50 * GIGABYTE / KILOBYTE)

// Special block IDs.  These don't really belong here because they're
// more magic constants than tunable parameters.

// The btree superblock, which has a reference to the root node block
// id.
#define SUPERBLOCK_ID                             0

// The ratio at which we should start GCing.  (HEY: What's the extra
// 0.000001 in MAX_GC_HIGH_RATIO for?  Is it because we told the user
// that 0.99 was too high?)
#define DEFAULT_GC_HIGH_RATIO                     0.20
// TODO: MAX_GC_HIGH_RATIO is unused.  Use it!
// TODO: Probably this value is way too high.
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MAX_GC_HIGH_RATIO                         0.990001

// The ratio at which we don't want to keep GC'ing.
#define DEFAULT_GC_LOW_RATIO                      0.15
// TODO: MIN_GC_LOW_RATIO is unused.  Use it!
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MIN_GC_LOW_RATIO                          0.099999

// What's the maximum number of "young" extents we can have?
#define GC_YOUNG_EXTENT_MAX_SIZE                  50
// What's the definition of a "young" extent in microseconds?
#define GC_YOUNG_EXTENT_TIMELIMIT_MICROS          50000

// If the size of the LBA on a given disk exceeds LBA_MIN_SIZE_FOR_GC, then the fraction of the
// entries that are live and not garbage should be at least LBA_MIN_UNGARBAGE_FRACTION.
// TODO: Maybe change this back to 20 megabytes?
#define LBA_MIN_SIZE_FOR_GC                       (MEGABYTE * 1)
// TODO: This used to be 0.15 but we figured why not do the opposite of our well-tested parameter?
#define LBA_MIN_UNGARBAGE_FRACTION                0.85

// How many LBA structures to have for each file
// TODO: LBA_SHARD_FACTOR used to be 16.
#define LBA_SHARD_FACTOR                          4

// How many bytes of buffering space we can use per disk when reading the LBA. If it's set
// too high, then RethinkDB will eat a lot of memory at startup. This is bad because tcmalloc
// doesn't return memory to the OS. If it's set too low, startup will take a longer time.
#define LBA_READ_BUFFER_SIZE                      GIGABYTE

#define COROUTINE_STACK_SIZE                      131072

#define MAX_COROS_PER_THREAD                      10000

// Size of a cache line (used in cache_line_padded_t).
#define CACHE_LINE_SIZE                           64

#endif  // CONFIG_ARGS_HPP_

<commit_msg>Set DEVICE_BLOCK_SIZE to 512.<commit_after>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef CONFIG_ARGS_HPP_
#define CONFIG_ARGS_HPP_

#define KILOBYTE 1024LL
#define MEGABYTE (KILOBYTE * 1024LL)
#define GIGABYTE (MEGABYTE * 1024LL)
#define TERABYTE (GIGABYTE * 1024LL)

#define THOUSAND 1000LL
#define MILLION (THOUSAND * THOUSAND)
#define BILLION (THOUSAND * MILLION)

/*!
 * Version strings
 */

#define SOFTWARE_NAME_STRING "RethinkDB"
#define SERIALIZER_VERSION_STRING "1.6"

/**
 * Basic configuration parameters.
 */

// Defines the maximum size of the batch of IO events to process on
// each loop iteration. A larger number will increase throughput but
// decrease concurrency
#define MAX_IO_EVENT_PROCESSING_BATCH_SIZE        50

// The io batch factor ensures a minimum number of i/o operations
// which are picked from any specific i/o account consecutively.
// A higher value might be advantageous for throughput if seek times
// matter on the underlying i/o system. A low value improves latency.
// The advantage only holds as long as each account has a tendentially
// sequential set of i/o operations though. If access patterns are random
// for all accounts, a low io batch factor does just as well (as bad) when it comes
// to the number of random seeks, but might still provide a lower latency
// than a high batch factor. Our serializer writes usually generate a sequential
// access pattern and therefore take considerable advantage from a high io batch
// factor.
#define DEFAULT_IO_BATCH_FACTOR                   8

// Currently, each cache uses two IO accounts:
// one account for writes, and one account for reads.
// By adjusting the priorities of these accounts, reads
// can be prioritized over writes or the other way around.
//
// This is a one-per-serializer/file priority.
// The per-cache priorities are dynamically derived by dividing these priorities
// by the number of slices on a specific file.
#define CACHE_READS_IO_PRIORITY                   512
#define CACHE_WRITES_IO_PRIORITY                  64

// Garbage Colletion uses its own two IO accounts.
// There is one low-priority account that is meant to guarantee
// (performance-wise) unintrusive garbage collection.
// If the garbage ratio keeps growing,
// GC starts using the high priority account instead, which
// might have a negative influence on database performance
// under i/o heavy workloads but guarantees that the database
// doesn't grow indefinitely.
//
// This is a one-per-serializer/file priority.
#define GC_IO_PRIORITY_NICE                       8
#define GC_IO_PRIORITY_HIGH                       (4 * CACHE_WRITES_IO_PRIORITY)

// Size of the buffer used to perform IO operations (in bytes).
#define IO_BUFFER_SIZE                            (4 * KILOBYTE)

// Size of the device block size (in bytes)
#define DEVICE_BLOCK_SIZE                         512

// Size of each btree node (in bytes) on disk
#define DEFAULT_BTREE_BLOCK_SIZE                  (4 * KILOBYTE)

// Maximum number of data blocks
#define MAX_DATA_EXTENTS                          (TERABYTE / (16 * KILOBYTE))

// Size of each extent (in bytes)
#define DEFAULT_EXTENT_SIZE                       (512 * KILOBYTE)

// Max number of blocks which can be read ahead in one i/o transaction (if enabled)

// Ratio of free ram to use for the cache by default
// TODO: DEFAULT_MAX_CACHE_RATIO is unused. Should it be deleted?
#define DEFAULT_MAX_CACHE_RATIO                   0.5

// Maximum number of threads we support
// TODO: make this dynamic where possible
#define MAX_THREADS                               128

// Ticks (in milliseconds) the internal timed tasks are performed at
#define TIMER_TICKS_IN_MS                         5

// How many milliseconds to allow changes to sit in memory before flushing to disk
#define DEFAULT_FLUSH_TIMER_MS                    1000

// If non-null disk_ack_signals are present, concurrent flushing can be used to reduce the
// latency of each single flush. max_concurrent_flushes controls how many flushes can be active
// on a specific slice at any given time.
#define DEFAULT_MAX_CONCURRENT_FLUSHES            1

// If more than this many bytes of dirty data accumulate in the cache, then write
// transactions will be throttled.  A value of 0 means that it will automatically be
// set to MAX_UNSAVED_DATA_LIMIT_FRACTION times the max cache size
#define DEFAULT_UNSAVED_DATA_LIMIT                (4096 * MEGABYTE)

// The unsaved data limit cannot exceed this fraction of the max cache size
#define MAX_UNSAVED_DATA_LIMIT_FRACTION           0.9

// We start flushing dirty pages as soon as we hit this fraction of the unsaved data limit
#define FLUSH_AT_FRACTION_OF_UNSAVED_DATA_LIMIT   0.2

// How many times the page replacement algorithm tries to find an eligible page before giving up.
// Note that (MAX_UNSAVED_DATA_LIMIT_FRACTION ** PAGE_REPL_NUM_TRIES) is the probability that the
// page replacement algorithm will succeed on a given try, and if that probability is less than 1/2
// then the page replacement algorithm will on average be unable to evict pages from the cache.
#define PAGE_REPL_NUM_TRIES                       10

// How large can the key be, in bytes?  This value needs to fit in a byte.
#define MAX_KEY_SIZE                              250

// Any values of this size or less will be directly stored in btree leaf nodes.
// Values greater than this size will be stored in overflow blocks. This value
// needs to fit in a byte.
#define MAX_IN_NODE_VALUE_SIZE                    250

// memcached specifies the maximum value size to be 1MB, but customers asked this to be much higher
#define MAX_VALUE_SIZE                            (10 * MEGABYTE)

// Values larger than this will be streamed in a get operation
#define MAX_BUFFERED_GET_SIZE                     MAX_VALUE_SIZE // streaming is too slow for now, so we disable it completely

// If a single connection sends this many 'noreply' commands, the next command will
// have to wait until the first one finishes
#define MAX_CONCURRENT_QUERIES_PER_CONNECTION     500

// The number of concurrent queries when loading memcached operations from a file.
#define MAX_CONCURRENT_QUEURIES_ON_IMPORT         1000

// How many timestamps we store in a leaf node.  We store the
// NUM_LEAF_NODE_EARLIER_TIMES+1 most-recent timestamps.
#define NUM_LEAF_NODE_EARLIER_TIMES               4

// We assume there will never be more than this many blocks. The value
// is computed by dividing 1 TB by the smallest reasonable block size.
// This value currently fits in 32 bits, and so block_id_t is a uint32_t.
#define MAX_BLOCK_ID                              (TERABYTE / KILOBYTE)

// We assume that there will never be more than this many blocks held in memory by the cache at
// any one time. The value is computed by dividing 50 GB by the smallest reasonable block size.
#define MAX_BLOCKS_IN_MEMORY                      (50 * GIGABYTE / KILOBYTE)

// Special block IDs.  These don't really belong here because they're
// more magic constants than tunable parameters.

// The btree superblock, which has a reference to the root node block
// id.
#define SUPERBLOCK_ID                             0

// The ratio at which we should start GCing.  (HEY: What's the extra
// 0.000001 in MAX_GC_HIGH_RATIO for?  Is it because we told the user
// that 0.99 was too high?)
#define DEFAULT_GC_HIGH_RATIO                     0.20
// TODO: MAX_GC_HIGH_RATIO is unused.  Use it!
// TODO: Probably this value is way too high.
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MAX_GC_HIGH_RATIO                         0.990001

// The ratio at which we don't want to keep GC'ing.
#define DEFAULT_GC_LOW_RATIO                      0.15
// TODO: MIN_GC_LOW_RATIO is unused.  Use it!
//  - Keeping this around because if it becomes configurable again, we
//    should have these limits.  Before then at least rassert it.
#define MIN_GC_LOW_RATIO                          0.099999

// What's the maximum number of "young" extents we can have?
#define GC_YOUNG_EXTENT_MAX_SIZE                  50
// What's the definition of a "young" extent in microseconds?
#define GC_YOUNG_EXTENT_TIMELIMIT_MICROS          50000

// If the size of the LBA on a given disk exceeds LBA_MIN_SIZE_FOR_GC, then the fraction of the
// entries that are live and not garbage should be at least LBA_MIN_UNGARBAGE_FRACTION.
// TODO: Maybe change this back to 20 megabytes?
#define LBA_MIN_SIZE_FOR_GC                       (MEGABYTE * 1)
// TODO: This used to be 0.15 but we figured why not do the opposite of our well-tested parameter?
#define LBA_MIN_UNGARBAGE_FRACTION                0.85

// How many LBA structures to have for each file
// TODO: LBA_SHARD_FACTOR used to be 16.
#define LBA_SHARD_FACTOR                          4

// How many bytes of buffering space we can use per disk when reading the LBA. If it's set
// too high, then RethinkDB will eat a lot of memory at startup. This is bad because tcmalloc
// doesn't return memory to the OS. If it's set too low, startup will take a longer time.
#define LBA_READ_BUFFER_SIZE                      GIGABYTE

#define COROUTINE_STACK_SIZE                      131072

#define MAX_COROS_PER_THREAD                      10000

// Size of a cache line (used in cache_line_padded_t).
#define CACHE_LINE_SIZE                           64

#endif  // CONFIG_ARGS_HPP_

<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h"

#include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
#include "tensorflow/compiler/xla/service/pattern_matcher.h"
#include "tensorflow/compiler/xla/shape_util.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"

namespace xla {
namespace gpu {

namespace {

bool IsFusile(const HloInstruction& hlo) {
  // Don't fuse get-tuple-element on GPU: We can, but it's slower than not
  // fusing.  We never generate kernels for unfused GTEs.  Instead, if an
  // unfused GTE is an input to a kernel (including a fusion kernel), we
  // compute the address of the GTE at the top of the kernel.  Often we know the
  // address of the GTE result statically, so we can do this without chasing any
  // pointers.
  return (hlo.IsElementwise() && hlo.operand_count() > 0) ||
         hlo.opcode() == HloOpcode::kBitcast ||
         hlo.opcode() == HloOpcode::kBroadcast ||
         hlo.opcode() == HloOpcode::kConcatenate ||
         hlo.opcode() == HloOpcode::kDynamicSlice ||
         hlo.opcode() == HloOpcode::kDynamicUpdateSlice ||
         hlo.opcode() == HloOpcode::kFusion ||
         hlo.opcode() == HloOpcode::kPad ||
         hlo.opcode() == HloOpcode::kReduce ||
         hlo.opcode() == HloOpcode::kReduceWindow ||
         hlo.opcode() == HloOpcode::kReshape ||
         hlo.opcode() == HloOpcode::kSlice ||
         hlo.opcode() == HloOpcode::kTranspose;
}

bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) {
  if (constant->opcode() != HloOpcode::kConstant ||
      !ShapeUtil::IsScalar(constant->shape())) {
    return false;
  }
  auto type = constant->shape().element_type();
  return type == F16 || type == F32 || type == F64;
}

}  // namespace

/*static*/ bool GpuInstructionFusion::IsExpensive(
    const HloInstruction& instruction) {
  switch (instruction.opcode()) {
    // We say that floating-point division is cheap on the GPU.
    case HloOpcode::kDivide:
      return !ShapeUtil::ElementIsFloating(instruction.shape()) &&
             InstructionFusion::IsExpensive(instruction);

    default:
      return InstructionFusion::IsExpensive(instruction);
  }
}

bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer,
                                      int64 operand_index) {
  HloInstruction* producer = consumer->mutable_operand(operand_index);

  // Check if we can use output fusion for (A @ B) * alpha
  if (producer->opcode() == HloOpcode::kDot ||
      (producer->opcode() == HloOpcode::kFusion &&
       producer->fused_expression_root()->opcode() == HloOpcode::kDot)) {
    int64 other_operand_index = 1 - operand_index;
    HloInstruction* op1 = nullptr;
    HloInstruction* op2 = nullptr;
    if (consumer->operand_count() == 1 &&
        consumer->opcode() == HloOpcode::kFusion &&
        consumer->fusion_kind() == HloInstruction::FusionKind::kLoop &&
        Match(consumer->fused_expression_root(),
              match::Op()
                  .WithOpcode(HloOpcode::kMultiply)
                  .WithOperand(0, match::Op(&op1))
                  .WithOperand(1, match::Op(&op2)))) {
      CHECK(op1 != nullptr && op2 != nullptr);
      // If 'consumer' is a fusion node, it should consist of a broadcast of a
      // scalar constant fused into a multiply, but nothing more. So one operand
      // should be a parameter, and the other should be a broadcast.
      if (op1->opcode() != HloOpcode::kParameter) {
        std::swap(op1, op2);
      }
      if (op1->opcode() != HloOpcode::kParameter ||
          op2->opcode() != HloOpcode::kBroadcast) {
        return false;
      }
      if (IsIEEEFloatingPointScalarConstant(op2->operand(0))) {
        return true;
      }
    } else if (consumer->operand_count() == 2 &&
               consumer->opcode() == HloOpcode::kMultiply) {
      const HloInstruction* alpha = consumer->operand(other_operand_index);
      // Fuse if 'alpha' is a broadcast of a scalar constant.
      if (alpha->opcode() == HloOpcode::kBroadcast &&
          alpha->dimensions().empty() &&
          IsIEEEFloatingPointScalarConstant(alpha->operand(0))) {
        return true;
      }
    }
  }

  // Only allow fusing transpose or broadcast into an output fusion that is
  // implemented as a Gemm call.
  if (consumer->opcode() == HloOpcode::kFusion &&
      consumer->fusion_kind() == HloInstruction::FusionKind::kOutput &&
      ImplementedAsGemm(*consumer)) {
    auto producer_operand_index = consumer->operand_index(producer);
    auto fused_parameter = consumer->fused_parameter(producer_operand_index);
    const std::vector<HloInstruction*>& fused_parameter_users =
        fused_parameter->users();
    if (fused_parameter_users.size() != 1) {
      return false;
    }
    if (producer->opcode() == HloOpcode::kTranspose) {
      // Check that the transpose is an operand of a dot.
      return fused_parameter_users[0]->opcode() == HloOpcode::kDot;
    }
    if (producer->opcode() == HloOpcode::kBroadcast) {
      // Check that the broadcast is a broadcast of a scalar constant into a
      // multiply.
      return producer->dimensions().empty() &&
             IsIEEEFloatingPointScalarConstant(producer->operand(0)) &&
             fused_parameter_users[0]->opcode() == HloOpcode::kMultiply;
    }
  }

  // Other output fusions are not currently supported on GPUs.
  if (producer->opcode() == HloOpcode::kFusion) {
    return false;
  }

  // RNG operations are not currently parallel-friendly on GPU.
  if (producer->opcode() == HloOpcode::kRng) {
    return false;
  }

  // Do not fuse to-vector reduction into other consumers. They should be
  // unfused or the root of a kInput fusion.
  if (IsReductionToVector(*producer)) {
    return false;
  }

  // We can't fuse library calls, so if a user of such an op could become a
  // bitcast, leave it unfused. See `xla::InstructionFusion::ShouldFuse` for
  // further rationale.
  if (producer->CouldBeBitcast() &&
      ImplementedAsLibraryCall(*producer->operand(0))) {
    return false;
  }

  // Cost condition: not fuse (simple, expensive producers) and (consumers who
  // reuse operand elements).
  if (producer->opcode() != HloOpcode::kFusion &&
      consumer->ReusesOperandElements(operand_index) &&
      is_expensive(*producer)) {
    return false;
  }

  // Fuse scalar constants into loop fusion nodes, this reduces the number of
  // parameters and makes matching scalar broadcasts easier.
  if (ShapeUtil::IsEffectiveScalar(producer->shape()) &&
      consumer->opcode() == HloOpcode::kFusion &&
      producer->opcode() == HloOpcode::kConstant) {
    return true;
  }

  return IsFusile(*producer) && IsFusile(*consumer) &&
         InstructionFusion::ShouldFuse(consumer, operand_index);
}

bool GpuInstructionFusion::ShouldFuseIntoMultiOutput(HloInstruction* consumer,
                                                     int64 operand_index) {
  const HloInstruction* producer = consumer->operand(operand_index);
  // The IR emitter has limited support for non-loop fusions with multi output
  // at present.
  // TODO(tjoerg): Relax this constraint to allow for arbitraty kinds of fusion.
  if (consumer->opcode() == HloOpcode::kFusion &&
      consumer->fusion_kind() != HloInstruction::FusionKind::kLoop) {
    return false;
  }
  // Multi-output fusion requires instructions with compatible shapes.
  if (!ShapeUtil::Compatible(producer->shape(), consumer->shape())) {
    return false;
  }
  // TODO(tjoerg): Stop calling `ShouldFuse` to relax the criteria for
  // multi-output fusion. In particular, do not check whether an instruction is
  // expensive to duplicate, since this doesn't matter here.
  return GpuInstructionFusion::ShouldFuse(consumer, operand_index);
}

HloInstruction::FusionKind GpuInstructionFusion::ChooseKind(
    const HloInstruction* producer, const HloInstruction* consumer) {
  if (IsReductionToVector(*consumer)) {
    return HloInstruction::FusionKind::kInput;
  }
  if (producer->opcode() == HloOpcode::kDot ||
      (producer->opcode() == HloOpcode::kFusion &&
       producer->fused_expression_root()->opcode() == HloOpcode::kDot)) {
    return HloInstruction::FusionKind::kOutput;
  }
  if (HloOpcode::kFusion == consumer->opcode()) {
    return consumer->fusion_kind();
  }
  return InstructionFusion::ChooseKind(producer, consumer);
}

}  // namespace gpu
}  // namespace xla
<commit_msg>Mark Gather as fusile.<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h"

#include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
#include "tensorflow/compiler/xla/service/pattern_matcher.h"
#include "tensorflow/compiler/xla/shape_util.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"

namespace xla {
namespace gpu {

namespace {

bool IsFusile(const HloInstruction& hlo) {
  // Don't fuse get-tuple-element on GPU: We can, but it's slower than not
  // fusing.  We never generate kernels for unfused GTEs.  Instead, if an
  // unfused GTE is an input to a kernel (including a fusion kernel), we
  // compute the address of the GTE at the top of the kernel.  Often we know the
  // address of the GTE result statically, so we can do this without chasing any
  // pointers.
  return (hlo.IsElementwise() && hlo.operand_count() > 0) ||
         hlo.opcode() == HloOpcode::kBitcast ||
         hlo.opcode() == HloOpcode::kBroadcast ||
         hlo.opcode() == HloOpcode::kConcatenate ||
         hlo.opcode() == HloOpcode::kDynamicSlice ||
         hlo.opcode() == HloOpcode::kDynamicUpdateSlice ||
         hlo.opcode() == HloOpcode::kFusion ||
         hlo.opcode() == HloOpcode::kGather ||
         hlo.opcode() == HloOpcode::kPad ||
         hlo.opcode() == HloOpcode::kReduce ||
         hlo.opcode() == HloOpcode::kReduceWindow ||
         hlo.opcode() == HloOpcode::kReshape ||
         hlo.opcode() == HloOpcode::kSlice ||
         hlo.opcode() == HloOpcode::kTranspose;
}

bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) {
  if (constant->opcode() != HloOpcode::kConstant ||
      !ShapeUtil::IsScalar(constant->shape())) {
    return false;
  }
  auto type = constant->shape().element_type();
  return type == F16 || type == F32 || type == F64;
}

}  // namespace

/*static*/ bool GpuInstructionFusion::IsExpensive(
    const HloInstruction& instruction) {
  switch (instruction.opcode()) {
    // We say that floating-point division is cheap on the GPU.
    case HloOpcode::kDivide:
      return !ShapeUtil::ElementIsFloating(instruction.shape()) &&
             InstructionFusion::IsExpensive(instruction);

    default:
      return InstructionFusion::IsExpensive(instruction);
  }
}

bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer,
                                      int64 operand_index) {
  HloInstruction* producer = consumer->mutable_operand(operand_index);

  // Check if we can use output fusion for (A @ B) * alpha
  if (producer->opcode() == HloOpcode::kDot ||
      (producer->opcode() == HloOpcode::kFusion &&
       producer->fused_expression_root()->opcode() == HloOpcode::kDot)) {
    int64 other_operand_index = 1 - operand_index;
    HloInstruction* op1 = nullptr;
    HloInstruction* op2 = nullptr;
    if (consumer->operand_count() == 1 &&
        consumer->opcode() == HloOpcode::kFusion &&
        consumer->fusion_kind() == HloInstruction::FusionKind::kLoop &&
        Match(consumer->fused_expression_root(),
              match::Op()
                  .WithOpcode(HloOpcode::kMultiply)
                  .WithOperand(0, match::Op(&op1))
                  .WithOperand(1, match::Op(&op2)))) {
      CHECK(op1 != nullptr && op2 != nullptr);
      // If 'consumer' is a fusion node, it should consist of a broadcast of a
      // scalar constant fused into a multiply, but nothing more. So one operand
      // should be a parameter, and the other should be a broadcast.
      if (op1->opcode() != HloOpcode::kParameter) {
        std::swap(op1, op2);
      }
      if (op1->opcode() != HloOpcode::kParameter ||
          op2->opcode() != HloOpcode::kBroadcast) {
        return false;
      }
      if (IsIEEEFloatingPointScalarConstant(op2->operand(0))) {
        return true;
      }
    } else if (consumer->operand_count() == 2 &&
               consumer->opcode() == HloOpcode::kMultiply) {
      const HloInstruction* alpha = consumer->operand(other_operand_index);
      // Fuse if 'alpha' is a broadcast of a scalar constant.
      if (alpha->opcode() == HloOpcode::kBroadcast &&
          alpha->dimensions().empty() &&
          IsIEEEFloatingPointScalarConstant(alpha->operand(0))) {
        return true;
      }
    }
  }

  // Only allow fusing transpose or broadcast into an output fusion that is
  // implemented as a Gemm call.
  if (consumer->opcode() == HloOpcode::kFusion &&
      consumer->fusion_kind() == HloInstruction::FusionKind::kOutput &&
      ImplementedAsGemm(*consumer)) {
    auto producer_operand_index = consumer->operand_index(producer);
    auto fused_parameter = consumer->fused_parameter(producer_operand_index);
    const std::vector<HloInstruction*>& fused_parameter_users =
        fused_parameter->users();
    if (fused_parameter_users.size() != 1) {
      return false;
    }
    if (producer->opcode() == HloOpcode::kTranspose) {
      // Check that the transpose is an operand of a dot.
      return fused_parameter_users[0]->opcode() == HloOpcode::kDot;
    }
    if (producer->opcode() == HloOpcode::kBroadcast) {
      // Check that the broadcast is a broadcast of a scalar constant into a
      // multiply.
      return producer->dimensions().empty() &&
             IsIEEEFloatingPointScalarConstant(producer->operand(0)) &&
             fused_parameter_users[0]->opcode() == HloOpcode::kMultiply;
    }
  }

  // Other output fusions are not currently supported on GPUs.
  if (producer->opcode() == HloOpcode::kFusion) {
    return false;
  }

  // RNG operations are not currently parallel-friendly on GPU.
  if (producer->opcode() == HloOpcode::kRng) {
    return false;
  }

  // Do not fuse to-vector reduction into other consumers. They should be
  // unfused or the root of a kInput fusion.
  if (IsReductionToVector(*producer)) {
    return false;
  }

  // We can't fuse library calls, so if a user of such an op could become a
  // bitcast, leave it unfused. See `xla::InstructionFusion::ShouldFuse` for
  // further rationale.
  if (producer->CouldBeBitcast() &&
      ImplementedAsLibraryCall(*producer->operand(0))) {
    return false;
  }

  // Cost condition: not fuse (simple, expensive producers) and (consumers who
  // reuse operand elements).
  if (producer->opcode() != HloOpcode::kFusion &&
      consumer->ReusesOperandElements(operand_index) &&
      is_expensive(*producer)) {
    return false;
  }

  // Fuse scalar constants into loop fusion nodes, this reduces the number of
  // parameters and makes matching scalar broadcasts easier.
  if (ShapeUtil::IsEffectiveScalar(producer->shape()) &&
      consumer->opcode() == HloOpcode::kFusion &&
      producer->opcode() == HloOpcode::kConstant) {
    return true;
  }

  return IsFusile(*producer) && IsFusile(*consumer) &&
         InstructionFusion::ShouldFuse(consumer, operand_index);
}

bool GpuInstructionFusion::ShouldFuseIntoMultiOutput(HloInstruction* consumer,
                                                     int64 operand_index) {
  const HloInstruction* producer = consumer->operand(operand_index);
  // The IR emitter has limited support for non-loop fusions with multi output
  // at present.
  // TODO(tjoerg): Relax this constraint to allow for arbitraty kinds of fusion.
  if (consumer->opcode() == HloOpcode::kFusion &&
      consumer->fusion_kind() != HloInstruction::FusionKind::kLoop) {
    return false;
  }
  // Multi-output fusion requires instructions with compatible shapes.
  if (!ShapeUtil::Compatible(producer->shape(), consumer->shape())) {
    return false;
  }
  // TODO(tjoerg): Stop calling `ShouldFuse` to relax the criteria for
  // multi-output fusion. In particular, do not check whether an instruction is
  // expensive to duplicate, since this doesn't matter here.
  return GpuInstructionFusion::ShouldFuse(consumer, operand_index);
}

HloInstruction::FusionKind GpuInstructionFusion::ChooseKind(
    const HloInstruction* producer, const HloInstruction* consumer) {
  if (IsReductionToVector(*consumer)) {
    return HloInstruction::FusionKind::kInput;
  }
  if (producer->opcode() == HloOpcode::kDot ||
      (producer->opcode() == HloOpcode::kFusion &&
       producer->fused_expression_root()->opcode() == HloOpcode::kDot)) {
    return HloInstruction::FusionKind::kOutput;
  }
  if (HloOpcode::kFusion == consumer->opcode()) {
    return consumer->fusion_kind();
  }
  return InstructionFusion::ChooseKind(producer, consumer);
}

}  // namespace gpu
}  // namespace xla
<|endoftext|>