<commit_before>/*
    incomingtransfer.cpp - msn p2p protocol

    Copyright (c) 2003-2005 by Olivier Goffart        <ogoffart@kde.org>
    Copyright (c) 2005      by Gregg Edghill          <gregg.edghill@gmail.com>

    Kopete    (c) 2002-2007 by the Kopete developers  <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include "incomingtransfer.h"
//Added by qt3to4:
#include <QByteArray>
using P2P::TransferContext;
using P2P::IncomingTransfer;
using P2P::Message;

// Kde includes
#include <k3bufferedsocket.h>
#include <kdebug.h>
#include <klocale.h>
#include <k3serversocket.h>
#include <kstandarddirs.h>
#include <ktemporaryfile.h>
using namespace KNetwork;

// Qt includes
#include <qfile.h>
#include <qregexp.h>

// Kopete includes
#include <kopetetransfermanager.h>

IncomingTransfer::IncomingTransfer(const QString& from, P2P::Dispatcher *dispatcher, quint32 sessionId)
: TransferContext(from,dispatcher,sessionId)
{
	m_direction = P2P::Incoming;
	m_listener  = 0l;
}

IncomingTransfer::~IncomingTransfer()
{
	kDebug(14140) ;
	if(m_listener)
	{
		delete m_listener;
		m_listener = 0l;
	}

	if(m_socket)
	{
		delete m_socket;
		m_socket = 0l;
	}
}

void IncomingTransfer::slotTransferAccepted(Kopete::Transfer* transfer, const QString& /*fileName*/)
{
	quint32 sessionId = transfer->info().internalId().toUInt();
	if(sessionId!=m_sessionId)
		return;

	QObject::connect(transfer , SIGNAL(transferCanceled()), this, SLOT(abort()));
	m_transfer = transfer;

	QString content = QString("SessionID: %1\r\n\r\n").arg(sessionId);
	sendMessage(OK, content);

	QObject::disconnect(Kopete::TransferManager::transferManager(), 0l, this, 0l);
}

void IncomingTransfer::slotTransferRefused(const Kopete::FileTransferInfo& info)
{
	quint32 sessionId = info.internalId().toUInt();
	if(sessionId!=m_sessionId)
		return;

	QString content = QString("SessionID: %1\r\n\r\n").arg(sessionId);
	// Send the sending client a cancellation message.
	sendMessage(DECLINE, content);
	m_state=Finished;

	QObject::disconnect(Kopete::TransferManager::transferManager(), 0l, this, 0l);
}

void IncomingTransfer::acknowledged()
{
	kDebug(14140) ;

	switch(m_state)
	{
		case Invitation:
				// NOTE UDI: base identifier acknowledge message, ignore.
				//      UDI: 200 OK message should follow.
				if(m_type == File)
				{
					// FT: 200 OK acknowledged message.
					// If this is the first connection between the two clients, a direct connection invitation
					// should follow. Otherwise, the file transfer may start right away.
					if(m_transfer)
					{
						QFile *destination = new QFile(m_transfer->destinationURL().path());
						if(!destination->open(QIODevice::WriteOnly))
						{
							m_transfer->slotError(KIO::ERR_CANNOT_OPEN_FOR_WRITING, i18n("Cannot open file for writing"));
							m_transfer = 0l;

							error();
							return;
						}
						m_file = destination;
					}
					m_state = Negotiation;
				}
			break;

		case Negotiation:
				// 200 OK acknowledge message.
			break;

		case DataTransfer:
			break;

		case Finished:
			// UDI: Bye acknowledge message.
			m_dispatcher->detach(this);
			break;
	}
}

void IncomingTransfer::processMessage(const Message& message)
{
	if(m_file && (message.header.flag == 0x20 || message.header.flag == 0x01000030))
	{
		if(m_state == Finished) {
			return;
		}
		// UserDisplayIcon data or File data is in this message.
		// Write the received data to the file.
		kDebug(14140) << QString("Received, %1 bytes").arg(message.header.dataSize);

		m_file->write(message.body.data(), message.header.dataSize);
		if(m_transfer){
			m_transfer->slotProcessed(message.header.dataOffset + message.header.dataSize);
		}

		if((message.header.dataOffset + message.header.dataSize) == message.header.totalDataSize)
		{
			// Transfer is complete.
			if(m_type == UserDisplayIcon){
				m_tempFile->close();
				m_dispatcher->displayIconReceived(m_tempFile, m_object);
				m_tempFile = 0l;
				m_file = 0l;
			}
			else
			{
				m_file->close();
			}

			m_isComplete = true;
			// Send data acknowledge message.
			acknowledge(message);

			if(m_type == UserDisplayIcon)
			{
				m_state = Finished;
				// Send BYE message.
				sendMessage(BYE, "\r\n");
			}
		}
	}
	//pidgin is probably broken since it sends 0 as appid but we don't need this check anyway
	else if(message.header.dataSize == 4 /*&& message.applicationIdentifier == 1*/)
	{
		// Data preparation message.
		m_tempFile = new KTemporaryFile();
		m_tempFile->setPrefix("msnpicture--");
		m_tempFile->setSuffix(".png");
		m_tempFile->open();
		m_file = m_tempFile;
		m_state = DataTransfer;
		// Send data preparation acknowledge message.
		acknowledge(message);
	}
	else
	{
		QString body =
			QByteArray(message.body.data(), message.header.dataSize);
//		kDebug(14140) << "received, " << body;

		if(body.startsWith("INVITE"))
		{
			// Retrieve some MSNSLP headers used when
			// replying to this INVITE message.
			QRegExp regex(";branch=\\{([0-9A-F\\-]*)\\}\r\n");
			regex.indexIn(body);
			m_branch = regex.cap(1);
			// NOTE Call-ID never changes.
			regex = QRegExp("Call-ID: \\{([0-9A-F\\-]*)\\}\r\n");
			regex.indexIn(body);
			m_callId = regex.cap(1);
			regex = QRegExp("Bridges: ([^\r\n]*)\r\n");
			regex.indexIn(body);
			QString bridges = regex.cap(1);
			// The NetID field is 0 if the Conn-Type is
			// Direct-Connect or Firewall, otherwise, it is
			// a randomly generated number.
			regex = QRegExp("NetID: (\\-?\\d+)\r\n");
			regex.indexIn(body);
			QString netId = regex.cap(1);
			kDebug(14140) << "net id, " << netId;
			// Connection Types
			// - Direct-Connect
			// - Port-Restrict-NAT
			// - IP-Restrict-NAT
			// - Symmetric-NAT
			// - Firewall
			regex = QRegExp("Conn-Type: ([^\r\n]+)\r\n");
			regex.indexIn(body);
			QString connType = regex.cap(1);

			bool wouldListen = false;
			if(netId.toUInt() == 0 && connType == "Direct-Connect"){
				wouldListen = true;

			}
			else if(connType == "IP-Restrict-NAT"){
				wouldListen = true;
			}
#if 1
			wouldListen = false; // TODO Direct connection support
#endif
			QString content;

			if(wouldListen)
			{
				// Create a listening socket for direct file transfer.
				m_listener = new KServerSocket("", "");
				m_listener->setResolutionEnabled(true);
				// Create the callback that will try to accept incoming connections.
				QObject::connect(m_listener, SIGNAL(readyAccept()), SLOT(slotAccept()));
				QObject::connect(m_listener, SIGNAL(gotError(int)), this, SLOT(slotListenError(int)));
				// Listen for incoming connections.
				bool isListening = m_listener->listen(1);
				kDebug(14140) << (isListening ? "listening" : "not listening");
				kDebug(14140)
					<< "local endpoint, " << m_listener->localAddress().nodeName()
					<< endl;

				content = "Bridge: TCPv1\r\n"
					"Listening: true\r\n" +
					QString("Hashed-Nonce: {%1}\r\n").arg(P2P::Uid::createUid()) +
					QString("IPv4Internal-Addrs: %1\r\n").arg(m_listener->localAddress().nodeName())   +
					QString("IPv4Internal-Port: %1\r\n").arg(m_listener->localAddress().serviceName()) +
					"\r\n";
			}
			else
			{
				content =
					"Bridge: TCPv1\r\n"
					"Listening: false\r\n"
					"Hashed-Nonce: {00000000-0000-0000-0000-000000000000}\r\n"
					"\r\n";
			}

			m_state = DataTransfer;

			if (m_type != File)
			{
				// NOTE For file transfers, the connection invite *must not* be acknowledged in any way
				//      as this trips MSN 7.5

				acknowledge(message);
				// Send 200 OK message to the sending client.
				sendMessage(OK, content);
			}
		}
		else if(body.startsWith("BYE"))
		{
			m_state = Finished;
			// Send the sending client an acknowledge message.
			acknowledge(message);

			if(m_file && m_transfer)
			{
				if(m_isComplete){
					// The transfer is complete.
					m_transfer->slotComplete();
				}
				else
				{
					// The transfer has been canceled remotely.
					if(m_transfer){
						// Inform the user of the file transfer cancellation.
						m_transfer->slotError(KIO::ERR_ABORTED, i18n("File transfer cancelled."));
					}
					// Remove the partially received file.
					m_file->remove();
				}
			}

			// Dispose of this transfer context.
			m_dispatcher->detach(this);
		}
		else if(body.startsWith("MSNSLP/1.0 200 OK"))
		{
			if(m_type == UserDisplayIcon){
				m_state = Negotiation;
				// Acknowledge the 200 OK message.
				acknowledge(message);
			}
		}
	}
}

void IncomingTransfer::slotListenError(int /*errorCode*/)
{
	kDebug(14140) << m_listener->errorString();
}

void IncomingTransfer::slotAccept()
{
	// Try to accept an incoming connection from the sending client.
	m_socket = static_cast<KBufferedSocket*>(m_listener->accept());
	if(!m_socket)
	{
		// NOTE If direct connection fails, the sending
		// client wil transfer the file data through the
		// existing session.
		kDebug(14140) << "Direct connection failed.";
		// Close the listening endpoint.
		m_listener->close();
		return;
	}

	kDebug(14140) << "Direct connection established.";

	// Set the socket to non blocking,
	// enable the ready read signal and disable
	// ready write signal.
	// NOTE readyWrite consumes too much cpu usage.
	m_socket->setBlocking(false);
	m_socket->enableRead(true);
	m_socket->enableWrite(false);

	// Create the callback that will try to read bytes from the accepted socket.
	QObject::connect(m_socket, SIGNAL(readyRead()),   this, SLOT(slotSocketRead()));
	// Create the callback that will try to handle the socket close event.
	QObject::connect(m_socket, SIGNAL(closed()),      this, SLOT(slotSocketClosed()));
	// Create the callback that will try to handle the socket error event.
	QObject::connect(m_socket, SIGNAL(gotError(int)), this, SLOT(slotSocketError(int)));
}

void IncomingTransfer::slotSocketRead()
{
	int available = m_socket->bytesAvailable();
	kDebug(14140) << available << ", bytes available.";
	if(available > 0)
	{
		QByteArray buffer = m_socket->read(available);
		if(QString(buffer) == "foo"){
			kDebug(14140) << "Connection Check.";
		}
	}
}

void IncomingTransfer::slotSocketClosed()
{
	kDebug(14140) ;
}

void IncomingTransfer::slotSocketError(int errorCode)
{
	kDebug(14140) << errorCode;
}

#include "incomingtransfer.moc"
<commit_msg>remove the png suffix when saving the temporary file, otherwise QImageReader/QImage tries to open jpeg as png<commit_after>/*
    incomingtransfer.cpp - msn p2p protocol

    Copyright (c) 2003-2005 by Olivier Goffart        <ogoffart@kde.org>
    Copyright (c) 2005      by Gregg Edghill          <gregg.edghill@gmail.com>

    Kopete    (c) 2002-2007 by the Kopete developers  <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include "incomingtransfer.h"
//Added by qt3to4:
#include <QByteArray>
using P2P::TransferContext;
using P2P::IncomingTransfer;
using P2P::Message;

// Kde includes
#include <k3bufferedsocket.h>
#include <kdebug.h>
#include <klocale.h>
#include <k3serversocket.h>
#include <kstandarddirs.h>
#include <ktemporaryfile.h>
using namespace KNetwork;

// Qt includes
#include <qfile.h>
#include <qregexp.h>

// Kopete includes
#include <kopetetransfermanager.h>

IncomingTransfer::IncomingTransfer(const QString& from, P2P::Dispatcher *dispatcher, quint32 sessionId)
: TransferContext(from,dispatcher,sessionId)
{
	m_direction = P2P::Incoming;
	m_listener  = 0l;
}

IncomingTransfer::~IncomingTransfer()
{
	kDebug(14140) ;
	if(m_listener)
	{
		delete m_listener;
		m_listener = 0l;
	}

	if(m_socket)
	{
		delete m_socket;
		m_socket = 0l;
	}
}

void IncomingTransfer::slotTransferAccepted(Kopete::Transfer* transfer, const QString& /*fileName*/)
{
	quint32 sessionId = transfer->info().internalId().toUInt();
	if(sessionId!=m_sessionId)
		return;

	QObject::connect(transfer , SIGNAL(transferCanceled()), this, SLOT(abort()));
	m_transfer = transfer;

	QString content = QString("SessionID: %1\r\n\r\n").arg(sessionId);
	sendMessage(OK, content);

	QObject::disconnect(Kopete::TransferManager::transferManager(), 0l, this, 0l);
}

void IncomingTransfer::slotTransferRefused(const Kopete::FileTransferInfo& info)
{
	quint32 sessionId = info.internalId().toUInt();
	if(sessionId!=m_sessionId)
		return;

	QString content = QString("SessionID: %1\r\n\r\n").arg(sessionId);
	// Send the sending client a cancellation message.
	sendMessage(DECLINE, content);
	m_state=Finished;

	QObject::disconnect(Kopete::TransferManager::transferManager(), 0l, this, 0l);
}

void IncomingTransfer::acknowledged()
{
	kDebug(14140) ;

	switch(m_state)
	{
		case Invitation:
				// NOTE UDI: base identifier acknowledge message, ignore.
				//      UDI: 200 OK message should follow.
				if(m_type == File)
				{
					// FT: 200 OK acknowledged message.
					// If this is the first connection between the two clients, a direct connection invitation
					// should follow. Otherwise, the file transfer may start right away.
					if(m_transfer)
					{
						QFile *destination = new QFile(m_transfer->destinationURL().path());
						if(!destination->open(QIODevice::WriteOnly))
						{
							m_transfer->slotError(KIO::ERR_CANNOT_OPEN_FOR_WRITING, i18n("Cannot open file for writing"));
							m_transfer = 0l;

							error();
							return;
						}
						m_file = destination;
					}
					m_state = Negotiation;
				}
			break;

		case Negotiation:
				// 200 OK acknowledge message.
			break;

		case DataTransfer:
			break;

		case Finished:
			// UDI: Bye acknowledge message.
			m_dispatcher->detach(this);
			break;
	}
}

void IncomingTransfer::processMessage(const Message& message)
{
	if(m_file && (message.header.flag == 0x20 || message.header.flag == 0x01000030))
	{
		if(m_state == Finished) {
			return;
		}
		// UserDisplayIcon data or File data is in this message.
		// Write the received data to the file.
		kDebug(14140) << QString("Received, %1 bytes").arg(message.header.dataSize);

		m_file->write(message.body.data(), message.header.dataSize);
		if(m_transfer){
			m_transfer->slotProcessed(message.header.dataOffset + message.header.dataSize);
		}

		if((message.header.dataOffset + message.header.dataSize) == message.header.totalDataSize)
		{
			// Transfer is complete.
			if(m_type == UserDisplayIcon){
				m_tempFile->close();
				m_dispatcher->displayIconReceived(m_tempFile, m_object);
				m_tempFile = 0l;
				m_file = 0l;
			}
			else
			{
				m_file->close();
			}

			m_isComplete = true;
			// Send data acknowledge message.
			acknowledge(message);

			if(m_type == UserDisplayIcon)
			{
				m_state = Finished;
				// Send BYE message.
				sendMessage(BYE, "\r\n");
			}
		}
	}
	//pidgin is probably broken since it sends 0 as appid but we don't need this check anyway
	else if(message.header.dataSize == 4 /*&& message.applicationIdentifier == 1*/)
	{
		// Data preparation message.
		m_tempFile = new KTemporaryFile();
		m_tempFile->setPrefix("msnpicture--");
// 		m_tempFile->setSuffix(".png");
		m_tempFile->open();
		m_file = m_tempFile;
		m_state = DataTransfer;
		// Send data preparation acknowledge message.
		acknowledge(message);
	}
	else
	{
		QString body =
			QByteArray(message.body.data(), message.header.dataSize);
//		kDebug(14140) << "received, " << body;

		if(body.startsWith("INVITE"))
		{
			// Retrieve some MSNSLP headers used when
			// replying to this INVITE message.
			QRegExp regex(";branch=\\{([0-9A-F\\-]*)\\}\r\n");
			regex.indexIn(body);
			m_branch = regex.cap(1);
			// NOTE Call-ID never changes.
			regex = QRegExp("Call-ID: \\{([0-9A-F\\-]*)\\}\r\n");
			regex.indexIn(body);
			m_callId = regex.cap(1);
			regex = QRegExp("Bridges: ([^\r\n]*)\r\n");
			regex.indexIn(body);
			QString bridges = regex.cap(1);
			// The NetID field is 0 if the Conn-Type is
			// Direct-Connect or Firewall, otherwise, it is
			// a randomly generated number.
			regex = QRegExp("NetID: (\\-?\\d+)\r\n");
			regex.indexIn(body);
			QString netId = regex.cap(1);
			kDebug(14140) << "net id, " << netId;
			// Connection Types
			// - Direct-Connect
			// - Port-Restrict-NAT
			// - IP-Restrict-NAT
			// - Symmetric-NAT
			// - Firewall
			regex = QRegExp("Conn-Type: ([^\r\n]+)\r\n");
			regex.indexIn(body);
			QString connType = regex.cap(1);

			bool wouldListen = false;
			if(netId.toUInt() == 0 && connType == "Direct-Connect"){
				wouldListen = true;

			}
			else if(connType == "IP-Restrict-NAT"){
				wouldListen = true;
			}
#if 1
			wouldListen = false; // TODO Direct connection support
#endif
			QString content;

			if(wouldListen)
			{
				// Create a listening socket for direct file transfer.
				m_listener = new KServerSocket("", "");
				m_listener->setResolutionEnabled(true);
				// Create the callback that will try to accept incoming connections.
				QObject::connect(m_listener, SIGNAL(readyAccept()), SLOT(slotAccept()));
				QObject::connect(m_listener, SIGNAL(gotError(int)), this, SLOT(slotListenError(int)));
				// Listen for incoming connections.
				bool isListening = m_listener->listen(1);
				kDebug(14140) << (isListening ? "listening" : "not listening");
				kDebug(14140)
					<< "local endpoint, " << m_listener->localAddress().nodeName()
					<< endl;

				content = "Bridge: TCPv1\r\n"
					"Listening: true\r\n" +
					QString("Hashed-Nonce: {%1}\r\n").arg(P2P::Uid::createUid()) +
					QString("IPv4Internal-Addrs: %1\r\n").arg(m_listener->localAddress().nodeName())   +
					QString("IPv4Internal-Port: %1\r\n").arg(m_listener->localAddress().serviceName()) +
					"\r\n";
			}
			else
			{
				content =
					"Bridge: TCPv1\r\n"
					"Listening: false\r\n"
					"Hashed-Nonce: {00000000-0000-0000-0000-000000000000}\r\n"
					"\r\n";
			}

			m_state = DataTransfer;

			if (m_type != File)
			{
				// NOTE For file transfers, the connection invite *must not* be acknowledged in any way
				//      as this trips MSN 7.5

				acknowledge(message);
				// Send 200 OK message to the sending client.
				sendMessage(OK, content);
			}
		}
		else if(body.startsWith("BYE"))
		{
			m_state = Finished;
			// Send the sending client an acknowledge message.
			acknowledge(message);

			if(m_file && m_transfer)
			{
				if(m_isComplete){
					// The transfer is complete.
					m_transfer->slotComplete();
				}
				else
				{
					// The transfer has been canceled remotely.
					if(m_transfer){
						// Inform the user of the file transfer cancellation.
						m_transfer->slotError(KIO::ERR_ABORTED, i18n("File transfer cancelled."));
					}
					// Remove the partially received file.
					m_file->remove();
				}
			}

			// Dispose of this transfer context.
			m_dispatcher->detach(this);
		}
		else if(body.startsWith("MSNSLP/1.0 200 OK"))
		{
			if(m_type == UserDisplayIcon){
				m_state = Negotiation;
				// Acknowledge the 200 OK message.
				acknowledge(message);
			}
		}
	}
}

void IncomingTransfer::slotListenError(int /*errorCode*/)
{
	kDebug(14140) << m_listener->errorString();
}

void IncomingTransfer::slotAccept()
{
	// Try to accept an incoming connection from the sending client.
	m_socket = static_cast<KBufferedSocket*>(m_listener->accept());
	if(!m_socket)
	{
		// NOTE If direct connection fails, the sending
		// client wil transfer the file data through the
		// existing session.
		kDebug(14140) << "Direct connection failed.";
		// Close the listening endpoint.
		m_listener->close();
		return;
	}

	kDebug(14140) << "Direct connection established.";

	// Set the socket to non blocking,
	// enable the ready read signal and disable
	// ready write signal.
	// NOTE readyWrite consumes too much cpu usage.
	m_socket->setBlocking(false);
	m_socket->enableRead(true);
	m_socket->enableWrite(false);

	// Create the callback that will try to read bytes from the accepted socket.
	QObject::connect(m_socket, SIGNAL(readyRead()),   this, SLOT(slotSocketRead()));
	// Create the callback that will try to handle the socket close event.
	QObject::connect(m_socket, SIGNAL(closed()),      this, SLOT(slotSocketClosed()));
	// Create the callback that will try to handle the socket error event.
	QObject::connect(m_socket, SIGNAL(gotError(int)), this, SLOT(slotSocketError(int)));
}

void IncomingTransfer::slotSocketRead()
{
	int available = m_socket->bytesAvailable();
	kDebug(14140) << available << ", bytes available.";
	if(available > 0)
	{
		QByteArray buffer = m_socket->read(available);
		if(QString(buffer) == "foo"){
			kDebug(14140) << "Connection Check.";
		}
	}
}

void IncomingTransfer::slotSocketClosed()
{
	kDebug(14140) ;
}

void IncomingTransfer::slotSocketError(int errorCode)
{
	kDebug(14140) << errorCode;
}

#include "incomingtransfer.moc"
<|endoftext|>


<commit_before>#include "../config.hpp"

#include <cassert>
#include <fstream>
#include <algorithm>

#if __has_include(<filesystem>)
	#include <filesystem>
	namespace filesystem = std::filesystem;
#elif __has_include(<experimental/filesystem>)
	#include <experimental/filesystem>
	namespace filesystem = std::experimental::filesystem;
#else
	#define STX_NO_FILESYSTEM
	#pragma message("No <filesystem> or <experimental/filesystem>, config will not support directories.")
#endif

namespace stx {

void config::set(std::string name, std::string value) noexcept {
	m_entries[std::move(name)] = std::move(value);
}

std::string config::get(std::string const& name) {
	std::string result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, std::string* into) noexcept {
	assert(into);

	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = iter->second;
	return true;
}
std::string config::get(std::string const& name, std::string const& fallback) noexcept {
	std::string result;
	return get(name, &result) ? result : fallback;
}

double config::getf(std::string const& name) {
	double result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, double* into) noexcept {
	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = std::stod(iter->second);
	return true;
}
double config::get(std::string const& name, double fallback) noexcept {
	double result;
	return get(name, &result) ? result : fallback;
}

bool config::getb(std::string const& name) {
	bool result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, bool* into) noexcept {
	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = iter->second == "true" || iter->second == "1";
	return true;
}
bool config::get(std::string const& name, bool fallback) noexcept {
	bool result;
	return get(name, &result) ? result : fallback;
}

void config::parseIni(std::string const& path) {
	auto status = filesystem::status(path);

	#ifndef STX_NO_FILESYSTEM
	// Is path a directory?
	if(status.type() == filesystem::file_type::directory) {
		// A directory

		// Read directory contents IN ALPHABETICAL ORDER
		std::vector<std::string> directory_contents;
		using iter_t = filesystem::directory_iterator;
		for(iter_t i = iter_t(path); i != iter_t(); i++) {
			directory_contents.push_back(i->path().string());
		}
		std::sort(directory_contents.begin(), directory_contents.end());
		for(auto& content : directory_contents) {
			parseIni(content);
		}

		return;
	}
	#endif // !defined(STX_NO_FILESYSTEM)

	// A file
	puts(path.c_str());

	std::ifstream file(path);
	parseIni(file);
}

void config::parseIni(std::istream& stream) {
	constexpr auto npos = std::string_view::npos;

	std::string section;

	std::string raw_line;
	while(std::getline(stream, raw_line)) {
		std::string_view line = raw_line;

		// Remove comment
		if(auto commentBegin = line.find_first_of(';'); commentBegin != npos) {
			line = line.substr(0, commentBegin);
		}

		// Trim leading whitespace
		while(!line.empty() && std::isspace(line.front())) line.remove_prefix(1);

		// Trim trailing whitespace
		while(!line.empty() && std::isspace(line.back())) line.remove_suffix(1);

		if(line.empty()) continue; // Line is empty

		// New Section?
		if(line.front() == '[' && line.back() == ']') {
			line.remove_prefix(1);
			line.remove_suffix(1);
			section = std::string(line) + ".";
			continue;
		}

		std::string_view key, value;
		value = line;
		while(value.size() && value.front() != '=')
			value.remove_prefix(1);
		if(value.empty()) throw std::runtime_error("Missing =");
		value.remove_prefix(1);

		key = line.substr(0, line.size() - value.size() - 1);

		set(section + std::string(key), std::string(value));
	}
}

void config::parseCmd(int argc, const char** argv) noexcept {
	for(int i = 0; i < argc; i++) {
		const char* arg = argv[i];
		if(arg[0] == '-' && arg[1] == '-') {
			std::string_view option(arg + 2);

			size_t equals_pos = option.find('=');
			if(equals_pos == std::string_view::npos) {
				set(std::string(option), "true");
			}
			else {
				std::string_view option_name = option.substr(0, equals_pos);
				std::string_view option_value = option.substr(equals_pos + 1);
				set(std::string(option_name), std::string(option_value));
			}
		}
	}

}

extern "C" char** environ;

void config::importEnv() noexcept {
	for(char** envar = environ; *envar; envar++) {
		std::string_view option(*envar);

		size_t equals_pos = option.find('=');
		if(equals_pos == std::string_view::npos) {
			set(std::string(option), "true");
		}
		else {
			std::string_view option_name = option.substr(0, equals_pos);
			std::string_view option_value = option.substr(equals_pos + 1);
			set(std::string(option_name), std::string(option_value));
		}
	}
}

} // namespace stx
<commit_msg>Appearently `for(auto& entry : std::directory_iterator(path))` is a thing :)<commit_after>#include "../config.hpp"

#include <cassert>
#include <fstream>
#include <algorithm>

#if __has_include(<filesystem>)
	#include <filesystem>
	namespace filesystem = std::filesystem;
#elif __has_include(<experimental/filesystem>)
	#include <experimental/filesystem>
	namespace filesystem = std::experimental::filesystem;
#else
	#define STX_NO_FILESYSTEM
	#pragma message("No <filesystem> or <experimental/filesystem>, config will not support directories.")
#endif

namespace stx {

void config::set(std::string name, std::string value) noexcept {
	m_entries[std::move(name)] = std::move(value);
}

std::string config::get(std::string const& name) {
	std::string result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, std::string* into) noexcept {
	assert(into);

	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = iter->second;
	return true;
}
std::string config::get(std::string const& name, std::string const& fallback) noexcept {
	std::string result;
	return get(name, &result) ? result : fallback;
}

double config::getf(std::string const& name) {
	double result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, double* into) noexcept {
	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = std::stod(iter->second);
	return true;
}
double config::get(std::string const& name, double fallback) noexcept {
	double result;
	return get(name, &result) ? result : fallback;
}

bool config::getb(std::string const& name) {
	bool result;
	if(!get(name, &result)) throw std::runtime_error("Couldn't find config entry '" + name + "'");
	return result;
}
bool config::get(std::string const& name, bool* into) noexcept {
	auto iter = m_entries.find(name);

	if(iter == m_entries.end()) return false; // Not found

	*into = iter->second == "true" || iter->second == "1";
	return true;
}
bool config::get(std::string const& name, bool fallback) noexcept {
	bool result;
	return get(name, &result) ? result : fallback;
}

void config::parseIni(std::string const& path) {
	auto status = filesystem::status(path);

	#ifndef STX_NO_FILESYSTEM
	// Is path a directory?
	if(status.type() == filesystem::file_type::directory) {
		// A directory

		// Read directory contents IN ALPHABETICAL ORDER
		std::vector<std::string> directory_contents;
		for(auto& entry : filesystem::recursive_directory_iterator(path)) {
			directory_contents.push_back(entry.path().string());
		}
		std::sort(directory_contents.begin(), directory_contents.end());
		for(auto& content : directory_contents) {
			parseIni(content);
		}

		return;
	}
	#endif // !defined(STX_NO_FILESYSTEM)

	// A file
	puts(path.c_str());

	std::ifstream file(path);
	parseIni(file);
}

void config::parseIni(std::istream& stream) {
	constexpr auto npos = std::string_view::npos;

	std::string section;

	std::string raw_line;
	while(std::getline(stream, raw_line)) {
		std::string_view line = raw_line;

		// Remove comment
		if(auto commentBegin = line.find_first_of(';'); commentBegin != npos) {
			line = line.substr(0, commentBegin);
		}

		// Trim leading whitespace
		while(!line.empty() && std::isspace(line.front())) line.remove_prefix(1);

		// Trim trailing whitespace
		while(!line.empty() && std::isspace(line.back())) line.remove_suffix(1);

		if(line.empty()) continue; // Line is empty

		// New Section?
		if(line.front() == '[' && line.back() == ']') {
			line.remove_prefix(1);
			line.remove_suffix(1);
			section = std::string(line) + ".";
			continue;
		}

		std::string_view key, value;
		value = line;
		while(value.size() && value.front() != '=')
			value.remove_prefix(1);
		if(value.empty()) throw std::runtime_error("Missing =");
		value.remove_prefix(1);

		key = line.substr(0, line.size() - value.size() - 1);

		set(section + std::string(key), std::string(value));
	}
}

void config::parseCmd(int argc, const char** argv) noexcept {
	for(int i = 0; i < argc; i++) {
		const char* arg = argv[i];
		if(arg[0] == '-' && arg[1] == '-') {
			std::string_view option(arg + 2);

			size_t equals_pos = option.find('=');
			if(equals_pos == std::string_view::npos) {
				set(std::string(option), "true");
			}
			else {
				std::string_view option_name = option.substr(0, equals_pos);
				std::string_view option_value = option.substr(equals_pos + 1);
				set(std::string(option_name), std::string(option_value));
			}
		}
	}

}

extern "C" char** environ;

void config::importEnv() noexcept {
	for(char** envar = environ; *envar; envar++) {
		std::string_view option(*envar);

		size_t equals_pos = option.find('=');
		if(equals_pos == std::string_view::npos) {
			set(std::string(option), "true");
		}
		else {
			std::string_view option_name = option.substr(0, equals_pos);
			std::string_view option_value = option.substr(equals_pos + 1);
			set(std::string(option_name), std::string(option_value));
		}
	}
}

} // namespace stx
<|endoftext|>


<commit_before>#include <execution_agent_new>
#include <execution_policy>
#include <__nested_execution_policy_new>

namespace std
{

template<class ExecutionAgent, class BulkExecutor, class ExecutionCategory, class Function>
void bulk_invoke_new(const __basic_execution_policy<ExecutionAgent,BulkExecutor,ExecutionCategory>& exec,
                     Function f)
{
  using traits = std::execution_agent_traits_new<ExecutionAgent>;

  auto param = exec.param();
  auto shape = traits::shape(param);
  auto shared_init = traits::make_shared_initializer(param);

  using executor_index = typename executor_traits<BulkExecutor>::index_type;
  using shared_param_type = typename executor_traits<BulkExecutor>::template shared_param_type<decltype(shared_init)>;

  return executor_traits<BulkExecutor>::bulk_invoke(exec.executor(), [=](executor_index agent_idx, shared_param_type shared_params)
  {
    traits::execute(agent_idx, param, f, shared_params);
  },
  shape,
  shared_init);
}

}

int main()
{
//  using agent_type = std::sequential_agent_new;
//  using traits = std::execution_agent_traits_new<agent_type>;
//  auto param = traits::param_type(0,1);
//
//  bulk_invoke_new(std::seq, [](agent_type& self)
//  {
//    std::cout << "self.index(): " << self.index() << std::endl;
//  });

//  std::mutex mut;
//  bulk_invoke_new(std::con(10), [&mut](std::concurrent_agent_new& self)
//  {
//    mut.lock();
//    std::cout << "self.index(): " << self.index() << " arriving at barrier" << std::endl;
//    mut.unlock();
//
//    self.wait();
//
//    mut.lock();
//    std::cout << "self.index(): " << self.index() << " departing barrier" << std::endl;
//    mut.unlock();
//  });

//  using inner_agent_type = std::sequential_agent_new;
//  using inner_traits = std::execution_agent_traits_new<inner_agent_type>;
//  auto inner_param = inner_traits::param_type(0,2);
//
//  using outer_agent_type = inner_agent_type;
//  using outer_traits = std::execution_agent_traits_new<outer_agent_type>;
//  auto outer_param = inner_traits::param_type(0,3);
//
//  using agent_type = std::sequential_group_new<inner_agent_type>;
//  using traits = std::execution_agent_traits_new<agent_type>;
//  auto param = traits::param_type(outer_param, inner_param);
//
//  using seq_seq_type = std::__nested_execution_policy_new<
//    std::sequential_execution_policy,
//    std::sequential_execution_policy
//  >;
//
//  auto seq_seq = seq_seq_type(std::seq(3), std::seq(2));
//
//  bulk_invoke_new(seq_seq, [](std::sequential_group_new<std::sequential_agent_new>& self)
//  {
//    std::cout << "index: (" << self.index() << ", " << self.inner().index() << ")" << std::endl;
//  });
//
//
//  using seq_seq_seq_type = std::__nested_execution_policy_new<
//    std::sequential_execution_policy,
//    seq_seq_type
//  >;
//
//  auto seq_seq_seq = seq_seq_seq_type(std::seq(4), seq_seq);
//
//  bulk_invoke_new(seq_seq_seq, [](std::sequential_group_new<std::sequential_group_new<std::sequential_agent_new>>& self)
//  {
//    std::cout << "index: (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ")" << std::endl;
//  });

  using seq_con_type = std::__nested_execution_policy_new<
    std::sequential_execution_policy,
    std::concurrent_execution_policy
  >;

  auto seq_con = seq_con_type(std::seq(2), std::con(2));

  using con_seq_con_type = std::__nested_execution_policy_new<
    std::concurrent_execution_policy,
    seq_con_type
  >;

  auto con_seq_con = con_seq_con_type(std::con(2), seq_con);

  std::mutex mut;
  bulk_invoke_new(con_seq_con, [&mut](std::concurrent_group_new<std::sequential_group_new<std::concurrent_agent_new>>& self)
  {
    // the first agent in the first subgroup waits at the top-level barrier
    if(self.inner().index() == 0 && self.inner().inner().index() == 0)
    {
      mut.lock();
      std::cout << "(" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") arriving at top-level barrier" << std::endl;
      mut.unlock();

      self.wait();

      mut.lock();
      std::cout << "(" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") departing top-level barrier" << std::endl;
      mut.unlock();
    }

    // every agent waits at the inner most barrier
    mut.lock();
    std::cout << "  (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") arriving at bottom-level barrier" << std::endl;
    mut.unlock();

    self.inner().inner().wait();

    mut.lock();
    std::cout << "  (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") departing bottom-level barrier" << std::endl;
    mut.unlock();
  });

  return 0;
}

<commit_msg>Reenable all tests in test_new_stuff.cpp<commit_after>#include <execution_agent_new>
#include <execution_policy>
#include <__nested_execution_policy_new>

namespace std
{

template<class ExecutionAgent, class BulkExecutor, class ExecutionCategory, class Function>
void bulk_invoke_new(const __basic_execution_policy<ExecutionAgent,BulkExecutor,ExecutionCategory>& exec,
                     Function f)
{
  using traits = std::execution_agent_traits_new<ExecutionAgent>;

  auto param = exec.param();
  auto shape = traits::shape(param);
  auto shared_init = traits::make_shared_initializer(param);

  using executor_index = typename executor_traits<BulkExecutor>::index_type;
  using shared_param_type = typename executor_traits<BulkExecutor>::template shared_param_type<decltype(shared_init)>;

  return executor_traits<BulkExecutor>::bulk_invoke(exec.executor(), [=](executor_index agent_idx, shared_param_type shared_params)
  {
    traits::execute(agent_idx, param, f, shared_params);
  },
  shape,
  shared_init);
}

}

int main()
{
  bulk_invoke_new(std::seq, [](std::sequential_agent_new& self)
  {
    std::cout << "self.index(): " << self.index() << std::endl;
  });

  std::mutex mut;
  bulk_invoke_new(std::con(10), [&mut](std::concurrent_agent_new& self)
  {
    mut.lock();
    std::cout << "self.index(): " << self.index() << " arriving at barrier" << std::endl;
    mut.unlock();

    self.wait();

    mut.lock();
    std::cout << "self.index(): " << self.index() << " departing barrier" << std::endl;
    mut.unlock();
  });

  using inner_agent_type = std::sequential_agent_new;
  using inner_traits = std::execution_agent_traits_new<inner_agent_type>;
  auto inner_param = inner_traits::param_type(0,2);

  using outer_agent_type = inner_agent_type;
  using outer_traits = std::execution_agent_traits_new<outer_agent_type>;
  auto outer_param = inner_traits::param_type(0,3);

  using agent_type = std::sequential_group_new<inner_agent_type>;
  using traits = std::execution_agent_traits_new<agent_type>;
  auto param = traits::param_type(outer_param, inner_param);

  using seq_seq_type = std::__nested_execution_policy_new<
    std::sequential_execution_policy,
    std::sequential_execution_policy
  >;

  auto seq_seq = seq_seq_type(std::seq(3), std::seq(2));

  bulk_invoke_new(seq_seq, [](std::sequential_group_new<std::sequential_agent_new>& self)
  {
    std::cout << "index: (" << self.index() << ", " << self.inner().index() << ")" << std::endl;
  });

  using seq_seq_seq_type = std::__nested_execution_policy_new<
    std::sequential_execution_policy,
    seq_seq_type
  >;

  auto seq_seq_seq = seq_seq_seq_type(std::seq(4), seq_seq);

  bulk_invoke_new(seq_seq_seq, [](std::sequential_group_new<std::sequential_group_new<std::sequential_agent_new>>& self)
  {
    std::cout << "index: (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ")" << std::endl;
  });

  using seq_con_type = std::__nested_execution_policy_new<
    std::sequential_execution_policy,
    std::concurrent_execution_policy
  >;

  auto seq_con = seq_con_type(std::seq(2), std::con(2));

  using con_seq_con_type = std::__nested_execution_policy_new<
    std::concurrent_execution_policy,
    seq_con_type
  >;

  auto con_seq_con = con_seq_con_type(std::con(2), seq_con);

  bulk_invoke_new(con_seq_con, [&mut](std::concurrent_group_new<std::sequential_group_new<std::concurrent_agent_new>>& self)
  {
    // the first agent in the first subgroup waits at the top-level barrier
    if(self.inner().index() == 0 && self.inner().inner().index() == 0)
    {
      mut.lock();
      std::cout << "(" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") arriving at top-level barrier" << std::endl;
      mut.unlock();

      self.wait();

      mut.lock();
      std::cout << "(" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") departing top-level barrier" << std::endl;
      mut.unlock();
    }

    // every agent waits at the inner most barrier
    mut.lock();
    std::cout << "  (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") arriving at bottom-level barrier" << std::endl;
    mut.unlock();

    self.inner().inner().wait();

    mut.lock();
    std::cout << "  (" << self.index() << ", " << self.inner().index() << ", " << self.inner().inner().index() << ") departing bottom-level barrier" << std::endl;
    mut.unlock();
  });

  return 0;
}

<|endoftext|>


<commit_before>/*******************************************************************************
 * include/util/histogram.hpp
 *
 * Copyright (C) 2017 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 * Copyright (C) 2018 Marvin Löbel <loebel.marvin@gmail.com>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <cstdint>
#include <limits>
#include <type_traits>
#include <unordered_map>

template <typename AlphabetType>
struct histogram_entry {
  const AlphabetType symbol;
  uint64_t frequency;

  histogram_entry(AlphabetType s, uint64_t f): symbol(s), frequency(f) {}

  friend std::ostream& operator <<(std::ostream& os,
    const histogram_entry& he) {

    return os << "[ " << he.symbol << "(" << uint64_t(he.symbol) << ") | "
              << he.frequency << " ]";
  }
}; // struct histogram_entry

// TODO: Optimize memory layout and frequency search

template <typename AlphabetType>
class histogram {

public:
  histogram() = default;

  histogram(const AlphabetType* text, const uint64_t size,
    const uint64_t reduced_sigma = 0) : max_symbol_(0){

    if (std::is_same<AlphabetType, uint8_t>::value) {
      const uint64_t max_char = std::max(
        static_cast<std::remove_cv_t<decltype(reduced_sigma)>>(
          std::numeric_limits<uint8_t>::max() + 1), reduced_sigma);
      std::vector<uint64_t> hist(max_char, 0);
      for (uint64_t pos = 0; pos < size; ++pos) {
        const AlphabetType cur_char = text[pos];
        max_symbol_ = std::max(max_symbol_, cur_char);
        ++hist[cur_char];
      }
      for (uint64_t pos = 0; pos < hist.size(); ++pos) {
        if (hist[pos] > 0) {
          data_.emplace_back(AlphabetType(pos), hist[pos]);
        }
      }
    } else {
      std::unordered_map<AlphabetType, uint64_t> symbol_list;
      for (uint64_t pos = 0; pos < size; ++pos) {
        const AlphabetType cur_char = text[pos];
        max_symbol_ = std::max(max_symbol_, cur_char);
        auto result = symbol_list.find(cur_char);
        if (result == symbol_list.end()) { symbol_list.emplace(cur_char, 1); }
        else { ++(result->second); }
      }
      for (const auto& symbol : symbol_list) {
        data_.emplace_back(symbol.first, symbol.second);
      }
    }
  }

  histogram(std::vector<histogram> const& others): max_symbol_(0) {
    for (auto& o: others) {
      max_symbol_ = std::max(max_symbol_, o.max_symbol_);
      for (size_t j = 0; j < o.size(); j++) {
        bool found = false;
        for (size_t i = 0; i < size(); i++) {
          if ((*this)[i].symbol == o[j].symbol) {
            (*this)[i].frequency += o[j].frequency;
            found = true;
            break;
          }
        }
        if (!found) {
          data_.emplace_back(o[j].symbol, o[j].frequency);
        }
      }
    }
  }

  AlphabetType max_symbol() const {
    return max_symbol_;
  };

  uint64_t size() const {
    return data_.size();
  }

  // Returns the frequency of a symbol. Does not check if symbol exists. If the
  // symbol does not exits, the behavior is undefined.
  uint64_t frequency(const AlphabetType symbol) const {
    for (const auto& he : data_) {
      if (he.symbol == symbol) {
        return he.frequency;
      }
    }
    return 0;
  }

  histogram_entry<AlphabetType>& operator [](const uint64_t index) {
    return data_[index];
  }

  histogram_entry<AlphabetType> operator [](const uint64_t index) const {
    return data_[index];
  }

private:
  AlphabetType max_symbol_;
  std::vector<histogram_entry<AlphabetType>> data_;
}; // class histogram

/******************************************************************************/
<commit_msg>rename variable<commit_after>/*******************************************************************************
 * include/util/histogram.hpp
 *
 * Copyright (C) 2017 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 * Copyright (C) 2018 Marvin Löbel <loebel.marvin@gmail.com>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <cstdint>
#include <limits>
#include <type_traits>
#include <unordered_map>

template <typename AlphabetType>
struct histogram_entry {
  const AlphabetType symbol;
  uint64_t frequency;

  histogram_entry(AlphabetType s, uint64_t f): symbol(s), frequency(f) {}

  friend std::ostream& operator <<(std::ostream& os,
    const histogram_entry& he) {

    return os << "[ " << he.symbol << "(" << uint64_t(he.symbol) << ") | "
              << he.frequency << " ]";
  }
}; // struct histogram_entry

// TODO: Optimize memory layout and frequency search

template <typename AlphabetType>
class histogram {

public:
  histogram() = default;

  histogram(const AlphabetType* text, const uint64_t size,
    const uint64_t reduced_sigma = 0) : max_symbol_(0){

    if (std::is_same<AlphabetType, uint8_t>::value) {
      const uint64_t alphabet_size = std::max(
        static_cast<std::remove_cv_t<decltype(reduced_sigma)>>(
          std::numeric_limits<uint8_t>::max() + 1), reduced_sigma);
      std::vector<uint64_t> hist(alphabet_size, 0);
      for (uint64_t pos = 0; pos < size; ++pos) {
        const AlphabetType cur_char = text[pos];
        max_symbol_ = std::max(max_symbol_, cur_char);
        ++hist[cur_char];
      }
      for (uint64_t pos = 0; pos < hist.size(); ++pos) {
        if (hist[pos] > 0) {
          data_.emplace_back(AlphabetType(pos), hist[pos]);
        }
      }
    } else {
      std::unordered_map<AlphabetType, uint64_t> symbol_list;
      for (uint64_t pos = 0; pos < size; ++pos) {
        const AlphabetType cur_char = text[pos];
        max_symbol_ = std::max(max_symbol_, cur_char);
        auto result = symbol_list.find(cur_char);
        if (result == symbol_list.end()) { symbol_list.emplace(cur_char, 1); }
        else { ++(result->second); }
      }
      for (const auto& symbol : symbol_list) {
        data_.emplace_back(symbol.first, symbol.second);
      }
    }
  }

  histogram(std::vector<histogram> const& others): max_symbol_(0) {
    for (auto& o: others) {
      max_symbol_ = std::max(max_symbol_, o.max_symbol_);
      for (size_t j = 0; j < o.size(); j++) {
        bool found = false;
        for (size_t i = 0; i < size(); i++) {
          if ((*this)[i].symbol == o[j].symbol) {
            (*this)[i].frequency += o[j].frequency;
            found = true;
            break;
          }
        }
        if (!found) {
          data_.emplace_back(o[j].symbol, o[j].frequency);
        }
      }
    }
  }

  AlphabetType max_symbol() const {
    return max_symbol_;
  };

  uint64_t size() const {
    return data_.size();
  }

  // Returns the frequency of a symbol. Does not check if symbol exists. If the
  // symbol does not exits, the behavior is undefined.
  uint64_t frequency(const AlphabetType symbol) const {
    for (const auto& he : data_) {
      if (he.symbol == symbol) {
        return he.frequency;
      }
    }
    return 0;
  }

  histogram_entry<AlphabetType>& operator [](const uint64_t index) {
    return data_[index];
  }

  histogram_entry<AlphabetType> operator [](const uint64_t index) const {
    return data_[index];
  }

private:
  AlphabetType max_symbol_;
  std::vector<histogram_entry<AlphabetType>> data_;
}; // class histogram

/******************************************************************************/
<|endoftext|>


<commit_before>/**
    httpparser.cpp
    HTTP Parser
    Will parse streaming data through parseHttp() method

    @author Bertrand Martel
    @version 1.0
*/
#include <cctype>

#include "httpparser.h"
#include <vector>
#include <sstream>
#include <QString>
#include <iostream>
#include "utils/stringutils.h"
#include "httpconstants.h"
#include "httpstates.h"
#include "protocol/inter/http/httpconsumer.h"
#include "protocol/inter/http/httpframe.h"

using namespace std;

std::vector<httpconstants::statusCodeStruct> httpconstants::http_status_code_list;

/**
 * @brief The HttpParser class
 *      Main HTTP Parser

 *      Will parse streaming data
 */
httpparser::httpparser()
{
    debug=false;
}

/**
 * @brief setDebug
 *      set debug mode for http parser
 * @param debug
 */
void httpparser::setDebug(bool debugArg)
{
    debug=debugArg;
}

/**
 * @brief parseHttp
 *      parse streaming data
 *
 * @param data
 *      streaming data
 */
void httpparser::parseHttp(QByteArray* data,httpconsumer *consumer)
{
    if (debug)
        cout << "http state init : " << consumer->getHttpState() << endl;

    switch (consumer->getHttpState())
    {
        case HTTP_STATE_INIT:
        {
            consumer->addNewHttpFrame(new httpframe());
            consumer->setBodyProcess(false);

            consumer->getCurrentHttpFrame()->setBody("");

            //TODO:test if pointer null here

            (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

            consumer->setBodyLength(0);
            consumer->setBodyIndex(0);
            consumer->getCurrentHttpFrame()->setFinishedProcessing(false);
            consumer->setFinishedProcessing(false);

            if (debug)
                cout << "HTTP STATE INIT" << endl;
        }
        case HTTP_STATE_VERSION:
        {
            if (debug)
                cout << "HTTP STATE VERSION" << endl;

            std::vector<std::string>  initStateLine =  stringutils::split(QString(data->data()).toStdString(), ' ');

            if (initStateLine.size()>2)
            {
                QString firstElement(initStateLine.at(0).data());

                QString thirdElement(initStateLine.at(2).data());

                if (firstElement.indexOf("HTTP/")!=-1)
                {
                    bool isMethodVal  = false;
                    bool isStatusNumVal = false;

                    consumer->getCurrentHttpFrame()->setMethod("");
                    consumer->getCurrentHttpFrame()->setStatusCode(0);
                    consumer->getCurrentHttpFrame()->setUri("");
                    consumer->getCurrentHttpFrame()->setMethod("");
                    consumer->getCurrentHttpFrame()->setQueryString("");

                    QString secondElement(initStateLine.at(1).data());
                    isMethodVal=isMethod(secondElement.toLocal8Bit().data());
                    if (isMethodVal)
                    {
                        consumer->getCurrentHttpFrame()->setMethod(secondElement.toStdString());
                        consumer->getCurrentHttpFrame()->setUri(initStateLine.at(2));
                    }
                    else
                    {
                        isStatusNumVal=isStatusNum(secondElement.toLocal8Bit().data());

                        if (isStatusNumVal)
                        {
                            consumer->getCurrentHttpFrame()->setStatusCode(atoi(secondElement.toLocal8Bit().data()));
                            consumer->getCurrentHttpFrame()->setQueryString(initStateLine.at(2).data());
                        }
                        else
                        {
                            if (debug)
                                cout  << "Http parse error occured. No http status number or method found." << endl;
                        }
                    }

                    (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

                    consumer->setHttpState(HTTP_STATE_HEADERS);
                }
                else if (thirdElement.indexOf("HTTP/")!=-1)
                {
                    bool isMethodVal  = false;

                    consumer->getCurrentHttpFrame()->setUri("");
                    consumer->getCurrentHttpFrame()->setMethod("");

                    QString firstElement(initStateLine.at(0).data());
                    isMethodVal=isMethod(firstElement.toLocal8Bit().data());

                    if (isMethodVal)
                    {
                        consumer->getCurrentHttpFrame()->setUri(initStateLine.at(1));
                        consumer->getCurrentHttpFrame()->setMethod(firstElement.toStdString());
                    }
                    else
                    {
                       if (debug)
                            cout  << "Http parse error occured. No method found." << endl;
                    }

                    (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

                    consumer->setHttpState(HTTP_STATE_HEADERS);
                }
                else
                {
                    if (debug)
                        cout  << "Http parse error occured. No http version was specified in header." << endl;
                }
            }
            else
            {
                if (debug)
                    cout  << "Http parse error occured. Http version header is undefined." << endl;
            }
            break;
        }
        case HTTP_STATE_HEADERS:
        {
            if (debug)
                cout << "HTTP STATE HEADERS" << endl;

            int indexOfPoint = data->indexOf(":");
            if (indexOfPoint!=-1)
            {
               string currentHeader(QString(data->data()).toStdString());

                (*consumer->getCurrentHttpFrame()->getHeaders())[currentHeader.substr(0,indexOfPoint)]=currentHeader.substr(indexOfPoint+1,currentHeader.length());
                if (debug)
                    cout << currentHeader.substr(0,indexOfPoint).data() << " => " <<currentHeader.substr(indexOfPoint+1,currentHeader.length()).data() <<endl;
            }
            else
            {
                if ((*consumer->getCurrentHttpFrame()->getHeaders()).find(HTTP_HEADERS_CONTENT_LENGTH)==(*consumer->getCurrentHttpFrame()->getHeaders()).end())
                {
                    if (debug)
                        cout << "return to HTTP_INIT state" << endl;
                    consumer->setBodyLength(0);
                    consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                    consumer->setFinishedProcessing(true);

                    consumer->setHttpState(HTTP_STATE_INIT);
                }
                else
                {
                    if (debug)
                        cout << "continue to HTTP_BODY state" << endl;
                    consumer->setBodyLength(atoi((*consumer->getCurrentHttpFrame()->getHeaders())[HTTP_HEADERS_CONTENT_LENGTH].data()));
                    consumer->setHttpState(HTTP_STATE_BODY);
                    consumer->setBodyProcess(true);
                }
            }
            break;
        }
        case HTTP_STATE_BODY:
        {
            if (debug)
                cout << "HTTP STATE BODY" << endl;
            if (consumer->getBodyLength()==0)
            {
                if (debug)
                    cout << "no body to read" << endl;
                consumer->setBodyProcess(false);
                consumer->setBodyLength(0);
                consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                consumer->setFinishedProcessing(true);
                consumer->setHttpState(HTTP_STATE_INIT);
            }
            else if (data->length()>=consumer->getBodyLength() && consumer->getBodyIndex()==0)
            {
                consumer->getCurrentHttpFrame()->setBody(QString(data->data()).toStdString().substr(0,consumer->getBodyLength()));

                consumer->setBodyProcess(false);
                consumer->setBodyLength(0);
                consumer->setBodyIndex(0);

                consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                consumer->setFinishedProcessing(true);
                consumer->setHttpState(HTTP_STATE_INIT);
            }
            else
            {
                if (debug)
                    cout << "not all body can be read";
                if (consumer->getBodyIndex()==0 && data->length()>=consumer->getBodyLength())
                {
                    consumer->getCurrentHttpFrame()->setBody(QString(data->data()).toStdString().substr(0,data->length()));

                    consumer->setBodyProcess(false);
                    consumer->setBodyLength(0);
                    consumer->setBodyIndex(0);

                    consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                    consumer->setFinishedProcessing(true);
                    consumer->setHttpState(HTTP_STATE_INIT);
                }
                else
                {
                    if (data->length()>=(consumer->getBodyLength()-consumer->getBodyIndex()))
                    {
                        std::string bodyTemp = consumer->getCurrentHttpFrame()->getBody();

                        bodyTemp+=QString(data->data()).toStdString().substr(0,consumer->getBodyLength()-consumer->getBodyIndex());

                        consumer->getCurrentHttpFrame()->setBody(bodyTemp);

                        consumer->setBodyProcess(false);
                        consumer->setBodyLength(0);
                        consumer->setBodyIndex(0);

                        consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                        consumer->setFinishedProcessing(true);
                        consumer->setHttpState(HTTP_STATE_INIT);
                    }
                    else
                    {
                        std::string bodyTemp = consumer->getCurrentHttpFrame()->getBody();
                        bodyTemp+=QString(data->data()).toStdString().substr(0,data->length());
                        consumer->getCurrentHttpFrame()->setBody(bodyTemp);

                        int bodyIndexTemp = consumer->getBodyIndex();
                        bodyIndexTemp+=data->length();
                        consumer->setBodyIndex(bodyIndexTemp);
                    }
                }
            }
            break;
        }
    }
}

/**
 * @brief isMethod
 *      check if input data is a valid HTTP rest method
 *
 * @param data
 *      data to be tested
 * @return
 *      true if data is valid REST HTTP method
 */
bool httpparser::isMethod(char* data)
{
    if (strcmp(data,HTTP_METHOD_GET)==0 || strcmp(data,HTTP_METHOD_POST)==0 || strcmp(data,HTTP_METHOD_PUT)==0 || strcmp(data,HTTP_METHOD_DELETE)==0)
    {
        return true;
    }
    return false;
}

/**
 * @brief isStatusNum
 *     check if input data is a valid HTTP status code
 * @param data
 *      data to be tested
 * @return
 *      true if data is valid HTTP status code
 */
bool httpparser::isStatusNum(char * data)
{

    if (stringutils::isNum(data))
    {
        int code = atoi(data);
        for(std::vector<httpconstants::statusCodeStruct>::iterator it = httpconstants::http_status_code_list.begin(); it != httpconstants::http_status_code_list.end(); ++it) {
            if (code==(*it).code_value)
                return true;
        }
    }
    return false;
}
<commit_msg>trim header in http parser<commit_after>/**
    httpparser.cpp
    HTTP Parser
    Will parse streaming data through parseHttp() method

    @author Bertrand Martel
    @version 1.0
*/
#include <cctype>

#include "httpparser.h"
#include <vector>
#include <sstream>
#include <QString>
#include <iostream>
#include "utils/stringutils.h"
#include "httpconstants.h"
#include "httpstates.h"
#include "protocol/inter/http/httpconsumer.h"
#include "protocol/inter/http/httpframe.h"

using namespace std;

std::vector<httpconstants::statusCodeStruct> httpconstants::http_status_code_list;

/**
 * @brief The HttpParser class
 *      Main HTTP Parser

 *      Will parse streaming data
 */
httpparser::httpparser()
{
    debug=false;
}

/**
 * @brief setDebug
 *      set debug mode for http parser
 * @param debug
 */
void httpparser::setDebug(bool debugArg)
{
    debug=debugArg;
}

/**
 * @brief parseHttp
 *      parse streaming data
 *
 * @param data
 *      streaming data
 */
void httpparser::parseHttp(QByteArray* data,httpconsumer *consumer)
{
    if (debug)
        cout << "http state init : " << consumer->getHttpState() << endl;

    switch (consumer->getHttpState())
    {
        case HTTP_STATE_INIT:
        {
            consumer->addNewHttpFrame(new httpframe());
            consumer->setBodyProcess(false);

            consumer->getCurrentHttpFrame()->setBody("");

            //TODO:test if pointer null here

            (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

            consumer->setBodyLength(0);
            consumer->setBodyIndex(0);
            consumer->getCurrentHttpFrame()->setFinishedProcessing(false);
            consumer->setFinishedProcessing(false);

            if (debug)
                cout << "HTTP STATE INIT" << endl;
        }
        case HTTP_STATE_VERSION:
        {
            if (debug)
                cout << "HTTP STATE VERSION" << endl;

            std::vector<std::string>  initStateLine =  stringutils::split(QString(data->data()).toStdString(), ' ');

            if (initStateLine.size()>2)
            {
                QString firstElement(initStateLine.at(0).data());

                QString thirdElement(initStateLine.at(2).data());

                if (firstElement.indexOf("HTTP/")!=-1)
                {
                    bool isMethodVal  = false;
                    bool isStatusNumVal = false;

                    consumer->getCurrentHttpFrame()->setMethod("");
                    consumer->getCurrentHttpFrame()->setStatusCode(0);
                    consumer->getCurrentHttpFrame()->setUri("");
                    consumer->getCurrentHttpFrame()->setMethod("");
                    consumer->getCurrentHttpFrame()->setQueryString("");

                    QString secondElement(initStateLine.at(1).data());
                    isMethodVal=isMethod(secondElement.toLocal8Bit().data());
                    if (isMethodVal)
                    {
                        consumer->getCurrentHttpFrame()->setMethod(secondElement.toStdString());
                        consumer->getCurrentHttpFrame()->setUri(initStateLine.at(2));
                    }
                    else
                    {
                        isStatusNumVal=isStatusNum(secondElement.toLocal8Bit().data());

                        if (isStatusNumVal)
                        {
                            consumer->getCurrentHttpFrame()->setStatusCode(atoi(secondElement.toLocal8Bit().data()));
                            consumer->getCurrentHttpFrame()->setQueryString(initStateLine.at(2).data());
                        }
                        else
                        {
                            if (debug)
                                cout  << "Http parse error occured. No http status number or method found." << endl;
                        }
                    }

                    (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

                    consumer->setHttpState(HTTP_STATE_HEADERS);
                }
                else if (thirdElement.indexOf("HTTP/")!=-1)
                {
                    bool isMethodVal  = false;

                    consumer->getCurrentHttpFrame()->setUri("");
                    consumer->getCurrentHttpFrame()->setMethod("");

                    QString firstElement(initStateLine.at(0).data());
                    isMethodVal=isMethod(firstElement.toLocal8Bit().data());

                    if (isMethodVal)
                    {
                        consumer->getCurrentHttpFrame()->setUri(initStateLine.at(1));
                        consumer->getCurrentHttpFrame()->setMethod(firstElement.toStdString());
                    }
                    else
                    {
                       if (debug)
                            cout  << "Http parse error occured. No method found." << endl;
                    }

                    (*consumer->getCurrentHttpFrame()->getHeaders()).clear();

                    consumer->setHttpState(HTTP_STATE_HEADERS);
                }
                else
                {
                    if (debug)
                        cout  << "Http parse error occured. No http version was specified in header." << endl;
                }
            }
            else
            {
                if (debug)
                    cout  << "Http parse error occured. Http version header is undefined." << endl;
            }
            break;
        }
        case HTTP_STATE_HEADERS:
        {
            if (debug)
                cout << "HTTP STATE HEADERS" << endl;

            int indexOfPoint = data->indexOf(":");
            if (indexOfPoint!=-1)
            {
               string currentHeader(QString(data->data()).trimmed().toStdString());

               (*consumer->getCurrentHttpFrame()->getHeaders())[currentHeader.substr(0,indexOfPoint)]=QString(currentHeader.substr(indexOfPoint+1,currentHeader.length()).data()).trimmed().toStdString();
                if (debug)
                    cout << currentHeader.substr(0,indexOfPoint).data() << " => " <<currentHeader.substr(indexOfPoint+1,currentHeader.length()).data() <<endl;
            }
            else
            {
                if ((*consumer->getCurrentHttpFrame()->getHeaders()).find(HTTP_HEADERS_CONTENT_LENGTH)==(*consumer->getCurrentHttpFrame()->getHeaders()).end())
                {
                    if (debug)
                        cout << "return to HTTP_INIT state" << endl;
                    consumer->setBodyLength(0);
                    consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                    consumer->setFinishedProcessing(true);

                    consumer->setHttpState(HTTP_STATE_INIT);
                }
                else
                {
                    if (debug)
                        cout << "continue to HTTP_BODY state" << endl;
                    consumer->setBodyLength(atoi((*consumer->getCurrentHttpFrame()->getHeaders())[HTTP_HEADERS_CONTENT_LENGTH].data()));
                    consumer->setHttpState(HTTP_STATE_BODY);
                    consumer->setBodyProcess(true);
                }
            }
            break;
        }
        case HTTP_STATE_BODY:
        {
            if (debug)
                cout << "HTTP STATE BODY" << endl;
            if (consumer->getBodyLength()==0)
            {
                if (debug)
                    cout << "no body to read" << endl;
                consumer->setBodyProcess(false);
                consumer->setBodyLength(0);
                consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                consumer->setFinishedProcessing(true);
                consumer->setHttpState(HTTP_STATE_INIT);
            }
            else if (data->length()>=consumer->getBodyLength() && consumer->getBodyIndex()==0)
            {
                consumer->getCurrentHttpFrame()->setBody(QString(data->data()).toStdString().substr(0,consumer->getBodyLength()));

                consumer->setBodyProcess(false);
                consumer->setBodyLength(0);
                consumer->setBodyIndex(0);

                consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                consumer->setFinishedProcessing(true);
                consumer->setHttpState(HTTP_STATE_INIT);
            }
            else
            {
                if (debug)
                    cout << "not all body can be read";
                if (consumer->getBodyIndex()==0 && data->length()>=consumer->getBodyLength())
                {
                    consumer->getCurrentHttpFrame()->setBody(QString(data->data()).toStdString().substr(0,data->length()));

                    consumer->setBodyProcess(false);
                    consumer->setBodyLength(0);
                    consumer->setBodyIndex(0);

                    consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                    consumer->setFinishedProcessing(true);
                    consumer->setHttpState(HTTP_STATE_INIT);
                }
                else
                {
                    if (data->length()>=(consumer->getBodyLength()-consumer->getBodyIndex()))
                    {
                        std::string bodyTemp = consumer->getCurrentHttpFrame()->getBody();

                        bodyTemp+=QString(data->data()).toStdString().substr(0,consumer->getBodyLength()-consumer->getBodyIndex());

                        consumer->getCurrentHttpFrame()->setBody(bodyTemp);

                        consumer->setBodyProcess(false);
                        consumer->setBodyLength(0);
                        consumer->setBodyIndex(0);

                        consumer->getCurrentHttpFrame()->setFinishedProcessing(true);
                        consumer->setFinishedProcessing(true);
                        consumer->setHttpState(HTTP_STATE_INIT);
                    }
                    else
                    {
                        std::string bodyTemp = consumer->getCurrentHttpFrame()->getBody();
                        bodyTemp+=QString(data->data()).toStdString().substr(0,data->length());
                        consumer->getCurrentHttpFrame()->setBody(bodyTemp);

                        int bodyIndexTemp = consumer->getBodyIndex();
                        bodyIndexTemp+=data->length();
                        consumer->setBodyIndex(bodyIndexTemp);
                    }
                }
            }
            break;
        }
    }
}

/**
 * @brief isMethod
 *      check if input data is a valid HTTP rest method
 *
 * @param data
 *      data to be tested
 * @return
 *      true if data is valid REST HTTP method
 */
bool httpparser::isMethod(char* data)
{
    if (strcmp(data,HTTP_METHOD_GET)==0 || strcmp(data,HTTP_METHOD_POST)==0 || strcmp(data,HTTP_METHOD_PUT)==0 || strcmp(data,HTTP_METHOD_DELETE)==0)
    {
        return true;
    }
    return false;
}

/**
 * @brief isStatusNum
 *     check if input data is a valid HTTP status code
 * @param data
 *      data to be tested
 * @return
 *      true if data is valid HTTP status code
 */
bool httpparser::isStatusNum(char * data)
{

    if (stringutils::isNum(data))
    {
        int code = atoi(data);
        for(std::vector<httpconstants::statusCodeStruct>::iterator it = httpconstants::http_status_code_list.begin(); it != httpconstants::http_status_code_list.end(); ++it) {
            if (code==(*it).code_value)
                return true;
        }
    }
    return false;
}
<|endoftext|>


<commit_before>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef BTREE_LEAF_NODE_HPP_
#define BTREE_LEAF_NODE_HPP_

#include <string>
#include <utility>
#include <vector>

#include "errors.hpp"
#include <boost/optional.hpp>

#include "arch/compiler.hpp"
#include "btree/types.hpp"
#include "buffer_cache/types.hpp"

class value_sizer_t;
struct btree_key_t;
class repli_timestamp_t;

// TODO: Could key_modification_proof_t not go in this file?

// Originally we thought that leaf node modification functions would
// take a key-modification callback, thus forcing every key/value
// modification to consider the callback.  The problem is that the
// user is supposed to call is_full before calling insert, and is_full
// depends on the value size and can cause a split to happen.  But
// what if the key modification then wants to change the value size?
// Then we would need to redo the logic considering whether we want to
// split the leaf node.  Instead the caller just provides evidence
// that they considered doing the appropriate value modifications, by
// constructing one of these dummy values.
class key_modification_proof_t {
public:
    static key_modification_proof_t real_proof() { return key_modification_proof_t(); }
};

namespace leaf {
class iterator;
class reverse_iterator;
} //namespace leaf

// The leaf node begins with the following struct layout.
ATTR_PACKED(struct leaf_node_t {
    // The value-type-specific magic value.  It's a bit of a hack, but
    // it's possible to construct a value_sizer_t based on this value.
    block_magic_t magic;

    // The size of pair_offsets.
    uint16_t num_pairs;

    // The total size (in bytes) of the live entries and their 2-byte
    // pair offsets in pair_offsets.  (Does not include the size of
    // the live entries' timestamps.)
    uint16_t live_size;

    // The frontmost offset.
    uint16_t frontmost;

    // The first offset whose entry is not accompanied by a timestamp.
    uint16_t tstamp_cutpoint;

    // The pair offsets.
    uint16_t pair_offsets[];

    //Iteration
    typedef leaf::iterator iterator;
    typedef leaf::reverse_iterator reverse_iterator;
});

namespace leaf {

leaf_node_t::iterator begin(const leaf_node_t &leaf_node);
leaf_node_t::iterator end(const leaf_node_t &leaf_node);

leaf_node_t::reverse_iterator rbegin(const leaf_node_t &leaf_node);
leaf_node_t::reverse_iterator rend(const leaf_node_t &leaf_node);

leaf_node_t::iterator inclusive_lower_bound(const btree_key_t *key, const leaf_node_t &leaf_node);
leaf_node_t::reverse_iterator exclusive_upper_bound(const btree_key_t *key, const leaf_node_t &leaf_node);

// We must maintain timestamps and deletion entries as best we can,
// with the following limitations.  The number of timestamps stored
// need not be more than the most `MANDATORY_TIMESTAMPS` recent
// timestamps.  The deletions stored need not be any more than what is
// necessary to fill `(block_size - offsetof(leaf_node_t,
// pair_offsets)) / DELETION_RESERVE_FRACTION` bytes.  For example,
// with a 4084 block size, if the five most recent operations were
// deletions of 250-byte keys, we would only be required to store the
// 2 most recent deletions and the 2 most recent timestamps.
//
// These parameters are in the header because some unit tests are
// based on them.
const int MANDATORY_TIMESTAMPS = 5;
const int DELETION_RESERVE_FRACTION = 10;

std::string strprint_leaf(value_sizer_t *sizer, const leaf_node_t *node);

void print(FILE *fp, value_sizer_t *sizer, const leaf_node_t *node);

class key_value_fscker_t {
public:
    key_value_fscker_t() { }

    // Returns true if there are no problems.
    virtual bool fsck(value_sizer_t *sizer, const btree_key_t *key,
                      const void *value, std::string *msg_out) = 0;

protected:
    virtual ~key_value_fscker_t() { }

    DISABLE_COPYING(key_value_fscker_t);
};

bool fsck(value_sizer_t *sizer, const btree_key_t *left_exclusive_or_null, const btree_key_t *right_inclusive_or_null, const leaf_node_t *node, key_value_fscker_t *fscker, std::string *msg_out);

void validate(value_sizer_t *sizer, const leaf_node_t *node);

void init(value_sizer_t *sizer, leaf_node_t *node);

bool is_empty(const leaf_node_t *node);

bool is_full(value_sizer_t *sizer, const leaf_node_t *node, const btree_key_t *key, const void *value);

bool is_underfull(value_sizer_t *sizer, const leaf_node_t *node);

void split(value_sizer_t *sizer, leaf_node_t *node, leaf_node_t *sibling,
           btree_key_t *median_out);

void merge(value_sizer_t *sizer, leaf_node_t *left, leaf_node_t *right);

// The pointers in `moved_values_out` point to positions in `node` and
// will be valid as long as `node` remains unchanged.
bool level(value_sizer_t *sizer, int nodecmp_node_with_sib, leaf_node_t *node,
           leaf_node_t *sibling, btree_key_t *replacement_key_out,
           std::vector<const void *> *moved_values_out);

bool is_mergable(value_sizer_t *sizer, const leaf_node_t *node, const leaf_node_t *sibling);

bool find_key(const leaf_node_t *node, const btree_key_t *key, int *index_out);

bool lookup(value_sizer_t *sizer, const leaf_node_t *node, const btree_key_t *key, void *value_out);

void insert(
        value_sizer_t *sizer,
        leaf_node_t *node,
        const btree_key_t *key,
        const void *value,
        repli_timestamp_t tstamp,
        /* the recency of the buf_t that `node` comes from */
        repli_timestamp_t maximum_existing_tstamp,
        UNUSED key_modification_proof_t km_proof);

/* `remove()` removes any preexisting value or tombstone, then creates a tombstone unless
`tstamp` is before `tstamp_cutpoint`. */
void remove(
        value_sizer_t *sizer,
        leaf_node_t *node,
        const btree_key_t *key,
        repli_timestamp_t tstamp,
        /* the recency of the buf_t that `node` comes from */
        repli_timestamp_t maximum_existing_tstamp,
        key_modification_proof_t km_proof);

/* `erase_presence()` removes any preexisting value or tombstone, but does not create a
new tombstone. */
void erase_presence(value_sizer_t *sizer, leaf_node_t *node, const btree_key_t *key, key_modification_proof_t km_proof);

/* Returns the smallest timestamp such that if a deletion had occurred with that
timestamp, the node would still have a record of it. */
repli_timestamp_t min_deletion_timestamp(
    value_sizer_t *sizer,
    const leaf_node_t *node,
    // the recency of the buf that `node` came from
    repli_timestamp_t maximum_existing_timestamp);

/* Removes all timestamps and deletions. Optionally: specify a `min_timestamp`
so that only timestamps and deletions earlier than the given timestamp are removed. */
void erase_deletions(
    value_sizer_t *sizer, leaf_node_t *node,
    boost::optional<repli_timestamp_t> min_timestamp);

/* Calls `cb` on every entry in the node, whether a real entry or a deletion. The calls
will be in order from most recent to least recent. For entries with no timestamp, the
callback will get `min_deletion_timestamp() - 1`. */
continue_bool_t visit_entries(
    value_sizer_t *sizer,
    const leaf_node_t *node,
    // the recency of the buf that `node` came from
    repli_timestamp_t maximum_existing_timestamp,
    const std::function<continue_bool_t(
        const btree_key_t *key,
        repli_timestamp_t timestamp,
        const void *value   /* null for deletion */
        )> &cb);

class iterator {
public:
    iterator();
    iterator(const leaf_node_t *node, int index);
    std::pair<const btree_key_t *, const void *> operator*() const;
    iterator &operator++();
    iterator &operator--();
    bool operator==(const iterator &other) const;
    bool operator!=(const iterator &other) const;
    bool operator<(const iterator &other) const;
    bool operator>(const iterator &other) const;
    bool operator<=(const iterator &other) const;
    bool operator>=(const iterator &other) const;
private:
    /* negative return => this < other */
    int cmp(const iterator &other) const;
    const leaf_node_t *node_;
    int index_;
};

class reverse_iterator {
public:
    reverse_iterator();
    reverse_iterator(const leaf_node_t *node, int index);
    std::pair<const btree_key_t *, const void *> operator*() const;
    reverse_iterator &operator++();
    reverse_iterator &operator--();
    bool operator==(const reverse_iterator &other) const;
    bool operator!=(const reverse_iterator &other) const;
    bool operator<(const reverse_iterator &other) const;
    bool operator>(const reverse_iterator &other) const;
    bool operator<=(const reverse_iterator &other) const;
    bool operator>=(const reverse_iterator &other) const;
private:
    iterator inner_;
};

}  // namespace leaf

#endif  // BTREE_LEAF_NODE_HPP_
<commit_msg>Compiler-specific attributes and macros<commit_after>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef BTREE_LEAF_NODE_HPP_
#define BTREE_LEAF_NODE_HPP_

#include <string>
#include <utility>
#include <vector>

#include "arch/compiler.hpp"
#include "errors.hpp"
#include <boost/optional.hpp>

#include "arch/compiler.hpp"
#include "btree/types.hpp"
#include "buffer_cache/types.hpp"

class value_sizer_t;
struct btree_key_t;
class repli_timestamp_t;

// TODO: Could key_modification_proof_t not go in this file?

// Originally we thought that leaf node modification functions would
// take a key-modification callback, thus forcing every key/value
// modification to consider the callback.  The problem is that the
// user is supposed to call is_full before calling insert, and is_full
// depends on the value size and can cause a split to happen.  But
// what if the key modification then wants to change the value size?
// Then we would need to redo the logic considering whether we want to
// split the leaf node.  Instead the caller just provides evidence
// that they considered doing the appropriate value modifications, by
// constructing one of these dummy values.
class key_modification_proof_t {
public:
    static key_modification_proof_t real_proof() { return key_modification_proof_t(); }
};

namespace leaf {
class iterator;
class reverse_iterator;
} //namespace leaf

// The leaf node begins with the following struct layout.
ATTR_PACKED(struct leaf_node_t {
    // The value-type-specific magic value.  It's a bit of a hack, but
    // it's possible to construct a value_sizer_t based on this value.
    block_magic_t magic;

    // The size of pair_offsets.
    uint16_t num_pairs;

    // The total size (in bytes) of the live entries and their 2-byte
    // pair offsets in pair_offsets.  (Does not include the size of
    // the live entries' timestamps.)
    uint16_t live_size;

    // The frontmost offset.
    uint16_t frontmost;

    // The first offset whose entry is not accompanied by a timestamp.
    uint16_t tstamp_cutpoint;

    // The pair offsets.
    uint16_t pair_offsets[];

    //Iteration
    typedef leaf::iterator iterator;
    typedef leaf::reverse_iterator reverse_iterator;
});

namespace leaf {

leaf_node_t::iterator begin(const leaf_node_t &leaf_node);
leaf_node_t::iterator end(const leaf_node_t &leaf_node);

leaf_node_t::reverse_iterator rbegin(const leaf_node_t &leaf_node);
leaf_node_t::reverse_iterator rend(const leaf_node_t &leaf_node);

leaf_node_t::iterator inclusive_lower_bound(const btree_key_t *key, const leaf_node_t &leaf_node);
leaf_node_t::reverse_iterator exclusive_upper_bound(const btree_key_t *key, const leaf_node_t &leaf_node);

// We must maintain timestamps and deletion entries as best we can,
// with the following limitations.  The number of timestamps stored
// need not be more than the most `MANDATORY_TIMESTAMPS` recent
// timestamps.  The deletions stored need not be any more than what is
// necessary to fill `(block_size - offsetof(leaf_node_t,
// pair_offsets)) / DELETION_RESERVE_FRACTION` bytes.  For example,
// with a 4084 block size, if the five most recent operations were
// deletions of 250-byte keys, we would only be required to store the
// 2 most recent deletions and the 2 most recent timestamps.
//
// These parameters are in the header because some unit tests are
// based on them.
const int MANDATORY_TIMESTAMPS = 5;
const int DELETION_RESERVE_FRACTION = 10;

std::string strprint_leaf(value_sizer_t *sizer, const leaf_node_t *node);

void print(FILE *fp, value_sizer_t *sizer, const leaf_node_t *node);

class key_value_fscker_t {
public:
    key_value_fscker_t() { }

    // Returns true if there are no problems.
    virtual bool fsck(value_sizer_t *sizer, const btree_key_t *key,
                      const void *value, std::string *msg_out) = 0;

protected:
    virtual ~key_value_fscker_t() { }

    DISABLE_COPYING(key_value_fscker_t);
};

bool fsck(value_sizer_t *sizer, const btree_key_t *left_exclusive_or_null, const btree_key_t *right_inclusive_or_null, const leaf_node_t *node, key_value_fscker_t *fscker, std::string *msg_out);

void validate(value_sizer_t *sizer, const leaf_node_t *node);

void init(value_sizer_t *sizer, leaf_node_t *node);

bool is_empty(const leaf_node_t *node);

bool is_full(value_sizer_t *sizer, const leaf_node_t *node, const btree_key_t *key, const void *value);

bool is_underfull(value_sizer_t *sizer, const leaf_node_t *node);

void split(value_sizer_t *sizer, leaf_node_t *node, leaf_node_t *sibling,
           btree_key_t *median_out);

void merge(value_sizer_t *sizer, leaf_node_t *left, leaf_node_t *right);

// The pointers in `moved_values_out` point to positions in `node` and
// will be valid as long as `node` remains unchanged.
bool level(value_sizer_t *sizer, int nodecmp_node_with_sib, leaf_node_t *node,
           leaf_node_t *sibling, btree_key_t *replacement_key_out,
           std::vector<const void *> *moved_values_out);

bool is_mergable(value_sizer_t *sizer, const leaf_node_t *node, const leaf_node_t *sibling);

bool find_key(const leaf_node_t *node, const btree_key_t *key, int *index_out);

bool lookup(value_sizer_t *sizer, const leaf_node_t *node, const btree_key_t *key, void *value_out);

void insert(
        value_sizer_t *sizer,
        leaf_node_t *node,
        const btree_key_t *key,
        const void *value,
        repli_timestamp_t tstamp,
        /* the recency of the buf_t that `node` comes from */
        repli_timestamp_t maximum_existing_tstamp,
        UNUSED key_modification_proof_t km_proof);

/* `remove()` removes any preexisting value or tombstone, then creates a tombstone unless
`tstamp` is before `tstamp_cutpoint`. */
void remove(
        value_sizer_t *sizer,
        leaf_node_t *node,
        const btree_key_t *key,
        repli_timestamp_t tstamp,
        /* the recency of the buf_t that `node` comes from */
        repli_timestamp_t maximum_existing_tstamp,
        key_modification_proof_t km_proof);

/* `erase_presence()` removes any preexisting value or tombstone, but does not create a
new tombstone. */
void erase_presence(value_sizer_t *sizer, leaf_node_t *node, const btree_key_t *key, key_modification_proof_t km_proof);

/* Returns the smallest timestamp such that if a deletion had occurred with that
timestamp, the node would still have a record of it. */
repli_timestamp_t min_deletion_timestamp(
    value_sizer_t *sizer,
    const leaf_node_t *node,
    // the recency of the buf that `node` came from
    repli_timestamp_t maximum_existing_timestamp);

/* Removes all timestamps and deletions. Optionally: specify a `min_timestamp`
so that only timestamps and deletions earlier than the given timestamp are removed. */
void erase_deletions(
    value_sizer_t *sizer, leaf_node_t *node,
    boost::optional<repli_timestamp_t> min_timestamp);

/* Calls `cb` on every entry in the node, whether a real entry or a deletion. The calls
will be in order from most recent to least recent. For entries with no timestamp, the
callback will get `min_deletion_timestamp() - 1`. */
continue_bool_t visit_entries(
    value_sizer_t *sizer,
    const leaf_node_t *node,
    // the recency of the buf that `node` came from
    repli_timestamp_t maximum_existing_timestamp,
    const std::function<continue_bool_t(
        const btree_key_t *key,
        repli_timestamp_t timestamp,
        const void *value   /* null for deletion */
        )> &cb);

class iterator {
public:
    iterator();
    iterator(const leaf_node_t *node, int index);
    std::pair<const btree_key_t *, const void *> operator*() const;
    iterator &operator++();
    iterator &operator--();
    bool operator==(const iterator &other) const;
    bool operator!=(const iterator &other) const;
    bool operator<(const iterator &other) const;
    bool operator>(const iterator &other) const;
    bool operator<=(const iterator &other) const;
    bool operator>=(const iterator &other) const;
private:
    /* negative return => this < other */
    int cmp(const iterator &other) const;
    const leaf_node_t *node_;
    int index_;
};

class reverse_iterator {
public:
    reverse_iterator();
    reverse_iterator(const leaf_node_t *node, int index);
    std::pair<const btree_key_t *, const void *> operator*() const;
    reverse_iterator &operator++();
    reverse_iterator &operator--();
    bool operator==(const reverse_iterator &other) const;
    bool operator!=(const reverse_iterator &other) const;
    bool operator<(const reverse_iterator &other) const;
    bool operator>(const reverse_iterator &other) const;
    bool operator<=(const reverse_iterator &other) const;
    bool operator>=(const reverse_iterator &other) const;
private:
    iterator inner_;
};

}  // namespace leaf

#endif  // BTREE_LEAF_NODE_HPP_
<|endoftext|>


<commit_before>#ifndef VSMC_CORE_PATH_HPP
#define VSMC_CORE_PATH_HPP

#include <vsmc/internal/common.hpp>

namespace vsmc {

/// \brief Monitor for path sampling
/// \ingroup Core
///
/// \tparam T Particle<T>::value_type
template <typename T>
class Path
{
    public :

    /// The type of the particle values
    typedef T value_type;

    /// The type of path sampling evaluation functor
    typedef cxx11::function<double (
            unsigned, const Particle<T> &, double *)> eval_type;

    /// The type of the index vector
    typedef std::vector<unsigned> index_type;

    /// The type of the integrand vector
    typedef std::vector<double> integrand_type;

    /// The type of the width vector
    typedef std::vector<double> width_type;

    /// The type of the grid vector
    typedef std::vector<double> grid_type;

    /// \brief Construct a Path with an evaluation function
    ///
    /// \param eval The functor used to compute the integrands
    explicit Path (const eval_type &eval = VSMC_NULLPTR) : eval_(eval) {}

    Path (const Path<T> &other) :
        eval_(other.eval_),
        index_(other.index_), integrand_(other.integrand_),
        width_(other.width_), grid_(other.grid_) {}

    Path<T> & operator= (const Path<T> &other)
    {
        if (&other != this) {
            eval_      = other.eval_;
            index_     = other.index_;
            integrand_ = other.integrand_;
            width_     = other.width_;
            grid_      = other.grid_;
        }

        return *this;
    }

    /// Size of records
    unsigned iter_size () const
    {
        return static_cast<unsigned>(index_.size());
    }

    /// \brief Test if the monitor is valid
    ///
    /// \note This operator will be \c explicit if the C++11 feature is enabled
#if VSMC_HAS_CXX11_EXPLICIT_CONVERSIONS
    explicit
#endif
        operator bool () const
    {
        return bool(eval_);
    }

    /// Iteration index
    const index_type &index () const
    {
        return index_;
    }

    /// Record of path sampling integrands
    const integrand_type &integrand () const
    {
        return integrand_;
    }

    /// Record of path sampling width
    const width_type &width () const
    {
        return width_;
    }

    /// Record of path sampling grid (accumulated width)
    const grid_type &grid () const
    {
        return grid_;
    }

    /// Set the evaluation functor
    void set_eval (const eval_type &new_eval)
    {
        eval_ = new_eval;
    }

    /// \brief Evaluate the integration
    ///
    /// \param iter The iteration number
    /// \param particle The particle set to be operated on by eval()
    void eval (unsigned iter, const Particle<T> &particle)
    {
        VSMC_RUNTIME_ASSERT((bool(eval_)),
                ("CALL **Path::eval** WITH AN INVALID "
                 "EVALUATION FUNCTOR"));

        buffer_.resize(particle.size());
        weight_.resize(particle.size());
        double w = eval_(iter, particle, &buffer_[0]);
        particle.read_weight(weight_.begin());
        double p = 0;
        for (std::vector<double>::size_type i = 0; i != weight_.size(); ++i)
            p += weight_[i] * buffer_[i];
        width_.push_back(w);
        integrand_.push_back(p);
        index_.push_back(iter);
        grid_.push_back(grid_.size() ?
                grid_.back() + width_.back() : width_.back());
    }

    /// Path sampling estimate of normalizing constant
    double zconst () const
    {
        double sum = 0;
        for (unsigned i = 1; i != iter_size(); ++i)
            sum += 0.5 * width_[i] * (integrand_[i-1] + integrand_[i]);

        return sum;
    }

    /// \brief Clear all recorded data
    ///
    /// \note The evaluation functor is not reset
    void clear ()
    {
        index_.clear();
        integrand_.clear();
        width_.clear();
        grid_.clear();
    }

    private :

    std::vector<double> buffer_;
    std::vector<double> weight_;
    eval_type eval_;
    index_type index_;
    integrand_type integrand_;
    width_type width_;
    grid_type grid_;
}; // class PathSampling

} // namespace vsmc

#endif // VSMC_CORE_PATH_HPP
<commit_msg>cleanup<commit_after>#ifndef VSMC_CORE_PATH_HPP
#define VSMC_CORE_PATH_HPP

#include <vsmc/internal/common.hpp>

namespace vsmc {

/// \brief Monitor for path sampling
/// \ingroup Core
///
/// \tparam T Particle<T>::value_type
template <typename T>
class Path
{
    public :

    /// The type of the particle values
    typedef T value_type;

    /// The type of path sampling evaluation functor
    typedef cxx11::function<double (
            unsigned, const Particle<T> &, double *)> eval_type;

    /// The type of the index vector
    typedef std::vector<unsigned> index_type;

    /// The type of the integrand vector
    typedef std::vector<double> integrand_type;

    /// The type of the width vector
    typedef std::vector<double> width_type;

    /// The type of the grid vector
    typedef std::vector<double> grid_type;

    /// \brief Construct a Path with an evaluation function
    ///
    /// \param eval The functor used to compute the integrands
    explicit Path (const eval_type &eval = VSMC_NULLPTR) : eval_(eval) {}

    Path (const Path<T> &other) :
        eval_(other.eval_),
        index_(other.index_), integrand_(other.integrand_),
        width_(other.width_), grid_(other.grid_) {}

    Path<T> & operator= (const Path<T> &other)
    {
        if (&other != this) {
            eval_      = other.eval_;
            index_     = other.index_;
            integrand_ = other.integrand_;
            width_     = other.width_;
            grid_      = other.grid_;
        }

        return *this;
    }

    /// Size of records
    unsigned iter_size () const
    {
        return static_cast<unsigned>(index_.size());
    }

    /// \brief Test if the monitor is valid
    ///
    /// \note This operator will be \c explicit if the C++11 feature is enabled
#if VSMC_HAS_CXX11_EXPLICIT_CONVERSIONS
    explicit
#endif
        operator bool () const
    {
        return bool(eval_);
    }

    /// Iteration index
    const index_type &index () const
    {
        return index_;
    }

    /// Record of path sampling integrands
    const integrand_type &integrand () const
    {
        return integrand_;
    }

    /// Record of path sampling width
    const width_type &width () const
    {
        return width_;
    }

    /// Record of path sampling grid (accumulated width)
    const grid_type &grid () const
    {
        return grid_;
    }

    /// Set the evaluation functor
    void set_eval (const eval_type &new_eval)
    {
        eval_ = new_eval;
    }

    /// \brief Evaluate the integration
    ///
    /// \param iter The iteration number
    /// \param particle The particle set to be operated on by eval()
    void eval (unsigned iter, const Particle<T> &particle)
    {
        VSMC_RUNTIME_ASSERT((bool(eval_)),
                ("CALL **Path::eval** WITH AN INVALID "
                 "EVALUATION FUNCTOR"));

        buffer_.resize(particle.size());
        weight_.resize(particle.size());
        double w = eval_(iter, particle, &buffer_[0]);
        particle.read_weight(weight_.begin());
        double p = 0;
        for (std::vector<double>::size_type i = 0; i != weight_.size(); ++i)
            p += weight_[i] * buffer_[i];

        index_.push_back(iter);
        integrand_.push_back(p);
        width_.push_back(w);
        grid_.push_back(grid_.size() ?
                grid_.back() + width_.back() : width_.back());
    }

    /// Path sampling estimate of normalizing constant
    double zconst () const
    {
        double sum = 0;
        for (unsigned i = 1; i != iter_size(); ++i)
            sum += 0.5 * width_[i] * (integrand_[i-1] + integrand_[i]);

        return sum;
    }

    /// \brief Clear all recorded data
    ///
    /// \note The evaluation functor is not reset
    void clear ()
    {
        index_.clear();
        integrand_.clear();
        width_.clear();
        grid_.clear();
    }

    private :

    std::vector<double> buffer_;
    std::vector<double> weight_;
    eval_type eval_;
    index_type index_;
    integrand_type integrand_;
    width_type width_;
    grid_type grid_;
}; // class PathSampling

} // namespace vsmc

#endif // VSMC_CORE_PATH_HPP
<|endoftext|>