<commit_before>/*
    irccontact.cpp - IRC Contact

    Copyright (c) 2002      by Nick Betcher <nbetcher@kde.org>
    Copyright (c) 2004      by Michel Hermier <michel.hermier@wanadoo.fr>

    Kopete    (c) 2002-2004 by the Kopete developers <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include <kdebug.h>
#include <klocale.h>
#include <qregexp.h>

#include <qtimer.h>
#include <qtextcodec.h>

#include "ircaccount.h"
#include "kopeteglobal.h"
#include "kopeteuiglobal.h"
#include "kopetemetacontact.h"
#include "kopeteview.h"
#include "ircusercontact.h"
#include "irccontact.h"
#include "ircprotocol.h"
#include "ircservercontact.h"
#include "irccontactmanager.h"
#include "ksparser.h"

IRCContact::IRCContact(IRCAccount *account, KIRC::EntityPtr entity, Kopete::MetaContact *metac, const QString& icon)
	: Kopete::Contact(account, entity->name(), metac, icon),
	  m_chatSession(0)
{
}

IRCContact::IRCContact(IRCContactManager *contactManager, const QString &nick, Kopete::MetaContact *metac, const QString& icon)
	: Kopete::Contact(contactManager->account(), nick, metac, icon),
	  m_nickName(nick),
	  m_chatSession(0)
{
	KIRC::Engine *engine = kircEngine();

	// Contact list display name
	setProperty( Kopete::Global::Properties::self()->nickName(), m_nickName );

	// IRCContactManager stuff
	QObject::connect(contactManager, SIGNAL(privateMessage(IRCContact *, IRCContact *, const QString &)),
			this, SLOT(privateMessage(IRCContact *, IRCContact *, const QString &)));

	// Kopete::ChatSessionManager stuff
	mMyself.append( static_cast<Kopete::Contact*>( this ) );

	// KIRC stuff
	QObject::connect(engine, SIGNAL(incomingNickChange(const QString &, const QString &)),
			this, SLOT( slotNewNickChange(const QString&, const QString&)));
	QObject::connect(engine, SIGNAL(successfullyChangedNick(const QString &, const QString &)),
			this, SLOT(slotNewNickChange(const QString &, const QString &)));
	QObject::connect(engine, SIGNAL(incomingQuitIRC(const QString &, const QString &)),
			this, SLOT( slotUserDisconnected(const QString&, const QString&)));

	QObject::connect(engine, SIGNAL(statusChanged(KIRC::Engine::Status)),
			this, SLOT(updateStatus()));

	engine->setCodec( m_nickName, codec() );
}

IRCContact::~IRCContact()
{
//	kdDebug(14120) << k_funcinfo << m_nickName << endl;
	if (metaContact() && metaContact()->isTemporary() && !isChatting(m_chatSession))
		metaContact()->deleteLater();

	emit destroyed(this);
}

IRCAccount *IRCContact::ircAccount() const
{
	return static_cast<IRCAccount *>(account());
}

KIRC::Engine *IRCContact::kircEngine() const
{
	return ircAccount()->engine();
}

bool IRCContact::isReachable()
{
	if (onlineStatus().status() != Kopete::OnlineStatus::Offline &&
		onlineStatus().status() != Kopete::OnlineStatus::Unknown)
		return true;

	return false;
}

const QString IRCContact::caption() const
{
	return QString::null;
}
/*
const QString IRCContact::formatedName() const
{
	return QString::null;
}
*/
void IRCContact::updateStatus()
{
}

void IRCContact::privateMessage(IRCContact *, IRCContact *, const QString &)
{
}

void IRCContact::setCodec(const QTextCodec *codec)
{
	kircEngine()->setCodec(m_nickName, codec);
	metaContact()->setPluginData(m_protocol, QString::fromLatin1("Codec"), QString::number(codec->mibEnum()));
}

const QTextCodec *IRCContact::codec()
{
	QString codecId = metaContact()->pluginData(m_protocol, QString::fromLatin1("Codec"));
	QTextCodec *codec = ircAccount()->codec();

	if( !codecId.isEmpty() )
	{
		bool test = true;
		uint mib = codecId.toInt(&test);
		if (test)
			codec = QTextCodec::codecForMib(mib);
		else
			codec = QTextCodec::codecForName(codecId.latin1());
	}

	if( !codec )
		return kircEngine()->codec();

	return codec;
}

Kopete::ChatSession *IRCContact::manager(Kopete::Contact::CanCreateFlags canCreate)
{
	IRCAccount *account = ircAccount();
	KIRC::Engine *engine = kircEngine();

	if (canCreate == Kopete::Contact::CanCreate && !m_chatSession)
	{
		if( engine->status() == KIRC::Engine::Idle && dynamic_cast<IRCServerContact*>(this) == 0 )
			account->connect();

		m_chatSession = Kopete::ChatSessionManager::self()->create(account->myself(), mMyself, account->protocol());
		m_chatSession->setDisplayName(caption());

		QObject::connect(m_chatSession, SIGNAL(messageSent(Kopete::Message&, Kopete::ChatSession *)),
			this, SLOT(slotSendMsg(Kopete::Message&, Kopete::ChatSession *)));
		QObject::connect(m_chatSession, SIGNAL(closing(Kopete::ChatSession *)),
			this, SLOT(chatSessionDestroyed()));

		initConversation();
	}

	return m_chatSession;
}

void IRCContact::chatSessionDestroyed()
{
	m_chatSession = 0;

	if (metaContact()->isTemporary() && !isChatting())
		deleteLater();
}

void IRCContact::slotUserDisconnected(const QString &user, const QString &reason)
{
	if (m_chatSession)
	{
		QString nickname = user.section('!', 0, 0);
		Kopete::Contact *c = locateUser( nickname );
		if ( c )
		{
			m_chatSession->removeContact(c, i18n("Quit: \"%1\" ").arg(reason), Kopete::Message::RichText);
			c->setOnlineStatus(m_protocol->m_UserStatusOffline);
		}
	}
}

void IRCContact::setNickName( const QString &nickname )
{
	kdDebug(14120) << k_funcinfo << m_nickName << " changed to " << nickname << endl;
	m_nickName = nickname;
	Kopete::Contact::setNickName( nickname );
}

void IRCContact::slotNewNickChange(const QString &oldnickname, const QString &newnickname)
{
	IRCAccount *account = ircAccount();

	IRCContact *user = static_cast<IRCContact*>( locateUser(oldnickname) );
	if( user )
	{
		user->setNickName( newnickname );

		//If the user is in our contact list, then change the notify list nickname
		if (!user->metaContact()->isTemporary())
		{
			account->contactManager()->removeFromNotifyList( oldnickname );
			account->contactManager()->addToNotifyList( newnickname );
		}
	}
}

void IRCContact::slotSendMsg(Kopete::Message &message, Kopete::ChatSession *)
{
	QString htmlString = message.escapedBody();

	if (htmlString.find(QString::fromLatin1(" -1)
	{
		QRegExp findTags( QString::fromLatin1("(.*)") );
		findTags.setMinimal( true );
		int pos = 0;

		while (pos >= 0)
		{
			pos = findTags.search(htmlString);
			if (pos > -1)
			{
				QString styleHTML = findTags.cap(1);
				QString replacement = findTags.cap(2);
				QStringList styleAttrs = QStringList::split(';', styleHTML);

				for (QStringList::Iterator attrPair = styleAttrs.begin(); attrPair != styleAttrs.end(); ++attrPair)
				{
					QString attribute = (*attrPair).section(':',0,0);
					QString value = (*attrPair).section(':',1);

					if( attribute == QString::fromLatin1("color") )
					{
						int ircColor = KSParser::colorForHTML( value );
						if( ircColor > -1 )
							replacement.prepend( QString( QChar(0x03) ).append( QString::number(ircColor) ) ).append( QChar( 0x03 ) );
					}
					else if( attribute == QString::fromLatin1("font-weight") && value == QString::fromLatin1("600") )
						replacement.prepend( QChar(0x02) ).append( QChar(0x02) );
					else if( attribute == QString::fromLatin1("text-decoration")  && value == QString::fromLatin1("underline") )
						replacement.prepend( QChar(31) ).append( QChar(31) );
				}

				htmlString = htmlString.left( pos ) + replacement + htmlString.mid( pos + findTags.matchedLength() );
			}
		}
	}

	htmlString = Kopete::Message::unescape(htmlString);

	if (htmlString.find('\n') > -1)
	{
		QStringList messages = QStringList::split( '\n', htmlString );

		for( QStringList::Iterator it = messages.begin(); it != messages.end(); ++it )
		{
			Kopete::Message msg(message.from(), message.to(), sendMessage(*it), message.direction(),
			                    Kopete::Message::RichText, CHAT_VIEW, message.type());

			msg.setBg(QColor());
			msg.setFg(QColor());

			appendMessage(msg);
			manager(Kopete::Contact::CanCreate)->messageSucceeded();
		}
	}
	else
	{
		message.setBody( sendMessage( htmlString ), Kopete::Message::RichText );

		message.setBg( QColor() );
		message.setFg( QColor() );

		appendMessage(message);
		manager(Kopete::Contact::CanCreate)->messageSucceeded();
	}
}

QString IRCContact::sendMessage( const QString &msg )
{
	QString newMessage = msg;
	uint trueLength = msg.length() + m_nickName.length() + 12;
	if( trueLength > 512 )
	{
		//TODO: tell them it is truncated
		kdWarning() << "Message was to long (" << trueLength << "), it has been truncated to 512 characters" << endl;
		newMessage.truncate( 512 - ( m_nickName.length() + 12 ) );
	}

	kircEngine()->privmsg(m_nickName, newMessage );

	return newMessage;
}

Kopete::Contact *IRCContact::locateUser(const QString &nick)
{
	IRCAccount *account = ircAccount();

	if (m_chatSession)
	{
		if( nick == account->mySelf()->nickName() )
			return account->mySelf();
		else
		{
			Kopete::ContactPtrList mMembers = m_chatSession->members();
			for (Kopete::Contact *it = mMembers.first(); it; it = mMembers.next())
			{
				if (static_cast<IRCContact*>(it)->nickName() == nick)
					return it;
			}
		}
	}
	return 0;
}

bool IRCContact::isChatting(Kopete::ChatSession *avoid) const
{
	IRCAccount *account = ircAccount();

	if (!account)
		return false;

	QValueList<Kopete::ChatSession*> sessions = Kopete::ChatSessionManager::self()->sessions();
	for (QValueList<Kopete::ChatSession*>::Iterator it= sessions.begin(); it!=sessions.end() ; ++it)
	{
	  if( (*it) != avoid && (*it)->account() == account &&
			   (*it)->members().contains(this) )
		{
			return true;
		}
	}
	return false;
}

void IRCContact::deleteContact()
{
	kdDebug(14120) << k_funcinfo << m_nickName << endl;

	delete m_chatSession;

	if (!isChatting())
	{
		kdDebug(14120) << k_funcinfo << "will delete " << m_nickName << endl;
		Kopete::Contact::deleteContact();
	}
	else
	{
		metaContact()->removeContact(this);
		Kopete::MetaContact *m = new Kopete::MetaContact();
		m->setTemporary(true);
		setMetaContact(m);
	}
}

void IRCContact::appendMessage(Kopete::Message &msg)
{
	manager(Kopete::Contact::CanCreate)->appendMessage(msg);
}

KopeteView *IRCContact::view()
{
	if (m_chatSession)
		return m_chatSession->view(false);
	return 0L;
}
void IRCContact::serialize(QMap<QString, QString> & /*serializedData*/, QMap<QString, QString> &addressBookData)
{
	// write the
	addressBookData[ protocol()->addressBookIndexField() ] = ( contactId() + QChar(0xE120) + account()->accountId() );
}

void IRCContact::receivedMessage( KIRC::Engine::ServerMessageType type,
				const KIRC::EntityPtr &from,
				const KIRC::EntityPtrList &to,
				const QString &msg)
{
	if (to.contains(m_entity))
	{
		IRCContact *fromContact = ircAccount()->getContact(from);
		Kopete::Message message(fromContact, manager()->members(), msg, Kopete::Message::Inbound,
					Kopete::Message::RichText, CHAT_VIEW);
		appendMessage(message);
	}
}

#include "irccontact.moc"
<commit_msg>Better escape a previoulsy unescaped message before sending it back to the GUI. (fix the < > bug)<commit_after>/*
    irccontact.cpp - IRC Contact

    Copyright (c) 2002      by Nick Betcher <nbetcher@kde.org>
    Copyright (c) 2004      by Michel Hermier <michel.hermier@wanadoo.fr>

    Kopete    (c) 2002-2004 by the Kopete developers <kopete-devel@kde.org>

    *************************************************************************
    *                                                                       *
    * This program is free software; you can redistribute it and/or modify  *
    * it under the terms of the GNU General Public License as published by  *
    * the Free Software Foundation; either version 2 of the License, or     *
    * (at your option) any later version.                                   *
    *                                                                       *
    *************************************************************************
*/

#include <kdebug.h>
#include <klocale.h>
#include <qregexp.h>

#include <qtimer.h>
#include <qtextcodec.h>

#include "ircaccount.h"
#include "kopeteglobal.h"
#include "kopeteuiglobal.h"
#include "kopetemetacontact.h"
#include "kopeteview.h"
#include "ircusercontact.h"
#include "irccontact.h"
#include "ircprotocol.h"
#include "ircservercontact.h"
#include "irccontactmanager.h"
#include "ksparser.h"

IRCContact::IRCContact(IRCAccount *account, KIRC::EntityPtr entity, Kopete::MetaContact *metac, const QString& icon)
	: Kopete::Contact(account, entity->name(), metac, icon),
	  m_chatSession(0)
{
}

IRCContact::IRCContact(IRCContactManager *contactManager, const QString &nick, Kopete::MetaContact *metac, const QString& icon)
	: Kopete::Contact(contactManager->account(), nick, metac, icon),
	  m_nickName(nick),
	  m_chatSession(0)
{
	KIRC::Engine *engine = kircEngine();

	// Contact list display name
	setProperty( Kopete::Global::Properties::self()->nickName(), m_nickName );

	// IRCContactManager stuff
	QObject::connect(contactManager, SIGNAL(privateMessage(IRCContact *, IRCContact *, const QString &)),
			this, SLOT(privateMessage(IRCContact *, IRCContact *, const QString &)));

	// Kopete::ChatSessionManager stuff
	mMyself.append( static_cast<Kopete::Contact*>( this ) );

	// KIRC stuff
	QObject::connect(engine, SIGNAL(incomingNickChange(const QString &, const QString &)),
			this, SLOT( slotNewNickChange(const QString&, const QString&)));
	QObject::connect(engine, SIGNAL(successfullyChangedNick(const QString &, const QString &)),
			this, SLOT(slotNewNickChange(const QString &, const QString &)));
	QObject::connect(engine, SIGNAL(incomingQuitIRC(const QString &, const QString &)),
			this, SLOT( slotUserDisconnected(const QString&, const QString&)));

	QObject::connect(engine, SIGNAL(statusChanged(KIRC::Engine::Status)),
			this, SLOT(updateStatus()));

	engine->setCodec( m_nickName, codec() );
}

IRCContact::~IRCContact()
{
//	kdDebug(14120) << k_funcinfo << m_nickName << endl;
	if (metaContact() && metaContact()->isTemporary() && !isChatting(m_chatSession))
		metaContact()->deleteLater();

	emit destroyed(this);
}

IRCAccount *IRCContact::ircAccount() const
{
	return static_cast<IRCAccount *>(account());
}

KIRC::Engine *IRCContact::kircEngine() const
{
	return ircAccount()->engine();
}

bool IRCContact::isReachable()
{
	if (onlineStatus().status() != Kopete::OnlineStatus::Offline &&
		onlineStatus().status() != Kopete::OnlineStatus::Unknown)
		return true;

	return false;
}

const QString IRCContact::caption() const
{
	return QString::null;
}
/*
const QString IRCContact::formatedName() const
{
	return QString::null;
}
*/
void IRCContact::updateStatus()
{
}

void IRCContact::privateMessage(IRCContact *, IRCContact *, const QString &)
{
}

void IRCContact::setCodec(const QTextCodec *codec)
{
	kircEngine()->setCodec(m_nickName, codec);
	metaContact()->setPluginData(m_protocol, QString::fromLatin1("Codec"), QString::number(codec->mibEnum()));
}

const QTextCodec *IRCContact::codec()
{
	QString codecId = metaContact()->pluginData(m_protocol, QString::fromLatin1("Codec"));
	QTextCodec *codec = ircAccount()->codec();

	if( !codecId.isEmpty() )
	{
		bool test = true;
		uint mib = codecId.toInt(&test);
		if (test)
			codec = QTextCodec::codecForMib(mib);
		else
			codec = QTextCodec::codecForName(codecId.latin1());
	}

	if( !codec )
		return kircEngine()->codec();

	return codec;
}

Kopete::ChatSession *IRCContact::manager(Kopete::Contact::CanCreateFlags canCreate)
{
	IRCAccount *account = ircAccount();
	KIRC::Engine *engine = kircEngine();

	if (canCreate == Kopete::Contact::CanCreate && !m_chatSession)
	{
		if( engine->status() == KIRC::Engine::Idle && dynamic_cast<IRCServerContact*>(this) == 0 )
			account->connect();

		m_chatSession = Kopete::ChatSessionManager::self()->create(account->myself(), mMyself, account->protocol());
		m_chatSession->setDisplayName(caption());

		QObject::connect(m_chatSession, SIGNAL(messageSent(Kopete::Message&, Kopete::ChatSession *)),
			this, SLOT(slotSendMsg(Kopete::Message&, Kopete::ChatSession *)));
		QObject::connect(m_chatSession, SIGNAL(closing(Kopete::ChatSession *)),
			this, SLOT(chatSessionDestroyed()));

		initConversation();
	}

	return m_chatSession;
}

void IRCContact::chatSessionDestroyed()
{
	m_chatSession = 0;

	if (metaContact()->isTemporary() && !isChatting())
		deleteLater();
}

void IRCContact::slotUserDisconnected(const QString &user, const QString &reason)
{
	if (m_chatSession)
	{
		QString nickname = user.section('!', 0, 0);
		Kopete::Contact *c = locateUser( nickname );
		if ( c )
		{
			m_chatSession->removeContact(c, i18n("Quit: \"%1\" ").arg(reason), Kopete::Message::RichText);
			c->setOnlineStatus(m_protocol->m_UserStatusOffline);
		}
	}
}

void IRCContact::setNickName( const QString &nickname )
{
	kdDebug(14120) << k_funcinfo << m_nickName << " changed to " << nickname << endl;
	m_nickName = nickname;
	Kopete::Contact::setNickName( nickname );
}

void IRCContact::slotNewNickChange(const QString &oldnickname, const QString &newnickname)
{
	IRCAccount *account = ircAccount();

	IRCContact *user = static_cast<IRCContact*>( locateUser(oldnickname) );
	if( user )
	{
		user->setNickName( newnickname );

		//If the user is in our contact list, then change the notify list nickname
		if (!user->metaContact()->isTemporary())
		{
			account->contactManager()->removeFromNotifyList( oldnickname );
			account->contactManager()->addToNotifyList( newnickname );
		}
	}
}

void IRCContact::slotSendMsg(Kopete::Message &message, Kopete::ChatSession *)
{
	QString htmlString = message.escapedBody();

	if (htmlString.find(QString::fromLatin1(" -1)
	{
		QRegExp findTags( QString::fromLatin1("(.*)") );
		findTags.setMinimal( true );
		int pos = 0;

		while (pos >= 0)
		{
			pos = findTags.search(htmlString);
			if (pos > -1)
			{
				QString styleHTML = findTags.cap(1);
				QString replacement = findTags.cap(2);
				QStringList styleAttrs = QStringList::split(';', styleHTML);

				for (QStringList::Iterator attrPair = styleAttrs.begin(); attrPair != styleAttrs.end(); ++attrPair)
				{
					QString attribute = (*attrPair).section(':',0,0);
					QString value = (*attrPair).section(':',1);

					if( attribute == QString::fromLatin1("color") )
					{
						int ircColor = KSParser::colorForHTML( value );
						if( ircColor > -1 )
							replacement.prepend( QString( QChar(0x03) ).append( QString::number(ircColor) ) ).append( QChar( 0x03 ) );
					}
					else if( attribute == QString::fromLatin1("font-weight") && value == QString::fromLatin1("600") )
						replacement.prepend( QChar(0x02) ).append( QChar(0x02) );
					else if( attribute == QString::fromLatin1("text-decoration")  && value == QString::fromLatin1("underline") )
						replacement.prepend( QChar(31) ).append( QChar(31) );
				}

				htmlString = htmlString.left( pos ) + replacement + htmlString.mid( pos + findTags.matchedLength() );
			}
		}
	}

	htmlString = Kopete::Message::unescape(htmlString);

	if (htmlString.find('\n') > -1)
	{
		QStringList messages = QStringList::split( '\n', htmlString );

		for( QStringList::Iterator it = messages.begin(); it != messages.end(); ++it )
		{
			Kopete::Message msg(message.from(), message.to(), Kopete::Message::escape(sendMessage(*it)), message.direction(),
			                    Kopete::Message::RichText, CHAT_VIEW, message.type());

			msg.setBg(QColor());
			msg.setFg(QColor());

			appendMessage(msg);
			manager(Kopete::Contact::CanCreate)->messageSucceeded();
		}
	}
	else
	{
		message.setBody( Kopete::Message::escape(sendMessage( htmlString )), Kopete::Message::RichText );

		message.setBg( QColor() );
		message.setFg( QColor() );

		appendMessage(message);
		manager(Kopete::Contact::CanCreate)->messageSucceeded();
	}
}

QString IRCContact::sendMessage( const QString &msg )
{
	QString newMessage = msg;
	uint trueLength = msg.length() + m_nickName.length() + 12;
	if( trueLength > 512 )
	{
		//TODO: tell them it is truncated
		kdWarning() << "Message was to long (" << trueLength << "), it has been truncated to 512 characters" << endl;
		newMessage.truncate( 512 - ( m_nickName.length() + 12 ) );
	}

	kircEngine()->privmsg(m_nickName, newMessage );

	return newMessage;
}

Kopete::Contact *IRCContact::locateUser(const QString &nick)
{
	IRCAccount *account = ircAccount();

	if (m_chatSession)
	{
		if( nick == account->mySelf()->nickName() )
			return account->mySelf();
		else
		{
			Kopete::ContactPtrList mMembers = m_chatSession->members();
			for (Kopete::Contact *it = mMembers.first(); it; it = mMembers.next())
			{
				if (static_cast<IRCContact*>(it)->nickName() == nick)
					return it;
			}
		}
	}
	return 0;
}

bool IRCContact::isChatting(Kopete::ChatSession *avoid) const
{
	IRCAccount *account = ircAccount();

	if (!account)
		return false;

	QValueList<Kopete::ChatSession*> sessions = Kopete::ChatSessionManager::self()->sessions();
	for (QValueList<Kopete::ChatSession*>::Iterator it= sessions.begin(); it!=sessions.end() ; ++it)
	{
	  if( (*it) != avoid && (*it)->account() == account &&
			   (*it)->members().contains(this) )
		{
			return true;
		}
	}
	return false;
}

void IRCContact::deleteContact()
{
	kdDebug(14120) << k_funcinfo << m_nickName << endl;

	delete m_chatSession;

	if (!isChatting())
	{
		kdDebug(14120) << k_funcinfo << "will delete " << m_nickName << endl;
		Kopete::Contact::deleteContact();
	}
	else
	{
		metaContact()->removeContact(this);
		Kopete::MetaContact *m = new Kopete::MetaContact();
		m->setTemporary(true);
		setMetaContact(m);
	}
}

void IRCContact::appendMessage(Kopete::Message &msg)
{
	manager(Kopete::Contact::CanCreate)->appendMessage(msg);
}

KopeteView *IRCContact::view()
{
	if (m_chatSession)
		return m_chatSession->view(false);
	return 0L;
}
void IRCContact::serialize(QMap<QString, QString> & /*serializedData*/, QMap<QString, QString> &addressBookData)
{
	// write the
	addressBookData[ protocol()->addressBookIndexField() ] = ( contactId() + QChar(0xE120) + account()->accountId() );
}

void IRCContact::receivedMessage( KIRC::Engine::ServerMessageType type,
				const KIRC::EntityPtr &from,
				const KIRC::EntityPtrList &to,
				const QString &msg)
{
	if (to.contains(m_entity))
	{
		IRCContact *fromContact = ircAccount()->getContact(from);
		Kopete::Message message(fromContact, manager()->members(), msg, Kopete::Message::Inbound,
					Kopete::Message::RichText, CHAT_VIEW);
		appendMessage(message);
	}
}

#include "irccontact.moc"
<|endoftext|>


<commit_before>#include <cycle.hpp>

#include "helpers.hpp"

#include <vector>
#include <string>
#include <iterator>

#include "catch.hpp"

using iter::cycle;

TEST_CASE("cycle: iterate twice", "[cycle]") {
    std::vector<int> ns {2,4,6};
    std::vector<int> v{};
    std::size_t count = 0;
    for (auto i : cycle(ns)) {
        v.push_back(i);
        ++count;
        if (count == ns.size() * 2) break;
    }

    auto vc = ns;
    vc.insert(std::end(vc), std::begin(ns), std::end(ns));
    REQUIRE( v == vc );
}
<commit_msg>makes sure cycle binds and moves correctly<commit_after>#include <cycle.hpp>

#include "helpers.hpp"

#include <vector>
#include <string>
#include <iterator>

#include "catch.hpp"

using iter::cycle;

TEST_CASE("cycle: iterate twice", "[cycle]") {
    std::vector<int> ns {2,4,6};
    std::vector<int> v{};
    std::size_t count = 0;
    for (auto i : cycle(ns)) {
        v.push_back(i);
        ++count;
        if (count == ns.size() * 2) break;
    }

    auto vc = ns;
    vc.insert(std::end(vc), std::begin(ns), std::end(ns));
    REQUIRE( v == vc );
}

TEST_CASE("cycle: empty cycle terminates", "[cycle]") {
    std::vector<int> ns;
    auto c = cycle(ns);
    std::vector<int> v(std::begin(c), std::end(c));
    REQUIRE( v.empty() );
}

TEST_CASE("cycle: binds to lvalues, moves rvalues", "[cycle]") {
    itertest::BasicIterable<char> bi{'x', 'y', 'z'};
    SECTION("binds to lvalues") {
        cycle(bi);
        REQUIRE_FALSE( bi.was_moved_from() );
    }
    SECTION("moves rvalues") {
        cycle(std::move(bi));
        REQUIRE( bi.was_moved_from() );
    }
}
<|endoftext|>


<commit_before>#include "cnn/nodes.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/gru.h"
#include "cnn/lstm.h"
#include "cnn/dglstm.h"
#include "cnn/dict.h"
# include "cnn/expr.h"
#include "cnn/cnn-helper.h"
#include <iostream>
#include <fstream>
#include <sstream>

#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

using namespace std;
using namespace cnn;

long LAYERS = 2;
long INPUT_DIM = 8;  //256
long HIDDEN_DIM = 24;  // 1024
long VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM});
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return Expression of total loss
  Expression BuildLMGraph(const vector<int>& sent, ComputationGraph& cg) {
    const unsigned slen = sent.size() - 1;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    Expression i_R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression i_bias = parameter(cg, p_bias);  // word bias
    vector<Expression> errs;
    for (unsigned t = 0; t < slen; ++t) {
      Expression i_x_t = lookup(cg, p_c, sent[t]);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t =  i_bias + i_R * i_y_t;

      // we can easily look at intermidiate values
//      std::vector<float> r_t = as_vector(i_r_t.value());
  //    for (float f : r_t) cout << f << " "; cout << endl;
    //  cout << "[" << as_scalar(pick(i_r_t, sent[t+1]).value()) << "]" << endl;

      // LogSoftmax followed by PickElement can be written in one step
      // using PickNegLogSoftmax
#if 0
      Expression i_ydist = logsoftmax(i_r_t);
      errs.push_back(pick(i_ydist, sent[t+1]));
#if 0
      Expression i_ydist = softmax(i_r_t);
      i_ydist = log(i_ydist)
      errs.push_back(pick(i_ydist, sent[t+1]));
#endif
#else
      Expression i_err = pickneglogsoftmax(i_r_t, sent[t+1]);
      errs.push_back(i_err);
#endif
    }
    Expression i_nerr = sum(errs);
#if 0
    return -i_nerr;
#else
    return i_nerr;
#endif
  }

  // return Expression for total loss
  void RandomSample(int max_len = 150) {
    cerr << endl;
    ComputationGraph cg;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();

    Expression i_R = parameter(cg, p_R);
    Expression i_bias = parameter(cg, p_bias);
    vector<Expression> errs;
    int len = 0;
    int cur = kSOS;
    while(len < max_len && cur != kEOS) {
      ++len;
      Expression i_x_t = lookup(cg, p_c, cur);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;

      Expression ydist = softmax(i_r_t);

      unsigned w = 0;
      while (w == 0 || (int)w == kSOS) {
        auto dist = as_vector(cg.incremental_forward());
        double p = rand01();
        for (; w < dist.size(); ++w) {
          p -= dist[w];
          if (p < 0.0) { break; }
        }
        if (w == dist.size()) w = kEOS;
      }
      cerr << (len == 1 ? "" : " ") << d.Convert(w);
      cur = w;
    }
    cerr << endl;
  }
};

template <class LM_t>
void train(Model &model, LM_t &lm,
    const vector<vector<int>>& training,
    const vector<vector<int>>& dev,
    Trainer *sgd, const string& fname,
    bool randomSample)
{
    double best = 9e+99;
    unsigned report_every_i = 50;
    unsigned dev_every_i_reports = 500;
    unsigned si = training.size();
    vector<unsigned> order(training.size());
    for (unsigned i = 0; i < order.size(); ++i) order[i] = i;
    bool first = true;
    int report = 0;
    unsigned lines = 0;
    while (1) {
        Timer iteration("completed in");
        double loss = 0;
        unsigned chars = 0;
        for (unsigned i = 0; i < report_every_i; ++i) {
            if (si == training.size()) {
                si = 0;
                if (first) { first = false; }
                else { sgd->update_epoch(); }
                cerr << "**SHUFFLE\n";
                shuffle(order.begin(), order.end(), *rndeng);
            }

            // build graph for this instance
            ComputationGraph cg;
            auto& sent = training[order[si]];
            chars += sent.size() - 1;
            ++si;
            lm.BuildLMGraph(sent, cg);
            loss += as_scalar(cg.forward());
            cg.backward();
            sgd->update();
            ++lines;
        }
        sgd->status();
        cerr << " E = " << (loss / chars) << " ppl=" << exp(loss / chars) << ' ';

        if (randomSample)
            lm.RandomSample();

        // show score on dev data?
        report++;
        if (report % dev_every_i_reports == 0) {
            double dloss = 0;
            int dchars = 0;
            for (auto& sent : dev) {
                ComputationGraph cg;
                lm.BuildLMGraph(sent, cg);
                dloss += as_scalar(cg.forward());
                dchars += sent.size() - 1;
            }
            if (dloss < best) {
                best = dloss;
                ofstream out(fname);
                boost::archive::text_oarchive oa(out);
                oa << model;
            }
            else{
                sgd->eta *= 0.5;
            }
            cerr << "\n***DEV [epoch=" << (lines / (double)training.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
        }
    }
}

template <class LM_t>
void testcorpus(Model &model, LM_t &lm,
    const vector<vector<int>>& dev)
{
    unsigned lines = 0;
    double dloss = 0;
    int dchars = 0;
    for (auto& sent : dev) {
        ComputationGraph cg;
        lm.BuildLMGraph(sent, cg);
        dloss += as_scalar(cg.forward());
        dchars += sent.size() - 1;
    }

    cerr << "\n***DEV [epoch=" << (lines / (double)dev.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
}

void initialise(Model &model, const string &filename)
{
    cerr << "Initialising model parameters from file: " << filename << endl;
    ifstream in(filename);
    boost::archive::text_iarchive ia(in);
    ia >> model;
}

int main(int argc, char** argv) {
  cnn::Initialize(argc, argv);

  // command line processing
  using namespace boost::program_options;
  variables_map vm;
  options_description opts("Allowed options");
  opts.add_options()
      ("help", "print help message")
      ("seed,s", value<int>()->default_value(217), "random seed number")
      ("train,t", value<string>(), "file containing training sentences")
      ("devel,d", value<string>(), "file containing development sentences.")
      ("test,T", value<string>(), "file containing testing source sentences")
      ("initialise,i", value<string>(), "load initial parameters from file")
      ("parameters,p", value<string>(), "save best parameters to this file")
      ("layers,l", value<int>()->default_value(LAYERS), "use <num> layers for RNN components")
      ("hidden,h", value<int>()->default_value(HIDDEN_DIM), "use <num> dimensions for recurrent hidden states")
      ("gru", "use Gated Recurrent Unit (GRU) for recurrent structure; default RNN")
      ("lstm", "use Long Short Term Memory (GRU) for recurrent structure; default RNN")
      ("dglstm", "use depth-gated LSTM for recurrent structure; default RNN")
      ("verbose,v", "be extremely chatty")
      ("generate,g", value<bool>()->default_value(false), "generate random samples")
      ;
  store(parse_command_line(argc, argv, opts), vm);

  string flavour;
  if (vm.count("gru"))	flavour = "gru";
  else if (vm.count("lstm"))	flavour = "lstm";
  else if (vm.count("rnnem"))	flavour = "rnnem";
  else if (vm.count("dglstm")) flavour = "dglstm";
  else			flavour = "rnn";

  LAYERS = vm["layers"].as<int>();
  HIDDEN_DIM = vm["hidden"].as<int>();

  bool generateSample = false;
  generateSample = vm["generate"].as<bool>();

  string fname;
  if (vm.count("parameters")) {
      fname = vm["parameters"].as<string>();
  }
  else {
      ostringstream os;
      os << "lm"
          << '_' << LAYERS
          << '_' << HIDDEN_DIM
          << '_' << flavour
          << "-pid" << getpid() << ".params";
      fname = os.str();
  }

  cerr << "Parameters will be written to: " << fname << endl;

  if (vm.count("help") || vm.count("train") != 1 || (vm.count("devel") != 1 && vm.count("test") != 1)) {
      cout << opts << "\n";
      return 1;
  }

  kSOS = d.Convert("<s>");
  kEOS = d.Convert("</s>");
  vector<vector<int>> training, dev, test;
  string line;
  int tlc = 0;
  int ttoks = 0;

  string infile = vm["train"].as<string>();
  cerr << "Reading training data from " << infile << "...\n";

  {
    ifstream in(infile);
    assert(in);
    while(getline(in, line)) {
      ++tlc;
      training.push_back(ReadSentence(line, &d));
      ttoks += training.back().size();
      if (training.back().front() != kSOS && training.back().back() != kEOS) {
        cerr << "Training sentence in " << infile << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << tlc << " lines, " << ttoks << " tokens, " << d.size() << " types\n";
  }
  d.Freeze(); // no new word types allowed
  VOCAB_SIZE = d.size();

  if (vm.count("devel") > 0)
  {
      int dlc = 0;
      int dtoks = 0;
      string devfile = vm["devel"].as<string>();
      cerr << "Reading training data from " << devfile << "...\n";
      {
          ifstream in(devfile);
          assert(in);
          while (getline(in, line)) {
              ++dlc;
              dev.push_back(ReadSentence(line, &d));
              dtoks += dev.back().size();
              if (dev.back().front() != kSOS && dev.back().back() != kEOS) {
                  cerr << "Dev sentence in " << devfile << ":" << tlc << " didn't start or end with <s>, </s>\n";
                  abort();
              }
          }
          cerr << dlc << " lines, " << dtoks << " tokens\n";
      }
  }

  Model model;
  bool use_momentum = false;
  Trainer* sgd = nullptr;
  //if (use_momentum)
  //  sgd = new MomentumSGDTrainer(&model);
  //else
  sgd = new SimpleSGDTrainer(&model);

  if (vm.count("test") == 0)
  {
      if (vm.count("lstm")) {
          cerr << "%% Using LSTM recurrent units" << endl;
          RNNLanguageModel<LSTMBuilder> lm(model);
          train(model, lm, training, dev, sgd, fname, generateSample);
      }
      else if (vm.count("dglstm")) {
          cerr << "%% Using DGLSTM recurrent units" << endl;
          RNNLanguageModel<DGLSTMBuilder> lm(model);
          train(model, lm, training, dev, sgd, fname, generateSample);
      }
  }
  else
  {
      string testfile = vm["test"].as<string>();
      int dlc = 0;
      int dtoks = 0;
      cerr << "Reading training data from " << testfile << "...\n";
      {
          ifstream in(testfile);
          assert(in);
          while (getline(in, line)) {
              ++dlc;
              test.push_back(ReadSentence(line, &d));
              dtoks += test.back().size();
              if (test.back().front() != kSOS && test.back().back() != kEOS) {
                  cerr << "Dev sentence in " << testfile << ":" << tlc << " didn't start or end with <s>, </s>\n";
                  abort();
              }
          }
          cerr << dlc << " lines, " << dtoks << " tokens\n";
      }

      if (vm.count("test"))
      {
          if (vm.count("lstm")){
              cerr << "%% using LSTM recurrent units" << endl;
              RNNLanguageModel<LSTMBuilder> lm(model);
              if (vm.count("initialise"))
                  initialise(model, vm["initialise"].as<string>());
              testcorpus(model, lm, test);
          }
          if (vm.count("dglstm")){
              cerr << "%% using LSTM recurrent units" << endl;
              RNNLanguageModel<DGLSTMBuilder> lm(model);
              if (vm.count("initialise"))
                  initialise(model, vm["initialise"].as<string>());
              testcorpus(model, lm, test);
          }
      }
  }

  //RNNLanguageModel<SimpleRNNBuilder> lm(model);
  if (argc == 4) {
    string fname = argv[3];
    ifstream in(fname);
    boost::archive::text_iarchive ia(in);
    ia >> model;
  }

  delete sgd;
}

<commit_msg>updated rnnlm2 code<commit_after>#include "cnn/nodes.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/gru.h"
#include "cnn/lstm.h"
#include "cnn/dglstm.h"
#include "cnn/dict.h"
# include "cnn/expr.h"
#include "cnn/cnn-helper.h"
#include <iostream>
#include <fstream>
#include <sstream>

#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

using namespace std;
using namespace cnn;

long LAYERS = 2;
long INPUT_DIM = 8;  //256
long HIDDEN_DIM = 24;  // 1024
long VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM});
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return Expression of total loss
  Expression BuildLMGraph(const vector<int>& sent, ComputationGraph& cg) {
    const unsigned slen = sent.size() - 1;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();
    Expression i_R = parameter(cg, p_R); // hidden -> word rep parameter
    Expression i_bias = parameter(cg, p_bias);  // word bias
    vector<Expression> errs;
    for (unsigned t = 0; t < slen; ++t) {
      Expression i_x_t = lookup(cg, p_c, sent[t]);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t =  i_bias + i_R * i_y_t;

      // we can easily look at intermidiate values
//      std::vector<float> r_t = as_vector(i_r_t.value());
  //    for (float f : r_t) cout << f << " "; cout << endl;
    //  cout << "[" << as_scalar(pick(i_r_t, sent[t+1]).value()) << "]" << endl;

      // LogSoftmax followed by PickElement can be written in one step
      // using PickNegLogSoftmax
#if 0
      Expression i_ydist = logsoftmax(i_r_t);
      errs.push_back(pick(i_ydist, sent[t+1]));
#if 0
      Expression i_ydist = softmax(i_r_t);
      i_ydist = log(i_ydist)
      errs.push_back(pick(i_ydist, sent[t+1]));
#endif
#else
      Expression i_err = pickneglogsoftmax(i_r_t, sent[t+1]);
      errs.push_back(i_err);
#endif
    }
    Expression i_nerr = sum(errs);
#if 0
    return -i_nerr;
#else
    return i_nerr;
#endif
  }

  // return Expression for total loss
  void RandomSample(int max_len = 150) {
    cerr << endl;
    ComputationGraph cg;
    builder.new_graph(cg);  // reset RNN builder for new graph
    builder.start_new_sequence();

    Expression i_R = parameter(cg, p_R);
    Expression i_bias = parameter(cg, p_bias);
    vector<Expression> errs;
    int len = 0;
    int cur = kSOS;
    while(len < max_len && cur != kEOS) {
      ++len;
      Expression i_x_t = lookup(cg, p_c, cur);
      // y_t = RNN(x_t)
      Expression i_y_t = builder.add_input(i_x_t);
      Expression i_r_t = i_bias + i_R * i_y_t;

      Expression ydist = softmax(i_r_t);

      unsigned w = 0;
      while (w == 0 || (int)w == kSOS) {
        auto dist = as_vector(cg.incremental_forward());
        double p = rand01();
        for (; w < dist.size(); ++w) {
          p -= dist[w];
          if (p < 0.0) { break; }
        }
        if (w == dist.size()) w = kEOS;
      }
      cerr << (len == 1 ? "" : " ") << d.Convert(w);
      cur = w;
    }
    cerr << endl;
  }
};

template <class LM_t>
void train(Model &model, LM_t &lm,
    const vector<vector<int>>& training,
    const vector<vector<int>>& dev,
    Trainer *sgd, const string& fname,
    bool randomSample)
{
    double best = 9e+99;
    unsigned report_every_i = 50;
    unsigned dev_every_i_reports = 500;
    unsigned si = training.size();
    vector<unsigned> order(training.size());
    for (unsigned i = 0; i < order.size(); ++i) order[i] = i;
    bool first = true;
    int report = 0;
    unsigned lines = 0;
    while (1) {
        Timer iteration("completed in");
        double loss = 0;
        unsigned chars = 0;
        for (unsigned i = 0; i < report_every_i; ++i) {
            if (si == training.size()) {
                si = 0;
                if (first) { first = false; }
                else { sgd->update_epoch(); }
                cerr << "**SHUFFLE\n";
                shuffle(order.begin(), order.end(), *rndeng);
            }

            // build graph for this instance
            ComputationGraph cg;
            auto& sent = training[order[si]];
            chars += sent.size() - 1;
            ++si;
            lm.BuildLMGraph(sent, cg);
            loss += as_scalar(cg.forward());
            cg.backward();
            sgd->update();
            ++lines;
        }
        sgd->status();
        cerr << " E = " << (loss / chars) << " ppl=" << exp(loss / chars) << ' ';

        if (randomSample)
            lm.RandomSample();

        // show score on dev data?
        report++;
        if (report % dev_every_i_reports == 0) {
            double dloss = 0;
            int dchars = 0;
            for (auto& sent : dev) {
                ComputationGraph cg;
                lm.BuildLMGraph(sent, cg);
                dloss += as_scalar(cg.forward());
                dchars += sent.size() - 1;
            }
            if (dloss < best) {
                best = dloss;
                ofstream out(fname);
                boost::archive::text_oarchive oa(out);
                oa << model;
            }
            else{
                sgd->eta *= 0.5;
            }
            cerr << "\n***TEST E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
        }
    }
}

template <class LM_t>
void testcorpus(Model &model, LM_t &lm,
    const vector<vector<int>>& dev)
{
    unsigned lines = 0;
    double dloss = 0;
    int dchars = 0;
    for (auto& sent : dev) {
        ComputationGraph cg;
        lm.BuildLMGraph(sent, cg);
        dloss += as_scalar(cg.forward());
        dchars += sent.size() - 1;
    }

    cerr << "\n***DEV [epoch=" << (lines / (double)dev.size()) << "] E = " << (dloss / dchars) << " ppl=" << exp(dloss / dchars) << ' ';
}

void initialise(Model &model, const string &filename)
{
    cerr << "Initialising model parameters from file: " << filename << endl;
    ifstream in(filename);
    boost::archive::text_iarchive ia(in);
    ia >> model;
}

int main(int argc, char** argv) {
  cnn::Initialize(argc, argv);

  // command line processing
  using namespace boost::program_options;
  variables_map vm;
  options_description opts("Allowed options");
  opts.add_options()
      ("help", "print help message")
      ("seed,s", value<int>()->default_value(217), "random seed number")
      ("train,t", value<string>(), "file containing training sentences")
      ("devel,d", value<string>(), "file containing development sentences.")
      ("test,T", value<string>(), "file containing testing source sentences")
      ("initialise,i", value<string>(), "load initial parameters from file")
      ("parameters,p", value<string>(), "save best parameters to this file")
      ("layers,l", value<int>()->default_value(LAYERS), "use <num> layers for RNN components")
      ("hidden,h", value<int>()->default_value(HIDDEN_DIM), "use <num> dimensions for recurrent hidden states")
      ("gru", "use Gated Recurrent Unit (GRU) for recurrent structure; default RNN")
      ("lstm", "use Long Short Term Memory (GRU) for recurrent structure; default RNN")
      ("dglstm", "use depth-gated LSTM for recurrent structure; default RNN")
      ("verbose,v", "be extremely chatty")
      ("generate,g", value<bool>()->default_value(false), "generate random samples")
      ;
  store(parse_command_line(argc, argv, opts), vm);

  string flavour;
  if (vm.count("gru"))	flavour = "gru";
  else if (vm.count("lstm"))	flavour = "lstm";
  else if (vm.count("rnnem"))	flavour = "rnnem";
  else if (vm.count("dglstm")) flavour = "dglstm";
  else			flavour = "rnn";

  LAYERS = vm["layers"].as<int>();
  HIDDEN_DIM = vm["hidden"].as<int>();

  bool generateSample = false;
  generateSample = vm["generate"].as<bool>();

  string fname;
  if (vm.count("parameters")) {
      fname = vm["parameters"].as<string>();
  }
  else {
      ostringstream os;
      os << "lm"
          << '_' << LAYERS
          << '_' << HIDDEN_DIM
          << '_' << flavour
          << "-pid" << getpid() << ".params";
      fname = os.str();
  }

  cerr << "Parameters will be written to: " << fname << endl;

  if (vm.count("help") || vm.count("train") != 1 || (vm.count("devel") != 1 && vm.count("test") != 1)) {
      cout << opts << "\n";
      return 1;
  }

  kSOS = d.Convert("<s>");
  kEOS = d.Convert("</s>");
  vector<vector<int>> training, dev, test;
  string line;
  int tlc = 0;
  int ttoks = 0;

  string infile = vm["train"].as<string>();
  cerr << "Reading training data from " << infile << "...\n";

  {
    ifstream in(infile);
    assert(in);
    while(getline(in, line)) {
      ++tlc;
      training.push_back(ReadSentence(line, &d));
      ttoks += training.back().size();
      if (training.back().front() != kSOS && training.back().back() != kEOS) {
        cerr << "Training sentence in " << infile << ":" << tlc << " didn't start or end with <s>, </s>\n";
        abort();
      }
    }
    cerr << tlc << " lines, " << ttoks << " tokens, " << d.size() << " types\n";
  }
  d.Freeze(); // no new word types allowed
  VOCAB_SIZE = d.size();

  if (vm.count("devel") > 0)
  {
      int dlc = 0;
      int dtoks = 0;
      string devfile = vm["devel"].as<string>();
      cerr << "Reading training data from " << devfile << "...\n";
      {
          ifstream in(devfile);
          assert(in);
          while (getline(in, line)) {
              ++dlc;
              dev.push_back(ReadSentence(line, &d));
              dtoks += dev.back().size();
              if (dev.back().front() != kSOS && dev.back().back() != kEOS) {
                  cerr << "Dev sentence in " << devfile << ":" << tlc << " didn't start or end with <s>, </s>\n";
                  abort();
              }
          }
          cerr << dlc << " lines, " << dtoks << " tokens\n";
      }
  }

  Model model;
  bool use_momentum = false;
  Trainer* sgd = nullptr;
  //if (use_momentum)
  //  sgd = new MomentumSGDTrainer(&model);
  //else
  sgd = new SimpleSGDTrainer(&model);

  if (vm.count("test") == 0)
  {
      if (vm.count("lstm")) {
          cerr << "%% Using LSTM recurrent units" << endl;
          RNNLanguageModel<LSTMBuilder> lm(model);
          train(model, lm, training, dev, sgd, fname, generateSample);
      }
      else if (vm.count("dglstm")) {
          cerr << "%% Using DGLSTM recurrent units" << endl;
          RNNLanguageModel<DGLSTMBuilder> lm(model);
          train(model, lm, training, dev, sgd, fname, generateSample);
      }
  }
  else
  {
      string testfile = vm["test"].as<string>();
      int dlc = 0;
      int dtoks = 0;
      cerr << "Reading training data from " << testfile << "...\n";
      {
          ifstream in(testfile);
          assert(in);
          while (getline(in, line)) {
              ++dlc;
              test.push_back(ReadSentence(line, &d));
              dtoks += test.back().size();
              if (test.back().front() != kSOS && test.back().back() != kEOS) {
                  cerr << "Dev sentence in " << testfile << ":" << tlc << " didn't start or end with <s>, </s>\n";
                  abort();
              }
          }
          cerr << dlc << " lines, " << dtoks << " tokens\n";
      }

      if (vm.count("test"))
      {
          if (vm.count("lstm")){
              cerr << "%% using LSTM recurrent units" << endl;
              RNNLanguageModel<LSTMBuilder> lm(model);
              if (vm.count("initialise"))
                  initialise(model, vm["initialise"].as<string>());
              testcorpus(model, lm, test);
          }
          if (vm.count("dglstm")){
              cerr << "%% using LSTM recurrent units" << endl;
              RNNLanguageModel<DGLSTMBuilder> lm(model);
              if (vm.count("initialise"))
                  initialise(model, vm["initialise"].as<string>());
              testcorpus(model, lm, test);
          }
      }
  }

  //RNNLanguageModel<SimpleRNNBuilder> lm(model);
  if (argc == 4) {
    string fname = argv[3];
    ifstream in(fname);
    boost::archive::text_iarchive ia(in);
    ia >> model;
  }

  delete sgd;
}

<|endoftext|>


<commit_before>/***************************Copyright-DO-NOT-REMOVE-THIS-LINE**
  *
  * Condor Software Copyright Notice
  * Copyright (C) 1990-2006, Condor Team, Computer Sciences Department,
  * University of Wisconsin-Madison, WI.
  *
  * This source code is covered by the Condor Public License, which can
  * be found in the accompanying LICENSE.TXT file, or online at
  * www.condorproject.org.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * AND THE UNIVERSITY OF WISCONSIN-MADISON "AS IS" AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY, OF SATISFACTORY QUALITY, AND FITNESS
  * FOR A PARTICULAR PURPOSE OR USE ARE DISCLAIMED. THE COPYRIGHT
  * HOLDERS AND CONTRIBUTORS AND THE UNIVERSITY OF WISCONSIN-MADISON
  * MAKE NO MAKE NO REPRESENTATION THAT THE SOFTWARE, MODIFICATIONS,
  * ENHANCEMENTS OR DERIVATIVE WORKS THEREOF, WILL NOT INFRINGE ANY
  * PATENT, COPYRIGHT, TRADEMARK, TRADE SECRET OR OTHER PROPRIETARY
  * RIGHT.
  *
  ****************************Copyright-DO-NOT-REMOVE-THIS-LINE**/

#include "condor_common.h"
#include "startd.h"
#include "basename.h"
#include "hook_utils.h"

// // // // // // // // // // // //
// StartdHookMgr
// // // // // // // // // // // //

StartdHookMgr::StartdHookMgr()
	: HookClientMgr(),
	  NUM_HOOKS(3),
	  UNDEFINED((char*)1),
	  m_slot_hook_keywords(resmgr->numSlots()),
	  m_keyword_hook_paths(MyStringHash)
{
	dprintf( D_FULLDEBUG, "Instantiating a StartdHookMgr\n" );
	m_slot_hook_keywords.setFiller(NULL);
	m_startd_job_hook_keyword = NULL;
}

StartdHookMgr::~StartdHookMgr()
{
	dprintf( D_FULLDEBUG, "Deleting the StartdHookMgr\n" );

		// Delete our copies of the paths for each hook.
	clearHookPaths();

}

void
StartdHookMgr::clearHookPaths()
{
	if (m_startd_job_hook_keyword) {
		free(m_startd_job_hook_keyword);
		m_startd_job_hook_keyword = NULL;
	}

	int i;
	for (i=0; i <= m_slot_hook_keywords.getlast(); i++) {
		if (m_slot_hook_keywords[i] && m_slot_hook_keywords[i] != UNDEFINED) {
			free(m_slot_hook_keywords[i]);
		}
		m_slot_hook_keywords[i] = NULL;
	}

	MyString key;
	char** hook_paths;
	m_keyword_hook_paths.startIterations();
	while (m_keyword_hook_paths.iterate(key, hook_paths)) {
		for (i=0; i<NUM_HOOKS; i++) {
			if (hook_paths[i] && hook_paths[i] != UNDEFINED) {
				free(hook_paths[i]);
			}
		}
		delete [] hook_paths;
	}
	m_keyword_hook_paths.clear();
}

bool
StartdHookMgr::initialize()
{
	reconfig();
	return HookClientMgr::initialize();
}

bool
StartdHookMgr::reconfig()
{
		// Clear out our old copies of each hook's path.
	clearHookPaths();

		// Grab the global setting if the per-slot keywords aren't defined.
	m_startd_job_hook_keyword = param("STARTD_JOB_HOOK_KEYWORD");

	return true;
}

char*
StartdHookMgr::getHookPath(HookType hook_type, Resource* rip)
{
	char* keyword = getHookKeyword(rip);

	if (!keyword) {
			// Nothing defined, bail now.
		return NULL;
	}

	int i;
	MyString key(keyword);
	char** hook_paths;
	if (m_keyword_hook_paths.lookup(key, hook_paths) < 0) {
			// No entry, initialize it.
		hook_paths = new char*[NUM_HOOKS];
		for (i=0; i<NUM_HOOKS; i++) {
			hook_paths[i] = NULL;
		}
		m_keyword_hook_paths.insert(key, hook_paths);
	}

	char* path = hook_paths[(int)hook_type];
	if (!path) {
		MyString _param;
		_param.sprintf("%s_HOOK_%s", keyword, getHookTypeString(hook_type));
		path = validateHookPath(_param.Value());
		if (!path) {
			hook_paths[(int)hook_type] = UNDEFINED;
		}
		else {
			hook_paths[(int)hook_type] = path;
		}
		rip->dprintf(D_FULLDEBUG, "Hook %s: %s\n", _param.Value(),
					 path ? path : "UNDEFINED");
	}
	else if (path == UNDEFINED) {
		path = NULL;
	}
	return path;
}

char*
StartdHookMgr::getHookKeyword(Resource* rip)
{
	int slot_id = rip->r_id;
	char* keyword = m_slot_hook_keywords[slot_id];
	if (!keyword) {
		MyString param_name;
		param_name.sprintf("%s_JOB_HOOK_KEYWORD", rip->r_id_str);
		keyword = param(param_name.Value());
		m_slot_hook_keywords[slot_id] = keyword ? keyword : UNDEFINED;
	}
	else if (keyword == UNDEFINED) {
		keyword = NULL;
	}
	return keyword ? keyword : m_startd_job_hook_keyword;
}

FetchClient*
StartdHookMgr::buildFetchClient(Resource* rip)
{
	char* hook_path = getHookPath(HOOK_FETCH_WORK, rip);
	if (!hook_path) {
			// No fetch hook defined for this slot, abort.
		return NULL;
	}
	FetchClient* new_client = new FetchClient(rip, hook_path);
	return new_client;
}

bool
StartdHookMgr::invokeHookFetchWork(Resource* rip)
{
	FetchClient* fetch_client = buildFetchClient(rip);
	if (!fetch_client) {
		return false;
	}
	return fetch_client->startFetch();
}

bool
StartdHookMgr::handleHookFetchWork(FetchClient* fetch_client)
{
	ClassAd* job_ad = NULL;
	Resource* rip = fetch_client->m_rip;
	float rank = 0;
	bool willing = true;
		// Are we currently in Claimed/Idle with a fetched claim?
	bool idle_fetch_claim = (rip->r_cur->type() == CLAIM_FETCH
							 && rip->state() == claimed_state
							 && rip->activity() == idle_act);

	if (!(job_ad = fetch_client->reply())) {
			// No work or error reading the reply, bail out.
			// Try other hooks?
		if (idle_fetch_claim) {
				// we're currently Claimed/Idle with a fetched
				// claim. If the fetch hook just returned no data, it
				// means we're out of work, we should evict this
				// claim, and return to the Owner state.
			rip->terminateFetchedWork();
		}
		return false;
	}

		// If we got here, we've got a ClassAd describing the job, so
		// see if this slot is willing to run it.
	if (!rip->willingToRun(job_ad)) {
		willing = false;
	}
	else {
		rank = compute_rank(rip->r_classad, job_ad);
		rip->dprintf(D_FULLDEBUG, "Rank of this fetched claim is: %f\n", rank);
		if (rip->state() == claimed_state && !idle_fetch_claim) {
				// Make sure it's got a high enough rank to preempt us.
			if (rank <= rip->r_cur->rank()) {
					// For fetched jobs, there's no user priority
					// preemption, so the newer claim has to have higher,
					// not just equal rank.
				rip->dprintf(D_ALWAYS, "Fetched claim doesn't have sufficient rank, refusing.\n");
				willing = false;
			}
		}
	}

		// Make sure that the job classad defines ATTR_HOOK_KEYWORD,
		// and if not, insert this slot's keyword.
	char buf[1];	// We don't care what it is, just if it's there.
	if (!job_ad->LookupString(ATTR_HOOK_KEYWORD, buf, 1)) {
		char* keyword = getHookKeyword(rip);
		ASSERT(keyword && keyword != UNDEFINED);
		MyString keyword_attr;
		keyword_attr.sprintf("%s = \"%s\"", ATTR_HOOK_KEYWORD, keyword);
		job_ad->Insert(keyword_attr.Value());
	}

		// No matter what, if the reply claim hook is configured, invoke it.
	hookReplyClaim(willing, job_ad, rip);

	if (!willing) {
			// TODO-fetch: matchmaking on other slots?
		if (idle_fetch_claim) {
				// The slot is Claimed/Idle with a fetch claim. If we
				// just fetched work and aren't willing to run it, we
				// need to evict this claim and return to Owner.
			rip->terminateFetchedWork();
		}
		return false;
	}

		// We're ready to start running the job, so we need to update
		// the current Claim and Client objects to remember this work.
	rip->createOrUpdateFetchClaim(job_ad, rank);

		// Once we've done that, the Claim object in the Resource has
		// control over the job classad, so we want to NULL-out our
		// copy here to avoid a double-free.
	fetch_client->clearReplyAd();

		// Now, depending on our current state, initiate a state change.
	if (rip->state() == claimed_state) {
		if (idle_fetch_claim) {
				// We've got an idle fetch claim and we just got more
				// work, so we should spawn it.
			rip->spawnFetchedWork();
			return true;
		}
			// We're already claimed, but not via an idle fetch claim,
			// so we need to preempt the current job first.
		rip->dprintf(D_ALWAYS, "State change: preempting claim based on "
					 "machine rank of fetched work.\n");

			// Force resource to take note of the preempting claim.
			// This results in a reversible transition to the
			// retiring activity.  If the preempting claim goes
			// away before the current claim retires, the current
			// claim can unretire and continue without any disturbance.
		rip->eval_state();
	}
	else {
			// Start moving towards Claimed so we actually spawn the job.
		dprintf(D_ALWAYS, "State change: Finished fetching work successfully\n");
		rip->r_state->set_destination(claimed_state);
	}
	return true;
}

void
StartdHookMgr::hookReplyClaim(bool claimed, ClassAd* job_ad, Resource* rip)
{
	char* hook_path = getHookPath(HOOK_REPLY_CLAIM, rip);
	if (!hook_path) {
		return;
	}

		// Since we're not saving the output, this can just live on
		// the stack and be destroyed as soon as we return.
	HookClient hook_client(HOOK_REPLY_CLAIM, hook_path, false);

		// Construct the output to write to STDIN.
	MyString hook_stdin;
	job_ad->sPrint(hook_stdin);
	hook_stdin += "-----\n";  // TODO-fetch: better delimiter?
	rip->r_classad->sPrint(hook_stdin);
	if (claimed) {
			// Also include the claim id in the slot classad.
		hook_stdin += ATTR_CLAIM_ID;
		hook_stdin += " = \"";
		hook_stdin += rip->r_cur->id();
		hook_stdin += "\"\n";
	}

	ArgList args;
	args.AppendArg((claimed ? "accept" : "reject"));

	spawn(&hook_client, &args, &hook_stdin);
}

void
StartdHookMgr::hookEvictClaim(Resource* rip)
{
	char* hook_path = getHookPath(HOOK_EVICT_CLAIM, rip);
	if (!hook_path) {
		return;
	}

		// Since we're not saving the output, this can just live on
		// the stack and be destroyed as soon as we return.
	HookClient hook_client(HOOK_EVICT_CLAIM, hook_path, false);

		// Construct the output to write to STDIN.
	MyString hook_stdin;
	rip->r_cur->ad()->sPrint(hook_stdin);
	hook_stdin += "-----\n";  // TODO-fetch: better delimiter?
	rip->r_classad->sPrint(hook_stdin);
		// Also include the claim id in the slot classad.
	hook_stdin += ATTR_CLAIM_ID;
	hook_stdin += " = \"";
	hook_stdin += rip->r_cur->id();
	hook_stdin += "\"\n";

	spawn(&hook_client, NULL, &hook_stdin);
}

// // // // // // // // // // // //
// FetchClient class
// // // // // // // // // // // //

FetchClient::FetchClient(Resource* rip, const char* hook_path)
	: HookClient(HOOK_FETCH_WORK, hook_path, true)
{
	m_rip = rip;
	m_job_ad = NULL;
}

FetchClient::~FetchClient()
{
	if (m_job_ad) {
		delete m_job_ad;
		m_job_ad = NULL;
	}
}

bool
FetchClient::startFetch()
{
	ASSERT(m_rip);
	ClassAd slot_ad;
	m_rip->publish(&slot_ad, A_ALL_PUB);
	MyString slot_ad_txt;
	slot_ad.sPrint(slot_ad_txt);
	resmgr->m_hook_mgr->spawn(this, NULL, &slot_ad_txt);
	m_rip->startedFetch();
	return true;
}

ClassAd*
FetchClient::reply()
{
	return m_job_ad;
}

void
FetchClient::hookExited(int exit_status) {
	HookClient::hookExited(exit_status);
	if (m_std_err.Length()) {
		dprintf(D_ALWAYS,
				"Warning, hook %s (pid %d) printed to stderr: %s\n",
				m_hook_path, (int)m_pid, m_std_err.Value());
	}
	if (m_std_out.Length()) {
		ASSERT(m_job_ad == NULL);
		m_job_ad = new ClassAd();
		m_std_out.Tokenize();
		const char* hook_line = NULL;
		while ((hook_line = m_std_out.GetNextToken("\n", true))) {
			if (!m_job_ad->Insert(hook_line)) {
				dprintf(D_ALWAYS, "Failed to insert \"%s\" into ClassAd, "
						"ignoring invalid hook output\n", hook_line);
					// TODO-pipe howto abort?
					// Tell the slot this fetch invocation completed.
				m_rip->fetchCompleted();
				return;
			}
		}
	}
	else {
		dprintf(D_FULLDEBUG, "Hook %s (pid %d) returned no data\n",
				m_hook_path, (int)m_pid);
	}
		// Let the work manager know this fetch result is done.
	resmgr->m_hook_mgr->handleHookFetchWork(this);

		// Tell the slot this fetch invocation completed.
	m_rip->fetchCompleted();
}

void
FetchClient::clearReplyAd(void) {
	m_job_ad = NULL;
}
<commit_msg>Fixed catastrophic test failures on some platforms due to seg faulting startd. The culprit was extArray: once you give it a size, it allocates (uninitialized) memory and calling setFiller() isn't enough, you actually need to use fill() to initialize everything yourself.<commit_after>/***************************Copyright-DO-NOT-REMOVE-THIS-LINE**
  *
  * Condor Software Copyright Notice
  * Copyright (C) 1990-2006, Condor Team, Computer Sciences Department,
  * University of Wisconsin-Madison, WI.
  *
  * This source code is covered by the Condor Public License, which can
  * be found in the accompanying LICENSE.TXT file, or online at
  * www.condorproject.org.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * AND THE UNIVERSITY OF WISCONSIN-MADISON "AS IS" AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY, OF SATISFACTORY QUALITY, AND FITNESS
  * FOR A PARTICULAR PURPOSE OR USE ARE DISCLAIMED. THE COPYRIGHT
  * HOLDERS AND CONTRIBUTORS AND THE UNIVERSITY OF WISCONSIN-MADISON
  * MAKE NO MAKE NO REPRESENTATION THAT THE SOFTWARE, MODIFICATIONS,
  * ENHANCEMENTS OR DERIVATIVE WORKS THEREOF, WILL NOT INFRINGE ANY
  * PATENT, COPYRIGHT, TRADEMARK, TRADE SECRET OR OTHER PROPRIETARY
  * RIGHT.
  *
  ****************************Copyright-DO-NOT-REMOVE-THIS-LINE**/

#include "condor_common.h"
#include "startd.h"
#include "basename.h"
#include "hook_utils.h"

// // // // // // // // // // // //
// StartdHookMgr
// // // // // // // // // // // //

StartdHookMgr::StartdHookMgr()
	: HookClientMgr(),
	  NUM_HOOKS(3),
	  UNDEFINED((char*)1),
	  m_slot_hook_keywords(resmgr->numSlots()),
	  m_keyword_hook_paths(MyStringHash)
{
	dprintf( D_FULLDEBUG, "Instantiating a StartdHookMgr\n" );
	m_slot_hook_keywords.fill(NULL);
	m_startd_job_hook_keyword = NULL;
}

StartdHookMgr::~StartdHookMgr()
{
	dprintf( D_FULLDEBUG, "Deleting the StartdHookMgr\n" );

		// Delete our copies of the paths for each hook.
	clearHookPaths();

}

void
StartdHookMgr::clearHookPaths()
{
	if (m_startd_job_hook_keyword) {
		free(m_startd_job_hook_keyword);
		m_startd_job_hook_keyword = NULL;
	}

	int i;
	for (i=0; i <= m_slot_hook_keywords.getlast(); i++) {
		if (m_slot_hook_keywords[i] && m_slot_hook_keywords[i] != UNDEFINED) {
			free(m_slot_hook_keywords[i]);
		}
		m_slot_hook_keywords[i] = NULL;
	}

	MyString key;
	char** hook_paths;
	m_keyword_hook_paths.startIterations();
	while (m_keyword_hook_paths.iterate(key, hook_paths)) {
		for (i=0; i<NUM_HOOKS; i++) {
			if (hook_paths[i] && hook_paths[i] != UNDEFINED) {
				free(hook_paths[i]);
			}
		}
		delete [] hook_paths;
	}
	m_keyword_hook_paths.clear();
}

bool
StartdHookMgr::initialize()
{
	reconfig();
	return HookClientMgr::initialize();
}

bool
StartdHookMgr::reconfig()
{
		// Clear out our old copies of each hook's path.
	clearHookPaths();

		// Grab the global setting if the per-slot keywords aren't defined.
	m_startd_job_hook_keyword = param("STARTD_JOB_HOOK_KEYWORD");

	return true;
}

char*
StartdHookMgr::getHookPath(HookType hook_type, Resource* rip)
{
	char* keyword = getHookKeyword(rip);

	if (!keyword) {
			// Nothing defined, bail now.
		return NULL;
	}

	int i;
	MyString key(keyword);
	char** hook_paths;
	if (m_keyword_hook_paths.lookup(key, hook_paths) < 0) {
			// No entry, initialize it.
		hook_paths = new char*[NUM_HOOKS];
		for (i=0; i<NUM_HOOKS; i++) {
			hook_paths[i] = NULL;
		}
		m_keyword_hook_paths.insert(key, hook_paths);
	}

	char* path = hook_paths[(int)hook_type];
	if (!path) {
		MyString _param;
		_param.sprintf("%s_HOOK_%s", keyword, getHookTypeString(hook_type));
		path = validateHookPath(_param.Value());
		if (!path) {
			hook_paths[(int)hook_type] = UNDEFINED;
		}
		else {
			hook_paths[(int)hook_type] = path;
		}
		rip->dprintf(D_FULLDEBUG, "Hook %s: %s\n", _param.Value(),
					 path ? path : "UNDEFINED");
	}
	else if (path == UNDEFINED) {
		path = NULL;
	}
	return path;
}

char*
StartdHookMgr::getHookKeyword(Resource* rip)
{
	int slot_id = rip->r_id;
	char* keyword = m_slot_hook_keywords[slot_id];
	if (!keyword) {
		MyString param_name;
		param_name.sprintf("%s_JOB_HOOK_KEYWORD", rip->r_id_str);
		keyword = param(param_name.Value());
		m_slot_hook_keywords[slot_id] = keyword ? keyword : UNDEFINED;
	}
	else if (keyword == UNDEFINED) {
		keyword = NULL;
	}
	return keyword ? keyword : m_startd_job_hook_keyword;
}

FetchClient*
StartdHookMgr::buildFetchClient(Resource* rip)
{
	char* hook_path = getHookPath(HOOK_FETCH_WORK, rip);
	if (!hook_path) {
			// No fetch hook defined for this slot, abort.
		return NULL;
	}
	FetchClient* new_client = new FetchClient(rip, hook_path);
	return new_client;
}

bool
StartdHookMgr::invokeHookFetchWork(Resource* rip)
{
	FetchClient* fetch_client = buildFetchClient(rip);
	if (!fetch_client) {
		return false;
	}
	return fetch_client->startFetch();
}

bool
StartdHookMgr::handleHookFetchWork(FetchClient* fetch_client)
{
	ClassAd* job_ad = NULL;
	Resource* rip = fetch_client->m_rip;
	float rank = 0;
	bool willing = true;
		// Are we currently in Claimed/Idle with a fetched claim?
	bool idle_fetch_claim = (rip->r_cur->type() == CLAIM_FETCH
							 && rip->state() == claimed_state
							 && rip->activity() == idle_act);

	if (!(job_ad = fetch_client->reply())) {
			// No work or error reading the reply, bail out.
			// Try other hooks?
		if (idle_fetch_claim) {
				// we're currently Claimed/Idle with a fetched
				// claim. If the fetch hook just returned no data, it
				// means we're out of work, we should evict this
				// claim, and return to the Owner state.
			rip->terminateFetchedWork();
		}
		return false;
	}

		// If we got here, we've got a ClassAd describing the job, so
		// see if this slot is willing to run it.
	if (!rip->willingToRun(job_ad)) {
		willing = false;
	}
	else {
		rank = compute_rank(rip->r_classad, job_ad);
		rip->dprintf(D_FULLDEBUG, "Rank of this fetched claim is: %f\n", rank);
		if (rip->state() == claimed_state && !idle_fetch_claim) {
				// Make sure it's got a high enough rank to preempt us.
			if (rank <= rip->r_cur->rank()) {
					// For fetched jobs, there's no user priority
					// preemption, so the newer claim has to have higher,
					// not just equal rank.
				rip->dprintf(D_ALWAYS, "Fetched claim doesn't have sufficient rank, refusing.\n");
				willing = false;
			}
		}
	}

		// Make sure that the job classad defines ATTR_HOOK_KEYWORD,
		// and if not, insert this slot's keyword.
	char buf[1];	// We don't care what it is, just if it's there.
	if (!job_ad->LookupString(ATTR_HOOK_KEYWORD, buf, 1)) {
		char* keyword = getHookKeyword(rip);
		ASSERT(keyword && keyword != UNDEFINED);
		MyString keyword_attr;
		keyword_attr.sprintf("%s = \"%s\"", ATTR_HOOK_KEYWORD, keyword);
		job_ad->Insert(keyword_attr.Value());
	}

		// No matter what, if the reply claim hook is configured, invoke it.
	hookReplyClaim(willing, job_ad, rip);

	if (!willing) {
			// TODO-fetch: matchmaking on other slots?
		if (idle_fetch_claim) {
				// The slot is Claimed/Idle with a fetch claim. If we
				// just fetched work and aren't willing to run it, we
				// need to evict this claim and return to Owner.
			rip->terminateFetchedWork();
		}
		return false;
	}

		// We're ready to start running the job, so we need to update
		// the current Claim and Client objects to remember this work.
	rip->createOrUpdateFetchClaim(job_ad, rank);

		// Once we've done that, the Claim object in the Resource has
		// control over the job classad, so we want to NULL-out our
		// copy here to avoid a double-free.
	fetch_client->clearReplyAd();

		// Now, depending on our current state, initiate a state change.
	if (rip->state() == claimed_state) {
		if (idle_fetch_claim) {
				// We've got an idle fetch claim and we just got more
				// work, so we should spawn it.
			rip->spawnFetchedWork();
			return true;
		}
			// We're already claimed, but not via an idle fetch claim,
			// so we need to preempt the current job first.
		rip->dprintf(D_ALWAYS, "State change: preempting claim based on "
					 "machine rank of fetched work.\n");

			// Force resource to take note of the preempting claim.
			// This results in a reversible transition to the
			// retiring activity.  If the preempting claim goes
			// away before the current claim retires, the current
			// claim can unretire and continue without any disturbance.
		rip->eval_state();
	}
	else {
			// Start moving towards Claimed so we actually spawn the job.
		dprintf(D_ALWAYS, "State change: Finished fetching work successfully\n");
		rip->r_state->set_destination(claimed_state);
	}
	return true;
}

void
StartdHookMgr::hookReplyClaim(bool claimed, ClassAd* job_ad, Resource* rip)
{
	char* hook_path = getHookPath(HOOK_REPLY_CLAIM, rip);
	if (!hook_path) {
		return;
	}

		// Since we're not saving the output, this can just live on
		// the stack and be destroyed as soon as we return.
	HookClient hook_client(HOOK_REPLY_CLAIM, hook_path, false);

		// Construct the output to write to STDIN.
	MyString hook_stdin;
	job_ad->sPrint(hook_stdin);
	hook_stdin += "-----\n";  // TODO-fetch: better delimiter?
	rip->r_classad->sPrint(hook_stdin);
	if (claimed) {
			// Also include the claim id in the slot classad.
		hook_stdin += ATTR_CLAIM_ID;
		hook_stdin += " = \"";
		hook_stdin += rip->r_cur->id();
		hook_stdin += "\"\n";
	}

	ArgList args;
	args.AppendArg((claimed ? "accept" : "reject"));

	spawn(&hook_client, &args, &hook_stdin);
}

void
StartdHookMgr::hookEvictClaim(Resource* rip)
{
	char* hook_path = getHookPath(HOOK_EVICT_CLAIM, rip);
	if (!hook_path) {
		return;
	}

		// Since we're not saving the output, this can just live on
		// the stack and be destroyed as soon as we return.
	HookClient hook_client(HOOK_EVICT_CLAIM, hook_path, false);

		// Construct the output to write to STDIN.
	MyString hook_stdin;
	rip->r_cur->ad()->sPrint(hook_stdin);
	hook_stdin += "-----\n";  // TODO-fetch: better delimiter?
	rip->r_classad->sPrint(hook_stdin);
		// Also include the claim id in the slot classad.
	hook_stdin += ATTR_CLAIM_ID;
	hook_stdin += " = \"";
	hook_stdin += rip->r_cur->id();
	hook_stdin += "\"\n";

	spawn(&hook_client, NULL, &hook_stdin);
}

// // // // // // // // // // // //
// FetchClient class
// // // // // // // // // // // //

FetchClient::FetchClient(Resource* rip, const char* hook_path)
	: HookClient(HOOK_FETCH_WORK, hook_path, true)
{
	m_rip = rip;
	m_job_ad = NULL;
}

FetchClient::~FetchClient()
{
	if (m_job_ad) {
		delete m_job_ad;
		m_job_ad = NULL;
	}
}

bool
FetchClient::startFetch()
{
	ASSERT(m_rip);
	ClassAd slot_ad;
	m_rip->publish(&slot_ad, A_ALL_PUB);
	MyString slot_ad_txt;
	slot_ad.sPrint(slot_ad_txt);
	resmgr->m_hook_mgr->spawn(this, NULL, &slot_ad_txt);
	m_rip->startedFetch();
	return true;
}

ClassAd*
FetchClient::reply()
{
	return m_job_ad;
}

void
FetchClient::hookExited(int exit_status) {
	HookClient::hookExited(exit_status);
	if (m_std_err.Length()) {
		dprintf(D_ALWAYS,
				"Warning, hook %s (pid %d) printed to stderr: %s\n",
				m_hook_path, (int)m_pid, m_std_err.Value());
	}
	if (m_std_out.Length()) {
		ASSERT(m_job_ad == NULL);
		m_job_ad = new ClassAd();
		m_std_out.Tokenize();
		const char* hook_line = NULL;
		while ((hook_line = m_std_out.GetNextToken("\n", true))) {
			if (!m_job_ad->Insert(hook_line)) {
				dprintf(D_ALWAYS, "Failed to insert \"%s\" into ClassAd, "
						"ignoring invalid hook output\n", hook_line);
					// TODO-pipe howto abort?
					// Tell the slot this fetch invocation completed.
				m_rip->fetchCompleted();
				return;
			}
		}
	}
	else {
		dprintf(D_FULLDEBUG, "Hook %s (pid %d) returned no data\n",
				m_hook_path, (int)m_pid);
	}
		// Let the work manager know this fetch result is done.
	resmgr->m_hook_mgr->handleHookFetchWork(this);

		// Tell the slot this fetch invocation completed.
	m_rip->fetchCompleted();
}

void
FetchClient::clearReplyAd(void) {
	m_job_ad = NULL;
}
<|endoftext|>


<commit_before>/****************************************************************************
 *
 *   Copyright (c) 2013, 2014 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/
/**
 * @file launchDetection.cpp
 * Auto Detection for different launch methods (e.g. catapult)
 *
 * @author Thomas Gubler <thomasgubler@gmail.com>
 */

#include "LaunchDetector.h"
#include "CatapultLaunchMethod.h"
#include <systemlib/err.h>

namespace launchdetection
{

LaunchDetector::LaunchDetector() :
	SuperBlock(NULL, "LAUN"),
	activeLaunchDetectionMethodIndex(-1),
	launchdetection_on(this, "ALL_ON"),
	throttlePreTakeoff(this, "THR_PRE")
{
	/* init all detectors */
	launchMethods[0] = new CatapultLaunchMethod(this);

	/* update all parameters of all detectors */
	updateParams();
}

LaunchDetector::~LaunchDetector()
{

}

void LaunchDetector::reset()
{
	/* Reset all detectors */
	for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod); i++) {
		launchMethods[i]->reset();
	}

	/* Reset active launchdetector */
	activeLaunchDetectionMethodIndex  = -1;

}

void LaunchDetector::update(float accel_x)
{
	if (launchdetection_on.get() == 1) {
		for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod); i++) {
			launchMethods[i]->update(accel_x);
		}
	}
}

LaunchDetectionResult LaunchDetector::getLaunchDetected()
{
	if (launchdetection_on.get() == 1) {
		if (activeLaunchDetectionMethodIndex < 0) {
		/* None of the active launchmethods has detected a launch, check all launchmethods */
			for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod); i++) {
				if(launchMethods[i]->getLaunchDetected() != LAUNCHDETECTION_RES_NONE) {
					warnx("selecting launchmethod %d", i);
					activeLaunchDetectionMethodIndex = i; // from now on only check this method
					return launchMethods[i]->getLaunchDetected();
				}
			}
		} else {
			return launchMethods[activeLaunchDetectionMethodIndex]->getLaunchDetected();
		}
	}

	return LAUNCHDETECTION_RES_NONE;
}

float LaunchDetector::getPitchMax(float pitchMaxDefault) {
	if (!launchdetection_on.get()) {
		return pitchMaxDefault;
	}

	/* if a lauchdetectionmethod is active or only one exists return the pitch limit from this method,
	 * otherwise use the default limit */
	if (activeLaunchDetectionMethodIndex < 0) {
		if (sizeof(launchMethods)/sizeof(LaunchMethod) > 1) {
			return pitchMaxDefault;
		} else {
			return launchMethods[0]->getPitchMax(pitchMaxDefault);
		}
	} else {
		return launchMethods[activeLaunchDetectionMethodIndex]->getPitchMax(pitchMaxDefault);
	}

}

}
<commit_msg>fix launch method iteration<commit_after>/****************************************************************************
 *
 *   Copyright (c) 2013, 2014 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/
/**
 * @file launchDetection.cpp
 * Auto Detection for different launch methods (e.g. catapult)
 *
 * @author Thomas Gubler <thomasgubler@gmail.com>
 */

#include "LaunchDetector.h"
#include "CatapultLaunchMethod.h"
#include <systemlib/err.h>

namespace launchdetection
{

LaunchDetector::LaunchDetector() :
	SuperBlock(NULL, "LAUN"),
	activeLaunchDetectionMethodIndex(-1),
	launchdetection_on(this, "ALL_ON"),
	throttlePreTakeoff(this, "THR_PRE")
{
	/* init all detectors */
	launchMethods[0] = new CatapultLaunchMethod(this);

	/* update all parameters of all detectors */
	updateParams();
}

LaunchDetector::~LaunchDetector()
{

}

void LaunchDetector::reset()
{
	/* Reset all detectors */
	for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod *); i++) {
		launchMethods[i]->reset();
	}

	/* Reset active launchdetector */
	activeLaunchDetectionMethodIndex  = -1;

}

void LaunchDetector::update(float accel_x)
{
	if (launchdetection_on.get() == 1) {
		for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod *); i++) {
			launchMethods[i]->update(accel_x);
		}
	}
}

LaunchDetectionResult LaunchDetector::getLaunchDetected()
{
	if (launchdetection_on.get() == 1) {
		if (activeLaunchDetectionMethodIndex < 0) {
		/* None of the active launchmethods has detected a launch, check all launchmethods */
			for (uint8_t i = 0; i < sizeof(launchMethods)/sizeof(LaunchMethod *); i++) {
				if(launchMethods[i]->getLaunchDetected() != LAUNCHDETECTION_RES_NONE) {
					warnx("selecting launchmethod %d", i);
					activeLaunchDetectionMethodIndex = i; // from now on only check this method
					return launchMethods[i]->getLaunchDetected();
				}
			}
		} else {
			return launchMethods[activeLaunchDetectionMethodIndex]->getLaunchDetected();
		}
	}

	return LAUNCHDETECTION_RES_NONE;
}

float LaunchDetector::getPitchMax(float pitchMaxDefault) {
	if (!launchdetection_on.get()) {
		return pitchMaxDefault;
	}

	/* if a lauchdetectionmethod is active or only one exists return the pitch limit from this method,
	 * otherwise use the default limit */
	if (activeLaunchDetectionMethodIndex < 0) {
		if (sizeof(launchMethods)/sizeof(LaunchMethod *) > 1) {
			return pitchMaxDefault;
		} else {
			return launchMethods[0]->getPitchMax(pitchMaxDefault);
		}
	} else {
		return launchMethods[activeLaunchDetectionMethodIndex]->getPitchMax(pitchMaxDefault);
	}

}

}
<|endoftext|>


<commit_before>#ifndef NEURO_TILE_HPP
#define NEURO_TILE_HPP

namespace neuro {

using Player=int;
using Orientation=int;

	/**
		* @brief All the possible types of tiles.
		*/
	enum class TileType {
		INSTANT_ACTION,
		HQ,
		MODULE,
		UNIT,
		FOUNDATION
	};

	/**
		* @brief A tile to be created in an army and later played.
		* @todo Lists of abilities:
		*  -playing
		*  -on board
		*  -permanent edge x6
		*  -battle edge x6
		*/
	class Tile : public ui::Observable<Tile> {
		public:
			/**
				* @brief Construct a tile with a specific type, health and initiative.
				* @param[in] type Type of the tile.
				* @param[in] health Health of the tile. Defaults to 0.
				* @param[in] initiative Initiative of the tile. Defaults to 0.
				*/
			Tile(TileType type, int health = 0, int initiative = 0) : type(type), health(health), initiative(initiative), orientation(0) {}

			/**
				* @brief Returns the type of the tile.
				*/
			TileType getType() const { return type; }

			/**
				* @brief Returns the owner of the tile.
				*/
			Player getOwner() const { return owner; }

			/**
				* @brief Returns the current controller of the tile.
				*/
			Player getController() const { return controller; }

			/**
				* @brief Returns the current orientation of the tile.
				*/
			Orientation getOrientation() const { return orientation; }

			/**
				* @brief Returns the current health of the tile.
				*/
			int getHealth() const { return health; }

			/**
				* @brief Returns the current initiative of the tile if positive, else 0.
				*/
			int getInitiative() const { return (initiative > 0) ? initiative : 0; }

			/**
				* @brief Tells whether the tile is considered a solid object.
				*/
			bool isSolid() const { return (type == TileType::INSTANT_ACTION) || (type == TileType::FOUNDATION); }

			/**
				* @brief Sets the owner and original controller of the tile.
				* @param[in] player The player to become the owner.
				*/
			void setOwner(Player player) {
				owner = player;
				controller = player;
			}

			/**
				* @brief Changes the orientation of the tile.
				* @param[in] orient The new orientation of the tile.
				*/
			void rotate(Orientation orient) {
				orientation = orient;
				sigModified(*this);
			}

			/**
				* @brief Changes the controller of the tile.
				* @param[in] player The new controller of the tile.
				*/
			void changeController(Player player) {
				controller = player;
				sigModified(*this);
			}

			/**
				* @brief Modifies the health of the tile.
				* @param[in] mod The amount of health to be added.
				*/
			void modifyHealth(int mod) {
				health += mod;
				sigModified(*this);
			}

			/**
				* @brief Modifies the initiative of the tile.
				* @param[in] mod The amount of initiative to be added.
				*/
			void modifyInitiative(int mod) {
				initiative += mod;
				sigModified(*this);
			}
		private:
			TileType type;
			Player owner;
			Player controller;
			Orientation orientation; // An integer between 0 and 5.
			int health;
			int initiative;
	};

}

#endif
<commit_msg>Added neccessary include in Tile<commit_after>#ifndef NEURO_TILE_HPP
#define NEURO_TILE_HPP

#include"ui/Observable.hpp"

namespace neuro {

using Player=int;
using Orientation=int;

	/**
		* @brief All the possible types of tiles.
		*/
	enum class TileType {
		INSTANT_ACTION,
		HQ,
		MODULE,
		UNIT,
		FOUNDATION
	};

	/**
		* @brief A tile to be created in an army and later played.
		* @todo Lists of abilities:
		*  -playing
		*  -on board
		*  -permanent edge x6
		*  -battle edge x6
		*/
	class Tile : public ui::Observable<Tile> {
		public:
			/**
				* @brief Construct a tile with a specific type, health and initiative.
				* @param[in] type Type of the tile.
				* @param[in] health Health of the tile. Defaults to 0.
				* @param[in] initiative Initiative of the tile. Defaults to 0.
				*/
			Tile(TileType type, int health = 0, int initiative = 0) : type(type), health(health), initiative(initiative), orientation(0) {}

			/**
				* @brief Returns the type of the tile.
				*/
			TileType getType() const { return type; }

			/**
				* @brief Returns the owner of the tile.
				*/
			Player getOwner() const { return owner; }

			/**
				* @brief Returns the current controller of the tile.
				*/
			Player getController() const { return controller; }

			/**
				* @brief Returns the current orientation of the tile.
				*/
			Orientation getOrientation() const { return orientation; }

			/**
				* @brief Returns the current health of the tile.
				*/
			int getHealth() const { return health; }

			/**
				* @brief Returns the current initiative of the tile if positive, else 0.
				*/
			int getInitiative() const { return (initiative > 0) ? initiative : 0; }

			/**
				* @brief Tells whether the tile is considered a solid object.
				*/
			bool isSolid() const { return (type == TileType::INSTANT_ACTION) || (type == TileType::FOUNDATION); }

			/**
				* @brief Sets the owner and original controller of the tile.
				* @param[in] player The player to become the owner.
				*/
			void setOwner(Player player) {
				owner = player;
				controller = player;
			}

			/**
				* @brief Changes the orientation of the tile.
				* @param[in] orient The new orientation of the tile.
				*/
			void rotate(Orientation orient) {
				orientation = orient;
				sigModified(*this);
			}

			/**
				* @brief Changes the controller of the tile.
				* @param[in] player The new controller of the tile.
				*/
			void changeController(Player player) {
				controller = player;
				sigModified(*this);
			}

			/**
				* @brief Modifies the health of the tile.
				* @param[in] mod The amount of health to be added.
				*/
			void modifyHealth(int mod) {
				health += mod;
				sigModified(*this);
			}

			/**
				* @brief Modifies the initiative of the tile.
				* @param[in] mod The amount of initiative to be added.
				*/
			void modifyInitiative(int mod) {
				initiative += mod;
				sigModified(*this);
			}
		private:
			TileType type;
			Player owner;
			Player controller;
			int health;
			int initiative;
			Orientation orientation; // An integer between 0 and 5.
	};

}

#endif
<|endoftext|>