<commit_before>// $Id$
// vim:tabstop=2

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/
#ifdef WIN32
#include <hash_set>
#else
#include <ext/hash_set>
#endif

#include <limits>
#include <cmath>
#include "Manager.h"
#include "TypeDef.h"
#include "Util.h"
#include "TargetPhrase.h"
#include "LatticePath.h"
#include "LatticePathCollection.h"
#include "TranslationOption.h"
#include "LMList.h"
#include "TranslationOptionCollection.h"

using namespace std;

Manager::Manager(InputType const& source, StaticData &staticData)
:m_source(source)
,m_hypoStack(source.GetSize() + 1)
,m_staticData(staticData)
,m_possibleTranslations(source.CreateTranslationOptionCollection())
,m_initialTargetPhrase(Output)
{
	std::vector < HypothesisCollection >::iterator iterStack;
	for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		HypothesisCollection &sourceHypoColl = *iterStack;
		sourceHypoColl.SetMaxHypoStackSize(m_staticData.GetMaxHypoStackSize());
		sourceHypoColl.SetBeamThreshold(m_staticData.GetBeamThreshold());
	}
}

Manager::~Manager()
{
  delete m_possibleTranslations;
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void Manager::ProcessSentence()
{
	m_staticData.ResetSentenceStats(m_source);
	list < DecodeStep* > &decodeStepList = m_staticData.GetDecodeStepList();
	// create list of all possible translations
	// this is only valid if:
	//		1. generation of source sentence is not done 1st
	//		2. initial hypothesis factors are given in the sentence
	//CreateTranslationOptions(m_source, phraseDictionary, lmListInitial);
	m_possibleTranslations->CreateTranslationOptions(decodeStepList
  														, m_staticData.GetFactorCollection());

	// initial seed hypothesis: nothing translated, no words produced
	{
		Hypothesis *hypo = Hypothesis::Create(m_source, m_initialTargetPhrase);
		m_hypoStack[0].AddPrune(hypo);
	}

	// go through each stack
	std::vector < HypothesisCollection >::iterator iterStack;
	for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		HypothesisCollection &sourceHypoColl = *iterStack;

		// the stack is pruned before processing (lazy pruning):
		VERBOSE(3,"processing hypothesis from next stack");
		sourceHypoColl.PruneToSize(m_staticData.GetMaxHypoStackSize());
		VERBOSE(3,std::endl);
		sourceHypoColl.InitializeArcs();

		// go through each hypothesis on the stack and try to expand it
		HypothesisCollection::const_iterator iterHypo;
		for (iterHypo = sourceHypoColl.begin() ; iterHypo != sourceHypoColl.end() ; ++iterHypo)
			{
				Hypothesis &hypothesis = **iterHypo;
				ProcessOneHypothesis(hypothesis); // expand the hypothesis
			}
		// some logging
		IFVERBOSE(2) { OutputHypoStackSize(); }
	}

	// some more logging
	VERBOSE(2,m_staticData.GetSentenceStats());
}

/** Find all translation options to expand one hypothesis, trigger expansion
 * this is mostly a check for overlap with already covered words, and for
 * violation of reordering limits.
 * \param hypothesis hypothesis to be expanded upon
 */
void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
{
	// since we check for reordering limits, its good to have that limit handy
	int maxDistortion = m_staticData.GetMaxDistortion();

	// no limit of reordering: only check for overlap
	if (maxDistortion < 0)
	{
		const WordsBitmap hypoBitmap	= hypothesis.GetWordsBitmap();
		const size_t hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
								, sourceSize			= m_source.GetSize();

		for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
		{
			for (size_t endPos = startPos ; endPos < sourceSize ; ++endPos)
			{
				if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
				{
					ExpandAllHypotheses(hypothesis
												, m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
				}
			}
		}

		return; // done with special case (no reordering limit)
	}

	// if there are reordering limits, make sure it is not violated
	// the coverage bitmap is handy here (and the position of the first gap)
	const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
	const size_t hypoWordCount		= hypoBitmap.GetNumWordsCovered()
							, hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
							, sourceSize			= m_source.GetSize();

	// MAIN LOOP. go through each possible hypo
	for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
	{
		for (size_t endPos = startPos ; endPos < sourceSize ; ++endPos)
		{
			// no gap so far => don't skip more than allowed limit
			if (hypoFirstGapPos == hypoWordCount)
				{
					if (startPos == hypoWordCount
							|| (startPos > hypoWordCount
									&& endPos <= hypoWordCount + maxDistortion)
					)
				{
					ExpandAllHypotheses(hypothesis
												,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
				}
			}
			// filling in gap => just check for overlap
			else if (startPos < hypoWordCount)
				{
					if (startPos >= hypoFirstGapPos
						&& !hypoBitmap.Overlap(WordsRange(startPos, endPos)))
					{
						ExpandAllHypotheses(hypothesis
													,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
					}
				}
			// ignoring, continuing forward => be limited by start of gap
			else
				{
					if (endPos <= hypoFirstGapPos + maxDistortion
						&& !hypoBitmap.Overlap(WordsRange(startPos, endPos)))
					{
						ExpandAllHypotheses(hypothesis
													,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
					}
				}
		}
	}
}

/**
 * Expand a hypothesis given a list of translation options
 * \param hypothesis hypothesis to be expanded upon
 * \param transOptList list of translation options to be applied
 */

void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
{
	TranslationOptionList::const_iterator iter;
	for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
	{
		ExpandHypothesis(hypothesis, **iter);
	}
}

/**
 * Expand one hypothesis with a translation option.
 * this involves initial creation, scoring and adding it to the proper stack
 * \param hypothesis hypothesis to be expanded upon
 * \param transOpt translation option (phrase translation)
 *        that is applied to create the new hypothesis
 */
void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
{
	// create hypothesis and calculate all its scores
	Hypothesis *newHypo = hypothesis.CreateNext(transOpt);
	newHypo->CalcScore(m_staticData, m_possibleTranslations->GetFutureScore());

	// logging for the curious
	IFVERBOSE(3) {
	  newHypo->PrintHypothesis(m_source, m_staticData.GetWeightDistortion(), m_staticData.GetWeightWordPenalty());
	}

	// add to hypothesis stack
	size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
	m_hypoStack[wordsTranslated].AddPrune(newHypo);
}

/**
 * Find best hypothesis on the last stack.
 * This is the end point of the best translation, which can be traced back from here
 */
const Hypothesis *Manager::GetBestHypothesis() const
{
	const HypothesisCollection &hypoColl = m_hypoStack.back();
	return hypoColl.GetBestHypothesis();
}

/**
 * Logging of hypothesis stack sizes
 */
void Manager::OutputHypoStackSize()
{
	std::vector < HypothesisCollection >::const_iterator iterStack = m_hypoStack.begin();
	TRACE_ERR("Stack sizes: " << (int)iterStack->size());
	for (++iterStack; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		TRACE_ERR ( ", " << (int)iterStack->size());
	}
	TRACE_ERR (endl);
}

/**
 * Logging of hypothesis stack contents
 * \param stack number of stack to be reported, report all stacks if 0
 */
void Manager::OutputHypoStack(int stack)
{
	if (stack >= 0)
	{
		TRACE_ERR ( "Stack " << stack << ": " << endl << m_hypoStack[stack] << endl);
	}
	else
	{ // all stacks
		int i = 0;
		vector < HypothesisCollection >::iterator iterStack;
		for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
		{
			HypothesisCollection &hypoColl = *iterStack;
			TRACE_ERR ( "Stack " << i++ << ": " << endl << hypoColl << endl);
		}
	}
}
void getSurfacePhrase(std::vector<size_t>& tphrase,LatticePath const& path)
{
	tphrase.clear();
	const std::vector<const Hypothesis *> &edges = path.GetEdges();
	for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
		{
			const Phrase &phrase = edges[currEdge]->GetTargetPhrase();
			for (size_t pos=0,size=phrase.GetSize() ; pos < size ; ++pos)
				{
					const Factor *factor = phrase.GetFactor(pos,0);
					assert(factor);
					tphrase.push_back(factor->GetId());
				}
		}
}

/**
 * After decoding, the hypotheses in the stacks and additional arcs
 * form a search graph that can be mined for n-best lists.
 * The heavy lifting is done in the LatticePath and LatticePathCollection
 * this function controls this for one sentence.
 *
 * \param count the number of n-best translations to produce
 * \param ret holds the n-best list that was calculated
 */
void Manager::CalcNBest(size_t count, LatticePathList &ret,bool onlyDistinct) const
{
	if (count <= 0)
		return;

	vector<const Hypothesis*> sortedPureHypo = m_hypoStack.back().GetSortedList();

	if (sortedPureHypo.size() == 0)
		return;

	LatticePathCollection contenders;

	set<std::vector<size_t> > distinctHyps;

	// add all pure paths
	vector<const Hypothesis*>::const_iterator iterBestHypo;
	for (iterBestHypo = sortedPureHypo.begin()
			; iterBestHypo != sortedPureHypo.end()
			; ++iterBestHypo)
	{
		contenders.Add(new LatticePath(*iterBestHypo));
	}

	// MAIN loop
	for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) <= count && contenders.GetSize() > 0 && (iteration < count * 20) ; iteration++)
	{
		// get next best from list of contenders
		LatticePath *path = *contenders.begin();
		assert(path);
		bool addPath = true;
		if(onlyDistinct)
		{
			// TODO - not entirely correct.
			// output phrase can't be assumed to only contain factor 0.
			// have to look in StaticData.GetOutputFactorOrder() to find out what output factors should be
			std::vector<size_t> tgtPhrase;
			getSurfacePhrase(tgtPhrase,*path);
			addPath = distinctHyps.insert(tgtPhrase).second;
		}

		if(addPath && ret.Add(path))
		{	// create deviations from current best
			path->CreateDeviantPaths(contenders);
		}

		contenders.Detach(contenders.begin());
	}
}

void Manager::CalcDecoderStatistics(const StaticData& staticData) const
{
  const Hypothesis *hypo = GetBestHypothesis();
	if (hypo != NULL)
  {
  	staticData.GetSentenceStats().CalcFinalStats(*hypo);
    IFVERBOSE(2) {
		 	if (hypo != NULL) {
		   	string buff;
		  	string buff2;
		   	TRACE_ERR ("Source and Target Units:");
		 		TRACE_ERR ((hypo->GetSourcePhrase()).ToString());
				buff2.insert(0,"] ");
				buff2.insert(0,(hypo->GetTargetPhrase()).ToString());
				buff2.insert(0,":");
				buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
				buff2.insert(0,"[");

				hypo = hypo->GetPrevHypo();
				while (hypo != NULL) {
					//dont print out the empty final hypo
				  buff.insert(0,buff2);
				  buff2.clear();
				  buff2.insert(0,"] ");
				  buff2.insert(0,(hypo->GetTargetPhrase()).ToString());
				  buff2.insert(0,":");
				  buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
				  buff2.insert(0,"[");
				  hypo = hypo->GetPrevHypo();
				}
				TRACE_ERR (buff);
				TRACE_ERR ("\n");
      }
    }
  }
}
<commit_msg>fix nbest list behavior to generate n best translations, not n+1.<commit_after>// $Id$
// vim:tabstop=2

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/
#ifdef WIN32
#include <hash_set>
#else
#include <ext/hash_set>
#endif

#include <limits>
#include <cmath>
#include "Manager.h"
#include "TypeDef.h"
#include "Util.h"
#include "TargetPhrase.h"
#include "LatticePath.h"
#include "LatticePathCollection.h"
#include "TranslationOption.h"
#include "LMList.h"
#include "TranslationOptionCollection.h"

using namespace std;

Manager::Manager(InputType const& source, StaticData &staticData)
:m_source(source)
,m_hypoStack(source.GetSize() + 1)
,m_staticData(staticData)
,m_possibleTranslations(source.CreateTranslationOptionCollection())
,m_initialTargetPhrase(Output)
{
	std::vector < HypothesisCollection >::iterator iterStack;
	for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		HypothesisCollection &sourceHypoColl = *iterStack;
		sourceHypoColl.SetMaxHypoStackSize(m_staticData.GetMaxHypoStackSize());
		sourceHypoColl.SetBeamThreshold(m_staticData.GetBeamThreshold());
	}
}

Manager::~Manager()
{
  delete m_possibleTranslations;
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void Manager::ProcessSentence()
{
	m_staticData.ResetSentenceStats(m_source);
	list < DecodeStep* > &decodeStepList = m_staticData.GetDecodeStepList();
	// create list of all possible translations
	// this is only valid if:
	//		1. generation of source sentence is not done 1st
	//		2. initial hypothesis factors are given in the sentence
	//CreateTranslationOptions(m_source, phraseDictionary, lmListInitial);
	m_possibleTranslations->CreateTranslationOptions(decodeStepList
  														, m_staticData.GetFactorCollection());

	// initial seed hypothesis: nothing translated, no words produced
	{
		Hypothesis *hypo = Hypothesis::Create(m_source, m_initialTargetPhrase);
		m_hypoStack[0].AddPrune(hypo);
	}

	// go through each stack
	std::vector < HypothesisCollection >::iterator iterStack;
	for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		HypothesisCollection &sourceHypoColl = *iterStack;

		// the stack is pruned before processing (lazy pruning):
		VERBOSE(3,"processing hypothesis from next stack");
		sourceHypoColl.PruneToSize(m_staticData.GetMaxHypoStackSize());
		VERBOSE(3,std::endl);
		sourceHypoColl.InitializeArcs();

		// go through each hypothesis on the stack and try to expand it
		HypothesisCollection::const_iterator iterHypo;
		for (iterHypo = sourceHypoColl.begin() ; iterHypo != sourceHypoColl.end() ; ++iterHypo)
			{
				Hypothesis &hypothesis = **iterHypo;
				ProcessOneHypothesis(hypothesis); // expand the hypothesis
			}
		// some logging
		IFVERBOSE(2) { OutputHypoStackSize(); }
	}

	// some more logging
	VERBOSE(2,m_staticData.GetSentenceStats());
}

/** Find all translation options to expand one hypothesis, trigger expansion
 * this is mostly a check for overlap with already covered words, and for
 * violation of reordering limits.
 * \param hypothesis hypothesis to be expanded upon
 */
void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
{
	// since we check for reordering limits, its good to have that limit handy
	int maxDistortion = m_staticData.GetMaxDistortion();

	// no limit of reordering: only check for overlap
	if (maxDistortion < 0)
	{
		const WordsBitmap hypoBitmap	= hypothesis.GetWordsBitmap();
		const size_t hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
								, sourceSize			= m_source.GetSize();

		for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
		{
			for (size_t endPos = startPos ; endPos < sourceSize ; ++endPos)
			{
				if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
				{
					ExpandAllHypotheses(hypothesis
												, m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
				}
			}
		}

		return; // done with special case (no reordering limit)
	}

	// if there are reordering limits, make sure it is not violated
	// the coverage bitmap is handy here (and the position of the first gap)
	const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
	const size_t hypoWordCount		= hypoBitmap.GetNumWordsCovered()
							, hypoFirstGapPos	= hypoBitmap.GetFirstGapPos()
							, sourceSize			= m_source.GetSize();

	// MAIN LOOP. go through each possible hypo
	for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
	{
		for (size_t endPos = startPos ; endPos < sourceSize ; ++endPos)
		{
			// no gap so far => don't skip more than allowed limit
			if (hypoFirstGapPos == hypoWordCount)
				{
					if (startPos == hypoWordCount
							|| (startPos > hypoWordCount
									&& endPos <= hypoWordCount + maxDistortion)
					)
				{
					ExpandAllHypotheses(hypothesis
												,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
				}
			}
			// filling in gap => just check for overlap
			else if (startPos < hypoWordCount)
				{
					if (startPos >= hypoFirstGapPos
						&& !hypoBitmap.Overlap(WordsRange(startPos, endPos)))
					{
						ExpandAllHypotheses(hypothesis
													,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
					}
				}
			// ignoring, continuing forward => be limited by start of gap
			else
				{
					if (endPos <= hypoFirstGapPos + maxDistortion
						&& !hypoBitmap.Overlap(WordsRange(startPos, endPos)))
					{
						ExpandAllHypotheses(hypothesis
													,m_possibleTranslations->GetTranslationOptionList(WordsRange(startPos, endPos)));
					}
				}
		}
	}
}

/**
 * Expand a hypothesis given a list of translation options
 * \param hypothesis hypothesis to be expanded upon
 * \param transOptList list of translation options to be applied
 */

void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
{
	TranslationOptionList::const_iterator iter;
	for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
	{
		ExpandHypothesis(hypothesis, **iter);
	}
}

/**
 * Expand one hypothesis with a translation option.
 * this involves initial creation, scoring and adding it to the proper stack
 * \param hypothesis hypothesis to be expanded upon
 * \param transOpt translation option (phrase translation)
 *        that is applied to create the new hypothesis
 */
void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
{
	// create hypothesis and calculate all its scores
	Hypothesis *newHypo = hypothesis.CreateNext(transOpt);
	newHypo->CalcScore(m_staticData, m_possibleTranslations->GetFutureScore());

	// logging for the curious
	IFVERBOSE(3) {
	  newHypo->PrintHypothesis(m_source, m_staticData.GetWeightDistortion(), m_staticData.GetWeightWordPenalty());
	}

	// add to hypothesis stack
	size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
	m_hypoStack[wordsTranslated].AddPrune(newHypo);
}

/**
 * Find best hypothesis on the last stack.
 * This is the end point of the best translation, which can be traced back from here
 */
const Hypothesis *Manager::GetBestHypothesis() const
{
	const HypothesisCollection &hypoColl = m_hypoStack.back();
	return hypoColl.GetBestHypothesis();
}

/**
 * Logging of hypothesis stack sizes
 */
void Manager::OutputHypoStackSize()
{
	std::vector < HypothesisCollection >::const_iterator iterStack = m_hypoStack.begin();
	TRACE_ERR("Stack sizes: " << (int)iterStack->size());
	for (++iterStack; iterStack != m_hypoStack.end() ; ++iterStack)
	{
		TRACE_ERR ( ", " << (int)iterStack->size());
	}
	TRACE_ERR (endl);
}

/**
 * Logging of hypothesis stack contents
 * \param stack number of stack to be reported, report all stacks if 0
 */
void Manager::OutputHypoStack(int stack)
{
	if (stack >= 0)
	{
		TRACE_ERR ( "Stack " << stack << ": " << endl << m_hypoStack[stack] << endl);
	}
	else
	{ // all stacks
		int i = 0;
		vector < HypothesisCollection >::iterator iterStack;
		for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
		{
			HypothesisCollection &hypoColl = *iterStack;
			TRACE_ERR ( "Stack " << i++ << ": " << endl << hypoColl << endl);
		}
	}
}
void getSurfacePhrase(std::vector<size_t>& tphrase,LatticePath const& path)
{
	tphrase.clear();
	const std::vector<const Hypothesis *> &edges = path.GetEdges();
	for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
		{
			const Phrase &phrase = edges[currEdge]->GetTargetPhrase();
			for (size_t pos=0,size=phrase.GetSize() ; pos < size ; ++pos)
				{
					const Factor *factor = phrase.GetFactor(pos,0);
					assert(factor);
					tphrase.push_back(factor->GetId());
				}
		}
}

/**
 * After decoding, the hypotheses in the stacks and additional arcs
 * form a search graph that can be mined for n-best lists.
 * The heavy lifting is done in the LatticePath and LatticePathCollection
 * this function controls this for one sentence.
 *
 * \param count the number of n-best translations to produce
 * \param ret holds the n-best list that was calculated
 */
void Manager::CalcNBest(size_t count, LatticePathList &ret,bool onlyDistinct) const
{
	if (count <= 0)
		return;

	vector<const Hypothesis*> sortedPureHypo = m_hypoStack.back().GetSortedList();

	if (sortedPureHypo.size() == 0)
		return;

	LatticePathCollection contenders;

	set<std::vector<size_t> > distinctHyps;

	// add all pure paths
	vector<const Hypothesis*>::const_iterator iterBestHypo;
	for (iterBestHypo = sortedPureHypo.begin()
			; iterBestHypo != sortedPureHypo.end()
			; ++iterBestHypo)
	{
		contenders.Add(new LatticePath(*iterBestHypo));
	}

	// MAIN loop
	for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * 20) ; iteration++)
	{
		// get next best from list of contenders
		LatticePath *path = *contenders.begin();
		assert(path);
		bool addPath = true;
		if(onlyDistinct)
		{
			// TODO - not entirely correct.
			// output phrase can't be assumed to only contain factor 0.
			// have to look in StaticData.GetOutputFactorOrder() to find out what output factors should be
			std::vector<size_t> tgtPhrase;
			getSurfacePhrase(tgtPhrase,*path);
			addPath = distinctHyps.insert(tgtPhrase).second;
		}

		if(addPath && ret.Add(path))
		{	// create deviations from current best
			path->CreateDeviantPaths(contenders);
		}

		contenders.Detach(contenders.begin());
	}
}

void Manager::CalcDecoderStatistics(const StaticData& staticData) const
{
  const Hypothesis *hypo = GetBestHypothesis();
	if (hypo != NULL)
  {
  	staticData.GetSentenceStats().CalcFinalStats(*hypo);
    IFVERBOSE(2) {
		 	if (hypo != NULL) {
		   	string buff;
		  	string buff2;
		   	TRACE_ERR ("Source and Target Units:");
		 		TRACE_ERR ((hypo->GetSourcePhrase()).ToString());
				buff2.insert(0,"] ");
				buff2.insert(0,(hypo->GetTargetPhrase()).ToString());
				buff2.insert(0,":");
				buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
				buff2.insert(0,"[");

				hypo = hypo->GetPrevHypo();
				while (hypo != NULL) {
					//dont print out the empty final hypo
				  buff.insert(0,buff2);
				  buff2.clear();
				  buff2.insert(0,"] ");
				  buff2.insert(0,(hypo->GetTargetPhrase()).ToString());
				  buff2.insert(0,":");
				  buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
				  buff2.insert(0,"[");
				  hypo = hypo->GetPrevHypo();
				}
				TRACE_ERR (buff);
				TRACE_ERR ("\n");
      }
    }
  }
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013-2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "portal.h"
#include "sock_utils.h"
#include <string.h>
#include <poll.h>
#include <errno.h>
#include <pthread.h>
#include <fcntl.h>

#ifndef NO_CPP_PORTAL_CODE
PortalPoller *defaultPoller = new PortalPoller();
uint64_t poll_enter_time, poll_return_time; // for performance measurement

PortalPoller::PortalPoller()
  : portal_wrappers(0), portal_fds(0), startThread(1), numWrappers(0), numFds(0), stopping(0)
{
    int rc = pipe(pipefd);
    if (rc != 0)
      fprintf(stderr, "[%s:%d] pipe error %d:%s\n", __FUNCTION__, __LINE__, errno, strerror(errno));
    sem_init(&sem_startup, 0, 0);
    pthread_mutex_init(&mutex, NULL);
    fcntl(pipefd[0], F_SETFL, O_NONBLOCK);
    addFd(pipefd[0]);

    timeout = -1;
#if defined(BSIM) || defined(BOARD_xsim)
    timeout = 100;
#endif
}

int PortalPoller::unregisterInstance(Portal *portal)
{
    int i = 0;
    pthread_mutex_lock(&mutex);
    while(i < numWrappers){
        if(portal_wrappers[i]->pint.fpga_number == portal->pint.fpga_number) {
	    //fprintf(stderr, "PortalPoller::unregisterInstance %d %d\n", i, portal->pint.fpga_number);
            break;
        }
        i++;
    }

    while(i < numWrappers-1){
        portal_wrappers[i] = portal_wrappers[i+1];
        i++;
    }
    numWrappers--;
    i = 0;
    while(i < numFds){
        if(portal_fds[i].fd == portal->pint.fpga_fd)
            break;
        i++;
    }

    while(i < numFds-1){
        portal_fds[i] = portal_fds[i+1];
        i++;
    }
    numFds--;
    pthread_mutex_unlock(&mutex);
    return 0;
}

void PortalPoller::addFd(int fd)
{
    /* this internal function assumes mutex locked by caller.
     * since it can be called from addFdToPoller(), which was called under mutex lock
     * event().
     */
    numFds++;
    portal_fds = (struct pollfd *)realloc(portal_fds, numFds*sizeof(struct pollfd));
    struct pollfd *pollfd = &portal_fds[numFds-1];
    memset(pollfd, 0, sizeof(struct pollfd));
    pollfd->fd = fd;
    pollfd->events = POLLIN;
}

int PortalPoller::registerInstance(Portal *portal)
{
    uint8_t ch = 0;
    pthread_mutex_lock(&mutex);
    int rc = write(pipefd[1], &ch, 1); // get poll to return, so that it is no long using portal_fds (which gets realloc'ed)
    if (rc < 0)
        fprintf(stderr, "[%s:%d] write error %d\n", __FUNCTION__, __LINE__, errno);
    numWrappers++;
    //fprintf(stderr, "Portal::registerInstance fpga%d fd %d clients %d\n", portal->pint.fpga_number, portal->pint.fpga_fd, portal->pint.client_fd_number);
    portal_wrappers = (Portal **)realloc(portal_wrappers, numWrappers*sizeof(Portal *));
    portal_wrappers[numWrappers-1] = portal;

    if (portal->pint.fpga_fd != -1)
        addFd(portal->pint.fpga_fd);
    for (int i = 0; i < portal->pint.client_fd_number; i++)
        addFd(portal->pint.client_fd[i]);
    portal->pint.item->enableint(&portal->pint, 1);
    pthread_mutex_unlock(&mutex);
    start();
    return 0;
}

void* PortalPoller::init(void)
{
#ifdef BSIM
    if (global_sockfd != -1) {
        pthread_mutex_lock(&mutex);
        addFd(global_sockfd);
        pthread_mutex_unlock(&mutex);
    }
#endif
    //fprintf(stderr, "portalExec::about to enter loop, numFds=%d\n", numFds);
    return NULL;
}
void PortalPoller::stop(void)
{
    uint8_t ch = 0;
    int rc;
    stopping = 1;
    startThread = 0;
    rc = write(pipefd[1], &ch, 1);
    if (rc < 0)
        fprintf(stderr, "[%s:%d] write error %d\n", __FUNCTION__, __LINE__, errno);
}
void PortalPoller::end(void)
{
    stopping = 1;
    fprintf(stderr, "%s: don't disable interrupts when stopping\n", __FUNCTION__);
    return;
    pthread_mutex_lock(&mutex);
    for (int i = 0; i < numWrappers; i++) {
        Portal *instance = portal_wrappers[i];
        fprintf(stderr, "portalExec::disabling interrupts portal %d fpga%d\n", i, instance->pint.fpga_number);
        instance->pint.item->enableint(&instance->pint, 0);
    }
    pthread_mutex_unlock(&mutex);
}

void* PortalPoller::pollFn(int timeout)
{
    long rc = 0;
    //printf("[%s:%d] before poll %d numFds %d\n", __FUNCTION__, __LINE__, timeout, numFds);
    //for (int i = 0; i < numFds; i++)
        //printf("%s: fd %d events %x\n", __FUNCTION__, portal_fds[i].fd, portal_fds[i].events);
    if (timeout != 0)
        rc = poll(portal_fds, numFds, timeout);
    if(rc < 0) {
        // return only in error case
        fprintf(stderr, "poll returned rc=%ld errno=%d:%s\n", rc, errno, strerror(errno));
    }
    return (void*)rc;
}

void* PortalPoller::event(void)
{
    uint8_t ch;
    pthread_mutex_lock(&mutex);
    size_t rc = read(pipefd[0], &ch, 1);
    if (rc < 0)
        fprintf(stderr, "[%s:%d] read error %d\n", __FUNCTION__, __LINE__, errno);
    for (int i = 0; i < numWrappers; i++) {
        if (!portal_wrappers)
            fprintf(stderr, "No portal_instances revents=%d\n", portal_fds[i].revents);
        Portal *instance = portal_wrappers[i];
        instance->pint.item->event(&instance->pint);
        if (instance->pint.handler) {
            // re-enable interrupt which was disabled by portal_isr
            instance->pint.item->enableint(&instance->pint, 1);
        }
    }
    pthread_mutex_unlock(&mutex);
    return NULL;
}
extern "C" void addFdToPoller(struct PortalPoller *poller, int fd)
{
    poller->addFd(fd);
}

void* PortalPoller::threadFn(void* __x)
{
    void *rc = init();
    sem_post(&sem_startup);
    while (!rc && !stopping) {
        rc = pollFn(timeout);
        if ((long) rc >= 0)
            rc = event();
    }
    end();
    fprintf(stderr, "[%s] thread ending\n", __FUNCTION__);
    return rc;
}

static void *pthread_worker(void *__x)
{
    ((PortalPoller *)__x)->threadFn(__x);
    return 0;
}

void PortalPoller::start()
{
    pthread_t threaddata;
    pthread_mutex_lock(&mutex);
    if (!startThread) {
        pthread_mutex_unlock(&mutex);
        return;
    }
    startThread = 0;
    pthread_mutex_unlock(&mutex);
    pthread_create(&threaddata, NULL, &pthread_worker, (void *)this);
    sem_wait(&sem_startup);
}
#endif // NO_CPP_PORTAL_CODE
<commit_msg>tracing poller<commit_after>// Copyright (c) 2012 Nokia, Inc.
// Copyright (c) 2013-2014 Quanta Research Cambridge, Inc.

// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use, copy,
// modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "portal.h"
#include "sock_utils.h"
#include <string.h>
#include <poll.h>
#include <errno.h>
#include <pthread.h>
#include <fcntl.h>

static int trace_poller=;//1;

#ifndef NO_CPP_PORTAL_CODE
PortalPoller *defaultPoller = new PortalPoller();
uint64_t poll_enter_time, poll_return_time; // for performance measurement

PortalPoller::PortalPoller()
  : portal_wrappers(0), portal_fds(0), startThread(1), numWrappers(0), numFds(0), stopping(0)
{
    int rc = pipe(pipefd);
    if (rc != 0)
      fprintf(stderr, "[%s:%d] pipe error %d:%s\n", __FUNCTION__, __LINE__, errno, strerror(errno));
    sem_init(&sem_startup, 0, 0);
    pthread_mutex_init(&mutex, NULL);
    fcntl(pipefd[0], F_SETFL, O_NONBLOCK);
    addFd(pipefd[0]);

    timeout = -1;
#if defined(BSIM) || defined(BOARD_xsim)
    timeout = 100;
#endif
}

int PortalPoller::unregisterInstance(Portal *portal)
{
    int i = 0;
    pthread_mutex_lock(&mutex);
    while(i < numWrappers){
        if(portal_wrappers[i]->pint.fpga_number == portal->pint.fpga_number) {
	    //fprintf(stderr, "PortalPoller::unregisterInstance %d %d\n", i, portal->pint.fpga_number);
            break;
        }
        i++;
    }

    while(i < numWrappers-1){
        portal_wrappers[i] = portal_wrappers[i+1];
        i++;
    }
    numWrappers--;
    i = 0;
    while(i < numFds){
        if(portal_fds[i].fd == portal->pint.fpga_fd)
            break;
        i++;
    }

    while(i < numFds-1){
        portal_fds[i] = portal_fds[i+1];
        i++;
    }
    numFds--;
    pthread_mutex_unlock(&mutex);
    return 0;
}

void PortalPoller::addFd(int fd)
{
    /* this internal function assumes mutex locked by caller.
     * since it can be called from addFdToPoller(), which was called under mutex lock
     * event().
     */
    numFds++;
    portal_fds = (struct pollfd *)realloc(portal_fds, numFds*sizeof(struct pollfd));
    struct pollfd *pollfd = &portal_fds[numFds-1];
    memset(pollfd, 0, sizeof(struct pollfd));
    pollfd->fd = fd;
    pollfd->events = POLLIN;
}

int PortalPoller::registerInstance(Portal *portal)
{
    uint8_t ch = 0;
    pthread_mutex_lock(&mutex);
    int rc = write(pipefd[1], &ch, 1); // get poll to return, so that it is no long using portal_fds (which gets realloc'ed)
    if (rc < 0)
        fprintf(stderr, "[%s:%d] write error %d\n", __FUNCTION__, __LINE__, errno);
    numWrappers++;
    if (trace_poller)
        fprintf(stderr, "Poller: registerInstance fpga%d fd %d clients %d\n", portal->pint.fpga_number, portal->pint.fpga_fd, portal->pint.client_fd_number);
    portal_wrappers = (Portal **)realloc(portal_wrappers, numWrappers*sizeof(Portal *));
    portal_wrappers[numWrappers-1] = portal;

    if (portal->pint.fpga_fd != -1)
        addFd(portal->pint.fpga_fd);
    for (int i = 0; i < portal->pint.client_fd_number; i++)
        addFd(portal->pint.client_fd[i]);
    portal->pint.item->enableint(&portal->pint, 1);
    pthread_mutex_unlock(&mutex);
    start();
    return 0;
}

void* PortalPoller::init(void)
{
#ifdef BSIM
    if (global_sockfd != -1) {
        pthread_mutex_lock(&mutex);
        addFd(global_sockfd);
        pthread_mutex_unlock(&mutex);
    }
#endif
    //fprintf(stderr, "Poller: about to enter loop, numFds=%d\n", numFds);
    return NULL;
}
void PortalPoller::stop(void)
{
    uint8_t ch = 0;
    int rc;
    stopping = 1;
    startThread = 0;
    rc = write(pipefd[1], &ch, 1);
    if (rc < 0)
        fprintf(stderr, "[%s:%d] write error %d\n", __FUNCTION__, __LINE__, errno);
}
void PortalPoller::end(void)
{
    stopping = 1;
    fprintf(stderr, "%s: don't disable interrupts when stopping\n", __FUNCTION__);
    return;
    pthread_mutex_lock(&mutex);
    for (int i = 0; i < numWrappers; i++) {
        Portal *instance = portal_wrappers[i];
        fprintf(stderr, "Poller::disabling interrupts portal %d fpga%d\n", i, instance->pint.fpga_number);
        instance->pint.item->enableint(&instance->pint, 0);
    }
    pthread_mutex_unlock(&mutex);
}

void* PortalPoller::pollFn(int timeout)
{
    long rc = 0;
    //printf("[%s:%d] before poll %d numFds %d\n", __FUNCTION__, __LINE__, timeout, numFds);
    //for (int i = 0; i < numFds; i++)
        //printf("%s: fd %d events %x\n", __FUNCTION__, portal_fds[i].fd, portal_fds[i].events);
    if (timeout != 0)
        rc = poll(portal_fds, numFds, timeout);
    if(rc < 0) {
        // return only in error case
        fprintf(stderr, "Poller: poll returned rc=%ld errno=%d:%s\n", rc, errno, strerror(errno));
    }
    return (void*)rc;
}

void* PortalPoller::event(void)
{
    uint8_t ch;
    pthread_mutex_lock(&mutex);
    size_t rc = read(pipefd[0], &ch, 1);
    if (rc < 0)
        fprintf(stderr, "[%s:%d] read error %d\n", __FUNCTION__, __LINE__, errno);
    for (int i = 0; i < numWrappers; i++) {
        if (!portal_wrappers)
            fprintf(stderr, "Poller: No portal_instances revents=%d\n", portal_fds[i].revents);
        Portal *instance = portal_wrappers[i];
        if (trace_poller)
            fprintf(stderr, "Poller: event tile %d fpga%d fd %d handler %p parent %p\n",
                instance->pint.fpga_tile, instance->pint.fpga_number, instance->pint.fpga_fd, instance->pint.handler, instance->pint.parent);
        instance->pint.item->event(&instance->pint);
        if (instance->pint.handler) {
            // re-enable interrupt which was disabled by portal_isr
            instance->pint.item->enableint(&instance->pint, 1);
        }
    }
    pthread_mutex_unlock(&mutex);
    return NULL;
}
extern "C" void addFdToPoller(struct PortalPoller *poller, int fd)
{
    poller->addFd(fd);
}

void* PortalPoller::threadFn(void* __x)
{
    void *rc = init();
    sem_post(&sem_startup);
    while (!rc && !stopping) {
        rc = pollFn(timeout);
        if ((long) rc >= 0)
            rc = event();
    }
    end();
    fprintf(stderr, "[%s] thread ending\n", __FUNCTION__);
    return rc;
}

static void *pthread_worker(void *__x)
{
    ((PortalPoller *)__x)->threadFn(__x);
    return 0;
}

void PortalPoller::start()
{
    pthread_t threaddata;
    pthread_mutex_lock(&mutex);
    if (!startThread) {
        pthread_mutex_unlock(&mutex);
        return;
    }
    startThread = 0;
    pthread_mutex_unlock(&mutex);
    pthread_create(&threaddata, NULL, &pthread_worker, (void *)this);
    sem_wait(&sem_startup);
}
#endif // NO_CPP_PORTAL_CODE
<|endoftext|>


<commit_before>// $Id$
// -*-C++-*-
// * BeginRiceCopyright *****************************************************
//
// Copyright ((c)) 2002, Rice University
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// * Neither the name of Rice University (RICE) nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// This software is provided by RICE and contributors "as is" and any
// express or implied warranties, including, but not limited to, the
// implied warranties of merchantability and fitness for a particular
// purpose are disclaimed. In no event shall RICE or contributors be
// liable for any direct, indirect, incidental, special, exemplary, or
// consequential damages (including, but not limited to, procurement of
// substitute goods or services; loss of use, data, or profits; or
// business interruption) however caused and on any theory of liability,
// whether in contract, strict liability, or tort (including negligence
// or otherwise) arising in any way out of the use of this software, even
// if advised of the possibility of such damage.
//
// ******************************************************* EndRiceCopyright *

//***************************************************************************
//
// File:
//    ProfileReader.C
//
// Purpose:
//    [The purpose of this file]
//
// Description:
//    [The set of functions, macros, etc. defined in the file]
//
//***************************************************************************

//************************* System Include Files ****************************

#include <iostream>
#include <fstream>
#include <sstream>
#include <list>
#include <vector>
#include <algorithm>

//*************************** User Include Files ****************************

#include "ProfileReader.h"
#include "PCProfile.h"
#include "DCPIProfile.h"

#include <lib/ISA/ISA.h>
#include <lib/ISA/AlphaISA.h>
#include <lib/xml/xml.h>

//*************************** Forward Declarations ***************************

using namespace xml;

using std::cerr;
using std::endl;
using std::hex;
using std::dec;

struct lt_PCProfileMetric
{
  // return true if m1 < m2; false otherwise
  bool operator()(const PCProfileMetric* m1, const PCProfileMetric* m2) const
  { return (strcmp(m1->GetName(), m2->GetName()) < 0); }
};

static bool
MetricNamesAreUnique(PCProfileMetricSet& mset);

static DCPIProfile::PMMode
DeterminePMMode(const char* pmcounter);

//****************************************************************************
//  public: ReadProfileFile:
//****************************************************************************

// Note: 'ReadProfileFile' guarantees that every 'PCProfileMetric' in the
// 'PCProfile' will have identical return values for both
// 'GetTxtStart' and 'GetTxtSz'.

PCProfile*
ProfileReader::ReadProfileFile(const char* profFile /* FIXME: type */)
{

  // Text or binary!!

  std::ifstream pFile(profFile);
  if ( !pFile.is_open() || pFile.fail()) {
    cerr << "Error opening file `" << profFile << "'" << endl;
    return NULL;
  }

  PCProfile* profData = NULL;

  // depending on file type, choose the correct reader
  profData = dynamic_cast<PCProfile*>(ReadProfileFile_DCPICat(pFile));

  if (!profData) {
    cerr << "Error reading file `" << profFile << "'" << endl;
  }

  pFile.close();
  return profData;
}

//****************************************************************************
//  private: DEC/Alpha/OSF1
//****************************************************************************

// 'ReadProfileFile_DCPICat' reads 'dcpicat' output.
DCPIProfile*
ProfileReader::ReadProfileFile_DCPICat(std::istream& pFile)
{
  String str;
  const int bufSz = 128;
  char buf[bufSz]; // holds single tokens guaranteed not to be too big

  ISA* isa = new AlphaISA(); // we are going to reference count this
  DCPIProfile* profData = NULL;
  std::stringstream hdr;   // header info
  String profiledFile;     // name of profiled file
  String profiledFilePath; // path to profiled file
  Addr txtStart = 0;       // starting address of text segment
  Addr txtSz    = 0;       // byte-size of text section

  // ------------------------------------------------------------------------
  // Read header
  // ------------------------------------------------------------------------

  // 'name' - name of profiled binary
  str = GetLine(pFile); hdr << (const char*)str << "\n";
  pFile.seekg(-(int)str.Length()-1, std::ios::cur); // reposition to reread
  pFile >> buf >> buf >> std::ws;              // read 'name' token and value
  profiledFile = buf;

  // 'image' - date and id information for profiled binary
  str = GetLine(pFile); hdr << (const char*)str << "\n";

  // 'label' (optional) - label for this profile file
  if (pFile.peek() == 'l') {
    str = GetLine(pFile); hdr << (const char*)str << "\n";
  }

  // 'path' - path for the profiled binary
  //  note: while the 'dcpicat' man-page does not indicate this,
  //   multiple 'path' entries are sometimes given.
  while ( (!pFile.eof() && !pFile.fail()) && pFile.peek() == 'p' ) {
    str = GetLine(pFile); hdr << (const char*)str << "\n";
    if (profiledFilePath.Empty()) {
      pFile.seekg(-(int)str.Length()-1, std::ios::cur); // reposition to reread
      pFile >> buf >> buf >> std::ws;            // read 'path' token and value
      profiledFilePath = buf;
    }
  }

  // 'epoch' - date the profile was made
  str = GetLine(pFile); hdr << (const char*)str << "\n";

  // 'platform' - name of machine on which the profile was made
  str = GetLine(pFile); hdr << (const char*)str << "\n";

  // 'text_start' - starting address of text section of profiled binary (hex)
  str = GetLine(pFile); hdr << (const char*)str << "\n";
  pFile.seekg(-(int)str.Length()-1, std::ios::cur); // reposition to reread
  pFile >> buf >> std::ws; Skip(pFile, "0x");  // read 'text_start' token...
  pFile >> hex >> txtStart >> dec >> std::ws;  // read 'text_start' value

  // 'text_size' - byte-size of the text section (hex)
  str = GetLine(pFile); hdr << (const char*)str << "\n";
  pFile.seekg(-(int)str.Length()-1, std::ios::cur); // reposition to reread
  pFile >> buf >> std::ws; Skip(pFile, "0x");  // read 'text_size' token...
  pFile >> hex >> txtSz >> dec >> std::ws;     // read 'text_size' value

  if (pFile.eof() || pFile.fail()) { return NULL; /* error */ }

  // 'event' - one line for each event type in the profile
  //   Format: 'event W x:X:Y:Z'
  //   (The following adapted from dcpicat man page.)
  //   'W' - sum of this event's count for all pc values recorded in profile
  //   'x' - (only ProfileMe) profileme sample set (attriute and trap bits)
  //   'X' - event name
  //         (or for ProfileMe) the counter name appended to the sample set
  //   'Y' - sampling period used to sample this event (one such event
  //         occurrance is recorded every Y occurrances of the event.)
  //   'Z' - 10000 times the fraction of the time the event was being
  //         monitored.  (When multiple events are being multiplexed onto
  //         the same hardware counter, "Z" will be less than 10000.
  //
  //  see 'dcpiprofileme' and 'dcpicat' man pages.

  // Create 'PCProfileMetric' from 'event' and collect them all into a queue
  PCProfileMetricList mlist;
  DCPIProfileMetric* curMetric;
  DCPIProfile::PMMode pmmode = DCPIProfile::PM_NONE;

  String X;
  PCProfileDatum W, Y, Z;
  bool invalidMetric = false;
  unsigned int metricCount = 0;

  while ( (!pFile.eof() && !pFile.fail()) && pFile.peek() == 'e' ) {

    pFile >> buf;  // read 'event' token (above: line begins with 'e')
    pFile >> W >> std::ws;          // read 'W' (total count)

    X = GetLine(pFile, ':');        // read 'x' or 'X' (event name)
    if ( !isdigit(pFile.peek()) ) { // read 'X' (for ProfileMe)
      String pmcounter = GetLine(pFile, ':');
      pmmode = DeterminePMMode(pmcounter);
      if (pmmode == DCPIProfile::PM_NONE) { invalidMetric = true; }
      X += ":" + pmcounter;
    }

    pFile >> Y; Skip(pFile, ":");   // read 'Y' (sampling period) and ':'
    pFile >> Z >> std::ws;          // read 'Z' (sampling time)

    curMetric = new DCPIProfileMetric(isa, X);
    isa->Detach(); // Remove our reference
    curMetric->SetTxtStart(txtStart);
    curMetric->SetTxtSz(txtSz);
    curMetric->SetTotalCount(W);
    curMetric->SetName(X);
    curMetric->SetPeriod(Y);
    if (! curMetric->GetDCPIDesc().IsValid()) {
      invalidMetric = true;
    }

    mlist.push_back(curMetric);
    metricCount++;
  }

  // Create 'PCProfile' object and add all events
  profData = new DCPIProfile(isa, metricCount);
  profData->SetHdrInfo( (hdr.str()).c_str() );
  if (!profiledFilePath.Empty()) {
    profData->SetProfiledFile(profiledFilePath);
  } else {
    profData->SetProfiledFile(profiledFile);
  }
  profData->SetPMMode(pmmode);

  while (!mlist.empty()) {
    PCProfileMetric* m = mlist.front();
    mlist.pop_front();
    profData->AddMetric(m);
  }

  if (invalidMetric || pFile.eof() || pFile.fail()) {
    goto DCPICat_CleanupAfterError;
  }

  // Check for duplicate 'event' names.  'event' names should be
  // unique, but we have seen at least one instance where it seems
  // duplicates were generated...
  MetricNamesAreUnique(*profData); // Print warning if necessary

  // ------------------------------------------------------------------------
  // Read sampling information
  // ------------------------------------------------------------------------

  // There are 1 PC-column + n count-columns where n is the 'event'
  // number; the columns are in the same order as the 'event' listing.
  // PC's are in increasing address order.  Skipped PC's have '0' for
  // all counts.  Because we are dealing with a non-VLIW architecture,
  // the 'opIndex' argument to 'AddPC(...)' and 'Insert(...)' is always 0.

  // Note: add a dummy block to prevent compiler warnings/errors
  // about the above 'goto' crossing the initialization of 'pcCount'
  {
    unsigned long pcCount = 0; // number of PC entries
    PCProfileDatum profileDatum;
    Addr PC;
    pFile >> std::ws;
    while ( (!pFile.eof() && !pFile.fail()) ) {
      Skip(pFile, "0x");         // eat up '0x' prefix
      pFile >> hex >> PC >> dec; // read 'PC' value, non VLIW instruction

      profData->AddPC(PC, 0); // there is some non-zero data at PC
      for (unsigned long i = 0; i < metricCount; i++) {
	pFile >> profileDatum;
	const PCProfileMetric* m = profData->GetMetric(i);
	const_cast<PCProfileMetric*>(m)->Insert(PC, 0, profileDatum);
      }

      pFile >> std::ws;
      pcCount++;
    }
  }

  if (pFile.fail()) { goto DCPICat_CleanupAfterError; }

  // ------------------------------------------------------------------------
  // Done!
  // ------------------------------------------------------------------------
  return profData;

 DCPICat_CleanupAfterError:
  delete profData;
  return NULL;
}

// Check to see if there are duplicates in 'mlist' while preserving order.
static bool
MetricNamesAreUnique(PCProfileMetricSet& mset)
{
  bool STATUS = true;

  if (mset.GetSz() > 1) {
    // Collect all metrics (pointers) into a temporary vector and sort it
    PCProfileMetricVec mvec(mset.GetSz());

    PCProfileMetricSetIterator setIt(mset);
    for (suint i = 0; setIt.IsValid(); ++setIt, ++i) {
      mvec[i] = setIt.Current();
    }

    std::sort(mvec.begin(), mvec.end(), lt_PCProfileMetric()); // ascending

    // Check for duplicates.  'mvec.size' is guaranteed to be >= 2
    PCProfileMetric* cur1, *cur2 = NULL;
    PCProfileMetricVecIt it = mvec.begin();
    do {
      cur1 = *it;
      cur2 = *(++it); // increment
      if (strcmp(cur1->GetName(), cur2->GetName()) == 0) {
	STATUS = false;
	cerr << "Warning: Duplicate metric names exist: '"
	     << cur1->GetName() << "'." << endl;
      }
    } while (it+1 != mvec.end());

    // Remove all metric pointers from 'mvec' to ensure they are not deleted
    for (it = mvec.begin(); it != mvec.end(); ++it) { *it = NULL; }
  }

  return STATUS;
}

// Given a ProfileMe counter name, determine which ProfileMe mode was used
static DCPIProfile::PMMode
DeterminePMMode(const char* pmcounter)
{
  if (strncmp("m0", pmcounter, 2) == 0) {
    return DCPIProfile::PM0;
  } else if (strncmp("m1", pmcounter, 2) == 0) {
    return DCPIProfile::PM1;
  } else if (strncmp("m2", pmcounter, 2) == 0) {
    return DCPIProfile::PM2;
  } else if (strncmp("m3", pmcounter, 2) == 0) {
    return DCPIProfile::PM3;
  }
  return DCPIProfile::PM_NONE;
}

<commit_msg>Rewrite dcpicat reader to enable reading from pipes. (Can't do any stream rewinding!).<commit_after>// $Id$
// -*-C++-*-
// * BeginRiceCopyright *****************************************************
//
// Copyright ((c)) 2002, Rice University
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// * Neither the name of Rice University (RICE) nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// This software is provided by RICE and contributors "as is" and any
// express or implied warranties, including, but not limited to, the
// implied warranties of merchantability and fitness for a particular
// purpose are disclaimed. In no event shall RICE or contributors be
// liable for any direct, indirect, incidental, special, exemplary, or
// consequential damages (including, but not limited to, procurement of
// substitute goods or services; loss of use, data, or profits; or
// business interruption) however caused and on any theory of liability,
// whether in contract, strict liability, or tort (including negligence
// or otherwise) arising in any way out of the use of this software, even
// if advised of the possibility of such damage.
//
// ******************************************************* EndRiceCopyright *

//***************************************************************************
//
// File:
//    ProfileReader.C
//
// Purpose:
//    [The purpose of this file]
//
// Description:
//    [The set of functions, macros, etc. defined in the file]
//
//***************************************************************************

//************************* System Include Files ****************************

#include <iostream>
#include <fstream>
#include <sstream>
#include <list>
#include <vector>
#include <algorithm>

#ifdef NO_STD_CHEADERS
# include <stdlib.h>
#else
# include <cstdlib>
using std::strtoul; // For compatibility with non-std C headers
#endif

//*************************** User Include Files ****************************

#include "ProfileReader.h"
#include "PCProfile.h"
#include "DCPIProfile.h"

#include <lib/ISA/ISA.h>
#include <lib/ISA/AlphaISA.h>
#include <lib/xml/xml.h>

//*************************** Forward Declarations ***************************

using namespace xml;

using std::cerr;
using std::endl;
using std::hex;
using std::dec;

struct lt_PCProfileMetric
{
  // return true if m1 < m2; false otherwise
  bool operator()(const PCProfileMetric* m1, const PCProfileMetric* m2) const
  { return (strcmp(m1->GetName(), m2->GetName()) < 0); }
};

static bool
MetricNamesAreUnique(PCProfileMetricSet& mset);

static DCPIProfile::PMMode
DeterminePMMode(const char* pmcounter);

static const char*
GetSecondSubstring(const char* str);

//****************************************************************************
//  public: ReadProfileFile:
//****************************************************************************

// ReadProfileFile: Given the name of a profile file 'profFile',
// creates a 'PCProfile' from the raw profile data.  If 'profFile' is
// NULL or the empty string, reads from cin.
//
// Note: Guarantees that every 'PCProfileMetric' in the 'PCProfile'
// will have identical return values for both 'GetTxtStart' and
// 'GetTxtSz'.
PCProfile*
ProfileReader::ReadProfileFile(const char* profFile /* FIXME: type */)
{
  PCProfile* profData = NULL;
  std::istream* is = NULL;
  bool usingStdin = (!profFile || profFile[0] == '\0');

  // ------------------------------------------------------------
  // Open an input stream and determine its type
  // ------------------------------------------------------------
  if (!usingStdin) {

    // Attempt to open as a file
    std::ifstream* pFile = new std::ifstream(profFile);
    is = pFile;
    if ( !pFile->is_open() || pFile->fail()) {
      cerr << "Error opening file `" << profFile << "'" << endl;
      goto ReadProfileFile_CleanupAfterError;
    }

  } else {
    // Read from cin
    is = &(std::cin);
  }

  // Figure out type (text, binary?)

  // ------------------------------------------------------------
  // Read the data
  // ------------------------------------------------------------

  // depending on file type, choose the correct reader
  profData = dynamic_cast<PCProfile*>(ReadProfileFile_DCPICat(*is));

  if (!profData) {
    if (!usingStdin) {
      cerr << "Error reading file `" << profFile << "'" << endl;
    } else {
      cerr << "Error reading from stdin" << endl;
    }
  }

  // ------------------------------------------------------------
  // Cleanup
  // ------------------------------------------------------------
 ReadProfileFile_CleanupAfterError:
  if (!usingStdin) {
    delete is;
  }

  return profData;
}

//****************************************************************************
//  private: DEC/Alpha/OSF1
//****************************************************************************

// 'ReadProfileFile_DCPICat' reads 'dcpicat' output.
DCPIProfile*
ProfileReader::ReadProfileFile_DCPICat(std::istream& is)
{
  String str;
  const char* str1;
  const int bufSz = 128;
  char buf[bufSz]; // holds single tokens guaranteed not to be too big

  ISA* isa = new AlphaISA(); // we are going to reference count this
  DCPIProfile* profData = NULL;
  std::stringstream hdr;   // header info
  String profiledFile;     // name of profiled file
  String profiledFilePath; // path to profiled file
  Addr txtStart = 0;       // starting address of text segment
  Addr txtSz    = 0;       // byte-size of text section

  // ------------------------------------------------------------
  // Read header
  // ------------------------------------------------------------

  // 'name' - name of profiled binary
  str = GetLine(is); hdr << (const char*)str << "\n";
  profiledFile = GetSecondSubstring(str);  // value of 'name'

  // 'image' - date and id information for profiled binary
  str = GetLine(is); hdr << (const char*)str << "\n";

  // 'label' (optional) - label for this profile file
  if (is.peek() == 'l') {
    str = GetLine(is); hdr << (const char*)str << "\n";
  }

  // 'path' - path for the profiled binary
  //  note: while the 'dcpicat' man-page does not indicate this,
  //  multiple 'path' entries are sometimes given.
  while ( (!is.eof() && !is.fail()) && is.peek() == 'p' ) {
    str = GetLine(is); hdr << (const char*)str << "\n";
    if (profiledFilePath.Empty()) {
      profiledFilePath = GetSecondSubstring(str);  // value of 'path'
    }
  }

  // 'epoch' - date the profile was made
  str = GetLine(is); hdr << (const char*)str << "\n";

  // 'platform' - name of machine on which the profile was made
  str = GetLine(is); hdr << (const char*)str << "\n";

  // 'text_start' - starting address of text section of profiled binary (hex)
  str = GetLine(is); hdr << (const char*)str << "\n";
  str1 = GetSecondSubstring(str);  // value of 'text_start'
  txtStart = strtoul(str1, (char **)NULL, 16);

  // 'text_size' - byte-size of the text section (hex)
  str = GetLine(is); hdr << (const char*)str << "\n";
  str1 = GetSecondSubstring(str);  // value of 'text_size'
  txtSz = strtoul(str1, (char **)NULL, 16);

  if (is.eof() || is.fail()) { return NULL; /* error */ }

  // 'event' - one line for each event type in the profile
  //   Format: 'event W x:X:Y:Z'
  //   (The following adapted from dcpicat man page.)
  //   'W' - sum of this event's count for all pc values recorded in profile
  //   'x' - (only ProfileMe) profileme sample set (attriute and trap bits)
  //   'X' - event name
  //         (or for ProfileMe) the counter name appended to the sample set
  //   'Y' - sampling period used to sample this event (one such event
  //         occurrance is recorded every Y occurrances of the event.)
  //   'Z' - 10000 times the fraction of the time the event was being
  //         monitored.  (When multiple events are being multiplexed onto
  //         the same hardware counter, "Z" will be less than 10000.
  //
  //  see 'dcpiprofileme' and 'dcpicat' man pages.

  // Create 'PCProfileMetric' from 'event' and collect them all into a queue
  PCProfileMetricList mlist;
  DCPIProfileMetric* curMetric;
  DCPIProfile::PMMode pmmode = DCPIProfile::PM_NONE;

  String X;
  PCProfileDatum W, Y, Z;
  bool invalidMetric = false;
  unsigned int metricCount = 0;

  while ( (!is.eof() && !is.fail()) && is.peek() == 'e' ) {

    is >> buf;  // read 'event' token (above: line begins with 'e')
    is >> W >> std::ws;          // read 'W' (total count)

    X = GetLine(is, ':');        // read 'x' or 'X' (event name)
    if ( !isdigit(is.peek()) ) { // read 'X' (for ProfileMe)
      String pmcounter = GetLine(is, ':');
      pmmode = DeterminePMMode(pmcounter);
      if (pmmode == DCPIProfile::PM_NONE) { invalidMetric = true; }
      X += ":" + pmcounter;
    }

    is >> Y; Skip(is, ":");   // read 'Y' (sampling period) and ':'
    is >> Z >> std::ws;          // read 'Z' (sampling time)

    curMetric = new DCPIProfileMetric(isa, X);
    curMetric->SetTxtStart(txtStart);
    curMetric->SetTxtSz(txtSz);
    curMetric->SetTotalCount(W);
    curMetric->SetName(X);
    curMetric->SetPeriod(Y);
    if (! curMetric->GetDCPIDesc().IsValid()) {
      invalidMetric = true;
    }

    mlist.push_back(curMetric);
    metricCount++;
  }

  // Create 'PCProfile' object and add all events
  profData = new DCPIProfile(isa, metricCount);
  isa->Detach(); // Remove our reference
  profData->SetHdrInfo( (hdr.str()).c_str() );
  if (!profiledFilePath.Empty()) {
    profData->SetProfiledFile(profiledFilePath);
  } else {
    profData->SetProfiledFile(profiledFile);
  }
  profData->SetPMMode(pmmode);

  while (!mlist.empty()) {
    PCProfileMetric* m = mlist.front();
    mlist.pop_front();
    profData->AddMetric(m);
  }

  if (invalidMetric || is.eof() || is.fail()) {
    goto DCPICat_CleanupAfterError;
  }

  // Check for duplicate 'event' names.  'event' names should be
  // unique, but we have seen at least one instance where it seems
  // duplicates were generated...
  MetricNamesAreUnique(*profData); // Print warning if necessary

  // ------------------------------------------------------------
  // Read sampling information
  // ------------------------------------------------------------

  // There are 1 PC-column + n count-columns where n is the 'event'
  // number; the columns are in the same order as the 'event' listing.
  // PC's are in increasing address order.  Skipped PC's have '0' for
  // all counts.  Because we are dealing with a non-VLIW architecture,
  // the 'opIndex' argument to 'AddPC(...)' and 'Insert(...)' is always 0.

  // Note: add a dummy block to prevent compiler warnings/errors
  // about the above 'goto' crossing the initialization of 'pcCount'
  {
    unsigned long pcCount = 0; // number of PC entries
    PCProfileDatum profileDatum;
    Addr PC;
    is >> std::ws;
    while ( (!is.eof() && !is.fail()) ) {
      Skip(is, "0x");         // eat up '0x' prefix
      is >> hex >> PC >> dec; // read 'PC' value, non VLIW instruction

      profData->AddPC(PC, 0); // there is some non-zero data at PC
      for (unsigned long i = 0; i < metricCount; i++) {
	is >> profileDatum;
	const PCProfileMetric* m = profData->GetMetric(i);
	const_cast<PCProfileMetric*>(m)->Insert(PC, 0, profileDatum);
      }

      is >> std::ws;
      pcCount++;
    }
  }

  if (is.fail()) { goto DCPICat_CleanupAfterError; }

  // ------------------------------------------------------------
  // Done!
  // ------------------------------------------------------------
  return profData;

 DCPICat_CleanupAfterError:
  delete profData;
  return NULL;
}

// Check to see if there are duplicates in 'mlist' while preserving order.
static bool
MetricNamesAreUnique(PCProfileMetricSet& mset)
{
  bool STATUS = true;

  if (mset.GetSz() > 1) {
    // Collect all metrics (pointers) into a temporary vector and sort it
    PCProfileMetricVec mvec(mset.GetSz());

    PCProfileMetricSetIterator setIt(mset);
    for (suint i = 0; setIt.IsValid(); ++setIt, ++i) {
      mvec[i] = setIt.Current();
    }

    std::sort(mvec.begin(), mvec.end(), lt_PCProfileMetric()); // ascending

    // Check for duplicates.  'mvec.size' is guaranteed to be >= 2
    PCProfileMetric* cur1, *cur2 = NULL;
    PCProfileMetricVecIt it = mvec.begin();
    do {
      cur1 = *it;
      cur2 = *(++it); // increment
      if (strcmp(cur1->GetName(), cur2->GetName()) == 0) {
	STATUS = false;
	cerr << "Warning: Duplicate metric names exist: '"
	     << cur1->GetName() << "'." << endl;
      }
    } while (it+1 != mvec.end());

    // Remove all metric pointers from 'mvec' to ensure they are not deleted
    for (it = mvec.begin(); it != mvec.end(); ++it) { *it = NULL; }
  }

  return STATUS;
}

// Given a ProfileMe counter name, determine which ProfileMe mode was used
static DCPIProfile::PMMode
DeterminePMMode(const char* pmcounter)
{
  if (strncmp("m0", pmcounter, 2) == 0) {
    return DCPIProfile::PM0;
  } else if (strncmp("m1", pmcounter, 2) == 0) {
    return DCPIProfile::PM1;
  } else if (strncmp("m2", pmcounter, 2) == 0) {
    return DCPIProfile::PM2;
  } else if (strncmp("m3", pmcounter, 2) == 0) {
    return DCPIProfile::PM3;
  }
  return DCPIProfile::PM_NONE;
}

// GetSecondString: Given a string composed of substrings separated by
// whitespace, return a pointer to the beginning of the second
// substring.  Ignores any whitespace at the beginning of the first
// substring. E.g.
//   name                 hydro
//   text_start           0x000120000000
static const char*
GetSecondSubstring(const char* str)
{
  if (!str) { return NULL; }

  const char* ptr = str;

  // ignore first bit of whitespace
  while (*ptr != '/0' && isspace(*ptr)) { ptr++; }

  // ignore first substring
  while (*ptr != '/0' && !isspace(*ptr)) { ptr++; }

  // ignore second bit of whitespace
  while (*ptr != '/0' && isspace(*ptr)) { ptr++; }

  return ptr; // beginning of second substring
}

<|endoftext|>


<commit_before>#include <algorithm>
#include <cmath>
#include <ctime>
#include <cstdlib>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include <openssl/sha.h>

using namespace std;

// Version
const char SPLASH_VERSION = '0';

// Debug mode
const bool DEBUG = false;

// Precision of floating point operations and representations
const double MZ_PRECISION = 6;
const long long MZ_PRECISION_FACTOR = static_cast<long long>(pow(10, MZ_PRECISION));

const double INTENSITY_PRECISION = 0;
const long long INTENSITY_PRECISION_FACTOR = static_cast<long long>(pow(10, INTENSITY_PRECISION));

const double EPS = 1.0e-6;

// Value to scale relative spectra
const double RELATIVE_INTENSITY_SCALE = 100.0;

// Separator for building spectrum strings
const char ION_SEPARATOR = ' ';

// Full spectrum hash properties
const char ION_PAIR_SEPARATOR = ':';
const int MAX_HASH_CHARATERS_ENCODED_SPECTRUM = 20;
const double EPS_CORRECTION = 1.0e-7;

// Prefilter properties
const int PREFILTER_BASE = 3;
const int PREFILTER_LENGTH = 10;
const int PREFILTER_BIN_SIZE = 5;

// Similarity histogram properties
const int SIMILARITY_BASE = 10;
const int SIMILARITY_LENGTH = 10;
const int SIMILARITY_BIN_SIZE = 100;

// Map to convert up to base 36
const char BASE_36_MAP[] = {
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c',
    'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
    'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'
};

string sha256(const string s) {
	unsigned char digest[SHA256_DIGEST_LENGTH];

	SHA256_CTX sha256;
	SHA256_Init(&sha256);
	SHA256_Update(&sha256, s.c_str(), s.size());
	SHA256_Final(digest, &sha256);

	stringstream ss;

	for(int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
		ss << hex << setw(2) << setfill('0') << (int)digest[i];
	}

	return ss.str();
}

// http://stackoverflow.com/a/236803
vector<string> split(const string &s, char delimeter) {
	vector<string> elements;
	stringstream ss(s);
	string element;

	while(getline(ss, element, delimeter)) {
		elements.push_back(element);
	}

	return elements;
}

bool ionPairMzComparator(const pair<double, double> a, const pair<double, double> b) {
	if(abs(a.first - b.first) < EPS) {
		return a.second > b.second;
	} else {
		return a.first < b.first;
	}
}

bool ionPairIntensityComparator(const pair<double, double> a, const pair<double, double> b) {
	if(abs(a.second - b.second) < EPS) {
		return a.first < b.first;
	} else {
		return a.second > b.second;
	}
}

string buildInitialBlock(vector<pair<double, double> > &spectrum, char spectrum_type) {
	stringstream ss;
	ss << "splash" << spectrum_type << SPLASH_VERSION;
	return ss.str();
}

string encodeSpectrum(vector<pair<double, double> > &spectrum, char spectrum_type) {
	sort(spectrum.begin(), spectrum.end(), ionPairMzComparator);

	int i = 0;
	stringstream ss;

	for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
		ss << static_cast<long long>(((*it).first + EPS_CORRECTION) * MZ_PRECISION_FACTOR)
		   << ION_PAIR_SEPARATOR
		   << static_cast<long long>(((*it).second + EPS_CORRECTION) * INTENSITY_PRECISION_FACTOR);

		if(++i < spectrum.size()) {
			ss << ION_SEPARATOR;
		}
	}

	if(DEBUG) {
		cerr << "Encoded Spectrum: '" << ss.str() << "'" << endl;
	}

	return sha256(ss.str()).substr(0, MAX_HASH_CHARATERS_ENCODED_SPECTRUM);
}

string calculateHistogram(vector<pair<double, double> > &spectrum, char spectrum_type, int base, int length, int bin_size) {
	double* histogram = new double[length]();
	double maxIntensity = 0;

	// Bin ions using the histogram wrapping strategy
	for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
		int idx = static_cast<int>((*it).first / bin_size) % length;
		histogram[idx] += (*it).second;

		if(histogram[idx] > maxIntensity)
			maxIntensity = histogram[idx];
	}

    // Normalize the histogram and scale to the provided base
    for (int i = 0; i < length; i++) {
        histogram[i] = (base - 1) * histogram[i] / maxIntensity;
    }

	stringstream ss;

	for(int i = 0; i < length; i++) {
		int bin = static_cast<int>(EPS_CORRECTION + histogram[i]);
		ss << BASE_36_MAP[bin];
	}

    // Return histogram
    return ss.str();
}

string translateBase(string number, int initialBase, int finalBase, int fill) {
	long int n = stoi(number, nullptr, initialBase);

	stringstream ss;
	int length = 0;

	while(n > 0) {
		ss << BASE_36_MAP[n % finalBase];
		n /= finalBase;
		length++;
	}

	for(int i = 0; i < fill - length; i++) {
		ss << "0";
	}

	string s = ss.str();
	reverse(s.begin(), s.end());

    return s;
}

string splashIt(vector<pair<double, double> > &spectrum, char spectrum_type) {
	stringstream ss;

	ss << buildInitialBlock(spectrum, spectrum_type) << '-';
	ss << translateBase(calculateHistogram(spectrum, spectrum_type, PREFILTER_BASE, PREFILTER_LENGTH, PREFILTER_BIN_SIZE), PREFILTER_BASE, 36, 4) << '-';
	ss << calculateHistogram(spectrum, spectrum_type, SIMILARITY_BASE, SIMILARITY_LENGTH, SIMILARITY_BIN_SIZE) << '-';
	ss << encodeSpectrum(spectrum, spectrum_type);

	return ss.str();
}

string splashIt(string spectrum_string, char spectrum_type) {
    // Convert spectrum to a vector of ion pairs and find the max intensity
    vector<string> ion_strings = split(spectrum_string, ' ');
    vector<pair<double, double> > spectrum;

    double maxIntensity = 0;

    for(vector<string>::iterator it = ion_strings.begin(); it != ion_strings.end(); ++it) {
        int delim_pos = (*it).find(':');

        double mz = atof((*it).substr(0, delim_pos).c_str());
        double intensity = atof((*it).substr(delim_pos + 1).c_str());

        if(intensity > maxIntensity)
            maxIntensity = intensity;

        // Store ion as a pair object, with 'first' corresponding to m/z
        // and 'second' to intensity
        spectrum.push_back(make_pair(mz, intensity));
    }

    // Normalize spectrum
    for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
        (*it).second = (*it).second / maxIntensity * RELATIVE_INTENSITY_SCALE;
    }

    // Return the calculated splash id
    return splashIt(spectrum, '1');
}<commit_msg>use string interpretation functions from the c++ string library<commit_after>#include <algorithm>
#include <cmath>
#include <ctime>
#include <cstdlib>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include <openssl/sha.h>

using namespace std;

// Version
const char SPLASH_VERSION = '0';

// Debug mode
const bool DEBUG = false;

// Precision of floating point operations and representations
const double MZ_PRECISION = 6;
const long long MZ_PRECISION_FACTOR = static_cast<long long>(pow(10, MZ_PRECISION));

const double INTENSITY_PRECISION = 0;
const long long INTENSITY_PRECISION_FACTOR = static_cast<long long>(pow(10, INTENSITY_PRECISION));

const double EPS = 1.0e-6;

// Value to scale relative spectra
const double RELATIVE_INTENSITY_SCALE = 100.0;

// Separator for building spectrum strings
const char ION_SEPARATOR = ' ';

// Full spectrum hash properties
const char ION_PAIR_SEPARATOR = ':';
const int MAX_HASH_CHARATERS_ENCODED_SPECTRUM = 20;
const double EPS_CORRECTION = 1.0e-7;

// Prefilter properties
const int PREFILTER_BASE = 3;
const int PREFILTER_LENGTH = 10;
const int PREFILTER_BIN_SIZE = 5;

// Similarity histogram properties
const int SIMILARITY_BASE = 10;
const int SIMILARITY_LENGTH = 10;
const int SIMILARITY_BIN_SIZE = 100;

// Map to convert up to base 36
const char BASE_36_MAP[] = {
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c',
    'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
    'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'
};

string sha256(const string s) {
	unsigned char digest[SHA256_DIGEST_LENGTH];

	SHA256_CTX sha256;
	SHA256_Init(&sha256);
	SHA256_Update(&sha256, s.c_str(), s.size());
	SHA256_Final(digest, &sha256);

	stringstream ss;

	for(int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
		ss << hex << setw(2) << setfill('0') << (int)digest[i];
	}

	return ss.str();
}

// http://stackoverflow.com/a/236803
vector<string> split(const string &s, char delimeter) {
	vector<string> elements;
	stringstream ss(s);
	string element;

	while(getline(ss, element, delimeter)) {
		elements.push_back(element);
	}

	return elements;
}

bool ionPairMzComparator(const pair<double, double> a, const pair<double, double> b) {
	if(abs(a.first - b.first) < EPS) {
		return a.second > b.second;
	} else {
		return a.first < b.first;
	}
}

bool ionPairIntensityComparator(const pair<double, double> a, const pair<double, double> b) {
	if(abs(a.second - b.second) < EPS) {
		return a.first < b.first;
	} else {
		return a.second > b.second;
	}
}

string buildInitialBlock(vector<pair<double, double> > &spectrum, char spectrum_type) {
	stringstream ss;
	ss << "splash" << spectrum_type << SPLASH_VERSION;
	return ss.str();
}

string encodeSpectrum(vector<pair<double, double> > &spectrum, char spectrum_type) {
	sort(spectrum.begin(), spectrum.end(), ionPairMzComparator);

	int i = 0;
	stringstream ss;

	for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
		ss << static_cast<long long>(((*it).first + EPS_CORRECTION) * MZ_PRECISION_FACTOR)
		   << ION_PAIR_SEPARATOR
		   << static_cast<long long>(((*it).second + EPS_CORRECTION) * INTENSITY_PRECISION_FACTOR);

		if(++i < spectrum.size()) {
			ss << ION_SEPARATOR;
		}
	}

	if(DEBUG) {
		cerr << "Encoded Spectrum: '" << ss.str() << "'" << endl;
	}

	return sha256(ss.str()).substr(0, MAX_HASH_CHARATERS_ENCODED_SPECTRUM);
}

string calculateHistogram(vector<pair<double, double> > &spectrum, char spectrum_type, int base, int length, int bin_size) {
	double* histogram = new double[length]();
	double maxIntensity = 0;

	// Bin ions using the histogram wrapping strategy
	for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
		int idx = static_cast<int>((*it).first / bin_size) % length;
		histogram[idx] += (*it).second;

		if(histogram[idx] > maxIntensity)
			maxIntensity = histogram[idx];
	}

    // Normalize the histogram and scale to the provided base
    for (int i = 0; i < length; i++) {
        histogram[i] = (base - 1) * histogram[i] / maxIntensity;
    }

	stringstream ss;

	for(int i = 0; i < length; i++) {
		int bin = static_cast<int>(EPS_CORRECTION + histogram[i]);
		ss << BASE_36_MAP[bin];
	}

    // Return histogram
    return ss.str();
}

string translateBase(string number, int initialBase, int finalBase, int fill) {
	long int n = stoi(number, nullptr, initialBase);

	stringstream ss;
	int length = 0;

	while(n > 0) {
		ss << BASE_36_MAP[n % finalBase];
		n /= finalBase;
		length++;
	}

	for(int i = 0; i < fill - length; i++) {
		ss << "0";
	}

	string s = ss.str();
	reverse(s.begin(), s.end());

    return s;
}

string splashIt(vector<pair<double, double> > &spectrum, char spectrum_type) {
	stringstream ss;

	ss << buildInitialBlock(spectrum, spectrum_type) << '-';
	ss << translateBase(calculateHistogram(spectrum, spectrum_type, PREFILTER_BASE, PREFILTER_LENGTH, PREFILTER_BIN_SIZE), PREFILTER_BASE, 36, 4) << '-';
	ss << calculateHistogram(spectrum, spectrum_type, SIMILARITY_BASE, SIMILARITY_LENGTH, SIMILARITY_BIN_SIZE) << '-';
	ss << encodeSpectrum(spectrum, spectrum_type);

	return ss.str();
}

string splashIt(string spectrum_string, char spectrum_type) {
    // Convert spectrum to a vector of ion pairs and find the max intensity
    vector<string> ion_strings = split(spectrum_string, ' ');
    vector<pair<double, double> > spectrum;

    double maxIntensity = 0;

    for(vector<string>::iterator it = ion_strings.begin(); it != ion_strings.end(); ++it) {
        int delim_pos = (*it).find(':');

        double mz = stod((*it).substr(0, delim_pos));
        double intensity = stod((*it).substr(delim_pos + 1));

        if(intensity > maxIntensity)
            maxIntensity = intensity;

        // Store ion as a pair object, with 'first' corresponding to m/z
        // and 'second' to intensity
        spectrum.push_back(make_pair(mz, intensity));
    }

    // Normalize spectrum
    for(vector<pair<double, double> >::iterator it = spectrum.begin(); it != spectrum.end(); ++it) {
        (*it).second = (*it).second / maxIntensity * RELATIVE_INTENSITY_SCALE;
    }

    // Return the calculated splash id
    return splashIt(spectrum, '1');
}<|endoftext|>


<commit_before>// Copyright 2021 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <memory>
#include <random>

#include "open_spiel/abseil-cpp/absl/flags/flag.h"
#include "open_spiel/abseil-cpp/absl/flags/parse.h"
#include "open_spiel/abseil-cpp/absl/random/uniform_int_distribution.h"
#include "open_spiel/spiel.h"
#include "open_spiel/spiel_utils.h"

ABSL_FLAG(std::string, game, "tic_tac_toe", "The name of the game to play.");
ABSL_FLAG(int, players, 0, "How many players in this game, 0 for default.");
ABSL_FLAG(bool, show_infostate, false, "Show the information state.");
ABSL_FLAG(int, seed, 0, "Seed for the random number generator. 0 for auto.");
ABSL_FLAG(bool, show_legals, false, "Show the legal moves.");

void PrintLegalActions(const open_spiel::State& state,
                       open_spiel::Player player,
                       const std::vector<open_spiel::Action>& movelist) {
  std::cerr << "Legal moves for player " << player << ":" << std::endl;
  for (open_spiel::Action action : movelist) {
    std::cerr << "  " << state.ActionToString(player, action) << std::endl;
  }
}

int main(int argc, char** argv) {
  absl::ParseCommandLine(argc, argv);

  std::string game_name = absl::GetFlag(FLAGS_game);
  auto players = absl::GetFlag(FLAGS_players);
  bool show_infostate = absl::GetFlag(FLAGS_show_infostate);
  int seed = absl::GetFlag(FLAGS_seed);
  bool show_legals = absl::GetFlag(FLAGS_show_legals);

  // Print out registered games.
  std::cerr << "Registered games:" << std::endl;
  std::vector<std::string> names = open_spiel::RegisteredGames();
  for (const std::string& name : names) {
    std::cerr << name << std::endl;
  }

  // Random number generator.
  std::mt19937 rng(seed ? seed : time(0));

  // Create the game.
  std::cerr << "Creating game..\n" << std::endl;

  // Add any specified parameters to override the defaults.
  open_spiel::GameParameters params;
  if (players > 0) {
    params["players"] = open_spiel::GameParameter(players);
  }
  std::shared_ptr<const open_spiel::Game> game =
      open_spiel::LoadGame(game_name, params);

  if (!game) {
    std::cerr << "problem with loading game, exiting..." << std::endl;
    return -1;
  }

  std::cerr << "Starting new game..." << std::endl;
  std::unique_ptr<open_spiel::State> state = game->NewInitialState();

  std::cerr << "Initial state:" << std::endl;
  std::cerr << "State:" << std::endl << state->ToString() << std::endl;

  while (!state->IsTerminal()) {
    std::cerr << "player " << state->CurrentPlayer() << std::endl;

    if (state->IsChanceNode()) {
      // Chance node; sample one according to underlying distribution.
      std::vector<std::pair<open_spiel::Action, double>> outcomes =
          state->ChanceOutcomes();
      open_spiel::Action action = open_spiel::SampleAction(outcomes, rng).first;
      std::cerr << "sampled outcome: "
                << state->ActionToString(open_spiel::kChancePlayerId, action)
                << std::endl;
      state->ApplyAction(action);
    } else if (state->IsSimultaneousNode()) {
      // open_spiel::Players choose simultaneously?
      std::vector<open_spiel::Action> joint_action;
      std::vector<float> infostate(game->InformationStateTensorSize());

      // Sample a action for each player
      for (auto player = open_spiel::Player{0}; player < game->NumPlayers();
           ++player) {
        if (show_infostate) {
          if (game->GetType().provides_information_state_tensor) {
            state->InformationStateTensor(player, absl::MakeSpan(infostate));
            std::cerr << "player " << player << ": "
                      << absl::StrJoin(infostate, " ") << std::endl;
          }
          if (game->GetType().provides_information_state_string) {
            std::cerr << "player " << player << ": "
                      << state->InformationStateString(player) << std::endl;
          }
        }

        std::vector<open_spiel::Action> actions = state->LegalActions(player);
        if (show_legals) {
          PrintLegalActions(*state, player, actions);
        }

        open_spiel::Action action = 0;
        if (!actions.empty()){
          absl::uniform_int_distribution<> dis(0, actions.size() - 1);
          action = actions[dis(rng)];
        }
        joint_action.push_back(action);
        std::cerr << "player " << player << " chose "
                  << state->ActionToString(player, action) << std::endl;
      }

      state->ApplyActions(joint_action);
    } else {
      // Decision node, sample one uniformly.
      auto player = state->CurrentPlayer();
      if (show_infostate) {
        if (game->GetType().provides_information_state_tensor) {
          std::vector<float> infostate;
          state->InformationStateTensor(player, absl::MakeSpan(infostate));
          std::cerr << "player " << player << ": "
                    << absl::StrJoin(infostate, " ") << std::endl;
        }
        if (game->GetType().provides_information_state_string) {
          std::cerr << "player " << player << ": "
                    << state->InformationStateString(player) << std::endl;
        }
      }

      std::vector<open_spiel::Action> actions = state->LegalActions();
      if (show_legals) {
        PrintLegalActions(*state, player, actions);
      }

      absl::uniform_int_distribution<> dis(0, actions.size() - 1);
      auto action = actions[dis(rng)];
      std::cerr << "chose action: " << state->ActionToString(player, action)
                << std::endl;
      state->ApplyAction(action);
    }

    std::cerr << "State: " << std::endl << state->ToString() << std::endl;
  }

  auto returns = state->Returns();
  for (auto p = open_spiel::Player{0}; p < game->NumPlayers(); p++) {
    std::cerr << "Final return to player " << p << " is " << returns[p]
              << std::endl;
  }
}
<commit_msg>Add mean field node in example.<commit_after>// Copyright 2021 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <memory>
#include <random>

#include "open_spiel/abseil-cpp/absl/flags/flag.h"
#include "open_spiel/abseil-cpp/absl/flags/parse.h"
#include "open_spiel/abseil-cpp/absl/random/uniform_int_distribution.h"
#include "open_spiel/spiel.h"
#include "open_spiel/spiel_utils.h"

ABSL_FLAG(std::string, game, "tic_tac_toe", "The name of the game to play.");
ABSL_FLAG(int, players, 0, "How many players in this game, 0 for default.");
ABSL_FLAG(bool, show_infostate, false, "Show the information state.");
ABSL_FLAG(int, seed, 0, "Seed for the random number generator. 0 for auto.");
ABSL_FLAG(bool, show_legals, false, "Show the legal moves.");

void PrintLegalActions(const open_spiel::State& state,
                       open_spiel::Player player,
                       const std::vector<open_spiel::Action>& movelist) {
  std::cerr << "Legal moves for player " << player << ":" << std::endl;
  for (open_spiel::Action action : movelist) {
    std::cerr << "  " << state.ActionToString(player, action) << std::endl;
  }
}

int main(int argc, char** argv) {
  absl::ParseCommandLine(argc, argv);

  std::string game_name = absl::GetFlag(FLAGS_game);
  auto players = absl::GetFlag(FLAGS_players);
  bool show_infostate = absl::GetFlag(FLAGS_show_infostate);
  int seed = absl::GetFlag(FLAGS_seed);
  bool show_legals = absl::GetFlag(FLAGS_show_legals);

  // Print out registered games.
  std::cerr << "Registered games:" << std::endl;
  std::vector<std::string> names = open_spiel::RegisteredGames();
  for (const std::string& name : names) {
    std::cerr << name << std::endl;
  }

  // Random number generator.
  std::mt19937 rng(seed ? seed : time(0));

  // Create the game.
  std::cerr << "Creating game..\n" << std::endl;

  // Add any specified parameters to override the defaults.
  open_spiel::GameParameters params;
  if (players > 0) {
    params["players"] = open_spiel::GameParameter(players);
  }
  std::shared_ptr<const open_spiel::Game> game =
      open_spiel::LoadGame(game_name, params);

  if (!game) {
    std::cerr << "problem with loading game, exiting..." << std::endl;
    return -1;
  }

  std::cerr << "Starting new game..." << std::endl;
  std::unique_ptr<open_spiel::State> state = game->NewInitialState();

  std::cerr << "Initial state:" << std::endl;
  std::cerr << "State:" << std::endl << state->ToString() << std::endl;

  while (!state->IsTerminal()) {
    std::cerr << "player " << state->CurrentPlayer() << std::endl;

    if (state->IsChanceNode()) {
      // Chance node; sample one according to underlying distribution.
      std::vector<std::pair<open_spiel::Action, double>> outcomes =
          state->ChanceOutcomes();
      open_spiel::Action action = open_spiel::SampleAction(outcomes, rng).first;
      std::cerr << "sampled outcome: "
                << state->ActionToString(open_spiel::kChancePlayerId, action)
                << std::endl;
      state->ApplyAction(action);
    } else if (state->IsMeanFieldNode()) {
      int num_states_distribution = state->DistributionSupport().size();
      if (num_states_distribution) {
        state->UpdateDistribution(std::vector<double>(
            num_states_distribution, 1.0 / num_states_distribution));
      }
    } else if (state->IsSimultaneousNode()) {
      // open_spiel::Players choose simultaneously?
      std::vector<open_spiel::Action> joint_action;
      std::vector<float> infostate(game->InformationStateTensorSize());

      // Sample a action for each player
      for (auto player = open_spiel::Player{0}; player < game->NumPlayers();
           ++player) {
        if (show_infostate) {
          if (game->GetType().provides_information_state_tensor) {
            state->InformationStateTensor(player, absl::MakeSpan(infostate));
            std::cerr << "player " << player << ": "
                      << absl::StrJoin(infostate, " ") << std::endl;
          }
          if (game->GetType().provides_information_state_string) {
            std::cerr << "player " << player << ": "
                      << state->InformationStateString(player) << std::endl;
          }
        }

        std::vector<open_spiel::Action> actions = state->LegalActions(player);
        if (show_legals) {
          PrintLegalActions(*state, player, actions);
        }

        open_spiel::Action action = 0;
        if (!actions.empty()) {
          absl::uniform_int_distribution<> dis(0, actions.size() - 1);
          action = actions[dis(rng)];
        }
        joint_action.push_back(action);
        std::cerr << "player " << player << " chose "
                  << state->ActionToString(player, action) << std::endl;
      }

      state->ApplyActions(joint_action);
    } else {
      // Decision node, sample one uniformly.
      auto player = state->CurrentPlayer();
      if (show_infostate) {
        if (game->GetType().provides_information_state_tensor) {
          std::vector<float> infostate;
          state->InformationStateTensor(player, absl::MakeSpan(infostate));
          std::cerr << "player " << player << ": "
                    << absl::StrJoin(infostate, " ") << std::endl;
        }
        if (game->GetType().provides_information_state_string) {
          std::cerr << "player " << player << ": "
                    << state->InformationStateString(player) << std::endl;
        }
      }

      std::vector<open_spiel::Action> actions = state->LegalActions();
      if (show_legals) {
        PrintLegalActions(*state, player, actions);
      }

      absl::uniform_int_distribution<> dis(0, actions.size() - 1);
      auto action = actions[dis(rng)];
      std::cerr << "chose action: " << state->ActionToString(player, action)
                << std::endl;
      state->ApplyAction(action);
    }

    std::cerr << "State: " << std::endl << state->ToString() << std::endl;
  }

  auto returns = state->Returns();
  for (auto p = open_spiel::Player{0}; p < game->NumPlayers(); p++) {
    std::cerr << "Final return to player " << p << " is " << returns[p]
              << std::endl;
  }
}
<|endoftext|>