<commit_before>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <iostream>
#include <fstream>
#include <sstream>
#include "boost/filesystem/operations.hpp" // includes boost/filesystem/path.hpp
#include "boost/filesystem/fstream.hpp"    // ditto
#include <boost/algorithm/string.hpp>
#include "Parameter.h"
#include "Util.h"
#include "InputFileStream.h"
#include "UserMessage.h"

using namespace std;

/** defines allowed parameters */
Parameter::Parameter()
{
	AddParam("config-file", "f", "location of the configuration file");
	AddParam("ttable-file", "location and properties of the translation tables");
	AddParam("lmodel-file", "location and properties of the language models");
	AddParam("ttable-limit", "maximum number of translation table entries per input phrase");
	AddParam("weight-d", "d", "weight(s) for distortion (reordering components)");
	AddParam("weight-l", "lm", "weight(s) for language models");
	AddParam("weight-t", "tm", "weights for translation model components");
	AddParam("weight-w", "w", "weight for word penalty");
	AddParam("weight-e", "e", "weight for word deletion"); //source word deletion overall weight
	AddParam("weight-generation", "g", "weight(s) for generation components");
	AddParam("weight-i", "I", "weight for word insertion");
	AddParam("mapping", "description of decoding steps");
	AddParam("n-best-list", "file and size of n-best-list to be generated");
	AddParam("beam-threshold", "threshold for threshold pruning");
	AddParam("distortion-limit", "dl", "distortion (reordering) limit in maximum number of words");
	AddParam("input-factors", "list of factors in the input");
	AddParam("output-factors", "list if factors in the output");
	AddParam("mysql", "(deprecated)");
	AddParam("input-file", "i", "location of the input file to be translated");
	AddParam("cache-path", "?");
 	AddParam("distortion-file", "location and properties of the factorized/lexicalized reordering table");
 	AddParam("distortion", "?");
	AddParam("generation-file", "location and properties of the generation table");
	AddParam("stack", "s", "maximum stack size for histogram pruning");
	AddParam("verbose", "v", "verbosity level of the logging");
	AddParam("report-source-span", "?");
	AddParam("report-all-factors", "?");
	AddParam("drop-unknown", "du", "drop unknown words instead of copying them");
	AddParam("inputtype", "text (0) or confusion network (1)");
	AddParam("translation-details", "T", "?");
	AddParam("max-trans-opt-per-coverage", "maximum number of translation options per input span (after applying mapping steps)");
	AddParam("max-partial-trans-opt", "maximum number of partial translation options per input span (during mapping steps)");
	AddParam("use-distortion-future-costs", "consider expected distortion cost in future cost estimation");
}

/** initializes a parameter, sub of constructor */
PARAM_VEC &Parameter::AddParam(const string &paramName, const string &description)
{
	m_valid[paramName] = true;
	m_description[paramName] = description;
	return m_setting[paramName];
}

/** initializes a parameter (including abbreviation), sub of constructor */
PARAM_VEC &Parameter::AddParam(const string &paramName, const string &abbrevName, const string &description)
{
	m_valid[paramName] = true;
	m_valid[abbrevName] = true;
	m_abbreviation[paramName] = abbrevName;
	m_description[paramName] = description;
	return m_setting[paramName];
}

/** prints descriptions of all parameters */
void Parameter::Explain() {
	cerr << "Usage:" << endl;
	for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			const string paramDescription = iterParam->second;
			cerr << "\t-" << paramName;
			PARAM_STRING::const_iterator iterAbbr = m_abbreviation.find( paramName );
			if ( iterAbbr != m_abbreviation.end() )
				cerr << " (" << iterAbbr->second << ")";
			cerr << ": " << paramDescription << endl;
		}
}

/** checks if an item on the command line is a switch or a value
 * \param token token on the command line to checked **/

bool Parameter::isOption(const char* token) {
  if (! token) return false;
  std::string tokenString(token);
  size_t length = tokenString.size();
  if (length > 0 && tokenString.substr(0,1) != "-") return false;
  if (length > 1 && tokenString.substr(1,1).find_first_not_of("0123456789") == 0) return true;
  return false;
}

/** load all parameters from the configuration file and the command line switches */
bool Parameter::LoadParam(int argc, char* argv[])
{
	// config file (-f) arg mandatory
	string configPath;
	if ( (configPath = FindParam("-f", argc, argv)) == ""
		&& (configPath = FindParam("-config", argc, argv)) == "")
	{
		UserMessage::Add("No configuration file was specified.  Use -config or -f");
		return false;
	}
	else
	{
		if (!ReadConfigFile(configPath))
		{
			UserMessage::Add("Could not read "+configPath);
			return false;
		}
	}

	// overwrite parameters with values from switches
	for(PARAM_MAP::const_iterator iterParam = m_setting.begin(); iterParam != m_setting.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			OverwriteParam("-" + paramName, paramName, argc, argv);
		}

	// ... also shortcuts
	for(PARAM_STRING::const_iterator iterParam = m_abbreviation.begin(); iterParam != m_abbreviation.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			const string paramShortName = iterParam->second;
			OverwriteParam("-" + paramShortName, paramName, argc, argv);
		}

	// logging of parameters that were set in either config or switch
	//int verbose = 0;
	//if (m_setting["verbose"].size() > 0)
	//  verbose = Scan<int>(m_setting["verbose"][0]);
	//  if (verbose >= 1) { // only if verbose
	cerr << "Defined parameters (per moses.ini or switch):" << endl;
	for(PARAM_MAP::const_iterator iterParam = m_setting.begin() ; iterParam != m_setting.end(); iterParam++) {
		if (iterParam->second.size() > 0) {
			cerr << "\t" << iterParam->first << ": ";
			for ( size_t i = 0; i < iterParam->second.size(); i++ )
				cerr << iterParam->second[i] << " ";
			cerr << endl;
		}
		//}
	}

	// check for illegal parameters
	bool noErrorFlag = true;
	for (int i = 0 ; i < argc ; i++)
		{
			if (isOption(argv[i]))
				{
					string paramSwitch = (string) argv[i];
					string paramName = paramSwitch.substr(1);
					if (m_valid.find(paramName) == m_valid.end())
						{
							UserMessage::Add("illegal switch: " + paramSwitch);
							noErrorFlag = false;
						}
				}
		}
  // check if parameters make sense
	return Validate() && noErrorFlag;
}

/** check if parameter settings make sense */
bool Parameter::Validate()
{
	bool noErrorFlag = true;

  // required parameters
	if (m_setting["ttable-file"].size() == 0)
	{
		UserMessage::Add("No phrase translation table (ttable-file)");
		noErrorFlag = false;
	}

	if (m_setting["lmodel-file"].size() == 0)
	{
		UserMessage::Add("No language model (lmodel-file)");
		noErrorFlag = false;
	}

	if (m_setting["lmodel-file"].size() != m_setting["weight-l"].size())
	{
		stringstream errorMsg("");
		errorMsg << "Config and parameters specify "
            << static_cast<int>(m_setting["lmodel-file"].size())
						<< " language model files (lmodel-file), but "
						<< static_cast<int>(m_setting["weight-l"].size())
						<< " weights (weight-l)";
    errorMsg << endl << "You might be giving '-lmodel-file TYPE FACTOR ORDER FILENAME' but you should be giving these four as a single argument, i.e. '-lmodel-file \"TYPE FACTOR ORDER FILENAME\"'";
		UserMessage::Add(errorMsg.str());
		noErrorFlag = false;
	}

  // do files exist?
	// phrase tables
	if (noErrorFlag)
		{
			std::vector<std::string> ext;
			// standard phrase table extension (i.e. full name has to be specified)
			ext.push_back("");
			// alternative file extension for binary phrase table format:
			ext.push_back(".binphr.idx");
			noErrorFlag = FilesExist("ttable-file", 3,ext);
		}
	// generation tables
	//	if (noErrorFlag)
	//		noErrorFlag = FilesExist("generation-file", 2);
	// language model
	if (noErrorFlag)
		noErrorFlag = FilesExist("lmodel-file", 3);
	// input file
	if (noErrorFlag)
		noErrorFlag = FilesExist("input-file", 0);

	return noErrorFlag;
}

/** checks if a file exist */
bool Parameter::FilesExist(const string &paramName, size_t tokenizeIndex,std::vector<std::string> const& extensions)
{
	using namespace boost::filesystem;

	typedef std::vector<std::string> StringVec;
	StringVec::const_iterator iter;

	PARAM_MAP::const_iterator iterParam = m_setting.find(paramName);
	if (iterParam == m_setting.end())
	{ // no param. therefore nothing to check
		return true;
	}
	const StringVec &pathVec = (*iterParam).second;
	for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter)
	{
		StringVec vec = Tokenize(*iter);
		if (tokenizeIndex >= vec.size())
		{
			stringstream errorMsg("");
			errorMsg << "Expected " << tokenizeIndex << " tokens per"
							<< " entry in '" << paramName << "', but only found "
							<< vec.size();
			UserMessage::Add(errorMsg.str());
			return false;
		}
		const string &pathStr = vec[tokenizeIndex];

		bool fileFound=0;
		for(size_t i=0;i<extensions.size() && !fileFound;++i)
			{
				path filePath(pathStr+extensions[i], native);
				fileFound|=exists(filePath);
			}
		if(!fileFound)
			{
				stringstream errorMsg("");
				errorMsg << "File " << pathStr << " does not exist";
				UserMessage::Add(errorMsg.str());
				return false;
			}

	}
	return true;
}

/** looks for a switch in arg, updates parameter */
// TODO arg parsing like this does not belong in the library, it belongs
// in moses-cmd
string Parameter::FindParam(const string &paramSwitch, int argc, char* argv[])
{
	for (int i = 0 ; i < argc ; i++)
	{
		if (string(argv[i]) == paramSwitch)
		{
			if (i+1 < argc)
			{
				return argv[i+1];
			} else {
				stringstream errorMsg("");
				errorMsg << "Option " << paramSwitch << " requires a parameter!";
				UserMessage::Add(errorMsg.str());
				// TODO return some sort of error, not the empty string
			}
		}
	}
	return "";
}

/** update parameter settings with command line switches
 * \param paramSwitch (potentially short) name of switch
 * \param paramName full name of parameter
 * \param argc number of arguments on command line
 * \param argv values of paramters on command line */
void Parameter::OverwriteParam(const string &paramSwitch, const string &paramName, int argc, char* argv[])
{
	int startPos = -1;
	for (int i = 0 ; i < argc ; i++)
	{
		if (string(argv[i]) == paramSwitch)
		{
			startPos = i+1;
			break;
		}
	}
	if (startPos < 0)
		return;

	int index = 0;
	while (startPos < argc && (!isOption(argv[startPos])))
	{
		if (m_setting[paramName].size() > (size_t)index)
			m_setting[paramName][index] = argv[startPos];
		else
			m_setting[paramName].push_back(argv[startPos]);
		index++;
		startPos++;
	}
}

/** read parameters from a configuration file */
bool Parameter::ReadConfigFile( string filePath )
{
	InputFileStream inFile(filePath);
	string line, paramName;
	while(getline(inFile, line))
	{
		// comments
		size_t comPos = line.find_first_of("#");
		if (comPos != string::npos)
			line = line.substr(0, comPos);
		// trim leading and trailing spaces/tabs
		boost::trim(line);

		if (line[0]=='[')
		{ // new parameter
			for (size_t currPos = 0 ; currPos < line.size() ; currPos++)
			{
				if (line[currPos] == ']')
				{
					paramName = line.substr(1, currPos - 1);
					break;
				}
			}
		}
    else if (line != "")
		{ // add value to parameter
			m_setting[paramName].push_back(line);
		}
	}
	return true;
}
<commit_msg>add description for -T option, fix so -config is the proper option, not -config-file (required for mert to work)<commit_after>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include <iostream>
#include <fstream>
#include <sstream>
#include "boost/filesystem/operations.hpp" // includes boost/filesystem/path.hpp
#include "boost/filesystem/fstream.hpp"    // ditto
#include <boost/algorithm/string.hpp>
#include "Parameter.h"
#include "Util.h"
#include "InputFileStream.h"
#include "UserMessage.h"

using namespace std;

/** defines allowed parameters */
Parameter::Parameter()
{
	AddParam("config", "f", "location of the configuration file");
	AddParam("ttable-file", "location and properties of the translation tables");
	AddParam("lmodel-file", "location and properties of the language models");
	AddParam("ttable-limit", "maximum number of translation table entries per input phrase");
	AddParam("weight-d", "d", "weight(s) for distortion (reordering components)");
	AddParam("weight-l", "lm", "weight(s) for language models");
	AddParam("weight-t", "tm", "weights for translation model components");
	AddParam("weight-w", "w", "weight for word penalty");
	AddParam("weight-e", "e", "weight for word deletion"); //source word deletion overall weight
	AddParam("weight-generation", "g", "weight(s) for generation components");
	AddParam("weight-i", "I", "weight for word insertion");
	AddParam("mapping", "description of decoding steps");
	AddParam("n-best-list", "file and size of n-best-list to be generated");
	AddParam("beam-threshold", "threshold for threshold pruning");
	AddParam("distortion-limit", "dl", "distortion (reordering) limit in maximum number of words");
	AddParam("input-factors", "list of factors in the input");
	AddParam("output-factors", "list if factors in the output");
	AddParam("mysql", "(deprecated)");
	AddParam("input-file", "i", "location of the input file to be translated");
	AddParam("cache-path", "?");
 	AddParam("distortion-file", "location and properties of the factorized/lexicalized reordering table");
 	AddParam("distortion", "?");
	AddParam("generation-file", "location and properties of the generation table");
	AddParam("stack", "s", "maximum stack size for histogram pruning");
	AddParam("verbose", "v", "verbosity level of the logging");
	AddParam("report-source-span", "?");
	AddParam("report-all-factors", "?");
	AddParam("drop-unknown", "du", "drop unknown words instead of copying them");
	AddParam("inputtype", "text (0) or confusion network (1)");
	AddParam("translation-details", "T", "for each best translation hypothesis, print out details about what sourcce spans were used, dropped");
	AddParam("max-trans-opt-per-coverage", "maximum number of translation options per input span (after applying mapping steps)");
	AddParam("max-partial-trans-opt", "maximum number of partial translation options per input span (during mapping steps)");
	AddParam("use-distortion-future-costs", "consider expected distortion cost in future cost estimation");
}

/** initializes a parameter, sub of constructor */
PARAM_VEC &Parameter::AddParam(const string &paramName, const string &description)
{
	m_valid[paramName] = true;
	m_description[paramName] = description;
	return m_setting[paramName];
}

/** initializes a parameter (including abbreviation), sub of constructor */
PARAM_VEC &Parameter::AddParam(const string &paramName, const string &abbrevName, const string &description)
{
	m_valid[paramName] = true;
	m_valid[abbrevName] = true;
	m_abbreviation[paramName] = abbrevName;
	m_description[paramName] = description;
	return m_setting[paramName];
}

/** prints descriptions of all parameters */
void Parameter::Explain() {
	cerr << "Usage:" << endl;
	for(PARAM_STRING::const_iterator iterParam = m_description.begin(); iterParam != m_description.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			const string paramDescription = iterParam->second;
			cerr << "\t-" << paramName;
			PARAM_STRING::const_iterator iterAbbr = m_abbreviation.find( paramName );
			if ( iterAbbr != m_abbreviation.end() )
				cerr << " (" << iterAbbr->second << ")";
			cerr << ": " << paramDescription << endl;
		}
}

/** checks if an item on the command line is a switch or a value
 * \param token token on the command line to checked **/

bool Parameter::isOption(const char* token) {
  if (! token) return false;
  std::string tokenString(token);
  size_t length = tokenString.size();
  if (length > 0 && tokenString.substr(0,1) != "-") return false;
  if (length > 1 && tokenString.substr(1,1).find_first_not_of("0123456789") == 0) return true;
  return false;
}

/** load all parameters from the configuration file and the command line switches */
bool Parameter::LoadParam(int argc, char* argv[])
{
	// config file (-f) arg mandatory
	string configPath;
	if ( (configPath = FindParam("-f", argc, argv)) == ""
		&& (configPath = FindParam("-config", argc, argv)) == "")
	{
		UserMessage::Add("No configuration file was specified.  Use -config or -f");
		return false;
	}
	else
	{
		if (!ReadConfigFile(configPath))
		{
			UserMessage::Add("Could not read "+configPath);
			return false;
		}
	}

	// overwrite parameters with values from switches
	for(PARAM_MAP::const_iterator iterParam = m_setting.begin(); iterParam != m_setting.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			OverwriteParam("-" + paramName, paramName, argc, argv);
		}

	// ... also shortcuts
	for(PARAM_STRING::const_iterator iterParam = m_abbreviation.begin(); iterParam != m_abbreviation.end(); iterParam++)
		{
			const string paramName = iterParam->first;
			const string paramShortName = iterParam->second;
			OverwriteParam("-" + paramShortName, paramName, argc, argv);
		}

	// logging of parameters that were set in either config or switch
	//int verbose = 0;
	//if (m_setting["verbose"].size() > 0)
	//  verbose = Scan<int>(m_setting["verbose"][0]);
	//  if (verbose >= 1) { // only if verbose
	cerr << "Defined parameters (per moses.ini or switch):" << endl;
	for(PARAM_MAP::const_iterator iterParam = m_setting.begin() ; iterParam != m_setting.end(); iterParam++) {
		if (iterParam->second.size() > 0) {
			cerr << "\t" << iterParam->first << ": ";
			for ( size_t i = 0; i < iterParam->second.size(); i++ )
				cerr << iterParam->second[i] << " ";
			cerr << endl;
		}
		//}
	}

	// check for illegal parameters
	bool noErrorFlag = true;
	for (int i = 0 ; i < argc ; i++)
		{
			if (isOption(argv[i]))
				{
					string paramSwitch = (string) argv[i];
					string paramName = paramSwitch.substr(1);
					if (m_valid.find(paramName) == m_valid.end())
						{
							UserMessage::Add("illegal switch: " + paramSwitch);
							noErrorFlag = false;
						}
				}
		}
  // check if parameters make sense
	return Validate() && noErrorFlag;
}

/** check if parameter settings make sense */
bool Parameter::Validate()
{
	bool noErrorFlag = true;

  // required parameters
	if (m_setting["ttable-file"].size() == 0)
	{
		UserMessage::Add("No phrase translation table (ttable-file)");
		noErrorFlag = false;
	}

	if (m_setting["lmodel-file"].size() == 0)
	{
		UserMessage::Add("No language model (lmodel-file)");
		noErrorFlag = false;
	}

	if (m_setting["lmodel-file"].size() != m_setting["weight-l"].size())
	{
		stringstream errorMsg("");
		errorMsg << "Config and parameters specify "
            << static_cast<int>(m_setting["lmodel-file"].size())
						<< " language model files (lmodel-file), but "
						<< static_cast<int>(m_setting["weight-l"].size())
						<< " weights (weight-l)";
    errorMsg << endl << "You might be giving '-lmodel-file TYPE FACTOR ORDER FILENAME' but you should be giving these four as a single argument, i.e. '-lmodel-file \"TYPE FACTOR ORDER FILENAME\"'";
		UserMessage::Add(errorMsg.str());
		noErrorFlag = false;
	}

  // do files exist?
	// phrase tables
	if (noErrorFlag)
		{
			std::vector<std::string> ext;
			// standard phrase table extension (i.e. full name has to be specified)
			ext.push_back("");
			// alternative file extension for binary phrase table format:
			ext.push_back(".binphr.idx");
			noErrorFlag = FilesExist("ttable-file", 3,ext);
		}
	// generation tables
	//	if (noErrorFlag)
	//		noErrorFlag = FilesExist("generation-file", 2);
	// language model
	if (noErrorFlag)
		noErrorFlag = FilesExist("lmodel-file", 3);
	// input file
	if (noErrorFlag)
		noErrorFlag = FilesExist("input-file", 0);

	return noErrorFlag;
}

/** checks if a file exist */
bool Parameter::FilesExist(const string &paramName, size_t tokenizeIndex,std::vector<std::string> const& extensions)
{
	using namespace boost::filesystem;

	typedef std::vector<std::string> StringVec;
	StringVec::const_iterator iter;

	PARAM_MAP::const_iterator iterParam = m_setting.find(paramName);
	if (iterParam == m_setting.end())
	{ // no param. therefore nothing to check
		return true;
	}
	const StringVec &pathVec = (*iterParam).second;
	for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter)
	{
		StringVec vec = Tokenize(*iter);
		if (tokenizeIndex >= vec.size())
		{
			stringstream errorMsg("");
			errorMsg << "Expected " << tokenizeIndex << " tokens per"
							<< " entry in '" << paramName << "', but only found "
							<< vec.size();
			UserMessage::Add(errorMsg.str());
			return false;
		}
		const string &pathStr = vec[tokenizeIndex];

		bool fileFound=0;
		for(size_t i=0;i<extensions.size() && !fileFound;++i)
			{
				path filePath(pathStr+extensions[i], native);
				fileFound|=exists(filePath);
			}
		if(!fileFound)
			{
				stringstream errorMsg("");
				errorMsg << "File " << pathStr << " does not exist";
				UserMessage::Add(errorMsg.str());
				return false;
			}

	}
	return true;
}

/** looks for a switch in arg, updates parameter */
// TODO arg parsing like this does not belong in the library, it belongs
// in moses-cmd
string Parameter::FindParam(const string &paramSwitch, int argc, char* argv[])
{
	for (int i = 0 ; i < argc ; i++)
	{
		if (string(argv[i]) == paramSwitch)
		{
			if (i+1 < argc)
			{
				return argv[i+1];
			} else {
				stringstream errorMsg("");
				errorMsg << "Option " << paramSwitch << " requires a parameter!";
				UserMessage::Add(errorMsg.str());
				// TODO return some sort of error, not the empty string
			}
		}
	}
	return "";
}

/** update parameter settings with command line switches
 * \param paramSwitch (potentially short) name of switch
 * \param paramName full name of parameter
 * \param argc number of arguments on command line
 * \param argv values of paramters on command line */
void Parameter::OverwriteParam(const string &paramSwitch, const string &paramName, int argc, char* argv[])
{
	int startPos = -1;
	for (int i = 0 ; i < argc ; i++)
	{
		if (string(argv[i]) == paramSwitch)
		{
			startPos = i+1;
			break;
		}
	}
	if (startPos < 0)
		return;

	int index = 0;
	while (startPos < argc && (!isOption(argv[startPos])))
	{
		if (m_setting[paramName].size() > (size_t)index)
			m_setting[paramName][index] = argv[startPos];
		else
			m_setting[paramName].push_back(argv[startPos]);
		index++;
		startPos++;
	}
}

/** read parameters from a configuration file */
bool Parameter::ReadConfigFile( string filePath )
{
	InputFileStream inFile(filePath);
	string line, paramName;
	while(getline(inFile, line))
	{
		// comments
		size_t comPos = line.find_first_of("#");
		if (comPos != string::npos)
			line = line.substr(0, comPos);
		// trim leading and trailing spaces/tabs
		boost::trim(line);

		if (line[0]=='[')
		{ // new parameter
			for (size_t currPos = 0 ; currPos < line.size() ; currPos++)
			{
				if (line[currPos] == ']')
				{
					paramName = line.substr(1, currPos - 1);
					break;
				}
			}
		}
    else if (line != "")
		{ // add value to parameter
			m_setting[paramName].push_back(line);
		}
	}
	return true;
}
<|endoftext|>


<commit_before>/*
 * Quantum++
 *
 * Copyright (c) 2013 - 2014 Vlad Gheorghiu (vgheorgh@gmail.com)
 *
 * This file is part of Quantum++.
 *
 * Quantum++ is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Quantum++ is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Quantum++.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "qpp.h"

//#include "MATLAB/matlab.h" // support for MATLAB

using namespace qpp;

int main()
{
    // Qudit Teleportation
    {
        std::size_t D = 3; // size of the system
        std::cout << std::endl << "**** Qudit Teleportation, D = " << D << " ****" << std::endl;
        ket mes_AB = ket::Zero(D * D); // maximally entangled state resource
        for (std::size_t i = 0; i < D; i++)
            mes_AB += mket({i, i}, D);
        mes_AB /= std::sqrt((double) D);
        // circuit that measures in the qudit Bell basis
        cmat Bell_aA = adjoint(gt.CTRL(gt.Xd(D), {0}, {1}, 2, D) * kron(gt.Fd(D), gt.Id(D)));
        ket psi_a = randket(D); // random state as input on a
        std::cout << ">> Initial state:" << std::endl;
        std::cout << disp(psi_a) << std::endl;
        ket input_aAB = kron(psi_a, mes_AB); // joint input state aAB
        // output before measurement
        ket output_aAB = apply(input_aAB, Bell_aA, {0, 1}, 3, D);
        auto measured_aA = measure(ptrace2(prj(output_aAB), {D * D, D}), gt.Id(D * D)); // measure on aA
        std::discrete_distribution<std::size_t> dd(measured_aA.first.begin(), measured_aA.first.end());
        std::cout << ">> Measurement probabilities: ";
        std::cout << disp(measured_aA.first, ", ") << std::endl;
        std::size_t m = dd(rdevs._rng); // sample
        auto midx = n2multiidx(m, {D, D});
        std::cout << ">> Measurement result: ";
        std::cout << disp(midx, " ") << std::endl;
        // conditional result on B before correction
        ket output_m_aAB = apply(output_aAB, prj(mket(midx, D)), {0, 1}, 3, D) / std::sqrt(measured_aA.first[m]);
        cmat correction_B = powm(gt.Zd(D), midx[0]) * powm(adjoint(gt.Xd(D)), midx[1]); // correction operator
        // apply correction on B
        output_aAB = apply(output_m_aAB, correction_B, {2}, 3, D);
        cmat rho_B = ptrace1(prj(output_aAB), {D * D, D});
        std::cout << ">> Bob's density operator: " << std::endl;
        std::cout << disp(rho_B) << std::endl;
        std::cout << ">> Norm difference: " << norm(rho_B - prj(psi_a)) << std::endl; // verification
    }

    // Qudit Dense Coding
    {
        std::size_t D = 3; // size of the system
        std::cout << std::endl << "**** Qudit Dense Coding, D = " << D << " ****" << std::endl;
        ket mes_AB = ket::Zero(D * D); // maximally entangled state resource
        for (std::size_t i = 0; i < D; i++)
            mes_AB += mket({i, i}, D);
        mes_AB /= std::sqrt((double) D);
        // circuit that measures in the qudit Bell basis
        cmat Bell_AB = adjoint(gt.CTRL(gt.Xd(D), {0}, {1}, 2, D) * kron(gt.Fd(D), gt.Id(D)));
        // equal probabilities of choosing a message
        std::uniform_int_distribution<std::size_t> uid(0, D * D - 1);
        std::size_t m_A = uid(rdevs._rng); // sample, obtain the message index
        auto midx = n2multiidx(m_A, {D, D});
        std::cout << ">> Alice sent: ";
        std::cout << disp(midx, " ") << std::endl;
        // Alice's operation
        cmat U_A = powm(gt.Zd(D), midx[0]) * powm(adjoint(gt.Xd(D)), midx[1]);
        // Alice encodes the message
        ket psi_AB = apply(mes_AB, U_A, {0}, 2, D);
        // Bob measures the joint system in the qudit Bell basis
        psi_AB = apply(psi_AB, Bell_AB, {0, 1}, 2, D);
        auto measured = measure(psi_AB, gt.Id(D * D));
        std::cout << ">> Bob measurement probabilities: ";
        std::cout << disp(measured.first, ", ") << std::endl;
        // Bob samples according to the measurement probabilities
        std::discrete_distribution<std::size_t> dd(measured.first.begin(), measured.first.end());
        std::size_t m_B = dd(rdevs._rng);
        std::cout << ">> Bob received: ";
        std::cout << disp(n2multiidx(m_B, {D, D}), " ") << std::endl;
    }

    // Grover's search algorithm, we time it
    {
        Timer t; // set a timer
        std::size_t n = 4; // number of qubits
        std::cout << std::endl << "**** Grover on n = " << n << " qubits ****" << std::endl;
        std::vector<std::size_t> dims(n, 2); // local dimensions
        std::size_t N = std::pow(2, n); // number of elements in the database
        std::cout << ">> Database size: " << N << std::endl;
        // mark an element randomly
        std::uniform_int_distribution<std::size_t> uid(0, N - 1);
        std::size_t marked = uid(rdevs._rng);
        std::cout << ">> Marked state: " << marked << " -> ";
        std::cout << disp(n2multiidx(marked, dims), " ") << std::endl;
        ket psi = mket(n2multiidx(0, dims)); // computational |0>^\otimes n
        psi = (kronpow(gt.H, n) * psi).eval(); // apply H^\otimes n, no aliasing
        cmat G = 2 * prj(psi) - gt.Id(N); // Diffusion operator
        // number of queries
        std::size_t nqueries = std::ceil(pi * std::sqrt(N) / 4.);
        std::cout << ">> We run " << nqueries << " queries" << std::endl;
        for (std::size_t i = 0; i < nqueries; i++)
        {
            psi(marked) = -psi(marked); // apply the oracle first, no aliasing
            psi = (G * psi).eval(); // then the diffusion operator, no aliasing
        }
        // we now measure the state in the computational basis
        auto measured = measure(psi, gt.Id(N));
        std::cout << ">> Probability of the marked state: " << measured.first[marked] << std::endl;
        std::cout << ">> Probability of all results: ";
        std::cout << disp(measured.first, ", ") << std::endl;
        std::cout << ">> Let's sample..." << std::endl;
        std::discrete_distribution<std::size_t> dd(measured.first.begin(), measured.first.end());
        std::size_t result = dd(rdevs._rng);
        if (result == marked)
            std::cout << ">> Hooray, we obtained the correct result: ";
        else
            std::cout << ">> Not there yet... we obtained: ";
        std::cout << result << " -> ";
        std::cout << disp(n2multiidx(result, dims), " ") << std::endl;
        // stop the timer and display it
        std::cout << ">> It took " << t.toc() << " seconds to simulate Grover on " << n << " qubits." << std::endl;
    }

    // Entanglement
    {
        std::cout << std::endl << "**** Entanglement ****" << std::endl;
        cmat rho = 0.2 * st.pb00 + 0.8 * st.pb11;
        std::cout << ">> rho: " << std::endl;
        std::cout << disp(rho) << std::endl;
        std::cout << ">> Concurrence of rho: " << concurrence(rho) << std::endl;
        std::cout << ">> Negativity of rho: " << negativity(rho, {2, 2}) << std::endl;
        std::cout << ">> Logarithimc negativity of rho: " << lognegativity(rho, {2, 2}) << std::endl;
        ket psi = 0.8 * mket({0, 0}) + 0.6 * mket({1, 1});
        // apply some local random unitaries
        psi = kron(randU(2), randU(2)) * psi;
        std::cout << ">> psi: " << std::endl;
        std::cout << disp(psi) << std::endl;
        std::cout << ">> Entanglement of psi: " << entanglement(psi, {2, 2}) << std::endl;
        std::cout << ">> Concurrence of psi: " << concurrence(prj(psi)) << std::endl;
        std::cout << ">> G-Concurrence of psi: " << gconcurrence(psi) << std::endl;
        std::cout << ">> Schmidt coefficients of psi: " << std::endl;
        std::cout << disp(schmidtcoeff(psi, {2, 2})) << std::endl;
        std::cout << ">> Schmidt probabilities of psi: " << std::endl;
        std::cout << disp(schmidtprob(psi, {2, 2})) << std::endl;
        cmat U = schmidtU(psi, {2, 2});
        cmat V = schmidtV(psi, {2, 2});
        std::cout << ">> Schmidt vectors on Alice's side: " << std::endl;
        std::cout << disp(U) << std::endl;
        std::cout << ">> Schmidt vectors on Bob's side: " << std::endl;
        std::cout << disp(V) << std::endl;
        std::cout << ">> State psi in the Schmidt basis: " << std::endl;
        std::cout << disp(adjoint(kron(U, V)) * psi) << std::endl;
        // reconstructed state
        ket psi_from_schmidt = schmidtcoeff(psi, {2, 2})(0) * kron(U.col(0), V.col(0)) +
                schmidtcoeff(psi, {2, 2})(1) * kron(U.col(1), V.col(1));
        std::cout << ">> State psi reconstructed from the Schmidt decomposition: " << std::endl;
        std::cout << disp(psi_from_schmidt) << std::endl;
        std::cout << ">> Norm difference: " << norm(psi - psi_from_schmidt) << std::endl;
    }

    // Quantum error correcting codes
    {
        std::cout << std::endl << "**** Quantum error correcting codes ****" << std::endl;
        ket a0 = codes.codeword(Codes::Type::FIVE_QUBIT, 0);
        ket a1 = codes.codeword(Codes::Type::FIVE_QUBIT, 1);
        ket b0 = codes.codeword(Codes::Type::SEVEN_QUBIT_STEANE, 0);
        ket b1 = codes.codeword(Codes::Type::SEVEN_QUBIT_STEANE, 1);
        ket c0 = codes.codeword(Codes::Type::NINE_QUBIT_SHOR, 0);
        ket c1 = codes.codeword(Codes::Type::NINE_QUBIT_SHOR, 1);
        std::cout << ">> Five qubit [[5, 1, 3]] code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(a0) * a1) << std::endl;
        std::cout << ">> Seven qubit [[7, 1, 3]] Steane code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(b0) * b1) << std::endl;
        std::cout << ">> Nine qubit [[9, 1, 3]] Shor code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(c0) * c1) << std::endl;
    }
}
<commit_msg>commit<commit_after>/*
 * Quantum++
 *
 * Copyright (c) 2013 - 2014 Vlad Gheorghiu (vgheorgh@gmail.com)
 *
 * This file is part of Quantum++.
 *
 * Quantum++ is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Quantum++ is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Quantum++.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "qpp.h"

//#include "MATLAB/matlab.h" // support for MATLAB

using namespace qpp;

int main()
{
    // Qudit Teleportation
    {
        std::size_t D = 3; // size of the system
        std::cout << std::endl << "**** Qudit Teleportation, D = " << D << " ****" << std::endl;
        ket mes_AB = ket::Zero(D * D); // maximally entangled state resource
        for (std::size_t i = 0; i < D; i++)
            mes_AB += mket({i, i}, D);
        mes_AB /= std::sqrt((double) D);
        // circuit that measures in the qudit Bell basis
        cmat Bell_aA = adjoint(gt.CTRL(gt.Xd(D), {0}, {1}, 2, D) * kron(gt.Fd(D), gt.Id(D)));
        ket psi_a = randket(D); // random state as input on a
        std::cout << ">> Initial state:" << std::endl;
        std::cout << disp(psi_a) << std::endl;
        ket input_aAB = kron(psi_a, mes_AB); // joint input state aAB
        // output before measurement
        ket output_aAB = apply(input_aAB, Bell_aA, {0, 1}, 3, D);
        auto measured_aA = measure(ptrace2(prj(output_aAB), {D * D, D}), gt.Id(D * D)); // measure on aA
        std::discrete_distribution<std::size_t> dd(measured_aA.first.begin(), measured_aA.first.end());
        std::cout << ">> Alice's measurement probabilities: ";
        std::cout << disp(measured_aA.first, ", ") << std::endl;
        std::size_t m = dd(rdevs._rng); // sample
        auto midx = n2multiidx(m, {D, D});
        std::cout << ">> Alice's measurement result: ";
        std::cout << disp(midx, " ") << std::endl;
        // conditional result on B before correction
        ket output_m_aAB = apply(output_aAB, prj(mket(midx, D)), {0, 1}, 3, D) / std::sqrt(measured_aA.first[m]);
        cmat correction_B = powm(gt.Zd(D), midx[0]) * powm(adjoint(gt.Xd(D)), midx[1]); // correction operator
        // apply correction on B
        output_aAB = apply(output_m_aAB, correction_B, {2}, 3, D);
        cmat rho_B = ptrace1(prj(output_aAB), {D * D, D});
        std::cout << ">> Bob's density operator: " << std::endl;
        std::cout << disp(rho_B) << std::endl;
        std::cout << ">> Norm difference: " << norm(rho_B - prj(psi_a)) << std::endl; // verification
    }

    // Qudit Dense Coding
    {
        std::size_t D = 3; // size of the system
        std::cout << std::endl << "**** Qudit Dense Coding, D = " << D << " ****" << std::endl;
        ket mes_AB = ket::Zero(D * D); // maximally entangled state resource
        for (std::size_t i = 0; i < D; i++)
            mes_AB += mket({i, i}, D);
        mes_AB /= std::sqrt((double) D);
        // circuit that measures in the qudit Bell basis
        cmat Bell_AB = adjoint(gt.CTRL(gt.Xd(D), {0}, {1}, 2, D) * kron(gt.Fd(D), gt.Id(D)));
        // equal probabilities of choosing a message
        std::uniform_int_distribution<std::size_t> uid(0, D * D - 1);
        std::size_t m_A = uid(rdevs._rng); // sample, obtain the message index
        auto midx = n2multiidx(m_A, {D, D});
        std::cout << ">> Alice sent: ";
        std::cout << disp(midx, " ") << std::endl;
        // Alice's operation
        cmat U_A = powm(gt.Zd(D), midx[0]) * powm(adjoint(gt.Xd(D)), midx[1]);
        // Alice encodes the message
        ket psi_AB = apply(mes_AB, U_A, {0}, 2, D);
        // Bob measures the joint system in the qudit Bell basis
        psi_AB = apply(psi_AB, Bell_AB, {0, 1}, 2, D);
        auto measured = measure(psi_AB, gt.Id(D * D));
        std::cout << ">> Bob's measurement probabilities: ";
        std::cout << disp(measured.first, ", ") << std::endl;
        // Bob samples according to the measurement probabilities
        std::discrete_distribution<std::size_t> dd(measured.first.begin(), measured.first.end());
        std::size_t m_B = dd(rdevs._rng);
        std::cout << ">> Bob received: ";
        std::cout << disp(n2multiidx(m_B, {D, D}), " ") << std::endl;
    }

    // Grover's search algorithm, we time it
    {
        Timer t; // set a timer
        std::size_t n = 4; // number of qubits
        std::cout << std::endl << "**** Grover on n = " << n << " qubits ****" << std::endl;
        std::vector<std::size_t> dims(n, 2); // local dimensions
        std::size_t N = std::pow(2, n); // number of elements in the database
        std::cout << ">> Database size: " << N << std::endl;
        // mark an element randomly
        std::uniform_int_distribution<std::size_t> uid(0, N - 1);
        std::size_t marked = uid(rdevs._rng);
        std::cout << ">> Marked state: " << marked << " -> ";
        std::cout << disp(n2multiidx(marked, dims), " ") << std::endl;
        ket psi = mket(n2multiidx(0, dims)); // computational |0>^\otimes n
        psi = (kronpow(gt.H, n) * psi).eval(); // apply H^\otimes n, no aliasing
        cmat G = 2 * prj(psi) - gt.Id(N); // Diffusion operator
        // number of queries
        std::size_t nqueries = std::ceil(pi * std::sqrt(N) / 4.);
        std::cout << ">> We run " << nqueries << " queries" << std::endl;
        for (std::size_t i = 0; i < nqueries; i++)
        {
            psi(marked) = -psi(marked); // apply the oracle first, no aliasing
            psi = (G * psi).eval(); // then the diffusion operator, no aliasing
        }
        // we now measure the state in the computational basis
        auto measured = measure(psi, gt.Id(N));
        std::cout << ">> Probability of the marked state: " << measured.first[marked] << std::endl;
        std::cout << ">> Probability of all results: ";
        std::cout << disp(measured.first, ", ") << std::endl;
        std::cout << ">> Let's sample..." << std::endl;
        std::discrete_distribution<std::size_t> dd(measured.first.begin(), measured.first.end());
        std::size_t result = dd(rdevs._rng);
        if (result == marked)
            std::cout << ">> Hooray, we obtained the correct result: ";
        else
            std::cout << ">> Not there yet... we obtained: ";
        std::cout << result << " -> ";
        std::cout << disp(n2multiidx(result, dims), " ") << std::endl;
        // stop the timer and display it
        std::cout << ">> It took " << t.toc() << " seconds to simulate Grover on " << n << " qubits." << std::endl;
    }

    // Entanglement
    {
        std::cout << std::endl << "**** Entanglement ****" << std::endl;
        cmat rho = 0.2 * st.pb00 + 0.8 * st.pb11;
        std::cout << ">> rho: " << std::endl;
        std::cout << disp(rho) << std::endl;
        std::cout << ">> Concurrence of rho: " << concurrence(rho) << std::endl;
        std::cout << ">> Negativity of rho: " << negativity(rho, {2, 2}) << std::endl;
        std::cout << ">> Logarithimc negativity of rho: " << lognegativity(rho, {2, 2}) << std::endl;
        ket psi = 0.8 * mket({0, 0}) + 0.6 * mket({1, 1});
        // apply some local random unitaries
        psi = kron(randU(2), randU(2)) * psi;
        std::cout << ">> psi: " << std::endl;
        std::cout << disp(psi) << std::endl;
        std::cout << ">> Entanglement of psi: " << entanglement(psi, {2, 2}) << std::endl;
        std::cout << ">> Concurrence of psi: " << concurrence(prj(psi)) << std::endl;
        std::cout << ">> G-Concurrence of psi: " << gconcurrence(psi) << std::endl;
        std::cout << ">> Schmidt coefficients of psi: " << std::endl;
        std::cout << disp(schmidtcoeff(psi, {2, 2})) << std::endl;
        std::cout << ">> Schmidt probabilities of psi: " << std::endl;
        std::cout << disp(schmidtprob(psi, {2, 2})) << std::endl;
        cmat U = schmidtU(psi, {2, 2});
        cmat V = schmidtV(psi, {2, 2});
        std::cout << ">> Schmidt vectors on Alice's side: " << std::endl;
        std::cout << disp(U) << std::endl;
        std::cout << ">> Schmidt vectors on Bob's side: " << std::endl;
        std::cout << disp(V) << std::endl;
        std::cout << ">> State psi in the Schmidt basis: " << std::endl;
        std::cout << disp(adjoint(kron(U, V)) * psi) << std::endl;
        // reconstructed state
        ket psi_from_schmidt = schmidtcoeff(psi, {2, 2})(0) * kron(U.col(0), V.col(0)) +
                schmidtcoeff(psi, {2, 2})(1) * kron(U.col(1), V.col(1));
        std::cout << ">> State psi reconstructed from the Schmidt decomposition: " << std::endl;
        std::cout << disp(psi_from_schmidt) << std::endl;
        std::cout << ">> Norm difference: " << norm(psi - psi_from_schmidt) << std::endl;
    }

    // Quantum error correcting codes
    {
        std::cout << std::endl << "**** Quantum error correcting codes ****" << std::endl;
        ket a0 = codes.codeword(Codes::Type::FIVE_QUBIT, 0);
        ket a1 = codes.codeword(Codes::Type::FIVE_QUBIT, 1);
        ket b0 = codes.codeword(Codes::Type::SEVEN_QUBIT_STEANE, 0);
        ket b1 = codes.codeword(Codes::Type::SEVEN_QUBIT_STEANE, 1);
        ket c0 = codes.codeword(Codes::Type::NINE_QUBIT_SHOR, 0);
        ket c1 = codes.codeword(Codes::Type::NINE_QUBIT_SHOR, 1);
        std::cout << ">> Five qubit [[5, 1, 3]] code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(a0) * a1) << std::endl;
        std::cout << ">> Seven qubit [[7, 1, 3]] Steane code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(b0) * b1) << std::endl;
        std::cout << ">> Nine qubit [[9, 1, 3]] Shor code. ";
        std::cout << "Checking codeword orthogonality." << std::endl;
        std::cout << ">> <0L | 1L> = ";
        std::cout << disp(adjoint(c0) * c1) << std::endl;
    }
}
<|endoftext|>


<commit_before>/**
 * This code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 * (c) Daniel Lemire, http://lemire.me/en/
 */
#include <stdlib.h>
#include "ewah.h"

void demoSerialization() {
    stringstream ss;
    EWAHBoolArray<uint64_t> myarray;
    myarray.add(234321);
    myarray.add(0);
    myarray.add(0);
    myarray.add(999999);
    //
    myarray.write(ss);
    //
    EWAHBoolArray<uint64_t> lmyarray;
    lmyarray.read(ss);
    //
    if (lmyarray == myarray)
        cout << "serialization works" << endl;
    else
        cout << "serialization does not works" << endl;
}

void demoCheckBitValue() {
	/**
	 * We construct a bitmap.
	 */
    EWAHBoolArray<uint64_t> myarray;
    myarray.set(0);
    myarray.set(1);
    myarray.set(999999);
    /**
     * We want to know if bit 1 is set.
     */
    EWAHBoolArray<uint64_t> tmp;
    tmp.set(1);
    EWAHBoolArray<uint64_t> a;
    myarray.logicaland(tmp,a);
    bool is1set = (a.numberOfOnes() == 1);
    /**
     * Yes, this is a bit painful, but it reflects
     * the fact that the format is not meant to
     * support random access.
     */
    cout<< is1set <<endl;
}

int main(void) {
    EWAHBoolArray<uint32_t> bitset1;
    bitset1.set(1);
    bitset1.set(2);
    bitset1.set(1000);
    bitset1.set(1001);
    bitset1.set(1002);
    bitset1.set(1003);
    bitset1.set(1007);
    bitset1.set(1009);
    bitset1.set(100000);
    cout << "first bitset : " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = bitset1.begin(); i
            != bitset1.end(); ++i)
        cout << *i << endl;
    cout << endl;
    EWAHBoolArray<uint32_t> bitset2;
    bitset2.set(1);
    bitset2.set(3);
    bitset2.set(1000);
    bitset2.set(1007);
    bitset2.set(100000);
    cout << "second bitset : " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = bitset2.begin(); i
            != bitset2.end(); ++i)
        cout << *i << endl;
    cout << endl;
    EWAHBoolArray<uint32_t> orbitset;
    EWAHBoolArray<uint32_t> andbitset;
    bitset1.logicalor(bitset2, orbitset);
    bitset1.logicaland(bitset2, andbitset);
    // we will display the or
    cout << "logical and: " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = andbitset.begin(); i
            != andbitset.end(); ++i)
        cout << *i << endl;
    cout << endl;
    cout << "memory usage of compressed bitset = " << andbitset.sizeInBytes()
            << " bytes" << endl;
    cout << endl;
    // we will display the and
    cout << "logical or: " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = orbitset.begin(); i
            != orbitset.end(); ++i)
        cout << *i << endl;
    cout << endl;
    cout << endl;
    cout << "memory usage of compressed bitset = " << orbitset.sizeInBytes()
            << " bytes" << endl;
    cout << endl;
    demoSerialization();
    return EXIT_SUCCESS;
}
<commit_msg>Adding a remark<commit_after>/**
 * This code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 * (c) Daniel Lemire, http://lemire.me/en/
 */
#include <stdlib.h>
#include "ewah.h"

void demoSerialization() {
    stringstream ss;
    EWAHBoolArray<uint64_t> myarray;
    myarray.add(234321);
    myarray.add(0);
    myarray.add(0);
    myarray.add(999999);
    //
    myarray.write(ss);
    //
    EWAHBoolArray<uint64_t> lmyarray;
    lmyarray.read(ss);
    //
    if (lmyarray == myarray)
        cout << "serialization works" << endl;
    else
        cout << "serialization does not works" << endl;
}

void demoCheckBitValue() {
	/**
	 * We construct a bitmap.
	 */
    EWAHBoolArray<uint64_t> myarray;
    myarray.set(0);
    myarray.set(1);
    myarray.set(999999);
    /**
     * We want to know if bit 1 is set.
     */
    EWAHBoolArray<uint64_t> tmp;
    tmp.set(1);
    EWAHBoolArray<uint64_t> a;
    myarray.logicaland(tmp,a);
    bool is1set = (a.numberOfOnes() == 1);
    /**
     * Yes, this is a bit painful, but it reflects
     * the fact that the format is not meant to
     * support random access.
     */
    cout<< is1set <<endl;
    /**
     * Truth: if enough people ask for it, we will
     * provide a convenience method, but with warnings
     * regarding performance.
     */
}

int main(void) {
    EWAHBoolArray<uint32_t> bitset1;
    bitset1.set(1);
    bitset1.set(2);
    bitset1.set(1000);
    bitset1.set(1001);
    bitset1.set(1002);
    bitset1.set(1003);
    bitset1.set(1007);
    bitset1.set(1009);
    bitset1.set(100000);
    cout << "first bitset : " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = bitset1.begin(); i
            != bitset1.end(); ++i)
        cout << *i << endl;
    cout << endl;
    EWAHBoolArray<uint32_t> bitset2;
    bitset2.set(1);
    bitset2.set(3);
    bitset2.set(1000);
    bitset2.set(1007);
    bitset2.set(100000);
    cout << "second bitset : " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = bitset2.begin(); i
            != bitset2.end(); ++i)
        cout << *i << endl;
    cout << endl;
    EWAHBoolArray<uint32_t> orbitset;
    EWAHBoolArray<uint32_t> andbitset;
    bitset1.logicalor(bitset2, orbitset);
    bitset1.logicaland(bitset2, andbitset);
    // we will display the or
    cout << "logical and: " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = andbitset.begin(); i
            != andbitset.end(); ++i)
        cout << *i << endl;
    cout << endl;
    cout << "memory usage of compressed bitset = " << andbitset.sizeInBytes()
            << " bytes" << endl;
    cout << endl;
    // we will display the and
    cout << "logical or: " << endl;
    for (EWAHBoolArray<uint32_t>::const_iterator i = orbitset.begin(); i
            != orbitset.end(); ++i)
        cout << *i << endl;
    cout << endl;
    cout << endl;
    cout << "memory usage of compressed bitset = " << orbitset.sizeInBytes()
            << " bytes" << endl;
    cout << endl;
    demoSerialization();
    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <iostream>
#include <set>
#include <random>
#include <cassert>
#include <fstream>
#include <sstream>
#include <chrono>

using namespace std;

// This is a constant so that we always generate the same file (=> benchmark more repeatable).
unsigned seed = 42;
std::default_random_engine generator(seed);

string getHeaderName(int n) {
  ostringstream stream;
  stream << "X" << n << ".h";
  return stream.str();
}

string getSourceName(int n) {
  ostringstream stream;
  stream << "X" << n << ".cpp";
  return stream.str();
}

string getObjectName(int n) {
  ostringstream stream;
  stream << "X" << n << ".o";
  return stream.str();
}

void add_node(int n, set<int> deps) {
  std::string headerName = getHeaderName(n);
  std::string sourceName = getSourceName(n);

  ofstream headerFile(headerName);
  headerFile << "#include <fruit/fruit.h>" << endl << endl;
  headerFile << "#ifndef X" << n << "_H" << endl;
  headerFile << "#define X" << n << "_H" << endl;
  for (auto dep : deps) {
    headerFile << "#include \"" << getHeaderName(dep) << "\"" << endl;
  }
  headerFile << "struct X" << n << " { INJECT(X" << n << "(";
  for (auto i = deps.begin(), i_end = deps.end(); i != i_end; ++i) {
    if (i != deps.begin()) {
      headerFile << ", ";
    }
    headerFile << "const X" << *i << "&";
  }
  headerFile << ")) {} };" << endl;
  headerFile << "fruit::Component<X" << n << "> getX" << n << "Component();" << endl;
  headerFile << "#endif // X" << n << "_H" << endl;

  ofstream sourceFile(sourceName);
  sourceFile << "#include \"" << headerName << "\"" << endl << endl;
  sourceFile << "fruit::Component<X" << n << "> getX" << n << "Component() {" << endl;
  sourceFile << "  return fruit::createComponent()" << endl;
  for (auto dep : deps) {
    sourceFile << "      .install(getX" << dep << "Component())" << endl;
  }
  sourceFile << "  ;" << endl;
  sourceFile << "}" << endl;
}

set<int> get_random_set(set<int>& pool, size_t desired_size) {
  assert(desired_size <= pool.size());
  set<int> result;
  while (result.size() != desired_size) {
    std::uniform_int_distribution<int> distribution(0, pool.size() - 1);
    int i = distribution(generator);
    auto itr = pool.begin();
    std::advance(itr, i);
    result.insert(*itr);
    pool.erase(itr);
  }
  return result;
}

int main(int argc, char* argv[]) {

  if (argc != 3) {
    cout << "Invalid invocation: " << argv[0];
    for (int i = 1; i < argc; i++) {
      cout << " " << argv[i];
    }
    cout << endl;
    cout << "Usage: " << argv[0] << " /path/to/compiler path/to/fruit/sources/root" << endl;
    return 1;
  }

  constexpr int num_types_with_no_deps = 250;
  constexpr int num_types_with_deps = 30;
  constexpr int num_deps = 8;
  constexpr int num_loops = 20;
  static_assert(num_types_with_no_deps >= num_types_with_deps * num_deps + 1, "Not enough types with no deps");

  int num_used_ids = 0;

  set<int> toplevel_types;

  for (int i = 0; i < num_types_with_no_deps; i++) {
    int id = num_used_ids++;
    add_node(id, {});
    toplevel_types.insert(id);
  }

  for (int i = 0; i < num_types_with_deps; i++) {
    int current_dep_id = num_used_ids++;
    auto deps = get_random_set(toplevel_types, num_deps);
    toplevel_types.insert(current_dep_id);
    add_node(current_dep_id, deps);
  }

  int toplevel_component = num_used_ids++;
  add_node(toplevel_component, toplevel_types);

  ofstream mainFile("main.cpp");
  mainFile << "#include \"" << getHeaderName(toplevel_component) << "\"" << endl;
  mainFile << "#include <ctime>" << endl;
  mainFile << "#include <iostream>" << endl;
  mainFile << "using namespace std;" << endl;

  mainFile << "int main() {" << endl;
  mainFile << "clock_t start_time = clock();" << endl;
  mainFile << "for (int i = 0; i < " << num_loops << "; i++) {" << endl;
  mainFile << "fruit::Injector<X" << toplevel_component << "> injector(getX" << toplevel_component << "Component());" << endl;
  mainFile << "injector.get<X" << toplevel_component << "*>();" << endl;
  mainFile << "}" << endl;
  mainFile << "clock_t end_time = clock();" << endl;
  mainFile << "cout << (end_time - start_time) / " << num_loops << " << endl;" << endl;
  mainFile << "return 0;" << endl;
  mainFile << "}" << endl;

  const string compiler = string(argv[1]) + " -std=c++11 -O2 -g -W -Wall -Werror -DNDEBUG -ftemplate-depth=1000 -I" + argv[2] + "/include";
  vector<string> fruit_srcs = {"component_storage", "demangle_type_name", "injector_storage"};

  ofstream buildFile("build.sh");
  buildFile << "#!/bin/bash" << endl;
  for (int i = 0; i < num_used_ids; i++) {
    buildFile << compiler << " -c " << getSourceName(i) << " -o " << getObjectName(i) << " &" << endl;
    if (i % 20 == 0) {
      // Avoids having too many parallel processes.
      buildFile << "wait || exit 1" << endl;
    }
  }
  buildFile << compiler << " -c main.cpp -o main.o &" << endl;
  for (string s : fruit_srcs) {
    buildFile << compiler << " -c " << argv[2] << "/src/" << s << ".cpp -o " << s << ".o &" << endl;
  }
  buildFile << "wait" << endl;
  buildFile << compiler << " main.o ";
  for (string s : fruit_srcs) {
    buildFile << s << ".o ";
  }
  for (int i = 0; i < num_used_ids; i++) {
    buildFile << getObjectName(i) << " ";
  }
  buildFile << " -o main" << endl;

  return 0;
}
<commit_msg>Update benchmark: change the number of injected types, reduce number of includes.<commit_after>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <iostream>
#include <set>
#include <random>
#include <cassert>
#include <fstream>
#include <sstream>
#include <chrono>

using namespace std;

// This is a constant so that we always generate the same file (=> benchmark more repeatable).
unsigned seed = 42;
std::default_random_engine generator(seed);

string getHeaderName(int n) {
  ostringstream stream;
  stream << "X" << n << ".h";
  return stream.str();
}

string getSourceName(int n) {
  ostringstream stream;
  stream << "X" << n << ".cpp";
  return stream.str();
}

string getObjectName(int n) {
  ostringstream stream;
  stream << "X" << n << ".o";
  return stream.str();
}

void add_node(int n, set<int> deps) {
  std::string headerName = getHeaderName(n);
  std::string sourceName = getSourceName(n);

  ofstream headerFile(headerName);
  headerFile << "#include <fruit/fruit.h>" << endl << endl;
  headerFile << "#ifndef X" << n << "_H" << endl;
  headerFile << "#define X" << n << "_H" << endl;
  for (auto dep : deps) {
    headerFile << "class X" << dep << ";" << endl;
  }
  headerFile << "struct X" << n << " { INJECT(X" << n << "(";
  for (auto i = deps.begin(), i_end = deps.end(); i != i_end; ++i) {
    if (i != deps.begin()) {
      headerFile << ", ";
    }
    headerFile << "const X" << *i << "&";
  }
  headerFile << ")) {} };" << endl;
  headerFile << "fruit::Component<X" << n << "> getX" << n << "Component();" << endl;
  headerFile << "#endif // X" << n << "_H" << endl;

  ofstream sourceFile(sourceName);
  sourceFile << "#include \"" << headerName << "\"" << endl << endl;
  for (auto dep : deps) {
    sourceFile << "#include \"" << getHeaderName(dep) << "\"" << endl;
  }
  sourceFile << "fruit::Component<X" << n << "> getX" << n << "Component() {" << endl;
  sourceFile << "  return fruit::createComponent()" << endl;
  for (auto dep : deps) {
    sourceFile << "      .install(getX" << dep << "Component())" << endl;
  }
  sourceFile << "  ;" << endl;
  sourceFile << "}" << endl;
}

set<int> get_random_set(set<int>& pool, size_t desired_size) {
  assert(desired_size <= pool.size());
  set<int> result;
  while (result.size() != desired_size) {
    std::uniform_int_distribution<int> distribution(0, pool.size() - 1);
    int i = distribution(generator);
    auto itr = pool.begin();
    std::advance(itr, i);
    result.insert(*itr);
    pool.erase(itr);
  }
  return result;
}

int main(int argc, char* argv[]) {

  if (argc != 3) {
    cout << "Invalid invocation: " << argv[0];
    for (int i = 1; i < argc; i++) {
      cout << " " << argv[i];
    }
    cout << endl;
    cout << "Usage: " << argv[0] << " /path/to/compiler path/to/fruit/sources/root" << endl;
    return 1;
  }

  constexpr int num_types_with_no_deps = 91;
  constexpr int num_types_with_deps = 10;
  constexpr int num_deps = 9;
  constexpr int num_loops = 20;
  static_assert(num_types_with_no_deps >= num_types_with_deps * num_deps + 1, "Not enough types with no deps");

  int num_used_ids = 0;

  set<int> toplevel_types;

  for (int i = 0; i < num_types_with_no_deps; i++) {
    int id = num_used_ids++;
    add_node(id, {});
    toplevel_types.insert(id);
  }

  for (int i = 0; i < num_types_with_deps; i++) {
    int current_dep_id = num_used_ids++;
    auto deps = get_random_set(toplevel_types, num_deps);
    toplevel_types.insert(current_dep_id);
    add_node(current_dep_id, deps);
  }

  int toplevel_component = num_used_ids++;
  add_node(toplevel_component, toplevel_types);

  ofstream mainFile("main.cpp");
  mainFile << "#include \"" << getHeaderName(toplevel_component) << "\"" << endl;
  mainFile << "#include <ctime>" << endl;
  mainFile << "#include <iostream>" << endl;
  mainFile << "using namespace std;" << endl;

  mainFile << "int main() {" << endl;
  mainFile << "clock_t start_time = clock();" << endl;
  mainFile << "for (int i = 0; i < " << num_loops << "; i++) {" << endl;
  mainFile << "fruit::Injector<X" << toplevel_component << "> injector(getX" << toplevel_component << "Component());" << endl;
  mainFile << "injector.get<X" << toplevel_component << "*>();" << endl;
  mainFile << "}" << endl;
  mainFile << "clock_t end_time = clock();" << endl;
  mainFile << "cout << (end_time - start_time) / " << num_loops << " << endl;" << endl;
  mainFile << "return 0;" << endl;
  mainFile << "}" << endl;

  const string compiler = string(argv[1]) + " -std=c++11 -O2 -g -W -Wall -Werror -DNDEBUG -ftemplate-depth=1000 -I" + argv[2] + "/include";
  vector<string> fruit_srcs = {"component_storage", "demangle_type_name", "injector_storage"};

  ofstream buildFile("build.sh");
  buildFile << "#!/bin/bash" << endl;
  for (int i = 0; i < num_used_ids; i++) {
    buildFile << compiler << " -c " << getSourceName(i) << " -o " << getObjectName(i) << " &" << endl;
    if (i % 20 == 0) {
      // Avoids having too many parallel processes.
      buildFile << "wait || exit 1" << endl;
    }
  }
  buildFile << compiler << " -c main.cpp -o main.o &" << endl;
  for (string s : fruit_srcs) {
    buildFile << compiler << " -c " << argv[2] << "/src/" << s << ".cpp -o " << s << ".o &" << endl;
  }
  buildFile << "wait" << endl;
  buildFile << compiler << " main.o ";
  for (string s : fruit_srcs) {
    buildFile << s << ".o ";
  }
  for (int i = 0; i < num_used_ids; i++) {
    buildFile << getObjectName(i) << " ";
  }
  buildFile << " -o main" << endl;

  return 0;
}
<|endoftext|>


<commit_before><commit_msg>fix few issues after SAL_INFO conversion<commit_after><|endoftext|>


<commit_before>/*
 * opencog/cython/PythonModule.cc
 *
 * Copyright (C) 2013 by OpenCog Foundation
 * All Rights Reserved
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include "PyIncludeWrapper.h"

#include <boost/filesystem/operations.hpp>

#include <opencog/util/Config.h>
#include <opencog/util/misc.h>
#include <opencog/atomspace/AtomSpace.h>

#include "PyMindAgent.h"
#include "PyRequest.h"
#include "PythonEval.h"
#include "PythonModule.h"

#include "opencog/agent_finder_types.h"
#include "opencog/agent_finder_api.h"

using std::vector;
using std::string;

using namespace opencog;

//#define DPRINTF printf
#define DPRINTF(...)

DECLARE_MODULE(PythonModule);

// Factories

Agent* PythonAgentFactory::create(CogServer& cs) const
{
    logger().info() << "Creating python agent " << _pySrcModuleName << "." << _pyClassName;
    PyMindAgent* pma = new PyMindAgent(cs, _pySrcModuleName, _pyClassName);
    return pma;
}

Request* PythonRequestFactory::create(CogServer& cs) const
{
    logger().info() << "Creating python request " << _pySrcModuleName << "." << _pyClassName;
    PyRequest* pma = new PyRequest(cs, _pySrcModuleName, _pyClassName, _cci);
    return pma;
}

// ------

PythonModule::PythonModule(CogServer& cs) : Module(cs)
{
}

static bool already_loaded = false;

PythonModule::~PythonModule()
{
    logger().info("[PythonModule] destructor");
    unregisterAgentsAndRequests();
    do_load_py_unregister();

    for (PythonAgentFactory* af : _agentFactories) delete af;
    for (PythonRequestFactory* rf : _requestFactories) delete rf;

    already_loaded = false;
}

bool PythonModule::unregisterAgentsAndRequests()
{
    // Requires GIL
    for (std::string s : _agentNames) {
        DPRINTF("Deleting all instances of %s\n", s.c_str());
        _cogserver.unregisterAgent(s);
    }
    for (std::string s : _requestNames) {
        DPRINTF("Unregistering requests of id %s\n", s.c_str());
        _cogserver.unregisterRequest(s);
    }

    return true;
}

void PythonModule::init()
{
    // Avoid hard crash if already loaded.
    if (already_loaded) return;
    already_loaded = true;

    logger().info("[PythonModule] Initialising Python CogServer module.");

    // Make sure that Python has been properly initialized.
    if (not Py_IsInitialized() || not PyEval_ThreadsInitialized()) {
            throw opencog::RuntimeException(TRACE_INFO,
                    "Python not initialized, missing global_python_init()");
    }

    // NOTE: Even though the Cython docs do not say that you need to call this
    // more than once, you need to call the import functions in each separate
    // shared library that accesses Cython defined api. If you don't then you
    // get a crash when you call an api function.
    import_opencog__agent_finder();

    if (config().has("PYTHON_PRELOAD")) preloadModules();
    do_load_py_register();

}

bool PythonModule::preloadModules()
{
    // requires GIL
    std::vector<std::string> pythonmodules;
    tokenize(config()["PYTHON_PRELOAD"], std::back_inserter(pythonmodules), ", ");
    for (std::vector<std::string>::const_iterator it = pythonmodules.begin();
         it != pythonmodules.end(); ++it) {
        std::list<std::string> args;
        args.push_back(*it);
        logger().info("[PythonModule] Preloading python module " + *it);
        std::string result = do_load_py(NULL,args);
        logger().info("[PythonModule] " + result);
    }
    return true;
}

/// do_load_py -- load python code, given a file name. (Implements the loadpy command)
///
/// It is expected that the file contains a python module. The module
/// should implement either a mind-agent, or it should contain a 'request'
/// (shell command, written in python). Mind agents must inherit from
/// the python class opencog.cogserver.MindAgent, while requests/commnds
/// must inherit from opencog.cogserver.Request
//
std::string PythonModule::do_load_py(Request *dummy, std::list<std::string> args)
{
    if (args.empty()) return "Please specify Python module to load.";
    std::string moduleName = args.front();

    std::ostringstream oss;
    if (moduleName.substr(moduleName.size()-3,3) == ".py") {
        oss << "Warning: Python module name should be "
            << "passed without .py extension" << std::endl;
        moduleName.replace(moduleName.size()-3,3,"");
    }

    requests_and_agents_t thingsInModule = load_req_agent_module(moduleName);
    if (thingsInModule.err_string.size() > 0) {
        return thingsInModule.err_string;
    }

    // If there are agents, load them.
    if (thingsInModule.agents.size() > 0) {
        bool first = true;
        oss << "Python MindAgents found: ";
        for (std::string s : thingsInModule.agents) {
            if (!first) {
                oss << ", ";
                first = false;
            }

            // Register agent with cogserver using dotted name:
            // module.AgentName
            std::string dottedName = moduleName + "." + s;

            // register the agent with a custom factory that knows how to
            // instantiate new Python MindAgents
            PythonAgentFactory* afact = new PythonAgentFactory(moduleName,s);
            _agentFactories.push_back(afact);
            _cogserver.registerAgent(dottedName, afact);

            // save a list of Python agents that we've added to the CogServer
            _agentNames.push_back(dottedName);
            oss << s;
        }
        oss << "." << std::endl;
    } else {
        oss << "No subclasses of opencog.cogserver.MindAgent found.\n";
    }

    // Now load the commands/requests, if any.
    if (thingsInModule.requests.size() > 0) {
        bool first = true;
        oss << "Python Requests found: ";
        for (size_t i=0; i< thingsInModule.requests.size(); i++) {
            std::string s = thingsInModule.requests[i];
            std::string short_desc = thingsInModule.req_summary[i];
            std::string long_desc = thingsInModule.req_description[i];
            bool is_shell = thingsInModule.req_is_shell[i];

            // CogServer commands in Python
            // Register request with cogserver using dotted name: module.RequestName
            std::string dottedName = moduleName + "." + s;
            // register the agent with a custom factory that knows how to
            PythonRequestFactory* fact =
                new PythonRequestFactory(moduleName, s, short_desc, long_desc, is_shell);
            _requestFactories.push_back(fact);
            _cogserver.registerRequest(dottedName, fact);
            // save a list of Python agents that we've added to the CogServer
            _requestNames.push_back(dottedName);

            if (!first) { oss << ", "; first = false; }
            oss << s;
        }
        oss << ".";
    } else {
        oss << "No subclasses of opencog.cogserver.Request found.\n";
    }

    // Return info on what requests and mindagents were found
    // This gets printed out to the user at the shell prompt.
    return oss.str();
}

<commit_msg>Remove PythonEval.h include, add missing \n<commit_after>/*
 * opencog/cython/PythonModule.cc
 *
 * Copyright (C) 2013 by OpenCog Foundation
 * All Rights Reserved
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include "PyIncludeWrapper.h"

#include <boost/filesystem/operations.hpp>

#include <opencog/util/Config.h>
#include <opencog/util/misc.h>
#include <opencog/atomspace/AtomSpace.h>

#include "PyMindAgent.h"
#include "PyRequest.h"
#include "PythonModule.h"

#include "opencog/agent_finder_types.h"
#include "opencog/agent_finder_api.h"

using std::vector;
using std::string;

using namespace opencog;

//#define DPRINTF printf
#define DPRINTF(...)

DECLARE_MODULE(PythonModule);

// Factories

Agent* PythonAgentFactory::create(CogServer& cs) const
{
    logger().info() << "Creating python agent " << _pySrcModuleName << "." << _pyClassName;
    PyMindAgent* pma = new PyMindAgent(cs, _pySrcModuleName, _pyClassName);
    return pma;
}

Request* PythonRequestFactory::create(CogServer& cs) const
{
    logger().info() << "Creating python request " << _pySrcModuleName << "." << _pyClassName;
    PyRequest* pma = new PyRequest(cs, _pySrcModuleName, _pyClassName, _cci);
    return pma;
}

// ------

PythonModule::PythonModule(CogServer& cs) : Module(cs)
{
}

static bool already_loaded = false;

PythonModule::~PythonModule()
{
    logger().info("[PythonModule] destructor");
    unregisterAgentsAndRequests();
    do_load_py_unregister();

    for (PythonAgentFactory* af : _agentFactories) delete af;
    for (PythonRequestFactory* rf : _requestFactories) delete rf;

    already_loaded = false;
}

bool PythonModule::unregisterAgentsAndRequests()
{
    // Requires GIL
    for (std::string s : _agentNames) {
        DPRINTF("Deleting all instances of %s\n", s.c_str());
        _cogserver.unregisterAgent(s);
    }
    for (std::string s : _requestNames) {
        DPRINTF("Unregistering requests of id %s\n", s.c_str());
        _cogserver.unregisterRequest(s);
    }

    return true;
}

void PythonModule::init()
{
    // Avoid hard crash if already loaded.
    if (already_loaded) return;
    already_loaded = true;

    logger().info("[PythonModule] Initialising Python CogServer module.");

    // Make sure that Python has been properly initialized.
    if (not Py_IsInitialized() || not PyEval_ThreadsInitialized()) {
            throw opencog::RuntimeException(TRACE_INFO,
                    "Python not initialized, missing global_python_init()");
    }

    // NOTE: Even though the Cython docs do not say that you need to call this
    // more than once, you need to call the import functions in each separate
    // shared library that accesses Cython defined api. If you don't then you
    // get a crash when you call an api function.
    import_opencog__agent_finder();

    if (config().has("PYTHON_PRELOAD")) preloadModules();
    do_load_py_register();

}

bool PythonModule::preloadModules()
{
    // requires GIL
    std::vector<std::string> pythonmodules;
    tokenize(config()["PYTHON_PRELOAD"], std::back_inserter(pythonmodules), ", ");
    for (std::vector<std::string>::const_iterator it = pythonmodules.begin();
         it != pythonmodules.end(); ++it) {
        std::list<std::string> args;
        args.push_back(*it);
        logger().info("[PythonModule] Preloading python module " + *it);
        std::string result = do_load_py(NULL,args);
        logger().info("[PythonModule] " + result);
    }
    return true;
}

/// do_load_py -- load python code, given a file name. (Implements the loadpy command)
///
/// It is expected that the file contains a python module. The module
/// should implement either a mind-agent, or it should contain a 'request'
/// (shell command, written in python). Mind agents must inherit from
/// the python class opencog.cogserver.MindAgent, while requests/commnds
/// must inherit from opencog.cogserver.Request
//
std::string PythonModule::do_load_py(Request *dummy, std::list<std::string> args)
{
    if (args.empty()) return "Please specify Python module to load.";
    std::string moduleName = args.front();

    std::ostringstream oss;
    if (moduleName.substr(moduleName.size()-3,3) == ".py") {
        oss << "Warning: Python module name should be "
            << "passed without .py extension" << std::endl;
        moduleName.replace(moduleName.size()-3,3,"");
    }

    requests_and_agents_t thingsInModule = load_req_agent_module(moduleName);
    if (thingsInModule.err_string.size() > 0) {
        return thingsInModule.err_string;
    }

    // If there are agents, load them.
    if (thingsInModule.agents.size() > 0) {
        bool first = true;
        oss << "Python MindAgents found: ";
        for (std::string s : thingsInModule.agents) {
            if (!first) {
                oss << ", ";
                first = false;
            }

            // Register agent with cogserver using dotted name:
            // module.AgentName
            std::string dottedName = moduleName + "." + s;

            // register the agent with a custom factory that knows how to
            // instantiate new Python MindAgents
            PythonAgentFactory* afact = new PythonAgentFactory(moduleName,s);
            _agentFactories.push_back(afact);
            _cogserver.registerAgent(dottedName, afact);

            // save a list of Python agents that we've added to the CogServer
            _agentNames.push_back(dottedName);
            oss << s;
        }
        oss << "." << std::endl;
    } else {
        oss << "No subclasses of opencog.cogserver.MindAgent found.\n";
    }

    // Now load the commands/requests, if any.
    if (thingsInModule.requests.size() > 0) {
        bool first = true;
        oss << "Python Requests found: ";
        for (size_t i=0; i< thingsInModule.requests.size(); i++) {
            std::string s = thingsInModule.requests[i];
            std::string short_desc = thingsInModule.req_summary[i];
            std::string long_desc = thingsInModule.req_description[i];
            bool is_shell = thingsInModule.req_is_shell[i];

            // CogServer commands in Python
            // Register request with cogserver using dotted name: module.RequestName
            std::string dottedName = moduleName + "." + s;
            // register the agent with a custom factory that knows how to
            PythonRequestFactory* fact =
                new PythonRequestFactory(moduleName, s, short_desc, long_desc, is_shell);
            _requestFactories.push_back(fact);
            _cogserver.registerRequest(dottedName, fact);
            // save a list of Python agents that we've added to the CogServer
            _requestNames.push_back(dottedName);

            if (!first) { oss << ", "; first = false; }
            oss << s;
        }
        oss << ".\n";
    } else {
        oss << "No subclasses of opencog.cogserver.Request found.\n";
    }

    // Return info on what requests and mindagents were found
    // This gets printed out to the user at the shell prompt.
    return oss.str();
}

<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: AccessibleDocumentBase.cxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-08 20:17:31 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _SC_ACCESSIBLEDOCUMENTBASE_HXX
#include "AccessibleDocumentBase.hxx"
#endif

#ifndef _COM_SUN_STAR_ACCESSIBILITY_XACCESSIBLEROLE_HPP_
#include <com/sun/star/accessibility/AccessibleRole.hpp>
#endif

using namespace ::com::sun::star;
using namespace ::com::sun::star::accessibility;

    //=====  internal  ========================================================

ScAccessibleDocumentBase::ScAccessibleDocumentBase(
        const uno::Reference<XAccessible>& rxParent)
    : ScAccessibleContextBase(rxParent, AccessibleRole::DOCUMENT)
{
}

ScAccessibleDocumentBase::~ScAccessibleDocumentBase(void)
{
}
<commit_msg>INTEGRATION: CWS pchfix01 (1.3.216); FILE MERGED 2006/07/12 10:02:17 kaib 1.3.216.1: #i67080# Converted cxx files in sc, added initial project level pch and stripped old PCH definitions.<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: AccessibleDocumentBase.cxx,v $
 *
 *  $Revision: 1.4 $
 *
 *  last change: $Author: kz $ $Date: 2006-07-21 13:05:45 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_sc.hxx"

#ifndef _SC_ACCESSIBLEDOCUMENTBASE_HXX
#include "AccessibleDocumentBase.hxx"
#endif

#ifndef _COM_SUN_STAR_ACCESSIBILITY_XACCESSIBLEROLE_HPP_
#include <com/sun/star/accessibility/AccessibleRole.hpp>
#endif

using namespace ::com::sun::star;
using namespace ::com::sun::star::accessibility;

    //=====  internal  ========================================================

ScAccessibleDocumentBase::ScAccessibleDocumentBase(
        const uno::Reference<XAccessible>& rxParent)
    : ScAccessibleContextBase(rxParent, AccessibleRole::DOCUMENT)
{
}

ScAccessibleDocumentBase::~ScAccessibleDocumentBase(void)
{
}
<|endoftext|>