<commit_before>// tsStego.cpp
// Released under the MIT License
//
// A simple implementation of stegonography in C++
// Uses the LodePNG library version 20140801 by Lode Vandevenne

#include <fstream>
#include <sstream>
#include <iostream>
#include <exception>
#include "lodepng.h"

// Read the plain text file in
void read_text_file(const char* filename, std::vector<unsigned char>& plaintext)
{
	try
	{
		std::fstream text_file(filename);
		while (text_file.good())
		{
			unsigned char c = text_file.get();
			if (text_file.good())
				plaintext.push_back(c);
		}
	}
	catch (...)
	{
		throw std::exception("Exception in read_text_file()");
		return;
	}
}

// Write the plain text file out
void write_text_file(const char* filename, std::vector<unsigned char>& plaintext)
{
	try
	{
		std::ofstream text_file(filename);
		for (auto& c : plaintext)
			text_file.put(c);
	}
	catch (...)
	{
		throw std::exception("Exception in write_text_file()");
		return;
	}
}

// Read the PNG file and save the data into the data structure
// Color values in the vector are 4 bytes per pixel, ordered RGBARGBA...
// On an error, throws an exception
void read_png_from_file(const char* filename, std::vector<unsigned char>& image, unsigned int& width, unsigned int& height)
{
	unsigned int error = 0;
	try
	{
		unsigned int error = lodepng::decode(image, width, height, filename);
	}
	catch (...)
	{
		throw std::exception("Exception in lodepng::decode()");
		return;
	}

	if (error)
	{
		std::stringstream err_desc;
		err_desc << "Decoder error " << error << ": " << lodepng_error_text(error);
		std::string s = err_desc.str();
		throw std::exception(s.c_str());
	}
}

// Write the PNG file from the data structure
// Color values in the vector are 4 bytes per pixel, ordered RGBARGBA...
// On an error, throws an exception
void write_png_to_file(const char* filename, std::vector<unsigned char>& image, unsigned int width, unsigned int height)
{
	unsigned int error = 0;

	try
	{
		unsigned error = lodepng::encode(filename, image, width, height);
	}
	catch (...)
	{
		throw std::exception("Exception in lodepng::decode()");
		return;
	}

	if (error)
	{
		std::stringstream err_desc;
		err_desc << "Encoder error " << error << ": " << lodepng_error_text(error);
		std::string s = err_desc.str();
		throw std::exception(s.c_str());
	}
}

// Take the 8 bits per char and split them 3-2-3, putting 3 bits into the Red, 2 bits into the Green,
// 3 bits into the Blue, and nothing in Alpha
// The bits will be placed starting from the LSB of each byte so as to make the least impact to the
// image when viewed by a human
// We put fewer bits into the Green channel because human eyes are more sensitive to a yellowish-green
void merge_plaintext_into_img_data(std::vector<unsigned char>& plaintext, std::vector<unsigned char>& img_data)
{
	// Bounds check
	// Using 4 * the plaintext size because each element in img_data is a color channel of a pixel, of which
	// there are 4 channels per pixel, and we're going to overwrite certain bits across three of the channels
	if (img_data.size() < 4 * plaintext.size())
		throw std::exception("Exception in merge_plaintext_into_img_data: image is too small to fit all the text");

	unsigned char tmp = 0;
	unsigned int img_index = 0;

	// Loop through all the characters
	for (auto& c : plaintext)
	{
		// First color channel is Red

		// First 3 MSBs from the character are put into the Red channel's 3 LSBs
		tmp = c >> 5; // shift those 3 bits to the right 5 to make them align with the 3 LSBs
		// merge the 5 bits from the original with the 3 shifted MSBs of the text
		img_data[img_index] = (img_data[img_index] & 0xF8) | (tmp & 0x7);

		img_index++; // Next color channel (Blue)

		// Next 3 bits from the character are put into the Blue channel's 3 LSBs
		tmp = c << 3; // shave off the 3 MSBs
		tmp = tmp >> 5; // shift back 5 to make them align with the 3 LSBs
		// merge the 5 bits from the original with the 3 shifted bits of the text
		img_data[img_index] = (img_data[img_index] & 0xF8) | (tmp & 0x7);

		img_index++; // Next color channel (Green)

		// The 2 LSBs from the character are put into the Green channel's 2 LSBs
		tmp = c << 6; // shave off the 6 MSBs
		tmp = tmp >> 6; // shift back 6 to make them align with the 2 LSBs
		// merge the 6 bits from the original with the 2 shifted bits of the text
		img_data[img_index] = (img_data[img_index] & 0xFC) | (tmp & 0x3);

		img_index++; // Next color channel (Alpha)
		img_index++; // Next color channel (Red of the next pixel...so we're ready for next char)
	}
}

// TODO:
//		- Load the PNG file into the image data structure [ DONE ]
//		- Load the plain text file [ DONE ]
//		- Save the image data structure as a new PNG file [ DONE ]
//		- Save the plain text as a new text file [ DONE ]
//		- Handle command line input
//		- Encipher the plain text
//		- Merge the cipher text into the image data structure
//		- Load the enciphered PNG file into the enciphered image data structure
//		- Extract the cipher text from the enciphered image data structure
//		- Decipher to plain text
int main(int argc, char** argv)
{
	// Testing the read file function
	std::vector<unsigned char>plain_text;
	read_text_file("example.txt", plain_text);

	plain_text[3] = 'F';
	write_text_file("output.txt", plain_text);

	// Just testing the LodePNG library for now...input, modification, and output
	std::vector<unsigned char> img_data;
	unsigned int h, w;

	read_png_from_file("planet.png", img_data, w, h);

	merge_plaintext_into_img_data(plain_text, img_data);

	write_png_to_file("output.png", img_data, w, h);

	return 0;
}<commit_msg>Modified merge function to support XOR Passing a usage flag allows the text  to be merged into the image using an XOR operation, requiring the original image to extract properly<commit_after>// tsStego.cpp
// Released under the MIT License
//
// A simple implementation of stegonography in C++
// Uses the LodePNG library version 20140801 by Lode Vandevenne

#include <fstream>
#include <sstream>
#include <iostream>
#include <exception>
#include "lodepng.h"

// Read the plain text file in
void read_text_file(const char* filename, std::vector<unsigned char>& plaintext)
{
	try
	{
		std::fstream text_file(filename);
		while (text_file.good())
		{
			unsigned char c = text_file.get();
			if (text_file.good())
				plaintext.push_back(c);
		}
	}
	catch (...)
	{
		throw std::exception("Exception in read_text_file()");
		return;
	}
}

// Write the plain text file out
void write_text_file(const char* filename, std::vector<unsigned char>& plaintext)
{
	try
	{
		std::ofstream text_file(filename);
		for (auto& c : plaintext)
			text_file.put(c);
	}
	catch (...)
	{
		throw std::exception("Exception in write_text_file()");
		return;
	}
}

// Read the PNG file and save the data into the data structure
// Color values in the vector are 4 bytes per pixel, ordered RGBARGBA...
// On an error, throws an exception
void read_png_from_file(const char* filename, std::vector<unsigned char>& image, unsigned int& width, unsigned int& height)
{
	unsigned int error = 0;
	try
	{
		unsigned int error = lodepng::decode(image, width, height, filename);
	}
	catch (...)
	{
		throw std::exception("Exception in lodepng::decode()");
		return;
	}

	if (error)
	{
		std::stringstream err_desc;
		err_desc << "Decoder error " << error << ": " << lodepng_error_text(error);
		std::string s = err_desc.str();
		throw std::exception(s.c_str());
	}
}

// Write the PNG file from the data structure
// Color values in the vector are 4 bytes per pixel, ordered RGBARGBA...
// On an error, throws an exception
void write_png_to_file(const char* filename, std::vector<unsigned char>& image, unsigned int width, unsigned int height)
{
	unsigned int error = 0;

	try
	{
		unsigned error = lodepng::encode(filename, image, width, height);
	}
	catch (...)
	{
		throw std::exception("Exception in lodepng::decode()");
		return;
	}

	if (error)
	{
		std::stringstream err_desc;
		err_desc << "Encoder error " << error << ": " << lodepng_error_text(error);
		std::string s = err_desc.str();
		throw std::exception(s.c_str());
	}
}

// Take the 8 bits per char and split them 3-2-3, putting 3 bits into the Red, 2 bits into the Green,
// 3 bits into the Blue, and nothing in Alpha
// The bits will be placed starting from the LSB of each byte so as to make the least impact to the
// image when viewed by a human
// We put fewer bits into the Green channel because human eyes are more sensitive to a yellowish-green
// The using_XOR flag allows the merge to either overwrite the destination bits in the img data, or XOR
// the source bits (from the text data) with the destination
// Throws exception on error
void merge_text_into_img_data(std::vector<unsigned char>& plaintext, std::vector<unsigned char>& img_data, bool using_XOR=false)
{
	// Bounds check
	// Using 4 * the plaintext size because each element in img_data is a color channel of a pixel, of which
	// there are 4 channels per pixel, and we're going to overwrite certain bits across three of the channels
	if (img_data.size() < 4 * plaintext.size())
		throw std::exception("Exception in merge_plaintext_into_img_data: image is too small to fit all the text");

	unsigned char tmp = 0;
	unsigned int img_index = 0;

	// Loop through all the characters
	for (auto& c : plaintext)
	{
		// First color channel is Red

		// First 3 MSBs from the character are put into the Red channel's 3 LSBs
		tmp = c >> 5; // shift those 3 bits to the right 5 to make them align with the 3 LSBs

		// Depending on the XOR state flag, either overwrite the data or XOR the text into it
		if (!using_XOR)
		{
			// merge the 5 bits from the original with the 3 shifted MSBs of the text
			img_data[img_index] = (img_data[img_index] & 0xF8) | (tmp & 0x7);
		}
		else // Otherwise XOR the bits in, requiring the original image's pixel data to extract
			img_data[img_index] ^= tmp;

		img_index++; // Next color channel (Blue)

		// Next 3 bits from the character are put into the Blue channel's 3 LSBs
		tmp = c << 3; // shave off the 3 MSBs
		tmp = tmp >> 5; // shift back 5 to make them align with the 3 LSBs
		// Depending on the XOR state flag, either overwrite the data or XOR the text into it
		if (!using_XOR)
		{
			// merge the 5 bits from the original with the 3 shifted MSBs of the text
			img_data[img_index] = (img_data[img_index] & 0xF8) | (tmp & 0x7);
		}
		else // Otherwise XOR the bits in, requiring the original image's pixel data to extract
			img_data[img_index] ^= tmp;

		img_index++; // Next color channel (Green)

		// The 2 LSBs from the character are put into the Green channel's 2 LSBs
		tmp = c << 6; // shave off the 6 MSBs
		tmp = tmp >> 6; // shift back 6 to make them align with the 2 LSBs
		// Depending on the XOR state flag, either overwrite the data or XOR the text into it
		if (!using_XOR)
		{
			// merge the 5 bits from the original with the 3 shifted MSBs of the text
			img_data[img_index] = (img_data[img_index] & 0xF8) | (tmp & 0x7);
		}
		else // Otherwise XOR the bits in, requiring the original image's pixel data to extract
			img_data[img_index] ^= tmp;

		img_index++; // Next color channel (Alpha)
		img_index++; // Next color channel (Red of the next pixel...so we're ready for next char)
	}
}

// TODO:
//		- Load the PNG file into the image data structure [ DONE ]
//		- Load the plain text file [ DONE ]
//		- Save the image data structure as a new PNG file [ DONE ]
//		- Save the plain text as a new text file [ DONE ]
//		- Handle command line input
//		- Encipher the plain text
//		- Merge the cipher text into the image data structure
//		- Load the enciphered PNG file into the enciphered image data structure
//		- Extract the cipher text from the enciphered image data structure
//		- Decipher to plain text
int main(int argc, char** argv)
{
	// Testing the read file function
	std::vector<unsigned char>plain_text;
	read_text_file("example.txt", plain_text);

	plain_text[3] = 'F';
	write_text_file("output.txt", plain_text);

	// Just testing the LodePNG library for now...input, modification, and output
	std::vector<unsigned char> img_data;
	unsigned int h, w;

	read_png_from_file("planet.png", img_data, w, h);

	merge_text_into_img_data(plain_text, img_data);

	write_png_to_file("output.png", img_data, w, h);

	return 0;
}<|endoftext|>


<commit_before>/**
Copyright (c) 2013, Philip Deegan.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
    * Neither the name of Philip Deegan nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _KUL_SCM_HPP_
#define _KUL_SCM_HPP_

#include "kul/os.hpp"
#include "kul/hash.hpp"
#include "kul/proc.hpp"
#include "kul/string.hpp"

namespace kul{ namespace scm{

class Exception : public kul::Exception{
	public:
		Exception(const char*f, const int l, const std::string& s) : kul::Exception(f, l, s){}
};

class NotFoundException : public kul::Exception{
	public:
		NotFoundException(const char*f, const int l, const std::string& s) : kul::Exception(f, l, s){}
};
}

class SCM{
	protected:
		SCM(){}
	public:
		virtual ~SCM(){}
		const std::string type() { return typeid(*this).name(); }
		virtual const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception) = 0;
		virtual void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception) = 0;
		virtual const std::string origin(const std::string& d) const = 0;
		virtual const std::string localVersion(const std::string& d, const std::string& b) const = 0;
		virtual const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& branch) const throw(Exception) = 0;

		virtual bool hasChanges(const std::string& d) const = 0;

		// virtual void setOrigin(const std::string& d, const std::string& r) const = 0;
		virtual void status(const std::string& d) const = 0;
		virtual void diff(const std::string& d) const = 0;
};

namespace scm{
class Git : public SCM{
	public:
		const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			Dir dr(d, true);
			kul::Process p("git");
			p.arg("clone").arg(r);
			if(!v.empty()) p.arg("-b").arg(v);
			try{
				p.arg(d).start();
			}catch(const kul::proc::ExitException& e){
				dr.rm();
				KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
			}
			return p.toString();
		}
		void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			if(!Dir(d).is()) co(d, r, v);
			else{
				kul::Process p("git", d);
				p.arg("pull");
				if(!v.empty()) p.arg(r).arg(v);
				try{
					p.start();
				}catch(const kul::proc::ExitException& e){
					KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
				}
			}
		}
		const std::string origin(const std::string& d) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("remote").arg("-v").start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR: Directory may not be git repository : " + d);
			return kul::String::split(kul::String::split(kul::String::lines(pc.outs())[0], "	")[1], " ")[0];
		}
		const std::string localVersion(const std::string& d, const std::string& b) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("rev-parse").arg(b).start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR: Directory may not be git repository : " + d);
			return kul::String::lines(pc.outs())[0];
		}

		const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& b) const throw(Exception){
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("ls-remote").arg(url).arg(b).start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR URL OR BRANCH MAY NOT EXIST: " + url + " / " + b);
			std::string s(kul::String::lines(pc.outs())[0]);
			kul::String::trim(s);
			return s.substr(0, s.find("	"));
		}

		bool hasChanges(const std::string& d) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("status").arg("-sb").start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			return kul::String::lines(pc.outs()).size() > 1;
		}
		void status(const std::string& d) const{
			kul::Process p("git", d);
			try{
				p.arg("status").start();
			}catch(const kul::proc::ExitException& e){
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
		}
		void diff(const std::string& d) const{
			kul::Process p("git", d);
			try{
				p.arg("diff").start();
			}catch(const kul::proc::ExitException& e){
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
		}
		// void setOrigin(const std::string& d, const std::string& r) const{
		// 	kul::Process p("git", d);
		// 	kul::ProcessCapture pc(p);
		// 	try{
		// 		p.arg("remote").arg("set-url").arg(r).start();
		// 	}catch(const kul::proc::ExitException& e){
		// 		KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
		// 	}
		// }
};

// SVN URL CHANGE: svn switch –relocate  <from URL> <to URL>
class Svn : public SCM{
	public:
		const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			Dir dr(d, true);
			kul::Process p("svn", d);
			p.arg("checkout");
			if(v.empty()) p.arg(r);
			else p.arg(r + "/" + v);
			try{
				p.arg(".").start();
			}catch(const kul::proc::ExitException& e){
				dr.rm();
				KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
			}
			return p.toString();
		};
		void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			if(!Dir(d).is()) co(d, r, v);
			else KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		};
		const std::string origin(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		const std::string localVersion(const std::string& d, const std::string& b) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& b) const throw(Exception){
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}

		bool hasChanges(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		void status(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		void diff(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		// void setOrigin(const std::string& d, const std::string& r) const;
};

class Manager{
	private:
		Manager(){
			git = std::make_unique<Git>();
			svn = std::make_unique<Svn>();
			SCMs.insert(std::pair<std::string, SCM*>("git", git.get()));
			SCMs.insert(std::pair<std::string, SCM*>("svn", svn.get()));
		}
		hash::map::S2T<SCM*> SCMs;
		std::unique_ptr<SCM> git;
		std::unique_ptr<SCM> svn;
	public:
		static Manager& INSTANCE(){
			static Manager instance;
			return instance;
		}
		const SCM& get(const std::string& s) throw(NotFoundException){
			if(SCMs.count(s) > 0) return *(*SCMs.find(s)).second;
			KEXCEPT(NotFoundException, "Source Control Management for " + s + " is not implemented");
		}
};

}// END NAMESPACE scm
}// END NAMESPACE kul
#endif /* _KUL_SCM_HPP_ */
<commit_msg>scm defaults<commit_after>/**
Copyright (c) 2013, Philip Deegan.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
    * Neither the name of Philip Deegan nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _KUL_SCM_HPP_
#define _KUL_SCM_HPP_

#include "kul/os.hpp"
#include "kul/hash.hpp"
#include "kul/proc.hpp"
#include "kul/string.hpp"

namespace kul{ namespace scm{

class Exception : public kul::Exception{
	public:
		Exception(const char*f, const int l, const std::string& s) : kul::Exception(f, l, s){}
};

class NotFoundException : public kul::Exception{
	public:
		NotFoundException(const char*f, const int l, const std::string& s) : kul::Exception(f, l, s){}
};
}

class SCM{
	protected:
		SCM(){}
	public:
		virtual ~SCM(){}
		const std::string type() { return typeid(*this).name(); }
		virtual const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception) = 0;
		virtual void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception) = 0;
		virtual const std::string origin(const std::string& d) const = 0;
		virtual const std::string localVersion(const std::string& d, const std::string& b) const = 0;
		virtual const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& branch) const throw(Exception) = 0;

		virtual bool hasChanges(const std::string& d) const = 0;

		// virtual void setOrigin(const std::string& d, const std::string& r) const = 0;
		virtual void status(const std::string& d) const = 0;
		virtual void diff(const std::string& d) const = 0;
};

namespace scm{
class Git : public SCM{
	public:
		const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			Dir dr(d, true);
			kul::Process p("git");
			p.arg("clone").arg(r);
			if(!v.empty()) p.arg("-b").arg(v);
			try{
				p.arg(d).start();
			}catch(const kul::proc::ExitException& e){
				dr.rm();
				KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
			}
			return p.toString();
		}
		void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			if(!Dir(d).is()) co(d, r, v);
			else{
				kul::Process p("git", d);
				p.arg("pull");
				if(!r.empty()) p.arg(r);
				if(!r.empty() && !v.empty()) p.arg(v);
				try{
					p.start();
				}catch(const kul::proc::ExitException& e){
					KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
				}
			}
		}
		const std::string origin(const std::string& d) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("remote").arg("-v").start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR: Directory may not be git repository : " + d);
			return kul::String::split(kul::String::split(kul::String::lines(pc.outs())[0], "	")[1], " ")[0];
		}
		const std::string localVersion(const std::string& d, const std::string& b) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("rev-parse").arg(b.empty() ? "HEAD" : b).start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR: Directory may not be git repository : " + d);
			return kul::String::lines(pc.outs())[0];
		}

		const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& b) const throw(Exception){
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("ls-remote").arg(url).arg(b).start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			if(pc.outs().empty()) KEXCEPT(Exception, "SCM ERROR URL OR BRANCH MAY NOT EXIST: " + url + " / " + b);
			std::string s(kul::String::lines(pc.outs())[0]);
			kul::String::trim(s);
			return s.substr(0, s.find("	"));
		}

		bool hasChanges(const std::string& d) const{
			kul::Process p("git", d);
			kul::ProcessCapture pc(p);
			try{
				p.arg("status").arg("-sb").start();
			}catch(const kul::proc::ExitException& e){
				KERR << pc.errs();
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
			return kul::String::lines(pc.outs()).size() > 1;
		}
		void status(const std::string& d) const{
			kul::Process p("git", d);
			try{
				p.arg("status").start();
			}catch(const kul::proc::ExitException& e){
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
		}
		void diff(const std::string& d) const{
			kul::Process p("git", d);
			try{
				p.arg("diff").start();
			}catch(const kul::proc::ExitException& e){
				KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
			}
		}
		// void setOrigin(const std::string& d, const std::string& r) const{
		// 	kul::Process p("git", d);
		// 	kul::ProcessCapture pc(p);
		// 	try{
		// 		p.arg("remote").arg("set-url").arg(r).start();
		// 	}catch(const kul::proc::ExitException& e){
		// 		KEXCEPT(Exception, "SCM ERROR " + std::string(e.what()));
		// 	}
		// }
};

class Svn : public SCM{
	public:
		const std::string co(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			Dir dr(d, true);
			kul::Process p("svn", d);
			p.arg("checkout");
			if(v.empty()) p.arg(r);
			else p.arg(r + "/" + v);
			try{
				p.arg(".").start();
			}catch(const kul::proc::ExitException& e){
				dr.rm();
				KEXCEPT(Exception, "SCM ERROR - Check remote dependency location / version");
			}
			return p.toString();
		}
		void up(const std::string& d, const std::string& r, const std::string& v) const throw(Exception){
			if(!Dir(d).is()) co(d, r, v);
			else KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		// svn info . | lines | split | ifeq "Repository Root" go | http://svnbook.red-bean.com/en/1.6/svn.ref.svn.c.info.html
		const std::string origin(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		const std::string localVersion(const std::string& d, const std::string& b) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		const std::string remoteVersion(const std::string& d, const std::string& url, const std::string& b) const throw(Exception){
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}

		bool hasChanges(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		void status(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		void diff(const std::string& d) const{
			KEXCEPT(Exception, "SCM ERROR - SVN NOT IMPLEMENTED");
		}
		// void setOrigin(const std::string& d, const std::string& r) const;
};

class Manager{
	private:
		Manager(){
			git = std::make_unique<Git>();
			svn = std::make_unique<Svn>();
			SCMs.insert(std::pair<std::string, SCM*>("git", git.get()));
			SCMs.insert(std::pair<std::string, SCM*>("svn", svn.get()));
		}
		hash::map::S2T<SCM*> SCMs;
		std::unique_ptr<SCM> git;
		std::unique_ptr<SCM> svn;
	public:
		static Manager& INSTANCE(){
			static Manager instance;
			return instance;
		}
		const SCM& get(const std::string& s) throw(NotFoundException){
			if(SCMs.count(s) > 0) return *(*SCMs.find(s)).second;
			KEXCEPT(NotFoundException, "Source Control Management for " + s + " is not implemented");
		}
};

}// END NAMESPACE scm
}// END NAMESPACE kul
#endif /* _KUL_SCM_HPP_ */
<|endoftext|>


<commit_before>// Copyright 2010 Google Inc. All Rights Reserved.
// Author: rays@google.com (Ray Smith)
///////////////////////////////////////////////////////////////////////
// File:        intfeaturemap.cpp
// Description: Encapsulation of IntFeatureSpace with IndexMapBiDi
//              to provide a subspace mapping and fast feature lookup.
// Created:     Tue Oct 26 08:58:30 PDT 2010
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#include "intfeaturemap.h"

#include "intfeaturespace.h"
#include "intfx.h"
// These includes do not exist yet, but will be coming soon.
//#include "sampleiterator.h"
//#include "trainingsample.h"
//#include "trainingsampleset.h"

namespace tesseract {

const int kMaxOffsetDist = 32;
const double kMinPCLengthIncrease = 1.0 / 1024;

IntFeatureMap::IntFeatureMap()
  : mapping_changed_(true), compact_size_(0) {
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    offset_plus_[dir] = NULL;
    offset_minus_[dir] = NULL;
  }
}

IntFeatureMap::~IntFeatureMap() {
  Clear();
}

// Pseudo-accessors.
int IntFeatureMap::IndexFeature(const INT_FEATURE_STRUCT& f) const {
  return feature_space_.Index(f);
}
int IntFeatureMap::MapFeature(const INT_FEATURE_STRUCT& f) const {
  return feature_map_.SparseToCompact(feature_space_.Index(f));
}
int IntFeatureMap::MapIndexFeature(int index_feature) const {
  return feature_map_.SparseToCompact(index_feature);
}
INT_FEATURE_STRUCT IntFeatureMap::InverseIndexFeature(int index_feature) const {
  return feature_space_.PositionFromIndex(index_feature);
}
INT_FEATURE_STRUCT IntFeatureMap::InverseMapFeature(int map_feature) const {
  int index = feature_map_.CompactToSparse(map_feature);
  return feature_space_.PositionFromIndex(index);
}
void IntFeatureMap::DeleteMapFeature(int map_feature) {
  feature_map_.Merge(-1, map_feature);
  mapping_changed_ = true;
}
bool IntFeatureMap::IsMapFeatureDeleted(int map_feature) const {
  return feature_map_.IsCompactDeleted(map_feature);
}

// Copies the given feature_space and uses it as the index feature map
// from INT_FEATURE_STRUCT.
void IntFeatureMap::Init(const IntFeatureSpace& feature_space) {
  feature_space_ = feature_space;
  mapping_changed_ = false;
  int sparse_size = feature_space_.Size();
  feature_map_.Init(sparse_size, true);
  feature_map_.Setup();
  compact_size_ = feature_map_.CompactSize();
  // Initialize look-up tables if needed.
  FCOORD dir = FeatureDirection(0);
  if (dir.x() == 0.0f && dir.y() == 0.0f)
    InitIntegerFX();
  // Compute look-up tables to generate offset features.
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    delete [] offset_plus_[dir];
    delete [] offset_minus_[dir];
    offset_plus_[dir] = new int[sparse_size];
    offset_minus_[dir] = new int[sparse_size];
  }
  for (int dir = 1; dir <= kNumOffsetMaps; ++dir) {
    for (int i = 0; i < sparse_size; ++i) {
      int offset_index = ComputeOffsetFeature(i, dir);
      offset_plus_[dir - 1][i] = offset_index;
      offset_index = ComputeOffsetFeature(i, -dir);
      offset_minus_[dir - 1][i] = offset_index;
    }
  }
}

// Helper to return an offset index feature. In this context an offset
// feature with a dir of +/-1 is a feature of a similar direction,
// but shifted perpendicular to the direction of the feature. An offset
// feature with a dir of +/-2 is feature at the same position, but rotated
// by +/- one [compact] quantum. Returns the index of the generated offset
// feature, or -1 if it doesn't exist. Dir should be in
// [-kNumOffsetMaps, kNumOffsetMaps] to indicate the relative direction.
// A dir of 0 is an identity transformation.
// Both input and output are from the index(sparse) feature space, not
// the mapped/compact feature space, but the offset feature is the minimum
// distance moved from the input to guarantee that it maps to the next
// available quantum in the mapped/compact space.
int IntFeatureMap::OffsetFeature(int index_feature, int dir) const {
  if (dir > 0 && dir <= kNumOffsetMaps)
    return offset_plus_[dir - 1][index_feature];
  else if (dir < 0 && -dir <= kNumOffsetMaps)
    return offset_minus_[-dir - 1][index_feature];
  else if (dir == 0)
    return index_feature;
  else
    return -1;
}

//#define EXPERIMENT_ON
#ifdef EXPERIMENT_ON  // This code is commented out as SampleIterator and
// TrainingSample are not reviewed/checked in yet, but these functions are a
// useful indicator of how an IntFeatureMap is setup.

// Computes the features used by the subset of samples defined by
// the iterator and sets up the feature mapping.
// Returns the size of the compacted feature space.
int IntFeatureMap::FindNZFeatureMapping(SampleIterator* it) {
  feature_map_.Init(feature_space_.Size(), false);
  int total_samples = 0;
  for (it->Begin(); !it->AtEnd(); it->Next()) {
    const TrainingSample& sample = it->GetSample();
    GenericVector<int> features;
    feature_space_.IndexAndSortFeatures(sample.features(),
                                        sample.num_features(),
                                        &features);
    int num_features = features.size();
    for (int f = 0; f < num_features; ++f)
      feature_map_.SetMap(features[f], true);
    ++total_samples;
  }
  feature_map_.Setup();
  compact_size_ = feature_map_.CompactSize();
  mapping_changed_ = true;
  FinalizeMapping(it);
  tprintf("%d non-zero features found in %d samples\n",
          compact_size_, total_samples);
  return compact_size_;
}
#endif

// After deleting some features, finish setting up the mapping, and map
// all the samples. Returns the size of the compacted feature space.
int IntFeatureMap::FinalizeMapping(SampleIterator* it) {
  if (mapping_changed_) {
    feature_map_.CompleteMerges();
    compact_size_ = feature_map_.CompactSize();
#ifdef EXPERIMENT_ON
    it->MapSampleFeatures(*this);
#endif
    mapping_changed_ = false;
  }
  return compact_size_;
}

// Prints the map features from the set in human-readable form.
void IntFeatureMap::DebugMapFeatures(
    const GenericVector<int>& map_features) const {
  for (int i = 0; i < map_features.size(); ++i) {
    INT_FEATURE_STRUCT f = InverseMapFeature(map_features[i]);
    f.print();
  }
}

void IntFeatureMap::Clear() {
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    delete [] offset_plus_[dir];
    delete [] offset_minus_[dir];
    offset_plus_[dir] = NULL;
    offset_minus_[dir] = NULL;
  }
}

// Helper to compute an offset index feature. In this context an offset
// feature with a dir of +/-1 is a feature of a similar direction,
// but shifted perpendicular to the direction of the feature. An offset
// feature with a dir of +/-2 is feature at the same position, but rotated
// by +/- one [compact] quantum. Returns the index of the generated offset
// feature, or -1 if it doesn't exist. Dir should be in
// [-kNumOffsetMaps, kNumOffsetMaps] to indicate the relative direction.
// A dir of 0 is an identity transformation.
// Both input and output are from the index(sparse) feature space, not
// the mapped/compact feature space, but the offset feature is the minimum
// distance moved from the input to guarantee that it maps to the next
// available quantum in the mapped/compact space.
int IntFeatureMap::ComputeOffsetFeature(int index_feature, int dir) const {
  INT_FEATURE_STRUCT f = InverseIndexFeature(index_feature);
  ASSERT_HOST(IndexFeature(f) == index_feature);
  if (dir == 0) {
    return index_feature;
  } else if (dir == 1 || dir == -1) {
    FCOORD feature_dir = FeatureDirection(f.Theta);
    FCOORD rotation90(0.0f, 1.0f);
    feature_dir.rotate(rotation90);
    // Find the nearest existing feature.
    for (int m = 1; m < kMaxOffsetDist; ++m) {
      double x_pos = f.X + feature_dir.x() * (m * dir);
      double y_pos = f.Y + feature_dir.y() * (m * dir);
      int x = IntCastRounded(x_pos);
      int y = IntCastRounded(y_pos);
      if (x >= 0 && x <= MAX_UINT8 && y >= 0 && y <= MAX_UINT8) {
        INT_FEATURE_STRUCT offset_f;
        offset_f.X = x;
        offset_f.Y = y;
        offset_f.Theta = f.Theta;
        int offset_index = IndexFeature(offset_f);
        if (offset_index != index_feature && offset_index >= 0)
          return offset_index;  // Found one.
      } else {
        return -1;  // Hit the edge of feature space.
      }
    }
  } else if (dir == 2 || dir == -2) {
    // Find the nearest existing index_feature.
    for (int m = 1; m < kMaxOffsetDist; ++m) {
      int theta = f.Theta + m * dir / 2;
      INT_FEATURE_STRUCT offset_f;
      offset_f.X = f.X;
      offset_f.Y = f.Y;
      offset_f.Theta = Modulo(theta, 256);
      int offset_index = IndexFeature(offset_f);
      if (offset_index != index_feature && offset_index >= 0)
        return offset_index;  // Found one.
    }
  }
  return -1;  // Nothing within the max distance.
}

}  // namespace tesseract.
<commit_msg>classify: Remove unused constant kMinPCLengthIncrease<commit_after>// Copyright 2010 Google Inc. All Rights Reserved.
// Author: rays@google.com (Ray Smith)
///////////////////////////////////////////////////////////////////////
// File:        intfeaturemap.cpp
// Description: Encapsulation of IntFeatureSpace with IndexMapBiDi
//              to provide a subspace mapping and fast feature lookup.
// Created:     Tue Oct 26 08:58:30 PDT 2010
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
///////////////////////////////////////////////////////////////////////

#include "intfeaturemap.h"

#include "intfeaturespace.h"
#include "intfx.h"
// These includes do not exist yet, but will be coming soon.
//#include "sampleiterator.h"
//#include "trainingsample.h"
//#include "trainingsampleset.h"

namespace tesseract {

const int kMaxOffsetDist = 32;

IntFeatureMap::IntFeatureMap()
  : mapping_changed_(true), compact_size_(0) {
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    offset_plus_[dir] = NULL;
    offset_minus_[dir] = NULL;
  }
}

IntFeatureMap::~IntFeatureMap() {
  Clear();
}

// Pseudo-accessors.
int IntFeatureMap::IndexFeature(const INT_FEATURE_STRUCT& f) const {
  return feature_space_.Index(f);
}
int IntFeatureMap::MapFeature(const INT_FEATURE_STRUCT& f) const {
  return feature_map_.SparseToCompact(feature_space_.Index(f));
}
int IntFeatureMap::MapIndexFeature(int index_feature) const {
  return feature_map_.SparseToCompact(index_feature);
}
INT_FEATURE_STRUCT IntFeatureMap::InverseIndexFeature(int index_feature) const {
  return feature_space_.PositionFromIndex(index_feature);
}
INT_FEATURE_STRUCT IntFeatureMap::InverseMapFeature(int map_feature) const {
  int index = feature_map_.CompactToSparse(map_feature);
  return feature_space_.PositionFromIndex(index);
}
void IntFeatureMap::DeleteMapFeature(int map_feature) {
  feature_map_.Merge(-1, map_feature);
  mapping_changed_ = true;
}
bool IntFeatureMap::IsMapFeatureDeleted(int map_feature) const {
  return feature_map_.IsCompactDeleted(map_feature);
}

// Copies the given feature_space and uses it as the index feature map
// from INT_FEATURE_STRUCT.
void IntFeatureMap::Init(const IntFeatureSpace& feature_space) {
  feature_space_ = feature_space;
  mapping_changed_ = false;
  int sparse_size = feature_space_.Size();
  feature_map_.Init(sparse_size, true);
  feature_map_.Setup();
  compact_size_ = feature_map_.CompactSize();
  // Initialize look-up tables if needed.
  FCOORD dir = FeatureDirection(0);
  if (dir.x() == 0.0f && dir.y() == 0.0f)
    InitIntegerFX();
  // Compute look-up tables to generate offset features.
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    delete [] offset_plus_[dir];
    delete [] offset_minus_[dir];
    offset_plus_[dir] = new int[sparse_size];
    offset_minus_[dir] = new int[sparse_size];
  }
  for (int dir = 1; dir <= kNumOffsetMaps; ++dir) {
    for (int i = 0; i < sparse_size; ++i) {
      int offset_index = ComputeOffsetFeature(i, dir);
      offset_plus_[dir - 1][i] = offset_index;
      offset_index = ComputeOffsetFeature(i, -dir);
      offset_minus_[dir - 1][i] = offset_index;
    }
  }
}

// Helper to return an offset index feature. In this context an offset
// feature with a dir of +/-1 is a feature of a similar direction,
// but shifted perpendicular to the direction of the feature. An offset
// feature with a dir of +/-2 is feature at the same position, but rotated
// by +/- one [compact] quantum. Returns the index of the generated offset
// feature, or -1 if it doesn't exist. Dir should be in
// [-kNumOffsetMaps, kNumOffsetMaps] to indicate the relative direction.
// A dir of 0 is an identity transformation.
// Both input and output are from the index(sparse) feature space, not
// the mapped/compact feature space, but the offset feature is the minimum
// distance moved from the input to guarantee that it maps to the next
// available quantum in the mapped/compact space.
int IntFeatureMap::OffsetFeature(int index_feature, int dir) const {
  if (dir > 0 && dir <= kNumOffsetMaps)
    return offset_plus_[dir - 1][index_feature];
  else if (dir < 0 && -dir <= kNumOffsetMaps)
    return offset_minus_[-dir - 1][index_feature];
  else if (dir == 0)
    return index_feature;
  else
    return -1;
}

//#define EXPERIMENT_ON
#ifdef EXPERIMENT_ON  // This code is commented out as SampleIterator and
// TrainingSample are not reviewed/checked in yet, but these functions are a
// useful indicator of how an IntFeatureMap is setup.

// Computes the features used by the subset of samples defined by
// the iterator and sets up the feature mapping.
// Returns the size of the compacted feature space.
int IntFeatureMap::FindNZFeatureMapping(SampleIterator* it) {
  feature_map_.Init(feature_space_.Size(), false);
  int total_samples = 0;
  for (it->Begin(); !it->AtEnd(); it->Next()) {
    const TrainingSample& sample = it->GetSample();
    GenericVector<int> features;
    feature_space_.IndexAndSortFeatures(sample.features(),
                                        sample.num_features(),
                                        &features);
    int num_features = features.size();
    for (int f = 0; f < num_features; ++f)
      feature_map_.SetMap(features[f], true);
    ++total_samples;
  }
  feature_map_.Setup();
  compact_size_ = feature_map_.CompactSize();
  mapping_changed_ = true;
  FinalizeMapping(it);
  tprintf("%d non-zero features found in %d samples\n",
          compact_size_, total_samples);
  return compact_size_;
}
#endif

// After deleting some features, finish setting up the mapping, and map
// all the samples. Returns the size of the compacted feature space.
int IntFeatureMap::FinalizeMapping(SampleIterator* it) {
  if (mapping_changed_) {
    feature_map_.CompleteMerges();
    compact_size_ = feature_map_.CompactSize();
#ifdef EXPERIMENT_ON
    it->MapSampleFeatures(*this);
#endif
    mapping_changed_ = false;
  }
  return compact_size_;
}

// Prints the map features from the set in human-readable form.
void IntFeatureMap::DebugMapFeatures(
    const GenericVector<int>& map_features) const {
  for (int i = 0; i < map_features.size(); ++i) {
    INT_FEATURE_STRUCT f = InverseMapFeature(map_features[i]);
    f.print();
  }
}

void IntFeatureMap::Clear() {
  for (int dir = 0; dir < kNumOffsetMaps; ++dir) {
    delete [] offset_plus_[dir];
    delete [] offset_minus_[dir];
    offset_plus_[dir] = NULL;
    offset_minus_[dir] = NULL;
  }
}

// Helper to compute an offset index feature. In this context an offset
// feature with a dir of +/-1 is a feature of a similar direction,
// but shifted perpendicular to the direction of the feature. An offset
// feature with a dir of +/-2 is feature at the same position, but rotated
// by +/- one [compact] quantum. Returns the index of the generated offset
// feature, or -1 if it doesn't exist. Dir should be in
// [-kNumOffsetMaps, kNumOffsetMaps] to indicate the relative direction.
// A dir of 0 is an identity transformation.
// Both input and output are from the index(sparse) feature space, not
// the mapped/compact feature space, but the offset feature is the minimum
// distance moved from the input to guarantee that it maps to the next
// available quantum in the mapped/compact space.
int IntFeatureMap::ComputeOffsetFeature(int index_feature, int dir) const {
  INT_FEATURE_STRUCT f = InverseIndexFeature(index_feature);
  ASSERT_HOST(IndexFeature(f) == index_feature);
  if (dir == 0) {
    return index_feature;
  } else if (dir == 1 || dir == -1) {
    FCOORD feature_dir = FeatureDirection(f.Theta);
    FCOORD rotation90(0.0f, 1.0f);
    feature_dir.rotate(rotation90);
    // Find the nearest existing feature.
    for (int m = 1; m < kMaxOffsetDist; ++m) {
      double x_pos = f.X + feature_dir.x() * (m * dir);
      double y_pos = f.Y + feature_dir.y() * (m * dir);
      int x = IntCastRounded(x_pos);
      int y = IntCastRounded(y_pos);
      if (x >= 0 && x <= MAX_UINT8 && y >= 0 && y <= MAX_UINT8) {
        INT_FEATURE_STRUCT offset_f;
        offset_f.X = x;
        offset_f.Y = y;
        offset_f.Theta = f.Theta;
        int offset_index = IndexFeature(offset_f);
        if (offset_index != index_feature && offset_index >= 0)
          return offset_index;  // Found one.
      } else {
        return -1;  // Hit the edge of feature space.
      }
    }
  } else if (dir == 2 || dir == -2) {
    // Find the nearest existing index_feature.
    for (int m = 1; m < kMaxOffsetDist; ++m) {
      int theta = f.Theta + m * dir / 2;
      INT_FEATURE_STRUCT offset_f;
      offset_f.X = f.X;
      offset_f.Y = f.Y;
      offset_f.Theta = Modulo(theta, 256);
      int offset_index = IndexFeature(offset_f);
      if (offset_index != index_feature && offset_index >= 0)
        return offset_index;  // Found one.
    }
  }
  return -1;  // Nothing within the max distance.
}

}  // namespace tesseract.
<|endoftext|>


<commit_before>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <string>
#include <cmath>
#include <x86intrin.h>

#include <utils.hpp>
#include <Observation.hpp>

#ifndef SNR_HPP
#define SNR_HPP

namespace PulsarSearch {

template< typename T > using snrFunc = void (*)(const AstroData::Observation< T > &, const float *, float *);

// Sequential SNR
template< typename T > void snrFoldedTS(const AstroData::Observation< T > & observation, const std::vector< T > & foldedTS, std::vector< T > & snrs);
// OpenCL SNR
template< typename T > std::string * getSNROpenCL(const unsigned int nrDMsPerBlock, const unsigned int nrPeriodsPerBlock, const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, std::string & dataType, const AstroData::Observation< T > & observation);
// SIMD SNR
std::string * getSNRSIMD(const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, bool phi = false);

// Implementations
template< typename T > void snrFoldedTS(AstroData::Observation< T > & observation, const std::vector< T > & foldedTS, std::vector< T > & snrs) {
	for ( unsigned int period = 0; period < observation.getNrPeriods(); period++ ) {
		for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
			T max = 0;
			float average = 0.0f;
			float rms = 0.0f;

			for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {
				T value = foldedTS[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + (period * observation.getNrPaddedDMs()) + dm];

				average += value;
				rms += (value * value);

				if ( value > max ) {
					max = value;
				}
			}
			average /= observation.getNrBins();
			rms = std::sqrt(rms / observation.getNrBins());

			snrs[(period * observation.getNrPaddedDMs()) + dm] = (max - average) / rms;
		}
	}
}

template< typename T > std::string * getSNROpenCL(const unsigned int nrDMsPerBlock, const unsigned int nrPeriodsPerBlock, const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, std::string & dataType, const AstroData::Observation< T > & observation) {
  std::string * code = new std::string();

  // Begin kernel's template
  std::string nrPaddedDMs_s = isa::utils::toString< unsigned int >(observation.getNrPaddedDMs());
  std::string nrBinsInverse_s = isa::utils::toString< float >(1.0f / observation.getNrBins());

  *code = "__kernel void snr(__global const " + dataType + " * const restrict foldedData, __global " + dataType + " * const restrict snrs) {\n"
    "<%DEF_DM%>"
    "<%DEF_PERIOD%>"
    + dataType + " globalItem = 0;\n"
    + dataType + " average = 0;\n"
    + dataType + " rms = 0;\n"
    + dataType + " max = 0;\n"
    "\n"
    "<%COMPUTE%>"
    "}\n";
    std::string defDMsTemplate = "const unsigned int dm<%DM_NUM%> = (get_group_id(0) * " + isa::utils::toString< unsigned int >(nrDMsPerBlock * nrDMsPerThread) + ") + get_local_id(0) + <%DM_NUM%>;\n";
  std::string defPeriodsTemplate = "const unsigned int period<%PERIOD_NUM%> = (get_group_id(1) * " + isa::utils::toString< unsigned int >(nrPeriodsPerBlock * nrPeriodsPerThread) + ") + get_local_id(1) + <%PERIOD_NUM%>;\n";
  std::string computeTemplate = "average = 0;\n"
    "rms = 0;\n"
    "max = 0;\n"
    "for ( unsigned int bin = 0; bin < " + isa::utils::toString< unsigned int >(observation.getNrBins()) + "; bin++ ) {\n"
    "globalItem = foldedData[(bin * " + isa::utils::toString< unsigned int >(observation.getNrPeriods()) + " * " + nrPaddedDMs_s + ") + (period<%PERIOD_NUM%> * " + nrPaddedDMs_s + ") + dm<%DM_NUM%>];\n"
    "average += globalItem;\n"
    "rms += (globalItem * globalItem);\n"
    "max = fmax(max, globalItem);\n"
    "}\n"
    "average *= " + nrBinsInverse_s + "f;\n"
    "rms *= " + nrBinsInverse_s + "f;\n"
    "snrs[(period<%PERIOD_NUM%> * " + nrPaddedDMs_s + ") + dm<%DM_NUM%>] = (max - average) / native_sqrt(rms);\n";
  // End kernel's template

  std::string * defDM_s = new std::string();
  std::string * defPeriod_s = new std::string();
  std::string * compute_s = new std::string();

  for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
    std::string dm_s = isa::utils::toString< unsigned int >(dm);
    std::string * temp_s = 0;

    temp_s = isa::utils::replace(&defDMsTemplate, "<%DM_NUM%>", dm_s);
    defDM_s->append(*temp_s);
    delete temp_s;
  }
  for ( unsigned int period = 0; period < nrPeriodsPerThread; period++ ) {
    std::string period_s = isa::utils::toString< unsigned int >(period);
    std::string * temp_s = 0;

    temp_s = isa::utils::replace(&defPeriodsTemplate, "<%PERIOD_NUM%>", period_s);
    defPeriod_s->append(*temp_s);
    delete temp_s;

    for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
      std::string dm_s = isa::utils::toString< unsigned int >(dm);

      temp_s = isa::utils::replace(&computeTemplate, "<%PERIOD_NUM%>", period_s);
      temp_s = isa::utils::replace(temp_s, "<%DM_NUM%>", dm_s, true);
      compute_s->append(*temp_s);
      delete temp_s;
    }
  }

  code = isa::utils::replace(code, "<%DEF_DM%>", *defDM_s, true);
  code = isa::utils::replace(code, "<%DEF_PERIOD%>", *defPeriod_s, true);
  code = isa::utils::replace(code, "<%COMPUTE%>", *compute_s, true);

  return code;
}

std::string * getSNRSIMD(const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, bool phi) {
  std::string * code = new std::string();
  std::string * computeTemplate = new string();

  // Begin kernel's template
  if ( !phi ) {
    *code = "namespace PulsarSearch {\n"
      "template< typename T > void snrAVX" + isa::utils::toString(nrDMsPerThread) + "x" + isa::utils::toString(nrPeriodsPerThread) + "(const AstroData::Observation< T > & observation, const float * const __restrict__ foldedData, float * const __restrict__ snrs) {\n"
      "{\n"
      "__m256 max = _mm256_setzero_ps();\n"
      "__m256 average = _mm256_setzero_ps();\n"
      "__m256 rms = _mm256_setzero_ps();\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int period = 0; period < observation.getNrPeriods(); period += " + isa::utils::toString(nrPeriodsPerThread) + ") {\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm += " + isa::utils::toString(nrPeriodsPerThread) + " * 8) {\n"
      "<%COMPUTE%>"
      "}\n"
      "}\n"
      "}\n"
      "}\n"
      "}\n";
    *computeTemplate = "max = _mm256_setzero_ps();\n"
      "average = _mm256_setzero_ps();\n"
      "rms = _mm256_setzero_ps();\n"
      "\n"
      "for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {\n"
      "__m256 value = _mm256_load_ps(&(foldedData[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + ((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]));\n"
      "\n"
      "average = _mm256_add_ps(average, value);\n"
      "rms = _mm256_add_ps(rms, _mm256_mul_ps(value, value));\n"
      "max = _mm256_max_ps(max, value);\n"
      "}\n"
      "average = _mm256_div_ps(average, _mm256_set1_ps(observation.getNrBins()));\n"
      "rms = _mm256_sqrt_ps(_mm256_div_ps(rms, _mm256_set1_ps(observation.getNrBins())));\n"
      "_mm256_store_ps(&(snrs[((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]),_mm256_div_ps(_mm256_sub_ps(max, average), rms));\n";
  } else {
    *code = "namespace PulsarSearch {\n"
      "template< typename T > void snrPhi" + isa::utils::toString(nrDMsPerThread) + "x" + isa::utils::toString(nrPeriodsPerThread) + "(const AstroData::Observation< T > & observation, const float * const __restrict__ foldedData, float * const __restrict__ snrs) {\n"
      "{\n"
      "__m512 max = _mm512_setzero_ps();\n"
      "__m512 average = _mm512_setzero_ps();\n"
      "__m512 rms = _mm512_setzero_ps();\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int period = 0; period < observation.getNrPeriods(); period += " + isa::utils::toString(nrPeriodsPerThread) + ") {\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm += " + isa::utils::toString(nrPeriodsPerThread) + " * 16) {\n"
      "<%COMPUTE%>"
      "}\n"
      "}\n"
      "}\n"
      "}\n"
      "}\n";
    *computeTemplate = "max = _mm512_setzero_ps();\n"
      "average = _mm512_setzero_ps();\n"
      "rms = _mm512_setzero_ps();\n"
      "\n"
      "for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {\n"
      "__m512 value = _mm512_load_ps(&(foldedData[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + ((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]));\n"
      "\n"
      "average = _mm512_add_ps(average, value);\n"
      "rms = _mm512_add_ps(rms, _mm512_mul_ps(value, value));\n"
      "max = _mm512_max_ps(max, value);\n"
      "}\n"
      "average = _mm512_div_ps(average, _mm512_set1_ps(observation.getNrBins()));\n"
      "rms = _mm512_sqrt_ps(_mm512_div_ps(rms, _mm512_set1_ps(observation.getNrBins())));\n"
      "_mm512_store_ps(&(snrs[((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]),_mm512_div_ps(_mm512_sub_ps(max, average), rms));\n";
  }
  // End kernel's template

  std::string * compute_s = new std::string();

  for ( unsigned int period = 0; period < nrPeriodsPerThread; period++ ) {
    std::string period_s = isa::utils::toString< unsigned int >(period);

    for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
      std::string dm_s = isa::utils::toString< unsigned int >(dm);
      std::string * temp_s = 0;

      temp_s = isa::utils::replace(computeTemplate, "<%PERIOD_NUM%>", period_s);
      temp_s = isa::utils::replace(temp_s, "<%DM_NUM%>", dm_s, true);
      compute_s->append(*temp_s);
      delete temp_s;
    }
  }
  code = isa::utils::replace(code, "<%COMPUTE%>", *compute_s, true);
  delete compute_s;

  delete computeTemplate;
  return code;
}

} // PulsarSearch

#endif // SNR_HPP

<commit_msg>Forgot a std:: before string.<commit_after>// Copyright 2014 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <string>
#include <cmath>
#include <x86intrin.h>

#include <utils.hpp>
#include <Observation.hpp>

#ifndef SNR_HPP
#define SNR_HPP

namespace PulsarSearch {

template< typename T > using snrFunc = void (*)(const AstroData::Observation< T > &, const float *, float *);

// Sequential SNR
template< typename T > void snrFoldedTS(const AstroData::Observation< T > & observation, const std::vector< T > & foldedTS, std::vector< T > & snrs);
// OpenCL SNR
template< typename T > std::string * getSNROpenCL(const unsigned int nrDMsPerBlock, const unsigned int nrPeriodsPerBlock, const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, std::string & dataType, const AstroData::Observation< T > & observation);
// SIMD SNR
std::string * getSNRSIMD(const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, bool phi = false);

// Implementations
template< typename T > void snrFoldedTS(AstroData::Observation< T > & observation, const std::vector< T > & foldedTS, std::vector< T > & snrs) {
	for ( unsigned int period = 0; period < observation.getNrPeriods(); period++ ) {
		for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm++ ) {
			T max = 0;
			float average = 0.0f;
			float rms = 0.0f;

			for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {
				T value = foldedTS[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + (period * observation.getNrPaddedDMs()) + dm];

				average += value;
				rms += (value * value);

				if ( value > max ) {
					max = value;
				}
			}
			average /= observation.getNrBins();
			rms = std::sqrt(rms / observation.getNrBins());

			snrs[(period * observation.getNrPaddedDMs()) + dm] = (max - average) / rms;
		}
	}
}

template< typename T > std::string * getSNROpenCL(const unsigned int nrDMsPerBlock, const unsigned int nrPeriodsPerBlock, const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, std::string & dataType, const AstroData::Observation< T > & observation) {
  std::string * code = new std::string();

  // Begin kernel's template
  std::string nrPaddedDMs_s = isa::utils::toString< unsigned int >(observation.getNrPaddedDMs());
  std::string nrBinsInverse_s = isa::utils::toString< float >(1.0f / observation.getNrBins());

  *code = "__kernel void snr(__global const " + dataType + " * const restrict foldedData, __global " + dataType + " * const restrict snrs) {\n"
    "<%DEF_DM%>"
    "<%DEF_PERIOD%>"
    + dataType + " globalItem = 0;\n"
    + dataType + " average = 0;\n"
    + dataType + " rms = 0;\n"
    + dataType + " max = 0;\n"
    "\n"
    "<%COMPUTE%>"
    "}\n";
    std::string defDMsTemplate = "const unsigned int dm<%DM_NUM%> = (get_group_id(0) * " + isa::utils::toString< unsigned int >(nrDMsPerBlock * nrDMsPerThread) + ") + get_local_id(0) + <%DM_NUM%>;\n";
  std::string defPeriodsTemplate = "const unsigned int period<%PERIOD_NUM%> = (get_group_id(1) * " + isa::utils::toString< unsigned int >(nrPeriodsPerBlock * nrPeriodsPerThread) + ") + get_local_id(1) + <%PERIOD_NUM%>;\n";
  std::string computeTemplate = "average = 0;\n"
    "rms = 0;\n"
    "max = 0;\n"
    "for ( unsigned int bin = 0; bin < " + isa::utils::toString< unsigned int >(observation.getNrBins()) + "; bin++ ) {\n"
    "globalItem = foldedData[(bin * " + isa::utils::toString< unsigned int >(observation.getNrPeriods()) + " * " + nrPaddedDMs_s + ") + (period<%PERIOD_NUM%> * " + nrPaddedDMs_s + ") + dm<%DM_NUM%>];\n"
    "average += globalItem;\n"
    "rms += (globalItem * globalItem);\n"
    "max = fmax(max, globalItem);\n"
    "}\n"
    "average *= " + nrBinsInverse_s + "f;\n"
    "rms *= " + nrBinsInverse_s + "f;\n"
    "snrs[(period<%PERIOD_NUM%> * " + nrPaddedDMs_s + ") + dm<%DM_NUM%>] = (max - average) / native_sqrt(rms);\n";
  // End kernel's template

  std::string * defDM_s = new std::string();
  std::string * defPeriod_s = new std::string();
  std::string * compute_s = new std::string();

  for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
    std::string dm_s = isa::utils::toString< unsigned int >(dm);
    std::string * temp_s = 0;

    temp_s = isa::utils::replace(&defDMsTemplate, "<%DM_NUM%>", dm_s);
    defDM_s->append(*temp_s);
    delete temp_s;
  }
  for ( unsigned int period = 0; period < nrPeriodsPerThread; period++ ) {
    std::string period_s = isa::utils::toString< unsigned int >(period);
    std::string * temp_s = 0;

    temp_s = isa::utils::replace(&defPeriodsTemplate, "<%PERIOD_NUM%>", period_s);
    defPeriod_s->append(*temp_s);
    delete temp_s;

    for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
      std::string dm_s = isa::utils::toString< unsigned int >(dm);

      temp_s = isa::utils::replace(&computeTemplate, "<%PERIOD_NUM%>", period_s);
      temp_s = isa::utils::replace(temp_s, "<%DM_NUM%>", dm_s, true);
      compute_s->append(*temp_s);
      delete temp_s;
    }
  }

  code = isa::utils::replace(code, "<%DEF_DM%>", *defDM_s, true);
  code = isa::utils::replace(code, "<%DEF_PERIOD%>", *defPeriod_s, true);
  code = isa::utils::replace(code, "<%COMPUTE%>", *compute_s, true);

  return code;
}

std::string * getSNRSIMD(const unsigned int nrDMsPerThread, const unsigned int nrPeriodsPerThread, bool phi) {
  std::string * code = new std::string();
  std::string * computeTemplate = new std::string();

  // Begin kernel's template
  if ( !phi ) {
    *code = "namespace PulsarSearch {\n"
      "template< typename T > void snrAVX" + isa::utils::toString(nrDMsPerThread) + "x" + isa::utils::toString(nrPeriodsPerThread) + "(const AstroData::Observation< T > & observation, const float * const __restrict__ foldedData, float * const __restrict__ snrs) {\n"
      "{\n"
      "__m256 max = _mm256_setzero_ps();\n"
      "__m256 average = _mm256_setzero_ps();\n"
      "__m256 rms = _mm256_setzero_ps();\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int period = 0; period < observation.getNrPeriods(); period += " + isa::utils::toString(nrPeriodsPerThread) + ") {\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm += " + isa::utils::toString(nrPeriodsPerThread) + " * 8) {\n"
      "<%COMPUTE%>"
      "}\n"
      "}\n"
      "}\n"
      "}\n"
      "}\n";
    *computeTemplate = "max = _mm256_setzero_ps();\n"
      "average = _mm256_setzero_ps();\n"
      "rms = _mm256_setzero_ps();\n"
      "\n"
      "for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {\n"
      "__m256 value = _mm256_load_ps(&(foldedData[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + ((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]));\n"
      "\n"
      "average = _mm256_add_ps(average, value);\n"
      "rms = _mm256_add_ps(rms, _mm256_mul_ps(value, value));\n"
      "max = _mm256_max_ps(max, value);\n"
      "}\n"
      "average = _mm256_div_ps(average, _mm256_set1_ps(observation.getNrBins()));\n"
      "rms = _mm256_sqrt_ps(_mm256_div_ps(rms, _mm256_set1_ps(observation.getNrBins())));\n"
      "_mm256_store_ps(&(snrs[((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]),_mm256_div_ps(_mm256_sub_ps(max, average), rms));\n";
  } else {
    *code = "namespace PulsarSearch {\n"
      "template< typename T > void snrPhi" + isa::utils::toString(nrDMsPerThread) + "x" + isa::utils::toString(nrPeriodsPerThread) + "(const AstroData::Observation< T > & observation, const float * const __restrict__ foldedData, float * const __restrict__ snrs) {\n"
      "{\n"
      "__m512 max = _mm512_setzero_ps();\n"
      "__m512 average = _mm512_setzero_ps();\n"
      "__m512 rms = _mm512_setzero_ps();\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int period = 0; period < observation.getNrPeriods(); period += " + isa::utils::toString(nrPeriodsPerThread) + ") {\n"
      "#pragma omp parallel for schedule(static)\n"
      "for ( unsigned int dm = 0; dm < observation.getNrDMs(); dm += " + isa::utils::toString(nrPeriodsPerThread) + " * 16) {\n"
      "<%COMPUTE%>"
      "}\n"
      "}\n"
      "}\n"
      "}\n"
      "}\n";
    *computeTemplate = "max = _mm512_setzero_ps();\n"
      "average = _mm512_setzero_ps();\n"
      "rms = _mm512_setzero_ps();\n"
      "\n"
      "for ( unsigned int bin = 0; bin < observation.getNrBins(); bin++ ) {\n"
      "__m512 value = _mm512_load_ps(&(foldedData[(bin * observation.getNrPeriods() * observation.getNrPaddedDMs()) + ((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]));\n"
      "\n"
      "average = _mm512_add_ps(average, value);\n"
      "rms = _mm512_add_ps(rms, _mm512_mul_ps(value, value));\n"
      "max = _mm512_max_ps(max, value);\n"
      "}\n"
      "average = _mm512_div_ps(average, _mm512_set1_ps(observation.getNrBins()));\n"
      "rms = _mm512_sqrt_ps(_mm512_div_ps(rms, _mm512_set1_ps(observation.getNrBins())));\n"
      "_mm512_store_ps(&(snrs[((period + <%PERIOD_NUM%>) * observation.getNrPaddedDMs()) + dm + <%DM_NUM%>]),_mm512_div_ps(_mm512_sub_ps(max, average), rms));\n";
  }
  // End kernel's template

  std::string * compute_s = new std::string();

  for ( unsigned int period = 0; period < nrPeriodsPerThread; period++ ) {
    std::string period_s = isa::utils::toString< unsigned int >(period);

    for ( unsigned int dm = 0; dm < nrDMsPerThread; dm++ ) {
      std::string dm_s = isa::utils::toString< unsigned int >(dm);
      std::string * temp_s = 0;

      temp_s = isa::utils::replace(computeTemplate, "<%PERIOD_NUM%>", period_s);
      temp_s = isa::utils::replace(temp_s, "<%DM_NUM%>", dm_s, true);
      compute_s->append(*temp_s);
      delete temp_s;
    }
  }
  code = isa::utils::replace(code, "<%COMPUTE%>", *compute_s, true);
  delete compute_s;

  delete computeTemplate;
  return code;
}

} // PulsarSearch

#endif // SNR_HPP

<|endoftext|>


<commit_before>/**
 * @file    any.hpp
 * @brief   A very simple variant for C++.
 */

#pragma once
#include <iostream>
#include <typeinfo>

/**
 * @namespace type
 */
namespace type {
  /**
   * @class any
   * @brief A very simple variant type for C++
   *
   * The Boost.any class is very feature complete, but also drags a large part of the Boost library
   * along with it. There is no easy way to separate that one concept without requiring a project
   * to depend on Boost itself.
   *
   * This class is an attempt to replicate most of the functionality of Boost.any in a self-
   * contained manner. There currently exist some limitations, such as not being able to create an
   * instance of type::any from another type::any instance. However this should work for most cases
   * where one would use Boost.any.
   */
  class any {
    public:
      /**
       * @brief Default constructor.
       */
      any() : _value(nullptr) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an object reference.
       * @tparam    T     Value type.
       * @param[in] value Object that the type::any instance represents.
       */
      template<typename T> any(T& value)
        : _value(new erasure_impl<typename std::decay<T&>::type>(value)) { }

      /**
       * @brief Clones an existing type::any.
       * @param[in] value The type::any instance to clone.
       */
      any(any& value) : _value(value._value->clone()) { }

      /**
       * @brief Constructs and initializes a new type::any instance from a constant.
       * @tparam T Value type.
       * @param[in] value Constant that the type::any instance represents.
       */
      template<typename T> any(T const& value)
        : _value(new erasure_impl<typename std::decay<T const&>::type>(value)) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an object pointer.
       * @tparam T Value type.
       * @param[in] value Object pointer that the type::any instance represents.
       */
      template<typename T> any(T* value)
        : _value(new erasure_impl<typename std::decay<T*>::type>(value)) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an array or const pointer.
       * @tparam T Value type.
       * @param[in] value Array or pointer that the type::any instance represents.
       */
      template<typename T> any(T const* value)
        : _value(new erasure_impl<typename std::decay<T const*>::type>(value)) { }

      /**
       * @brief Object destructor.
       */
      ~any() { if(_value) delete _value; }

      /**
       * @brief Sets the value of the type::any instance from an object reference.
       * @tparam T Value type.
       * @param[in] value Object that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T& value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T&>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from a constant.
       * @tparam T Value type.
       * @param[in] value Constant that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T const& value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T const&>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from an object pointer.
       * @tparam T Value type.
       * @param[in] value Object pointer that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T* value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T*>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from an array or const pointer.
       * @tparam T Value type.
       * @param[in] value Array or const pointer that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T const* value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T const*>::type>(value);
        return *this;
      }

      /**
       * @brief Retrieves the value represented by the type::any object.
       * @tparam T Type to cast the represented value to.
       * @return Reference to the represented value.
       */
      template<typename T> T& as() const {
        if(_value) return static_cast<erasure_impl<T>*>(_value)->value();
        throw std::bad_cast();
      }

      /**
       * @brief Retrieves the type of the value represented by the type::any object.
       * @return Reference to the std::type_info structure that describes the value type.
       */
      std::type_info const& type() const {
        if(_value) return _value->type();
        return typeid(nullptr);
      }

    private:

      struct erasure {
        virtual ~erasure() = 0;
        virtual erasure* clone() = 0;
        virtual std::type_info const& type() const = 0;
      };

      template<typename T> class erasure_impl : public erasure {
        public:
          erasure_impl(T value) : _value(value) { }
          virtual erasure* clone() { return new erasure_impl<T>(_value); }
          virtual std::type_info const& type() const { return typeid(T); }
          T& value() { return _value; }
        private:
          T _value;
      };

      erasure* _value;
  };

  any::erasure::~erasure() { }
}
<commit_msg>Added a copy constructor.<commit_after>/**
 * @file    any.hpp
 * @brief   A very simple variant for C++.
 */

#pragma once
#include <iostream>
#include <typeinfo>

/**
 * @namespace type
 */
namespace type {
  /**
   * @class any
   * @brief A very simple variant type for C++
   *
   * The Boost.any class is very feature complete, but also drags a large part of the Boost library
   * along with it. There is no easy way to separate that one concept without requiring a project
   * to depend on Boost itself.
   *
   * This class is an attempt to replicate most of the functionality of Boost.any in a self-
   * contained manner. There currently exist some limitations, such as not being able to create an
   * instance of type::any from another type::any instance. However this should work for most cases
   * where one would use Boost.any.
   */
  class any {
    public:
      /**
       * @brief Default constructor.
       */
      any() : _value(nullptr) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an object reference.
       * @tparam    T     Value type.
       * @param[in] value Object that the type::any instance represents.
       */
      template<typename T> any(T& value)
        : _value(new erasure_impl<typename std::decay<T&>::type>(value)) { }

      /**
       * @brief Copy constructor.
       * @param[in] value The type::any instance to copy.
       */
      any(any const& value) : _value(value._value->clone()) { }

      /**
       * @brief Clones an existing type::any.
       * @param[in] value The type::any instance to clone.
       */
      any(any& value) : _value(value._value->clone()) { }

      /**
       * @brief Constructs and initializes a new type::any instance from a constant.
       * @tparam T Value type.
       * @param[in] value Constant that the type::any instance represents.
       */
      template<typename T> any(T const& value)
        : _value(new erasure_impl<typename std::decay<T const&>::type>(value)) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an object pointer.
       * @tparam T Value type.
       * @param[in] value Object pointer that the type::any instance represents.
       */
      template<typename T> any(T* value)
        : _value(new erasure_impl<typename std::decay<T*>::type>(value)) { }

      /**
       * @brief Constructs and initializes a new type::any instance from an array or const pointer.
       * @tparam T Value type.
       * @param[in] value Array or pointer that the type::any instance represents.
       */
      template<typename T> any(T const* value)
        : _value(new erasure_impl<typename std::decay<T const*>::type>(value)) { }

      /**
       * @brief Object destructor.
       */
      ~any() { if(_value) delete _value; }

      /**
       * @brief Sets the value of the type::any instance from an object reference.
       * @tparam T Value type.
       * @param[in] value Object that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T& value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T&>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from a constant.
       * @tparam T Value type.
       * @param[in] value Constant that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T const& value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T const&>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from an object pointer.
       * @tparam T Value type.
       * @param[in] value Object pointer that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T* value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T*>::type>(value);
        return *this;
      }

      /**
       * @brief Sets the value of the type::any instance from an array or const pointer.
       * @tparam T Value type.
       * @param[in] value Array or const pointer that the type::any instance represents.
       * @return Reference to the type::any instance.
       */
      template<typename T> any& operator=(T const* value) {
        if(_value) delete _value;
        _value = new erasure_impl<typename std::decay<T const*>::type>(value);
        return *this;
      }

      /**
       * @brief Retrieves the value represented by the type::any object.
       * @tparam T Type to cast the represented value to.
       * @return Reference to the represented value.
       */
      template<typename T> T& as() const {
        if(_value) return static_cast<erasure_impl<T>*>(_value)->value();
        throw std::bad_cast();
      }

      /**
       * @brief Retrieves the type of the value represented by the type::any object.
       * @return Reference to the std::type_info structure that describes the value type.
       */
      std::type_info const& type() const {
        if(_value) return _value->type();
        return typeid(nullptr);
      }

    private:

      struct erasure {
        virtual ~erasure() = 0;
        virtual erasure* clone() = 0;
        virtual std::type_info const& type() const = 0;
      };

      template<typename T> class erasure_impl : public erasure {
        public:
          erasure_impl(T value) : _value(value) { }
          virtual erasure* clone() { return new erasure_impl<T>(_value); }
          virtual std::type_info const& type() const { return typeid(T); }
          T& value() { return _value; }
        private:
          T _value;
      };

      erasure* _value;
  };

  any::erasure::~erasure() { }
}
<|endoftext|>


<commit_before>#include "../data_generator.hpp"
#include "../feature_generator.hpp"
#include "../feature_sorter.hpp"
#include "../update_generator.hpp"
#include "../borders_generator.hpp"
#include "../borders_loader.hpp"
#include "../dumper.hpp"
#include "../statistics.hpp"
#include "../unpack_mwm.hpp"
#include "../generate_info.hpp"

#include "../../indexer/drawing_rules.hpp"
#include "../../indexer/classificator_loader.hpp"
#include "../../indexer/data_header.hpp"
#include "../../indexer/features_vector.hpp"
#include "../../indexer/index_builder.hpp"
#include "../../indexer/search_index_builder.hpp"

#include "../../defines.hpp"

#include "../../platform/platform.hpp"

#include "../../3party/gflags/src/gflags/gflags.h"

#include "../../std/iostream.hpp"
#include "../../std/iomanip.hpp"
#include "../../std/numeric.hpp"

#include "../../version/version.hpp"

DEFINE_bool(version, false, "Display version");
DEFINE_bool(generate_update, false,
              "If specified, update.maps file will be generated from cells in the data path");

DEFINE_bool(generate_classif, false, "Generate classificator.");
DEFINE_bool(preprocess_xml, false, "1st pass - create nodes/ways/relations data");
DEFINE_bool(make_coasts, false, "create intermediate file with coasts data");
DEFINE_bool(emit_coasts, false, "push coasts features from intermediate file to out files/countries");

DEFINE_bool(generate_features, false, "2nd pass - generate intermediate features");
DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry and triangles for features");
DEFINE_bool(generate_index, false, "4rd pass - generate index");
DEFINE_bool(generate_search_index, false, "5th pass - generate search index");
DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files");
DEFINE_bool(use_light_nodes, false,
            "If true, use temporary vector of nodes, instead of huge temp file");
DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty.");
DEFINE_string(output, "", "Prefix of filenames of outputted .dat and .idx files.");
DEFINE_string(intermediate_data_path, "", "Path to store nodes, ways, relations.");
DEFINE_bool(generate_world, false, "Generate separate world file");
DEFINE_bool(split_by_polygons, false, "Use countries borders to split planet by regions and countries");
DEFINE_string(generate_borders, "",
            "Create binary country .borders file for osm xml file given in 'output' parameter,"
            "specify tag name and optional value: ISO3166-1 or admin_level=4");
DEFINE_bool(dump_types, false, "If defined, prints all types combinations and their total count");
DEFINE_bool(dump_prefixes, false, "If defined, prints statistics on feature name prefixes");
DEFINE_bool(dump_search_tokens, false, "Print statistics on search tokens.");
DEFINE_bool(unpack_mwm, false, "Unpack each section of mwm into a separate file with name filePath.sectionName.");
DEFINE_bool(generate_packed_borders, false, "Generate packed file with country polygons");

string AddSlashIfNeeded(string const & str)
{
  string result(str);
  size_t const size = result.size();
  if (size)
  {
    if (result.find_last_of('\\') == size - 1)
      result[size - 1] = '/';
    else
      if (result.find_last_of('/') != size - 1)
        result.push_back('/');
  }
  return result;
}

int main(int argc, char ** argv)
{
  google::SetUsageMessage(
      "Takes OSM XML data from stdin and creates data and index files in several passes.");

  google::ParseCommandLineFlags(&argc, &argv, true);

  Platform & pl = GetPlatform();

  string const path =
      FLAGS_data_path.empty() ? pl.WritableDir() : AddSlashIfNeeded(FLAGS_data_path);

  if (FLAGS_version)
  {
    cout << "Tool version: " << VERSION_STRING << endl;
    cout << "Built on: " << VERSION_DATE_STRING << endl;
  }

  // Make a classificator
  if (FLAGS_generate_classif)
  {
    drule::RulesHolder & rules = drule::rules();

    string buffer;
    ModelReaderPtr(pl.GetReader(DRAWING_RULES_TXT_FILE)).ReadAsString(buffer);

    rules.LoadFromTextProto(buffer);

    ofstream s((path + DRAWING_RULES_BIN_FILE).c_str(), ios::out | ios::binary);
    rules.SaveToBinaryProto(buffer, s);
  }

  // Generating intermediate files
  if (FLAGS_preprocess_xml)
  {
    LOG(LINFO, ("Generating intermediate data ...."));
    if (!data::GenerateToFile(FLAGS_intermediate_data_path, FLAGS_use_light_nodes))
      return -1;
  }

  feature::GenerateInfo genInfo;
  genInfo.m_tmpDir = FLAGS_intermediate_data_path;

  // load classificator only if necessary
  if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry ||
      FLAGS_generate_index || FLAGS_generate_search_index ||
      FLAGS_calc_statistics || FLAGS_dump_types || FLAGS_dump_prefixes)
  {
    classificator::Load();
    classif().SortClassificator();
  }

  // Generate dat file
  if (FLAGS_generate_features || FLAGS_make_coasts)
  {
    LOG(LINFO, ("Generating final data ..."));

    if (FLAGS_split_by_polygons)
    {
      // do split by countries
      genInfo.m_datFilePrefix = path;
    }
    else
    {
      // used for one country generation - append destination file
      CHECK ( !FLAGS_output.empty(), ("No destination specified. Did you forget --output?") );
      genInfo.m_datFilePrefix = path + FLAGS_output;
    }

    genInfo.m_datFileSuffix = DATA_FILE_EXTENSION_TMP;

    genInfo.m_splitByPolygons = FLAGS_split_by_polygons;
    genInfo.m_createWorld = FLAGS_generate_world;
    genInfo.m_makeCoasts = FLAGS_make_coasts;
    genInfo.m_emitCoasts = FLAGS_emit_coasts;

    if (!feature::GenerateFeatures(genInfo, FLAGS_use_light_nodes))
      return -1;

    // without --spit_by_polygons, we have empty name country as result - assign it
    if (genInfo.m_bucketNames.size() == 1 && genInfo.m_bucketNames[0].empty())
      genInfo.m_bucketNames[0] = FLAGS_output;

    if (FLAGS_generate_world)
    {
      genInfo.m_bucketNames.push_back(WORLD_FILE_NAME);
      genInfo.m_bucketNames.push_back(WORLD_COASTS_FILE_NAME);
    }
  }
  else
  {
    if (!FLAGS_output.empty())
      genInfo.m_bucketNames.push_back(FLAGS_output);
  }

  // Enumerate over all dat files that were created.
  size_t const count = genInfo.m_bucketNames.size();
  for (size_t i = 0; i < count; ++i)
  {
    string const & country = genInfo.m_bucketNames[i];

    if (FLAGS_generate_geometry)
    {
      LOG(LINFO, ("Generating result features for file", country));

      int mapType = feature::DataHeader::country;
      if (country == WORLD_FILE_NAME)
        mapType = feature::DataHeader::world;
      if (country == WORLD_COASTS_FILE_NAME)
        mapType = feature::DataHeader::worldcoasts;

      if (!feature::GenerateFinalFeatures(path, country, mapType))
      {
        // If error - move to next bucket without index generation
        continue;
      }
    }

    string const datFile = path + country + DATA_FILE_EXTENSION;

    if (FLAGS_generate_index)
    {
      LOG(LINFO, ("Generating index for ", datFile));
      if (!indexer::BuildIndexFromDatFile(datFile, FLAGS_intermediate_data_path + country))
      {
        LOG(LCRITICAL, ("Error generating index."));
      }
    }

    if (FLAGS_generate_search_index)
    {
      LOG(LINFO, ("Generating search index for ", datFile));
      if (!indexer::BuildSearchIndexFromDatFile(datFile))
      {
        LOG(LCRITICAL, ("Error generating search index."));
      }
    }
  }

  // Create http update list for countries and corresponding files
  if (FLAGS_generate_update)
  {
    LOG(LINFO, ("Updating countries file..."));
    update::UpdateCountries(path);
  }

  if (!FLAGS_generate_borders.empty())
  {
    if (!FLAGS_output.empty())
    {
      osm::GenerateBordersFromOsm(FLAGS_generate_borders,
                                  path + FLAGS_output + ".osm",
                                  path + FLAGS_output + ".borders");
    }
    else
    {
      LOG(LINFO, ("Please specify osm country borders file in 'output' command line parameter."));
    }
  }

  string const datFile = path + FLAGS_output + DATA_FILE_EXTENSION;

  if (FLAGS_calc_statistics)
  {
    LOG(LINFO, ("Calculating statistics for ", datFile));

    stats::FileContainerStatistic(datFile);

    stats::MapInfo info;
    stats::CalcStatistic(datFile, info);
    stats::PrintStatistic(info);
  }

  if (FLAGS_dump_types)
    feature::DumpTypes(datFile);

  if (FLAGS_dump_prefixes)
    feature::DumpPrefixes(datFile);

  if (FLAGS_dump_search_tokens)
    feature::DumpSearchTokens(datFile);

  if (FLAGS_unpack_mwm)
    UnpackMwm(datFile);

  if (FLAGS_generate_packed_borders)
    borders::GeneratePackedBorders(path);

  return 0;
}
<commit_msg>Remove dummy CHECK.<commit_after>#include "../data_generator.hpp"
#include "../feature_generator.hpp"
#include "../feature_sorter.hpp"
#include "../update_generator.hpp"
#include "../borders_generator.hpp"
#include "../borders_loader.hpp"
#include "../dumper.hpp"
#include "../statistics.hpp"
#include "../unpack_mwm.hpp"
#include "../generate_info.hpp"

#include "../../indexer/drawing_rules.hpp"
#include "../../indexer/classificator_loader.hpp"
#include "../../indexer/data_header.hpp"
#include "../../indexer/features_vector.hpp"
#include "../../indexer/index_builder.hpp"
#include "../../indexer/search_index_builder.hpp"

#include "../../defines.hpp"

#include "../../platform/platform.hpp"

#include "../../3party/gflags/src/gflags/gflags.h"

#include "../../std/iostream.hpp"
#include "../../std/iomanip.hpp"
#include "../../std/numeric.hpp"

#include "../../version/version.hpp"

DEFINE_bool(version, false, "Display version");
DEFINE_bool(generate_update, false,
              "If specified, update.maps file will be generated from cells in the data path");

DEFINE_bool(generate_classif, false, "Generate classificator.");
DEFINE_bool(preprocess_xml, false, "1st pass - create nodes/ways/relations data");
DEFINE_bool(make_coasts, false, "create intermediate file with coasts data");
DEFINE_bool(emit_coasts, false, "push coasts features from intermediate file to out files/countries");

DEFINE_bool(generate_features, false, "2nd pass - generate intermediate features");
DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry and triangles for features");
DEFINE_bool(generate_index, false, "4rd pass - generate index");
DEFINE_bool(generate_search_index, false, "5th pass - generate search index");
DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files");
DEFINE_bool(use_light_nodes, false,
            "If true, use temporary vector of nodes, instead of huge temp file");
DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty.");
DEFINE_string(output, "", "Prefix of filenames of outputted .dat and .idx files.");
DEFINE_string(intermediate_data_path, "", "Path to store nodes, ways, relations.");
DEFINE_bool(generate_world, false, "Generate separate world file");
DEFINE_bool(split_by_polygons, false, "Use countries borders to split planet by regions and countries");
DEFINE_string(generate_borders, "",
            "Create binary country .borders file for osm xml file given in 'output' parameter,"
            "specify tag name and optional value: ISO3166-1 or admin_level=4");
DEFINE_bool(dump_types, false, "If defined, prints all types combinations and their total count");
DEFINE_bool(dump_prefixes, false, "If defined, prints statistics on feature name prefixes");
DEFINE_bool(dump_search_tokens, false, "Print statistics on search tokens.");
DEFINE_bool(unpack_mwm, false, "Unpack each section of mwm into a separate file with name filePath.sectionName.");
DEFINE_bool(generate_packed_borders, false, "Generate packed file with country polygons");

string AddSlashIfNeeded(string const & str)
{
  string result(str);
  size_t const size = result.size();
  if (size)
  {
    if (result.find_last_of('\\') == size - 1)
      result[size - 1] = '/';
    else
      if (result.find_last_of('/') != size - 1)
        result.push_back('/');
  }
  return result;
}

int main(int argc, char ** argv)
{
  google::SetUsageMessage(
      "Takes OSM XML data from stdin and creates data and index files in several passes.");

  google::ParseCommandLineFlags(&argc, &argv, true);

  Platform & pl = GetPlatform();

  string const path =
      FLAGS_data_path.empty() ? pl.WritableDir() : AddSlashIfNeeded(FLAGS_data_path);

  if (FLAGS_version)
  {
    cout << "Tool version: " << VERSION_STRING << endl;
    cout << "Built on: " << VERSION_DATE_STRING << endl;
  }

  // Make a classificator
  if (FLAGS_generate_classif)
  {
    drule::RulesHolder & rules = drule::rules();

    string buffer;
    ModelReaderPtr(pl.GetReader(DRAWING_RULES_TXT_FILE)).ReadAsString(buffer);

    rules.LoadFromTextProto(buffer);

    ofstream s((path + DRAWING_RULES_BIN_FILE).c_str(), ios::out | ios::binary);
    rules.SaveToBinaryProto(buffer, s);
  }

  // Generating intermediate files
  if (FLAGS_preprocess_xml)
  {
    LOG(LINFO, ("Generating intermediate data ...."));
    if (!data::GenerateToFile(FLAGS_intermediate_data_path, FLAGS_use_light_nodes))
      return -1;
  }

  feature::GenerateInfo genInfo;
  genInfo.m_tmpDir = FLAGS_intermediate_data_path;

  // load classificator only if necessary
  if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry ||
      FLAGS_generate_index || FLAGS_generate_search_index ||
      FLAGS_calc_statistics || FLAGS_dump_types || FLAGS_dump_prefixes)
  {
    classificator::Load();
    classif().SortClassificator();
  }

  // Generate dat file
  if (FLAGS_generate_features || FLAGS_make_coasts)
  {
    LOG(LINFO, ("Generating final data ..."));

    if (FLAGS_split_by_polygons)
    {
      // do split by countries
      genInfo.m_datFilePrefix = path;
    }
    else
    {
      // Used for one country generation - append destination file.
      // FLAGS_output may be empty for FLAGS_make_coasts flag.
      genInfo.m_datFilePrefix = path + FLAGS_output;
    }

    genInfo.m_datFileSuffix = DATA_FILE_EXTENSION_TMP;

    genInfo.m_splitByPolygons = FLAGS_split_by_polygons;
    genInfo.m_createWorld = FLAGS_generate_world;
    genInfo.m_makeCoasts = FLAGS_make_coasts;
    genInfo.m_emitCoasts = FLAGS_emit_coasts;

    if (!feature::GenerateFeatures(genInfo, FLAGS_use_light_nodes))
      return -1;

    // without --spit_by_polygons, we have empty name country as result - assign it
    if (genInfo.m_bucketNames.size() == 1 && genInfo.m_bucketNames[0].empty())
      genInfo.m_bucketNames[0] = FLAGS_output;

    if (FLAGS_generate_world)
    {
      genInfo.m_bucketNames.push_back(WORLD_FILE_NAME);
      genInfo.m_bucketNames.push_back(WORLD_COASTS_FILE_NAME);
    }
  }
  else
  {
    if (!FLAGS_output.empty())
      genInfo.m_bucketNames.push_back(FLAGS_output);
  }

  // Enumerate over all dat files that were created.
  size_t const count = genInfo.m_bucketNames.size();
  for (size_t i = 0; i < count; ++i)
  {
    string const & country = genInfo.m_bucketNames[i];

    if (FLAGS_generate_geometry)
    {
      LOG(LINFO, ("Generating result features for file", country));

      int mapType = feature::DataHeader::country;
      if (country == WORLD_FILE_NAME)
        mapType = feature::DataHeader::world;
      if (country == WORLD_COASTS_FILE_NAME)
        mapType = feature::DataHeader::worldcoasts;

      if (!feature::GenerateFinalFeatures(path, country, mapType))
      {
        // If error - move to next bucket without index generation
        continue;
      }
    }

    string const datFile = path + country + DATA_FILE_EXTENSION;

    if (FLAGS_generate_index)
    {
      LOG(LINFO, ("Generating index for ", datFile));
      if (!indexer::BuildIndexFromDatFile(datFile, FLAGS_intermediate_data_path + country))
      {
        LOG(LCRITICAL, ("Error generating index."));
      }
    }

    if (FLAGS_generate_search_index)
    {
      LOG(LINFO, ("Generating search index for ", datFile));
      if (!indexer::BuildSearchIndexFromDatFile(datFile))
      {
        LOG(LCRITICAL, ("Error generating search index."));
      }
    }
  }

  // Create http update list for countries and corresponding files
  if (FLAGS_generate_update)
  {
    LOG(LINFO, ("Updating countries file..."));
    update::UpdateCountries(path);
  }

  if (!FLAGS_generate_borders.empty())
  {
    if (!FLAGS_output.empty())
    {
      osm::GenerateBordersFromOsm(FLAGS_generate_borders,
                                  path + FLAGS_output + ".osm",
                                  path + FLAGS_output + ".borders");
    }
    else
    {
      LOG(LINFO, ("Please specify osm country borders file in 'output' command line parameter."));
    }
  }

  string const datFile = path + FLAGS_output + DATA_FILE_EXTENSION;

  if (FLAGS_calc_statistics)
  {
    LOG(LINFO, ("Calculating statistics for ", datFile));

    stats::FileContainerStatistic(datFile);

    stats::MapInfo info;
    stats::CalcStatistic(datFile, info);
    stats::PrintStatistic(info);
  }

  if (FLAGS_dump_types)
    feature::DumpTypes(datFile);

  if (FLAGS_dump_prefixes)
    feature::DumpPrefixes(datFile);

  if (FLAGS_dump_search_tokens)
    feature::DumpSearchTokens(datFile);

  if (FLAGS_unpack_mwm)
    UnpackMwm(datFile);

  if (FLAGS_generate_packed_borders)
    borders::GeneratePackedBorders(path);

  return 0;
}
<|endoftext|>