<commit_before>/*

Copyright (c) 2007, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef TORRENT_INTRUSIVE_PTR_BASE
#define TORRENT_INTRUSIVE_PTR_BASE

#include <boost/detail/atomic_count.hpp>
#include "libtorrent/config.hpp"
#include "libtorrent/assert.hpp"

namespace libtorrent
{
	template<class T>
	struct intrusive_ptr_base
	{
		intrusive_ptr_base(intrusive_ptr_base<T> const&)
			: m_refs(0) {}

		friend void intrusive_ptr_add_ref(intrusive_ptr_base<T> const* s)
		{
			assert(s->m_refs >= 0);
			assert(s != 0);
			++s->m_refs;
		}

		friend void intrusive_ptr_release(intrusive_ptr_base<T> const* s)
		{
			assert(s->m_refs > 0);
			assert(s != 0);
			if (--s->m_refs == 0)
				delete static_cast<T const*>(s);
		}

		boost::intrusive_ptr<T> self()
		{ return boost::intrusive_ptr<T>((T*)this); }

		boost::intrusive_ptr<const T> self() const
		{ return boost::intrusive_ptr<const T>((T const*)this); }

		int refcount() const { return m_refs; }

		intrusive_ptr_base(): m_refs(0) {}
	private:
		// reference counter for intrusive_ptr
		mutable boost::detail::atomic_count m_refs;
	};

}

#endif

<commit_msg>fix to make torrent_info assignable<commit_after>/*

Copyright (c) 2007, Arvid Norberg
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef TORRENT_INTRUSIVE_PTR_BASE
#define TORRENT_INTRUSIVE_PTR_BASE

#include <boost/detail/atomic_count.hpp>
#include "libtorrent/config.hpp"
#include "libtorrent/assert.hpp"

namespace libtorrent
{
	template<class T>
	struct intrusive_ptr_base
	{
		intrusive_ptr_base(intrusive_ptr_base<T> const&)
			: m_refs(0) {}

		intrusive_ptr_base& operator=(intrusive_ptr_base const& rhs) {}

		friend void intrusive_ptr_add_ref(intrusive_ptr_base<T> const* s)
		{
			assert(s->m_refs >= 0);
			assert(s != 0);
			++s->m_refs;
		}

		friend void intrusive_ptr_release(intrusive_ptr_base<T> const* s)
		{
			assert(s->m_refs > 0);
			assert(s != 0);
			if (--s->m_refs == 0)
				delete static_cast<T const*>(s);
		}

		boost::intrusive_ptr<T> self()
		{ return boost::intrusive_ptr<T>((T*)this); }

		boost::intrusive_ptr<const T> self() const
		{ return boost::intrusive_ptr<const T>((T const*)this); }

		int refcount() const { return m_refs; }

		intrusive_ptr_base(): m_refs(0) {}
	private:
		// reference counter for intrusive_ptr
		mutable boost::detail::atomic_count m_refs;
	};

}

#endif

<|endoftext|>


<commit_before>#include "PublishFilePanel.h"
PublishFilePanel::PublishFilePanel(BMessenger* target)
	:	BFilePanel(B_SAVE_PANEL)
	//	BFilePanel(B_SAVE_PANEL, BMessenger*, const entry_ref*, long unsigned int, bool, BMessage*, BRefFilter*, bool, bool)
{
	publishTypeMenu = new BMenu("Type");
	publishTypeMenu->AddItem(new BMenuItem("HTML", new BMessage(PUBLISH_TYPE)));
	publishTypeMenu->AddItem(new BMenuItem("LaTeX", new BMessage(PUBLISH_TYPE)));
	publishTypeMenu->SetLabelFromMarked(true);
	publishTypeMenu->ItemAt(0)->SetMarked(true);
	/*
	how to position item below something.
	r = fTargetText->Frame();
	r.OffsetBy(0,r.Height() + 10.0);
	*/
	publishTypeMenuField = new BMenuField(BRect(10, 10, 100, 40), "pubtype", "File Type:", publishTypeMenu, B_FOLLOW_ALL, B_WILL_DRAW | B_NAVIGABLE);
	publishTypeMenuField->SetDivider(publishTypeMenuField->StringWidth("File Type:") + 5.0);
	BWindow *w;
	w = Window();
	if(w->Lock())
	{
		BRect btnrect;
		BView* v = w->ChildAt(0);
		BView *cancelBtn = v->FindView("cancel button");

		if(cancelBtn)
		{
			BView* parentview;
			float charWidth;

			charWidth = cancelBtn->StringWidth("Select Current Directory");
			btnrect = cancelBtn->Frame();
			btnrect.right = btnrect.left - 10;
			btnrect.left = btnrect.right - charWidth - 40;
			fCurrentDirBtn = new BButton(btnrect, "current dir button", "Select Current Directory", NULL, B_FOLLOW_RIGHT | B_FOLLOW_BOTTOM);

			fCurrentDirBtn->SetTarget(*target);

			parentview = cancelBtn->Parent();
			//parentview->AddChild(fCurrentDirBtn);
			parentview->AddChild(publishTypeMenuField);

			entry_ref ref;
			BMessage* msg = new BMessage('slct');
			GetPanelDirectory(&ref);
			msg->AddRef("refs", &ref);
			fCurrentDirBtn->SetMessage(msg);
		}

		w->Unlock();
	}
}

void PublishFilePanel::SelectionChanged()
{
  BWindow* wind;

  wind = Window();

  if (wind->Lock()) {
    entry_ref currRef;

    GetPanelDirectory(&currRef);

    //  modify the btn's msg

    BMessage* msg = new BMessage('slct');
    msg->AddRef("refs",&currRef);
    fCurrentDirBtn->SetMessage(msg);

    wind->Unlock();
  }
}
/*
class TDirFilter : public BRefFilter {

public:
  bool Filter(const entry_ref* e, BNode* n, struct stat* s,
              const char* mimetype);
};

bool
TDirFilter::Filter(const entry_ref* e, BNode* n,
                   struct stat* s, const char* mimetype)
{
  if (strcmp("application/x-vnd.Be-directory",
             mimetype) == 0)
    return true;
  else if (strcmp("application/x-vnd.Be-volume",
                  mimetype) == 0)
    return true;
  else
    return false;
}
*/
<commit_msg>working on publishfilepanel implementation of filepanel save panel so i  can publish the file to somewhere<commit_after>#include "PublishFilePanel.h"
PublishFilePanel::PublishFilePanel(BMessenger* target)
	:	BFilePanel(B_SAVE_PANEL)
	//	BFilePanel(B_SAVE_PANEL, BMessenger*, const entry_ref*, long unsigned int, bool, BMessage*, BRefFilter*, bool, bool)
{
	BWindow *w;
	w = Window();
	if(w->Lock())
	{
		BRect btnrect;
		BView* v = w->ChildAt(0);
		BView *cancelBtn = v->FindView("cancel button");

		if(cancelBtn)
		{
			BView* parentview;
			float charWidth;

			charWidth = cancelBtn->StringWidth("Select Current Directory");
			btnrect = cancelBtn->Frame();
			btnrect.right = btnrect.left - 10;
			btnrect.left = btnrect.right - charWidth - 40;
			fCurrentDirBtn = new BButton(btnrect, "current dir button", "Select Current Directory", NULL, B_FOLLOW_RIGHT | B_FOLLOW_BOTTOM);

			fCurrentDirBtn->SetTarget(*target);

			parentview = cancelBtn->Parent();

			publishTypeMenu = new BMenu("Type");
			publishTypeMenu->AddItem(new BMenuItem("HTML", new BMessage(PUBLISH_TYPE)));
			publishTypeMenu->AddItem(new BMenuItem("LaTeX", new BMessage(PUBLISH_TYPE)));
			publishTypeMenu->SetLabelFromMarked(true);
			publishTypeMenu->ItemAt(0)->SetMarked(true);
			/*
			how to position item below something.
			r = fTargetText->Frame();
			r.OffsetBy(0,r.Height() + 10.0);
			*/
			publishTypeMenuField = new BMenuField(BRect(10, 10, 100, 40), "pubtype", "File Type:", publishTypeMenu, B_FOLLOW_ALL, B_WILL_DRAW | B_NAVIGABLE);
			publishTypeMenuField->SetDivider(publishTypeMenuField->StringWidth("File Type:") + 5.0);
			parentview->AddChild(publishTypeMenuField);

			entry_ref ref;
			BMessage* msg = new BMessage('slct');
			GetPanelDirectory(&ref);
			msg->AddRef("refs", &ref);
			fCurrentDirBtn->SetMessage(msg);
		}

		w->Unlock();
	}
}

void PublishFilePanel::SelectionChanged()
{
  BWindow* wind;

  wind = Window();

  if (wind->Lock()) {
    entry_ref currRef;

    GetPanelDirectory(&currRef);

    //  modify the btn's msg

    BMessage* msg = new BMessage('slct');
    msg->AddRef("refs",&currRef);
    fCurrentDirBtn->SetMessage(msg);

    wind->Unlock();
  }
}
/*
class TDirFilter : public BRefFilter {

public:
  bool Filter(const entry_ref* e, BNode* n, struct stat* s,
              const char* mimetype);
};

bool
TDirFilter::Filter(const entry_ref* e, BNode* n,
                   struct stat* s, const char* mimetype)
{
  if (strcmp("application/x-vnd.Be-directory",
             mimetype) == 0)
    return true;
  else if (strcmp("application/x-vnd.Be-volume",
                  mimetype) == 0)
    return true;
  else
    return false;
}
*/
<|endoftext|>


<commit_before>//
// Copyright (c) 2013-2014 Christoph Malek
// See LICENSE for more information.
//

#ifndef RJ_GAME_ENTITY_HANDLER_HPP
#define RJ_GAME_ENTITY_HANDLER_HPP

#include "components/player.hpp"
#include "collision.hpp"
#include "entity.hpp"

#include <mlk/log/log.h>
#include <mlk/types/types.h>

#include <vector>

namespace rj
{
	template<typename T>
	using entity_ptr = mlk::sptr<T>;
	using entity_base_ptr = entity_ptr<entity_base>;
	using player_ptr = entity_ptr<player>;

	class game;
	class entity_handler
	{
		game& m_game;

		player_ptr m_player{nullptr};
		std::vector<entity_base_ptr> m_entities;
		std::size_t m_max_entities;
		std::size_t m_current_id{0};

		static constexpr float m_despawn_zone{0.f};

	public:
		entity_handler(game& g, std::size_t max_entities = 1000) :
			m_game{g},
			m_max_entities{max_entities}
		{ }

		bool create_entity(const entity_base_ptr& e) noexcept
		{
			if(!this->is_entity_valid(e))
				return false;

			this->create_entity_impl(e); // add/create entity in handler
			return true;
		}

		bool create_entity(const player_ptr& p) noexcept
		{
			if(!this->is_entity_valid(p) || this->is_player_registered())
				return false;

			this->create_entity_impl(p);
			return true;
		}

		void update(dur duration)
		{
			// update player
			if(this->is_player_registered())
				m_player->update(duration);

			// update other
			for(auto& a : m_entities)
			{
				a->update(duration);
				if(a->right_out() <= m_despawn_zone)
					a->destroy();
			}

			this->check_collision();

			// erase flagged entities
			this->erase_destroyed();
		}

		void render()
		{
			// render player
			if(this->is_player_registered())
				m_player->render();

			// render other
			for(auto& a : m_entities)
				a->render();
		}

		void clear() noexcept
		{m_entities.clear();}

		std::size_t num_entities() const noexcept
		{return m_entities.size() + this->is_player_registered();}

	private:
		void check_collision() noexcept
		{
			if(!this->is_player_registered())
				return;

			bool collided{false};

			for(auto& a : m_entities)
			{
				if(is_colliding(*m_player, *a))
				{
					collided = true;

					if(m_player->bottom_out() - 2 <= a->top_out())
					{
						m_player->on_collision(a->top_out());
						m_player->render_object().setFillColor({0, 255, 0});
					}
					else
						m_player->render_object().setFillColor({255, 0, 0});

					if(a->has_propertie(entity_propertie::death))
					{
						// player touched death entity
						m_player->render_object().setFillColor({255, 0, 0});
					}
				}
			}
			if(!collided)
				m_player->on_collision_end();
		}

		// checking the entities
		bool is_entity_valid(const entity_base_ptr& e) const noexcept
		{
			if(m_entities.size() >= m_max_entities)
			{
				mlk::lout("rj::entity_handler") << "max_entities limit is reached, can't add more entities";
				return false;
			}
			if(e->is_registered())
			{
				mlk::lout("rj::entity_handler") << "entity with id '" << e->m_id << "' exists already in entity handler, ignoring";
				return false;
			}
			return true;
		}

		bool is_player_registered() const noexcept
		{return m_player != nullptr;}

		// create the entities
		void create_entity_impl(const entity_base_ptr& e) noexcept
		{
			this->register_impl(e);
			m_entities.push_back(e);
		}

		void create_entity_impl(const player_ptr& p) noexcept
		{
			this->register_impl(p);
			m_player = p;
		}

		void register_impl(const entity_base_ptr& e) noexcept
		{
			// important: set game and init
			e->handler_register(&m_game, m_current_id);
			e->init();
			++m_current_id;
		}

		void erase_destroyed() noexcept
		{
			m_entities.erase(std::remove_if(std::begin(m_entities), std::end(m_entities),
			[](const entity_base_ptr& entity)
			{return entity->m_destroyed;}), end(m_entities));
		}
	};
}

#endif // RJ_GAME_ENTITY_HANDLER_HPP
<commit_msg>added missing std::<commit_after>//
// Copyright (c) 2013-2014 Christoph Malek
// See LICENSE for more information.
//

#ifndef RJ_GAME_ENTITY_HANDLER_HPP
#define RJ_GAME_ENTITY_HANDLER_HPP

#include "components/player.hpp"
#include "collision.hpp"
#include "entity.hpp"

#include <mlk/log/log.h>
#include <mlk/types/types.h>

#include <vector>

namespace rj
{
	template<typename T>
	using entity_ptr = mlk::sptr<T>;
	using entity_base_ptr = entity_ptr<entity_base>;
	using player_ptr = entity_ptr<player>;

	class game;
	class entity_handler
	{
		game& m_game;

		player_ptr m_player{nullptr};
		std::vector<entity_base_ptr> m_entities;
		std::size_t m_max_entities;
		std::size_t m_current_id{0};

		static constexpr float m_despawn_zone{0.f};

	public:
		entity_handler(game& g, std::size_t max_entities = 1000) :
			m_game{g},
			m_max_entities{max_entities}
		{ }

		bool create_entity(const entity_base_ptr& e) noexcept
		{
			if(!this->is_entity_valid(e))
				return false;

			this->create_entity_impl(e); // add/create entity in handler
			return true;
		}

		bool create_entity(const player_ptr& p) noexcept
		{
			if(!this->is_entity_valid(p) || this->is_player_registered())
				return false;

			this->create_entity_impl(p);
			return true;
		}

		void update(dur duration)
		{
			// update player
			if(this->is_player_registered())
				m_player->update(duration);

			// update other
			for(auto& a : m_entities)
			{
				a->update(duration);
				if(a->right_out() <= m_despawn_zone)
					a->destroy();
			}

			this->check_collision();

			// erase flagged entities
			this->erase_destroyed();
		}

		void render()
		{
			// render player
			if(this->is_player_registered())
				m_player->render();

			// render other
			for(auto& a : m_entities)
				a->render();
		}

		void clear() noexcept
		{m_entities.clear();}

		std::size_t num_entities() const noexcept
		{return m_entities.size() + this->is_player_registered();}

	private:
		void check_collision() noexcept
		{
			if(!this->is_player_registered())
				return;

			bool collided{false};

			for(auto& a : m_entities)
			{
				if(is_colliding(*m_player, *a))
				{
					collided = true;

					if(m_player->bottom_out() - 2 <= a->top_out())
					{
						m_player->on_collision(a->top_out());
						m_player->render_object().setFillColor({0, 255, 0});
					}
					else
						m_player->render_object().setFillColor({255, 0, 0});

					if(a->has_propertie(entity_propertie::death))
					{
						// player touched death entity
						m_player->render_object().setFillColor({255, 0, 0});
					}
				}
			}
			if(!collided)
				m_player->on_collision_end();
		}

		// checking the entities
		bool is_entity_valid(const entity_base_ptr& e) const noexcept
		{
			if(m_entities.size() >= m_max_entities)
			{
				mlk::lout("rj::entity_handler") << "max_entities limit is reached, can't add more entities";
				return false;
			}
			if(e->is_registered())
			{
				mlk::lout("rj::entity_handler") << "entity with id '" << e->m_id << "' exists already in entity handler, ignoring";
				return false;
			}
			return true;
		}

		bool is_player_registered() const noexcept
		{return m_player != nullptr;}

		// create the entities
		void create_entity_impl(const entity_base_ptr& e) noexcept
		{
			this->register_impl(e);
			m_entities.push_back(e);
		}

		void create_entity_impl(const player_ptr& p) noexcept
		{
			this->register_impl(p);
			m_player = p;
		}

		void register_impl(const entity_base_ptr& e) noexcept
		{
			// important: set game and init
			e->handler_register(&m_game, m_current_id);
			e->init();
			++m_current_id;
		}

		void erase_destroyed() noexcept
		{
			m_entities.erase(std::remove_if(std::begin(m_entities), std::end(m_entities),
			[](const entity_base_ptr& entity)
			{return entity->m_destroyed;}), std::end(m_entities));
		}
	};
}

#endif // RJ_GAME_ENTITY_HANDLER_HPP
<|endoftext|>


<commit_before>#include "PySoundGenerator.hpp"

/**
 * @brief PySoundGenerator::PySoundGenerator
 * @param progName
 * @param pyInstructions
 *
 * The constructor of the PySoundGeneratorclass.
 * Sets up the python interpreter, the instructions with
 * which it will be fed, the class variables and the break shortcut.
 */
PySoundGenerator::PySoundGenerator(char* progName, char* pyInstructions){
    if(pyInstructions == QString()){
        emit doneSignal(tr("File is empty. Nothing to execute."), -1);
        return;
    }

    Py_SetProgramName(progName);
    Py_Initialize();
    ownExcept = QString();
    exceptNum = -1;
    abortAction = new QAction(this);
    abortAction->setShortcut(QKeySequence("Ctrl-C"));
    connect(abortAction, SIGNAL(triggered()), this, SLOT(terminated()));

    PyObject* module = PyImport_AddModule("__main__");
    sys = PyImport_ImportModule("sys");
//    PyObject *path = PyObject_GetAttrString(sys, "path");
//    PyList_Append(path, PyString_FromString("../../../"));
//    PyList_Append(path, PyString_FromString("."));
    dict = PyModule_GetDict(module);
    if(!dict){
        exceptionOccurred();
        emit doneSignal(ownExcept, -1);
        return;
    }
    execute("import AudioPython");
    execute(pyInstructions);
    execute("samples = AudioPython.compute_samples(channels, None)");

    device = new AudioOutputProcessor();
    connect(device, SIGNAL(startWriting()), this, SLOT(setReady()));
    ready = true;
    device->start();
}

/**
 * @brief PySoundGenerator::~PySoundGenerator
 *
 * The destructor of the PySoundGenerator class.
 * Kills the python interpreter.
 */
PySoundGenerator::~PySoundGenerator(){
    Py_DECREF(sys);
    delete abortAction;
    Py_Finalize();
    device->exit();
    // Wait for the end of QThread before deletion, otherwise it raises an error
    while(device->isRunning())
        ;
    delete device;
}

/**
 * @brief PySoundGenerator::run
 *
 * The main loop. Calls the user code executing method
 * and emits a signal when it's done.
 */
void PySoundGenerator::run(){
    write();
    while(!ready)
        QCoreApplication::processEvents();
}

/**
 * @brief PySoundGenerator::write
 *
 * Gets samples and does error handling. The actual
 * main loop, so to say.
 */
void PySoundGenerator::write(){
    while(ready){
        PyObject* check = execute("AudioPython.yield_raw(samples, None)");
        if(!check){
            exceptionOccurred();
            emit doneSignal(ownExcept, exceptNum);
            break;
        }
        if(PyBytes_Check(check))
            stream(check);
    }
}

/**
 * @brief PySoundGenerator::execute
 * @return PyObject* / NULL if there was an exception
 *         in the python interpreter.
 *
 * executes the python code in the interpreter.
 */
PyObject* PySoundGenerator::execute(QString instruct){
    return PyRun_String(instruct.toLocal8Bit().data(),
                        Py_file_input, dict, dict);
}

/**
 * @brief PySoundGenerator::stream
 * @param process
 *
 * writes the generated bytes to the IODevice
 */
void PySoundGenerator::stream(PyObject* data){
    ready = device->write(PyBytes_AsString(data), PyBytes_Size(data));
}

/**
 * @brief PySoundGenerator::updateCode
 * @param filename
 * @param instructions
 * @return true if the name of the program could be changed,
 *         false otherwise
 *
 * Updates the code of the currently running Python interpreter.
 */
bool PySoundGenerator::updateCode(QString filename, QString instructions){
    Py_SetProgramName(filename.toLocal8Bit().data());
    execute(instructions.toLocal8Bit().data());
    return true;
}

/**
 * @brief PySoundGenerator::exceptionOccurred
 * @return PythonException
 *
 * Fetches the Python Exception and translates it to a QString
 * and an int representing exception and line number.
 */
void PySoundGenerator::exceptionOccurred(){
    PyObject *errtype, *errvalue, *traceback;
    PyObject *mod;
    PyObject *ret, *list, *string;
    PyErr_Fetch(&errtype, &errvalue, &traceback);
    PyErr_NormalizeException(&errtype, &errvalue, &traceback);
    QString exceptionText = QString(PyString_AsString(PyObject_Str(errtype)));
    exceptionText.append(": '");
    exceptionText.append(PyString_AsString(PyObject_Str(errvalue)));
    QRegExp line("line [0-9]+");
    if(line.indexIn(exceptionText) != -1){
        QString text = line.capturedTexts().at(0);
        exceptionText.replace(" (<string>, " + text + ")", "");
        exceptionText.append("' at " + text);
        text.replace("line ", "");
        exceptNum = text.toInt();
    } else{
        mod = PyImport_ImportModule("traceback");
        list = PyObject_CallMethod(mod, (char*)"format_exception",
                               (char*)"OOO", errtype, errvalue, traceback);
        if(list == 0){
            ownExcept = exceptionText;
            exceptNum = -1;
            return;
        }
        string = PyString_FromString("\n");
        ret = _PyString_Join(string, list);
        if(line.indexIn(PyString_AsString(ret)) != -1){
            QString text = line.capturedTexts().at(0);
            exceptionText.append("' at " + text);
            text.replace("line ", "");
            exceptNum = text.toInt();
        } else{
            exceptNum = -1;
        }
        Py_DECREF(list);
        Py_DECREF(string);
        Py_DECREF(ret);
    }
    PyErr_Clear();
    ownExcept = exceptionText;
}

/**
 * @brief PySoundGenerator::terminated
 *
 * SLOT that is called when the user interrupt(CTRL-C) SIGNAL
 * is emitted.
 */
void PySoundGenerator::terminated(){
    ownExcept = tr("User Terminated.");
}

/**
 * @brief PySoundGenerator::setReady
 * @param set
 *
 * Setter for the ready-variable that indicates
 * whether writing to the IODevice is allowed.
 */
void PySoundGenerator::setReady(){
    ready = true;
    write();
}
<commit_msg>Updated for Python3<commit_after>#include "PySoundGenerator.hpp"

#if PY_MAJOR_VERSION >= 3
/**
 * @brief PySoundGenerator::PySoundGenerator
 * @param progName
 * @param pyInstructions
 *
 * The constructor of the PySoundGeneratorclass.
 * Sets up the python interpreter, the instructions with
 * which it will be fed, the class variables and the break shortcut.
 */
PySoundGenerator::PySoundGenerator(char* progName, char* pyInstructions){
    if(pyInstructions == QString()){
        emit doneSignal(tr("File is empty. Nothing to execute."), -1);
        return;
    }

    Py_SetProgramName(QString(progName).toWCharArray());
    Py_Initialize();
    ownExcept = QString();
    exceptNum = -1;
    abortAction = new QAction(this);
    abortAction->setShortcut(QKeySequence("Ctrl-C"));
    connect(abortAction, SIGNAL(triggered()), this, SLOT(terminated()));

    PyObject* module = PyImport_AddModule("__main__");
    sys = PyImport_ImportModule("sys");
//    PyObject *path = PyObject_GetAttrString(sys, "path");
//    PyList_Append(path, PyString_FromString("../../../"));
//    PyList_Append(path, PyString_FromString("."));
    dict = PyModule_GetDict(module);
    if(!dict){
        exceptionOccurred();
        emit doneSignal(ownExcept, -1);
        return;
    }
    execute("import AudioPython");
    execute(pyInstructions);
    execute("samples = AudioPython.compute_samples(channels, None)");

    device = new AudioOutputProcessor();
    connect(device, SIGNAL(startWriting()), this, SLOT(setReady()));
    ready = true;
    device->start();
}

/**
 * @brief PySoundGenerator::~PySoundGenerator
 *
 * The destructor of the PySoundGenerator class.
 * Kills the python interpreter.
 */
PySoundGenerator::~PySoundGenerator(){
    Py_DECREF(sys);
    delete abortAction;
    Py_Finalize();
    device->exit();
    // Wait for the end of QThread before deletion, otherwise it raises an error
    while(device->isRunning())
        ;
    delete device;
}

/**
 * @brief PySoundGenerator::run
 *
 * The main loop. Calls the user code executing method
 * and emits a signal when it's done.
 */
void PySoundGenerator::run(){
    write();
    while(!ready)
        QCoreApplication::processEvents();
}

/**
 * @brief PySoundGenerator::write
 *
 * Gets samples and does error handling. The actual
 * main loop, so to say.
 */
void PySoundGenerator::write(){
    while(ready){
        PyObject* check = execute("AudioPython.yield_raw(samples, None)");
        if(!check){
            exceptionOccurred();
            emit doneSignal(ownExcept, exceptNum);
            break;
        }
        if(PyBytes_Check(check))
            stream(check);
    }
}

/**
 * @brief PySoundGenerator::execute
 * @return PyObject* / NULL if there was an exception
 *         in the python interpreter.
 *
 * executes the python code in the interpreter.
 */
PyObject* PySoundGenerator::execute(QString instruct){
    return PyRun_String(instruct.toLocal8Bit().data(),
                        Py_file_input, dict, dict);
}

/**
 * @brief PySoundGenerator::stream
 * @param process
 *
 * writes the generated bytes to the IODevice
 */
void PySoundGenerator::stream(PyObject* data){
    ready = device->write(PyBytes_AsString(data), PyBytes_Size(data));
}

/**
 * @brief PySoundGenerator::updateCode
 * @param filename
 * @param instructions
 * @return true if the name of the program could be changed,
 *         false otherwise
 *
 * Updates the code of the currently running Python interpreter.
 */
bool PySoundGenerator::updateCode(QString filename, QString instructions){
    Py_SetProgramName(filename.toWCharArray());
    execute(instructions.toLocal8Bit().data());
    return true;
}

/**
 * @brief PySoundGenerator::exceptionOccurred
 * @return PythonException
 *
 * Fetches the Python Exception and translates it to a QString
 * and an int representing exception and line number.
 */
void PySoundGenerator::exceptionOccurred(){
    PyObject *errtype, *errvalue, *traceback;
    PyObject *mod;
    PyObject *ret, *list, *string;
    PyErr_Fetch(&errtype, &errvalue, &traceback);
    PyErr_NormalizeException(&errtype, &errvalue, &traceback);
    QString exceptionText = QString(PyBytes_AsString(PyObject_Str(errtype)));
    exceptionText.append(": '");
    exceptionText.append(PyBytes_AsString(PyObject_Str(errvalue)));
    QRegExp line("line [0-9]+");
    if(line.indexIn(exceptionText) != -1){
        QString text = line.capturedTexts().at(0);
        exceptionText.replace(" (<string>, " + text + ")", "");
        exceptionText.append("' at " + text);
        text.replace("line ", "");
        exceptNum = text.toInt();
    } else{
        mod = PyImport_ImportModule("traceback");
        list = PyObject_CallMethod(mod, (char*)"format_exception",
                               (char*)"OOO", errtype, errvalue, traceback);
        if(list == 0){
            ownExcept = exceptionText;
            exceptNum = -1;
            return;
        }
        string = PyBytes_FromString("\n");
        ret = _PyBytes_Join(string, list);
        if(line.indexIn(PyBytes_AsString(ret)) != -1){
            QString text = line.capturedTexts().at(0);
            exceptionText.append("' at " + text);
            text.replace("line ", "");
            exceptNum = text.toInt();
        } else{
            exceptNum = -1;
        }
        Py_DECREF(list);
        Py_DECREF(string);
        Py_DECREF(ret);
    }
    PyErr_Clear();
    ownExcept = exceptionText;
}

/**
 * @brief PySoundGenerator::terminated
 *
 * SLOT that is called when the user interrupt(CTRL-C) SIGNAL
 * is emitted.
 */
void PySoundGenerator::terminated(){
    ownExcept = tr("User Terminated.");
}

/**
 * @brief PySoundGenerator::setReady
 * @param set
 *
 * Setter for the ready-variable that indicates
 * whether writing to the IODevice is allowed.
 */
void PySoundGenerator::setReady(){
    ready = true;
    write();
}
#else
/**
 * @brief PySoundGenerator::PySoundGenerator
 * @param progName
 * @param pyInstructions
 *
 * The constructor of the PySoundGeneratorclass.
 * Sets up the python interpreter, the instructions with
 * which it will be fed, the class variables and the break shortcut.
 */
PySoundGenerator::PySoundGenerator(char* progName, char* pyInstructions){
    if(pyInstructions == QString()){
        emit doneSignal(tr("File is empty. Nothing to execute."), -1);
        return;
    }

    Py_SetProgramName(progName);
    Py_Initialize();
    ownExcept = QString();
    exceptNum = -1;
    abortAction = new QAction(this);
    abortAction->setShortcut(QKeySequence("Ctrl-C"));
    connect(abortAction, SIGNAL(triggered()), this, SLOT(terminated()));

    PyObject* module = PyImport_AddModule("__main__");
    sys = PyImport_ImportModule("sys");
//    PyObject *path = PyObject_GetAttrString(sys, "path");
//    PyList_Append(path, PyString_FromString("../../../"));
//    PyList_Append(path, PyString_FromString("."));
    dict = PyModule_GetDict(module);
    if(!dict){
        exceptionOccurred();
        emit doneSignal(ownExcept, -1);
        return;
    }
    execute("import AudioPython");
    execute(pyInstructions);
    execute("samples = AudioPython.compute_samples(channels, None)");

    device = new AudioOutputProcessor();
    connect(device, SIGNAL(startWriting()), this, SLOT(setReady()));
    ready = true;
    device->start();
}

/**
 * @brief PySoundGenerator::~PySoundGenerator
 *
 * The destructor of the PySoundGenerator class.
 * Kills the python interpreter.
 */
PySoundGenerator::~PySoundGenerator(){
    Py_DECREF(sys);
    delete abortAction;
    Py_Finalize();
    device->exit();
    // Wait for the end of QThread before deletion, otherwise it raises an error
    while(device->isRunning())
        ;
    delete device;
}

/**
 * @brief PySoundGenerator::run
 *
 * The main loop. Calls the user code executing method
 * and emits a signal when it's done.
 */
void PySoundGenerator::run(){
    write();
    while(!ready)
        QCoreApplication::processEvents();
}

/**
 * @brief PySoundGenerator::write
 *
 * Gets samples and does error handling. The actual
 * main loop, so to say.
 */
void PySoundGenerator::write(){
    while(ready){
        PyObject* check = execute("AudioPython.yield_raw(samples, None)");
        if(!check){
            exceptionOccurred();
            emit doneSignal(ownExcept, exceptNum);
            break;
        }
        if(PyBytes_Check(check))
            stream(check);
    }
}

/**
 * @brief PySoundGenerator::execute
 * @return PyObject* / NULL if there was an exception
 *         in the python interpreter.
 *
 * executes the python code in the interpreter.
 */
PyObject* PySoundGenerator::execute(QString instruct){
    return PyRun_String(instruct.toLocal8Bit().data(),
                        Py_file_input, dict, dict);
}

/**
 * @brief PySoundGenerator::stream
 * @param process
 *
 * writes the generated bytes to the IODevice
 */
void PySoundGenerator::stream(PyObject* data){
    ready = device->write(PyBytes_AsString(data), PyBytes_Size(data));
}

/**
 * @brief PySoundGenerator::updateCode
 * @param filename
 * @param instructions
 * @return true if the name of the program could be changed,
 *         false otherwise
 *
 * Updates the code of the currently running Python interpreter.
 */
bool PySoundGenerator::updateCode(QString filename, QString instructions){
    Py_SetProgramName(filename.toLocal8Bit().data());
    execute(instructions.toLocal8Bit().data());
    return true;
}

/**
 * @brief PySoundGenerator::exceptionOccurred
 * @return PythonException
 *
 * Fetches the Python Exception and translates it to a QString
 * and an int representing exception and line number.
 */
void PySoundGenerator::exceptionOccurred(){
    PyObject *errtype, *errvalue, *traceback;
    PyObject *mod;
    PyObject *ret, *list, *string;
    PyErr_Fetch(&errtype, &errvalue, &traceback);
    PyErr_NormalizeException(&errtype, &errvalue, &traceback);
    QString exceptionText = QString(PyString_AsString(PyObject_Str(errtype)));
    exceptionText.append(": '");
    exceptionText.append(PyString_AsString(PyObject_Str(errvalue)));
    QRegExp line("line [0-9]+");
    if(line.indexIn(exceptionText) != -1){
        QString text = line.capturedTexts().at(0);
        exceptionText.replace(" (<string>, " + text + ")", "");
        exceptionText.append("' at " + text);
        text.replace("line ", "");
        exceptNum = text.toInt();
    } else{
        mod = PyImport_ImportModule("traceback");
        list = PyObject_CallMethod(mod, (char*)"format_exception",
                               (char*)"OOO", errtype, errvalue, traceback);
        if(list == 0){
            ownExcept = exceptionText;
            exceptNum = -1;
            return;
        }
        string = PyString_FromString("\n");
        ret = _PyString_Join(string, list);
        if(line.indexIn(PyString_AsString(ret)) != -1){
            QString text = line.capturedTexts().at(0);
            exceptionText.append("' at " + text);
            text.replace("line ", "");
            exceptNum = text.toInt();
        } else{
            exceptNum = -1;
        }
        Py_DECREF(list);
        Py_DECREF(string);
        Py_DECREF(ret);
    }
    PyErr_Clear();
    ownExcept = exceptionText;
}

/**
 * @brief PySoundGenerator::terminated
 *
 * SLOT that is called when the user interrupt(CTRL-C) SIGNAL
 * is emitted.
 */
void PySoundGenerator::terminated(){
    ownExcept = tr("User Terminated.");
}

/**
 * @brief PySoundGenerator::setReady
 * @param set
 *
 * Setter for the ready-variable that indicates
 * whether writing to the IODevice is allowed.
 */
void PySoundGenerator::setReady(){
    ready = true;
    write();
}
#endif
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
//#include <codecvt>
//#include <cstddef>
#include <locale>
#include <string>
#include <iomanip>

#include "cp-neural.h"

using std::wcout; using std::wstring;
using std::cerr; using std::vector;

class Text {
private:
    bool isinit=false;
    bool readDataFile(string inputfile) {
        std::wifstream txtfile(inputfile, std::ios::binary);
        if (!txtfile.is_open()) {
            return false;
        }
        txtfile.imbue(std::locale("en_US.UTF8"));
        wstring ttext((std::istreambuf_iterator<wchar_t>(txtfile)),
                     std::istreambuf_iterator<wchar_t>());
        txtfile.close();
        text=ttext;
        filename=inputfile;
        return true;
    }
public:
    wstring text;
    string filename;
    std::map<wchar_t,int> freq;
    std::map<wchar_t,int> w2v;
    std::map<int,wchar_t> v2w;
    Text(string filename) {
        if (readDataFile(filename)) {
            for (auto wc : text) {
                freq[wc]++;
            }
            int it=0;
            for (auto wc : freq) {
                w2v[wc.first]=it;
                v2w[it]=wc.first;
                ++it;
            }
            isinit=true;
            cerr << "Freq:" << freq.size() << ", w2v:" << w2v.size() << ", v2w:" << v2w.size() << endl;
        }
    }
    ~Text() {

    }
    bool isInit() {
        return isinit;
    }
    int vocsize() {
        if (!isinit) return 0;
        return v2w.size();
    }
};

int main(int argc, char *argv[]) {
    std::setlocale (LC_ALL, "");
    wcout << L"Rnn-Readär" << std::endl;

    bool allOk=true;
    if (argc!=2) {
        cerr << "rnnreader <path-to-text-file>" << endl;
        exit(-1);
    }
    Text txt(argv[1]);
    if (!txt.isInit()) {
        cerr << "Cannot initialize Text object from inputfile: " << argv[1] << endl;
        exit(-1);
    }

    wcout << L"Text size: " << txt.text.size() << endl;

    wcout << L"Size of vocabulary:" << txt.freq.size() << std::endl;
/*    for (auto f : txt.freq) {
        int c=(int)f.first;
        wstring wc(1,f.first);
        wcout << wc << L"|" <<  wchar_t(f.first) << L"(0x" << std::hex << c << L")" ": " << std::dec <<  f.second << endl;
    }
*/
    Color::Modifier red(Color::FG_RED);
    Color::Modifier green(Color::FG_GREEN);
    Color::Modifier def(Color::FG_DEFAULT);

    int T=4;
    int N=txt.text.size()-T-1;

    MatrixN Xr(N,T);
    MatrixN yr(N,T);

    wstring chunk,chunky;
    int n=0;
    for (int i=0; i<N-T-1; i+=T) {
        chunk=txt.text.substr(i,T);
        chunky=txt.text.substr(i+1,T);
        for (int t=0; t<T; t++) {
            Xr(i,t)=txt.w2v[chunk[t]];
            yr(i,t)=txt.w2v[chunky[t]];
            ++n;
        }
    }

    int maxN = 50000;
    if (n>maxN) n=maxN;
    int n1=n*0.9;
    int dn=(n-n1)/2;

    cerr << n1 << " datasets" << endl;

/*    MatrixN X(n1,T);
    MatrixN y(n1,T);
    MatrixN Xv(dn,T);
    MatrixN yv(dn,T);
    MatrixN Xt(dn,T);
    MatrixN yt(dn,T);
*/
    MatrixN X=Xr.block(0,0,n1,T);
    MatrixN y=yr.block(0,0,n1,T);
    MatrixN Xv=Xr.block(n1,0,dn,T);
    MatrixN yv=yr.block(n1,0,dn,T);
    MatrixN Xt=Xr.block(n1+dn,0,dn,T);
    MatrixN yt=yr.block(n1+dn,0,dn,T);
/*

    X=Xr.block(0,0,100000,T);
    y=yr.block(0,0,100000,T);
    Xv=Xr.block(100000,0,10000,T);
    yv=yr.block(100000,0,10000,T);
    Xt=Xr.block(110000,0,10000,T);
    yt=yr.block(110000,0,10000,T);
*/
/*
    X=Xr.block(0,0,10000,T);
    y=yr.block(0,0,10000,T);
    Xv=Xr.block(10000,0,1000,T);
    yv=yr.block(10000,0,1000,T);
    Xt=Xr.block(11000,0,1000,T);
    yt=yr.block(11000,0,1000,T);
*/
    cpInitCompute("Rnnreader");
    registerLayers();

    LayerBlock lb("{name='rnnreader';init='orthonormal';initfactor=0.001}");
    int VS=txt.vocsize();
    int H=64;
    int D=128;
    int BS=64;
    float clip=5.0;

    CpParams cp0;
    cp0.setPar("inputShape",vector<int>{T});
    cp0.setPar("V",VS);
    cp0.setPar("D",D);
    lb.addLayer("WordEmbedding","WE0",cp0,{"input"});

    CpParams cp1;
    cp1.setPar("inputShape",vector<int>{D,T});
    cp1.setPar("N",BS);
    cp1.setPar("H",H);
    cp1.setPar("clip",clip);
    lb.addLayer("RNN","rnn1",cp1,{"WE0"});

    CpParams cp2;
    cp2.setPar("inputShape",vector<int>{H,T});
    cp2.setPar("N",BS);
    cp2.setPar("H",H);
    cp2.setPar("clip",clip);
    lb.addLayer("RNN","rnn2",cp2,{"rnn1"});

    CpParams cp3;
    cp3.setPar("inputShape",vector<int>{H,T});
    cp3.setPar("N",BS);
    cp3.setPar("H",H);
    cp3.setPar("clip",clip);
    lb.addLayer("RNN","rnn3",cp3,{"rnn2"});

    CpParams cp10;
    cp10.setPar("inputShape",vector<int>{H,T});
    //cp10.setPar("T",T);
    //cp10.setPar("D",H);
    cp10.setPar("M",VS);
    lb.addLayer("TemporalAffine","af1",cp10,{"rnn3"});

    CpParams cp11;
    cp11.setPar("inputShape",vector<int>{VS,T});
    lb.addLayer("TemporalSoftmax","sm1",cp11,{"af1"});

    if (!lb.checkTopology(true)) {
        allOk=false;
        cerr << red << "Topology-check for LayerBlock: ERROR." << def << endl;
    } else {
        cerr << green << "Topology-check for LayerBlock: ok." << def << endl;
    }

/*    wstring chunk;
    chunk = txt.text.substr(512,128);
    wcout << chunk << endl;
*/
    CpParams cpo("{verbose=true;epsilon=1e-8}");
    // CpParams cpo("{verbose=false;epsilon=1e-8}");
    cpo.setPar("learning_rate", (floatN)1e-2); //2.2e-2);
    cpo.setPar("lr_decay", (floatN)0.95);
    //cpo.setPar("regularization", (floatN)1.);

    floatN dep=1.0;
    floatN sep=0.0;
    cpo.setPar("epochs",(floatN)dep);
    cpo.setPar("batch_size",BS);

    MatrixN xg(1,T);
    MatrixN xg2(1,T);
    wstring sg;
    for (int i=0; i<10000; i++) {

        cpo.setPar("startepoch", (floatN)sep);
        cpo.setPar("maxthreads", (int)1); // RNN can't cope with threads.

        Layer *prnn1=lb.layerMap["rnn1"];
        prnn1->params["ho"]->setZero();
        /*Layer *prnn2=lb.layerMap["rnn2"];
        prnn2->params["ho"]->setZero();
        Layer *prnn3=lb.layerMap["rnn3"];
        prnn3->params["ho"]->setZero();
*/
        floatN cAcc=lb.train(X, y, Xv, yv, "Adam", cpo);
        sep+=dep;

        int pos=rand() % 1000 + 5000;
        wstring instr=txt.text.substr(pos,T);

        for (int i=0; i<T; i++) {
            if (i<instr.size()) {
                xg(0,i)=txt.w2v[instr[i]];
            } else {
                xg(0,i)=txt.w2v[L' '];
            }
        }
        /*
        for (auto wc : instr) {
            sg[0]=wc;
            xg(0,0)=txt.w2v[sg[0]];
            MatrixN z(0,0);
            MatrixN yg=lb.forward(xg,z,nullptr);
        }
        */
        //xg(0,0)=txt.w2v[sg[0]];
        for (int rep=0; rep<3; rep++) {
            //Layer *prnn=lb.layerMap["rnn1"];
            //prnn->params["ho"]->setZero();
            cerr << "------------" << rep << "--------------------------" << endl;
            for (int g=0; g<200; g++) {
                //wcout << g << L">";
                MatrixN z(0,0);

                //for (int i; i<xg.cols(); i++) wcout << txt.v2w[xg(0,i)];
                //wcout << L"<" << endl << L">";
                MatrixN probst=lb.forward(xg,z,nullptr);
                MatrixN probsd=MatrixN(N*T,VS);
                for (int n=0; n<1; n++) {
                    for (int t=0; t<T; t++) {
                        for (int d=0; d<VS; d++) {
                            probsd(n*T+t,d)=probst(n,t*VS+d);
                        }
                    }
                }

                for (int t=0; t<T; t++) {
                    vector<floatN> probs(VS);
                    vector<floatN> index(VS);
                    for (int d=0; d<VS; d++) {
                        probs[d]=probsd(0*T+t,d);
                        index[d]=d;
                    }
                    int ind=(int)index[randomChoice(index, probs)];
/*                    float mx=-1000.0;
                    int ind=-1;
                    for (int d=0; d<VS; d++) {
                        floatN p=yg(0,t*D+d);
                        floatN pr=p*((floatN)(rand()%100)/5000.0+0.98);
                        if (pr>mx) {
                            mx=pr; // yg(0,t*D+d);
                            ind=d;
                        }
                    }
*/
                    wchar_t cw=txt.v2w[ind];
                    //if (t==0) wcout << L"[" << cw << L"<";
                    //wcout << L"<" << cw << L">";
                    if (t==0) wcout << cw;
                    xg2(0,t)=ind;
                }
                //wcout << L"<" << endl;

                //for (int t=T-1; t>0; t--) xg(0,t)=xg(0,t-1);
                for (int t=0; t< T-1; t++) xg(0,t)=xg(0,t+1);
                xg(0,T-1)=xg2(0,0);

                //xg=xg2;
            }
            wcout << endl;
        }
    }
    /*
    floatN train_err, val_err, test_err;
    bool evalFinal=true;
    if (evalFinal) {
        train_err=lb.test(X, y, cpo.getPar("batch_size", 50));
        val_err=lb.test(Xv, yv, cpo.getPar("batch_size", 50));
        test_err=lb.test(Xt, yt, cpo.getPar("batch_size", 50));

        cerr << "Final results on RnnReader after " << cpo.getPar("epochs",(floatN)0.0) << " epochs:" << endl;
        cerr << "      Train-error: " << train_err << " train-acc: " << 1.0-train_err << endl;
        cerr << " Validation-error: " << val_err <<   "   val-acc: " << 1.0-val_err << endl;
        cerr << "       Test-error: " << test_err <<  "  test-acc: " << 1.0-test_err << endl;
    }
    //return cAcc;
    */
    cpExitCompute();
}
<commit_msg>stage<commit_after>#include <iostream>
#include <fstream>
//#include <codecvt>
//#include <cstddef>
#include <locale>
#include <string>
#include <iomanip>

#include "cp-neural.h"

using std::wcout; using std::wstring;
using std::cerr; using std::vector;

class Text {
private:
    bool isinit=false;
    bool readDataFile(string inputfile) {
        std::wifstream txtfile(inputfile, std::ios::binary);
        if (!txtfile.is_open()) {
            return false;
        }
        txtfile.imbue(std::locale("en_US.UTF8"));
        wstring ttext((std::istreambuf_iterator<wchar_t>(txtfile)),
                     std::istreambuf_iterator<wchar_t>());
        txtfile.close();
        text=ttext;
        filename=inputfile;
        return true;
    }
public:
    wstring text;
    string filename;
    std::map<wchar_t,int> freq;
    std::map<wchar_t,int> w2v;
    std::map<int,wchar_t> v2w;
    Text(string filename) {
        if (readDataFile(filename)) {
            for (auto wc : text) {
                freq[wc]++;
            }
            int it=0;
            for (auto wc : freq) {
                w2v[wc.first]=it;
                v2w[it]=wc.first;
                ++it;
            }
            isinit=true;
            cerr << "Freq:" << freq.size() << ", w2v:" << w2v.size() << ", v2w:" << v2w.size() << endl;
        }
    }
    ~Text() {

    }
    bool isInit() {
        return isinit;
    }
    int vocsize() {
        if (!isinit) return 0;
        return v2w.size();
    }
};

int main(int argc, char *argv[]) {
    std::setlocale (LC_ALL, "");
    wcout << L"Rnn-Readär" << std::endl;

    bool allOk=true;
    if (argc!=2) {
        cerr << "rnnreader <path-to-text-file>" << endl;
        exit(-1);
    }
    Text txt(argv[1]);
    if (!txt.isInit()) {
        cerr << "Cannot initialize Text object from inputfile: " << argv[1] << endl;
        exit(-1);
    }

    wcout << L"Text size: " << txt.text.size() << endl;

    wcout << L"Size of vocabulary:" << txt.freq.size() << std::endl;
/*    for (auto f : txt.freq) {
        int c=(int)f.first;
        wstring wc(1,f.first);
        wcout << wc << L"|" <<  wchar_t(f.first) << L"(0x" << std::hex << c << L")" ": " << std::dec <<  f.second << endl;
    }
*/
    Color::Modifier red(Color::FG_RED);
    Color::Modifier green(Color::FG_GREEN);
    Color::Modifier def(Color::FG_DEFAULT);

    int T=2;
    int N=txt.text.size()-T-1;

    MatrixN Xr(N,T);
    MatrixN yr(N,T);

    wstring chunk,chunky;
    int n=0;
    for (int i=0; i<N-T-1; i+=T) {
        chunk=txt.text.substr(i,T);
        chunky=txt.text.substr(i+1,T);
        for (int t=0; t<T; t++) {
            Xr(i,t)=txt.w2v[chunk[t]];
            yr(i,t)=txt.w2v[chunky[t]];
            ++n;
        }
    }

    int maxN = 50000;
    if (n>maxN) n=maxN;
    int n1=n*0.9;
    int dn=(n-n1)/2;

    cerr << n1 << " datasets" << endl;

/*    MatrixN X(n1,T);
    MatrixN y(n1,T);
    MatrixN Xv(dn,T);
    MatrixN yv(dn,T);
    MatrixN Xt(dn,T);
    MatrixN yt(dn,T);
*/
    MatrixN X=Xr.block(0,0,n1,T);
    MatrixN y=yr.block(0,0,n1,T);
    MatrixN Xv=Xr.block(n1,0,dn,T);
    MatrixN yv=yr.block(n1,0,dn,T);
    MatrixN Xt=Xr.block(n1+dn,0,dn,T);
    MatrixN yt=yr.block(n1+dn,0,dn,T);
/*

    X=Xr.block(0,0,100000,T);
    y=yr.block(0,0,100000,T);
    Xv=Xr.block(100000,0,10000,T);
    yv=yr.block(100000,0,10000,T);
    Xt=Xr.block(110000,0,10000,T);
    yt=yr.block(110000,0,10000,T);
*/
/*
    X=Xr.block(0,0,10000,T);
    y=yr.block(0,0,10000,T);
    Xv=Xr.block(10000,0,1000,T);
    yv=yr.block(10000,0,1000,T);
    Xt=Xr.block(11000,0,1000,T);
    yt=yr.block(11000,0,1000,T);
*/
    cpInitCompute("Rnnreader");
    registerLayers();

    LayerBlock lb("{name='rnnreader';init='orthonormal';initfactor=0.001}");
    int VS=txt.vocsize();
    int H=64;
    int D=128;
    int BS=64;
    float clip=5.0;

    CpParams cp0;
    cp0.setPar("inputShape",vector<int>{T});
    cp0.setPar("V",VS);
    cp0.setPar("D",D);
    lb.addLayer("WordEmbedding","WE0",cp0,{"input"});

    CpParams cp1;
    cp1.setPar("inputShape",vector<int>{D,T});
    cp1.setPar("N",BS);
    cp1.setPar("H",H);
    cp1.setPar("clip",clip);
    lb.addLayer("RNN","rnn1",cp1,{"WE0"});

    CpParams cp2;
    cp2.setPar("inputShape",vector<int>{H,T});
    cp2.setPar("N",BS);
    cp2.setPar("H",H);
    cp2.setPar("clip",clip);
    lb.addLayer("RNN","rnn2",cp2,{"rnn1"});

    CpParams cp3;
    cp3.setPar("inputShape",vector<int>{H,T});
    cp3.setPar("N",BS);
    cp3.setPar("H",H);
    cp3.setPar("clip",clip);
    lb.addLayer("RNN","rnn3",cp3,{"rnn2"});

    CpParams cp10;
    cp10.setPar("inputShape",vector<int>{H,T});
    //cp10.setPar("T",T);
    //cp10.setPar("D",H);
    cp10.setPar("M",VS);
    lb.addLayer("TemporalAffine","af1",cp10,{"rnn3"});

    CpParams cp11;
    cp11.setPar("inputShape",vector<int>{VS,T});
    lb.addLayer("TemporalSoftmax","sm1",cp11,{"af1"});

    if (!lb.checkTopology(true)) {
        allOk=false;
        cerr << red << "Topology-check for LayerBlock: ERROR." << def << endl;
    } else {
        cerr << green << "Topology-check for LayerBlock: ok." << def << endl;
    }

/*    wstring chunk;
    chunk = txt.text.substr(512,128);
    wcout << chunk << endl;
*/
    CpParams cpo("{verbose=true;epsilon=1e-8}");
    // CpParams cpo("{verbose=false;epsilon=1e-8}");
    cpo.setPar("learning_rate", (floatN)1e-2); //2.2e-2);
    cpo.setPar("lr_decay", (floatN)0.95);
    //cpo.setPar("regularization", (floatN)1.);

    floatN dep=1.0;
    floatN sep=0.0;
    cpo.setPar("epochs",(floatN)dep);
    cpo.setPar("batch_size",BS);

    MatrixN xg(1,T);
    MatrixN xg2(1,T);
    wstring sg;
    for (int i=0; i<10000; i++) {

        cpo.setPar("startepoch", (floatN)sep);
        cpo.setPar("maxthreads", (int)1); // RNN can't cope with threads.

        Layer *prnn1=lb.layerMap["rnn1"];
        prnn1->params["ho"]->setZero();
        /*Layer *prnn2=lb.layerMap["rnn2"];
        prnn2->params["ho"]->setZero();
        Layer *prnn3=lb.layerMap["rnn3"];
        prnn3->params["ho"]->setZero();
*/
        floatN cAcc=lb.train(X, y, Xv, yv, "Adam", cpo);
        sep+=dep;

        int pos=rand() % 1000 + 5000;
        wstring instr=txt.text.substr(pos,T);

        for (int i=0; i<T; i++) {
            if (i<instr.size()) {
                xg(0,i)=txt.w2v[instr[i]];
            } else {
                xg(0,i)=txt.w2v[L' '];
            }
        }
        /*
        for (auto wc : instr) {
            sg[0]=wc;
            xg(0,0)=txt.w2v[sg[0]];
            MatrixN z(0,0);
            MatrixN yg=lb.forward(xg,z,nullptr);
        }
        */
        //xg(0,0)=txt.w2v[sg[0]];
        for (int rep=0; rep<3; rep++) {
            //Layer *prnn=lb.layerMap["rnn1"];
            //prnn->params["ho"]->setZero();
            cerr << "------------" << rep << "--------------------------" << endl;
            for (int g=0; g<200; g++) {
                //wcout << g << L">";
                MatrixN z(0,0);

                //for (int i; i<xg.cols(); i++) wcout << txt.v2w[xg(0,i)];
                //wcout << L"<" << endl << L">";
                MatrixN probst=lb.forward(xg,z,nullptr);
                MatrixN probsd=MatrixN(N*T,VS);
                for (int n=0; n<1; n++) {
                    for (int t=0; t<T; t++) {
                        for (int d=0; d<VS; d++) {
                            probsd(n*T+t,d)=probst(n,t*VS+d);
                        }
                    }
                }

                for (int t=0; t<T; t++) {
                    vector<floatN> probs(VS);
                    vector<floatN> index(VS);
                    for (int d=0; d<VS; d++) {
                        probs[d]=probsd(0*T+t,d);
                        index[d]=d;
                    }
                    int ind=(int)index[randomChoice(index, probs)];
/*                    float mx=-1000.0;
                    int ind=-1;
                    for (int d=0; d<VS; d++) {
                        floatN p=yg(0,t*D+d);
                        floatN pr=p*((floatN)(rand()%100)/5000.0+0.98);
                        if (pr>mx) {
                            mx=pr; // yg(0,t*D+d);
                            ind=d;
                        }
                    }
*/
                    wchar_t cw=txt.v2w[ind];
                    //if (t==0) wcout << L"[" << cw << L"<";
                    //wcout << L"<" << cw << L">";
                    if (t==0) wcout << cw;
                    xg2(0,t)=ind;
                }
                //wcout << L"<" << endl;

                //for (int t=T-1; t>0; t--) xg(0,t)=xg(0,t-1);
                for (int t=0; t< T-1; t++) xg(0,t)=xg(0,t+1);
                xg(0,T-1)=xg2(0,0);

                //xg=xg2;
            }
            wcout << endl;
        }
    }
    /*
    floatN train_err, val_err, test_err;
    bool evalFinal=true;
    if (evalFinal) {
        train_err=lb.test(X, y, cpo.getPar("batch_size", 50));
        val_err=lb.test(Xv, yv, cpo.getPar("batch_size", 50));
        test_err=lb.test(Xt, yt, cpo.getPar("batch_size", 50));

        cerr << "Final results on RnnReader after " << cpo.getPar("epochs",(floatN)0.0) << " epochs:" << endl;
        cerr << "      Train-error: " << train_err << " train-acc: " << 1.0-train_err << endl;
        cerr << " Validation-error: " << val_err <<   "   val-acc: " << 1.0-val_err << endl;
        cerr << "       Test-error: " << test_err <<  "  test-acc: " << 1.0-test_err << endl;
    }
    //return cAcc;
    */
    cpExitCompute();
}
<|endoftext|>


<commit_before>#include "index_graph.hpp"

#include "base/assert.hpp"
#include "base/exception.hpp"

#include "std/limits.hpp"

namespace routing
{
IndexGraph::IndexGraph(unique_ptr<GeometryLoader> loader, shared_ptr<EdgeEstimator> estimator)
  : m_geometry(move(loader)), m_estimator(move(estimator))
{
  ASSERT(m_estimator, ());
}

void IndexGraph::GetEdgeList(Segment const & segment, bool isOutgoing, vector<SegmentEdge> & edges)
{
  RoadPoint const roadPoint = segment.GetRoadPoint(isOutgoing);
  Joint::Id const jointId = m_roadIndex.GetJointId(roadPoint);

  if (jointId != Joint::kInvalidId)
  {
    m_jointIndex.ForEachPoint(jointId, [&](RoadPoint const & rp) {
      GetNeighboringEdges(segment, rp, isOutgoing, edges);
    });
  }
  else
  {
    GetNeighboringEdges(segment, roadPoint, isOutgoing, edges);
  }
}

void IndexGraph::Build(uint32_t numJoints) { m_jointIndex.Build(m_roadIndex, numJoints); }

void IndexGraph::Import(vector<Joint> const & joints)
{
  m_roadIndex.Import(joints);
  CHECK_LESS_OR_EQUAL(joints.size(), numeric_limits<uint32_t>::max(), ());
  Build(static_cast<uint32_t>(joints.size()));
}

double IndexGraph::CalcSegmentWeight(Segment const & segment)
{
  return m_estimator->CalcSegmentWeight(segment, m_geometry.GetRoad(segment.GetFeatureId()));
}

void IndexGraph::GetNeighboringEdges(Segment const & from, RoadPoint const & rp, bool isOutgoing,
                                     vector<SegmentEdge> & edges)
{
  RoadGeometry const & road = m_geometry.GetRoad(rp.GetFeatureId());

  if (!road.IsValid())
    return;

  bool const bidirectional = !road.IsOneWay();

  if ((isOutgoing || bidirectional) && rp.GetPointId() + 1 < road.GetPointsCount())
  {
    GetNeighboringEdge(from, Segment(rp.GetFeatureId(), rp.GetPointId(), isOutgoing), isOutgoing,
                       edges);
  }

  if ((!isOutgoing || bidirectional) && rp.GetPointId() > 0)
  {
    GetNeighboringEdge(from, Segment(rp.GetFeatureId(), rp.GetPointId() - 1, !isOutgoing),
                       isOutgoing, edges);
  }
}

void IndexGraph::GetNeighboringEdge(Segment const & from, Segment const & to, bool isOutgoing,
                                    vector<SegmentEdge> & edges)
{
  RoadPoint const rp(from.GetPointId(isOutgoing), from.GetFeatureId());
  if (from.GetFeatureId() == to.GetFeatureId() && from.GetSegmentIdx() == to.GetSegmentIdx()
      && from.IsForward() != to.IsForward()
      && m_roadIndex.GetJointId(rp) == Joint::kInvalidId
      && rp.GetPointId() != 0
      && rp.GetPointId() + 1 != m_geometry.GetRoad(from.GetFeatureId()).GetPointsCount())
  {
    return;
  }

  double const weight = CalcSegmentWeight(isOutgoing ? to : from);
  edges.emplace_back(to, weight);
}
}  // namespace routing
<commit_msg>Review fixes.<commit_after>#include "index_graph.hpp"

#include "base/assert.hpp"
#include "base/exception.hpp"

#include "std/limits.hpp"

namespace routing
{
IndexGraph::IndexGraph(unique_ptr<GeometryLoader> loader, shared_ptr<EdgeEstimator> estimator)
  : m_geometry(move(loader)), m_estimator(move(estimator))
{
  ASSERT(m_estimator, ());
}

void IndexGraph::GetEdgeList(Segment const & segment, bool isOutgoing, vector<SegmentEdge> & edges)
{
  RoadPoint const roadPoint = segment.GetRoadPoint(isOutgoing);
  Joint::Id const jointId = m_roadIndex.GetJointId(roadPoint);

  if (jointId != Joint::kInvalidId)
  {
    m_jointIndex.ForEachPoint(jointId, [&](RoadPoint const & rp) {
      GetNeighboringEdges(segment, rp, isOutgoing, edges);
    });
  }
  else
  {
    GetNeighboringEdges(segment, roadPoint, isOutgoing, edges);
  }
}

void IndexGraph::Build(uint32_t numJoints) { m_jointIndex.Build(m_roadIndex, numJoints); }

void IndexGraph::Import(vector<Joint> const & joints)
{
  m_roadIndex.Import(joints);
  CHECK_LESS_OR_EQUAL(joints.size(), numeric_limits<uint32_t>::max(), ());
  Build(static_cast<uint32_t>(joints.size()));
}

double IndexGraph::CalcSegmentWeight(Segment const & segment)
{
  return m_estimator->CalcSegmentWeight(segment, m_geometry.GetRoad(segment.GetFeatureId()));
}

void IndexGraph::GetNeighboringEdges(Segment const & from, RoadPoint const & rp, bool isOutgoing,
                                     vector<SegmentEdge> & edges)
{
  RoadGeometry const & road = m_geometry.GetRoad(rp.GetFeatureId());

  if (!road.IsValid())
    return;

  bool const bidirectional = !road.IsOneWay();

  if ((isOutgoing || bidirectional) && rp.GetPointId() + 1 < road.GetPointsCount())
  {
    GetNeighboringEdge(from, Segment(rp.GetFeatureId(), rp.GetPointId(), isOutgoing), isOutgoing,
                       edges);
  }

  if ((!isOutgoing || bidirectional) && rp.GetPointId() > 0)
  {
    GetNeighboringEdge(from, Segment(rp.GetFeatureId(), rp.GetPointId() - 1, !isOutgoing),
                       isOutgoing, edges);
  }
}

void IndexGraph::GetNeighboringEdge(Segment const & from, Segment const & to, bool isOutgoing,
                                    vector<SegmentEdge> & edges)
{
  RoadPoint const rp = from.GetRoadPoint(isOutgoing);
  if (from.GetFeatureId() == to.GetFeatureId() && from.GetSegmentIdx() == to.GetSegmentIdx()
      && from.IsForward() != to.IsForward()
      && m_roadIndex.GetJointId(rp) == Joint::kInvalidId
      && rp.GetPointId() != 0
      && rp.GetPointId() + 1 != m_geometry.GetRoad(from.GetFeatureId()).GetPointsCount())
  {
    return;
  }

  double const weight = CalcSegmentWeight(isOutgoing ? to : from);
  edges.emplace_back(to, weight);
}
}  // namespace routing
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit (ITK)
  Module:
  Language:  C++
  Date:
  Version:

Copyright (c) 2000 National Library of Medicine
All rights reserved.

See COPYRIGHT.txt for copyright details.

=========================================================================*/
#if defined(_MSC_VER)
#pragma warning ( disable : 4786 )
#endif

#include <iostream>
// This file has been generated by BuildHeaderTest.tcl
// Test to include each header file for Insight

#include "itkAcosImageAdaptor.h"
#include "itkAffineTransform.txx"
#include "itkArray.txx"
#include "itkArray2D.txx"
#include "itkAsinImageAdaptor.h"
#include "itkAtanImageAdaptor.h"
#include "itkAutoPointer.h"
#include "itkAzimuthElevationToCartesianTransform.txx"
#include "itkBSplineDeformableTransform.txx"
#include "itkBSplineDerivativeKernelFunction.h"
#include "itkBSplineInterpolationWeightFunction.txx"
#include "itkBSplineKernelFunction.h"
#include "itkBackwardDifferenceOperator.txx"
#include "itkBinaryBallStructuringElement.txx"
#include "itkBinaryThresholdImageFunction.txx"
#include "itkBloxBoundaryPointImage.txx"
#include "itkBloxBoundaryPointItem.txx"
#include "itkBloxBoundaryPointPixel.txx"
#include "itkBloxBoundaryProfileImage.txx"
#include "itkBloxBoundaryProfileItem.txx"
#include "itkBloxBoundaryProfilePixel.txx"
#include "itkBloxCoreAtomImage.txx"
#include "itkBloxCoreAtomItem.txx"
#include "itkBloxCoreAtomPixel.txx"
#include "itkBloxImage.txx"
#include "itkBloxItem.h"
#include "itkBloxPixel.txx"
#include "itkBluePixelAccessor.h"
#include "itkBoundingBox.txx"
#include "itkByteSwapper.txx"
#include "itkCellBoundary.txx"
#include "itkCellInterface.txx"
#include "itkCellInterfaceVisitor.h"
#include "itkCentralDifferenceImageFunction.txx"
#include "itkColorTable.txx"
#include "itkCommand.h"
#include "itkConceptChecking.h"
#include "itkConditionalConstIterator.txx"
#include "itkConditionalIterator.txx"
#include "itkConicShellInteriorExteriorSpatialFunction.txx"
#include "itkConstNeighborhoodIterator.txx"
#include "itkConstRandomAccessNeighborhoodIterator.txx"
#include "itkConstSliceIterator.h"
#include "itkConstSmartNeighborhoodIterator.txx"
#include "itkConstantBoundaryCondition.h"
#include "itkContinuousIndex.h"
#include "itkCosImageAdaptor.h"
#include "itkCovariantVector.txx"
#include "itkCreateObjectFunction.h"
#include "itkDataObject.h"
#include "itkDecisionRuleBase.h"
#include "itkDefaultDynamicMeshTraits.h"
#include "itkDefaultImageTraits.h"
#include "itkDefaultPixelAccessor.h"
#include "itkDefaultStaticMeshTraits.h"
#include "itkDenseFiniteDifferenceImageFilter.txx"
#include "itkDerivativeOperator.txx"
#include "itkDifferenceImageFilter.txx"
#include "itkDirectory.h"
#include "itkDynamicLoader.h"
#include "itkElasticBodyReciprocalSplineKernelTransform.txx"
#include "itkElasticBodySplineKernelTransform.txx"
#include "itkEllipsoidInteriorExteriorSpatialFunction.txx"
#include "itkEntropyPreservingGradientMagnitudeImageFunction.txx"
#include "itkEuler2DTransform.txx"
#include "itkEventObject.h"
#include "itkExceptionObject.h"
#include "itkExpImageAdaptor.h"
#include "itkFastMutexLock.h"
#include "itkFileOutputWindow.h"
#include "itkFiniteDifferenceFunction.txx"
#include "itkFiniteDifferenceImageFilter.txx"
#include "itkFixedArray.txx"
#include "itkFloodFilledFunctionConditionalConstIterator.txx"
#include "itkFloodFilledFunctionConditionalIterator.txx"
#include "itkFloodFilledImageFunctionConditionalConstIterator.txx"
#include "itkFloodFilledImageFunctionConditionalIterator.txx"
#include "itkFloodFilledSpatialFunctionConditionalConstIterator.txx"
#include "itkFloodFilledSpatialFunctionConditionalIterator.txx"
#include "itkForwardDifferenceOperator.txx"
#include "itkFrustumSpatialFunction.txx"
#include "itkFunctionBase.h"
#include "itkGaussianKernelFunction.h"
#include "itkGaussianOperator.txx"
#include "itkGaussianSpatialFunction.txx"
#include "itkGreenPixelAccessor.h"
#include "itkHexahedronCell.txx"
#include "itkHexahedronCellTopology.h"
#include "itkIdentityTransform.h"
#include "itkImage.txx"
#include "itkImageAdaptor.txx"
#include "itkImageBase.txx"
#include "itkImageBoundaryCondition.h"
#include "itkImageConstIterator.txx"
#include "itkImageConstIteratorWithIndex.txx"
#include "itkImageContainerInterface.h"
#include "itkImageFunction.txx"
#include "itkImageIO.h"
#include "itkImageIOCommon.h"
#include "itkImageIterator.txx"
#include "itkImageIteratorWithIndex.txx"
#include "itkImageLinearConstIteratorWithIndex.txx"
#include "itkImageLinearIteratorWithIndex.txx"
#include "itkImageRandomConstIteratorWithIndex.txx"
#include "itkImageRandomIteratorWithIndex.txx"
#include "itkImageRegion.txx"
#include "itkImageRegionConstIterator.txx"
#include "itkImageRegionConstIteratorWithIndex.txx"
#include "itkImageRegionExclusionConstIteratorWithIndex.txx"
#include "itkImageRegionExclusionIteratorWithIndex.txx"
#include "itkImageRegionIterator.txx"
#include "itkImageRegionIteratorWithIndex.txx"
#include "itkImageRegionMultidimensionalSplitter.txx"
#include "itkImageRegionReverseIterator.txx"
#include "itkImageRegionSplitter.txx"
#include "itkImageReverseIterator.txx"
#include "itkImageSliceConstIteratorWithIndex.txx"
#include "itkImageSliceIteratorWithIndex.txx"
#include "itkImageSource.txx"
#include "itkImageToImageFilter.txx"
#include "itkImageToImageFilterDetail.h"
#include "itkImportImageContainer.txx"
#include "itkIndent.h"
#include "itkIndex.h"
#include "itkIndexedContainerInterface.h"
#include "itkIntTypes.h"
#include "itkInteriorExteriorSpatialFunction.txx"
#include "itkInterpolateImageFunction.h"
#include "itkKLMSegmentationBorder.h"
#include "itkKLMSegmentationRegion.h"
#include "itkKernelFunction.h"
#include "itkKernelTransform.txx"
#include "itkLaplacianOperator.txx"
#include "itkLevelSet.h"
#include "itkLevelSetCurvatureFunction.txx"
#include "itkLevelSetFunction.txx"
#include "itkLevelSetFunctionBase.txx"
#include "itkLightObject.h"
#include "itkLightProcessObject.h"
#include "itkLineCell.txx"
#include "itkLinearInterpolateImageFunction.txx"
#include "itkLog10ImageAdaptor.h"
#include "itkLogImageAdaptor.h"
#include "itkMacro.h"
#include "itkMapContainer.txx"
#include "itkMatrix.txx"
#include "itkMaximumDecisionRule.h"
#include "itkMeanImageFunction.txx"
#include "itkMedianImageFunction.txx"
#include "itkMesh.txx"
#include "itkMeshRegion.h"
#include "itkMeshSource.txx"
#include "itkMeshToMeshFilter.txx"
#include "itkMinimumDecisionRule.h"
#include "itkMultiThreader.h"
#include "itkMutexLock.h"
#include "itkMutexLockHolder.h"
#include "itkNearestNeighborInterpolateImageFunction.h"
#include "itkNeighborhood.txx"
#include "itkNeighborhoodAlgorithm.txx"
#include "itkNeighborhoodAllocator.h"
#include "itkNeighborhoodBinaryThresholdImageFunction.txx"
#include "itkNeighborhoodInnerProduct.txx"
#include "itkNeighborhoodIterator.txx"
#include "itkNeighborhoodOperator.txx"
#include "itkNthElementImageAdaptor.h"
#include "itkNthElementPixelAccessor.h"
#include "itkNumericTraits.h"
#include "itkObject.h"
#include "itkObjectFactory.h"
#include "itkObjectFactoryBase.h"
#include "itkObjectStore.txx"
#include "itkOffset.h"
#include "itkOutputWindow.h"
#include "itkPixelAccessor.h"
#include "itkPixelTraits.h"
#include "itkPoint.txx"
#include "itkPointLocator.txx"
#include "itkPointSet.txx"
#include "itkPolygonCell.txx"
#include "itkProcessObject.h"
#include "itkProgressReporter.h"
#include "itkQuadraticEdgeCell.txx"
#include "itkQuadraticTriangleCell.txx"
#include "itkQuadraticTriangleCellTopology.h"
#include "itkQuadrilateralCell.txx"
#include "itkQuadrilateralCellTopology.h"
#include "itkQuaternionRigidTransform.txx"
#include "itkRGBAPixel.txx"
#include "itkRGBPixel.txx"
#include "itkRandomAccessNeighborhoodIterator.txx"
#include "itkRedPixelAccessor.h"
#include "itkRegion.h"
#include "itkRigid2DTransform.txx"
#include "itkRigid3DPerspectiveTransform.txx"
#include "itkRigid3DTransform.txx"
#include "itkScalarVector.h"
#include "itkScaleTransform.txx"
#include "itkSegmentationBorder.h"
#include "itkSegmentationLevelSetFunction.txx"
#include "itkSegmentationRegion.h"
#include "itkSimilarity2DTransform.txx"
#include "itkSimpleFastMutexLock.h"
#include "itkSimpleImageRegionConstIterator.txx"
#include "itkSimpleImageRegionIterator.txx"
#include "itkSinImageAdaptor.h"
#include "itkSize.h"
#include "itkSliceIterator.h"
#include "itkSmartNeighborhoodIterator.txx"
#include "itkSmartPointer.h"
#include "itkSmartPointerForwardReference.txx"
#include "itkSobelOperator.txx"
#include "itkSpatialFunction.txx"
#include "itkSphereSpatialFunction.txx"
#include "itkSqrtImageAdaptor.h"
#include "itkSymmetricEllipsoidInteriorExteriorSpatialFunction.txx"
#include "itkTanImageAdaptor.h"
#include "itkTetrahedronCell.txx"
#include "itkTetrahedronCellTopology.h"
#include "itkTextOutput.h"
#include "itkThinPlateR2LogRSplineKernelTransform.txx"
#include "itkThinPlateSplineKernelTransform.txx"
#include "itkTimeProbe.h"
#include "itkTimeProbesCollectorBase.h"
#include "itkTimeStamp.h"
#include "itkTorusInteriorExteriorSpatialFunction.txx"
#include "itkTransform.txx"
#include "itkTranslationTransform.txx"
#include "itkTriangleCell.txx"
#include "itkTriangleCellTopology.h"
#include "itkUpwindDerivativeImageFunction.txx"
#include "itkValarrayImageContainer.h"
#include "itkVarianceImageFunction.txx"
#include "itkVector.txx"
#include "itkVectorContainer.txx"
#include "itkVectorInterpolateImageFunction.h"
#include "itkVectorLinearInterpolateImageFunction.txx"
#include "itkVectorNeighborhoodInnerProduct.txx"
#include "itkVersion.h"
#include "itkVersor.txx"
#include "itkVersorRigid3DTransform.txx"
#include "itkVersorTransform.txx"
#include "itkVertexCell.txx"
#include "itkVolumeSplineKernelTransform.txx"
#include "itkWeakPointer.h"
#include "itkXMLFileOutputWindow.h"
#include "itkZeroFluxNeumannBoundaryCondition.txx"
#include "itk_alloc.h"
#include "itk_hash_map.h"
#include "itk_hash_set.h"
#include "itk_hashtable.h"
#include "vcl_alloc.h"

int main ( int argc, char* argv )
{

  return 0;
}

<commit_msg>ENH: updated to latest headers.<commit_after>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit (ITK)
  Module:
  Language:  C++
  Date:
  Version:

Copyright (c) 2000 National Library of Medicine
All rights reserved.

See COPYRIGHT.txt for copyright details.

=========================================================================*/
#if defined(_MSC_VER)
#pragma warning ( disable : 4786 )
#endif

#include <iostream>
// This file has been generated by BuildHeaderTest.tcl
// Test to include each header file for Insight

#include "itkAcosImageAdaptor.h"
#include "itkAffineTransform.txx"
#include "itkArray.txx"
#include "itkArray2D.txx"
#include "itkAsinImageAdaptor.h"
#include "itkAtanImageAdaptor.h"
#include "itkAutoPointer.h"
#include "itkAzimuthElevationToCartesianTransform.txx"
#include "itkBSplineDeformableTransform.txx"
#include "itkBSplineDerivativeKernelFunction.h"
#include "itkBSplineInterpolationWeightFunction.txx"
#include "itkBSplineKernelFunction.h"
#include "itkBackwardDifferenceOperator.txx"
#include "itkBinaryBallStructuringElement.txx"
#include "itkBinaryThresholdImageFunction.txx"
#include "itkBloxBoundaryPointImage.txx"
#include "itkBloxBoundaryPointItem.txx"
#include "itkBloxBoundaryPointPixel.txx"
#include "itkBloxBoundaryProfileImage.txx"
#include "itkBloxBoundaryProfileItem.txx"
#include "itkBloxBoundaryProfilePixel.txx"
#include "itkBloxCoreAtomImage.txx"
#include "itkBloxCoreAtomItem.txx"
#include "itkBloxCoreAtomPixel.txx"
#include "itkBloxImage.txx"
#include "itkBloxItem.h"
#include "itkBloxPixel.txx"
#include "itkBluePixelAccessor.h"
#include "itkBoundingBox.txx"
#include "itkByteSwapper.txx"
#include "itkCellBoundary.txx"
#include "itkCellInterface.txx"
#include "itkCellInterfaceVisitor.h"
#include "itkCentralDifferenceImageFunction.txx"
#include "itkColorTable.txx"
#include "itkCommand.h"
#include "itkConceptChecking.h"
#include "itkConditionalConstIterator.txx"
#include "itkConditionalIterator.txx"
#include "itkConicShellInteriorExteriorSpatialFunction.txx"
#include "itkConstNeighborhoodIterator.txx"
#include "itkConstRandomAccessNeighborhoodIterator.txx"
#include "itkConstSliceIterator.h"
#include "itkConstSmartNeighborhoodIterator.txx"
#include "itkConstantBoundaryCondition.h"
#include "itkContinuousIndex.h"
#include "itkCosImageAdaptor.h"
#include "itkCovariantVector.txx"
#include "itkCreateObjectFunction.h"
#include "itkDataObject.h"
#include "itkDecisionRuleBase.h"
#include "itkDefaultDynamicMeshTraits.h"
#include "itkDefaultImageTraits.h"
#include "itkDefaultPixelAccessor.h"
#include "itkDefaultStaticMeshTraits.h"
#include "itkDenseFiniteDifferenceImageFilter.txx"
#include "itkDerivativeOperator.txx"
#include "itkDifferenceImageFilter.txx"
#include "itkDirectory.h"
#include "itkDynamicLoader.h"
#include "itkElasticBodyReciprocalSplineKernelTransform.txx"
#include "itkElasticBodySplineKernelTransform.txx"
#include "itkEllipsoidInteriorExteriorSpatialFunction.txx"
#include "itkEntropyPreservingGradientMagnitudeImageFunction.txx"
#include "itkEuler2DTransform.txx"
#include "itkEventObject.h"
#include "itkExceptionObject.h"
#include "itkExpImageAdaptor.h"
#include "itkFastMutexLock.h"
#include "itkFileOutputWindow.h"
#include "itkFiniteDifferenceFunction.txx"
#include "itkFiniteDifferenceImageFilter.txx"
#include "itkFixedArray.txx"
#include "itkFloodFilledFunctionConditionalConstIterator.txx"
#include "itkFloodFilledFunctionConditionalIterator.txx"
#include "itkFloodFilledImageFunctionConditionalConstIterator.txx"
#include "itkFloodFilledImageFunctionConditionalIterator.txx"
#include "itkFloodFilledSpatialFunctionConditionalConstIterator.txx"
#include "itkFloodFilledSpatialFunctionConditionalIterator.txx"
#include "itkForwardDifferenceOperator.txx"
#include "itkFrustumSpatialFunction.txx"
#include "itkFunctionBase.h"
#include "itkGaussianKernelFunction.h"
#include "itkGaussianOperator.txx"
#include "itkGaussianSpatialFunction.txx"
#include "itkGreenPixelAccessor.h"
#include "itkHexahedronCell.txx"
#include "itkHexahedronCellTopology.h"
#include "itkIdentityTransform.h"
#include "itkImage.txx"
#include "itkImageAdaptor.txx"
#include "itkImageBase.txx"
#include "itkImageBoundaryCondition.h"
#include "itkImageConstIterator.txx"
#include "itkImageConstIteratorWithIndex.txx"
#include "itkImageContainerInterface.h"
#include "itkImageFunction.txx"
#include "itkImageIO.h"
#include "itkImageIOCommon.h"
#include "itkImageIterator.txx"
#include "itkImageIteratorWithIndex.txx"
#include "itkImageLinearConstIteratorWithIndex.txx"
#include "itkImageLinearIteratorWithIndex.txx"
#include "itkImageRandomConstIteratorWithIndex.txx"
#include "itkImageRandomIteratorWithIndex.txx"
#include "itkImageRegion.txx"
#include "itkImageRegionConstIterator.txx"
#include "itkImageRegionConstIteratorWithIndex.txx"
#include "itkImageRegionExclusionConstIteratorWithIndex.txx"
#include "itkImageRegionExclusionIteratorWithIndex.txx"
#include "itkImageRegionIterator.txx"
#include "itkImageRegionIteratorWithIndex.txx"
#include "itkImageRegionMultidimensionalSplitter.txx"
#include "itkImageRegionReverseIterator.txx"
#include "itkImageRegionSplitter.txx"
#include "itkImageReverseIterator.txx"
#include "itkImageSliceConstIteratorWithIndex.txx"
#include "itkImageSliceIteratorWithIndex.txx"
#include "itkImageSource.txx"
#include "itkImageToImageFilter.txx"
#include "itkImageToImageFilterDetail.h"
#include "itkImportImageContainer.txx"
#include "itkIndent.h"
#include "itkIndex.h"
#include "itkIndexedContainerInterface.h"
#include "itkIntTypes.h"
#include "itkInteriorExteriorSpatialFunction.txx"
#include "itkInterpolateImageFunction.h"
#include "itkKLMSegmentationBorder.h"
#include "itkKLMSegmentationRegion.h"
#include "itkKernelFunction.h"
#include "itkKernelTransform.txx"
#include "itkLaplacianOperator.txx"
#include "itkLevelSet.h"
#include "itkLevelSetCurvatureFunction.txx"
#include "itkLevelSetFunction.txx"
#include "itkLevelSetFunctionBase.txx"
#include "itkLightObject.h"
#include "itkLightProcessObject.h"
#include "itkLineCell.txx"
#include "itkLinearInterpolateImageFunction.txx"
#include "itkLog10ImageAdaptor.h"
#include "itkLogImageAdaptor.h"
#include "itkMacro.h"
#include "itkMapContainer.txx"
#include "itkMatrix.txx"
#include "itkMaximumDecisionRule.h"
#include "itkMeanImageFunction.txx"
#include "itkMedianImageFunction.txx"
#include "itkMesh.txx"
#include "itkMeshRegion.h"
#include "itkMeshSource.txx"
#include "itkMeshToMeshFilter.txx"
#include "itkMinimumDecisionRule.h"
#include "itkMultiThreader.h"
#include "itkMutexLock.h"
#include "itkMutexLockHolder.h"
#include "itkNearestNeighborInterpolateImageFunction.h"
#include "itkNeighborhood.txx"
#include "itkNeighborhoodAlgorithm.txx"
#include "itkNeighborhoodAllocator.h"
#include "itkNeighborhoodBinaryThresholdImageFunction.txx"
#include "itkNeighborhoodInnerProduct.txx"
#include "itkNeighborhoodIterator.txx"
#include "itkNeighborhoodOperator.txx"
#include "itkNthElementImageAdaptor.h"
#include "itkNthElementPixelAccessor.h"
#include "itkNumericTraits.h"
#include "itkObject.h"
#include "itkObjectFactory.h"
#include "itkObjectFactoryBase.h"
#include "itkObjectStore.txx"
#include "itkOffset.h"
#include "itkOutputWindow.h"
#include "itkPixelAccessor.h"
#include "itkPixelTraits.h"
#include "itkPoint.txx"
#include "itkPointLocator.txx"
#include "itkPointSet.txx"
#include "itkPolygonCell.txx"
#include "itkProcessObject.h"
#include "itkProgressReporter.h"
#include "itkQuadraticEdgeCell.txx"
#include "itkQuadraticTriangleCell.txx"
#include "itkQuadraticTriangleCellTopology.h"
#include "itkQuadrilateralCell.txx"
#include "itkQuadrilateralCellTopology.h"
#include "itkQuaternionRigidTransform.txx"
#include "itkRGBAPixel.txx"
#include "itkRGBPixel.txx"
#include "itkRandomAccessNeighborhoodIterator.txx"
#include "itkRedPixelAccessor.h"
#include "itkRegion.h"
#include "itkRigid2DTransform.txx"
#include "itkRigid3DPerspectiveTransform.txx"
#include "itkRigid3DTransform.txx"
#include "itkScalarVector.h"
#include "itkScaleTransform.txx"
#include "itkSegmentationBorder.h"
#include "itkSegmentationLevelSetFunction.txx"
#include "itkSegmentationRegion.h"
#include "itkSimilarity2DTransform.txx"
#include "itkSimpleFastMutexLock.h"
#include "itkSimpleImageRegionConstIterator.txx"
#include "itkSinImageAdaptor.h"
#include "itkSize.h"
#include "itkSliceIterator.h"
#include "itkSmartNeighborhoodIterator.txx"
#include "itkSmartPointer.h"
#include "itkSmartPointerForwardReference.txx"
#include "itkSobelOperator.txx"
#include "itkSpatialFunction.txx"
#include "itkSphereSpatialFunction.txx"
#include "itkSqrtImageAdaptor.h"
#include "itkSymmetricEllipsoidInteriorExteriorSpatialFunction.txx"
#include "itkTanImageAdaptor.h"
#include "itkTetrahedronCell.txx"
#include "itkTetrahedronCellTopology.h"
#include "itkTextOutput.h"
#include "itkThinPlateR2LogRSplineKernelTransform.txx"
#include "itkThinPlateSplineKernelTransform.txx"
#include "itkTimeProbe.h"
#include "itkTimeProbesCollectorBase.h"
#include "itkTimeStamp.h"
#include "itkTorusInteriorExteriorSpatialFunction.txx"
#include "itkTransform.txx"
#include "itkTranslationTransform.txx"
#include "itkTriangleCell.txx"
#include "itkTriangleCellTopology.h"
#include "itkUpwindDerivativeImageFunction.txx"
#include "itkValarrayImageContainer.h"
#include "itkVarianceImageFunction.txx"
#include "itkVector.txx"
#include "itkVectorContainer.txx"
#include "itkVectorInterpolateImageFunction.h"
#include "itkVectorLinearInterpolateImageFunction.txx"
#include "itkVectorNeighborhoodInnerProduct.txx"
#include "itkVersion.h"
#include "itkVersor.txx"
#include "itkVersorRigid3DTransform.txx"
#include "itkVersorTransform.txx"
#include "itkVertexCell.txx"
#include "itkVolumeSplineKernelTransform.txx"
#include "itkWeakPointer.h"
#include "itkXMLFileOutputWindow.h"
#include "itkZeroFluxNeumannBoundaryCondition.txx"
#include "itk_alloc.h"
#include "itk_hash_map.h"
#include "itk_hash_set.h"
#include "itk_hashtable.h"
#include "vcl_alloc.h"

int main ( int argc, char* argv )
{

  return 0;
}

<|endoftext|>


<commit_before>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        encdec->set_dropout(opts.dropout);
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        trainer->eta0 = opts.learning_rate;
        trainer->eta_decay = opts.lr_decay;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        float align_w = opts.guided_alignment_weight;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align[t]);
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update(1.0 / double(one_batch.src.at(0).at(0).size()));
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            trainer->update_epoch();
            trainer->status();
            std::cerr << std::endl;
            align_w *= opts.guided_alignment_decay;
            para_corp_train.reset_index();
            epoch++;
            // dev
            std::cerr << "dev" << std::endl;
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        encdec->set_dropout(opts.dropout);
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<commit_msg>disable droput on dev and test times<commit_after>#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/dict.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/encdec.hpp"
#include "s2s/decode.hpp"
#include "s2s/define.hpp"
#include "s2s/comp.hpp"
#include "s2s/metrics.hpp"
#include "s2s/options.hpp"

namespace s2s {

    void train(const s2s_options &opts){
        s2s::dicts dicts;
        s2s::parallel_corpus para_corp_train;
        s2s::parallel_corpus para_corp_dev;
        dicts.set(opts);
        para_corp_train.load_src(opts.srcfile, dicts);
        para_corp_train.load_trg(opts.trgfile, dicts);
        para_corp_train.load_check();
        para_corp_dev.load_src(opts.srcvalfile, dicts);
        para_corp_dev.load_trg(opts.trgvalfile, dicts);
        para_corp_dev.load_check();
        if(opts.guided_alignment == true){
            para_corp_train.load_align(opts.alignfile);
            para_corp_train.load_check_with_align();
            para_corp_dev.load_align(opts.alignvalfile);
            para_corp_dev.load_check_with_align();
        }
        // for debug
        dicts.save(opts);
        // for debug
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        encdec->set_dropout(opts.dropout);
        dynet::Trainer* trainer = nullptr;
        if(opts.optim == "sgd"){
            trainer = new dynet::SimpleSGDTrainer(model);
        }else if(opts.optim == "momentum_sgd"){
            trainer = new dynet::MomentumSGDTrainer(model);
        }else if(opts.optim == "adagrad"){
            trainer = new dynet::AdagradTrainer(model);
        }else if(opts.optim == "adadelta"){
            trainer = new dynet::AdadeltaTrainer(model);
        }else if(opts.optim == "adam"){
            trainer = new dynet::AdamTrainer(model);
        }else{
            std::cerr << "Trainer does not exist !"<< std::endl;
            assert(false);
        }
        trainer->eta0 = opts.learning_rate;
        trainer->eta_decay = opts.lr_decay;
        trainer->clip_threshold = opts.clip_threshold;
        trainer->clipping_enabled = opts.clipping_enabled;
        unsigned int epoch = 0;
        float align_w = opts.guided_alignment_weight;
        while(epoch < opts.epochs){
            // train
            para_corp_train.shuffle();
            batch one_batch;
            unsigned int bid = 0;
            while(para_corp_train.next_batch_para(one_batch, opts.max_batch_train, dicts)){
                bid++;
                auto chrono_start = std::chrono::system_clock::now();
                dynet::ComputationGraph cg;
                std::vector<dynet::expr::Expression> errs_att;
                std::vector<dynet::expr::Expression> errs_out;
                float loss_att = 0.0;
                float loss_out = 0.0;
                std::vector<dynet::expr::Expression> i_enc = encdec->encoder(one_batch, cg);
                std::vector<dynet::expr::Expression> i_feed = encdec->init_feed(one_batch, cg);
                for (unsigned int t = 0; t < one_batch.trg.size() - 1; ++t) {
                    dynet::expr::Expression i_att_t = encdec->decoder_attention(cg, one_batch.trg[t], i_feed[t], i_enc[0]);
                    if(opts.guided_alignment == true){
                        dynet::expr::Expression i_err = pickneglogsoftmax(i_att_t, one_batch.align[t]);
                        errs_att.push_back(i_err);
                    }
                    std::vector<dynet::expr::Expression> i_out_t = encdec->decoder_output(cg, i_att_t, i_enc[1]);
                    i_feed.push_back(i_out_t[1]);
                    dynet::expr::Expression i_err = pickneglogsoftmax(i_out_t[0], one_batch.trg[t+1]);
                    errs_out.push_back(i_err);
                }
                dynet::expr::Expression i_nerr_out = sum_batches(sum(errs_out));
                loss_out = as_scalar(cg.forward(i_nerr_out));
                dynet::expr::Expression i_nerr_all;
                if(opts.guided_alignment == true){
                    dynet::expr::Expression i_nerr_att = sum_batches(sum(errs_att));
                    loss_att = as_scalar(cg.incremental_forward(i_nerr_att));
                    i_nerr_all = i_nerr_out + align_w * i_nerr_att;
                }else{
                    i_nerr_all = i_nerr_out;
                }
                cg.incremental_forward(i_nerr_all);
                cg.backward(i_nerr_all);
                //cg.print_graphviz();
                trainer->update(1.0 / double(one_batch.src.at(0).at(0).size()));
                auto chrono_end = std::chrono::system_clock::now();
                auto time_used = (double)std::chrono::duration_cast<std::chrono::milliseconds>(chrono_end - chrono_start).count() / (double)1000;
                std::cerr << "batch: " << bid;
                std::cerr << ",\tsize: " << one_batch.src.at(0).at(0).size();
                std::cerr << ",\toutput loss: " << loss_out;
                std::cerr << ",\tattention loss: " << loss_att;
                std::cerr << ",\tsource length: " << one_batch.src.size();
                std::cerr << ",\ttarget length: " << one_batch.trg.size();
                std::cerr << ",\ttime: " << time_used << " [s]" << std::endl;
                std::cerr << "[epoch=" << trainer->epoch << " eta=" << trainer->eta << " clips=" << trainer->clips_since_status << " updates=" << trainer->updates_since_status << "] " << std::endl;
            }
            trainer->update_epoch();
            trainer->status();
            std::cerr << std::endl;
            align_w *= opts.guided_alignment_decay;
            para_corp_train.reset_index();
            epoch++;
            // dev
            encdec->disable_dropout();
            std::cerr << "dev" << std::endl;
            ofstream dev_sents(opts.rootdir + "/dev_" + to_string(epoch) + ".txt");
            while(para_corp_dev.next_batch_para(one_batch, opts.max_batch_pred, dicts)){
                std::vector<std::vector<unsigned int> > osent;
                s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
                dev_sents << s2s::print_sents(osent, dicts);
            }
            dev_sents.close();
            para_corp_dev.reset_index();
            encdec->set_dropout(opts.dropout);
            // save Model
            ofstream model_out(opts.rootdir + "/" + opts.save_file + "_" + to_string(epoch) + ".model");
            boost::archive::text_oarchive model_oa(model_out);
            model_oa << model << *encdec;
            model_out.close();
        }
    }

    void predict(const s2s_options &opts){
        s2s::dicts dicts;
        dicts.load(opts);
        // load model
        dynet::Model model;
        encoder_decoder* encdec = new encoder_decoder(model, &opts);
        encdec->disable_dropout();
        ifstream model_in(opts.modelfile);
        boost::archive::text_iarchive model_ia(model_in);
        model_ia >> model >> *encdec;
        model_in.close();
        encdec->set_dropout(opts.dropout);
        // predict
        s2s::monoling_corpus mono_corp;
        mono_corp.load_src(opts.srcfile, dicts);
        batch one_batch;
        ofstream predict_sents(opts.trgfile);
        while(mono_corp.next_batch_mono(one_batch, opts.max_batch_pred, dicts)){
            std::vector<std::vector<unsigned int> > osent;
            s2s::greedy_decode(one_batch, osent, encdec, dicts, opts);
            predict_sents << s2s::print_sents(osent, dicts);
        }
        predict_sents.close();
    }

};

int main(int argc, char** argv) {
    namespace po = boost::program_options;
    po::options_description bpo("h");
    s2s::s2s_options opts;
    s2s::set_s2s_options(&bpo, &opts);
    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, bpo), vm);
    po::notify(vm);
    if(vm.at("mode").as<std::string>() == "train"){
        s2s::add_s2s_options_train(&vm, &opts);
        s2s::check_s2s_options_train(&vm, opts);
        std::string file_name = opts.rootdir + "/options.txt";
        struct stat st;
        if(stat(opts.rootdir.c_str(), &st) != 0){
            mkdir(opts.rootdir.c_str(), 0775);
        }
        ofstream out(file_name);
        boost::archive::text_oarchive oa(out);
        oa << opts;
        out.close();
        dynet::initialize(argc, argv);
        s2s::train(opts);
    }else if(vm.at("mode").as<std::string>() == "predict"){
        ifstream in(opts.rootdir + "/options.txt");
        boost::archive::text_iarchive ia(in);
        ia >> opts;
        in.close();
        s2s::check_s2s_options_predict(&vm, opts);
        dynet::initialize(argc, argv);
        s2s::predict(opts);
    }else if(vm.at("mode").as<std::string>() == "test"){

    }else{
        std::cerr << "Mode does not exist !"<< std::endl;
        assert(false);
    }
}
<|endoftext|>