<commit_before>/*
 * Copyright (c) 2012,2017-2018 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __BASE_COMPILER_HH__
#define __BASE_COMPILER_HH__

#include <memory>

// http://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html

#if defined(__GNUC__) // clang or gcc
#  define M5_ATTR_NORETURN  __attribute__((noreturn))
#  define M5_DUMMY_RETURN
#  define M5_VAR_USED __attribute__((unused))
#  define M5_ATTR_PACKED __attribute__ ((__packed__))
#  define M5_NO_INLINE __attribute__ ((__noinline__))
#  define M5_DEPRECATED __attribute__((deprecated))
#  define M5_DEPRECATED_MSG(MSG) __attribute__((deprecated(MSG)))
#  define M5_UNREACHABLE __builtin_unreachable()
#  define M5_PUBLIC __attribute__ ((visibility ("default")))
#  define M5_LOCAL __attribute__ ((visibility ("hidden")))
#endif

#if defined(__clang__)
#  define M5_CLASS_VAR_USED M5_VAR_USED
#else
#  define M5_CLASS_VAR_USED
#endif

// This can be removed once all compilers support C++17
#if defined __has_cpp_attribute
    // Note: We must separate this if statement because GCC < 5.0 doesn't
    //       support the function-like syntax in #if statements.
    #if __has_cpp_attribute(fallthrough)
        #define M5_FALLTHROUGH [[fallthrough]]
    #else
        #define M5_FALLTHROUGH
    #endif

    #if __has_cpp_attribute(nodiscard)
        #define M5_NODISCARD [[nodiscard]]
    #else
        #define M5_NODISCARD
    #endif
#else
    // Unsupported (and no warning) on GCC < 7.
    #define M5_FALLTHROUGH

    #define M5_NODISCARD
#endif

// std::make_unique redefined for C++11 compilers
namespace m5
{

#if __cplusplus >= 201402L // C++14

using std::make_unique;

#else // C++11

/** Defining custom version of make_unique: m5::make_unique<>() */
template<typename T, typename... Args>
std::unique_ptr<T>
make_unique( Args&&... constructor_args )
{
    return std::unique_ptr<T>(
               new T( std::forward<Args>(constructor_args)... )
           );
}

#endif // __cplusplus >= 201402L

} //namespace m5

#endif // __BASE_COMPILER_HH__
<commit_msg>base: Add M5_LIKELY and M5_UNLIKELY macros to compiler.hh.<commit_after>/*
 * Copyright (c) 2012,2017-2018 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __BASE_COMPILER_HH__
#define __BASE_COMPILER_HH__

#include <memory>

// http://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html

#if defined(__GNUC__) // clang or gcc
#  define M5_ATTR_NORETURN  __attribute__((noreturn))
#  define M5_DUMMY_RETURN
#  define M5_VAR_USED __attribute__((unused))
#  define M5_ATTR_PACKED __attribute__ ((__packed__))
#  define M5_NO_INLINE __attribute__ ((__noinline__))
#  define M5_DEPRECATED __attribute__((deprecated))
#  define M5_DEPRECATED_MSG(MSG) __attribute__((deprecated(MSG)))
#  define M5_UNREACHABLE __builtin_unreachable()
#  define M5_PUBLIC __attribute__ ((visibility ("default")))
#  define M5_LOCAL __attribute__ ((visibility ("hidden")))
#  define M5_LIKELY(cond) __builtin_expect(!!(cond), 1)
#  define M5_UNLIKELY(cond) __builtin_expect(!!(cond), 0)
#endif

#if defined(__clang__)
#  define M5_CLASS_VAR_USED M5_VAR_USED
#else
#  define M5_CLASS_VAR_USED
#endif

// This can be removed once all compilers support C++17
#if defined __has_cpp_attribute
    // Note: We must separate this if statement because GCC < 5.0 doesn't
    //       support the function-like syntax in #if statements.
    #if __has_cpp_attribute(fallthrough)
        #define M5_FALLTHROUGH [[fallthrough]]
    #else
        #define M5_FALLTHROUGH
    #endif

    #if __has_cpp_attribute(nodiscard)
        #define M5_NODISCARD [[nodiscard]]
    #else
        #define M5_NODISCARD
    #endif
#else
    // Unsupported (and no warning) on GCC < 7.
    #define M5_FALLTHROUGH

    #define M5_NODISCARD
#endif

// std::make_unique redefined for C++11 compilers
namespace m5
{

#if __cplusplus >= 201402L // C++14

using std::make_unique;

#else // C++11

/** Defining custom version of make_unique: m5::make_unique<>() */
template<typename T, typename... Args>
std::unique_ptr<T>
make_unique( Args&&... constructor_args )
{
    return std::unique_ptr<T>(
               new T( std::forward<Args>(constructor_args)... )
           );
}

#endif // __cplusplus >= 201402L

} //namespace m5

#endif // __BASE_COMPILER_HH__
<|endoftext|>


<commit_before>/**
 * @file
 *
 * @brief
 *
 * @copyright BSD License (see doc/LICENSE.md or http://www.libelektra.org)
 */

#include "complete.hpp"

#include <functional>
#include <iostream>
#include <stack>

#include "cmdline.hpp"
#include <kdb.hpp>
#include <keysetio.hpp>

using namespace kdb;
using namespace std;

CompleteCommand::CompleteCommand ()
{
}

int CompleteCommand::execute (const Cmdline & cl)
{
	if (cl.arguments.size () != 1)
	{
		throw invalid_argument ("wrong number of arguments, 1 needed");
	}
	if (cl.maxDepth <= cl.minDepth)
	{
		throw invalid_argument ("the maximum depth has to be larger than the minimum depth");
	}
	if (cl.maxDepth < 0)
	{
		throw invalid_argument ("the maximum depth has to be a positive number");
	}
	if (cl.minDepth < 0)
	{
		throw invalid_argument ("the minimum depth has to be a positive number");
	}

	cout.setf (ios_base::unitbuf);
	if (cl.null)
	{
		cout.unsetf (ios_base::skipws);
	}

	// Determine the actual root key, as for completion purpose originalRoot may not exist
	KDB kdb;
	const Key originalRoot = cl.createKey (cl.arguments.size () - 1);
	Key root = originalRoot;
	KeySet ks;
	printWarnings (cerr, root);

	kdb.get (ks, root);
	if (!ks.lookup (root) && !root.getBaseName ().empty ())
	{
		if (cl.verbose)
		{
			cout << originalRoot << " does not exist or is a cascading key, using " << root << " as the current completion path"
			     << endl;
		}
		root = getParentKey (root);
	}
	ks = ks.cut (root);

	// Now analyze the completion possibilities and print the results
	addMountpoints (ks, root);
	KeySet virtualKeys;
	printResult (originalRoot, root, analyze (ks, root, virtualKeys, cl), virtualKeys, cl);
	printWarnings (cerr, root);

	return 0;
}

void CompleteCommand::addMountpoints (KeySet & ks, const Key root)
{
	KDB kdb;
	Key mountpointPath ("system/elektra/mountpoints", KEY_END);
	KeySet mountpoints;

	kdb.get (mountpoints, mountpointPath);
	mountpoints = mountpoints.cut (mountpointPath);

	for (const Key mountpoint : mountpoints)
	{
		if (mountpoint.isDirectBelow (mountpointPath))
		{
			const string actualName = mountpoints.lookup (mountpoint.getFullName () + "/mountpoint").getString ();
			Key mountpointKey (actualName, KEY_END);
			if (mountpointKey.isBelow (root))
			{
				ks.append (mountpointKey);
			}
		}
	}

	printWarnings (cerr, mountpointPath);
}

/*
 * McCabe complexity of 13, 3 caused by debug switches so actually its ok
 */
const map<Key, pair<int, int>> CompleteCommand::analyze (const KeySet & ks, const Key root, KeySet & virtualKeys, const Cmdline & cl)
{
	map<Key, pair<int, int>> hierarchy;
	stack<Key> keyStack;
	Key parent;
	Key last;

	ks.rewind ();
	if (!(ks.next ()))
	{
		return hierarchy;
	}

	int curDepth = 0;
	keyStack.push (ks.current ());
	while (!keyStack.empty ())
	{
		Key current = keyStack.top ();
		keyStack.pop ();
		if (current.isDirectBelow (last))
		{ // down in the hierarchy, last element is new parent
			parent = last;
			curDepth++;
		}

		if (cl.debug)
		{
			cout << "Analyzing " << current << ", the last processed key is " << last << " and the parent is " << parent
			     << " at depth " << curDepth << endl;
		}

		if (current.isDirectBelow (parent))
		{ // hierarchy continues at the current level
			increaseCount (hierarchy, parent, [](int p) { return p; });
			increaseCount (hierarchy, current, [=](int) { return curDepth; });
		}
		else
		{ // hierarchy does not fit the current parent, expand the current key to the stack to find the new parent
			while (current.isBelow (root) && !current.getBaseName ().empty () && hierarchy[current].first == 0)
			{ // Go back up in the hierarchy until we encounter a known key or are back at the namespace level
				if (cl.debug)
				{
					cout << "Expanding " << current << endl;
				}
				keyStack.push (current);
				virtualKeys.append (current);
				current = getParentKey (current);
			}
			parent = getParentKey (current);
			curDepth = hierarchy[current].second;
			if (cl.debug)
			{
				cout << "Finished expanding, resume at " << current << " with parent " << parent << " and depth "
				     << curDepth << endl;
			}
		}

		if (keyStack.empty () && (ks.next ()))
		{ // Current hierarchy processed, we can resume with the next
			keyStack.push (ks.current ());
		}
		last = current;
	}

	return hierarchy;
}

const Key CompleteCommand::getParentKey (const Key key)
{
	return Key (key.getFullName ().erase (key.getFullName ().size () - key.getBaseName ().size ()), KEY_END);
}

void CompleteCommand::increaseCount (map<Key, pair<int, int>> & hierarchy, const Key key, const function<int(int)> depthIncreaser)
{
	const pair<int, int> prev = hierarchy[key];
	hierarchy[key] = pair<int, int> (prev.first + 1, depthIncreaser (prev.second));
}

/*
 * McCabe complexity of 12, 2 caused by debug/verbose switches so actually its ok
 */
void CompleteCommand::printResult (const Key originalRoot, const Key root, const map<Key, pair<int, int>> & hierarchy,
				   const KeySet & virtualKeys, const Cmdline & cl)
{
	const function<bool(string)> filterPredicate = determineFilterPredicate (originalRoot, root);

	// Adjust the output offset, in case the given string exists in the hierarchy but not in the original ks
	const bool limitMaxDepth = cl.maxDepth != numeric_limits<int>::max ();
	const int offset = originalRoot != root && virtualKeys.lookup (originalRoot);
	const int minDepth = cl.minDepth + offset;
	const int maxDepth = limitMaxDepth ? cl.maxDepth + offset : cl.maxDepth;
	if (cl.verbose)
	{
		cout << "Showing results for a minimum depth of " << minDepth;
		if (limitMaxDepth)
		{
			cout << " and a maximum depth of " << maxDepth;
		}
		else
		{
			cout << " and no maximum depth";
		}
		cout << endl;
	}

	if (cl.debug)
	{
		cout << endl << "Dumping whole analyzation results" << endl;
		for (const auto & it : hierarchy)
		{
			cout << it.first << " " << it.second.first << " " << it.second.second << endl;
		}
		cout << endl;
	}

	for (const auto & it : hierarchy)
	{
		if (filterPredicate (it.first.getFullName ()) && it.second.second > minDepth && it.second.second <= maxDepth)
		{
			cout << it.first << (it.second.first > 1 ? " node " + to_string (it.second.first - 1) : " leaf") << endl;
		}
	}
}

const function<bool(string)> CompleteCommand::determineFilterPredicate (const Key originalRoot, const Key root)
{
	if (root == originalRoot)
	{
		return [](string) { return true; };
	}
	const string fullName = originalRoot.getFullName ();
	return [=](string test) {
		// For cascading keys, we ignore the preceding namespace for filtering
		const int cascadationOffset = (root.isCascading () ? test.find ("/") : 0);
		return fullName.size () <= test.size () && equal (fullName.begin (), fullName.end (), test.begin () + cascadationOffset);
	};
}

CompleteCommand::~CompleteCommand ()
{
}
<commit_msg>complete: fix line endings+add limits header<commit_after>/**
 * @file
 *
 * @brief
 *
 * @copyright BSD License (see doc/LICENSE.md or http://www.libelektra.org)
 */

#include "complete.hpp"

#include <functional>
#include <iostream>
#include <limits>
#include <stack>

#include "cmdline.hpp"
#include <kdb.hpp>
#include <keysetio.hpp>

using namespace kdb;
using namespace std;

CompleteCommand::CompleteCommand ()
{
}

int CompleteCommand::execute (const Cmdline & cl)
{
	if (cl.arguments.size () != 1)
	{
		throw invalid_argument ("wrong number of arguments, 1 needed");
	}
	if (cl.maxDepth <= cl.minDepth)
	{
		throw invalid_argument ("the maximum depth has to be larger than the minimum depth");
	}
	if (cl.maxDepth < 0)
	{
		throw invalid_argument ("the maximum depth has to be a positive number");
	}
	if (cl.minDepth < 0)
	{
		throw invalid_argument ("the minimum depth has to be a positive number");
	}

	cout.setf (ios_base::unitbuf);
	if (cl.null)
	{
		cout.unsetf (ios_base::skipws);
	}

	// Determine the actual root key, as for completion purpose originalRoot may not exist
	KDB kdb;
	const Key originalRoot = cl.createKey (cl.arguments.size () - 1);
	Key root = originalRoot;
	KeySet ks;
	printWarnings (cerr, root);

	kdb.get (ks, root);
	if (!ks.lookup (root) && !root.getBaseName ().empty ())
	{
		if (cl.verbose)
		{
			cout << originalRoot << " does not exist or is a cascading key, using " << root << " as the current completion path"
			     << endl;
		}
		root = getParentKey (root);
	}
	ks = ks.cut (root);

	// Now analyze the completion possibilities and print the results
	addMountpoints (ks, root);
	KeySet virtualKeys;
	printResult (originalRoot, root, analyze (ks, root, virtualKeys, cl), virtualKeys, cl);
	printWarnings (cerr, root);

	return 0;
}

void CompleteCommand::addMountpoints (KeySet & ks, const Key root)
{
	KDB kdb;
	Key mountpointPath ("system/elektra/mountpoints", KEY_END);
	KeySet mountpoints;

	kdb.get (mountpoints, mountpointPath);
	mountpoints = mountpoints.cut (mountpointPath);

	for (const Key mountpoint : mountpoints)
	{
		if (mountpoint.isDirectBelow (mountpointPath))
		{
			const string actualName = mountpoints.lookup (mountpoint.getFullName () + "/mountpoint").getString ();
			Key mountpointKey (actualName, KEY_END);
			if (mountpointKey.isBelow (root))
			{
				ks.append (mountpointKey);
			}
		}
	}

	printWarnings (cerr, mountpointPath);
}

/*
 * McCabe complexity of 13, 3 caused by debug switches so actually its ok
 */
const map<Key, pair<int, int>> CompleteCommand::analyze (const KeySet & ks, const Key root, KeySet & virtualKeys, const Cmdline & cl)
{
	map<Key, pair<int, int>> hierarchy;
	stack<Key> keyStack;
	Key parent;
	Key last;

	ks.rewind ();
	if (!(ks.next ()))
	{
		return hierarchy;
	}

	int curDepth = 0;
	keyStack.push (ks.current ());
	while (!keyStack.empty ())
	{
		Key current = keyStack.top ();
		keyStack.pop ();
		if (current.isDirectBelow (last))
		{ // down in the hierarchy, last element is new parent
			parent = last;
			curDepth++;
		}

		if (cl.debug)
		{
			cout << "Analyzing " << current << ", the last processed key is " << last << " and the parent is " << parent
			     << " at depth " << curDepth << endl;
		}

		if (current.isDirectBelow (parent))
		{ // hierarchy continues at the current level
			increaseCount (hierarchy, parent, [](int p) { return p; });
			increaseCount (hierarchy, current, [=](int) { return curDepth; });
		}
		else
		{ // hierarchy does not fit the current parent, expand the current key to the stack to find the new parent
			while (current.isBelow (root) && !current.getBaseName ().empty () && hierarchy[current].first == 0)
			{ // Go back up in the hierarchy until we encounter a known key or are back at the namespace level
				if (cl.debug)
				{
					cout << "Expanding " << current << endl;
				}
				keyStack.push (current);
				virtualKeys.append (current);
				current = getParentKey (current);
			}
			parent = getParentKey (current);
			curDepth = hierarchy[current].second;
			if (cl.debug)
			{
				cout << "Finished expanding, resume at " << current << " with parent " << parent << " and depth "
				     << curDepth << endl;
			}
		}

		if (keyStack.empty () && (ks.next ()))
		{ // Current hierarchy processed, we can resume with the next
			keyStack.push (ks.current ());
		}
		last = current;
	}

	return hierarchy;
}

const Key CompleteCommand::getParentKey (const Key key)
{
	return Key (key.getFullName ().erase (key.getFullName ().size () - key.getBaseName ().size ()), KEY_END);
}

void CompleteCommand::increaseCount (map<Key, pair<int, int>> & hierarchy, const Key key, const function<int(int)> depthIncreaser)
{
	const pair<int, int> prev = hierarchy[key];
	hierarchy[key] = pair<int, int> (prev.first + 1, depthIncreaser (prev.second));
}

/*
 * McCabe complexity of 12, 2 caused by debug/verbose switches so actually its ok
 */
void CompleteCommand::printResult (const Key originalRoot, const Key root, const map<Key, pair<int, int>> & hierarchy,
				   const KeySet & virtualKeys, const Cmdline & cl)
{
	const function<bool(string)> filterPredicate = determineFilterPredicate (originalRoot, root);

	// Adjust the output offset, in case the given string exists in the hierarchy but not in the original ks
	const bool limitMaxDepth = cl.maxDepth != numeric_limits<int>::max ();
	const int offset = originalRoot != root && virtualKeys.lookup (originalRoot);
	const int minDepth = cl.minDepth + offset;
	const int maxDepth = limitMaxDepth ? cl.maxDepth + offset : cl.maxDepth;
	if (cl.verbose)
	{
		cout << "Showing results for a minimum depth of " << minDepth;
		if (limitMaxDepth)
		{
			cout << " and a maximum depth of " << maxDepth;
		}
		else
		{
			cout << " and no maximum depth";
		}
		cout << endl;
	}

	if (cl.debug)
	{
		cout << endl << "Dumping whole analyzation results" << endl;
		for (const auto & it : hierarchy)
		{
			cout << it.first << " " << it.second.first << " " << it.second.second << endl;
		}
		cout << endl;
	}

	for (const auto & it : hierarchy)
	{
		if (filterPredicate (it.first.getFullName ()) && it.second.second > minDepth && it.second.second <= maxDepth)
		{
			cout << it.first << (it.second.first > 1 ? " node " + to_string (it.second.first - 1) : " leaf") << endl;
		}
	}
}

const function<bool(string)> CompleteCommand::determineFilterPredicate (const Key originalRoot, const Key root)
{
	if (root == originalRoot)
	{
		return [](string) { return true; };
	}
	const string fullName = originalRoot.getFullName ();
	return [=](string test) {
		// For cascading keys, we ignore the preceding namespace for filtering
		const int cascadationOffset = (root.isCascading () ? test.find ("/") : 0);
		return fullName.size () <= test.size () && equal (fullName.begin (), fullName.end (), test.begin () + cascadationOffset);
	};
}

CompleteCommand::~CompleteCommand ()
{
}
<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/gtk/status_bubble_gtk.h"

#include <gtk/gtk.h>

#include "base/gfx/gtk_util.h"
#include "base/message_loop.h"
#include "base/string_util.h"
#include "chrome/browser/gtk/gtk_theme_provider.h"
#include "chrome/browser/gtk/slide_animator_gtk.h"
#include "chrome/common/gtk_util.h"
#include "chrome/common/notification_service.h"
#include "googleurl/src/gurl.h"

namespace {

const GdkColor kTextColor = GDK_COLOR_RGB(100, 100, 100);
const GdkColor kBackgroundColor = GDK_COLOR_RGB(0xe6, 0xed, 0xf4);
const GdkColor kFrameBorderColor = GDK_COLOR_RGB(0xbe, 0xc8, 0xd4);

// Inner padding between the border and the text label.
const int kInternalTopBottomPadding = 1;
const int kInternalLeftRightPadding = 2;

// Border of color kFrameBorderColor around the status bubble.
const int kBorderPadding = 1;

// Milliseconds before we hide the status bubble widget when you mouseout.
static const int kHideDelay = 250;

}  // namespace

StatusBubbleGtk::StatusBubbleGtk(Profile* profile)
    : theme_provider_(GtkThemeProvider::GetFrom(profile)),
      timer_factory_(this) {
  InitWidgets();

  registrar_.Add(this, NotificationType::BROWSER_THEME_CHANGED,
                 NotificationService::AllSources());
}

StatusBubbleGtk::~StatusBubbleGtk() {
  container_.Destroy();
}

void StatusBubbleGtk::SetStatus(const std::wstring& status_text_wide) {
  std::string status_text = WideToUTF8(status_text_wide);
  if (status_text_ == status_text)
    return;

  status_text_ = status_text;
  if (!status_text_.empty()) {
    SetStatusTextTo(status_text_);
  } else if (!url_text_.empty()) {
    SetStatusTextTo(url_text_);
  } else {
    SetStatusTextTo(std::string());
  }
}

void StatusBubbleGtk::SetURL(const GURL& url, const std::wstring& languages) {
  // If we want to clear a displayed URL but there is a status still to
  // display, display that status instead.
  if (url.is_empty() && !status_text_.empty()) {
    url_text_ = std::string();
    SetStatusTextTo(status_text_);
    return;
  }

  // TODO(erg): We probably want to elide the text from the GURL object.
  url_text_ = url.possibly_invalid_spec();

  SetStatusTextTo(url_text_);
}

void StatusBubbleGtk::Show() {
  // If we were going to hide, stop.
  timer_factory_.RevokeAll();

  gtk_widget_show_all(container_.get());

  if (container_->window)
    gdk_window_raise(container_->window);
}

void StatusBubbleGtk::Hide() {
  gtk_widget_hide_all(container_.get());
}

void StatusBubbleGtk::SetStatusTextTo(const std::string& status_utf8) {
  if (status_utf8.empty()) {
    HideInASecond();
  } else {
    gtk_label_set_text(GTK_LABEL(label_), status_utf8.c_str());
    Show();
  }
}

void StatusBubbleGtk::HideInASecond() {
  if (!timer_factory_.empty())
    timer_factory_.RevokeAll();

  MessageLoop::current()->PostDelayedTask(FROM_HERE,
      timer_factory_.NewRunnableMethod(&StatusBubbleGtk::Hide),
      kHideDelay);
}

void StatusBubbleGtk::MouseMoved() {
  // We can't do that fancy sliding behaviour where the status bubble slides
  // out of the window because the window manager gets in the way. So totally
  // ignore this message for now.
  //
  // TODO(erg): At least get some sliding behaviour so that it slides out of
  // the way to hide the status bubble on mouseover.
}

void StatusBubbleGtk::Observe(NotificationType type,
                              const NotificationSource& source,
                              const NotificationDetails& details) {
  if (type == NotificationType::BROWSER_THEME_CHANGED) {
    UserChangedTheme();
  }
}

void StatusBubbleGtk::InitWidgets() {
  label_ = gtk_label_new(NULL);

  GtkWidget* padding = gtk_alignment_new(0, 0, 1, 1);
  gtk_alignment_set_padding(GTK_ALIGNMENT(padding),
      kInternalTopBottomPadding, kInternalTopBottomPadding,
      kInternalLeftRightPadding, kInternalLeftRightPadding);
  gtk_container_add(GTK_CONTAINER(padding), label_);

  bg_box_ = gtk_event_box_new();
  gtk_container_add(GTK_CONTAINER(bg_box_), padding);

  container_.Own(gtk_util::CreateGtkBorderBin(bg_box_, &kFrameBorderColor,
          kBorderPadding, kBorderPadding, kBorderPadding, kBorderPadding));
  gtk_widget_set_name(container_.get(), "status-bubble");
  gtk_widget_set_app_paintable(container_.get(), TRUE);

  UserChangedTheme();
}

void StatusBubbleGtk::UserChangedTheme() {
  if (theme_provider_->UseGtkTheme()) {
    gtk_widget_modify_fg(label_, GTK_STATE_NORMAL, NULL);
    gtk_widget_modify_bg(bg_box_, GTK_STATE_NORMAL, NULL);
  } else {
    // TODO(erg): This is the closest to "text that will look good on a
    // toolbar" that I can find. Maybe in later iterations of the theme system,
    // there will be a better color to pick.
    GdkColor bookmark_text =
        theme_provider_->GetGdkColor(BrowserThemeProvider::COLOR_BOOKMARK_TEXT);
    gtk_widget_modify_fg(label_, GTK_STATE_NORMAL, &bookmark_text);

    GdkColor toolbar_color =
        theme_provider_->GetGdkColor(BrowserThemeProvider::COLOR_TOOLBAR);
    gtk_widget_modify_bg(bg_box_, GTK_STATE_NORMAL, &toolbar_color);
  }

  // TODO(erg): I don't know what to do with the status bubble border
  // (|container_|). There needs to be a border in GTK mode, and I'm not sure
  // which BrowserThemeProvider::COLOR I'm supposed to use here since the Views
  // implementation still uses constants in the equivalent, and it's used for
  // alpha blending instead of drawing a real border.
  //
  // This doesn't really matter because this part of the UI needs to be
  // rewritten per the UI review anyway; we should be matching windows with a
  // semi-transparent, rounded border instead of our constantly
  // CreateGtkBorderBin() usage.
}
<commit_msg>Re-land the patch to elide the status bubble to 1/3 the width of the window.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/gtk/status_bubble_gtk.h"

#include <gtk/gtk.h>

#include "app/gfx/text_elider.h"
#include "base/gfx/gtk_util.h"
#include "base/message_loop.h"
#include "base/string_util.h"
#include "chrome/browser/gtk/gtk_theme_provider.h"
#include "chrome/browser/gtk/slide_animator_gtk.h"
#include "chrome/common/gtk_util.h"
#include "chrome/common/notification_service.h"
#include "googleurl/src/gurl.h"

namespace {

const GdkColor kTextColor = GDK_COLOR_RGB(100, 100, 100);
const GdkColor kBackgroundColor = GDK_COLOR_RGB(0xe6, 0xed, 0xf4);
const GdkColor kFrameBorderColor = GDK_COLOR_RGB(0xbe, 0xc8, 0xd4);

// Inner padding between the border and the text label.
const int kInternalTopBottomPadding = 1;
const int kInternalLeftRightPadding = 2;

// Border of color kFrameBorderColor around the status bubble.
const int kBorderPadding = 1;

// Milliseconds before we hide the status bubble widget when you mouseout.
static const int kHideDelay = 250;

}  // namespace

StatusBubbleGtk::StatusBubbleGtk(Profile* profile)
    : theme_provider_(GtkThemeProvider::GetFrom(profile)),
      timer_factory_(this) {
  InitWidgets();

  registrar_.Add(this, NotificationType::BROWSER_THEME_CHANGED,
                 NotificationService::AllSources());
}

StatusBubbleGtk::~StatusBubbleGtk() {
  container_.Destroy();
}

void StatusBubbleGtk::SetStatus(const std::wstring& status_text_wide) {
  std::string status_text = WideToUTF8(status_text_wide);
  if (status_text_ == status_text)
    return;

  status_text_ = status_text;
  if (!status_text_.empty()) {
    SetStatusTextTo(status_text_);
  } else if (!url_text_.empty()) {
    SetStatusTextTo(url_text_);
  } else {
    SetStatusTextTo(std::string());
  }
}

void StatusBubbleGtk::SetURL(const GURL& url, const std::wstring& languages) {
  // If we want to clear a displayed URL but there is a status still to
  // display, display that status instead.
  if (url.is_empty() && !status_text_.empty()) {
    url_text_ = std::string();
    SetStatusTextTo(status_text_);
    return;
  }

  // Set Elided Text corresponding to the GURL object.  We limit the width of
  // the URL to a third of the width of the browser window (matching the width
  // on Windows).
  GtkWidget* parent = gtk_widget_get_parent(container_.get());
  int window_width = parent->allocation.width;
  // TODO(tc): We don't actually use gfx::Font as the font in the status
  // bubble.  We should extend gfx::ElideUrl to take some sort of pango font.
  url_text_ = WideToUTF8(gfx::ElideUrl(url, gfx::Font(), window_width / 3,
                                       languages));
  SetStatusTextTo(url_text_);
}

void StatusBubbleGtk::Show() {
  // If we were going to hide, stop.
  timer_factory_.RevokeAll();

  gtk_widget_show_all(container_.get());

  if (container_->window)
    gdk_window_raise(container_->window);
}

void StatusBubbleGtk::Hide() {
  gtk_widget_hide_all(container_.get());
}

void StatusBubbleGtk::SetStatusTextTo(const std::string& status_utf8) {
  if (status_utf8.empty()) {
    HideInASecond();
  } else {
    gtk_label_set_text(GTK_LABEL(label_), status_utf8.c_str());
    Show();
  }
}

void StatusBubbleGtk::HideInASecond() {
  if (!timer_factory_.empty())
    timer_factory_.RevokeAll();

  MessageLoop::current()->PostDelayedTask(FROM_HERE,
      timer_factory_.NewRunnableMethod(&StatusBubbleGtk::Hide),
      kHideDelay);
}

void StatusBubbleGtk::MouseMoved() {
  // We can't do that fancy sliding behaviour where the status bubble slides
  // out of the window because the window manager gets in the way. So totally
  // ignore this message for now.
  //
  // TODO(erg): At least get some sliding behaviour so that it slides out of
  // the way to hide the status bubble on mouseover.
}

void StatusBubbleGtk::Observe(NotificationType type,
                              const NotificationSource& source,
                              const NotificationDetails& details) {
  if (type == NotificationType::BROWSER_THEME_CHANGED) {
    UserChangedTheme();
  }
}

void StatusBubbleGtk::InitWidgets() {
  label_ = gtk_label_new(NULL);

  GtkWidget* padding = gtk_alignment_new(0, 0, 1, 1);
  gtk_alignment_set_padding(GTK_ALIGNMENT(padding),
      kInternalTopBottomPadding, kInternalTopBottomPadding,
      kInternalLeftRightPadding, kInternalLeftRightPadding);
  gtk_container_add(GTK_CONTAINER(padding), label_);

  bg_box_ = gtk_event_box_new();
  gtk_container_add(GTK_CONTAINER(bg_box_), padding);

  container_.Own(gtk_util::CreateGtkBorderBin(bg_box_, &kFrameBorderColor,
          kBorderPadding, kBorderPadding, kBorderPadding, kBorderPadding));
  gtk_widget_set_name(container_.get(), "status-bubble");
  gtk_widget_set_app_paintable(container_.get(), TRUE);

  UserChangedTheme();
}

void StatusBubbleGtk::UserChangedTheme() {
  if (theme_provider_->UseGtkTheme()) {
    gtk_widget_modify_fg(label_, GTK_STATE_NORMAL, NULL);
    gtk_widget_modify_bg(bg_box_, GTK_STATE_NORMAL, NULL);
  } else {
    // TODO(erg): This is the closest to "text that will look good on a
    // toolbar" that I can find. Maybe in later iterations of the theme system,
    // there will be a better color to pick.
    GdkColor bookmark_text =
        theme_provider_->GetGdkColor(BrowserThemeProvider::COLOR_BOOKMARK_TEXT);
    gtk_widget_modify_fg(label_, GTK_STATE_NORMAL, &bookmark_text);

    GdkColor toolbar_color =
        theme_provider_->GetGdkColor(BrowserThemeProvider::COLOR_TOOLBAR);
    gtk_widget_modify_bg(bg_box_, GTK_STATE_NORMAL, &toolbar_color);
  }

  // TODO(erg): I don't know what to do with the status bubble border
  // (|container_|). There needs to be a border in GTK mode, and I'm not sure
  // which BrowserThemeProvider::COLOR I'm supposed to use here since the Views
  // implementation still uses constants in the equivalent, and it's used for
  // alpha blending instead of drawing a real border.
  //
  // This doesn't really matter because this part of the UI needs to be
  // rewritten per the UI review anyway; we should be matching windows with a
  // semi-transparent, rounded border instead of our constantly
  // CreateGtkBorderBin() usage.
}
<|endoftext|>


<commit_before>#include <sstream>
#include <boost/algorithm/string.hpp>
#include "WordTranslationFeature.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ChartHypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "moses/InputPath.h"
#include "util/string_piece_hash.hh"
#include "util/exception.hh"

using namespace std;

namespace Moses
{

WordTranslationFeature::WordTranslationFeature(const std::string &line)
  :StatelessFeatureFunction(0, line)
  ,m_unrestricted(true)
  ,m_simple(true)
  ,m_sourceContext(false)
  ,m_targetContext(false)
  ,m_domainTrigger(false)
  ,m_ignorePunctuation(false)
{
  VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ...");
  ReadParameters();

  if (m_simple == 1) VERBOSE(1, " Using simple word translations.");
  if (m_sourceContext == 1) VERBOSE(1, " Using source context.");
  if (m_targetContext == 1) VERBOSE(1, " Using target context.");
  if (m_domainTrigger == 1) VERBOSE(1, " Using domain triggers.");

  // compile a list of punctuation characters
  if (m_ignorePunctuation) {
    VERBOSE(1, " Ignoring punctuation for triggers.");
    char punctuation[] = "\"'!?¿·()#_,.:;•&@‑/\\0123456789~=";
    for (size_t i=0; i < sizeof(punctuation)-1; ++i) {
      m_punctuationHash[punctuation[i]] = 1;
    }
  }

  VERBOSE(1, " Done." << std::endl);

  // TODO not sure about this
  /*
  if (weight[0] != 1) {
    AddSparseProducer(wordTranslationFeature);
    VERBOSE(1, "wt sparse producer weight: " << weight[0] << std::endl);
    if (m_mira)
      m_metaFeatureProducer = new MetaFeatureProducer("wt");
  }

  if (m_parameter->GetParam("report-sparse-features").size() > 0) {
    wordTranslationFeature->SetSparseFeatureReporting();
  }
  */

}

void WordTranslationFeature::SetParameter(const std::string& key, const std::string& value)
{
  if (key == "input-factor") {
    m_factorTypeSource = Scan<FactorType>(value);
  } else if (key == "output-factor") {
    m_factorTypeTarget = Scan<FactorType>(value);
  } else if (key == "simple") {
    m_simple = Scan<bool>(value);
  } else if (key == "source-context") {
    m_sourceContext = Scan<bool>(value);
  } else if (key == "target-context") {
    m_targetContext = Scan<bool>(value);
  } else if (key == "ignore-punctuation") {
    m_ignorePunctuation = Scan<bool>(value);
  } else if (key == "domain-trigger") {
    m_domainTrigger = Scan<bool>(value);
  } else if (key == "texttype") {
    //texttype = value; TODO not used
  } else if (key == "source-path") {
    m_filePathSource = value;
  } else if (key == "target-path") {
    m_filePathTarget = value;
  } else {
    StatelessFeatureFunction::SetParameter(key, value);
  }
}

void WordTranslationFeature::Load(AllOptions::ptr const& opts)
{
  m_options = opts;
  // load word list for restricted feature set
  if (m_filePathSource.empty()) {
    return;
  } //else if (tokens.size() == 8) {

  FEATUREVERBOSE(1, "Loading word translation word lists from " << m_filePathSource << " and " << m_filePathTarget << std::endl);
  if (m_domainTrigger) {
    // domain trigger terms for each input document
    ifstream inFileSource(m_filePathSource.c_str());
    UTIL_THROW_IF2(!inFileSource, "could not open file " << m_filePathSource);

    std::string line;
    while (getline(inFileSource, line)) {
      m_vocabDomain.resize(m_vocabDomain.size() + 1);
      vector<string> termVector;
      boost::split(termVector, line, boost::is_any_of("\t "));
      for (size_t i=0; i < termVector.size(); ++i)
        m_vocabDomain.back().insert(termVector[i]);
    }

    inFileSource.close();
  } else if (!m_filePathSource.empty() || !m_filePathTarget.empty()) {
    return;
    // restricted source word vocabulary
    ifstream inFileSource(m_filePathSource.c_str());
    UTIL_THROW_IF2(!inFileSource, "could not open file " << m_filePathSource);

    std::string line;
    while (getline(inFileSource, line)) {
      m_vocabSource.insert(line);
    }

    inFileSource.close();

    // restricted target word vocabulary
    ifstream inFileTarget(m_filePathTarget.c_str());
    UTIL_THROW_IF2(!inFileTarget, "could not open file " << m_filePathTarget);

    while (getline(inFileTarget, line)) {
      m_vocabTarget.insert(line);
    }

    inFileTarget.close();

    m_unrestricted = false;
  }
}

void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
    , const InputPath &inputPath
    , const TargetPhrase &targetPhrase
    , const StackVec *stackVec
    , ScoreComponentCollection &scoreBreakdown
    , ScoreComponentCollection *estimatedScores) const
{
  const Sentence& sentence = static_cast<const Sentence&>(input);
  const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();

  // process aligned words
  for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) {
    const Phrase& sourcePhrase = inputPath.GetPhrase();
    int sourceIndex = alignmentPoint->first;
    int targetIndex = alignmentPoint->second;
    Word ws = sourcePhrase.GetWord(sourceIndex);
    if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue;
    Word wt = targetPhrase.GetWord(targetIndex);
    if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue;
    StringPiece sourceWord = ws.GetFactor(m_factorTypeSource)->GetString();
    StringPiece targetWord = wt.GetFactor(m_factorTypeTarget)->GetString();
    if (m_ignorePunctuation) {
      // check if source or target are punctuation
      char firstChar = sourceWord[0];
      CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
      if(charIterator != m_punctuationHash.end())
        continue;
      firstChar = targetWord[0];
      charIterator = m_punctuationHash.find( firstChar );
      if(charIterator != m_punctuationHash.end())
        continue;
    }

    if (!m_unrestricted) {
      if (FindStringPiece(m_vocabSource, sourceWord) == m_vocabSource.end())
        sourceWord = "OTHER";
      if (FindStringPiece(m_vocabTarget, targetWord) == m_vocabTarget.end())
        targetWord = "OTHER";
    }

    if (m_simple) {
      // construct feature name
      util::StringStream featureName;
      featureName << m_description << "_";
      featureName << sourceWord;
      featureName << "~";
      featureName << targetWord;
      scoreBreakdown.SparsePlusEquals(featureName.str(), 1);
    }
    if (m_domainTrigger && !m_sourceContext) {
      const bool use_topicid = sentence.GetUseTopicId();
      const bool use_topicid_prob = sentence.GetUseTopicIdAndProb();
      if (use_topicid || use_topicid_prob) {
        if(use_topicid) {
          // use topicid as trigger
          const long topicid = sentence.GetTopicId();
          util::StringStream feature;
          feature << m_description << "_";
          if (topicid == -1)
            feature << "unk";
          else
            feature << topicid;

          feature << "_";
          feature << sourceWord;
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        } else {
          // use topic probabilities
          const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
          if (atol(topicid_prob[0].c_str()) == -1) {
            util::StringStream feature;
            feature << m_description << "_unk_";
            feature << sourceWord;
            feature << "~";
            feature << targetWord;
            scoreBreakdown.SparsePlusEquals(feature.str(), 1);
          } else {
            for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
              util::StringStream feature;
              feature << m_description << "_";
              feature << topicid_prob[i];
              feature << "_";
              feature << sourceWord;
              feature << "~";
              feature << targetWord;
              scoreBreakdown.SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str()));
            }
          }
        }
      } else {
        // range over domain trigger words (keywords)
        const long docid = input.GetDocumentId();
        for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
          string sourceTrigger = *p;
          util::StringStream feature;
          feature << m_description << "_";
          feature << sourceTrigger;
          feature << "_";
          feature << sourceWord;
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        }
      }
    }
    if (m_sourceContext) {
      size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex;
      if (!m_domainTrigger && globalSourceIndex == 0) {
        // add <s> trigger feature for source
        util::StringStream feature;
        feature << m_description << "_";
        feature << "<s>,";
        feature << sourceWord;
        feature << "~";
        feature << targetWord;
        scoreBreakdown.SparsePlusEquals(feature.str(), 1);
      }

      // range over source words to get context
      for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
        if (contextIndex == globalSourceIndex) continue;
        StringPiece sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString();
        if (m_ignorePunctuation) {
          // check if trigger is punctuation
          char firstChar = sourceTrigger[0];
          CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
          if(charIterator != m_punctuationHash.end())
            continue;
        }

        const long docid = input.GetDocumentId();
        bool sourceTriggerExists = false;
        if (m_domainTrigger)
          sourceTriggerExists = FindStringPiece(m_vocabDomain[docid], sourceTrigger ) != m_vocabDomain[docid].end();
        else if (!m_unrestricted)
          sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();

        if (m_domainTrigger) {
          if (sourceTriggerExists) {
            util::StringStream feature;
            feature << m_description << "_";
            feature << sourceTrigger;
            feature << "_";
            feature << sourceWord;
            feature << "~";
            feature << targetWord;
            scoreBreakdown.SparsePlusEquals(feature.str(), 1);
          }
        } else if (m_unrestricted || sourceTriggerExists) {
          util::StringStream feature;
          feature << m_description << "_";
          if (contextIndex < globalSourceIndex) {
            feature << sourceTrigger;
            feature << ",";
            feature << sourceWord;
          } else {
            feature << sourceWord;
            feature << ",";
            feature << sourceTrigger;
          }
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        }
      }
    }
    if (m_targetContext) {
      throw runtime_error("Can't use target words outside current translation option in a stateless feature");
      /*
      size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex;
      if (globalTargetIndex == 0) {
      	// add <s> trigger feature for source
      	stringstream feature;
      	feature << "wt_";
      	feature << sourceWord;
      	feature << "~";
      	feature << "<s>,";
      	feature << targetWord;
      	accumulator->SparsePlusEquals(feature.str(), 1);
      }

      // range over target words (up to current position) to get context
      for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) {
      	string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
      	if (m_ignorePunctuation) {
      		// check if trigger is punctuation
      		char firstChar = targetTrigger.at(0);
      		CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
      		if(charIterator != m_punctuationHash.end())
      			continue;
      	}

      	bool targetTriggerExists = false;
      	if (!m_unrestricted)
      		targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end();

      	if (m_unrestricted || targetTriggerExists) {
      		stringstream feature;
      		feature << "wt_";
      		feature << sourceWord;
      		feature << "~";
      		feature << targetTrigger;
      		feature << ",";
      		feature << targetWord;
      		accumulator->SparsePlusEquals(feature.str(), 1);
      	}
      }*/
    }
  }
}

bool WordTranslationFeature::IsUseable(const FactorMask &mask) const
{
  bool ret = mask[m_factorTypeTarget];
  return ret;
}

}
<commit_msg>WordTranslationFeature: fix loading of word lists<commit_after>#include <sstream>
#include <boost/algorithm/string.hpp>
#include "WordTranslationFeature.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ChartHypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "moses/InputPath.h"
#include "util/string_piece_hash.hh"
#include "util/exception.hh"

using namespace std;

namespace Moses
{

WordTranslationFeature::WordTranslationFeature(const std::string &line)
  :StatelessFeatureFunction(0, line)
  ,m_unrestricted(true)
  ,m_simple(true)
  ,m_sourceContext(false)
  ,m_targetContext(false)
  ,m_domainTrigger(false)
  ,m_ignorePunctuation(false)
{
  VERBOSE(1, "Initializing feature " << GetScoreProducerDescription() << " ...");
  ReadParameters();

  if (m_simple == 1) VERBOSE(1, " Using simple word translations.");
  if (m_sourceContext == 1) VERBOSE(1, " Using source context.");
  if (m_targetContext == 1) VERBOSE(1, " Using target context.");
  if (m_domainTrigger == 1) VERBOSE(1, " Using domain triggers.");

  // compile a list of punctuation characters
  if (m_ignorePunctuation) {
    VERBOSE(1, " Ignoring punctuation for triggers.");
    char punctuation[] = "\"'!?¿·()#_,.:;•&@‑/\\0123456789~=";
    for (size_t i=0; i < sizeof(punctuation)-1; ++i) {
      m_punctuationHash[punctuation[i]] = 1;
    }
  }

  VERBOSE(1, " Done." << std::endl);

  // TODO not sure about this
  /*
  if (weight[0] != 1) {
    AddSparseProducer(wordTranslationFeature);
    VERBOSE(1, "wt sparse producer weight: " << weight[0] << std::endl);
    if (m_mira)
      m_metaFeatureProducer = new MetaFeatureProducer("wt");
  }

  if (m_parameter->GetParam("report-sparse-features").size() > 0) {
    wordTranslationFeature->SetSparseFeatureReporting();
  }
  */

}

void WordTranslationFeature::SetParameter(const std::string& key, const std::string& value)
{
  if (key == "input-factor") {
    m_factorTypeSource = Scan<FactorType>(value);
  } else if (key == "output-factor") {
    m_factorTypeTarget = Scan<FactorType>(value);
  } else if (key == "simple") {
    m_simple = Scan<bool>(value);
  } else if (key == "source-context") {
    m_sourceContext = Scan<bool>(value);
  } else if (key == "target-context") {
    m_targetContext = Scan<bool>(value);
  } else if (key == "ignore-punctuation") {
    m_ignorePunctuation = Scan<bool>(value);
  } else if (key == "domain-trigger") {
    m_domainTrigger = Scan<bool>(value);
  } else if (key == "texttype") {
    //texttype = value; TODO not used
  } else if (key == "source-path") {
    m_filePathSource = value;
  } else if (key == "target-path") {
    m_filePathTarget = value;
  } else {
    StatelessFeatureFunction::SetParameter(key, value);
  }
}

void WordTranslationFeature::Load(AllOptions::ptr const& opts)
{
  m_options = opts;
  // load word list for restricted feature set
  if (m_filePathSource.empty()) {
    return;
  } //else if (tokens.size() == 8) {

  FEATUREVERBOSE(1, "Loading word translation word lists from " << m_filePathSource << " and " << m_filePathTarget << std::endl);
  if (m_domainTrigger) {
    // domain trigger terms for each input document
    ifstream inFileSource(m_filePathSource.c_str());
    UTIL_THROW_IF2(!inFileSource, "could not open file " << m_filePathSource);

    std::string line;
    while (getline(inFileSource, line)) {
      m_vocabDomain.resize(m_vocabDomain.size() + 1);
      vector<string> termVector;
      boost::split(termVector, line, boost::is_any_of("\t "));
      for (size_t i=0; i < termVector.size(); ++i)
        m_vocabDomain.back().insert(termVector[i]);
    }

    inFileSource.close();
  } else {
    // restricted source word vocabulary
    ifstream inFileSource(m_filePathSource.c_str());
    UTIL_THROW_IF2(!inFileSource, "could not open file " << m_filePathSource);

    std::string line;
    while (getline(inFileSource, line)) {
      m_vocabSource.insert(line);
    }

    inFileSource.close();

    // restricted target word vocabulary
    ifstream inFileTarget(m_filePathTarget.c_str());
    UTIL_THROW_IF2(!inFileTarget, "could not open file " << m_filePathTarget);

    while (getline(inFileTarget, line)) {
      m_vocabTarget.insert(line);
    }

    inFileTarget.close();

    m_unrestricted = false;
  }
}

void WordTranslationFeature::EvaluateWithSourceContext(const InputType &input
    , const InputPath &inputPath
    , const TargetPhrase &targetPhrase
    , const StackVec *stackVec
    , ScoreComponentCollection &scoreBreakdown
    , ScoreComponentCollection *estimatedScores) const
{
  const Sentence& sentence = static_cast<const Sentence&>(input);
  const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();

  // process aligned words
  for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) {
    const Phrase& sourcePhrase = inputPath.GetPhrase();
    int sourceIndex = alignmentPoint->first;
    int targetIndex = alignmentPoint->second;
    Word ws = sourcePhrase.GetWord(sourceIndex);
    if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue;
    Word wt = targetPhrase.GetWord(targetIndex);
    if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue;
    StringPiece sourceWord = ws.GetFactor(m_factorTypeSource)->GetString();
    StringPiece targetWord = wt.GetFactor(m_factorTypeTarget)->GetString();
    if (m_ignorePunctuation) {
      // check if source or target are punctuation
      char firstChar = sourceWord[0];
      CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
      if(charIterator != m_punctuationHash.end())
        continue;
      firstChar = targetWord[0];
      charIterator = m_punctuationHash.find( firstChar );
      if(charIterator != m_punctuationHash.end())
        continue;
    }

    if (!m_unrestricted) {
      if (FindStringPiece(m_vocabSource, sourceWord) == m_vocabSource.end())
        sourceWord = "OTHER";
      if (FindStringPiece(m_vocabTarget, targetWord) == m_vocabTarget.end())
        targetWord = "OTHER";
    }

    if (m_simple) {
      // construct feature name
      util::StringStream featureName;
      featureName << m_description << "_";
      featureName << sourceWord;
      featureName << "~";
      featureName << targetWord;
      scoreBreakdown.SparsePlusEquals(featureName.str(), 1);
    }
    if (m_domainTrigger && !m_sourceContext) {
      const bool use_topicid = sentence.GetUseTopicId();
      const bool use_topicid_prob = sentence.GetUseTopicIdAndProb();
      if (use_topicid || use_topicid_prob) {
        if(use_topicid) {
          // use topicid as trigger
          const long topicid = sentence.GetTopicId();
          util::StringStream feature;
          feature << m_description << "_";
          if (topicid == -1)
            feature << "unk";
          else
            feature << topicid;

          feature << "_";
          feature << sourceWord;
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        } else {
          // use topic probabilities
          const vector<string> &topicid_prob = *(input.GetTopicIdAndProb());
          if (atol(topicid_prob[0].c_str()) == -1) {
            util::StringStream feature;
            feature << m_description << "_unk_";
            feature << sourceWord;
            feature << "~";
            feature << targetWord;
            scoreBreakdown.SparsePlusEquals(feature.str(), 1);
          } else {
            for (size_t i=0; i+1 < topicid_prob.size(); i+=2) {
              util::StringStream feature;
              feature << m_description << "_";
              feature << topicid_prob[i];
              feature << "_";
              feature << sourceWord;
              feature << "~";
              feature << targetWord;
              scoreBreakdown.SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str()));
            }
          }
        }
      } else {
        // range over domain trigger words (keywords)
        const long docid = input.GetDocumentId();
        for (boost::unordered_set<std::string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) {
          string sourceTrigger = *p;
          util::StringStream feature;
          feature << m_description << "_";
          feature << sourceTrigger;
          feature << "_";
          feature << sourceWord;
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        }
      }
    }
    if (m_sourceContext) {
      size_t globalSourceIndex = inputPath.GetWordsRange().GetStartPos() + sourceIndex;
      if (!m_domainTrigger && globalSourceIndex == 0) {
        // add <s> trigger feature for source
        util::StringStream feature;
        feature << m_description << "_";
        feature << "<s>,";
        feature << sourceWord;
        feature << "~";
        feature << targetWord;
        scoreBreakdown.SparsePlusEquals(feature.str(), 1);
      }

      // range over source words to get context
      for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
        if (contextIndex == globalSourceIndex) continue;
        StringPiece sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString();
        if (m_ignorePunctuation) {
          // check if trigger is punctuation
          char firstChar = sourceTrigger[0];
          CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
          if(charIterator != m_punctuationHash.end())
            continue;
        }

        const long docid = input.GetDocumentId();
        bool sourceTriggerExists = false;
        if (m_domainTrigger)
          sourceTriggerExists = FindStringPiece(m_vocabDomain[docid], sourceTrigger ) != m_vocabDomain[docid].end();
        else if (!m_unrestricted)
          sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();

        if (m_domainTrigger) {
          if (sourceTriggerExists) {
            util::StringStream feature;
            feature << m_description << "_";
            feature << sourceTrigger;
            feature << "_";
            feature << sourceWord;
            feature << "~";
            feature << targetWord;
            scoreBreakdown.SparsePlusEquals(feature.str(), 1);
          }
        } else if (m_unrestricted || sourceTriggerExists) {
          util::StringStream feature;
          feature << m_description << "_";
          if (contextIndex < globalSourceIndex) {
            feature << sourceTrigger;
            feature << ",";
            feature << sourceWord;
          } else {
            feature << sourceWord;
            feature << ",";
            feature << sourceTrigger;
          }
          feature << "~";
          feature << targetWord;
          scoreBreakdown.SparsePlusEquals(feature.str(), 1);
        }
      }
    }
    if (m_targetContext) {
      throw runtime_error("Can't use target words outside current translation option in a stateless feature");
      /*
      size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex;
      if (globalTargetIndex == 0) {
      	// add <s> trigger feature for source
      	stringstream feature;
      	feature << "wt_";
      	feature << sourceWord;
      	feature << "~";
      	feature << "<s>,";
      	feature << targetWord;
      	accumulator->SparsePlusEquals(feature.str(), 1);
      }

      // range over target words (up to current position) to get context
      for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) {
      	string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
      	if (m_ignorePunctuation) {
      		// check if trigger is punctuation
      		char firstChar = targetTrigger.at(0);
      		CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
      		if(charIterator != m_punctuationHash.end())
      			continue;
      	}

      	bool targetTriggerExists = false;
      	if (!m_unrestricted)
      		targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end();

      	if (m_unrestricted || targetTriggerExists) {
      		stringstream feature;
      		feature << "wt_";
      		feature << sourceWord;
      		feature << "~";
      		feature << targetTrigger;
      		feature << ",";
      		feature << targetWord;
      		accumulator->SparsePlusEquals(feature.str(), 1);
      	}
      }*/
    }
  }
}

bool WordTranslationFeature::IsUseable(const FactorMask &mask) const
{
  bool ret = mask[m_factorTypeTarget];
  return ret;
}

}
<|endoftext|>


<commit_before>#include "TranslationRequest.h"
#include "moses/ContextScope.h"
#include <boost/foreach.hpp>

namespace MosesServer
{
using namespace std;
using Moses::Hypothesis;
using Moses::StaticData;
using Moses::WordsRange;
using Moses::ChartHypothesis;
using Moses::Phrase;
using Moses::Manager;
using Moses::SearchGraphNode;
using Moses::TrellisPathList;
using Moses::TranslationOptionCollection;
using Moses::TranslationOptionList;
using Moses::TranslationOption;
using Moses::TargetPhrase;
using Moses::FValue;
using Moses::PhraseDictionaryMultiModel;
using Moses::FindPhraseDictionary;
using Moses::Sentence;

boost::shared_ptr<TranslationRequest>
TranslationRequest::
create(xmlrpc_c::paramList const& paramList,
       boost::condition_variable& cond,
       boost::mutex& mut)
{
  boost::shared_ptr<TranslationRequest> ret;
  ret.reset(new TranslationRequest(paramList,cond, mut));
  ret->m_self = ret;
  ret->m_scope.reset(new Moses::ContextScope);
  return ret;
}

void
TranslationRequest::
Run()
{
  parse_request(m_paramList.getStruct(0));

  Moses::StaticData const& SD = Moses::StaticData::Instance();

  //Make sure alternative paths are retained, if necessary
  if (m_withGraphInfo || m_nbestSize>0)
    // why on earth is this a global variable? Is this even thread-safe???? UG
    (const_cast<Moses::StaticData&>(SD)).SetOutputSearchGraph(true);

  std::stringstream out, graphInfo, transCollOpts;

  if (SD.IsSyntax())
    run_chart_decoder();
  else
    run_phrase_decoder();

  XVERBOSE(1,"Output: " << out.str() << endl);
  {
    boost::lock_guard<boost::mutex> lock(m_mutex);
    m_done = true;
  }
  m_cond.notify_one();

}

/// add phrase alignment information from a Hypothesis
void
TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{
  if (!m_withAlignInfo) return;
  WordsRange const& trg = h.GetCurrTargetWordsRange();
  WordsRange const& src = h.GetCurrSourceWordsRange();

  std::map<std::string, xmlrpc_c::value> pAlnInfo;
  pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos());
  pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
  pAlnInfo["src-end"]   = xmlrpc_c::value_int(src.GetEndPos());
  aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
}

void
TranslationRequest::
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
{
  Phrase outPhrase(20);
  hypo->GetOutputPhrase(outPhrase);

  // delete 1st & last
  assert(outPhrase.GetSize() >= 2);
  outPhrase.RemoveWord(0);
  outPhrase.RemoveWord(outPhrase.GetSize() - 1);
  for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
    out << *outPhrase.GetFactor(pos, 0) << " ";
}

bool
TranslationRequest::
compareSearchGraphNode(const Moses::SearchGraphNode& a,
                       const Moses::SearchGraphNode& b)
{
  return a.hypo->GetId() < b.hypo->GetId();
}

void
TranslationRequest::
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
{
  using xmlrpc_c::value_int;
  using xmlrpc_c::value_double;
  using xmlrpc_c::value_struct;
  using xmlrpc_c::value_string;
  vector<xmlrpc_c::value> searchGraphXml;
  vector<SearchGraphNode> searchGraph;
  manager.GetSearchGraph(searchGraph);
  std::sort(searchGraph.begin(), searchGraph.end());
  BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
    map<string, xmlrpc_c::value> x; // search graph xml node
    x["forward"] = value_double(n.forward);
    x["fscore"] = value_double(n.fscore);
    const Hypothesis* hypo = n.hypo;
    x["hyp"] = value_int(hypo->GetId());
    x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
    if (hypo->GetId() != 0) {
      const Hypothesis *prevHypo = hypo->GetPrevHypo();
      x["back"] = value_int(prevHypo->GetId());
      x["score"] = value_double(hypo->GetScore());
      x["transition"] = value_double(hypo->GetScore() - prevHypo->GetScore());
      if (n.recombinationHypo)
        x["recombined"] = value_int(n.recombinationHypo->GetId());
      x["cover-start"] = value_int(hypo->GetCurrSourceWordsRange().GetStartPos());
      x["cover-end"] = value_int(hypo->GetCurrSourceWordsRange().GetEndPos());
      x["out"] = value_string(hypo->GetCurrTargetPhrase().GetStringRep(StaticData::Instance().GetOutputFactorOrder()));
    }
    searchGraphXml.push_back(value_struct(x));
  }
  retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
}

void
TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const
{
  if (!m_reportAllFactors) {
    for (size_t i = 0 ; i < phrase.GetSize(); ++i)
      out << *phrase.GetFactor(i, 0) << " ";
  } else out << phrase;
}

void
TranslationRequest::
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
{
  TrellisPathList nBestList;
  vector<xmlrpc_c::value> nBestXml;
  manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct);

  BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
    vector<const Hypothesis *> const& E = path->GetEdges();
    if (!E.size()) continue;
    std::map<std::string, xmlrpc_c::value> nBestXmlItem;
    pack_hypothesis(E, "hyp", nBestXmlItem);
    if (m_withScoreBreakdown) {
      // should the score breakdown be reported in a more structured manner?
      ostringstream buf;
      path->GetScoreBreakdown()->OutputAllFeatureScores(buf);
      nBestXmlItem["fvals"] = xmlrpc_c::value_string(buf.str());
    }

    // weighted score
    nBestXmlItem["totalScore"] = xmlrpc_c::value_double(path->GetTotalScore());
    nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
  }
  retData["nbest"] = xmlrpc_c::value_array(nBestXml);
}

void
TranslationRequest::
insertTranslationOptions(Moses::Manager& manager,
                         std::map<std::string, xmlrpc_c::value>& retData)
{
  const TranslationOptionCollection* toptsColl
  = manager.getSntTranslationOptions();
  vector<xmlrpc_c::value> toptsXml;
  size_t const stop = toptsColl->GetSource().GetSize();
  TranslationOptionList const* tol;
  for (size_t s = 0 ; s < stop ; ++s) {
    for (size_t e = s;
         (tol = toptsColl->GetTranslationOptionList(s,e)) != NULL;
         ++e) {
      BOOST_FOREACH(TranslationOption const* topt, *tol) {
        std::map<std::string, xmlrpc_c::value> toptXml;
        TargetPhrase const& tp = topt->GetTargetPhrase();
        StaticData const& GLOBAL = StaticData::Instance();
        std::string tphrase = tp.GetStringRep(GLOBAL.GetOutputFactorOrder());
        toptXml["phrase"] = xmlrpc_c::value_string(tphrase);
        toptXml["fscore"] = xmlrpc_c::value_double(topt->GetFutureScore());
        toptXml["start"]  = xmlrpc_c::value_int(s);
        toptXml["end"]    = xmlrpc_c::value_int(e);
        vector<xmlrpc_c::value> scoresXml;
        const std::valarray<FValue> &scores
        = topt->GetScoreBreakdown().getCoreFeatures();
        for (size_t j = 0; j < scores.size(); ++j)
          scoresXml.push_back(xmlrpc_c::value_double(scores[j]));

        toptXml["scores"] = xmlrpc_c::value_array(scoresXml);
        toptsXml.push_back(xmlrpc_c::value_struct(toptXml));
      }
    }
  }
  retData["topt"] = xmlrpc_c::value_array(toptsXml);
}

bool
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
{
  std::map<std::string, xmlrpc_c::value>::const_iterator m;
  return (params.find(key) != params.end());
}

TranslationRequest::
TranslationRequest(xmlrpc_c::paramList const& paramList,
                   boost::condition_variable& cond, boost::mutex& mut)
  : m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
  , m_nbestSize(0)
{ }

void
TranslationRequest::
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
{
  // parse XMLRPC request
  // params_t const params = m_paramList.getStruct(0);
  m_paramList.verifyEnd(1); // ??? UG

  // source text must be given, or we don't know what to translate
  typedef std::map<std::string, xmlrpc_c::value> params_t;
  params_t::const_iterator si = params.find("text");
  if (si == params.end())
    throw xmlrpc_c::fault("Missing source text", xmlrpc_c::fault::CODE_PARSE);
  m_source_string = xmlrpc_c::value_string(si->second);
  XVERBOSE(1,"Input: " << m_source_string << endl);

  m_withAlignInfo       = check(params, "align");
  m_withWordAlignInfo   = check(params, "word-align");
  m_withGraphInfo       = check(params, "sg");
  m_withTopts           = check(params, "topt");
  m_reportAllFactors    = check(params, "report-all-factors");
  m_nbestDistinct       = check(params, "nbest-distinct");
  m_withScoreBreakdown  = check(params, "add-score-breakdown");
  m_source.reset(new Sentence(0,m_source_string));
  si = params.find("lambda");
  if (si != params.end()) {
    // muMo = multiModel
    xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
    vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
    vector<float> w(muMoValVec.size());
    for (size_t i = 0; i < muMoValVec.size(); ++i)
      w[i] = xmlrpc_c::value_double(muMoValVec[i]);
    if (w.size() && (si = params.find("model_name")) != params.end()) {
      string const model_name = xmlrpc_c::value_string(si->second);
      PhraseDictionaryMultiModel* pdmm
      = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
      // Moses::PhraseDictionaryMultiModel* pdmm
      // = FindPhraseDictionary(model_name);
      pdmm->SetTemporaryMultiModelWeightsVector(w);
    }
  }

  si = params.find("nbest");
  if (si != params.end())
    m_nbestSize = xmlrpc_c::value_int(si->second);

  si = params.find("context");
  if (si != params.end()) {
    string context = xmlrpc_c::value_string(si->second);
    m_context.reset(new std::vector<std::string>(1,context));
  }
  // // biased sampling for suffix-array-based sampling phrase table?
  // if ((si = params.find("bias")) != params.end())
  //   {
  // 	std::vector<xmlrpc_c::value> tmp
  // 	  = xmlrpc_c::value_array(si->second).cvalue();
  // 	for (size_t i = 1; i < tmp.size(); i += 2)
  // 	  m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]);
  //   }
} // end of Translationtask::parse_request()

void
TranslationRequest::
run_chart_decoder()
{
  Moses::TreeInput tinput;
  istringstream buf(m_source_string + "\n");
  tinput.Read(buf, StaticData::Instance().GetInputFactorOrder());

  Moses::ChartManager manager(this->self());
  manager.Decode();

  const Moses::ChartHypothesis *hypo = manager.GetBestHypothesis();
  ostringstream out;
  outputChartHypo(out,hypo);

  m_target_string = out.str();
  m_retData["text"] = xmlrpc_c::value_string(m_target_string);

  if (m_withGraphInfo) {
    std::ostringstream sgstream;
    manager.OutputSearchGraphMoses(sgstream);
    m_retData["sg"] =  xmlrpc_c::value_string(sgstream.str());
  }
} // end of TranslationRequest::run_chart_decoder()

void
TranslationRequest::
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
                map<string, xmlrpc_c::value> & dest) const
{
  // target string
  ostringstream target;
  BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
  output_phrase(target, e->GetCurrTargetPhrase());
  dest[key] = xmlrpc_c::value_string(target.str());

  if (m_withAlignInfo) {
    // phrase alignment, if requested

    vector<xmlrpc_c::value> p_aln;
    BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
    add_phrase_aln_info(*e, p_aln);
    dest["align"] = xmlrpc_c::value_array(p_aln);
  }

  if (m_withWordAlignInfo) {
    // word alignment, if requested
    vector<xmlrpc_c::value> w_aln;
    BOOST_FOREACH(Hypothesis const* e, edges)
    e->OutputLocalWordAlignment(w_aln);
    dest["word-align"] = xmlrpc_c::value_array(w_aln);
  }
}

void
TranslationRequest::
pack_hypothesis(Hypothesis const* h, string const& key,
                map<string, xmlrpc_c::value>& dest) const
{
  using namespace std;
  vector<Hypothesis const*> edges;
  for (; h; h = h->GetPrevHypo())
    edges.push_back(h);
  pack_hypothesis(edges, key, dest);
}

void
TranslationRequest::
run_phrase_decoder()
{
  Manager manager(this->self());
  // if (m_bias.size()) manager.SetBias(&m_bias);
  manager.Decode();

  pack_hypothesis(manager.GetBestHypothesis(), "text", m_retData);

  if (m_withGraphInfo) insertGraphInfo(manager,m_retData);
  if (m_withTopts) insertTranslationOptions(manager,m_retData);
  if (m_nbestSize) outputNBest(manager, m_retData);

  (const_cast<StaticData&>(Moses::StaticData::Instance()))
  .SetOutputSearchGraph(false);
  // WTF? one more reason not to have this as global variable! --- UG

}
}
<commit_msg>Reporting of context parameter in TranslationRequest constructor.<commit_after>#include "TranslationRequest.h"
#include "moses/ContextScope.h"
#include <boost/foreach.hpp>

namespace MosesServer
{
using namespace std;
using Moses::Hypothesis;
using Moses::StaticData;
using Moses::WordsRange;
using Moses::ChartHypothesis;
using Moses::Phrase;
using Moses::Manager;
using Moses::SearchGraphNode;
using Moses::TrellisPathList;
using Moses::TranslationOptionCollection;
using Moses::TranslationOptionList;
using Moses::TranslationOption;
using Moses::TargetPhrase;
using Moses::FValue;
using Moses::PhraseDictionaryMultiModel;
using Moses::FindPhraseDictionary;
using Moses::Sentence;

boost::shared_ptr<TranslationRequest>
TranslationRequest::
create(xmlrpc_c::paramList const& paramList,
       boost::condition_variable& cond,
       boost::mutex& mut)
{
  boost::shared_ptr<TranslationRequest> ret;
  ret.reset(new TranslationRequest(paramList,cond, mut));
  ret->m_self = ret;
  ret->m_scope.reset(new Moses::ContextScope);
  return ret;
}

void
TranslationRequest::
Run()
{
  parse_request(m_paramList.getStruct(0));

  Moses::StaticData const& SD = Moses::StaticData::Instance();

  //Make sure alternative paths are retained, if necessary
  if (m_withGraphInfo || m_nbestSize>0)
    // why on earth is this a global variable? Is this even thread-safe???? UG
    (const_cast<Moses::StaticData&>(SD)).SetOutputSearchGraph(true);

  std::stringstream out, graphInfo, transCollOpts;

  if (SD.IsSyntax())
    run_chart_decoder();
  else
    run_phrase_decoder();

  XVERBOSE(1,"Output: " << out.str() << endl);
  {
    boost::lock_guard<boost::mutex> lock(m_mutex);
    m_done = true;
  }
  m_cond.notify_one();

}

/// add phrase alignment information from a Hypothesis
void
TranslationRequest::
add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
{
  if (!m_withAlignInfo) return;
  WordsRange const& trg = h.GetCurrTargetWordsRange();
  WordsRange const& src = h.GetCurrSourceWordsRange();

  std::map<std::string, xmlrpc_c::value> pAlnInfo;
  pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos());
  pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
  pAlnInfo["src-end"]   = xmlrpc_c::value_int(src.GetEndPos());
  aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
}

void
TranslationRequest::
outputChartHypo(ostream& out, const ChartHypothesis* hypo)
{
  Phrase outPhrase(20);
  hypo->GetOutputPhrase(outPhrase);

  // delete 1st & last
  assert(outPhrase.GetSize() >= 2);
  outPhrase.RemoveWord(0);
  outPhrase.RemoveWord(outPhrase.GetSize() - 1);
  for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
    out << *outPhrase.GetFactor(pos, 0) << " ";
}

bool
TranslationRequest::
compareSearchGraphNode(const Moses::SearchGraphNode& a,
                       const Moses::SearchGraphNode& b)
{
  return a.hypo->GetId() < b.hypo->GetId();
}

void
TranslationRequest::
insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
{
  using xmlrpc_c::value_int;
  using xmlrpc_c::value_double;
  using xmlrpc_c::value_struct;
  using xmlrpc_c::value_string;
  vector<xmlrpc_c::value> searchGraphXml;
  vector<SearchGraphNode> searchGraph;
  manager.GetSearchGraph(searchGraph);
  std::sort(searchGraph.begin(), searchGraph.end());
  BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
    map<string, xmlrpc_c::value> x; // search graph xml node
    x["forward"] = value_double(n.forward);
    x["fscore"] = value_double(n.fscore);
    const Hypothesis* hypo = n.hypo;
    x["hyp"] = value_int(hypo->GetId());
    x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
    if (hypo->GetId() != 0) {
      const Hypothesis *prevHypo = hypo->GetPrevHypo();
      x["back"] = value_int(prevHypo->GetId());
      x["score"] = value_double(hypo->GetScore());
      x["transition"] = value_double(hypo->GetScore() - prevHypo->GetScore());
      if (n.recombinationHypo)
        x["recombined"] = value_int(n.recombinationHypo->GetId());
      x["cover-start"] = value_int(hypo->GetCurrSourceWordsRange().GetStartPos());
      x["cover-end"] = value_int(hypo->GetCurrSourceWordsRange().GetEndPos());
      x["out"] = value_string(hypo->GetCurrTargetPhrase().GetStringRep(StaticData::Instance().GetOutputFactorOrder()));
    }
    searchGraphXml.push_back(value_struct(x));
  }
  retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
}

void
TranslationRequest::
output_phrase(ostream& out, Phrase const& phrase) const
{
  if (!m_reportAllFactors) {
    for (size_t i = 0 ; i < phrase.GetSize(); ++i)
      out << *phrase.GetFactor(i, 0) << " ";
  } else out << phrase;
}

void
TranslationRequest::
outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
{
  TrellisPathList nBestList;
  vector<xmlrpc_c::value> nBestXml;
  manager.CalcNBest(m_nbestSize, nBestList, m_nbestDistinct);

  BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
    vector<const Hypothesis *> const& E = path->GetEdges();
    if (!E.size()) continue;
    std::map<std::string, xmlrpc_c::value> nBestXmlItem;
    pack_hypothesis(E, "hyp", nBestXmlItem);
    if (m_withScoreBreakdown) {
      // should the score breakdown be reported in a more structured manner?
      ostringstream buf;
      path->GetScoreBreakdown()->OutputAllFeatureScores(buf);
      nBestXmlItem["fvals"] = xmlrpc_c::value_string(buf.str());
    }

    // weighted score
    nBestXmlItem["totalScore"] = xmlrpc_c::value_double(path->GetTotalScore());
    nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
  }
  retData["nbest"] = xmlrpc_c::value_array(nBestXml);
}

void
TranslationRequest::
insertTranslationOptions(Moses::Manager& manager,
                         std::map<std::string, xmlrpc_c::value>& retData)
{
  const TranslationOptionCollection* toptsColl
  = manager.getSntTranslationOptions();
  vector<xmlrpc_c::value> toptsXml;
  size_t const stop = toptsColl->GetSource().GetSize();
  TranslationOptionList const* tol;
  for (size_t s = 0 ; s < stop ; ++s) {
    for (size_t e = s;
         (tol = toptsColl->GetTranslationOptionList(s,e)) != NULL;
         ++e) {
      BOOST_FOREACH(TranslationOption const* topt, *tol) {
        std::map<std::string, xmlrpc_c::value> toptXml;
        TargetPhrase const& tp = topt->GetTargetPhrase();
        StaticData const& GLOBAL = StaticData::Instance();
        std::string tphrase = tp.GetStringRep(GLOBAL.GetOutputFactorOrder());
        toptXml["phrase"] = xmlrpc_c::value_string(tphrase);
        toptXml["fscore"] = xmlrpc_c::value_double(topt->GetFutureScore());
        toptXml["start"]  = xmlrpc_c::value_int(s);
        toptXml["end"]    = xmlrpc_c::value_int(e);
        vector<xmlrpc_c::value> scoresXml;
        const std::valarray<FValue> &scores
        = topt->GetScoreBreakdown().getCoreFeatures();
        for (size_t j = 0; j < scores.size(); ++j)
          scoresXml.push_back(xmlrpc_c::value_double(scores[j]));

        toptXml["scores"] = xmlrpc_c::value_array(scoresXml);
        toptsXml.push_back(xmlrpc_c::value_struct(toptXml));
      }
    }
  }
  retData["topt"] = xmlrpc_c::value_array(toptsXml);
}

bool
check(std::map<std::string, xmlrpc_c::value> const& params, std::string const key)
{
  std::map<std::string, xmlrpc_c::value>::const_iterator m;
  return (params.find(key) != params.end());
}

TranslationRequest::
TranslationRequest(xmlrpc_c::paramList const& paramList,
                   boost::condition_variable& cond, boost::mutex& mut)
  : m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
  , m_nbestSize(0)
{ }

void
TranslationRequest::
parse_request(std::map<std::string, xmlrpc_c::value> const& params)
{
  // parse XMLRPC request
  // params_t const params = m_paramList.getStruct(0);
  m_paramList.verifyEnd(1); // ??? UG

  // source text must be given, or we don't know what to translate
  typedef std::map<std::string, xmlrpc_c::value> params_t;
  params_t::const_iterator si = params.find("text");
  if (si == params.end())
    throw xmlrpc_c::fault("Missing source text", xmlrpc_c::fault::CODE_PARSE);
  m_source_string = xmlrpc_c::value_string(si->second);
  XVERBOSE(1,"Input: " << m_source_string << endl);

  m_withAlignInfo       = check(params, "align");
  m_withWordAlignInfo   = check(params, "word-align");
  m_withGraphInfo       = check(params, "sg");
  m_withTopts           = check(params, "topt");
  m_reportAllFactors    = check(params, "report-all-factors");
  m_nbestDistinct       = check(params, "nbest-distinct");
  m_withScoreBreakdown  = check(params, "add-score-breakdown");
  m_source.reset(new Sentence(0,m_source_string));
  si = params.find("lambda");
  if (si != params.end()) {
    // muMo = multiModel
    xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
    vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
    vector<float> w(muMoValVec.size());
    for (size_t i = 0; i < muMoValVec.size(); ++i)
      w[i] = xmlrpc_c::value_double(muMoValVec[i]);
    if (w.size() && (si = params.find("model_name")) != params.end()) {
      string const model_name = xmlrpc_c::value_string(si->second);
      PhraseDictionaryMultiModel* pdmm
      = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
      // Moses::PhraseDictionaryMultiModel* pdmm
      // = FindPhraseDictionary(model_name);
      pdmm->SetTemporaryMultiModelWeightsVector(w);
    }
  }

  si = params.find("nbest");
  if (si != params.end())
    m_nbestSize = xmlrpc_c::value_int(si->second);

  si = params.find("context");
  if (si != params.end()) {
    string context = xmlrpc_c::value_string(si->second);
    VERBOSE(1,"CONTEXT " << context);
    m_context.reset(new std::vector<std::string>(1,context));
  }
  // // biased sampling for suffix-array-based sampling phrase table?
  // if ((si = params.find("bias")) != params.end())
  //   {
  // 	std::vector<xmlrpc_c::value> tmp
  // 	  = xmlrpc_c::value_array(si->second).cvalue();
  // 	for (size_t i = 1; i < tmp.size(); i += 2)
  // 	  m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]);
  //   }
} // end of Translationtask::parse_request()

void
TranslationRequest::
run_chart_decoder()
{
  Moses::TreeInput tinput;
  istringstream buf(m_source_string + "\n");
  tinput.Read(buf, StaticData::Instance().GetInputFactorOrder());

  Moses::ChartManager manager(this->self());
  manager.Decode();

  const Moses::ChartHypothesis *hypo = manager.GetBestHypothesis();
  ostringstream out;
  outputChartHypo(out,hypo);

  m_target_string = out.str();
  m_retData["text"] = xmlrpc_c::value_string(m_target_string);

  if (m_withGraphInfo) {
    std::ostringstream sgstream;
    manager.OutputSearchGraphMoses(sgstream);
    m_retData["sg"] =  xmlrpc_c::value_string(sgstream.str());
  }
} // end of TranslationRequest::run_chart_decoder()

void
TranslationRequest::
pack_hypothesis(vector<Hypothesis const* > const& edges, string const& key,
                map<string, xmlrpc_c::value> & dest) const
{
  // target string
  ostringstream target;
  BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
  output_phrase(target, e->GetCurrTargetPhrase());
  dest[key] = xmlrpc_c::value_string(target.str());

  if (m_withAlignInfo) {
    // phrase alignment, if requested

    vector<xmlrpc_c::value> p_aln;
    BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
    add_phrase_aln_info(*e, p_aln);
    dest["align"] = xmlrpc_c::value_array(p_aln);
  }

  if (m_withWordAlignInfo) {
    // word alignment, if requested
    vector<xmlrpc_c::value> w_aln;
    BOOST_FOREACH(Hypothesis const* e, edges)
    e->OutputLocalWordAlignment(w_aln);
    dest["word-align"] = xmlrpc_c::value_array(w_aln);
  }
}

void
TranslationRequest::
pack_hypothesis(Hypothesis const* h, string const& key,
                map<string, xmlrpc_c::value>& dest) const
{
  using namespace std;
  vector<Hypothesis const*> edges;
  for (; h; h = h->GetPrevHypo())
    edges.push_back(h);
  pack_hypothesis(edges, key, dest);
}

void
TranslationRequest::
run_phrase_decoder()
{
  Manager manager(this->self());
  // if (m_bias.size()) manager.SetBias(&m_bias);
  manager.Decode();

  pack_hypothesis(manager.GetBestHypothesis(), "text", m_retData);

  if (m_withGraphInfo) insertGraphInfo(manager,m_retData);
  if (m_withTopts) insertTranslationOptions(manager,m_retData);
  if (m_nbestSize) outputNBest(manager, m_retData);

  (const_cast<StaticData&>(Moses::StaticData::Instance()))
  .SetOutputSearchGraph(false);
  // WTF? one more reason not to have this as global variable! --- UG

}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "Client.hxx"
#include "Marshal.hxx"
#include "translation/Parser.hxx"
#include "translation/Request.hxx"
#include "translation/Response.hxx"
#include "translation/Handler.hxx"
#include "event/net/BufferedSocket.hxx"
#include "stopwatch.hxx"
#include "pool/pool.hxx"
#include "system/Error.hxx"
#include "util/Cancellable.hxx"
#include "util/Exception.hxx"
#include "lease.hxx"

#include <stdexcept>

#include <assert.h>
#include <string.h>

static const uint8_t PROTOCOL_VERSION = 3;

class TranslateClient final : BufferedSocketHandler, Cancellable {
	const StopwatchPtr stopwatch;

	BufferedSocket socket;
	LeasePtr lease_ref;

	/** the marshalled translate request */
	GrowingBufferReader request;

	TranslateHandler &handler;

	TranslateParser parser;

public:
	TranslateClient(AllocatorPtr alloc, EventLoop &event_loop,
			StopwatchPtr &&_stopwatch,
			SocketDescriptor fd, Lease &lease,
			const TranslateRequest &request2,
			GrowingBuffer &&_request,
			TranslateHandler &_handler,
			CancellablePointer &cancel_ptr) noexcept;

	bool TryWrite() noexcept;

private:
	void Destroy() noexcept {
		this->~TranslateClient();
	}

	void ReleaseSocket(bool reuse) noexcept;

	void Fail(std::exception_ptr ep) noexcept;

	BufferedResult Feed(const uint8_t *data, size_t length) noexcept;

	/* virtual methods from class BufferedSocketHandler */
	BufferedResult OnBufferedData() override {
		auto r = socket.ReadBuffer();
		assert(!r.empty());
		return Feed((const uint8_t *)r.data, r.size);
	}

	bool OnBufferedClosed() noexcept override {
		ReleaseSocket(false);
		return true;
	}

	bool OnBufferedWrite() override {
		return TryWrite();
	}

	void OnBufferedError(std::exception_ptr ep) noexcept override {
		Fail(NestException(ep,
				   std::runtime_error("Translation server connection failed")));
	}

	/* virtual methods from class Cancellable */
	void Cancel() noexcept override {
		stopwatch.RecordEvent("cancel");
		ReleaseSocket(false);
		Destroy();
	}
};

static constexpr auto translate_read_timeout = std::chrono::minutes(1);
static constexpr auto translate_write_timeout = std::chrono::seconds(10);

void
TranslateClient::ReleaseSocket(bool reuse) noexcept
{
	assert(socket.IsConnected());

	socket.Abandon();
	socket.Destroy();

	lease_ref.Release(reuse);
}

void
TranslateClient::Fail(std::exception_ptr ep) noexcept
{
	stopwatch.RecordEvent("error");

	ReleaseSocket(false);

	auto &_handler = handler;

	Destroy();

	_handler.OnTranslateError(ep);
}

/*
 * receive response
 *
 */

inline BufferedResult
TranslateClient::Feed(const uint8_t *data, size_t length) noexcept
try {
	size_t consumed = 0;
	while (consumed < length) {
		size_t nbytes = parser.Feed(data + consumed, length - consumed);
		if (nbytes == 0)
			/* need more data */
			break;

		consumed += nbytes;
		socket.DisposeConsumed(nbytes);

		auto result = parser.Process();
		switch (result) {
		case TranslateParser::Result::MORE:
			break;

		case TranslateParser::Result::DONE:
			ReleaseSocket(true);

			{
				auto &_handler = handler;
				auto &response = parser.GetResponse();
				Destroy();
				_handler.OnTranslateResponse(response);
			}

			return BufferedResult::CLOSED;
		}
	}

	return BufferedResult::MORE;
} catch (...) {
	Fail(std::current_exception());
	return BufferedResult::CLOSED;
}

/*
 * send requests
 *
 */

bool
TranslateClient::TryWrite() noexcept
{
	auto src = request.Read();
	assert(!src.IsNull());

	ssize_t nbytes = socket.Write(src.data, src.size);
	if (gcc_unlikely(nbytes < 0)) {
		if (gcc_likely(nbytes == WRITE_BLOCKING))
			return true;

		Fail(std::make_exception_ptr(MakeErrno("write error to translation server")));
		return false;
	}

	request.Consume(nbytes);
	if (request.IsEOF()) {
		/* the buffer is empty, i.e. the request has been sent */

		stopwatch.RecordEvent("request_end");

		socket.UnscheduleWrite();
		return socket.Read(true);
	}

	socket.ScheduleWrite();
	return true;
}

/*
 * constructor
 *
 */

inline
TranslateClient::TranslateClient(AllocatorPtr alloc, EventLoop &event_loop,
				 StopwatchPtr &&_stopwatch,
				 SocketDescriptor fd, Lease &lease,
				 const TranslateRequest &request2,
				 GrowingBuffer &&_request,
				 TranslateHandler &_handler,
				 CancellablePointer &cancel_ptr) noexcept
	:stopwatch(std::move(_stopwatch)),
	 socket(event_loop), lease_ref(lease),
	 request(std::move(_request)),
	 handler(_handler),
	 parser(alloc, request2, *alloc.New<TranslateResponse>())
{
	socket.Init(fd, FdType::FD_SOCKET,
		    translate_read_timeout,
		    translate_write_timeout,
		    *this);

	cancel_ptr = *this;

	socket.DeferWrite();
}

void
translate(AllocatorPtr alloc, EventLoop &event_loop,
	  StopwatchPtr stopwatch,
	  SocketDescriptor fd, Lease &lease,
	  const TranslateRequest &request,
	  TranslateHandler &handler,
	  CancellablePointer &cancel_ptr) noexcept
try {
	assert(fd.IsDefined());
	assert(request.uri != nullptr || request.widget_type != nullptr ||
	       request.http_auth != nullptr ||
	       request.token_auth != nullptr ||
	       request.chain != nullptr ||
	       request.pool != nullptr ||
	       (!request.content_type_lookup.IsNull() &&
		request.suffix != nullptr));

	GrowingBuffer gb = MarshalTranslateRequest(PROTOCOL_VERSION,
						   request);

	alloc.New<TranslateClient>(alloc, event_loop,
				   std::move(stopwatch),
				   fd, lease,
				   request, std::move(gb),
				   handler, cancel_ptr);
} catch (...) {
	lease.ReleaseLease(true);

	handler.OnTranslateError(std::current_exception());
}
<commit_msg>translation/Parser: use std::byte<commit_after>/*
 * Copyright 2007-2021 CM4all GmbH
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "Client.hxx"
#include "Marshal.hxx"
#include "translation/Parser.hxx"
#include "translation/Request.hxx"
#include "translation/Response.hxx"
#include "translation/Handler.hxx"
#include "event/net/BufferedSocket.hxx"
#include "stopwatch.hxx"
#include "pool/pool.hxx"
#include "system/Error.hxx"
#include "util/Cancellable.hxx"
#include "util/Exception.hxx"
#include "lease.hxx"

#include <stdexcept>

#include <assert.h>
#include <string.h>

static const uint8_t PROTOCOL_VERSION = 3;

class TranslateClient final : BufferedSocketHandler, Cancellable {
	const StopwatchPtr stopwatch;

	BufferedSocket socket;
	LeasePtr lease_ref;

	/** the marshalled translate request */
	GrowingBufferReader request;

	TranslateHandler &handler;

	TranslateParser parser;

public:
	TranslateClient(AllocatorPtr alloc, EventLoop &event_loop,
			StopwatchPtr &&_stopwatch,
			SocketDescriptor fd, Lease &lease,
			const TranslateRequest &request2,
			GrowingBuffer &&_request,
			TranslateHandler &_handler,
			CancellablePointer &cancel_ptr) noexcept;

	bool TryWrite() noexcept;

private:
	void Destroy() noexcept {
		this->~TranslateClient();
	}

	void ReleaseSocket(bool reuse) noexcept;

	void Fail(std::exception_ptr ep) noexcept;

	BufferedResult Feed(const std::byte *data, size_t length) noexcept;

	/* virtual methods from class BufferedSocketHandler */
	BufferedResult OnBufferedData() override {
		auto r = socket.ReadBuffer();
		assert(!r.empty());
		return Feed((const std::byte *)r.data, r.size);
	}

	bool OnBufferedClosed() noexcept override {
		ReleaseSocket(false);
		return true;
	}

	bool OnBufferedWrite() override {
		return TryWrite();
	}

	void OnBufferedError(std::exception_ptr ep) noexcept override {
		Fail(NestException(ep,
				   std::runtime_error("Translation server connection failed")));
	}

	/* virtual methods from class Cancellable */
	void Cancel() noexcept override {
		stopwatch.RecordEvent("cancel");
		ReleaseSocket(false);
		Destroy();
	}
};

static constexpr auto translate_read_timeout = std::chrono::minutes(1);
static constexpr auto translate_write_timeout = std::chrono::seconds(10);

void
TranslateClient::ReleaseSocket(bool reuse) noexcept
{
	assert(socket.IsConnected());

	socket.Abandon();
	socket.Destroy();

	lease_ref.Release(reuse);
}

void
TranslateClient::Fail(std::exception_ptr ep) noexcept
{
	stopwatch.RecordEvent("error");

	ReleaseSocket(false);

	auto &_handler = handler;

	Destroy();

	_handler.OnTranslateError(ep);
}

/*
 * receive response
 *
 */

inline BufferedResult
TranslateClient::Feed(const std::byte *data, size_t length) noexcept
try {
	size_t consumed = 0;
	while (consumed < length) {
		size_t nbytes = parser.Feed((const uint8_t *)(data + consumed), length - consumed);
		if (nbytes == 0)
			/* need more data */
			break;

		consumed += nbytes;
		socket.DisposeConsumed(nbytes);

		auto result = parser.Process();
		switch (result) {
		case TranslateParser::Result::MORE:
			break;

		case TranslateParser::Result::DONE:
			ReleaseSocket(true);

			{
				auto &_handler = handler;
				auto &response = parser.GetResponse();
				Destroy();
				_handler.OnTranslateResponse(response);
			}

			return BufferedResult::CLOSED;
		}
	}

	return BufferedResult::MORE;
} catch (...) {
	Fail(std::current_exception());
	return BufferedResult::CLOSED;
}

/*
 * send requests
 *
 */

bool
TranslateClient::TryWrite() noexcept
{
	auto src = request.Read();
	assert(!src.IsNull());

	ssize_t nbytes = socket.Write(src.data, src.size);
	if (gcc_unlikely(nbytes < 0)) {
		if (gcc_likely(nbytes == WRITE_BLOCKING))
			return true;

		Fail(std::make_exception_ptr(MakeErrno("write error to translation server")));
		return false;
	}

	request.Consume(nbytes);
	if (request.IsEOF()) {
		/* the buffer is empty, i.e. the request has been sent */

		stopwatch.RecordEvent("request_end");

		socket.UnscheduleWrite();
		return socket.Read(true);
	}

	socket.ScheduleWrite();
	return true;
}

/*
 * constructor
 *
 */

inline
TranslateClient::TranslateClient(AllocatorPtr alloc, EventLoop &event_loop,
				 StopwatchPtr &&_stopwatch,
				 SocketDescriptor fd, Lease &lease,
				 const TranslateRequest &request2,
				 GrowingBuffer &&_request,
				 TranslateHandler &_handler,
				 CancellablePointer &cancel_ptr) noexcept
	:stopwatch(std::move(_stopwatch)),
	 socket(event_loop), lease_ref(lease),
	 request(std::move(_request)),
	 handler(_handler),
	 parser(alloc, request2, *alloc.New<TranslateResponse>())
{
	socket.Init(fd, FdType::FD_SOCKET,
		    translate_read_timeout,
		    translate_write_timeout,
		    *this);

	cancel_ptr = *this;

	socket.DeferWrite();
}

void
translate(AllocatorPtr alloc, EventLoop &event_loop,
	  StopwatchPtr stopwatch,
	  SocketDescriptor fd, Lease &lease,
	  const TranslateRequest &request,
	  TranslateHandler &handler,
	  CancellablePointer &cancel_ptr) noexcept
try {
	assert(fd.IsDefined());
	assert(request.uri != nullptr || request.widget_type != nullptr ||
	       request.http_auth != nullptr ||
	       request.token_auth != nullptr ||
	       request.chain != nullptr ||
	       request.pool != nullptr ||
	       (!request.content_type_lookup.IsNull() &&
		request.suffix != nullptr));

	GrowingBuffer gb = MarshalTranslateRequest(PROTOCOL_VERSION,
						   request);

	alloc.New<TranslateClient>(alloc, event_loop,
				   std::move(stopwatch),
				   fd, lease,
				   request, std::move(gb),
				   handler, cancel_ptr);
} catch (...) {
	lease.ReleaseLease(true);

	handler.OnTranslateError(std::current_exception());
}
<|endoftext|>