<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */

#include "paddle/framework/backward.h"

#include <list>
#include "paddle/framework/op_registry.h"
#include "paddle/operators/net_op.h"

namespace paddle {
namespace framework {

static bool AllInSet(const std::vector<std::string>& names,
                     const std::string& suffix,
                     const std::unordered_set<std::string>& set) {
  for (auto& name : names) {
    if (set.find(name + suffix) == set.end()) {
      return false;
    }
  }
  return true;
}

static std::shared_ptr<OperatorBase> NOP() {
  auto net_op = std::make_shared<operators::NetOp>();
  net_op->type_ = "@NOP@";
  net_op->CompleteAddOp();
  return net_op;
}

//  Get backward operator from a forward operator, recursively implementation.
//
//  no_grad_names the gradient variable names without gradient calculating.
//
//  uniq_id is a unique index used inside recursively calling
//  BackwardRecursive. use `uid = uniq_id++;` to get the unique index, and
//  pass `uniq_id` through recursive calling.
//
//  returns The backward operator. For simple situation, it is a simple
//  operator. For complex situation, it is a NetOp.
//
//  See Backward.h for details
static std::shared_ptr<OperatorBase> BackwardRecursive(
    const OperatorBase& forwardOp,
    std::unordered_set<std::string>& no_grad_names, size_t& uniq_id);
std::shared_ptr<OperatorBase> BackwardRecursive(
    const OperatorBase& forwardOp,
    std::unordered_set<std::string>& no_grad_names, size_t& uniq_id) {
  //  If all input gradients of forwarding operator do not need to calculate,
  //  just return an NOP. Not return null ptr because NOP does not take
  //  too much time for calculation, but it is useful for simplifying logic.
  if (AllInSet(forwardOp.inputs_, kGradVarSuffix, no_grad_names)) {
    return NOP();
  }

  //  All output gradients of forwarding operator do not need to calculate.
  //  Then all input gradients cannot be computed at all, and we put them into
  //  `no_grad_names` set. Return an NOP.
  if (AllInSet(forwardOp.outputs_, kGradVarSuffix, no_grad_names)) {
    for (auto& name : forwardOp.inputs_) {
      // Mark all input is not need
      no_grad_names.insert(name + kGradVarSuffix);
    }
    return NOP();
  }

  // Returned gradient network
  auto net = std::make_shared<operators::NetOp>();

  if (forwardOp.IsNetOp()) {
    // Because forwardOp is a net op, it can static_cast.
    auto& forwardNet = static_cast<const operators::NetOp&>(forwardOp);

    // Map from output gradient variable name to operator's indices in
    // backward net. That operator generates that variable.
    std::unordered_map<std::string, std::vector<size_t>> dup_output_ops;

    size_t local_op_id = 0;
    // reversely travel forwardNet
    for (auto it = forwardNet.ops_.rbegin(); it != forwardNet.ops_.rend();
         ++it, ++local_op_id) {
      auto fwd = *it;
      auto bwd = BackwardRecursive(*fwd, no_grad_names, uniq_id);
      net->AddOp(bwd);
      for (auto& out : bwd->outputs_) {
        dup_output_ops[out].emplace_back(local_op_id);
      }
    }
    // Get unique ID for this method.
    auto uid = uniq_id++;
    // TODO(dzh): more comment
    using Pos = std::pair<size_t, std::shared_ptr<OperatorBase>>;
    std::list<Pos> insert_position;
    for (auto& dup_output_op : dup_output_ops) {
      const std::string& name = dup_output_op.first;
      auto& dup_op = dup_output_op.second;
      if (dup_op.size() == 1) continue;
      std::vector<std::string> dup_outputs;

      for (size_t i = 0; i < dup_op.size(); ++i) {
        auto op_offset = dup_op[i];
        dup_outputs.push_back(name + "@RENAME@" + std::to_string(uid) + "@" +
                              std::to_string(i));
        net->ops_[op_offset]->Rename(name, dup_outputs.back());
      }
      insert_position.push_back(
          {dup_op.back(),
           OpRegistry::CreateOp(
               "add", {dup_outputs}, {name},
               {{"input_format",
                 std::vector<int>{0, static_cast<int>(dup_outputs.size())}}})});
    }

    insert_position.sort(
        [](const Pos& l, const Pos& r) { return l.first > r.first; });

    for (auto& pos : insert_position) {
      net->InsertOp(pos.first + 1, pos.second);
    }

  } else {
    std::shared_ptr<OperatorBase> grad_op = OpRegistry::CreateGradOp(forwardOp);
    for (std::string& grad_input : grad_op->inputs_) {
      if (no_grad_names.count(grad_input)) {
        std::string prefix = grad_input.substr(
            0, grad_input.size() - sizeof(kGradVarSuffix) / sizeof(char));
        grad_input = prefix + kZeroVarSuffix;

        // If part of input gradient of that operator is not calculated, fill
        // zero variables to that input gradient.
        net->AddOp(OpRegistry::CreateOp("fill_zeros_like", {prefix},
                                        {grad_input}, {}));
      }
    }

    for (std::string& grad_output : grad_op->outputs_) {
      if (no_grad_names.count(grad_output)) {
        grad_output = kEmptyVarName;
      }
    }

    if (net->ops_.empty()) {  // Current no aux op is added to network
      return grad_op;
    }
    net->AddOp(grad_op);
  }
  net->type_ = "@GENERATED_BACKWARD@";
  net->CompleteAddOp();
  return net;
}

// See header for comments
std::shared_ptr<OperatorBase> Backward(
    const OperatorBase& forwardOp,
    const std::unordered_set<std::string>& no_grad_vars) {
  std::unordered_set<std::string> no_grad_names;
  no_grad_names.reserve(no_grad_vars.size());

  no_grad_names.insert(std::string(kEmptyVarName) + kGradVarSuffix);

  for (auto& name : no_grad_vars) {
    no_grad_names.insert(name + kGradVarSuffix);
  }
  size_t uid = 0;
  return BackwardRecursive(forwardOp, no_grad_names, uid);
}

}  // namespace framework
}  // namespace paddle
<commit_msg>Fix TravisCI test<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */

#include "paddle/framework/backward.h"

#include <list>
#include "paddle/framework/op_registry.h"
#include "paddle/operators/net_op.h"

namespace paddle {
namespace framework {

static bool AllInSet(const std::vector<std::string>& names,
                     const std::string& suffix,
                     const std::unordered_set<std::string>& set) {
  for (auto& name : names) {
    if (set.find(name + suffix) == set.end()) {
      return false;
    }
  }
  return true;
}

static std::shared_ptr<OperatorBase> NOP() {
  auto net_op = std::make_shared<operators::NetOp>();
  net_op->type_ = "@NOP@";
  net_op->CompleteAddOp();
  return net_op;
}

//  Get backward operator from a forward operator, recursively implementation.
//
//  no_grad_names the gradient variable names without gradient calculating.
//
//  uniq_id is a unique index used inside recursively calling
//  BackwardRecursive. use `uid = uniq_id++;` to get the unique index, and
//  pass `uniq_id` through recursive calling.
//
//  returns The backward operator. For simple situation, it is a simple
//  operator. For complex situation, it is a NetOp.
//
//  See Backward.h for details
static std::shared_ptr<OperatorBase> BackwardRecursive(
    const OperatorBase& forwardOp,
    std::unordered_set<std::string>& no_grad_names, size_t& uniq_id);
std::shared_ptr<OperatorBase> BackwardRecursive(
    const OperatorBase& forwardOp,
    std::unordered_set<std::string>& no_grad_names, size_t& uniq_id) {
  //  If all input gradients of forwarding operator do not need to calculate,
  //  just return an NOP. Not return null ptr because NOP does not take
  //  too much time for calculation, but it is useful for simplifying logic.
  if (AllInSet(forwardOp.inputs_, kGradVarSuffix, no_grad_names)) {
    return NOP();
  }

  //  All output gradients of forwarding operator do not need to calculate.
  //  Then all input gradients cannot be computed at all, and we put them into
  //  `no_grad_names` set. Return an NOP.
  if (AllInSet(forwardOp.outputs_, kGradVarSuffix, no_grad_names)) {
    for (auto& name : forwardOp.inputs_) {
      // Mark all input is not need
      no_grad_names.insert(name + kGradVarSuffix);
    }
    return NOP();
  }

  // Returned gradient network
  auto net = std::make_shared<operators::NetOp>();

  if (forwardOp.IsNetOp()) {
    // Because forwardOp is a net op, it can static_cast.
    auto& forwardNet = static_cast<const operators::NetOp&>(forwardOp);

    // Map from output gradient variable name to operator's indices in
    // backward net. That operator generates that variable.
    std::unordered_map<std::string, std::vector<size_t>> dup_output_ops;

    size_t local_op_id = 0;
    // reversely travel forwardNet
    for (auto it = forwardNet.ops_.rbegin(); it != forwardNet.ops_.rend();
         ++it, ++local_op_id) {
      auto fwd = *it;
      auto bwd = BackwardRecursive(*fwd, no_grad_names, uniq_id);
      net->AddOp(bwd);
      for (auto& out : bwd->outputs_) {
        dup_output_ops[out].emplace_back(local_op_id);
      }
    }
    // Get unique ID for this method.
    auto uid = uniq_id++;
    // TODO(dzh): more comment
    using Pos = std::pair<size_t, std::shared_ptr<OperatorBase>>;
    std::list<Pos> insert_position;
    for (auto& dup_output_op : dup_output_ops) {
      const std::string& name = dup_output_op.first;
      auto& dup_op = dup_output_op.second;
      if (dup_op.size() == 1) continue;
      std::vector<std::string> dup_outputs;

      for (size_t i = 0; i < dup_op.size(); ++i) {
        auto op_offset = dup_op[i];
        dup_outputs.push_back(name + "@RENAME@" + std::to_string(uid) + "@" +
                              std::to_string(i));
        net->ops_[op_offset]->Rename(name, dup_outputs.back());
      }
      insert_position.push_back(
          {dup_op.back(),
           OpRegistry::CreateOp(
               "add", {dup_outputs}, {name},
               {{"input_format",
                 std::vector<int>{0, static_cast<int>(dup_outputs.size())}}})});
    }

    insert_position.sort(
        [](const Pos& l, const Pos& r) { return l.first > r.first; });

    for (auto& pos : insert_position) {
      net->InsertOp(pos.first + 1, pos.second);
    }

  } else {
    std::shared_ptr<OperatorBase> grad_op = OpRegistry::CreateGradOp(forwardOp);
    for (std::string& grad_input : grad_op->inputs_) {
      if (no_grad_names.count(grad_input)) {
        // +1 for \0
        std::string prefix = grad_input.substr(
            0, grad_input.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1);
        grad_input = prefix + kZeroVarSuffix;

        // If part of input gradient of that operator is not calculated, fill
        // zero variables to that input gradient.
        net->AddOp(OpRegistry::CreateOp("fill_zeros_like", {prefix},
                                        {grad_input}, {}));
      }
    }

    for (std::string& grad_output : grad_op->outputs_) {
      if (no_grad_names.count(grad_output)) {
        grad_output = kEmptyVarName;
      }
    }

    if (net->ops_.empty()) {  // Current no aux op is added to network
      return grad_op;
    }
    net->AddOp(grad_op);
  }
  net->type_ = "@GENERATED_BACKWARD@";
  net->CompleteAddOp();
  return net;
}

// See header for comments
std::shared_ptr<OperatorBase> Backward(
    const OperatorBase& forwardOp,
    const std::unordered_set<std::string>& no_grad_vars) {
  std::unordered_set<std::string> no_grad_names;
  no_grad_names.reserve(no_grad_vars.size());

  no_grad_names.insert(std::string(kEmptyVarName) + kGradVarSuffix);

  for (auto& name : no_grad_vars) {
    no_grad_names.insert(name + kGradVarSuffix);
  }
  size_t uid = 0;
  return BackwardRecursive(forwardOp, no_grad_names, uid);
}

}  // namespace framework
}  // namespace paddle
<|endoftext|>


<commit_before>// -*- Mode: C++; tab-width: 2; -*-
// vi: set ts=2:
//
// $Id: regularData1DWidget.C,v 1.7 2004/01/26 16:16:45 anne Exp $
//

#include <BALL/VIEW/WIDGETS/regularData1DWidget.h>
#include <BALL/VIEW/KERNEL/mainControl.h>
#include <qpointarray.h>
#include <qpainter.h>
#include <qwmatrix.h>
#include <qlayout.h>
#include <qpointarray.h>

namespace BALL
{
	namespace VIEW
	{
		// Constructor
		RegularData1DWidget::RegularData1DWidget(const RegularData1D& data, QWidget *parent)
			throw()
			:CanvasWidget(parent),
			 data_(data),
			 diagram_color_(QColor(red)),
			background_color_(QColor(white)) //,
			 //	 height_(0),//400),
			 //	 line_style_(true)
		{
		}

		//Copyconstructor
		//!!!!!!!!!!!!!!!TODO cant work until copyconstructor of pixmap works

		/*	RegularData1DWidget::RegularData1DWidget(RegularData1DWidget* widget)
			throw()
			:pixmapWidget(widget),
			data_(widget->data),
			diagram_color_(widget->diagram_color_),
			background_color_(widget->background_color_)//,
		//	heigth_(widget->heigth_),
		//	line_style_(widget->line_style)
		{
		}*/

		// Destructor
		RegularData1DWidget::~RegularData1DWidget()
		 throw()
		{
		}

		// creates a polygon from a given vector RegularData1D * data

		void RegularData1DWidget::createPolygon()
			throw()
		{
 			// no data => no polygon
			if (data_.size() == 0) return;

			// find the data min and max
			float min = std::numeric_limits<float>::max();
			float max = std::numeric_limits<float>::min();
			float dif_min = min;
			float old = min; //last point

			for (int i=0; i<(int)data_.size(); i++)
			{
				if (data_[i] >= max)
					max = data_[i];
				if (data_[i] <= min)
					min = data_[i];

				float dif = data_[i] - old;

				if ( (dif > 1e-3) && (dif < dif_min) )
				{
					dif_min = dif;
				}

				old = data_[i];
			}

			height_ = (int)ceil(((max-min) / dif_min) * 5);

      if (height_ > 1000)
	    {
				// the resulting picture would be much too large
				height_ = 1000;
				dif_min = (max - min)/200.;
			}

			// resize the canvas to fit the data
			canvas_.resize(5*(data_.size()+2), height_+10);
//			resizeContents(canvas_.width(), canvas_.height());
			int x_new;
			int y_new;
			int x_old = 5;
			int y_old = height_+5 - (int)round(((data_[0]-min)/dif_min)*5);
			QCanvasLine *ql;
			QBrush br(QColor(black));
			//			ql.setBrush(br);

			for (int i=1; i<(int)data_.size(); i++)
			{
				x_new = 5*(i+1);
				y_new = height_+5 - (int)round(((data_[i]-min)/dif_min)*5);

				ql = new QCanvasLine(&canvas_);
				ql->setPoints(x_old, y_old, x_new, y_new);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
			//	ql->show();

				x_old = x_new;
				y_old = y_new;
			}
/*
			//add the x-axis
				ql = new QCanvasLine(&canvas_);
				int startx = 5;
				int starty = height_+5 - (int)round(((startx-min)/dif_min)*5);
				int endx   = data_.size()*5;
				int endy   = height_+5 - (int)round(((data_[endx]-min)/dif_min)*5);

				ql->setPoints(startx, starty, endx, endy);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
	*/
			 //QCanvasPolygon* =qp;

			//add the y-axis
		/*		ql = new QCanvasLine(&canvas_);
				startx = 0;
				starty = height_+5 - (int)round(((0-min)/dif_min)*5);
				endx   = data_.size()*5;
				endy   = height_+5 - (int)round(((data_[endx]-min)/dif_min)*5);
				ql->setPoints(startx, starty, endx, endy);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
	*/		}

	}//end of namespace VIEW

}//end of namespace BALL
<commit_msg>no message<commit_after>// -*- Mode: C++; tab-width: 2; -*-
// vi: set ts=2:
//
// $Id: regularData1DWidget.C,v 1.8 2004/03/03 18:49:06 amoll Exp $
//

#include <BALL/VIEW/WIDGETS/regularData1DWidget.h>
#include <BALL/VIEW/KERNEL/mainControl.h>
#include <BALL/COMMON/limits.h>
#include <qpointarray.h>
#include <qpainter.h>
#include <qwmatrix.h>
#include <qlayout.h>
#include <qpointarray.h>

namespace BALL
{
	namespace VIEW
	{
		// Constructor
		RegularData1DWidget::RegularData1DWidget(const RegularData1D& data, QWidget *parent)
			throw()
			:CanvasWidget(parent),
			 data_(data),
			 diagram_color_(QColor(red)),
			background_color_(QColor(white)) //,
			 //	 height_(0),//400),
			 //	 line_style_(true)
		{
		}

		//Copyconstructor
		//!!!!!!!!!!!!!!!TODO cant work until copyconstructor of pixmap works

		/*	RegularData1DWidget::RegularData1DWidget(RegularData1DWidget* widget)
			throw()
			:pixmapWidget(widget),
			data_(widget->data),
			diagram_color_(widget->diagram_color_),
			background_color_(widget->background_color_)//,
		//	heigth_(widget->heigth_),
		//	line_style_(widget->line_style)
		{
		}*/

		// Destructor
		RegularData1DWidget::~RegularData1DWidget()
		 throw()
		{
		}

		// creates a polygon from a given vector RegularData1D * data

		void RegularData1DWidget::createPolygon()
			throw()
		{
 			// no data => no polygon
			if (data_.size() == 0) return;

			// find the data min and max
			float min = Limits<float>::max();
			float max = Limits<float>::min();
			float dif_min = min;
			float old = min; //last point

			for (int i=0; i<(int)data_.size(); i++)
			{
				if (data_[i] >= max)
					max = data_[i];
				if (data_[i] <= min)
					min = data_[i];

				float dif = data_[i] - old;

				if ( (dif > 1e-3) && (dif < dif_min) )
				{
					dif_min = dif;
				}

				old = data_[i];
			}

			height_ = (int)ceil(((max-min) / dif_min) * 5);

      if (height_ > 1000)
	    {
				// the resulting picture would be much too large
				height_ = 1000;
				dif_min = (max - min)/200.;
			}

			// resize the canvas to fit the data
			canvas_.resize(5*(data_.size()+2), height_+10);
//			resizeContents(canvas_.width(), canvas_.height());
			int x_new;
			int y_new;
			int x_old = 5;
			int y_old = height_+5 - (int)/*round*/(((data_[0]-min)/dif_min)*5);
			QCanvasLine *ql;
			QBrush br(QColor(black));
			//			ql.setBrush(br);

			for (int i=1; i<(int)data_.size(); i++)
			{
				x_new = 5*(i+1);
				y_new = height_+5 - (int)/*round*/(((data_[i]-min)/dif_min)*5);

				ql = new QCanvasLine(&canvas_);
				ql->setPoints(x_old, y_old, x_new, y_new);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
			//	ql->show();

				x_old = x_new;
				y_old = y_new;
			}
/*
			//add the x-axis
				ql = new QCanvasLine(&canvas_);
				int startx = 5;
				int starty = height_+5 - (int)round(((startx-min)/dif_min)*5);
				int endx   = data_.size()*5;
				int endy   = height_+5 - (int)round(((data_[endx]-min)/dif_min)*5);

				ql->setPoints(startx, starty, endx, endy);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
	*/
			 //QCanvasPolygon* =qp;

			//add the y-axis
		/*		ql = new QCanvasLine(&canvas_);
				startx = 0;
				starty = height_+5 - (int)round(((0-min)/dif_min)*5);
				endx   = data_.size()*5;
				endy   = height_+5 - (int)round(((data_[endx]-min)/dif_min)*5);
				ql->setPoints(startx, starty, endx, endy);
				objects_.push_back(dynamic_cast<QCanvasItem*> (ql));
	*/		}

	}//end of namespace VIEW

}//end of namespace BALL
<|endoftext|>


<commit_before>#pragma once
//=====================================================================//
/*!	@file
	@brief	GUI widget_chip クラス @n
			※「バルーンヘルプ」のようなＧＵＩ
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2018 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/glfw_app/blob/master/LICENSE
*/
//=====================================================================//
#include "widgets/widget_director.hpp"
#include "widgets/widget_utils.hpp"

namespace gui {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief	GUI widget_chip クラス
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	struct widget_chip : public widget {

		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		/*!
			@brief	オフセット・タイプ
		*/
		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		enum class offset_type {
			RIGHT_UP,		///< 右上
			RIGHT_DOWN,		///< 右下
			DOWN,			///< 下
			LEFT_DOWN,		///< 左下
			LEFT_UP,		///< 左上
			UP,				///< 上
			CENTER_UP,		///< 中心の上
		};

		typedef widget_chip value_type;

		typedef std::function< void(uint32_t) > select_func_type;

		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		/*!
			@brief	widget_chip パラメーター
		*/
		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		struct param {
			plate_param		plate_param_;	///< 平面描画パラメーター
			color_param		color_param_;	///< 頂点カラーで変調する場合のパラメーター
			text_param		text_param_;	///< テキスト描画のパラメータ
			uint32_t		id_;			///< セレクト ID （押された回数）

			vtx::ipos		skid_;			///< 隙間
			uint32_t		active_;		///< 有効な時間

			select_func_type	select_func_;	///< セレクト関数

			param(const std::string& text = "") :
				plate_param_(15, 0), color_param_(widget_director::default_chip_color_),
				text_param_(text, img::rgba8(255, 255), img::rgba8(0, 255)),
				id_(0), skid_(10, 10), active_(0),
				select_func_(nullptr)
				{ }
		};

	private:
		widget_director&	wd_;

		param				param_;

		gl::mobj::handle	objh_;

		uint32_t			id_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief	コンストラクター
		*/
		//-----------------------------------------------------------------//
		widget_chip(widget_director& wd, const widget::param& bp, const param& p) :
			widget(bp), wd_(wd), param_(p), objh_(0), id_(0) { }

		//-----------------------------------------------------------------//
		/*!
			@brief	デストラクター
		*/
		//-----------------------------------------------------------------//
		virtual ~widget_chip() { }

		//-----------------------------------------------------------------//
		/*!
			@brief	型を取得
		*/
		//-----------------------------------------------------------------//
		type_id type() const override { return get_type_id<value_type>(); }

		//-----------------------------------------------------------------//
		/*!
			@brief	widget 型の基本名称を取得
			@return widget 型の基本名称
		*/
		//-----------------------------------------------------------------//
		const char* type_name() const override { return "chip"; }

		//-----------------------------------------------------------------//
		/*!
			@brief	ハイブリッド・ウィジェットのサイン
			@return ハイブリッド・ウィジェットの場合「true」を返す。
		*/
		//-----------------------------------------------------------------//
		bool hybrid() const override { return false; }

		//-----------------------------------------------------------------//
		/*!
			@brief	個別パラメーターへの取得(ro)
			@return 個別パラメーター
		*/
		//-----------------------------------------------------------------//
		const param& get_local_param() const { return param_; }

		//-----------------------------------------------------------------//
		/*!
			@brief	個別パラメーターへの取得
			@return 個別パラメーター
		*/
		//-----------------------------------------------------------------//
		param& at_local_param() { return param_; }

		//-----------------------------------------------------------------//
		/*!
			@brief	アクティブの有効無効
			@param[in]	time	アクティブにしている時間 @n
								「０」を指定すると、無効
		*/
		//-----------------------------------------------------------------//
		void active(uint32_t time)
		{
			if(time == 0) {
				set_state(state::ENABLE, false);
			} else {
				set_state(state::ENABLE);
			}
			param_.active_ = time;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を参照
			@param[in]	pos	位置
		*/
		//-----------------------------------------------------------------//
		vtx::ipos& at_org() { return at_param().rect_.org; }

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を設定
			@param[in]	src	ソース・ウィジェット
			@param[in]	ofs	相対位置
		*/
		//-----------------------------------------------------------------//
		void set_offset(widget* src, const vtx::ipos& ofs)
		{
			if(src == nullptr) return;

			auto pos = src->get_rect().org;
			widget* w = src->get_param().parents_;
			while(w != nullptr && w != get_param().parents_) {
				pos += w->get_rect().org;
				w = w->get_param().parents_;
			}
			pos += ofs;
			at_org() = pos;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を設定
			@param[in]	src	ソース・ウィジェット
			@param[in]	ot	オフセット・タイプ
		*/
		//-----------------------------------------------------------------//
		void set_offset(widget* src, offset_type ot = offset_type::RIGHT_UP)
		{
			vtx::ipos ofs(0);
			switch(ot) {
			case offset_type::RIGHT_UP:
				ofs.x = src->get_rect().size.x;
				ofs.y = -get_rect().size.y;
				break;
			case offset_type::RIGHT_DOWN:
				ofs.x = src->get_rect().size.x;
				ofs.y = src->get_rect().size.y;
				break;
			case offset_type::CENTER_UP:
				ofs.x = src->get_rect().size.x / 2;
				ofs.y = -get_rect().size.y;
				break;
			case offset_type::DOWN:
			case offset_type::LEFT_DOWN:
			case offset_type::LEFT_UP:
			case offset_type::UP:
			default:
				break;
			}
			set_offset(src, ofs);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	テキストを設定
			@param[in]	text	テキスト
			@param[in]	asz		サイズを自動設定しない場合「false」
		*/
		//-----------------------------------------------------------------//
		void set_text(const std::string& text, bool asz = true) {
			param_.text_param_.set_text(text);
			if(asz) {
				auto sz = get_text_size(param_.text_param_);
				at_param().rect_.size = sz + (param_.skid_ * 2);
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	初期化
		*/
		//-----------------------------------------------------------------//
		void initialize() override
		{
			// サイズ固定
			at_param().state_.set(widget::state::SERVICE);
			at_param().state_.set(widget::state::POSITION_LOCK);
			at_param().state_.set(widget::state::SIZE_LOCK);
			at_param().state_.set(widget::state::MOVE_STALL);
			at_param().action_.set(widget::action::SELECT_SCALE, false);

			using namespace img;
//			param_.text_param_.placement_ =
			param_.plate_param_.resizeble_ = true;
			vtx::spos size;
			if(param_.plate_param_.resizeble_) {
				vtx::spos rsz = param_.plate_param_.grid_ * 3;
				if(get_param().rect_.size.x >= rsz.x) size.x = rsz.x;
				else size.x = get_param().rect_.size.x;
				if(get_param().rect_.size.y >= rsz.y) size.y = rsz.y;
				else size.y = get_param().rect_.size.y;
			} else {
				size = get_param().rect_.size;
			}
			share_t t;
			t.size_ = size;
			t.color_param_ = param_.color_param_;
			t.plate_param_ = param_.plate_param_;
			objh_ = wd_.share_add(t);

			active(0);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	アップデート
		*/
		//-----------------------------------------------------------------//
		void update() override
		{
			if(get_selected()) {
				++param_.id_;
			}
			if(param_.active_ > 0) {
				--param_.active_;
				if(param_.active_ == 0) {
					active(0);
				}
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	サービス
		*/
		//-----------------------------------------------------------------//
		void service() override
		{
			if(!get_state(state::ENABLE)) {
				return;
			}

			if(id_ != param_.id_) {
				if(param_.select_func_ != nullptr) {
					param_.select_func_(param_.id_);
				}
				id_ = param_.id_;
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	レンダリング
		*/
		//-----------------------------------------------------------------//
		void render() override
		{
			if(objh_ == 0) return;

			if(param_.plate_param_.resizeble_) {
				wd_.at_mobj().resize(objh_, get_param().rect_.size);
			}

			render_text(wd_, objh_, get_param(), param_.text_param_, param_.plate_param_);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	状態のセーブ
			@param[in]	pre	プリファレンス参照
			@return エラーが無い場合「true」
		*/
		//-----------------------------------------------------------------//
		bool save(sys::preference& pre) override {
			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	状態のロード
			@param[in]	pre	プリファレンス参照
			@return エラーが無い場合「true」
		*/
		//-----------------------------------------------------------------//
		bool load(const sys::preference& pre) override {
			return true;
		}
	};
}
<commit_msg>update: turn enable state for active time manage<commit_after>#pragma once
//=====================================================================//
/*!	@file
	@brief	GUI widget_chip クラス @n
			※「バルーンヘルプ」のようなＧＵＩ
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2018 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/glfw_app/blob/master/LICENSE
*/
//=====================================================================//
#include "widgets/widget_director.hpp"
#include "widgets/widget_utils.hpp"

namespace gui {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief	GUI widget_chip クラス
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	struct widget_chip : public widget {

		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		/*!
			@brief	オフセット・タイプ
		*/
		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		enum class offset_type {
			RIGHT_UP,		///< 右上
			RIGHT_DOWN,		///< 右下
			DOWN,			///< 下
			LEFT_DOWN,		///< 左下
			LEFT_UP,		///< 左上
			UP,				///< 上
			CENTER_UP,		///< 中心の上
		};

		typedef widget_chip value_type;

		typedef std::function< void(uint32_t) > select_func_type;

		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		/*!
			@brief	widget_chip パラメーター
		*/
		//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
		struct param {
			plate_param		plate_param_;	///< 平面描画パラメーター
			color_param		color_param_;	///< 頂点カラーで変調する場合のパラメーター
			text_param		text_param_;	///< テキスト描画のパラメータ
			uint32_t		id_;			///< セレクト ID （押された回数）

			vtx::ipos		skid_;			///< 隙間
			uint32_t		active_;		///< 有効な時間

			select_func_type	select_func_;	///< セレクト関数

			param(const std::string& text = "") :
				plate_param_(15, 0), color_param_(widget_director::default_chip_color_),
				text_param_(text, img::rgba8(255, 255), img::rgba8(0, 255)),
				id_(0), skid_(10, 10), active_(0),
				select_func_(nullptr)
				{ }
		};

	private:
		widget_director&	wd_;

		param				param_;

		gl::mobj::handle	objh_;

		uint32_t			id_;

		bool				ena_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief	コンストラクター
		*/
		//-----------------------------------------------------------------//
		widget_chip(widget_director& wd, const widget::param& bp, const param& p) :
			widget(bp), wd_(wd), param_(p), objh_(0), id_(0), ena_(false) { }

		//-----------------------------------------------------------------//
		/*!
			@brief	デストラクター
		*/
		//-----------------------------------------------------------------//
		virtual ~widget_chip() { }

		//-----------------------------------------------------------------//
		/*!
			@brief	型を取得
		*/
		//-----------------------------------------------------------------//
		type_id type() const override { return get_type_id<value_type>(); }

		//-----------------------------------------------------------------//
		/*!
			@brief	widget 型の基本名称を取得
			@return widget 型の基本名称
		*/
		//-----------------------------------------------------------------//
		const char* type_name() const override { return "chip"; }

		//-----------------------------------------------------------------//
		/*!
			@brief	ハイブリッド・ウィジェットのサイン
			@return ハイブリッド・ウィジェットの場合「true」を返す。
		*/
		//-----------------------------------------------------------------//
		bool hybrid() const override { return false; }

		//-----------------------------------------------------------------//
		/*!
			@brief	個別パラメーターへの取得(ro)
			@return 個別パラメーター
		*/
		//-----------------------------------------------------------------//
		const param& get_local_param() const { return param_; }

		//-----------------------------------------------------------------//
		/*!
			@brief	個別パラメーターへの取得
			@return 個別パラメーター
		*/
		//-----------------------------------------------------------------//
		param& at_local_param() { return param_; }

		//-----------------------------------------------------------------//
		/*!
			@brief	アクティブの有効無効
			@param[in]	time	アクティブにしている時間 @n
								「０」を指定すると、無効
		*/
		//-----------------------------------------------------------------//
		void active(uint32_t time)
		{
			if(time == 0) {
				set_state(state::ENABLE, false);
			} else {
				set_state(state::ENABLE);
			}
			param_.active_ = time;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を参照
			@param[in]	pos	位置
		*/
		//-----------------------------------------------------------------//
		vtx::ipos& at_org() { return at_param().rect_.org; }

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を設定
			@param[in]	src	ソース・ウィジェット
			@param[in]	ofs	相対位置
		*/
		//-----------------------------------------------------------------//
		void set_offset(widget* src, const vtx::ipos& ofs)
		{
			if(src == nullptr) return;

			auto pos = src->get_rect().org;
			widget* w = src->get_param().parents_;
			while(w != nullptr && w != get_param().parents_) {
				pos += w->get_rect().org;
				w = w->get_param().parents_;
			}
			pos += ofs;
			at_org() = pos;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	位置を設定
			@param[in]	src	ソース・ウィジェット
			@param[in]	ot	オフセット・タイプ
		*/
		//-----------------------------------------------------------------//
		void set_offset(widget* src, offset_type ot = offset_type::RIGHT_UP)
		{
			vtx::ipos ofs(0);
			switch(ot) {
			case offset_type::RIGHT_UP:
				ofs.x = src->get_rect().size.x;
				ofs.y = -get_rect().size.y;
				break;
			case offset_type::RIGHT_DOWN:
				ofs.x = src->get_rect().size.x;
				ofs.y = src->get_rect().size.y;
				break;
			case offset_type::CENTER_UP:
				ofs.x = src->get_rect().size.x / 2;
				ofs.y = -get_rect().size.y;
				break;
			case offset_type::DOWN:
			case offset_type::LEFT_DOWN:
			case offset_type::LEFT_UP:
			case offset_type::UP:
			default:
				break;
			}
			set_offset(src, ofs);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	テキストを設定
			@param[in]	text	テキスト
			@param[in]	asz		サイズを自動設定しない場合「false」
		*/
		//-----------------------------------------------------------------//
		void set_text(const std::string& text, bool asz = true) {
			param_.text_param_.set_text(text);
			if(asz) {
				auto sz = get_text_size(param_.text_param_);
				at_param().rect_.size = sz + (param_.skid_ * 2);
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	初期化
		*/
		//-----------------------------------------------------------------//
		void initialize() override
		{
			// サイズ固定
			at_param().state_.set(widget::state::SERVICE);
			at_param().state_.set(widget::state::POSITION_LOCK);
			at_param().state_.set(widget::state::SIZE_LOCK);
			at_param().state_.set(widget::state::MOVE_STALL);
			at_param().action_.set(widget::action::SELECT_SCALE, false);

			using namespace img;
//			param_.text_param_.placement_ =
			param_.plate_param_.resizeble_ = true;
			vtx::spos size;
			if(param_.plate_param_.resizeble_) {
				vtx::spos rsz = param_.plate_param_.grid_ * 3;
				if(get_param().rect_.size.x >= rsz.x) size.x = rsz.x;
				else size.x = get_param().rect_.size.x;
				if(get_param().rect_.size.y >= rsz.y) size.y = rsz.y;
				else size.y = get_param().rect_.size.y;
			} else {
				size = get_param().rect_.size;
			}
			share_t t;
			t.size_ = size;
			t.color_param_ = param_.color_param_;
			t.plate_param_ = param_.plate_param_;
			objh_ = wd_.share_add(t);

			active(0);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	アップデート
		*/
		//-----------------------------------------------------------------//
		void update() override
		{
			if(!ena_ && get_enable()) {
				if(param_.active_ == 0) {
					set_state(state::ENABLE, false);
					return;
				}
			}
			ena_ = get_enable();

			if(get_selected()) {
				++param_.id_;
			}
			if(param_.active_ > 0) {
				--param_.active_;
				if(param_.active_ == 0) {
					set_state(state::ENABLE, false);
				}
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	サービス
		*/
		//-----------------------------------------------------------------//
		void service() override
		{
			if(!get_state(state::ENABLE)) {
				return;
			}

			if(id_ != param_.id_) {
				if(param_.select_func_ != nullptr) {
					param_.select_func_(param_.id_);
				}
				id_ = param_.id_;
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	レンダリング
		*/
		//-----------------------------------------------------------------//
		void render() override
		{
			if(objh_ == 0) return;

			if(param_.plate_param_.resizeble_) {
				wd_.at_mobj().resize(objh_, get_param().rect_.size);
			}

			render_text(wd_, objh_, get_param(), param_.text_param_, param_.plate_param_);
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	状態のセーブ
			@param[in]	pre	プリファレンス参照
			@return エラーが無い場合「true」
		*/
		//-----------------------------------------------------------------//
		bool save(sys::preference& pre) override {
			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief	状態のロード
			@param[in]	pre	プリファレンス参照
			@return エラーが無い場合「true」
		*/
		//-----------------------------------------------------------------//
		bool load(const sys::preference& pre) override {
			return true;
		}
	};
}
<|endoftext|>


<commit_before><commit_msg>Refactor of the example: reduce a list of included headers<commit_after><|endoftext|>


<commit_before>#include "vast/partition.h"

#include <caf/all.hpp>
#include "vast/event.h"
#include "vast/indexer.h"
#include "vast/task_tree.h"
#include "vast/io/serialization.h"
#include "vast/source/dechunkifier.h"

using namespace caf;

namespace vast {

struct partition::dispatcher
{
  dispatcher(partition& pa)
    : part_{pa}
  {
  }

  template <typename T>
  std::vector<actor> operator()(T const&)
  {
    return {};
  }

  template <typename T, typename U>
  std::vector<actor> operator()(T const&, U const&)
  {
    return {};
  }

  std::vector<actor> operator()(predicate const& p)
  {
    op_ = p.op;
    return visit(*this, p.lhs, p.rhs);

  }

  std::vector<actor> operator()(event_extractor const&, data const&)
  {
    return {part_.load_name_indexer()};
  }

  std::vector<actor> operator()(time_extractor const&, data const&)
  {
    return {part_.load_time_indexer()};
  }

  std::vector<actor> operator()(type_extractor const& e, data const&)
  {
    std::vector<actor> indexes;
    for (auto& t : part_.schema_)
      if (auto r = get<type::record>(t))
      {
        auto attempt = r->each(
            [&](type::record::trace const& tr, offset const& o) -> trial<void>
            {
              if (tr.back()->type == e.type)
              {
                auto a = part_.load_data_indexer(t, e.type, o);
                if (! a)
                  return a.error();
                if (*a)
                  indexes.push_back(std::move(*a));
              }

              return nothing;
            });

        if (! attempt)
        {
          VAST_LOG_ERROR(attempt.error());
          return {};
        }
      }
      else
      {
        auto a = part_.load_data_indexer(t, t, {});
        if (! a)
        {
          VAST_LOG_ERROR(a.error());
          return {};
        }
        if (*a)
          indexes.push_back(std::move(*a));
      }

    return indexes;
  }

  std::vector<actor> operator()(schema_extractor const& e, data const& d)
  {
    std::vector<actor> indexes;
    for (auto& t : part_.schema_)
      if (auto r = get<type::record>(t))
      {
        for (auto& pair : r->find_suffix(e.key))
        {
          auto& o = pair.first;
          auto lhs_type = r->at(o);
          assert(lhs_type);

          if (! compatible(*lhs_type, op_, type::derive(d)))
          {
            VAST_LOG_WARN("incompatible types: LHS = " << *lhs_type <<
                          " <--> RHS = " << type::derive(d));
            return {};
          }

          auto a = part_.load_data_indexer(t, *lhs_type, o);
          if (! a)
            VAST_LOG_ERROR(a.error());
          else if (*a)
            indexes.push_back(std::move(*a));
        }
      }
      else if (e.key.size() == 1 && pattern::glob(e.key[0]).match(t.name()))
      {
        auto a = part_.load_data_indexer(t, t, {});
        if (! a)
        {
          VAST_LOG_ERROR(a.error());
          return {};
        }

        if (*a)
          indexes.push_back(std::move(*a));;
      }

    return indexes;
  }

  template <typename T>
  std::vector<actor> operator()(data const& d, T const& e)
  {
    return (*this)(e, d);
  }

  relational_operator op_;
  partition& part_;
};

partition::partition(path const& index_dir, uuid id, size_t batch_size)
  : dir_{index_dir / to_string(id)},
    id_{std::move(id)},
    batch_size_{batch_size}
{
}

void partition::at_down(down_msg const&)
{
  if (last_sender() == dechunkifier_)
  {
    dechunkifier_ = invalid_actor;
    chunks_.pop();

    if (chunks_.size() == 10 - 1)
    {
      VAST_LOG_ACTOR_DEBUG("signals underload");
      for (auto& a : upstream())
        send(a, message_priority::high, flow_control::underload{});
    }

    send(this, atom("unpack"));
  }
  else
  {
    VAST_LOG_ACTOR_DEBUG("got DOWN from " << last_sender());

    stats_.erase(last_sender());
    for (auto i = indexers_.begin(); i != indexers_.end(); ++i)
      if (i->second == last_sender())
      {
        indexers_.erase(i);
        break;
      }
  }
}

void partition::at_exit(exit_msg const& msg)
{
  if (msg.reason == exit::kill)
  {
    quit(msg.reason);
  }
  else if (exit_reason_ == 0 && ! chunks_.empty())
  {
    exit_reason_ = msg.reason; // Wait for unpacker to finish.
  }
  else
  {
    auto tree = spawn<task_tree>(this);
    send(tree, atom("notify"), this);
    send(tree, this, this);
    send(this, atom("flush"), tree);
    exit_reason_ = msg.reason;
  }
}

message_handler partition::make_handler()
{
  trap_exit(true);

  if (exists(dir_))
  {
    auto t = io::unarchive(dir_ / "schema", schema_);
    if (! t)
    {
      VAST_LOG_ACTOR_ERROR("failed to load schema: " << t.error());
      quit(exit::error);
      return {};
    }
  }

  attach_functor(
      [=](uint32_t reason)
      {
        if (dechunkifier_)
          anon_send_exit(dechunkifier_, reason);
        dechunkifier_ = invalid_actor;

        for (auto& p : indexers_)
          anon_send_exit(p.second, reason);
        indexers_.clear();
      });

  send(this, atom("stats"), atom("show"));

  return
  {
    on(atom("done")) >> [=]
    {
      // We only spawn one task tree upon exiting. Once we get notified we can
      // safely terminate with the last exit reason.
      quit(exit_reason_);
    },
    [=](expression const& pred, actor idx)
    {
      VAST_LOG_ACTOR_DEBUG("got predicate " << pred);

      auto indexers = visit(dispatcher{*this}, pred);
      uint64_t n = indexers.size();
      send(idx, pred, id_, n);

      if (n == 0)
      {
        VAST_LOG_ACTOR_DEBUG("did not find a matching indexer for " << pred);
        send(idx, pred, id_, bitstream{});
      }
      else
      {
        auto t = make_message(pred, id_, idx);
        for (auto& a : indexers)
          send_tuple(a, t);
      }
    },
    on(atom("flush"), arg_match) >> [=](actor tree)
    {
      VAST_LOG_ACTOR_DEBUG("got request to flush indexes");

      for (auto& p : indexers_)
        if (p.second)
        {
          send(tree, this, p.second);
          send(p.second, atom("flush"), tree);
        }

      if (! schema_.empty())
      {
        auto t = io::archive(dir_ / "schema", schema_);
        if (! t)
        {
          VAST_LOG_ACTOR_ERROR("failed to save schema: " << t.error());
          quit(exit::error);
          return;
        }
      }

      send(tree, atom("done"));
    },
    [=](chunk const& c)
    {
      VAST_LOG_ACTOR_DEBUG("got chunk with " << c.events() << " events");

      auto sch = schema::merge(schema_, c.meta().schema);
      if (! sch)
      {
        VAST_LOG_ACTOR_ERROR("failed to merge schema: " << sch.error());
        quit(exit::error);
        return;
      }
      else if (indexers_.empty() || *sch != schema_)
      {
        schema_ = std::move(*sch);

        load_time_indexer();
        load_name_indexer();

        for (auto& tp : c.meta().schema)
          // FIXME: Adjust after having switched to the new record indexer.
          if (auto r = get<type::record>(tp))
          {
            auto attempt = r->each(
                [&](type::record::trace const& t, offset const& o) -> trial<void>
                {
                  if (t.back()->type.find_attribute(type::attribute::skip))
                    return nothing;

                  auto a = create_data_indexer(tp, t.back()->type, o);
                  if (! a)
                    return a.error();
                  return nothing;
                });

            if (! attempt)
            {
              VAST_LOG_ACTOR_ERROR(attempt.error());
              quit(exit::error);
              return;
            }
          }
          else if (! tp.find_attribute(type::attribute::skip))
          {
            auto t = create_data_indexer(tp, tp, {});
            if (! t)
            {
              VAST_LOG_ACTOR_ERROR(t.error());
              quit(exit::error);
              return;
            }
          }
      }

      chunks_.push(c);
      send(this, atom("unpack"));

      if (chunks_.size() == 10)
      {
        VAST_LOG_ACTOR_DEBUG("signals overload");
        for (auto& a : upstream())
          send(a, message_priority::high, flow_control::overload{});
      }
    },
    [=](std::vector<event> const& events)
    {
      for (auto& p : indexers_)
        send_tuple(p.second, last_dequeued());

      for (auto& p : stats_)
        p.second.backlog += events.size();
    },
    on(atom("unpack")) >> [=]
    {
      if (! dechunkifier_)
      {
        if (! chunks_.empty())
        {
          VAST_LOG_ACTOR_DEBUG("begins unpacking a chunk (" <<
                               chunks_.size() << " remaining)");

          dechunkifier_ =
            spawn<source::dechunkifier, monitored>(chunks_.front());

          send(dechunkifier_, atom("sink"), this);
          send(dechunkifier_, atom("batch size"), batch_size_);
          send(dechunkifier_, atom("run"));
        }
        else if (exit_reason_ != 0)
        {
          send_exit(this, exit_reason_);
        }
      }
    },
    [=](uint64_t processed, uint64_t indexed, uint64_t rate, uint64_t mean)
    {
      auto i = stats_.find(last_sender());
      assert(i != stats_.end());
      auto& s = i->second;

      s.backlog -= processed;
      s.value_total += indexed;
      s.value_rate = rate;
      s.value_rate_mean = mean;

      updated_ = true;
    },
    on(atom("stats"), atom("show")) >> [=]
    {
      delayed_send_tuple(this, std::chrono::seconds(3), last_dequeued());

      if (updated_)
        updated_ = false;
      else
        return;

      std::pair<uint64_t, actor> max_backlog = {0, invalid_actor};
      uint64_t value_total = 0;
      uint64_t value_rate = 0;
      uint64_t value_rate_mean = 0;
      uint64_t event_rate_min = -1;
      uint64_t event_rate_max = 0;

      auto n = 0;
      for (auto& p : indexers_)
      {
        ++n;

        auto& s = stats_[p.second.address()];
        if (s.backlog > max_backlog.first)
          max_backlog = {s.backlog, p.second};

        if (s.value_rate < event_rate_min)
          event_rate_min = s.value_rate;
        if (s.value_rate > event_rate_max)
          event_rate_max = s.value_rate;

        value_total += s.value_total;
        value_rate += s.value_rate;
        value_rate_mean += s.value_rate_mean;
      }

      if (value_rate > 0 || max_backlog.first > 0)
        VAST_LOG_ACTOR_VERBOSE(
            "indexes at " << value_rate << " values/sec" <<
            " (mean " << value_rate_mean << ") and " <<
            (value_rate / n) << " events/sec" <<
            " (" << event_rate_min << '/' << event_rate_max << '/' <<
            (value_rate_mean / n) << " min/max/mean) with max backlog of " <<
            max_backlog.first << " at " << max_backlog.second);
    }
  };
}

std::string partition::name() const
{
  return "partition";
}

actor partition::load_time_indexer()
{
  auto p = dir_ / "meta" / "time" / "index";
  auto& s = indexers_[p];
  if (! s)
  {
    s = spawn<event_time_indexer<default_bitstream>>(std::move(p));
    monitor(s);
    stats_[s.address()];
  }

  return s;
}

actor partition::load_name_indexer()
{
  auto p = dir_ / "meta" / "name" / "index";
  auto& s = indexers_[p];
  if (! s)
  {
    s = spawn<event_name_indexer<default_bitstream>>(std::move(p));
    monitor(s);
    stats_[s.address()];
  }

  return s;
}

trial<actor> partition::load_data_indexer(
    type const& et, type const& t, offset const& o)
{
  auto i = indexers_.find(dir_ / path{"types"} / et.name());
  if (i != indexers_.end())
  {
    assert(i->second);
    return i->second;
  }

  return create_data_indexer(et, t, o);
}

trial<actor> partition::create_data_indexer(
    type const& et, type const& t, offset const& o)
{
  auto abs = dir_ / path{"types"} / et.name();

  // FIXME: Remove after having switched to the new record indexer.
  auto r = get<type::record>(et);
  if (r)
  {
    auto fs = r->resolve(o);
    assert(fs);
    for (auto& f : *fs)
      abs /= f;
  }

  abs /= "index";

  auto& s = indexers_[abs];
  if (! s)
  {
    auto a = make_event_data_indexer<default_bitstream>(abs, et, t, o);
    if (! a)
      return a;

    s = *a;
    monitor(s);
    stats_[s.address()];
  }

  return s;
}
} // namespace vast
<commit_msg>Fix argument permutation bug.<commit_after>#include "vast/partition.h"

#include <caf/all.hpp>
#include "vast/event.h"
#include "vast/indexer.h"
#include "vast/task_tree.h"
#include "vast/io/serialization.h"
#include "vast/source/dechunkifier.h"

using namespace caf;

namespace vast {

struct partition::dispatcher
{
  dispatcher(partition& pa)
    : part_{pa}
  {
  }

  template <typename T>
  std::vector<actor> operator()(T const&)
  {
    return {};
  }

  template <typename T, typename U>
  std::vector<actor> operator()(T const&, U const&)
  {
    return {};
  }

  std::vector<actor> operator()(predicate const& p)
  {
    op_ = p.op;
    return visit(*this, p.lhs, p.rhs);

  }

  std::vector<actor> operator()(event_extractor const&, data const&)
  {
    return {part_.load_name_indexer()};
  }

  std::vector<actor> operator()(time_extractor const&, data const&)
  {
    return {part_.load_time_indexer()};
  }

  std::vector<actor> operator()(type_extractor const& e, data const&)
  {
    std::vector<actor> indexes;
    for (auto& t : part_.schema_)
      if (auto r = get<type::record>(t))
      {
        auto attempt = r->each(
            [&](type::record::trace const& tr, offset const& o) -> trial<void>
            {
              if (tr.back()->type == e.type)
              {
                auto a = part_.load_data_indexer(t, e.type, o);
                if (! a)
                  return a.error();
                if (*a)
                  indexes.push_back(std::move(*a));
              }

              return nothing;
            });

        if (! attempt)
        {
          VAST_LOG_ERROR(attempt.error());
          return {};
        }
      }
      else
      {
        auto a = part_.load_data_indexer(t, t, {});
        if (! a)
        {
          VAST_LOG_ERROR(a.error());
          return {};
        }
        if (*a)
          indexes.push_back(std::move(*a));
      }

    return indexes;
  }

  std::vector<actor> operator()(schema_extractor const& e, data const& d)
  {
    std::vector<actor> indexes;
    for (auto& t : part_.schema_)
      if (auto r = get<type::record>(t))
      {
        for (auto& pair : r->find_suffix(e.key))
        {
          auto& o = pair.first;
          auto lhs_type = r->at(o);
          assert(lhs_type);

          if (! compatible(*lhs_type, op_, type::derive(d)))
          {
            VAST_LOG_WARN("incompatible types: LHS = " << *lhs_type <<
                          " <--> RHS = " << type::derive(d));
            return {};
          }

          auto a = part_.load_data_indexer(t, *lhs_type, o);
          if (! a)
            VAST_LOG_ERROR(a.error());
          else if (*a)
            indexes.push_back(std::move(*a));
        }
      }
      else if (e.key.size() == 1 && pattern::glob(e.key[0]).match(t.name()))
      {
        auto a = part_.load_data_indexer(t, t, {});
        if (! a)
        {
          VAST_LOG_ERROR(a.error());
          return {};
        }

        if (*a)
          indexes.push_back(std::move(*a));;
      }

    return indexes;
  }

  template <typename T>
  std::vector<actor> operator()(data const& d, T const& e)
  {
    return (*this)(e, d);
  }

  relational_operator op_;
  partition& part_;
};

partition::partition(path const& index_dir, uuid id, size_t batch_size)
  : dir_{index_dir / to_string(id)},
    id_{std::move(id)},
    batch_size_{batch_size}
{
}

void partition::at_down(down_msg const&)
{
  if (last_sender() == dechunkifier_)
  {
    dechunkifier_ = invalid_actor;
    chunks_.pop();

    if (chunks_.size() == 10 - 1)
    {
      VAST_LOG_ACTOR_DEBUG("signals underload");
      for (auto& a : upstream())
        send(message_priority::high, a, flow_control::underload{});
    }

    send(this, atom("unpack"));
  }
  else
  {
    VAST_LOG_ACTOR_DEBUG("got DOWN from " << last_sender());

    stats_.erase(last_sender());
    for (auto i = indexers_.begin(); i != indexers_.end(); ++i)
      if (i->second == last_sender())
      {
        indexers_.erase(i);
        break;
      }
  }
}

void partition::at_exit(exit_msg const& msg)
{
  if (msg.reason == exit::kill)
  {
    quit(msg.reason);
  }
  else if (exit_reason_ == 0 && ! chunks_.empty())
  {
    exit_reason_ = msg.reason; // Wait for unpacker to finish.
  }
  else
  {
    auto tree = spawn<task_tree>(this);
    send(tree, atom("notify"), this);
    send(tree, this, this);
    send(this, atom("flush"), tree);
    exit_reason_ = msg.reason;
  }
}

message_handler partition::make_handler()
{
  trap_exit(true);

  if (exists(dir_))
  {
    auto t = io::unarchive(dir_ / "schema", schema_);
    if (! t)
    {
      VAST_LOG_ACTOR_ERROR("failed to load schema: " << t.error());
      quit(exit::error);
      return {};
    }
  }

  attach_functor(
      [=](uint32_t reason)
      {
        if (dechunkifier_)
          anon_send_exit(dechunkifier_, reason);
        dechunkifier_ = invalid_actor;

        for (auto& p : indexers_)
          anon_send_exit(p.second, reason);
        indexers_.clear();
      });

  send(this, atom("stats"), atom("show"));

  return
  {
    on(atom("done")) >> [=]
    {
      // We only spawn one task tree upon exiting. Once we get notified we can
      // safely terminate with the last exit reason.
      quit(exit_reason_);
    },
    [=](expression const& pred, actor idx)
    {
      VAST_LOG_ACTOR_DEBUG("got predicate " << pred);

      auto indexers = visit(dispatcher{*this}, pred);
      uint64_t n = indexers.size();
      send(idx, pred, id_, n);

      if (n == 0)
      {
        VAST_LOG_ACTOR_DEBUG("did not find a matching indexer for " << pred);
        send(idx, pred, id_, bitstream{});
      }
      else
      {
        auto t = make_message(pred, id_, idx);
        for (auto& a : indexers)
          send_tuple(a, t);
      }
    },
    on(atom("flush"), arg_match) >> [=](actor tree)
    {
      VAST_LOG_ACTOR_DEBUG("got request to flush indexes");

      for (auto& p : indexers_)
        if (p.second)
        {
          send(tree, this, p.second);
          send(p.second, atom("flush"), tree);
        }

      if (! schema_.empty())
      {
        auto t = io::archive(dir_ / "schema", schema_);
        if (! t)
        {
          VAST_LOG_ACTOR_ERROR("failed to save schema: " << t.error());
          quit(exit::error);
          return;
        }
      }

      send(tree, atom("done"));
    },
    [=](chunk const& c)
    {
      VAST_LOG_ACTOR_DEBUG("got chunk with " << c.events() << " events");

      auto sch = schema::merge(schema_, c.meta().schema);
      if (! sch)
      {
        VAST_LOG_ACTOR_ERROR("failed to merge schema: " << sch.error());
        quit(exit::error);
        return;
      }
      else if (indexers_.empty() || *sch != schema_)
      {
        schema_ = std::move(*sch);

        load_time_indexer();
        load_name_indexer();

        for (auto& tp : c.meta().schema)
          // FIXME: Adjust after having switched to the new record indexer.
          if (auto r = get<type::record>(tp))
          {
            auto attempt = r->each(
                [&](type::record::trace const& t, offset const& o) -> trial<void>
                {
                  if (t.back()->type.find_attribute(type::attribute::skip))
                    return nothing;

                  auto a = create_data_indexer(tp, t.back()->type, o);
                  if (! a)
                    return a.error();
                  return nothing;
                });

            if (! attempt)
            {
              VAST_LOG_ACTOR_ERROR(attempt.error());
              quit(exit::error);
              return;
            }
          }
          else if (! tp.find_attribute(type::attribute::skip))
          {
            auto t = create_data_indexer(tp, tp, {});
            if (! t)
            {
              VAST_LOG_ACTOR_ERROR(t.error());
              quit(exit::error);
              return;
            }
          }
      }

      chunks_.push(c);
      send(this, atom("unpack"));

      if (chunks_.size() == 10)
      {
        VAST_LOG_ACTOR_DEBUG("signals overload");
        for (auto& a : upstream())
          send(message_priority::high, a, flow_control::overload{});
      }
    },
    [=](std::vector<event> const& events)
    {
      for (auto& p : indexers_)
        send_tuple(p.second, last_dequeued());

      for (auto& p : stats_)
        p.second.backlog += events.size();
    },
    on(atom("unpack")) >> [=]
    {
      if (! dechunkifier_)
      {
        if (! chunks_.empty())
        {
          VAST_LOG_ACTOR_DEBUG("begins unpacking a chunk (" <<
                               chunks_.size() << " remaining)");

          dechunkifier_ =
            spawn<source::dechunkifier, monitored>(chunks_.front());

          send(dechunkifier_, atom("sink"), this);
          send(dechunkifier_, atom("batch size"), batch_size_);
          send(dechunkifier_, atom("run"));
        }
        else if (exit_reason_ != 0)
        {
          send_exit(this, exit_reason_);
        }
      }
    },
    [=](uint64_t processed, uint64_t indexed, uint64_t rate, uint64_t mean)
    {
      auto i = stats_.find(last_sender());
      assert(i != stats_.end());
      auto& s = i->second;

      s.backlog -= processed;
      s.value_total += indexed;
      s.value_rate = rate;
      s.value_rate_mean = mean;

      updated_ = true;
    },
    on(atom("stats"), atom("show")) >> [=]
    {
      delayed_send_tuple(this, std::chrono::seconds(3), last_dequeued());

      if (updated_)
        updated_ = false;
      else
        return;

      std::pair<uint64_t, actor> max_backlog = {0, invalid_actor};
      uint64_t value_total = 0;
      uint64_t value_rate = 0;
      uint64_t value_rate_mean = 0;
      uint64_t event_rate_min = -1;
      uint64_t event_rate_max = 0;

      auto n = 0;
      for (auto& p : indexers_)
      {
        ++n;

        auto& s = stats_[p.second.address()];
        if (s.backlog > max_backlog.first)
          max_backlog = {s.backlog, p.second};

        if (s.value_rate < event_rate_min)
          event_rate_min = s.value_rate;
        if (s.value_rate > event_rate_max)
          event_rate_max = s.value_rate;

        value_total += s.value_total;
        value_rate += s.value_rate;
        value_rate_mean += s.value_rate_mean;
      }

      if (value_rate > 0 || max_backlog.first > 0)
        VAST_LOG_ACTOR_VERBOSE(
            "indexes at " << value_rate << " values/sec" <<
            " (mean " << value_rate_mean << ") and " <<
            (value_rate / n) << " events/sec" <<
            " (" << event_rate_min << '/' << event_rate_max << '/' <<
            (value_rate_mean / n) << " min/max/mean) with max backlog of " <<
            max_backlog.first << " at " << max_backlog.second);
    }
  };
}

std::string partition::name() const
{
  return "partition";
}

actor partition::load_time_indexer()
{
  auto p = dir_ / "meta" / "time" / "index";
  auto& s = indexers_[p];
  if (! s)
  {
    s = spawn<event_time_indexer<default_bitstream>>(std::move(p));
    monitor(s);
    stats_[s.address()];
  }

  return s;
}

actor partition::load_name_indexer()
{
  auto p = dir_ / "meta" / "name" / "index";
  auto& s = indexers_[p];
  if (! s)
  {
    s = spawn<event_name_indexer<default_bitstream>>(std::move(p));
    monitor(s);
    stats_[s.address()];
  }

  return s;
}

trial<actor> partition::load_data_indexer(
    type const& et, type const& t, offset const& o)
{
  auto i = indexers_.find(dir_ / path{"types"} / et.name());
  if (i != indexers_.end())
  {
    assert(i->second);
    return i->second;
  }

  return create_data_indexer(et, t, o);
}

trial<actor> partition::create_data_indexer(
    type const& et, type const& t, offset const& o)
{
  auto abs = dir_ / path{"types"} / et.name();

  // FIXME: Remove after having switched to the new record indexer.
  auto r = get<type::record>(et);
  if (r)
  {
    auto fs = r->resolve(o);
    assert(fs);
    for (auto& f : *fs)
      abs /= f;
  }

  abs /= "index";

  auto& s = indexers_[abs];
  if (! s)
  {
    auto a = make_event_data_indexer<default_bitstream>(abs, et, t, o);
    if (! a)
      return a;

    s = *a;
    monitor(s);
    stats_[s.address()];
  }

  return s;
}
} // namespace vast
<|endoftext|>


<commit_before>/*
 *  AdIndexManager.cpp
 */

#include "AdIndexManager.h"
#include "AdStreamSubscriber.h"
#include "AdSelector.h"
#include "AdClickPredictor.h"
#include "AdFeedbackMgr.h"
#include "AdSearchService.h"
#include "sponsored-ad-search/AdSponsoredMgr.h"
#include <common/ResultType.h>
#include <mining-manager/group-manager/GroupManager.h>
#include <query-manager/ActionItem.h>
#include <query-manager/SearchKeywordOperation.h>
#include <search-manager/SearchBase.h>
#include <search-manager/HitQueue.h>
#include <util/ustring/UString.h>
#include <algorithm>

namespace sf1r
{

static const int MAX_SEARCH_AD_COUNT = 20000;
static const int MAX_SELECT_AD_COUNT = 20;
static const int MAX_RECOMMEND_ITEM_NUM = 10;
static const std::string adlog_topic = "b5manlog";

using namespace sponsored;

AdIndexManager::AdIndexManager(
        const std::string& ad_resource_path,
        const std::string& ad_data_path,
        bool enable_ad_selector,
        bool enable_ad_rec,
        bool enable_ad_sponsored_search,
        const std::string& dmp_ip,
        uint16_t dmp_port,
        const std::string& stream_log_ip,
        uint16_t stream_log_port,
        boost::shared_ptr<DocumentManager>& dm,
        izenelib::ir::idmanager::IDManager* id_manager,
        NumericPropertyTableBuilder* ntb,
        AdSearchService* searcher,
        faceted::GroupManager* grp_mgr)
    : indexPath_(ad_data_path + "/index.bin"),
      clickPredictorWorkingPath_(ad_data_path + "/ctr_predictor"),
      ad_res_path_(ad_resource_path),
      ad_data_path_(ad_data_path),
      enable_ad_selector_(enable_ad_selector),
      enable_ad_rec_(enable_ad_rec),
      enable_ad_sponsored_search_(enable_ad_sponsored_search),
      documentManager_(dm),
      id_manager_(id_manager),
      numericTableBuilder_(ntb),
      ad_searcher_(searcher),
      groupManager_(grp_mgr)
{
    AdFeedbackMgr::get()->init(dmp_ip, dmp_port, ad_resource_path + "/feedback-schema.json");
    AdStreamSubscriber::get()->init(stream_log_ip, stream_log_port);
}

AdIndexManager::~AdIndexManager()
{
    // unsubscribe should make sure all callback finished and
    // no any callback will be send later.
    AdStreamSubscriber::get()->unsubscribe(adlog_topic);

    if(adMiningTask_)
        delete adMiningTask_;

    if (ad_click_predictor_)
        ad_click_predictor_->stop();

    AdStreamSubscriber::get()->stop();
}

bool AdIndexManager::buildMiningTask()
{
    ad_dnf_index_.reset(new AdDNFIndexType());

    ad_click_predictor_ = AdClickPredictor::get();
    ad_click_predictor_->init(clickPredictorWorkingPath_);

    std::ifstream ifs(indexPath_.c_str(), std::ios_base::binary);
    if(ifs.good())
    {
        try
        {
            boost::unique_lock<boost::shared_mutex> lock(rwMutex_);
            ad_dnf_index_->load_binary(ifs);
        }
        catch(const std::exception& e)
        {
            LOG(ERROR) << "exception in read file: " << e.what()
                << ", path: "<< indexPath_<<endl;
        }
    }

    if (enable_ad_selector_)
    {
        ad_selector_.reset(new AdSelector());
        ad_selector_->init(ad_res_path_ + "/ad_selector", ad_data_path_ + "/ad_selector",
            ad_data_path_ + "/ad_rec", enable_ad_rec_, ad_click_predictor_, groupManager_,
            documentManager_.get());
    }

    if (enable_ad_sponsored_search_)
    {
        ad_sponsored_mgr_.reset(new AdSponsoredMgr());
        ad_sponsored_mgr_->init(ad_res_path_ + "/ad_sponsored", ad_data_path_ + "ad_sponsored",
            groupManager_, documentManager_.get(), id_manager_, ad_searcher_);
    }

    adMiningTask_ = new AdMiningTask(indexPath_, documentManager_, ad_dnf_index_, ad_selector_, rwMutex_);
    adMiningTask_->setPostProcessFunc(boost::bind(&AdIndexManager::postMining, this, _1, _2));

    bool ret = AdStreamSubscriber::get()->subscribe(adlog_topic, boost::bind(&AdIndexManager::onAdStreamMessage, this, _1));
    if (!ret)
    {
        LOG(ERROR) << "subscribe the click log failed !!!!!!";
    }

    return true;
}

void AdIndexManager::onAdStreamMessage(const std::vector<AdMessage>& msg_list)
{
    static int cnt = 0;
    if (cnt % 10000 == 0)
    {
        for (size_t i = 0; i < msg_list.size(); ++i)
        {
            LOG(INFO) << "stream data: " << msg_list[i].body;
        }
        LOG(INFO) << "got ad stream data. size: " << msg_list.size() << ", total " << cnt;
    }
    cnt += msg_list.size();
    AdFeedbackMgr::FeedbackInfo feedback_info;
    AdClickPredictor::AssignmentT ad_feature;
    AdClickPredictor::AssignmentT user_feature;
    // read from stream msg and convert it to assignment list.
    for (size_t i = 0; i < msg_list.size(); ++i)
    {
        bool ret = AdFeedbackMgr::get()->parserFeedbackLogForAVRO(msg_list[i].body, feedback_info);
        if (!ret)
            continue;

        std::map<std::string, double>::const_iterator it = feedback_info.user_profiles.profile_data.begin();
        std::size_t feature_index = 0;
        user_feature.resize(feedback_info.user_profiles.profile_data.size());
        for(; it != feedback_info.user_profiles.profile_data.end(); ++it)
        {
            // TODO:
        }
        bool is_clicked = feedback_info.action > AdFeedbackMgr::View;
        if (enable_ad_sponsored_search_)
        {
            if (is_clicked)
            {
                ad_sponsored_mgr_->updateAuctionLogData(feedback_info.ad_id,
                    feedback_info.click_cost, feedback_info.click_slot);
            }
        }
        if (enable_ad_selector_)
        {
            docid_t docid = 0;
            uint128_t num_docid = Utilities::md5ToUint128(feedback_info.ad_id);
            bool ret = id_manager_->getDocIdByDocName(num_docid, docid, false);
            if (ret)
            {
                ad_selector_->getAdFeatureList(docid, ad_feature);
                ad_click_predictor_->update(user_feature, ad_feature,
                    is_clicked);
            }
            if (is_clicked)
            {
                if (!feedback_info.ad_id.empty())
                {
                    ad_selector_->updateClicked(docid);
                }
            }
            ad_selector_->updateSegments(user_feature, AdSelector::UserSeg);
            if (enable_ad_rec_)
            {
                ad_selector_->trainOnlineRecommender(feedback_info.user_id,
                    user_feature, feedback_info.ad_id, is_clicked);
            }
        }
    }
}

void AdIndexManager::postMining(docid_t startid, docid_t endid)
{
    LOG(INFO) << "ad mining finished from: " << startid << " to " << endid;
    if (ad_selector_)
    {
        ad_selector_->miningAdSegmentStr(startid, endid);
    }
    if (ad_sponsored_mgr_)
    {
        ad_sponsored_mgr_->miningAdCreatives(startid, endid);
    }
}

void AdIndexManager::rankAndSelect(const FeatureT& userinfo,
    std::vector<docid_t>& docids,
    std::vector<float>& topKRankScoreList,
    std::size_t& totalCount)
{
    LOG(INFO) << "begin rank ads from searched result : " << docids.size();
    typedef std::vector<docid_t>::iterator InputIterator;

    std::string propName = "Price";
    std::string propName_mode = "BidMode";
    PropSharedLockSet propSharedLockSet;

    boost::shared_ptr<NumericPropertyTableBase> numericTable =
        numericTableBuilder_->createPropertyTable(propName);
    boost::shared_ptr<NumericPropertyTableBase> numericTable_mode =
        numericTableBuilder_->createPropertyTable(propName_mode);

    if ( numericTable )
        propSharedLockSet.insertSharedLock(numericTable.get());
    if ( numericTable_mode )
        propSharedLockSet.insertSharedLock(numericTable_mode.get());

    boost::shared_ptr<HitQueue> scoreItemQueue;
    std::vector<docid_t>  cpc_ads_result;
    std::vector<AdSelector::FeatureMapT> cpc_ads_features;
    cpc_ads_result.reserve(docids.size());
    cpc_ads_features.reserve(docids.size());

    uint32_t heapSize = std::min((std::size_t)MAX_SELECT_AD_COUNT, docids.size());
    scoreItemQueue.reset(new ScoreSortedHitQueue(heapSize));

    for(InputIterator it = docids.begin();
            it != docids.end(); it++ )
    {
        if(!documentManager_->isDeleted(*it))
        {
            float price = 0.0;
            float score = 0.0;
            int32_t mode = 0;
            if(numericTable_mode)
            {
                numericTable_mode->getInt32Value(*it, mode, false);
            }
            if(mode == 0)
            {
                if(numericTable)
                {
                    numericTable->getFloatValue(*it, price, false);
                }
                score = price;
            }
            else if(mode == 1)
            {
                cpc_ads_result.push_back(*it);
                continue;
            }
            ScoreDoc scoreItem(*it, score);
            scoreItemQueue->insert(scoreItem);
        }
    }
    LOG(INFO) << "begin select ads from cpc cand result : " << cpc_ads_result.size();
    // select some ads using some strategy to maximize the CPC.
    std::vector<double> score_list;
    if (ad_selector_)
    {
        ad_selector_->select(userinfo, MAX_SELECT_AD_COUNT,
            cpc_ads_result, score_list, propSharedLockSet);
    }
    LOG(INFO) << "end select ads from result.";
    // calculate eCPM
    for (std::size_t i = 0; i < cpc_ads_result.size(); ++i)
    {
        float price = 0;
        if(numericTable)
        {
            numericTable->getFloatValue(cpc_ads_result[i], price, false);
        }
        double score = score_list[i] * price * 1000;
        ScoreDoc item(cpc_ads_result[i], score);
        scoreItemQueue->insert(item);
    }

    unsigned int scoreSize = scoreItemQueue->size();

    docids.resize(scoreSize);
    topKRankScoreList.resize(scoreSize);

    for (int i = scoreSize - 1; i >= 0; --i)
    {
        const ScoreDoc& item = scoreItemQueue->pop();
        docids[i] = item.docId;
        topKRankScoreList[i] = item.score;
    }
    totalCount = docids.size();
}

bool AdIndexManager::searchByQuery(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_searcher_ || !ad_selector_)
        return false;
    // using the DNF as Attributes.
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    bool ret = ad_searcher_->search(actionOperation.actionItem_, searchResult);
    const std::vector<std::pair<std::string, std::string> > userinfo;
    if (ret)
    {
        searchResult.adCachedTopKDocs_ = searchResult.topKDocs_;
        rankAndSelect(userinfo, searchResult.topKDocs_, searchResult.topKRankScoreList_, searchResult.totalCount_);
    }
    return ret;
}

bool AdIndexManager::searchByDNF(const FeatureT& info,
    std::vector<docid_t>& docids,
    std::vector<float>& topKRankScoreList,
    std::size_t& totalCount)
{
    boost::unordered_set<uint32_t> dnfIDs;

    {
        boost::shared_lock<boost::shared_mutex> lock(rwMutex_);
        ad_dnf_index_->retrieve(info, dnfIDs);
    }

    LOG(INFO)<< "dnfIDs.size(): "<< dnfIDs.size() << std::endl;

    docids.resize(dnfIDs.size());

    std::size_t i = 0;
    for(boost::unordered_set<uint32_t>::iterator it = dnfIDs.begin();
            it != dnfIDs.end(); it++ )
    {
        docids[i++] = *it;
    }

    rankAndSelect(info, docids, topKRankScoreList, totalCount);
    return true;
}

bool AdIndexManager::searchByRecommend(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_searcher_ || !ad_selector_)
        return false;
    const std::vector<std::pair<std::string, std::string> > userinfo;
    std::vector<double> score_list;
    std::vector<std::string> rec_items;
    ad_selector_->selectFromRecommend(userinfo, MAX_RECOMMEND_ITEM_NUM,
        rec_items, score_list);
    std::string rec_query_str;
    for (size_t i = 0; i < rec_items.size(); ++i)
    {
        rec_query_str += rec_items[i] + " ";
    }
    LOG(INFO) << "recommend feature items are: ";
    //KeywordSearchActionItem rec_search_query(actionOperation.actionItem_);
    //rec_search_query.env_.queryString_ = rec_query_str;
    //rec_search_query.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.env_.queryString_ = rec_query_str;
    bool ret = ad_searcher_->search(actionOperation.actionItem_, searchResult);
    if (ret)
    {
        rankAndSelect(userinfo, searchResult.topKDocs_, searchResult.topKRankScoreList_, searchResult.totalCount_);
    }
    return ret;
}

bool AdIndexManager::sponsoredAdSearch(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_sponsored_mgr_)
        return false;
    return ad_sponsored_mgr_->sponsoredAdSearch(actionOperation, searchResult);
}

} //namespace sf1r
<commit_msg>chage log toppic<commit_after>/*
 *  AdIndexManager.cpp
 */

#include "AdIndexManager.h"
#include "AdStreamSubscriber.h"
#include "AdSelector.h"
#include "AdClickPredictor.h"
#include "AdFeedbackMgr.h"
#include "AdSearchService.h"
#include "sponsored-ad-search/AdSponsoredMgr.h"
#include <common/ResultType.h>
#include <mining-manager/group-manager/GroupManager.h>
#include <query-manager/ActionItem.h>
#include <query-manager/SearchKeywordOperation.h>
#include <search-manager/SearchBase.h>
#include <search-manager/HitQueue.h>
#include <util/ustring/UString.h>
#include <algorithm>

namespace sf1r
{

static const int MAX_SEARCH_AD_COUNT = 20000;
static const int MAX_SELECT_AD_COUNT = 20;
static const int MAX_RECOMMEND_ITEM_NUM = 10;
static const std::string adlog_topic = "log_b5t_sf1";

using namespace sponsored;

AdIndexManager::AdIndexManager(
        const std::string& ad_resource_path,
        const std::string& ad_data_path,
        bool enable_ad_selector,
        bool enable_ad_rec,
        bool enable_ad_sponsored_search,
        const std::string& dmp_ip,
        uint16_t dmp_port,
        const std::string& stream_log_ip,
        uint16_t stream_log_port,
        boost::shared_ptr<DocumentManager>& dm,
        izenelib::ir::idmanager::IDManager* id_manager,
        NumericPropertyTableBuilder* ntb,
        AdSearchService* searcher,
        faceted::GroupManager* grp_mgr)
    : indexPath_(ad_data_path + "/index.bin"),
      clickPredictorWorkingPath_(ad_data_path + "/ctr_predictor"),
      ad_res_path_(ad_resource_path),
      ad_data_path_(ad_data_path),
      enable_ad_selector_(enable_ad_selector),
      enable_ad_rec_(enable_ad_rec),
      enable_ad_sponsored_search_(enable_ad_sponsored_search),
      documentManager_(dm),
      id_manager_(id_manager),
      numericTableBuilder_(ntb),
      ad_searcher_(searcher),
      groupManager_(grp_mgr)
{
    AdFeedbackMgr::get()->init(dmp_ip, dmp_port, ad_resource_path + "/feedback-schema.json");
    AdStreamSubscriber::get()->init(stream_log_ip, stream_log_port);
}

AdIndexManager::~AdIndexManager()
{
    // unsubscribe should make sure all callback finished and
    // no any callback will be send later.
    AdStreamSubscriber::get()->unsubscribe(adlog_topic);

    if(adMiningTask_)
        delete adMiningTask_;

    if (ad_click_predictor_)
        ad_click_predictor_->stop();

    AdStreamSubscriber::get()->stop();
}

bool AdIndexManager::buildMiningTask()
{
    ad_dnf_index_.reset(new AdDNFIndexType());

    ad_click_predictor_ = AdClickPredictor::get();
    ad_click_predictor_->init(clickPredictorWorkingPath_);

    std::ifstream ifs(indexPath_.c_str(), std::ios_base::binary);
    if(ifs.good())
    {
        try
        {
            boost::unique_lock<boost::shared_mutex> lock(rwMutex_);
            ad_dnf_index_->load_binary(ifs);
        }
        catch(const std::exception& e)
        {
            LOG(ERROR) << "exception in read file: " << e.what()
                << ", path: "<< indexPath_<<endl;
        }
    }

    if (enable_ad_selector_)
    {
        ad_selector_.reset(new AdSelector());
        ad_selector_->init(ad_res_path_ + "/ad_selector", ad_data_path_ + "/ad_selector",
            ad_data_path_ + "/ad_rec", enable_ad_rec_, ad_click_predictor_, groupManager_,
            documentManager_.get());
    }

    if (enable_ad_sponsored_search_)
    {
        ad_sponsored_mgr_.reset(new AdSponsoredMgr());
        ad_sponsored_mgr_->init(ad_res_path_ + "/ad_sponsored", ad_data_path_ + "ad_sponsored",
            groupManager_, documentManager_.get(), id_manager_, ad_searcher_);
    }

    adMiningTask_ = new AdMiningTask(indexPath_, documentManager_, ad_dnf_index_, ad_selector_, rwMutex_);
    adMiningTask_->setPostProcessFunc(boost::bind(&AdIndexManager::postMining, this, _1, _2));

    bool ret = AdStreamSubscriber::get()->subscribe(adlog_topic, boost::bind(&AdIndexManager::onAdStreamMessage, this, _1));
    if (!ret)
    {
        LOG(ERROR) << "subscribe the click log failed !!!!!!";
    }

    return true;
}

void AdIndexManager::onAdStreamMessage(const std::vector<AdMessage>& msg_list)
{
    static int cnt = 0;
    if (cnt % 10000 == 0)
    {
        for (size_t i = 0; i < msg_list.size(); ++i)
        {
            LOG(INFO) << "stream data: " << msg_list[i].body;
        }
        LOG(INFO) << "got ad stream data. size: " << msg_list.size() << ", total " << cnt;
    }
    cnt += msg_list.size();
    AdFeedbackMgr::FeedbackInfo feedback_info;
    AdClickPredictor::AssignmentT ad_feature;
    AdClickPredictor::AssignmentT user_feature;
    // read from stream msg and convert it to assignment list.
    for (size_t i = 0; i < msg_list.size(); ++i)
    {
        bool ret = AdFeedbackMgr::get()->parserFeedbackLogForAVRO(msg_list[i].body, feedback_info);
        if (!ret)
            continue;

        std::map<std::string, double>::const_iterator it = feedback_info.user_profiles.profile_data.begin();
        std::size_t feature_index = 0;
        user_feature.resize(feedback_info.user_profiles.profile_data.size());
        for(; it != feedback_info.user_profiles.profile_data.end(); ++it)
        {
            // TODO:
        }
        bool is_clicked = feedback_info.action > AdFeedbackMgr::View;
        if (enable_ad_sponsored_search_)
        {
            if (is_clicked)
            {
                ad_sponsored_mgr_->updateAuctionLogData(feedback_info.ad_id,
                    feedback_info.click_cost, feedback_info.click_slot);
            }
        }
        if (enable_ad_selector_)
        {
            docid_t docid = 0;
            uint128_t num_docid = Utilities::md5ToUint128(feedback_info.ad_id);
            bool ret = id_manager_->getDocIdByDocName(num_docid, docid, false);
            if (ret)
            {
                ad_selector_->getAdFeatureList(docid, ad_feature);
                ad_click_predictor_->update(user_feature, ad_feature,
                    is_clicked);
            }
            if (is_clicked)
            {
                if (!feedback_info.ad_id.empty())
                {
                    ad_selector_->updateClicked(docid);
                }
            }
            ad_selector_->updateSegments(user_feature, AdSelector::UserSeg);
            if (enable_ad_rec_)
            {
                ad_selector_->trainOnlineRecommender(feedback_info.user_id,
                    user_feature, feedback_info.ad_id, is_clicked);
            }
        }
    }
}

void AdIndexManager::postMining(docid_t startid, docid_t endid)
{
    LOG(INFO) << "ad mining finished from: " << startid << " to " << endid;
    if (ad_selector_)
    {
        ad_selector_->miningAdSegmentStr(startid, endid);
    }
    if (ad_sponsored_mgr_)
    {
        ad_sponsored_mgr_->miningAdCreatives(startid, endid);
    }
}

void AdIndexManager::rankAndSelect(const FeatureT& userinfo,
    std::vector<docid_t>& docids,
    std::vector<float>& topKRankScoreList,
    std::size_t& totalCount)
{
    LOG(INFO) << "begin rank ads from searched result : " << docids.size();
    typedef std::vector<docid_t>::iterator InputIterator;

    std::string propName = "Price";
    std::string propName_mode = "BidMode";
    PropSharedLockSet propSharedLockSet;

    boost::shared_ptr<NumericPropertyTableBase> numericTable =
        numericTableBuilder_->createPropertyTable(propName);
    boost::shared_ptr<NumericPropertyTableBase> numericTable_mode =
        numericTableBuilder_->createPropertyTable(propName_mode);

    if ( numericTable )
        propSharedLockSet.insertSharedLock(numericTable.get());
    if ( numericTable_mode )
        propSharedLockSet.insertSharedLock(numericTable_mode.get());

    boost::shared_ptr<HitQueue> scoreItemQueue;
    std::vector<docid_t>  cpc_ads_result;
    std::vector<AdSelector::FeatureMapT> cpc_ads_features;
    cpc_ads_result.reserve(docids.size());
    cpc_ads_features.reserve(docids.size());

    uint32_t heapSize = std::min((std::size_t)MAX_SELECT_AD_COUNT, docids.size());
    scoreItemQueue.reset(new ScoreSortedHitQueue(heapSize));

    for(InputIterator it = docids.begin();
            it != docids.end(); it++ )
    {
        if(!documentManager_->isDeleted(*it))
        {
            float price = 0.0;
            float score = 0.0;
            int32_t mode = 0;
            if(numericTable_mode)
            {
                numericTable_mode->getInt32Value(*it, mode, false);
            }
            if(mode == 0)
            {
                if(numericTable)
                {
                    numericTable->getFloatValue(*it, price, false);
                }
                score = price;
            }
            else if(mode == 1)
            {
                cpc_ads_result.push_back(*it);
                continue;
            }
            ScoreDoc scoreItem(*it, score);
            scoreItemQueue->insert(scoreItem);
        }
    }
    LOG(INFO) << "begin select ads from cpc cand result : " << cpc_ads_result.size();
    // select some ads using some strategy to maximize the CPC.
    std::vector<double> score_list;
    if (ad_selector_)
    {
        ad_selector_->select(userinfo, MAX_SELECT_AD_COUNT,
            cpc_ads_result, score_list, propSharedLockSet);
    }
    LOG(INFO) << "end select ads from result.";
    // calculate eCPM
    for (std::size_t i = 0; i < cpc_ads_result.size(); ++i)
    {
        float price = 0;
        if(numericTable)
        {
            numericTable->getFloatValue(cpc_ads_result[i], price, false);
        }
        double score = score_list[i] * price * 1000;
        ScoreDoc item(cpc_ads_result[i], score);
        scoreItemQueue->insert(item);
    }

    unsigned int scoreSize = scoreItemQueue->size();

    docids.resize(scoreSize);
    topKRankScoreList.resize(scoreSize);

    for (int i = scoreSize - 1; i >= 0; --i)
    {
        const ScoreDoc& item = scoreItemQueue->pop();
        docids[i] = item.docId;
        topKRankScoreList[i] = item.score;
    }
    totalCount = docids.size();
}

bool AdIndexManager::searchByQuery(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_searcher_ || !ad_selector_)
        return false;
    // using the DNF as Attributes.
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    bool ret = ad_searcher_->search(actionOperation.actionItem_, searchResult);
    const std::vector<std::pair<std::string, std::string> > userinfo;
    if (ret)
    {
        searchResult.adCachedTopKDocs_ = searchResult.topKDocs_;
        rankAndSelect(userinfo, searchResult.topKDocs_, searchResult.topKRankScoreList_, searchResult.totalCount_);
    }
    return ret;
}

bool AdIndexManager::searchByDNF(const FeatureT& info,
    std::vector<docid_t>& docids,
    std::vector<float>& topKRankScoreList,
    std::size_t& totalCount)
{
    boost::unordered_set<uint32_t> dnfIDs;

    {
        boost::shared_lock<boost::shared_mutex> lock(rwMutex_);
        ad_dnf_index_->retrieve(info, dnfIDs);
    }

    LOG(INFO)<< "dnfIDs.size(): "<< dnfIDs.size() << std::endl;

    docids.resize(dnfIDs.size());

    std::size_t i = 0;
    for(boost::unordered_set<uint32_t>::iterator it = dnfIDs.begin();
            it != dnfIDs.end(); it++ )
    {
        docids[i++] = *it;
    }

    rankAndSelect(info, docids, topKRankScoreList, totalCount);
    return true;
}

bool AdIndexManager::searchByRecommend(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_searcher_ || !ad_selector_)
        return false;
    const std::vector<std::pair<std::string, std::string> > userinfo;
    std::vector<double> score_list;
    std::vector<std::string> rec_items;
    ad_selector_->selectFromRecommend(userinfo, MAX_RECOMMEND_ITEM_NUM,
        rec_items, score_list);
    std::string rec_query_str;
    for (size_t i = 0; i < rec_items.size(); ++i)
    {
        rec_query_str += rec_items[i] + " ";
    }
    LOG(INFO) << "recommend feature items are: ";
    //KeywordSearchActionItem rec_search_query(actionOperation.actionItem_);
    //rec_search_query.env_.queryString_ = rec_query_str;
    //rec_search_query.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.searchingMode_.mode_ = SearchingMode::SUFFIX_MATCH;
    (*const_cast<SearchKeywordOperation*>(&actionOperation)).actionItem_.env_.queryString_ = rec_query_str;
    bool ret = ad_searcher_->search(actionOperation.actionItem_, searchResult);
    if (ret)
    {
        rankAndSelect(userinfo, searchResult.topKDocs_, searchResult.topKRankScoreList_, searchResult.totalCount_);
    }
    return ret;
}

bool AdIndexManager::sponsoredAdSearch(const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult)
{
    if (!ad_sponsored_mgr_)
        return false;
    return ad_sponsored_mgr_->sponsoredAdSearch(actionOperation, searchResult);
}

} //namespace sf1r
<|endoftext|>