<commit_before>/**
 * @file seaicing.cpp
 *
 *  Created on: Jan 03, 2013
 *  @author aaltom
 */

#include "seaicing.h"
#include "logger_factory.h"
#include <boost/lexical_cast.hpp>
#include "level.h"
#include "forecast_time.h"

using namespace std;
using namespace himan::plugin;

seaicing::seaicing()
	: global(false)
{
	itsClearTextFormula = "SeaIcing = FF * ( -sIndex -T2m ) / ( 1 + 0.3 * ( T0 + sIndex ))";

	itsLogger = logger_factory::Instance()->GetLog("seaicing");

}

void seaicing::Process(std::shared_ptr<const plugin_configuration> conf)
{
	Init(conf);

	if (itsConfiguration->Exists("global") && itsConfiguration->GetValue("global") == "true")
	{
		SetParams({param("SSICING-N", 10059, 0, 0, 2)});
		global = true;
	}
	else
	{
		// By default baltic sea
		SetParams({param("ICING-N", 480, 0, 0, 2)});
		global = false;
	}

	Start();

}

/*
 * Calculate()
 *
 * This function does the actual calculation.
 */

void seaicing::Calculate(shared_ptr<info> myTargetInfo, unsigned short theThreadIndex)
{

  const params TParam = {param("T-K"), param("TG-K")};
  const level TLevel(himan::kHeight, 2, "HEIGHT");
  const param FfParam("FF-MS"); // 10 meter wind
  const level FfLevel(himan::kHeight, 10, "HEIGHT");

  level ground;
  double saltinessIndex = 0.35;

  if (global)
  {
  		saltinessIndex = 1.5;
  }

  // this will come back to us
  if ( itsConfiguration->SourceProducer().Id() == 131)
  {
       ground = level(himan::kGndLayer, 0, "GNDLAYER");
  }
  else
  {
       ground = level(himan::kHeight, 0, "HEIGHT");
  }

	auto myThreadedLogger = logger_factory::Instance()->GetLog("seaicingThread #" + boost::lexical_cast<string> (theThreadIndex));

	forecast_time forecastTime = myTargetInfo->Time();
	level forecastLevel = myTargetInfo->Level();

	myThreadedLogger->Info("Calculating time " + static_cast<string>(*forecastTime.ValidDateTime()) + " level " + static_cast<string> (forecastLevel));

	info_t TInfo = Fetch(forecastTime, TLevel, TParam, false);
	info_t TgInfo = Fetch(forecastTime, ground, TParam, false);
	info_t FfInfo = Fetch(forecastTime, FfLevel, FfParam, false);

	if (!TInfo || !TgInfo || !FfInfo)
	{
		myThreadedLogger->Warning("Skipping step " + boost::lexical_cast<string> (forecastTime.Step()) + ", level " + static_cast<string> (forecastLevel));
		return;
	}

	string deviceType = "CPU";

	LOCKSTEP(myTargetInfo, TInfo, TgInfo, FfInfo)
	{
		double T = TInfo->Value();
		double Tg = TgInfo->Value();
		double Ff = FfInfo->Value();

		if (T == kFloatMissing || Tg == kFloatMissing || Ff == kFloatMissing)
		{
			myTargetInfo->Value(-10);
			continue;
		}

		double seaIcing;
		double TBase = 273.15;

		T = T - TBase;
		Tg = Tg - TBase;

		if (Tg < -2 )
		{
			seaIcing = -10;
		}
		else
		{
			seaIcing = Ff * ( -saltinessIndex -T ) / ( 1 + 0.3 * ( Tg + saltinessIndex ));

			if (seaIcing > 100)
			{
				seaIcing = 100;
			}
		}

		myTargetInfo->Value(seaIcing);
	}

	myThreadedLogger->Info("[" + deviceType + "] Missing values: " + boost::lexical_cast<string> (myTargetInfo->Data()->MissingCount()) + "/" + boost::lexical_cast<string> (myTargetInfo->Data()->Size()));

}
<commit_msg>Calculate index instead of values.<commit_after>/**
 * @file seaicing.cpp
 *
 *  Created on: Jan 03, 2013
 *  @author aaltom
 */

#include "seaicing.h"
#include "logger_factory.h"
#include <boost/lexical_cast.hpp>
#include "level.h"
#include "forecast_time.h"

using namespace std;
using namespace himan::plugin;

seaicing::seaicing()
	: global(false)
{
	itsClearTextFormula = "SeaIcing = FF * ( -sIndex -T2m ) / ( 1 + 0.3 * ( T0 + sIndex ))";

	itsLogger = logger_factory::Instance()->GetLog("seaicing");

}

void seaicing::Process(std::shared_ptr<const plugin_configuration> conf)
{
	Init(conf);

	if (itsConfiguration->Exists("global") && itsConfiguration->GetValue("global") == "true")
	{
		SetParams({param("SSICING-N", 10059, 0, 0, 2)});
		global = true;
	}
	else
	{
		// By default baltic sea
		SetParams({param("ICING-N", 480, 0, 0, 2)});
		global = false;
	}

	Start();

}

/*
 * Calculate()
 *
 * This function does the actual calculation.
 */

void seaicing::Calculate(shared_ptr<info> myTargetInfo, unsigned short theThreadIndex)
{

  const params TParam = {param("T-K"), param("TG-K")};
  const level TLevel(himan::kHeight, 2, "HEIGHT");
  const param FfParam("FF-MS"); // 10 meter wind
  const level FfLevel(himan::kHeight, 10, "HEIGHT");

  level ground;
  double saltinessIndex = 0.35;

  if (global)
  {
	saltinessIndex = 1.5;
  }

  // this will come back to us
  if ( itsConfiguration->SourceProducer().Id() == 131)
  {
       ground = level(himan::kGndLayer, 0, "GNDLAYER");
  }
  else
  {
       ground = level(himan::kHeight, 0, "HEIGHT");
  }

	auto myThreadedLogger = logger_factory::Instance()->GetLog("seaicingThread #" + boost::lexical_cast<string> (theThreadIndex));

	forecast_time forecastTime = myTargetInfo->Time();
	level forecastLevel = myTargetInfo->Level();

	myThreadedLogger->Info("Calculating time " + static_cast<string>(*forecastTime.ValidDateTime()) + " level " + static_cast<string> (forecastLevel));

	info_t TInfo = Fetch(forecastTime, TLevel, TParam, false);
	info_t TgInfo = Fetch(forecastTime, ground, TParam, false);
	info_t FfInfo = Fetch(forecastTime, FfLevel, FfParam, false);

	if (!TInfo || !TgInfo || !FfInfo)
	{
		myThreadedLogger->Warning("Skipping step " + boost::lexical_cast<string> (forecastTime.Step()) + ", level " + static_cast<string> (forecastLevel));
		return;
	}

	string deviceType = "CPU";

	LOCKSTEP(myTargetInfo, TInfo, TgInfo, FfInfo)
	{
		double T = TInfo->Value();
		double Tg = TgInfo->Value();
		double Ff = FfInfo->Value();

		if (T == kFloatMissing || Tg == kFloatMissing || Ff == kFloatMissing)
		{
			myTargetInfo->Value(-10);
			continue;
		}

		double seaIcing;
		double TBase = 273.15;

		T = T - TBase;
		Tg = Tg - TBase;

		if (Tg < -2 )
		{
			seaIcing = -10;
		}
		else
		{
			seaIcing = Ff * ( -saltinessIndex -T ) / ( 1 + 0.3 * ( Tg + saltinessIndex ));

			if (seaIcing > 100)
			{
				seaIcing = 100;
			}
		}

		// Change values to index
		// Index by Antonios Niros: Vessel icing forecast and services: further development and perspectives.

		if (seaIcing <= 0)
		{ // No icing
			seaIcing = 1;
		}
		else if (seaIcing > 0 && seaIcing < 22.4)
		{ // Light icing ja icing rate <0.7cm/h
			seaIcing = 2;
		}
		else if (seaIcing >= 22.4 && seaIcing < 53.3)
                { // Moderate icing ja icing rate between 0.7cm/h-2cm/h
                        seaIcing = 3;
                }
		else if (seaIcing >= 53.3 && seaIcing < 83)
                { //  Heavy icing ja icing rate between 2.0cm/h-4.0cm/h
                        seaIcing = 4;
                }
		else if (seaIcing >= 83)
                { // Extreme icing ja icing rate >4cm/h
                        seaIcing = 5;
                }

		myTargetInfo->Value(seaIcing);
	}

	myThreadedLogger->Info("[" + deviceType + "] Missing values: " + boost::lexical_cast<string> (myTargetInfo->Data()->MissingCount()) + "/" + boost::lexical_cast<string> (myTargetInfo->Data()->Size()));

}
<|endoftext|>


<commit_before>/* Copyright 2015 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// See docs in ../ops/data_flow_ops.cc.

#include <deque>
#include <vector>

#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/padding_fifo_queue.h"
#include "tensorflow/core/kernels/queue_base.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/port.h"
#include "tensorflow/core/public/tensor.h"
#include "tensorflow/core/public/tensor_shape.h"

namespace tensorflow {

PaddingFIFOQueue::PaddingFIFOQueue(
    int capacity, const DataTypeVector& component_dtypes,
    const std::vector<PartialTensorShape>& partial_shapes, const string& name)
    : FIFOQueue(capacity, component_dtypes,
                ConvertShapesPartialDimensionsToZero(partial_shapes), name),
      partial_shapes_(partial_shapes) {}

Status PaddingFIFOQueue::Initialize() {
  Status s = FIFOQueue::Initialize();
  if (!s.ok()) return s;

  if (component_dtypes_.size() != partial_shapes_.size()) {
    return errors::InvalidArgument(
        "Shapes must be provided for all components, but received ",
        component_dtypes_.size(), " dtypes and ", partial_shapes_.size(),
        " shapes.");
  }

  return Status::OK();
}

/* static */
Status PaddingFIFOQueue::GetElementComponent(
    const PaddingFIFOQueue::Tuple& tuple, int component, OpKernelContext* ctx,
    PersistentTensor* out_tensor) {
  TensorShape element_shape(tuple[component].shape());
  Tensor* element_access = nullptr;
  TF_RETURN_IF_ERROR(ctx->allocate_persistent(
      tuple[component].dtype(), element_shape, out_tensor, &element_access));
  *element_access = tuple[component];
  return Status::OK();
}

void PaddingFIFOQueue::TryDequeueMany(int num_elements, OpKernelContext* ctx,
                                      CallbackWithTuple callback) {
  if (num_elements == 0) {
    Tuple tuple;
    tuple.reserve(num_components());
    for (int i = 0; i < num_components(); ++i) {
      // TODO(josh11b,misard): Switch to allocate_output().
      // See similar comment in fifo_queue.cc
      Tensor element;
      // Here, ManyOutShape returns zeros for undetermined shapes,
      // which is exactly what we want to use.
      ctx->allocate_temp(component_dtypes_[i], ManyOutShape(i, 0), &element);
      tuple.emplace_back(element);
    }
    callback(tuple);
    return;
  }

  CancellationManager* cm = ctx->cancellation_manager();
  CancellationToken token = cm->get_cancellation_token();
  bool already_cancelled;
  {
    mutex_lock l(mu_);
    already_cancelled = !cm->RegisterCallback(
        token, [this, token]() { Cancel(kDequeue, token); });
    if (!already_cancelled) {
      // TODO(josh11b): This makes two copies of callback, avoid this if possible.
      dequeue_attempts_.emplace_back(
          num_elements, [callback]() { callback(Tuple()); }, ctx, token,
          [callback, this](Attempt* attempt) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
            int32 s = queues_[0].size();
            if (closed_ && s < attempt->elements_requested) {
              attempt->context->SetStatus(errors::OutOfRange(
                  "PaddingFIFOQueue '", name_, "' is closed and has ",
                  "insufficient elements (requested ",
                  attempt->elements_requested, ", current size ", s, ")"));

              // TODO(mrry): Add support for producing a partial batch as
              // output when the queue is closed.
              if (!attempt->tuples.empty()) {
                // Restore already-dequeued elements to the front of the queue.
                for (int64 i = attempt->tuples.size() - 1; i >= 0; --i) {
                  for (int j = 0; j < num_components(); ++j) {
                    PersistentTensor element;
                    Status s = GetElementComponent(attempt->tuples[i], j,
                                                   attempt->context, &element);
                    if (!s.ok()) {
                      attempt->context->SetStatus(
                          errors::DataLoss("Failed to restore element from "
                                           "partially-dequeued batch "
                                           "to PaddingFIFOQueue: ",
                                           s.error_message()));
                    }
                    queues_[j].push_front(element);
                  }
                }
              }
              return kComplete;
            }

            RunResult result = kNoProgress;
            for (; s > 0; --s) {
              result = kProgress;
              Tuple tuple;
              DequeueLocked(attempt->context, &tuple);
              attempt->tuples.push_back(tuple);
              tuple.clear();
              --attempt->elements_requested;

              if (attempt->elements_requested == 0) {
                // Finished.  Allocate attempt->tuple and
                // copy from attempt->tuples to attempt->tuple.
                attempt->tuple.reserve(num_components());
                const std::vector<Tuple>& tuples = attempt->tuples;

                std::vector<bool> dynamic_shape;
                const int64 batch_size = tuples.size();

                for (int i = 0; i < num_components(); ++i) {
                  const PartialTensorShape partial_shape =
                      PartialTensorShape({batch_size})
                          .Concatenate(partial_shapes_[i]);
                  TensorShape shape({batch_size});

                  for (int j = 0; j < partial_shape.dims() - 1; ++j) {
                    if (partial_shape.dim_size(j + 1) > -1) {
                      shape.AddDim(partial_shape.dim_size(j + 1));
                    } else {
                      // Expand sizes to match.
                      int64 max_val = 0;
                      for (const Tuple& t : tuples) {
                        max_val = max(max_val, t[i].shape().dim_size(j));
                      }
                      shape.AddDim(max_val);
                    }
                  }

                  Tensor element;
                  attempt->context->allocate_temp(component_dtypes_[i], shape,
                                                  &element);

                  bool has_dynamic_shape = !partial_shape.IsFullyDefined();
                  if (has_dynamic_shape) {
                    // Set all values to zero because not all values
                    // will get written over.
                    attempt->context->SetStatus(SetElementZero(&element));
                    if (!attempt->context->status().ok()) return kComplete;
                  }

                  dynamic_shape.push_back(has_dynamic_shape);

                  // TODO(ebrevdo): should this be a persistent tensor?
                  attempt->tuple.emplace_back(element);
                }

                for (int index = 0; index < tuples.size(); ++index) {
                  for (int i = 0; i < num_components(); ++i) {
                    if (dynamic_shape[i]) {
                      // Slightly slower copy operation
                      attempt->context->SetStatus(CopyElementToLargerSlice(
                          tuples[index][i], &attempt->tuple[i], index));
                    } else {
                      attempt->context->SetStatus(CopyElementToSlice(
                          tuples[index][i], &attempt->tuple[i], index));
                    }
                    if (!attempt->context->status().ok()) return kComplete;
                  }
                }
                tuple = attempt->tuple;
                attempt->tuples.clear();
                attempt->done_callback = [callback, tuple]() {
                  callback(tuple);
                };
                return kComplete;
              }
            }
            return result;
          });
    }
  }
  if (!already_cancelled) {
    FlushUnlocked();
  } else {
    ctx->SetStatus(errors::Cancelled("Dequeue operation was cancelled"));
    callback(Tuple());
  }
}

Status PaddingFIFOQueue::ValidateTuple(const Tuple& tuple) {
  TF_RETURN_IF_ERROR(ValidateTupleCommon(tuple));
  for (size_t i = 0; i < tuple.size(); ++i) {
    if (!partial_shapes_[i].IsCompatibleWith(tuple[i].shape())) {
      return errors::InvalidArgument("Shape mismatch in tuple component ", i,
                                     ". Expected ",
                                     partial_shapes_[i].DebugString(), ", got ",
                                     tuple[i].shape().ShortDebugString());
    }
  }
  return Status::OK();
}

Status PaddingFIFOQueue::ValidateManyTuple(const Tuple& tuple) {
  TF_RETURN_IF_ERROR(ValidateTupleCommon(tuple));
  const int64 batch_size = tuple[0].dim_size(0);
  for (size_t i = 0; i < tuple.size(); ++i) {
    // Expected shape is [batch_size] + partial_shapes_[i]
    const PartialTensorShape expected_shape =
        PartialTensorShape({batch_size}).Concatenate(partial_shapes_[i]);
    if (!expected_shape.IsCompatibleWith(tuple[i].shape())) {
      return errors::InvalidArgument("Shape mismatch in tuple component ", i,
                                     ". Expected ",
                                     expected_shape.DebugString(), ", got ",
                                     tuple[i].shape().ShortDebugString());
    }
  }
  return Status::OK();
}

Status PaddingFIFOQueue::CompatibleNodeDefShapes(
    const NodeDef& node_def) const {
  std::vector<PartialTensorShape> requested_shapes;
  TF_RETURN_IF_ERROR(GetNodeAttr(node_def, "shapes", &requested_shapes));
  if (!PartialTensorShapeUtils::AreCompatible(requested_shapes,
                                              partial_shapes_)) {
    return errors::InvalidArgument(
        "Shared queue '", name_, "' has component shapes ",
        PartialTensorShapeUtils::PartialShapeListString(partial_shapes_),
        " but requested component shapes were ",
        PartialTensorShapeUtils::PartialShapeListString(requested_shapes));
  } else {
    return Status::OK();
  }
}

Status PaddingFIFOQueue::MatchesNodeDef(const NodeDef& node_def) {
  TF_RETURN_IF_ERROR(MatchesNodeDefOp(node_def, "PaddingFIFOQueue"));
  TF_RETURN_IF_ERROR(MatchesNodeDefCapacity(node_def, capacity_));
  TF_RETURN_IF_ERROR(MatchesNodeDefTypes(node_def));
  TF_RETURN_IF_ERROR(CompatibleNodeDefShapes(node_def));
  return Status::OK();
}

template <typename T, int NDIMS>
Status HandleElementToLargerSlice(const Tensor& element, Tensor* parent,
                                  int index) {
  DCHECK_NE(parent->dim_size(0), 0);
  if (element.NumElements() > (parent->NumElements() / parent->dim_size(0))) {
    TensorShape chip_shape = parent->shape();
    chip_shape.RemoveDim(0);
    return errors::Internal(
        "HandleElementToLargerSlice Cannot copy slice: number of entries in "
        "element is greater than number of elements in parent slice.  ",
        "Shapes are: [element]: ", element.shape().DebugString(),
        ", [parent slice]: ", chip_shape.DebugString());
  }
  auto element_t = element.tensor<T, NDIMS>();
  auto parent_t = parent->tensor<T, NDIMS + 1>();
  Eigen::DSizes<Eigen::DenseIndex, NDIMS + 1> slice_indices;
  slice_indices[0] = index;
  Eigen::DSizes<Eigen::DenseIndex, NDIMS + 1> slice_size;
  slice_size[0] = 1;
  for (int i = 1; i < slice_size.size(); ++i) {
    slice_size[i] = element_t.dimension(i - 1);
  }
  parent_t.slice(slice_indices, slice_size) = element_t.reshape(slice_size);
  return Status::OK();
}

namespace {

template <int NDIMS>
Status HandleElementToLargerSliceWithRank(const Tensor& element, Tensor* parent,
                                          int index) {
#define HANDLE_TYPE(T)                                                   \
  case DataTypeToEnum<T>::value: {                                       \
    return HandleElementToLargerSlice<T, NDIMS>(element, parent, index); \
  }

  switch (element.dtype()) {
    TF_CALL_ALL_TYPES(HANDLE_TYPE);
#undef HANDLE_TYPE
    default:
      return errors::Unimplemented(
          "HandleElementToLargerSliceWithRank Unhandled data type: ",
          element.dtype());
  }
}

}  // namespace

Status PaddingFIFOQueue::CopyElementToLargerSlice(const Tensor& element,
                                                  Tensor* parent, int index) {
  if (parent->dims() != element.dims() + 1) {
    return errors::Internal(
        "Mismatched ranks.  Element's rank is: ", element.dims(),
        " but element is meant to be a slice in output Tensor having rank: ",
        parent->dims(), " (should be: ", element.dims() + 1, ")");
  }

#define HANDLE_DIMS(NDIMS)                                                  \
  case NDIMS: {                                                             \
    TF_RETURN_IF_ERROR(                                                     \
        HandleElementToLargerSliceWithRank<NDIMS>(element, parent, index)); \
    return Status::OK();                                                    \
  }

  switch (element.dims()) {
    HANDLE_DIMS(0);
    HANDLE_DIMS(1);
    HANDLE_DIMS(2);
    HANDLE_DIMS(3);
    HANDLE_DIMS(4);
#undef HANDLE_DIMS
    default:
      return errors::Unimplemented("CopyElementToLargerSlice Unhandled rank: ",
                                   element.dims());
  }
}

// Static method
Status PaddingFIFOQueue::SetElementZero(Tensor* element) {
#define HANDLE_TYPE(T)                                \
  if (element->dtype() == DataTypeToEnum<T>::value) { \
    element->flat<T>().setConstant(T());              \
    return Status::OK();                              \
  }
  TF_CALL_ALL_TYPES(HANDLE_TYPE);
#undef HANDLE_TYPE
  return errors::Unimplemented("SetElementZero Unhandled data type: ",
                               element->dtype());
}

std::vector<TensorShape> PaddingFIFOQueue::ConvertShapesPartialDimensionsToZero(
    const gtl::ArraySlice<PartialTensorShape>& partial_shapes) {
  std::vector<TensorShape> shapes(partial_shapes.size());
  for (int i = 0; i < shapes.size(); ++i) {
    const PartialTensorShape& partial = partial_shapes[i];
    TensorShape& shape = shapes[i];
    for (int64 s : partial.dim_sizes()) shape.AddDim(s < 0 ? 0 : s);
  }
  return shapes;
}

}  // namespace tensorflow
<commit_msg>Change: 112640197<commit_after>/* Copyright 2015 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// See docs in ../ops/data_flow_ops.cc.

#include <deque>
#include <vector>

#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/padding_fifo_queue.h"
#include "tensorflow/core/kernels/queue_base.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/port.h"
#include "tensorflow/core/public/tensor.h"
#include "tensorflow/core/public/tensor_shape.h"

namespace tensorflow {

PaddingFIFOQueue::PaddingFIFOQueue(
    int capacity, const DataTypeVector& component_dtypes,
    const std::vector<PartialTensorShape>& partial_shapes, const string& name)
    : FIFOQueue(capacity, component_dtypes,
                ConvertShapesPartialDimensionsToZero(partial_shapes), name),
      partial_shapes_(partial_shapes) {}

Status PaddingFIFOQueue::Initialize() {
  Status s = FIFOQueue::Initialize();
  if (!s.ok()) return s;

  if (component_dtypes_.size() != partial_shapes_.size()) {
    return errors::InvalidArgument(
        "Shapes must be provided for all components, but received ",
        component_dtypes_.size(), " dtypes and ", partial_shapes_.size(),
        " shapes.");
  }

  return Status::OK();
}

/* static */
Status PaddingFIFOQueue::GetElementComponent(
    const PaddingFIFOQueue::Tuple& tuple, int component, OpKernelContext* ctx,
    PersistentTensor* out_tensor) {
  TensorShape element_shape(tuple[component].shape());
  Tensor* element_access = nullptr;
  TF_RETURN_IF_ERROR(ctx->allocate_persistent(
      tuple[component].dtype(), element_shape, out_tensor, &element_access));
  *element_access = tuple[component];
  return Status::OK();
}

void PaddingFIFOQueue::TryDequeueMany(int num_elements, OpKernelContext* ctx,
                                      CallbackWithTuple callback) {
  if (num_elements == 0) {
    Tuple tuple;
    tuple.reserve(num_components());
    for (int i = 0; i < num_components(); ++i) {
      // TODO(josh11b,misard): Switch to allocate_output().
      // See similar comment in fifo_queue.cc
      Tensor element;
      // Here, ManyOutShape returns zeros for undetermined shapes,
      // which is exactly what we want to use.
      ctx->allocate_temp(component_dtypes_[i], ManyOutShape(i, 0), &element);
      tuple.emplace_back(element);
    }
    callback(tuple);
    return;
  }

  CancellationManager* cm = ctx->cancellation_manager();
  CancellationToken token = cm->get_cancellation_token();
  bool already_cancelled;
  {
    mutex_lock l(mu_);
    already_cancelled = !cm->RegisterCallback(
        token, [this, token]() { Cancel(kDequeue, token); });
    if (!already_cancelled) {
      // TODO(josh11b): This makes two copies of callback, avoid this if possible.
      dequeue_attempts_.emplace_back(
          num_elements, [callback]() { callback(Tuple()); }, ctx, token,
          [callback, this](Attempt* attempt) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
            int32 s = queues_[0].size();
            if (closed_ && s < attempt->elements_requested) {
              attempt->context->SetStatus(errors::OutOfRange(
                  "PaddingFIFOQueue '", name_, "' is closed and has ",
                  "insufficient elements (requested ",
                  attempt->elements_requested, ", current size ", s, ")"));

              // TODO(mrry): Add support for producing a partial batch as
              // output when the queue is closed.
              if (!attempt->tuples.empty()) {
                // Restore already-dequeued elements to the front of the queue.
                for (int64 i = attempt->tuples.size() - 1; i >= 0; --i) {
                  for (int j = 0; j < num_components(); ++j) {
                    PersistentTensor element;
                    Status s = GetElementComponent(attempt->tuples[i], j,
                                                   attempt->context, &element);
                    if (!s.ok()) {
                      attempt->context->SetStatus(
                          errors::DataLoss("Failed to restore element from "
                                           "partially-dequeued batch "
                                           "to PaddingFIFOQueue: ",
                                           s.error_message()));
                    }
                    queues_[j].push_front(element);
                  }
                }
              }
              return kComplete;
            }

            RunResult result = kNoProgress;
            for (; s > 0; --s) {
              result = kProgress;
              Tuple tuple;
              DequeueLocked(attempt->context, &tuple);
              attempt->tuples.push_back(tuple);
              tuple.clear();
              --attempt->elements_requested;

              if (attempt->elements_requested == 0) {
                // Finished.  Allocate attempt->tuple and
                // copy from attempt->tuples to attempt->tuple.
                attempt->tuple.reserve(num_components());
                const std::vector<Tuple>& tuples = attempt->tuples;

                std::vector<bool> dynamic_shape;
                const int64 batch_size = tuples.size();

                for (int i = 0; i < num_components(); ++i) {
                  const PartialTensorShape partial_shape =
                      PartialTensorShape({batch_size})
                          .Concatenate(partial_shapes_[i]);
                  TensorShape shape({batch_size});

                  for (int j = 0; j < partial_shape.dims() - 1; ++j) {
                    if (partial_shape.dim_size(j + 1) > -1) {
                      shape.AddDim(partial_shape.dim_size(j + 1));
                    } else {
                      // Expand sizes to match.
                      int64 max_val = 0;
                      for (const Tuple& t : tuples) {
                        max_val = std::max(max_val, t[i].shape().dim_size(j));
                      }
                      shape.AddDim(max_val);
                    }
                  }

                  Tensor element;
                  attempt->context->allocate_temp(component_dtypes_[i], shape,
                                                  &element);

                  bool has_dynamic_shape = !partial_shape.IsFullyDefined();
                  if (has_dynamic_shape) {
                    // Set all values to zero because not all values
                    // will get written over.
                    attempt->context->SetStatus(SetElementZero(&element));
                    if (!attempt->context->status().ok()) return kComplete;
                  }

                  dynamic_shape.push_back(has_dynamic_shape);

                  // TODO(ebrevdo): should this be a persistent tensor?
                  attempt->tuple.emplace_back(element);
                }

                for (int index = 0; index < tuples.size(); ++index) {
                  for (int i = 0; i < num_components(); ++i) {
                    if (dynamic_shape[i]) {
                      // Slightly slower copy operation
                      attempt->context->SetStatus(CopyElementToLargerSlice(
                          tuples[index][i], &attempt->tuple[i], index));
                    } else {
                      attempt->context->SetStatus(CopyElementToSlice(
                          tuples[index][i], &attempt->tuple[i], index));
                    }
                    if (!attempt->context->status().ok()) return kComplete;
                  }
                }
                tuple = attempt->tuple;
                attempt->tuples.clear();
                attempt->done_callback = [callback, tuple]() {
                  callback(tuple);
                };
                return kComplete;
              }
            }
            return result;
          });
    }
  }
  if (!already_cancelled) {
    FlushUnlocked();
  } else {
    ctx->SetStatus(errors::Cancelled("Dequeue operation was cancelled"));
    callback(Tuple());
  }
}

Status PaddingFIFOQueue::ValidateTuple(const Tuple& tuple) {
  TF_RETURN_IF_ERROR(ValidateTupleCommon(tuple));
  for (size_t i = 0; i < tuple.size(); ++i) {
    if (!partial_shapes_[i].IsCompatibleWith(tuple[i].shape())) {
      return errors::InvalidArgument("Shape mismatch in tuple component ", i,
                                     ". Expected ",
                                     partial_shapes_[i].DebugString(), ", got ",
                                     tuple[i].shape().ShortDebugString());
    }
  }
  return Status::OK();
}

Status PaddingFIFOQueue::ValidateManyTuple(const Tuple& tuple) {
  TF_RETURN_IF_ERROR(ValidateTupleCommon(tuple));
  const int64 batch_size = tuple[0].dim_size(0);
  for (size_t i = 0; i < tuple.size(); ++i) {
    // Expected shape is [batch_size] + partial_shapes_[i]
    const PartialTensorShape expected_shape =
        PartialTensorShape({batch_size}).Concatenate(partial_shapes_[i]);
    if (!expected_shape.IsCompatibleWith(tuple[i].shape())) {
      return errors::InvalidArgument("Shape mismatch in tuple component ", i,
                                     ". Expected ",
                                     expected_shape.DebugString(), ", got ",
                                     tuple[i].shape().ShortDebugString());
    }
  }
  return Status::OK();
}

Status PaddingFIFOQueue::CompatibleNodeDefShapes(
    const NodeDef& node_def) const {
  std::vector<PartialTensorShape> requested_shapes;
  TF_RETURN_IF_ERROR(GetNodeAttr(node_def, "shapes", &requested_shapes));
  if (!PartialTensorShapeUtils::AreCompatible(requested_shapes,
                                              partial_shapes_)) {
    return errors::InvalidArgument(
        "Shared queue '", name_, "' has component shapes ",
        PartialTensorShapeUtils::PartialShapeListString(partial_shapes_),
        " but requested component shapes were ",
        PartialTensorShapeUtils::PartialShapeListString(requested_shapes));
  } else {
    return Status::OK();
  }
}

Status PaddingFIFOQueue::MatchesNodeDef(const NodeDef& node_def) {
  TF_RETURN_IF_ERROR(MatchesNodeDefOp(node_def, "PaddingFIFOQueue"));
  TF_RETURN_IF_ERROR(MatchesNodeDefCapacity(node_def, capacity_));
  TF_RETURN_IF_ERROR(MatchesNodeDefTypes(node_def));
  TF_RETURN_IF_ERROR(CompatibleNodeDefShapes(node_def));
  return Status::OK();
}

template <typename T, int NDIMS>
Status HandleElementToLargerSlice(const Tensor& element, Tensor* parent,
                                  int index) {
  DCHECK_NE(parent->dim_size(0), 0);
  if (element.NumElements() > (parent->NumElements() / parent->dim_size(0))) {
    TensorShape chip_shape = parent->shape();
    chip_shape.RemoveDim(0);
    return errors::Internal(
        "HandleElementToLargerSlice Cannot copy slice: number of entries in "
        "element is greater than number of elements in parent slice.  ",
        "Shapes are: [element]: ", element.shape().DebugString(),
        ", [parent slice]: ", chip_shape.DebugString());
  }
  auto element_t = element.tensor<T, NDIMS>();
  auto parent_t = parent->tensor<T, NDIMS + 1>();
  Eigen::DSizes<Eigen::DenseIndex, NDIMS + 1> slice_indices;
  slice_indices[0] = index;
  Eigen::DSizes<Eigen::DenseIndex, NDIMS + 1> slice_size;
  slice_size[0] = 1;
  for (int i = 1; i < slice_size.size(); ++i) {
    slice_size[i] = element_t.dimension(i - 1);
  }
  parent_t.slice(slice_indices, slice_size) = element_t.reshape(slice_size);
  return Status::OK();
}

namespace {

template <int NDIMS>
Status HandleElementToLargerSliceWithRank(const Tensor& element, Tensor* parent,
                                          int index) {
#define HANDLE_TYPE(T)                                                   \
  case DataTypeToEnum<T>::value: {                                       \
    return HandleElementToLargerSlice<T, NDIMS>(element, parent, index); \
  }

  switch (element.dtype()) {
    TF_CALL_ALL_TYPES(HANDLE_TYPE);
#undef HANDLE_TYPE
    default:
      return errors::Unimplemented(
          "HandleElementToLargerSliceWithRank Unhandled data type: ",
          element.dtype());
  }
}

}  // namespace

Status PaddingFIFOQueue::CopyElementToLargerSlice(const Tensor& element,
                                                  Tensor* parent, int index) {
  if (parent->dims() != element.dims() + 1) {
    return errors::Internal(
        "Mismatched ranks.  Element's rank is: ", element.dims(),
        " but element is meant to be a slice in output Tensor having rank: ",
        parent->dims(), " (should be: ", element.dims() + 1, ")");
  }

#define HANDLE_DIMS(NDIMS)                                                  \
  case NDIMS: {                                                             \
    TF_RETURN_IF_ERROR(                                                     \
        HandleElementToLargerSliceWithRank<NDIMS>(element, parent, index)); \
    return Status::OK();                                                    \
  }

  switch (element.dims()) {
    HANDLE_DIMS(0);
    HANDLE_DIMS(1);
    HANDLE_DIMS(2);
    HANDLE_DIMS(3);
    HANDLE_DIMS(4);
#undef HANDLE_DIMS
    default:
      return errors::Unimplemented("CopyElementToLargerSlice Unhandled rank: ",
                                   element.dims());
  }
}

// Static method
Status PaddingFIFOQueue::SetElementZero(Tensor* element) {
#define HANDLE_TYPE(T)                                \
  if (element->dtype() == DataTypeToEnum<T>::value) { \
    element->flat<T>().setConstant(T());              \
    return Status::OK();                              \
  }
  TF_CALL_ALL_TYPES(HANDLE_TYPE);
#undef HANDLE_TYPE
  return errors::Unimplemented("SetElementZero Unhandled data type: ",
                               element->dtype());
}

std::vector<TensorShape> PaddingFIFOQueue::ConvertShapesPartialDimensionsToZero(
    const gtl::ArraySlice<PartialTensorShape>& partial_shapes) {
  std::vector<TensorShape> shapes(partial_shapes.size());
  for (int i = 0; i < shapes.size(); ++i) {
    const PartialTensorShape& partial = partial_shapes[i];
    TensorShape& shape = shapes[i];
    for (int64 s : partial.dim_sizes()) shape.AddDim(s < 0 ? 0 : s);
  }
  return shapes;
}

}  // namespace tensorflow
<|endoftext|>


<commit_before>
#include <iostream>
#include <boost/test/included/unit_test.hpp>
#include "protocol_module_control.h"
//#include "../../src/protocol_module_control.cpp"

#define	PM1	"test1"
#define	PM2	"test2"

class	l7vs::protocol_module_control;

using namespace boost::unit_test;

//test case1.
void	protocol_module_control_test(){
	l7vs::protocol_module_base*		protomod = NULL;
	l7vs::protocol_module_control& control = l7vs::protocol_module_control::getInstance();
	//call load_module before initialize
	try{
		protomod = control.load_module( PM1 );
	}
//	catch( l7vs::module_control_error& err ){
//		std::cout << err.message << std::endl;
//	}
	catch(...){
	}

	//test initialize and finalize
	control.initialize( "./" );
	control.finalize();

}

test_suite*	init_unit_test_suite( int argc, char* argv[] ){

	// create unit test suite
	test_suite* ts = BOOST_TEST_SUITE( "protocol_module_control" );
	ts->add( BOOST_TEST_CASE( &protocol_module_control_test ) );

	framework::master_test_suite().add( ts );

	return 0;
}

<commit_msg>テスト項目追加<commit_after>
#include <iostream>
#include <boost/test/included/unit_test.hpp>
#include "protocol_module_control.h"
//#include "../../src/protocol_module_control.cpp"

#define	PM1	"test1"
#define	PM2	"test2"

class	l7vs::protocol_module_control;

using namespace boost::unit_test;

//test case1.
void	protocol_module_control_test(){
	//============================================
	//protocol_module_control
	//1
	//getInstanceメソッドのテスト
	//getInstanceによって取得したcontrolとcontrol_2のインスタンスが同一であることを確認する。
	//============================================
	l7vs::protocol_module_control& control = l7vs::protocol_module_control::getInstance();
	l7vs::protocol_module_control& control_2 = l7vs::protocol_module_control::getInstance();

	BOOST_CHECK_EQUAL( &control, &control_2 );

	//test initialize and finalize
	control.initialize( "./" );
	control.finalize();

	//============================================
	//protocol_module_control
	//2
	//load_moduleメソッドのテスト(正常系その１)
	//ProtocolModuleクラスのインスタンスが取得できること
	//例外が発生しないこと
	//指定したモジュール名と取得したモジュール名が同じこと
	//============================================
	control.initialize( "./" );
	l7vs::protocol_module_base*		protomod = NULL;
	try{
		protomod = control.load_module( PM1 );
	}
	catch(...){
		BOOST_ERROR( "exception : load_module" );
	}
	BOOST_CHECK( NULL != protomod );
//	BOOST_CHECK_EQUAL( PM1, protomod.get_name() );

}

//test case2
void	protocol_module_control_test_thread(){

}

test_suite*	init_unit_test_suite( int argc, char* argv[] ){

	// create unit test suite
	test_suite* ts = BOOST_TEST_SUITE( "protocol_module_control" );

	// add test case to test suite
	ts->add( BOOST_TEST_CASE( &protocol_module_control_test ) );
	ts->add( BOOST_TEST_CASE( &protocol_module_control_test_thread ) );

	framework::master_test_suite().add( ts );

	return 0;
}

<|endoftext|>


<commit_before>/*
 * Copyright 2017, Andrej Kislovskij
 *
 * This is PUBLIC DOMAIN software so use at your own risk as it comes
 * with no warranties. This code is yours to share, use and modify without
 * any restrictions or obligations.
 *
 * For more information see conwrap/LICENSE or refer refer to http://unlicense.org
 *
 * Author: gimesketvirtadieni at gmail dot com (Andrej Kislovskij)
 */

#pragma once

#include <algorithm>
#include <atomic>
#include <chrono>
#include <conwrap/ProcessorProxy.hpp>
#include <cstddef>  // std::size_t
#include <functional>
#include <memory>
#include <optional>
#include <sstream>  // std::stringstream
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>

#include "slim/Chunk.hpp"
#include "slim/Consumer.hpp"
#include "slim/ContainerBase.hpp"
#include "slim/EncoderBuilder.hpp"
#include "slim/Exception.hpp"
#include "slim/log/log.hpp"
#include "slim/proto/Command.hpp"
#include "slim/proto/CommandSession.hpp"
#include "slim/proto/StreamingSession.hpp"

namespace slim
{
	namespace proto
	{
		template<typename ConnectionType>
		class Streamer : public Consumer
		{
			template<typename SessionType>
			using SessionsMap          = std::unordered_map<ConnectionType*, std::unique_ptr<SessionType>>;
			using TimePoint            = std::chrono::time_point<std::chrono::steady_clock>;
			using CommandSessionType   = CommandSession<ConnectionType>;
			using StreamingSessionType = StreamingSession<ConnectionType>;

			public:
				Streamer(unsigned int sp, EncoderBuilder eb, std::optional<unsigned int> g)
				: streamingPort{sp}
				, encoderBuilder{eb}
				, gain{g}
				, monitorThread{[&]
				{
					LOG(DEBUG) << LABELS{"proto"} << "Monitor thread was started (id=" << std::this_thread::get_id() << ")";

					for(unsigned int counter{0}; !monitorFinish; counter++, std::this_thread::sleep_for(std::chrono::milliseconds{200}))
			        {
						// TODO: make configurable
						if (counter > 24)
						{
							auto processorProxyPtr{getProcessorProxy()};
							if (processorProxyPtr)
							{
								processorProxyPtr->process([&]
								{
									// sending ping command to measure round-trip latency
									for (auto& entry : commandSessions)
									{
										entry.second->ping();
									}
								});
							}

							counter = 0;
						}
			        }

					LOG(DEBUG) << LABELS{"proto"} << "Monitor thread was stopped (id=" << std::this_thread::get_id() << ")";
				}}
				{
					LOG(DEBUG) << LABELS{"proto"} << "Streamer object was created (id=" << this << ")";
				}

				virtual ~Streamer()
				{
					// stopping monitor thread
					monitorFinish = true;
					if (monitorThread.joinable())
					{
						monitorThread.join();
					}

					LOG(DEBUG) << LABELS{"proto"} << "Streamer object was deleted (id=" << this << ")";
				}

				// TODO: non-movable is required due to usage in server callbacks; consider refactoring
				Streamer(const Streamer&) = delete;             // non-copyable
				Streamer& operator=(const Streamer&) = delete;  // non-assignable
				Streamer(Streamer&& rhs) = delete;              // non-movable
				Streamer& operator=(Streamer&& rhs) = delete;   // non-movable-assinable

				virtual bool consumeChunk(Chunk& chunk) override
				{
					auto result{false};
					auto chunkSamplingRate{chunk.getSamplingRate()};

					if (samplingRate != chunkSamplingRate)
					{
						if (samplingRate)
						{
							// propogating end-of-stream to all the clients
							stop();

							// changing state to not streaming
							streaming = false;
						}

						// assigning new sampling rate
						samplingRate = chunkSamplingRate;

						// if this is the beginning of a stream
						if (chunkSamplingRate)
						{
							start();
						}
					}
					else if (samplingRate)
					{
						if (!streaming)
						{
							// checking if all conditions for streaming were met
							streaming = isReadyToStream();
						}
						if (streaming)
						{
							// sending out Chunk to all the clients
							distributeChunk(chunk);

							// it will make Chunk to be consumed
							result = true;
						}
					}
					else
					{
						// consuming Chunk in case when samplingRate == chunkSamplingRate == 0
						result = true;
					}

					return result;
				}

				void onHTTPClose(ConnectionType& connection)
				{
					LOG(INFO) << LABELS{"proto"} << "HTTP close callback (connection=" << &connection << ")";

					auto streamingSessionPtr{removeSession(streamingSessions, connection)};
					if (streamingSessionPtr)
					{
						// if there is a relevant SlimProto connection found
						auto clientID{streamingSessionPtr->getClientID()};
						auto commandSession{findSessionByID(commandSessions, clientID)};
						if (commandSession.has_value())
						{
							// resetting HTTP session in its relevant SlimProto session
							commandSession.value()->setStreamingSession(nullptr);
						}
						else
						{
							LOG(WARNING) << LABELS{"proto"} << "Could not find SlimProto session by client ID (clientID=" << clientID << ")";
						}
					}
					else
					{
						LOG(WARNING) << LABELS{"proto"} << "Could not find HTTP session object";
					}
				}

				void onHTTPData(ConnectionType& connection, unsigned char* buffer, std::size_t receivedSize)
				{
					if (!applyToSession(streamingSessions, connection, [&](StreamingSessionType& session)
					{
						// processing request by a proper Streaming session mapped to this connection
						session.onRequest(buffer, receivedSize);
					}))
					{
						// parsing client ID
						auto clientID = StreamingSessionType::parseClientID(std::string{(char*)buffer, receivedSize});
						if (!clientID.has_value())
						{
							throw slim::Exception("Missing client ID in streaming session request");
						}

						LOG(INFO) << LABELS{"proto"} << "Client ID was parsed (clientID=" << clientID.value() << ")";

						// configuring an encoder builder
						encoderBuilder.setSamplingRate(samplingRate);
						encoderBuilder.setWriter(&connection);

						// creating streaming session object
						auto streamingSessionPtr{std::make_unique<StreamingSessionType>(std::ref<ConnectionType>(connection), std::move(encoderBuilder.build()), samplingRate, clientID.value())};

						// saving Streaming session reference in the relevant Command session
						auto commandSessionPtr{findSessionByID(commandSessions, clientID.value())};
						if (!commandSessionPtr.has_value())
						{
							throw slim::Exception("Could not correlate provided client ID with a valid SlimProto session");
						}
						commandSessionPtr.value()->setStreamingSession(streamingSessionPtr.get());

						// processing request by a proper Streaming session mapped to this connection
						streamingSessionPtr->onRequest(buffer, receivedSize);

						// saving Streaming session as a part of this Streamer
						addSession(streamingSessions, connection, std::move(streamingSessionPtr));
					}
				}

				void onHTTPOpen(ConnectionType& connection)
				{
					LOG(INFO) << LABELS{"proto"} << "New HTTP session request received (connection=" << &connection << ")";
				}

				void onSlimProtoClose(ConnectionType& connection)
				{
					removeSession(commandSessions, connection);
				}

				void onSlimProtoData(ConnectionType& connection, unsigned char* buffer, std::size_t size)
				{
					try
					{
						if (!applyToSession(commandSessions, connection, [&](CommandSessionType& session)
						{
							session.onRequest(buffer, size);
						}))
						{
							throw slim::Exception("Could not find SlimProto session object");
						}
					}
					catch (const slim::Exception& error)
					{
						LOG(ERROR) << LABELS{"proto"} << "Error while processing SlimProto command: " << error.what();
						connection.stop();
					}
				}

				void onSlimProtoOpen(ConnectionType& connection)
				{
					// using regular counter for session ID's instead of MAC's; it allows running multiple players on one host
					std::stringstream ss;
					ss << (++nextID);

					// creating command session object
					auto commandSessionPtr{std::make_unique<CommandSessionType>(std::ref<ConnectionType>(connection), ss.str(), streamingPort, encoderBuilder.getFormat(), gain)};

					// enable streaming for this session if required
					if (streaming)
					{
						// TODO: this is temporary solution
						commandSessionPtr->setSamplingRate(samplingRate);
						commandSessionPtr->start();
					}

					// saving command session in the map
					addSession(commandSessions, connection, std::move(commandSessionPtr));
				}

				virtual void start() override
				{
					for (auto& entry : commandSessions)
					{
						// TODO: this is a temporary solution
						entry.second->setSamplingRate(samplingRate);
						entry.second->start();
					}

					startedAt = std::chrono::steady_clock::now();
				}

				virtual void stop(bool gracefully = true) override
				{
					// stopping SlimProto session will send end-of-stream command which normally triggers close of HTTP connection
					// TODO: implement 'safety' logic to close HTTP connection in case client does not
					for (auto& entry : commandSessions)
					{
						entry.second->stop();
					}
				}

			protected:
				template<typename SessionType>
				inline auto& addSession(SessionsMap<SessionType>& sessions, ConnectionType& connection, std::unique_ptr<SessionType> sessionPtr)
				{
					auto         found{sessions.find(&connection)};
					SessionType* s{nullptr};

					if (found == sessions.end())
					{
						s = sessionPtr.get();

						// saving session in a map; using pointer to a relevant connection as an ID
						sessions[&connection] = std::move(sessionPtr);
						LOG(DEBUG) << LABELS{"proto"} << "New session was added (id=" << s << ", sessions=" << sessions.size() << ")";
					}
					else
					{
						s = (*found).second.get();
						LOG(WARNING) << LABELS{"proto"} << "Session already exists";
					}

					return *s;
				}

				template<typename SessionType, typename FunctionType>
				inline bool applyToSession(SessionsMap<SessionType>& sessions, ConnectionType& connection, FunctionType fun)
				{
					auto found{sessions.find(&connection)};

					if (found != sessions.end())
					{
						fun(*(*found).second);
					}

					return (found != sessions.end());
				}

				inline void distributeChunk(Chunk& chunk)
				{
					// sending chunk to all HTTP sessions
					auto counter{commandSessions.size()};
					for (auto& entry : commandSessions)
					{
						auto streamingSessionPtr{entry.second->getStreamingSession()};
						if (streamingSessionPtr)
						{
							streamingSessionPtr->onChunk(chunk);
							counter--;
						}
					}

					// if there are command sessions without relevant HTTP session
					if (counter > 0)
					{
						LOG(WARNING) << LABELS{"proto"} << "A chunk was not delivered to all clients (clients=" << commandSessions.size() << ", skipped=" << counter << ", size=" << chunk.getSize() << ")";
					}
					else
					{
						LOG(DEBUG) << LABELS{"proto"} << "A chunk was delivered (clients=" << commandSessions.size() << ", size=" << chunk.getSize() << ")";
					}
				}

				template<typename SessionType>
				auto findSessionByID(SessionsMap<SessionType>& sessions,  std::string clientID)
				{
					auto result{std::optional<SessionType*>{std::nullopt}};
					auto found{std::find_if(sessions.begin(), sessions.end(), [&](auto& entry) -> bool
					{
						auto found{false};

						if (entry.second->getClientID() == clientID)
						{
							found = true;
						}

						return found;
					})};

					if (found != sessions.end())
					{
						result = (*found).second.get();
					}

					return result;
				}

				inline auto isReadyToStream()
				{
					// TODO: deferring time-out should be configurable
					auto result{500 < std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startedAt).count()};
					auto missingSessionsTotal{std::count_if(commandSessions.begin(), commandSessions.end(), [&](auto& entry)
					{
						auto streamingSessionPtr{entry.second->getStreamingSession()};

						return !(streamingSessionPtr && samplingRate == streamingSessionPtr->getSamplingRate());
					})};

					// if deferring time-out has expired
					if (result && missingSessionsTotal)
					{
						LOG(WARNING) << LABELS{"proto"} << "Could not defer chunk processing due to reached threashold";
					}
					else if (!result)
					{
						// if all HTTP sessions were established
						if (!missingSessionsTotal)
						{
							result = true;
						}
						else
						{
							LOG(DEBUG) << LABELS{"proto"} << "Deferring chunk transmition due to missing HTTP sessions";

							// TODO: implement cruise control; for now sleep is good enough
							// this sleep prevents from busy spinning until all HTTP sessions reconnect
							std::this_thread::sleep_for(std::chrono::milliseconds{20});
						}
					}

					return result;
				}

				template<typename SessionType>
				inline auto removeSession(SessionsMap<SessionType>& sessions, ConnectionType& connection)
				{
					std::unique_ptr<SessionType> sessionPtr{};
					auto                         found{sessions.find(&connection)};

					if (found != sessions.end())
					{
						sessionPtr = std::move((*found).second);
						sessions.erase(found);
						LOG(DEBUG) << LABELS{"proto"} << "Session was removed (id=" << sessionPtr.get() << ", sessions=" << sessions.size() << ")";
					}

					return std::move(sessionPtr);
				}

			private:
				unsigned int                      streamingPort;
				EncoderBuilder                    encoderBuilder;
				std::optional<unsigned int>       gain;
				SessionsMap<CommandSessionType>   commandSessions;
				SessionsMap<StreamingSessionType> streamingSessions;
				bool                              streaming{false};
				unsigned int                      samplingRate{0};
				unsigned long                     nextID{0};
				std::atomic<bool>                 monitorFinish{false};
				std::thread                       monitorThread;
				TimePoint                         startedAt;
		};
	}
}
<commit_msg>Minor changes<commit_after>/*
 * Copyright 2017, Andrej Kislovskij
 *
 * This is PUBLIC DOMAIN software so use at your own risk as it comes
 * with no warranties. This code is yours to share, use and modify without
 * any restrictions or obligations.
 *
 * For more information see conwrap/LICENSE or refer refer to http://unlicense.org
 *
 * Author: gimesketvirtadieni at gmail dot com (Andrej Kislovskij)
 */

#pragma once

#include <algorithm>
#include <atomic>
#include <chrono>
#include <conwrap/ProcessorProxy.hpp>
#include <cstddef>  // std::size_t
#include <functional>
#include <memory>
#include <optional>
#include <sstream>  // std::stringstream
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>

#include "slim/Chunk.hpp"
#include "slim/Consumer.hpp"
#include "slim/ContainerBase.hpp"
#include "slim/EncoderBuilder.hpp"
#include "slim/Exception.hpp"
#include "slim/log/log.hpp"
#include "slim/proto/Command.hpp"
#include "slim/proto/CommandSession.hpp"
#include "slim/proto/StreamingSession.hpp"

namespace slim
{
	namespace proto
	{
		template<typename ConnectionType>
		class Streamer : public Consumer
		{
			template<typename SessionType>
			using SessionsMap          = std::unordered_map<ConnectionType*, std::unique_ptr<SessionType>>;
			using TimePoint            = std::chrono::time_point<std::chrono::steady_clock>;
			using CommandSessionType   = CommandSession<ConnectionType>;
			using StreamingSessionType = StreamingSession<ConnectionType>;

			public:
				Streamer(unsigned int sp, EncoderBuilder eb, std::optional<unsigned int> g)
				: streamingPort{sp}
				, encoderBuilder{eb}
				, gain{g}
				, monitorThread{[&]
				{
					LOG(DEBUG) << LABELS{"proto"} << "Monitor thread was started (id=" << std::this_thread::get_id() << ")";

					for(unsigned int counter{0}; !monitorFinish; counter++, std::this_thread::sleep_for(std::chrono::milliseconds{200}))
			        {
						// TODO: make configurable
						if (counter > 24)
						{
							auto processorProxyPtr{getProcessorProxy()};
							if (processorProxyPtr)
							{
								processorProxyPtr->process([&]
								{
									// sending ping command to measure round-trip latency
									for (auto& entry : commandSessions)
									{
										entry.second->ping();
									}
								});
							}

							counter = 0;
						}
			        }

					LOG(DEBUG) << LABELS{"proto"} << "Monitor thread was stopped (id=" << std::this_thread::get_id() << ")";
				}}
				{
					LOG(DEBUG) << LABELS{"proto"} << "Streamer object was created (id=" << this << ")";
				}

				virtual ~Streamer()
				{
					// stopping monitor thread
					monitorFinish = true;
					if (monitorThread.joinable())
					{
						monitorThread.join();
					}

					LOG(DEBUG) << LABELS{"proto"} << "Streamer object was deleted (id=" << this << ")";
				}

				// TODO: non-movable is required due to usage in server callbacks; consider refactoring
				Streamer(const Streamer&) = delete;             // non-copyable
				Streamer& operator=(const Streamer&) = delete;  // non-assignable
				Streamer(Streamer&& rhs) = delete;              // non-movable
				Streamer& operator=(Streamer&& rhs) = delete;   // non-movable-assinable

				virtual bool consumeChunk(Chunk& chunk) override
				{
					auto result{false};
					auto chunkSamplingRate{chunk.getSamplingRate()};

					if (samplingRate != chunkSamplingRate)
					{
						if (samplingRate)
						{
							// propogating end-of-stream to all the clients
							stop();

							// changing state to not streaming
							streaming = false;
						}

						// assigning new sampling rate
						samplingRate = chunkSamplingRate;

						// if this is the beginning of a stream
						if (chunkSamplingRate)
						{
							start();
						}
					}
					else if (samplingRate)
					{
						if (!streaming)
						{
							// checking if all conditions for streaming were met
							streaming = isReadyToStream();
						}
						if (streaming)
						{
							// sending out Chunk to all the clients
							distributeChunk(chunk);

							// it will make Chunk to be consumed
							result = true;
						}
					}
					else
					{
						// consuming Chunk in case when samplingRate == chunkSamplingRate == 0
						result = true;
					}

					return result;
				}

				void onHTTPClose(ConnectionType& connection)
				{
					LOG(INFO) << LABELS{"proto"} << "HTTP close callback (connection=" << &connection << ")";

					auto streamingSessionPtr{removeSession(streamingSessions, connection)};
					if (streamingSessionPtr)
					{
						// if there is a relevant SlimProto connection found
						auto clientID{streamingSessionPtr->getClientID()};
						auto commandSession{findSessionByID(commandSessions, clientID)};
						if (commandSession.has_value())
						{
							// resetting HTTP session in its relevant SlimProto session
							commandSession.value()->setStreamingSession(nullptr);
						}
						else
						{
							LOG(WARNING) << LABELS{"proto"} << "Could not find SlimProto session by client ID (clientID=" << clientID << ")";
						}
					}
					else
					{
						LOG(WARNING) << LABELS{"proto"} << "Could not find HTTP session object";
					}
				}

				void onHTTPData(ConnectionType& connection, unsigned char* buffer, std::size_t receivedSize)
				{
					if (!applyToSession(streamingSessions, connection, [&](StreamingSessionType& session)
					{
						// processing request by a proper Streaming session mapped to this connection
						session.onRequest(buffer, receivedSize);
					}))
					{
						// parsing client ID
						auto clientID = StreamingSessionType::parseClientID(std::string{(char*)buffer, receivedSize});
						if (!clientID.has_value())
						{
							throw slim::Exception("Missing client ID in streaming session request");
						}

						LOG(INFO) << LABELS{"proto"} << "Client ID was parsed (clientID=" << clientID.value() << ")";

						// configuring an encoder builder
						encoderBuilder.setSamplingRate(samplingRate);
						encoderBuilder.setWriter(&connection);

						// creating streaming session object
						auto streamingSessionPtr{std::make_unique<StreamingSessionType>(std::ref<ConnectionType>(connection), std::move(encoderBuilder.build()), samplingRate, clientID.value())};

						// saving Streaming session reference in the relevant Command session
						auto commandSessionPtr{findSessionByID(commandSessions, clientID.value())};
						if (!commandSessionPtr.has_value())
						{
							throw slim::Exception("Could not correlate provided client ID with a valid SlimProto session");
						}
						commandSessionPtr.value()->setStreamingSession(streamingSessionPtr.get());

						// processing request by a proper Streaming session mapped to this connection
						streamingSessionPtr->onRequest(buffer, receivedSize);

						// saving Streaming session as a part of this Streamer
						addSession(streamingSessions, connection, std::move(streamingSessionPtr));
					}
				}

				void onHTTPOpen(ConnectionType& connection)
				{
					LOG(INFO) << LABELS{"proto"} << "New HTTP session request received (connection=" << &connection << ")";
				}

				void onSlimProtoClose(ConnectionType& connection)
				{
					removeSession(commandSessions, connection);
				}

				void onSlimProtoData(ConnectionType& connection, unsigned char* buffer, std::size_t size)
				{
					try
					{
						if (!applyToSession(commandSessions, connection, [&](CommandSessionType& session)
						{
							session.onRequest(buffer, size);
						}))
						{
							throw slim::Exception("Could not find SlimProto session object");
						}
					}
					catch (const slim::Exception& error)
					{
						LOG(ERROR) << LABELS{"proto"} << "Error while processing SlimProto command: " << error.what();
						connection.stop();
					}
				}

				void onSlimProtoOpen(ConnectionType& connection)
				{
					// using regular counter for session ID's instead of MAC's; it allows running multiple players on one host
					std::stringstream ss;
					ss << (++nextID);

					// creating command session object
					auto commandSessionPtr{std::make_unique<CommandSessionType>(std::ref<ConnectionType>(connection), ss.str(), streamingPort, encoderBuilder.getFormat(), gain)};

					// enable streaming for this session if required
					if (streaming)
					{
						commandSessionPtr->setSamplingRate(samplingRate);
						commandSessionPtr->start();
					}

					// saving command session in the map
					addSession(commandSessions, connection, std::move(commandSessionPtr));
				}

				virtual void start() override
				{
					for (auto& entry : commandSessions)
					{
						// TODO: this is a temporary solution
						entry.second->setSamplingRate(samplingRate);
						entry.second->start();
					}

					// saving when streaming was started - required for calculating defer time-out
					startedAt = std::chrono::steady_clock::now();
				}

				virtual void stop(bool gracefully = true) override
				{
					// stopping SlimProto session will send end-of-stream command which normally triggers close of HTTP connection
					// TODO: implement 'safety' logic to close HTTP connection in case client does not
					for (auto& entry : commandSessions)
					{
						entry.second->stop();
					}
				}

			protected:
				template<typename SessionType>
				inline auto& addSession(SessionsMap<SessionType>& sessions, ConnectionType& connection, std::unique_ptr<SessionType> sessionPtr)
				{
					auto         found{sessions.find(&connection)};
					SessionType* s{nullptr};

					if (found == sessions.end())
					{
						s = sessionPtr.get();

						// saving session in a map; using pointer to a relevant connection as an ID
						sessions[&connection] = std::move(sessionPtr);
						LOG(DEBUG) << LABELS{"proto"} << "New session was added (id=" << s << ", sessions=" << sessions.size() << ")";
					}
					else
					{
						s = (*found).second.get();
						LOG(WARNING) << LABELS{"proto"} << "Session already exists";
					}

					return *s;
				}

				template<typename SessionType, typename FunctionType>
				inline bool applyToSession(SessionsMap<SessionType>& sessions, ConnectionType& connection, FunctionType fun)
				{
					auto found{sessions.find(&connection)};

					if (found != sessions.end())
					{
						fun(*(*found).second);
					}

					return (found != sessions.end());
				}

				inline void distributeChunk(Chunk& chunk)
				{
					// sending chunk to all HTTP sessions
					auto counter{commandSessions.size()};
					for (auto& entry : commandSessions)
					{
						auto streamingSessionPtr{entry.second->getStreamingSession()};
						if (streamingSessionPtr)
						{
							streamingSessionPtr->onChunk(chunk);
							counter--;
						}
					}

					// if there are command sessions without relevant HTTP session
					if (counter > 0)
					{
						LOG(WARNING) << LABELS{"proto"} << "A chunk was not delivered to all clients (clients=" << commandSessions.size() << ", skipped=" << counter << ", size=" << chunk.getSize() << ")";
					}
					else
					{
						LOG(DEBUG) << LABELS{"proto"} << "A chunk was delivered (clients=" << commandSessions.size() << ", size=" << chunk.getSize() << ")";
					}
				}

				template<typename SessionType>
				auto findSessionByID(SessionsMap<SessionType>& sessions,  std::string clientID)
				{
					auto result{std::optional<SessionType*>{std::nullopt}};
					auto found{std::find_if(sessions.begin(), sessions.end(), [&](auto& entry) -> bool
					{
						auto found{false};

						if (entry.second->getClientID() == clientID)
						{
							found = true;
						}

						return found;
					})};

					if (found != sessions.end())
					{
						result = (*found).second.get();
					}

					return result;
				}

				inline auto isReadyToStream()
				{
					// TODO: deferring time-out should be configurable
					auto result{500 < std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startedAt).count()};
					auto missingSessionsTotal{std::count_if(commandSessions.begin(), commandSessions.end(), [&](auto& entry)
					{
						auto streamingSessionPtr{entry.second->getStreamingSession()};

						return !(streamingSessionPtr && samplingRate == streamingSessionPtr->getSamplingRate());
					})};

					// if deferring time-out has expired
					if (result && missingSessionsTotal)
					{
						LOG(WARNING) << LABELS{"proto"} << "Could not defer chunk processing due to reached threashold";
					}
					else if (!result)
					{
						// if all HTTP sessions were established
						if (!missingSessionsTotal)
						{
							result = true;
						}
						else
						{
							LOG(DEBUG) << LABELS{"proto"} << "Deferring chunk transmition due to missing HTTP sessions";

							// TODO: implement cruise control; for now sleep is good enough
							// this sleep prevents from busy spinning until all HTTP sessions reconnect
							std::this_thread::sleep_for(std::chrono::milliseconds{20});
						}
					}

					return result;
				}

				template<typename SessionType>
				inline auto removeSession(SessionsMap<SessionType>& sessions, ConnectionType& connection)
				{
					std::unique_ptr<SessionType> sessionPtr{};
					auto                         found{sessions.find(&connection)};

					if (found != sessions.end())
					{
						sessionPtr = std::move((*found).second);
						sessions.erase(found);
						LOG(DEBUG) << LABELS{"proto"} << "Session was removed (id=" << sessionPtr.get() << ", sessions=" << sessions.size() << ")";
					}

					return std::move(sessionPtr);
				}

			private:
				unsigned int                      streamingPort;
				EncoderBuilder                    encoderBuilder;
				std::optional<unsigned int>       gain;
				SessionsMap<CommandSessionType>   commandSessions;
				SessionsMap<StreamingSessionType> streamingSessions;
				bool                              streaming{false};
				unsigned int                      samplingRate{0};
				unsigned long                     nextID{0};
				std::atomic<bool>                 monitorFinish{false};
				std::thread                       monitorThread;
				TimePoint                         startedAt;
		};
	}
}
<|endoftext|>


<commit_before>#pragma once
#include <eos/chain/block.hpp>
#include <eos/chain/types.hpp>

namespace eos {
   using namespace chain;
   using namespace fc;

   struct handshake_message {
      int16_t         network_version = 0;
      chain_id_type   chain_id; ///< used to identify chain
      fc::sha256      node_id; ///< used to identify peers and prevent self-connect
      string          p2p_address;
      uint32_t        last_irreversible_block_num = 0;
      block_id_type   last_irreversible_block_id;
      uint32_t        head_num = 0;
      block_id_type   head_id;
      string          os;
      string          agent;
   };

   struct notice_message {
      vector<transaction_id_type> known_trx;
      vector<block_id_type>       known_blocks;
   };

   struct request_message {
      vector<transaction_id_type> req_trx;
      vector<block_id_type>       req_blocks;
   };

   struct block_summary_message {
      signed_block                block;
      vector<transaction_id_type> trx_ids;
   };

   struct sync_request_message {
      uint32_t start_block;
      uint32_t end_block;
   };

   struct peer_message {
     vector<fc::ip::endpoint> peers;
   };

   using net_message = static_variant<handshake_message,
                                      peer_message,
                                      notice_message,
                                      request_message,
                                      sync_request_message,
                                      block_summary_message,
                                      SignedTransaction,
                                      signed_block>;

} // namespace eos

FC_REFLECT( eos::handshake_message,
            (network_version)(chain_id)(node_id)
            (p2p_address)
            (last_irreversible_block_num)(last_irreversible_block_id)
            (head_num)(head_id)
            (os)(agent) )

FC_REFLECT( eos::block_summary_message, (block)(trx_ids) )
FC_REFLECT( eos::notice_message, (known_trx)(known_blocks) )
FC_REFLECT( eos::request_message, (req_trx)(req_blocks) )
FC_REFLECT( eos::sync_request_message, (start_block)(end_block) )
FC_REFLECT( eos::peer_message, (peers) )

/**
 *
Goals of Network Code
1. low latency to minimize missed blocks and potentially reduce block interval
2. minimize redundant data between blocks and transactions.
3. enable rapid sync of a new node
4. update to new boost / fc

State:
   All nodes know which blocks and transactions they have
   All nodes know which blocks and transactions their peers have
   A node knows which blocks and transactions it has requested
   All nodes know when they learned of a transaction

   send hello message
   write loop (true)
      if peer knows the last irreversible block {
         if peer does not know you know a block or transactions
            send the ids you know (so they don't send it to you)
            yield continue
         if peer does not know about a block
            send transactions in block peer doesn't know then send block summary
            yield continue
         if peer does not know about new public endpoints that you have verified
            relay new endpoints to peer
            yield continue
         if peer does not know about transactions
            sends the oldest transactions that is not known by the remote peer
            yield continue
         wait for new validated block, transaction, or peer signal from network fiber
      } else {
         we assume peer is in sync mode in which case it is operating on a
         request / response basis

         wait for notice of sync from the read loop
      }

    read loop
      if hello message
         verify that peers Last Ir Block is in our state or disconnect, they are on fork
         verify peer network protocol

      if notice message update list of transactions known by remote peer
      if trx message then insert into global state as unvalidated
      if blk summary message then insert into global state *if* we know of all dependent transactions
         else close connection

    if my head block < the LIB of a peer and my head block age > block interval * round_size/2 then
    enter sync mode...
        divide the block numbers you need to fetch among peers and send fetch request
        if peer does not respond to request in a timely manner then make request to another peer
        ensure that there is a constant queue of requests in flight and everytime a request is filled
        send of another request.

     Once you have caught up to all peers, notify all peers of your head block so they know that you
     know the LIB and will start sending you real time tranasctions

parallel fetches, request in groups

only relay transactions to peers if we don't already know about it.

send a notification rather than a transaaction if the txn is > 3mtu size.

*/
<commit_msg>Update protocol.hpp<commit_after>#pragma once
#include <eos/chain/block.hpp>
#include <eos/chain/types.hpp>

namespace eos {
   using namespace chain;
   using namespace fc;

   struct handshake_message {
      int16_t         network_version = 0;
      chain_id_type   chain_id; ///< used to identify chain
      fc::sha256      node_id; ///< used to identify peers and prevent self-connect
      string          p2p_address;
      uint32_t        last_irreversible_block_num = 0;
      block_id_type   last_irreversible_block_id;
      uint32_t        head_num = 0;
      block_id_type   head_id;
      string          os;
      string          agent;
   };

   struct notice_message {
      vector<transaction_id_type> known_trx;
      vector<block_id_type>       known_blocks;
   };

   struct request_message {
      vector<transaction_id_type> req_trx;
      vector<block_id_type>       req_blocks;
   };

   struct block_summary_message {
      signed_block                block;
      vector<transaction_id_type> trx_ids;
   };

   struct sync_request_message {
      uint32_t start_block;
      uint32_t end_block;
   };

   struct peer_message {
     vector<fc::ip::endpoint> peers;
   };

   using net_message = static_variant<handshake_message,
                                      peer_message,
                                      notice_message,
                                      request_message,
                                      sync_request_message,
                                      block_summary_message,
                                      SignedTransaction,
                                      signed_block>;

} // namespace eos

FC_REFLECT( eos::handshake_message,
            (network_version)(chain_id)(node_id)
            (p2p_address)
            (last_irreversible_block_num)(last_irreversible_block_id)
            (head_num)(head_id)
            (os)(agent) )

FC_REFLECT( eos::block_summary_message, (block)(trx_ids) )
FC_REFLECT( eos::notice_message, (known_trx)(known_blocks) )
FC_REFLECT( eos::request_message, (req_trx)(req_blocks) )
FC_REFLECT( eos::sync_request_message, (start_block)(end_block) )
FC_REFLECT( eos::peer_message, (peers) )

/**
 *
Goals of Network Code
1. low latency to minimize missed blocks and potentially reduce block interval
2. minimize redundant data between blocks and transactions.
3. enable rapid sync of a new node
4. update to new boost / fc

State:
   All nodes know which blocks and transactions they have
   All nodes know which blocks and transactions their peers have
   A node knows which blocks and transactions it has requested
   All nodes know when they learned of a transaction

   send hello message
   write loop (true)
      if peer knows the last irreversible block {
         if peer does not know you know a block or transactions
            send the ids you know (so they don't send it to you)
            yield continue
         if peer does not know about a block
            send transactions in block peer doesn't know then send block summary
            yield continue
         if peer does not know about new public endpoints that you have verified
            relay new endpoints to peer
            yield continue
         if peer does not know about transactions
            sends the oldest transactions that is not known by the remote peer
            yield continue
         wait for new validated block, transaction, or peer signal from network fiber
      } else {
         we assume peer is in sync mode in which case it is operating on a
         request / response basis

         wait for notice of sync from the read loop
      }

    read loop
      if hello message
         verify that peers Last Ir Block is in our state or disconnect, they are on fork
         verify peer network protocol

      if notice message update list of transactions known by remote peer
      if trx message then insert into global state as unvalidated
      if blk summary message then insert into global state *if* we know of all dependent transactions
         else close connection

    if my head block < the LIB of a peer and my head block age > block interval * round_size/2 then
    enter sync mode...
        divide the block numbers you need to fetch among peers and send fetch request
        if peer does not respond to request in a timely manner then make request to another peer
        ensure that there is a constant queue of requests in flight and everytime a request is filled
        send of another request.

     Once you have caught up to all peers, notify all peers of your head block so they know that you
     know the LIB and will start sending you real time tranasctions

parallel fetches, request in groups

only relay transactions to peers if we don't already know about it.

send a notification rather than a transaction if the txn is > 3mtu size.

*/
<|endoftext|>