<commit_before>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/dlpack/dlpack_converter.h"

namespace onnxruntime {
namespace dlpack {

namespace {

DLDataType GetDlpackDataType(const OrtValue& ort_value) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only tensor-type OrtValues are supported");
  DLDataType dtype;
  dtype.lanes = 1;
  const Tensor& tensor = ort_value.Get<Tensor>();
  switch (tensor.GetElementType()) {
    case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(double);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(float);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT8:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int8_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT16:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int16_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT32:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT64:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int64_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(MLFloat16);
      break;
    // Currently bool is same as uint8 on both code and bits.
    // PyTorch's to_dlpack also does this, but in from_dlpack,
    // a torch.uint8 tensor is generated. This limitation from
    // PyTorch means we cannot create a torch.bool tensor
    // from a DLPack tensor.
    case ONNX_NAMESPACE::TensorProto_DataType_BOOL:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(bool);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint8_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT16:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint16_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT32:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint32_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT64:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint64_t);
      break;
    default:
      ORT_THROW("Unexpected data type of ", tensor.GetElementType());
  }

  dtype.bits *= 8;  // bits.
  return dtype;
}

DLContext GetDlpackContext(const OrtValue& ort_value, const int64_t& device_id) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only OrtValues that are Tensors are currently supported");
  DLContext ctx;
  ctx.device_id = static_cast<int>(device_id);
  const Tensor& tensor = ort_value.Get<Tensor>();
  const auto& location = tensor.Location();
  switch (location.device.Type()) {
    case OrtDevice::CPU:
      ctx.device_type = DLDeviceType::kDLCPU;
      break;
    case OrtDevice::GPU:
#ifdef USE_ROCM
      ctx.device_type = DLDeviceType::kDLROCM;
#else
      ctx.device_type = DLDeviceType::kDLGPU;
#endif
      break;
    default:
      ORT_THROW("Cannot pack tensors on this device.");
  }

  return ctx;
}

struct OrtDLManagedTensor {
  OrtValue handle;
  DLManagedTensor tensor;
};

void DlpackDeleter(DLManagedTensor* arg) { delete static_cast<OrtDLManagedTensor*>(arg->manager_ctx); }

OrtDevice GetOrtDevice(const DLContext& ctx) {
  switch (ctx.device_type) {
    case DLDeviceType::kDLCPU:
      return OrtDevice();
    case DLDeviceType::kDLGPU:
    case DLDeviceType::kDLROCM:
      return OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, static_cast<OrtDevice::DeviceId>(ctx.device_id));
    default:
      ORT_THROW("Unsupported device type");
  }
}

MLDataType GetOrtValueDataType(const DLDataType& dtype, bool is_bool_tensor) {
  if (dtype.lanes != 1) ORT_THROW("ORT does not support lanes != 1");
  switch (dtype.code) {
    case DLDataTypeCode::kDLUInt:
      switch (dtype.bits) {
        case 8:
          return is_bool_tensor ? DataTypeImpl::GetType<bool>() : DataTypeImpl::GetType<uint8_t>();
        case 16:
          return DataTypeImpl::GetType<uint16_t>();
        case 32:
          return DataTypeImpl::GetType<uint32_t>();
        case 64:
          return DataTypeImpl::GetType<uint64_t>();
        default:
          ORT_THROW("Unsupported kUInt bits " + std::to_string(dtype.bits));
      }
    case DLDataTypeCode::kDLInt:
      switch (dtype.bits) {
        case 8:
          return DataTypeImpl::GetType<int8_t>();
        case 16:
          return DataTypeImpl::GetType<int16_t>();
        case 32:
          return DataTypeImpl::GetType<int32_t>();
        case 64:
          return DataTypeImpl::GetType<int64_t>();
        default:
          ORT_THROW("Unsupported kInt bits " + std::to_string(dtype.bits));
      }
    case DLDataTypeCode::kDLFloat:
      switch (dtype.bits) {
        case 16:
          return DataTypeImpl::GetType<MLFloat16>();
        case 32:
          return DataTypeImpl::GetType<float>();
        case 64:
          return DataTypeImpl::GetType<double>();
        default:
          ORT_THROW("Unsupported kFloat bits " + std::to_string(dtype.bits));
      }
    default:
      ORT_THROW("Unsupported code " + std::to_string(dtype.code));
  }
}

const char* GetOrtDeviceName(const OrtDevice& device) {
  switch (device.Type()) {
    case OrtDevice::CPU:
      return CPU;
    case OrtDevice::GPU:
      return CUDA;
    default:
      ORT_THROW("Unknown device type: ", device.Type());
  }
}

bool IsContiguousTensor(const DLTensor& tensor) {
  if (!tensor.strides) {
    return true;
  }

  int64_t running_size = 1;
  for (int i = tensor.ndim - 1; i >= 0; i--) {
    if (tensor.shape[i] == 0) {
      return true;
    }

    if (tensor.shape[i] != 1 && tensor.strides[i] != running_size) {
      return false;
    }

    running_size *= tensor.shape[i];
  }

  return true;
}

}  // namespace

// This function returns a pointer to DLManagedTensor constructed from an OrtValue
// The OrtValue inside OrtDLManagedTensor will increase its own buffer's ref count by one
// When the consumer of DLManagedTensor is done with the tensor, it should invoke the deleter.
DLManagedTensor* OrtValueToDlpack(OrtValue& ort_value) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only tensor type OrtValues are supported");
  OrtDLManagedTensor* ort_dlmanaged_tensor(new OrtDLManagedTensor);
  Tensor& tensor = *ort_value.GetMutable<Tensor>();
  ort_dlmanaged_tensor->handle = ort_value;
  ort_dlmanaged_tensor->tensor.manager_ctx = ort_dlmanaged_tensor;
  ort_dlmanaged_tensor->tensor.deleter = &DlpackDeleter;
  ort_dlmanaged_tensor->tensor.dl_tensor.data = (tensor.MutableDataRaw());
  ort_dlmanaged_tensor->tensor.dl_tensor.ctx = GetDlpackContext(ort_value, tensor.Location().device.Id());
  ort_dlmanaged_tensor->tensor.dl_tensor.ndim = static_cast<int>(tensor.Shape().NumDimensions());
  ort_dlmanaged_tensor->tensor.dl_tensor.dtype = GetDlpackDataType(ort_value);
  ort_dlmanaged_tensor->tensor.dl_tensor.shape =
      tensor.Shape().NumDimensions() > 0 ? const_cast<int64_t*>(&tensor.Shape()[0]) : nullptr;
  ort_dlmanaged_tensor->tensor.dl_tensor.strides = nullptr;
  ort_dlmanaged_tensor->tensor.dl_tensor.byte_offset = 0;
  return &(ort_dlmanaged_tensor->tensor);
}

OrtValue DlpackToOrtValue(DLManagedTensor* dlpack, bool is_bool_tensor) {
  // ORT only supports contiguous tensor for now.
  ORT_ENFORCE(IsContiguousTensor(dlpack->dl_tensor), "ORT only supports contiguous tensor for now.");
  OrtDevice device = GetOrtDevice(dlpack->dl_tensor.ctx);
  MLDataType data_type = GetOrtValueDataType(dlpack->dl_tensor.dtype, is_bool_tensor);
  std::function<void(void*)> deleter = [dlpack](void*) { dlpack->deleter((dlpack)); };
  OrtMemoryInfo info(GetOrtDeviceName(device), OrtDeviceAllocator, device, device.Id());
  std::unique_ptr<Tensor> p_tensor = std::make_unique<Tensor>(
      data_type, TensorShape(dlpack->dl_tensor.shape, static_cast<size_t>(dlpack->dl_tensor.ndim)),
      dlpack->dl_tensor.data, info);

  OrtValue ort_value;
  ort_value.Init(p_tensor.release(), DataTypeImpl::GetType<Tensor>(), deleter);
  return ort_value;
}

}  // namespace dlpack
}  // namespace onnxruntime
<commit_msg>Fix Memory Leak from DlpackToOrtValue (#8029)<commit_after>// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "core/dlpack/dlpack_converter.h"

namespace onnxruntime {
namespace dlpack {

namespace {

DLDataType GetDlpackDataType(const OrtValue& ort_value) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only tensor-type OrtValues are supported");
  DLDataType dtype;
  dtype.lanes = 1;
  const Tensor& tensor = ort_value.Get<Tensor>();
  switch (tensor.GetElementType()) {
    case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(double);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(float);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT8:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int8_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT16:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int16_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT32:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_INT64:
      dtype.code = DLDataTypeCode::kDLInt;
      dtype.bits = sizeof(int64_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
      dtype.code = DLDataTypeCode::kDLFloat;
      dtype.bits = sizeof(MLFloat16);
      break;
    // Currently bool is same as uint8 on both code and bits.
    // PyTorch's to_dlpack also does this, but in from_dlpack,
    // a torch.uint8 tensor is generated. This limitation from
    // PyTorch means we cannot create a torch.bool tensor
    // from a DLPack tensor.
    case ONNX_NAMESPACE::TensorProto_DataType_BOOL:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(bool);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint8_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT16:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint16_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT32:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint32_t);
      break;
    case ONNX_NAMESPACE::TensorProto_DataType_UINT64:
      dtype.code = DLDataTypeCode::kDLUInt;
      dtype.bits = sizeof(uint64_t);
      break;
    default:
      ORT_THROW("Unexpected data type of ", tensor.GetElementType());
  }

  dtype.bits *= 8;  // bits.
  return dtype;
}

DLContext GetDlpackContext(const OrtValue& ort_value, const int64_t& device_id) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only OrtValues that are Tensors are currently supported");
  DLContext ctx;
  ctx.device_id = static_cast<int>(device_id);
  const Tensor& tensor = ort_value.Get<Tensor>();
  const auto& location = tensor.Location();
  switch (location.device.Type()) {
    case OrtDevice::CPU:
      ctx.device_type = DLDeviceType::kDLCPU;
      break;
    case OrtDevice::GPU:
#ifdef USE_ROCM
      ctx.device_type = DLDeviceType::kDLROCM;
#else
      ctx.device_type = DLDeviceType::kDLGPU;
#endif
      break;
    default:
      ORT_THROW("Cannot pack tensors on this device.");
  }

  return ctx;
}

struct OrtDLManagedTensor {
  OrtValue handle;
  DLManagedTensor tensor;
};

void DlpackDeleter(DLManagedTensor* arg) { delete static_cast<OrtDLManagedTensor*>(arg->manager_ctx); }

OrtDevice GetOrtDevice(const DLContext& ctx) {
  switch (ctx.device_type) {
    case DLDeviceType::kDLCPU:
      return OrtDevice();
    case DLDeviceType::kDLGPU:
    case DLDeviceType::kDLROCM:
      return OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, static_cast<OrtDevice::DeviceId>(ctx.device_id));
    default:
      ORT_THROW("Unsupported device type");
  }
}

MLDataType GetOrtValueDataType(const DLDataType& dtype, bool is_bool_tensor) {
  if (dtype.lanes != 1) ORT_THROW("ORT does not support lanes != 1");
  switch (dtype.code) {
    case DLDataTypeCode::kDLUInt:
      switch (dtype.bits) {
        case 8:
          return is_bool_tensor ? DataTypeImpl::GetType<bool>() : DataTypeImpl::GetType<uint8_t>();
        case 16:
          return DataTypeImpl::GetType<uint16_t>();
        case 32:
          return DataTypeImpl::GetType<uint32_t>();
        case 64:
          return DataTypeImpl::GetType<uint64_t>();
        default:
          ORT_THROW("Unsupported kUInt bits " + std::to_string(dtype.bits));
      }
    case DLDataTypeCode::kDLInt:
      switch (dtype.bits) {
        case 8:
          return DataTypeImpl::GetType<int8_t>();
        case 16:
          return DataTypeImpl::GetType<int16_t>();
        case 32:
          return DataTypeImpl::GetType<int32_t>();
        case 64:
          return DataTypeImpl::GetType<int64_t>();
        default:
          ORT_THROW("Unsupported kInt bits " + std::to_string(dtype.bits));
      }
    case DLDataTypeCode::kDLFloat:
      switch (dtype.bits) {
        case 16:
          return DataTypeImpl::GetType<MLFloat16>();
        case 32:
          return DataTypeImpl::GetType<float>();
        case 64:
          return DataTypeImpl::GetType<double>();
        default:
          ORT_THROW("Unsupported kFloat bits " + std::to_string(dtype.bits));
      }
    default:
      ORT_THROW("Unsupported code " + std::to_string(dtype.code));
  }
}

const char* GetOrtDeviceName(const OrtDevice& device) {
  switch (device.Type()) {
    case OrtDevice::CPU:
      return CPU;
    case OrtDevice::GPU:
      return CUDA;
    default:
      ORT_THROW("Unknown device type: ", device.Type());
  }
}

bool IsContiguousTensor(const DLTensor& tensor) {
  if (!tensor.strides) {
    return true;
  }

  int64_t running_size = 1;
  for (int i = tensor.ndim - 1; i >= 0; i--) {
    if (tensor.shape[i] == 0) {
      return true;
    }

    if (tensor.shape[i] != 1 && tensor.strides[i] != running_size) {
      return false;
    }

    running_size *= tensor.shape[i];
  }

  return true;
}

}  // namespace

// This function returns a pointer to DLManagedTensor constructed from an OrtValue
// The OrtValue inside OrtDLManagedTensor will increase its own buffer's ref count by one
// When the consumer of DLManagedTensor is done with the tensor, it should invoke the deleter.
DLManagedTensor* OrtValueToDlpack(OrtValue& ort_value) {
  ORT_ENFORCE(ort_value.IsTensor(), "Only tensor type OrtValues are supported");
  OrtDLManagedTensor* ort_dlmanaged_tensor(new OrtDLManagedTensor);
  Tensor& tensor = *ort_value.GetMutable<Tensor>();
  ort_dlmanaged_tensor->handle = ort_value;
  ort_dlmanaged_tensor->tensor.manager_ctx = ort_dlmanaged_tensor;
  ort_dlmanaged_tensor->tensor.deleter = &DlpackDeleter;
  ort_dlmanaged_tensor->tensor.dl_tensor.data = (tensor.MutableDataRaw());
  ort_dlmanaged_tensor->tensor.dl_tensor.ctx = GetDlpackContext(ort_value, tensor.Location().device.Id());
  ort_dlmanaged_tensor->tensor.dl_tensor.ndim = static_cast<int>(tensor.Shape().NumDimensions());
  ort_dlmanaged_tensor->tensor.dl_tensor.dtype = GetDlpackDataType(ort_value);
  ort_dlmanaged_tensor->tensor.dl_tensor.shape =
      tensor.Shape().NumDimensions() > 0 ? const_cast<int64_t*>(&tensor.Shape()[0]) : nullptr;
  ort_dlmanaged_tensor->tensor.dl_tensor.strides = nullptr;
  ort_dlmanaged_tensor->tensor.dl_tensor.byte_offset = 0;
  return &(ort_dlmanaged_tensor->tensor);
}

OrtValue DlpackToOrtValue(DLManagedTensor* dlpack, bool is_bool_tensor) {
  // ORT only supports contiguous tensor for now.
  ORT_ENFORCE(IsContiguousTensor(dlpack->dl_tensor), "ORT only supports contiguous tensor for now.");
  OrtDevice device = GetOrtDevice(dlpack->dl_tensor.ctx);
  MLDataType data_type = GetOrtValueDataType(dlpack->dl_tensor.dtype, is_bool_tensor);
  OrtMemoryInfo info(GetOrtDeviceName(device), OrtDeviceAllocator, device, device.Id());
  std::unique_ptr<Tensor> p_tensor = std::make_unique<Tensor>(
      data_type, TensorShape(dlpack->dl_tensor.shape, static_cast<size_t>(dlpack->dl_tensor.ndim)),
      dlpack->dl_tensor.data, info);

  OrtValue ort_value;
  std::function<void(void*)> deleter = [dlpack](void* p) {
    dlpack->deleter(dlpack);
    DataTypeImpl::GetType<Tensor>()->GetDeleteFunc()(p);
  };

  ort_value.Init(p_tensor.release(), DataTypeImpl::GetType<Tensor>(), deleter);
  return ort_value;
}

}  // namespace dlpack
}  // namespace onnxruntime
<|endoftext|>


<commit_before>/*
 * SQLUUID.cc
 * TLB & UUID management.
 *
 * Copyright (c) 2008,2009,2013,2017 Linas Vepstas <linas@linas.org>
 *
 * LICENSE:
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include <stdlib.h>
#include <unistd.h>

#include <opencog/atoms/base/Atom.h>
#include <opencog/atomspace/AtomSpace.h>
#include <opencog/atomspaceutils/TLB.h>

#include "SQLAtomStorage.h"
#include "SQLResponse.h"

using namespace opencog;

/* ================================================================== */

void SQLAtomStorage::registerWith(AtomSpace* as)
{
	_tlbuf.set_resolver(&as->get_atomtable());

#ifdef NOT_NEEDED_RIGHT_NOW
	// The goal here is to avoid cluttering the TLB with lots of
	// extra junk that has been removed from the atomspace. This
	// can happen in several ways:
	//
	// 1) User code adds atoms to the atomspace, saves them to the
	//	database, then deletes them from the atomspace.  If this is
	//	done, then pointers to those atoms will continue on, here in
	//	the TLB, chewing up RAM.
	// 2) The above happens unintentionally, due to a bug in the code.
	//
	// The callback just deletes stuff not in the atomspace, as, chances
	// are, they'll never be used again.
	//
	_extract_sig = as->atomRemovedAtomSignal().connect(
		std::bind(&SQLAtomStorage::extract_callback, this,
			std::placeholders::_1));
#endif // NOT_NEEDED_RIGHT_NOW
}

void SQLAtomStorage::unregisterWith(AtomSpace* as)
{
	flushStoreQueue();
	_tlbuf.clear_resolver(&as->get_atomtable());

#ifdef NOT_NEEDED_RIGHT_NOW
	_extract_sig.disconnect();
#endif // NOT_NEEDED_RIGHT_NOW
}

void SQLAtomStorage::extract_callback(const AtomPtr& atom)
{
	_tlbuf.removeAtom(atom);
}

/* ================================================================== */
/// Return the UUID of the handle, if it is known.
/// If the handle is in the database, then the correct UUID is returned.
/// If the handle is NOT in the database, this returns the invalid UUID.
UUID SQLAtomStorage::check_uuid(const Handle& h)
{
	UUID uuid = _tlbuf.getUUID(h);
	if (TLB::INVALID_UUID != uuid) return uuid;

	// Optimize for bulk stores. That is, we know for a fact that
	// the database cannot possibly contain this atom yet, so do
	// not query for it!
	if (bulk_store) return TLB::INVALID_UUID;

	// Ooops. We need to look in the database to find out what this is.
	Handle dbh;
	if (h->is_node())
	{
		dbh = doGetNode(h->get_type(), h->get_name().c_str());
	}
	else
	{
		dbh = doGetLink(h->get_type(), h->getOutgoingSet());
	}
	// If it was found in the database, then the TLB got updated.
	if (dbh) return _tlbuf.getUUID(h);

	// If it was not found in the database, then say so.
	return TLB::INVALID_UUID;
}

/// Return the UUID of the handle, if it is known, else throw exception.
/// If the handle is in the database, then the correct UUID is returned.
/// If the handle is NOT in the database, this throws a silent exception.
UUID SQLAtomStorage::get_uuid(const Handle& h)
{
	UUID uuid = check_uuid(h);
	if (TLB::INVALID_UUID != uuid) return uuid;

	// Throw a silent exception; don't clutter log-files with this!
	throw NotFoundException(TRACE_INFO, "");

	return TLB::INVALID_UUID;
}

/// Add an atom to the TLB.
void SQLAtomStorage::add_atom(const Handle& h, UUID uuid)
{
	OC_ASSERT(TLB::INVALID_UUID != uuid);
	_tlbuf.addAtom(h, uuid);
}

/// Issue a new UUID and put the atom to the TLB (under that UUID)
UUID SQLAtomStorage::issue_atom(const Handle& h)
{
	UUID uuid = _next_unused_uuid.fetch_add(1);
	while (_uuid_pool_top < uuid)
	{
		refill_uuid_pool();
		uuid = _next_unused_uuid.fetch_add(1);
	}
	return _tlbuf.addAtom(h, uuid);
}

/* ================================================================ */

UUID SQLAtomStorage::getMaxObservedUUID(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT uuid FROM Atoms ORDER BY uuid DESC LIMIT 1;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	return rp.intval;
}

SQLAtomStorage::VUID SQLAtomStorage::getMaxObservedVUID(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT vuid FROM Values ORDER BY vuid DESC LIMIT 1;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	return rp.intval;
}

UUID SQLAtomStorage::reserve(void)
{
	UUID max_observed_id = getMaxObservedUUID();
	logger().debug("SQLAtomStorage::reserve(): Max observed UUID is %lu\n",
		 max_observed_id);
	_tlbuf.reserve_upto(max_observed_id);
	return max_observed_id;
}

void SQLAtomStorage::clear_cache(void)
{
	_tlbuf.clear();
	reserve();
}

/* ================================================================ */
/* Pool management */

/// Reset the uuid_pool sequence, if and only if we are the only
/// ones who are connected.  Otherwise, no reset can take place,
/// because we are not the only user, and we must respect the currently
/// issued UUID bundle.
void SQLAtomStorage::reset_uuid_pool(void)
{
	// Create the missing sequences, if they do not exist. This is
	// for backwards compatibility with older databases that do not
	// have these sequences in them.
	UUID maxuuid = getMaxObservedUUID();
	UUID maxvuid = getMaxObservedVUID();
	std::string create =
		"CREATE SEQUENCE IF NOT EXISTS uuid_pool START WITH "
		+ std::to_string(maxuuid+1) +
		" INCREMENT BY 400;"
		"CREATE SEQUENCE IF NOT EXISTS vuid_pool START WITH "
		+ std::to_string(maxvuid+1) +
		" INCREMENT BY 400;";

	Response rp(conn_pool);
	rp.exec(create.c_str());

	std::string reset =
		"DO $$"
		"DECLARE "
		"   maxuuid BIGINT;"
		"   under BIGINT;"
		"   maxvu BIGINT := 0;"
		"   vnder BIGINT;"
		"BEGIN"
		"   IF (SELECT count(*) FROM pg_stat_activity WHERE"
		"           datname=(SELECT current_database())) = "
		+ std::to_string(_initial_conn_pool_size) +
		" THEN"
		"      maxuuid := (SELECT uuid FROM Atoms ORDER BY uuid DESC LIMIT 1);"
		"      maxvu := (SELECT vuid FROM Values ORDER BY vuid DESC LIMIT 1);"
		"      under := maxuuid - (SELECT increment_by FROM uuid_pool) + 1;"
		"      vnder := maxvu - (SELECT increment_by FROM vuid_pool) + 1;"
		"      RAISE NOTICE 'Max UUID=% under=%', maxuuid, under;"
		"      RAISE NOTICE 'Max VUID=% vnder=%', maxvu, vnder;"
		"      PERFORM (SELECT setval('uuid_pool',under));"
		"      PERFORM (SELECT setval('vuid_pool',vnder));"
		"   END IF;"
		"END $$;";

	rp.exec(reset.c_str());

	rp.intval = 0;
	rp.exec("SELECT increment_by FROM uuid_pool;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	_uuid_pool_increment = rp.intval;

	rp.intval = 0;
	rp.exec("SELECT increment_by FROM vuid_pool;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	_vuid_pool_increment = rp.intval;

	// Prepare to issue UUID's
	_uuid_pool_top = getMaxObservedUUID();
	_tlbuf.reserve_upto(_uuid_pool_top);
	_next_unused_uuid = _uuid_pool_top + 1;
}

/// Obtain a block of unused UUID's from the SQL Sequence
void SQLAtomStorage::refill_uuid_pool(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT nextval('uuid_pool');");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	// top is the last one that can be issued without requiring a refill.
	_next_unused_uuid = rp.intval;
	_uuid_pool_top = rp.intval + _uuid_pool_increment - 1;
}

/* ============================= END OF FILE ================= */
<commit_msg>Avoid error on ODBC interfaces<commit_after>/*
 * SQLUUID.cc
 * TLB & UUID management.
 *
 * Copyright (c) 2008,2009,2013,2017 Linas Vepstas <linas@linas.org>
 *
 * LICENSE:
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#include <stdlib.h>
#include <unistd.h>

#include <opencog/atoms/base/Atom.h>
#include <opencog/atomspace/AtomSpace.h>
#include <opencog/atomspaceutils/TLB.h>

#include "SQLAtomStorage.h"
#include "SQLResponse.h"

using namespace opencog;

/* ================================================================== */

void SQLAtomStorage::registerWith(AtomSpace* as)
{
	_tlbuf.set_resolver(&as->get_atomtable());

#ifdef NOT_NEEDED_RIGHT_NOW
	// The goal here is to avoid cluttering the TLB with lots of
	// extra junk that has been removed from the atomspace. This
	// can happen in several ways:
	//
	// 1) User code adds atoms to the atomspace, saves them to the
	//	database, then deletes them from the atomspace.  If this is
	//	done, then pointers to those atoms will continue on, here in
	//	the TLB, chewing up RAM.
	// 2) The above happens unintentionally, due to a bug in the code.
	//
	// The callback just deletes stuff not in the atomspace, as, chances
	// are, they'll never be used again.
	//
	_extract_sig = as->atomRemovedAtomSignal().connect(
		std::bind(&SQLAtomStorage::extract_callback, this,
			std::placeholders::_1));
#endif // NOT_NEEDED_RIGHT_NOW
}

void SQLAtomStorage::unregisterWith(AtomSpace* as)
{
	flushStoreQueue();
	_tlbuf.clear_resolver(&as->get_atomtable());

#ifdef NOT_NEEDED_RIGHT_NOW
	_extract_sig.disconnect();
#endif // NOT_NEEDED_RIGHT_NOW
}

void SQLAtomStorage::extract_callback(const AtomPtr& atom)
{
	_tlbuf.removeAtom(atom);
}

/* ================================================================== */
/// Return the UUID of the handle, if it is known.
/// If the handle is in the database, then the correct UUID is returned.
/// If the handle is NOT in the database, this returns the invalid UUID.
UUID SQLAtomStorage::check_uuid(const Handle& h)
{
	UUID uuid = _tlbuf.getUUID(h);
	if (TLB::INVALID_UUID != uuid) return uuid;

	// Optimize for bulk stores. That is, we know for a fact that
	// the database cannot possibly contain this atom yet, so do
	// not query for it!
	if (bulk_store) return TLB::INVALID_UUID;

	// Ooops. We need to look in the database to find out what this is.
	Handle dbh;
	if (h->is_node())
	{
		dbh = doGetNode(h->get_type(), h->get_name().c_str());
	}
	else
	{
		dbh = doGetLink(h->get_type(), h->getOutgoingSet());
	}
	// If it was found in the database, then the TLB got updated.
	if (dbh) return _tlbuf.getUUID(h);

	// If it was not found in the database, then say so.
	return TLB::INVALID_UUID;
}

/// Return the UUID of the handle, if it is known, else throw exception.
/// If the handle is in the database, then the correct UUID is returned.
/// If the handle is NOT in the database, this throws a silent exception.
UUID SQLAtomStorage::get_uuid(const Handle& h)
{
	UUID uuid = check_uuid(h);
	if (TLB::INVALID_UUID != uuid) return uuid;

	// Throw a silent exception; don't clutter log-files with this!
	throw NotFoundException(TRACE_INFO, "");

	return TLB::INVALID_UUID;
}

/// Add an atom to the TLB.
void SQLAtomStorage::add_atom(const Handle& h, UUID uuid)
{
	OC_ASSERT(TLB::INVALID_UUID != uuid);
	_tlbuf.addAtom(h, uuid);
}

/// Issue a new UUID and put the atom to the TLB (under that UUID)
UUID SQLAtomStorage::issue_atom(const Handle& h)
{
	UUID uuid = _next_unused_uuid.fetch_add(1);
	while (_uuid_pool_top < uuid)
	{
		refill_uuid_pool();
		uuid = _next_unused_uuid.fetch_add(1);
	}
	return _tlbuf.addAtom(h, uuid);
}

/* ================================================================ */

UUID SQLAtomStorage::getMaxObservedUUID(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT uuid FROM Atoms ORDER BY uuid DESC LIMIT 1;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	return rp.intval;
}

SQLAtomStorage::VUID SQLAtomStorage::getMaxObservedVUID(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT vuid FROM Values ORDER BY vuid DESC LIMIT 1;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	return rp.intval;
}

UUID SQLAtomStorage::reserve(void)
{
	UUID max_observed_id = getMaxObservedUUID();
	logger().debug("SQLAtomStorage::reserve(): Max observed UUID is %lu\n",
		 max_observed_id);
	_tlbuf.reserve_upto(max_observed_id);
	return max_observed_id;
}

void SQLAtomStorage::clear_cache(void)
{
	_tlbuf.clear();
	reserve();
}

/* ================================================================ */
/* Pool management */

/// Reset the uuid_pool sequence, if and only if we are the only
/// ones who are connected.  Otherwise, no reset can take place,
/// because we are not the only user, and we must respect the currently
/// issued UUID bundle.
void SQLAtomStorage::reset_uuid_pool(void)
{
	// Create the missing sequences, if they do not exist. This is
	// for backwards compatibility with older databases that do not
	// have these sequences in them.
	UUID maxuuid = getMaxObservedUUID();
	UUID maxvuid = getMaxObservedVUID();
	std::string create =
		"CREATE SEQUENCE IF NOT EXISTS uuid_pool START WITH "
		+ std::to_string(maxuuid+1) +
		" INCREMENT BY 400;"
		"CREATE SEQUENCE IF NOT EXISTS vuid_pool START WITH "
		+ std::to_string(maxvuid+1) +
		" INCREMENT BY 400;";

	Response rp(conn_pool);
	rp.exec(create.c_str());

	std::string reset =
		"DO $$"
		"DECLARE "
		"   maxuuid BIGINT;"
		"   under BIGINT;"
		"   maxvu BIGINT := 0;"
		"   vnder BIGINT;"
		"BEGIN"
		"   IF (SELECT count(*) FROM pg_stat_activity WHERE"
		"           datname=(SELECT current_database())) = "
		+ std::to_string(_initial_conn_pool_size) +
		" THEN"
		"      maxuuid := (SELECT uuid FROM Atoms ORDER BY uuid DESC LIMIT 1);"
		"      maxvu := (SELECT vuid FROM Values ORDER BY vuid DESC LIMIT 1);"
		"      under := maxuuid - (SELECT increment_by FROM uuid_pool) + 1;"
		"      vnder := maxvu - (SELECT increment_by FROM vuid_pool) + 1;"
		"      RAISE NOTICE 'Max UUID=% under=%', maxuuid, under;"
		"      RAISE NOTICE 'Max VUID=% vnder=%', maxvu, vnder;"
		"      PERFORM (SELECT setval('uuid_pool',under));"
		"      PERFORM (SELECT setval('vuid_pool',vnder));"
		"   END IF;"
		"END $$;";

	// ODBC cannot run the above query, so we punt.  Mostly this
	// won't matter, except that ODBC databases will have ranges
	// unused UUID's in them.
	std::string::size_type odbc = _uri.find("odbc://");
	if (0 != odbc)
		rp.exec(reset.c_str());

	rp.intval = 0;
	rp.exec("SELECT increment_by FROM uuid_pool;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	_uuid_pool_increment = rp.intval;

	rp.intval = 0;
	rp.exec("SELECT increment_by FROM vuid_pool;");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	_vuid_pool_increment = rp.intval;

	// Prepare to issue UUID's
	_uuid_pool_top = getMaxObservedUUID();
	_tlbuf.reserve_upto(_uuid_pool_top);
	_next_unused_uuid = _uuid_pool_top + 1;
}

/// Obtain a block of unused UUID's from the SQL Sequence
void SQLAtomStorage::refill_uuid_pool(void)
{
	Response rp(conn_pool);
	rp.intval = 0;
	rp.exec("SELECT nextval('uuid_pool');");
	rp.rs->foreach_row(&Response::intval_cb, &rp);
	// top is the last one that can be issued without requiring a refill.
	_next_unused_uuid = rp.intval;
	_uuid_pool_top = rp.intval + _uuid_pool_increment - 1;
}

/* ============================= END OF FILE ================= */
<|endoftext|>


<commit_before>#include "TruthValue_wrap.h"
#include <boost/python/enum.hpp>
#include <boost/python/manage_new_object.hpp>
#include <boost/python/copy_const_reference.hpp>
#include <boost/python/class.hpp>
#include <boost/python/pure_virtual.hpp>
#include <boost/python/operators.hpp>

using namespace opencog;
using namespace boost::python;

void init_TruthValue_py()
{
    enum_<TruthValueType>("TruthValueType")
        .value("SIMPLE_TRUTH_VALUE", SIMPLE_TRUTH_VALUE)
        .value("COUNT_TRUTH_VALUE", COUNT_TRUTH_VALUE)
        .value("INDEFINITE_TRUTH_VALUE", INDEFINITE_TRUTH_VALUE)
        .value("COMPOSITE_TRUTH_VALUE", COMPOSITE_TRUTH_VALUE)
        .value("NUMBER_OF_TRUTH_VALUE_TYPES", NUMBER_OF_TRUTH_VALUE_TYPES)
    ;

    class_<TruthValueWrap, boost::noncopyable>("TruthValue")
        .def("NULL_TV", &TruthValue::NULL_TV,
            //return_value_policy<reference_existing_object>())
            return_value_policy<copy_const_reference>())
        .def("DEFAULT_TV", &TruthValue::DEFAULT_TV,
            return_value_policy<copy_const_reference>())
        .def("TRUE_TV", &TruthValue::TRUE_TV,
            return_value_policy<copy_const_reference>())
        .def("FALSE_TV", &TruthValue::FALSE_TV,
            return_value_policy<copy_const_reference>())
        .def("TRIVIAL_TV", &TruthValue::TRIVIAL_TV,
            return_value_policy<copy_const_reference>())
        /*.def("getTVTypeName", &TruthValue::getTVTypeName,
            return_value_policy<manage_new_object>())*/
        .def("getMean", pure_virtual(&TruthValue::getMean))
        .def("getCount", pure_virtual(&TruthValue::getCount))
        .def("getConfidence", pure_virtual(&TruthValue::getConfidence))
        .def("toFloat", pure_virtual(&TruthValue::toFloat))
        .def("toString", pure_virtual(&TruthValue::toString))
        .def("getType", pure_virtual(&TruthValue::getType))
        .def("clone", pure_virtual(&TruthValue::clone),
            return_value_policy<manage_new_object>())
        .def(self == self)
        .def("merge", &TruthValue::merge, &TruthValueWrap::default_merge,
            return_value_policy<manage_new_object>())
        .def("isNullTv", &TruthValue::isNullTv,
            &TruthValueWrap::default_isNullTv)
        .def("typeToStr", &TruthValue::typeToStr)
        .def("strToType", &TruthValue::strToType)
        .def("factory", factoryx1,
            return_value_policy<manage_new_object>())
        .def("factory", factoryx2,
            return_value_policy<manage_new_object>())
        /* Cannot make work because this function is protected
        .def("DeleteAndSetDefaultTVIfPertinent",
            &TruthValue::DeleteAndSetDefaultTVIfPertinent)*/
        .staticmethod("NULL_TV")
        .staticmethod("DEFAULT_TV")
        .staticmethod("TRUE_TV")
        .staticmethod("FALSE_TV")
        .staticmethod("TRIVIAL_TV")
        //.staticmethod("getTVTypeName")
        .staticmethod("typeToStr")
        .staticmethod("strToType")
        .staticmethod("factory")
        //.staticmethod("DeleteAndSetDefaultTVIfPertinent")
    ;
}

// Pure virtual functions.

strength_t TruthValueWrap::getMean() const
{ return this->get_override("getMean")(); }
count_t TruthValueWrap::getCount() const
{ return this->get_override("getCount")(); }
confidence_t TruthValueWrap::getConfidence() const
{ return this->get_override("getConfidence")(); }
float TruthValueWrap::toFloat() const
{ return this->get_override("toFloat")(); }
std::string TruthValueWrap::toString() const
{ return this->get_override("toString")(); }
TruthValueType TruthValueWrap::getType() const
{ return this->get_override("getType")(); }
TruthValue* TruthValueWrap::clone() const
{ return this->get_override("clone")(); }
TruthValue& TruthValueWrap::operator=(const TruthValue& rhs)
{ return this->get_override("operator=")(); }
bool TruthValueWrap::operator==(const TruthValue& rhs) const
{ return this->get_override("operator==")(); }

// Non-pure virtual functions.

TruthValue* TruthValueWrap::merge(const TruthValue& other) const
{
    if (override merge = this->get_override("merge"))
        return merge(other);

    return TruthValue::merge(other);
}
TruthValue* TruthValueWrap::default_merge(const TruthValue& other) const
{
    return this->TruthValue::merge(other);
}

bool TruthValueWrap::isNullTv() const
{
    if (override isNullTv = this->get_override("isNullTv"))
        return isNullTv();

    return TruthValue::isNullTv();
}
bool TruthValueWrap::default_isNullTv() const
{
    return this->TruthValue::isNullTv();
}

//TruthValue* (*factoryx1)(const char*) = &TruthValue::factory;
//TruthValue* (*factoryx2)(TruthValueType, const char*) = &TruthValue::factory;
/*factoryx1 = TruthValue::factory;
factoryx2 = TruthValue::factory;*/
<commit_msg>Removed unneeded comments.<commit_after>#include "TruthValue_wrap.h"
#include <boost/python/enum.hpp>
#include <boost/python/manage_new_object.hpp>
#include <boost/python/copy_const_reference.hpp>
#include <boost/python/class.hpp>
#include <boost/python/pure_virtual.hpp>
#include <boost/python/operators.hpp>

using namespace opencog;
using namespace boost::python;

void init_TruthValue_py()
{
    enum_<TruthValueType>("TruthValueType")
        .value("SIMPLE_TRUTH_VALUE", SIMPLE_TRUTH_VALUE)
        .value("COUNT_TRUTH_VALUE", COUNT_TRUTH_VALUE)
        .value("INDEFINITE_TRUTH_VALUE", INDEFINITE_TRUTH_VALUE)
        .value("COMPOSITE_TRUTH_VALUE", COMPOSITE_TRUTH_VALUE)
        .value("NUMBER_OF_TRUTH_VALUE_TYPES", NUMBER_OF_TRUTH_VALUE_TYPES)
    ;

    class_<TruthValueWrap, boost::noncopyable>("TruthValue")
        .def("NULL_TV", &TruthValue::NULL_TV,
            //return_value_policy<reference_existing_object>())
            return_value_policy<copy_const_reference>())
        .def("DEFAULT_TV", &TruthValue::DEFAULT_TV,
            return_value_policy<copy_const_reference>())
        .def("TRUE_TV", &TruthValue::TRUE_TV,
            return_value_policy<copy_const_reference>())
        .def("FALSE_TV", &TruthValue::FALSE_TV,
            return_value_policy<copy_const_reference>())
        .def("TRIVIAL_TV", &TruthValue::TRIVIAL_TV,
            return_value_policy<copy_const_reference>())
        /*.def("getTVTypeName", &TruthValue::getTVTypeName,
            return_value_policy<manage_new_object>())*/
        .def("getMean", pure_virtual(&TruthValue::getMean))
        .def("getCount", pure_virtual(&TruthValue::getCount))
        .def("getConfidence", pure_virtual(&TruthValue::getConfidence))
        .def("toFloat", pure_virtual(&TruthValue::toFloat))
        .def("toString", pure_virtual(&TruthValue::toString))
        .def("getType", pure_virtual(&TruthValue::getType))
        .def("clone", pure_virtual(&TruthValue::clone),
            return_value_policy<manage_new_object>())
        .def(self == self)
        .def("merge", &TruthValue::merge, &TruthValueWrap::default_merge,
            return_value_policy<manage_new_object>())
        .def("isNullTv", &TruthValue::isNullTv,
            &TruthValueWrap::default_isNullTv)
        .def("typeToStr", &TruthValue::typeToStr)
        .def("strToType", &TruthValue::strToType)
        .def("factory", factoryx1,
            return_value_policy<manage_new_object>())
        .def("factory", factoryx2,
            return_value_policy<manage_new_object>())
        /* Cannot make work because this function is protected
        .def("DeleteAndSetDefaultTVIfPertinent",
            &TruthValue::DeleteAndSetDefaultTVIfPertinent)*/
        .staticmethod("NULL_TV")
        .staticmethod("DEFAULT_TV")
        .staticmethod("TRUE_TV")
        .staticmethod("FALSE_TV")
        .staticmethod("TRIVIAL_TV")
        //.staticmethod("getTVTypeName")
        .staticmethod("typeToStr")
        .staticmethod("strToType")
        .staticmethod("factory")
        //.staticmethod("DeleteAndSetDefaultTVIfPertinent")
    ;
}

// Pure virtual functions.

strength_t TruthValueWrap::getMean() const
{ return this->get_override("getMean")(); }
count_t TruthValueWrap::getCount() const
{ return this->get_override("getCount")(); }
confidence_t TruthValueWrap::getConfidence() const
{ return this->get_override("getConfidence")(); }
float TruthValueWrap::toFloat() const
{ return this->get_override("toFloat")(); }
std::string TruthValueWrap::toString() const
{ return this->get_override("toString")(); }
TruthValueType TruthValueWrap::getType() const
{ return this->get_override("getType")(); }
TruthValue* TruthValueWrap::clone() const
{ return this->get_override("clone")(); }
TruthValue& TruthValueWrap::operator=(const TruthValue& rhs)
{ return this->get_override("operator=")(); }
bool TruthValueWrap::operator==(const TruthValue& rhs) const
{ return this->get_override("operator==")(); }

// Non-pure virtual functions.

TruthValue* TruthValueWrap::merge(const TruthValue& other) const
{
    if (override merge = this->get_override("merge"))
        return merge(other);

    return TruthValue::merge(other);
}
TruthValue* TruthValueWrap::default_merge(const TruthValue& other) const
{
    return this->TruthValue::merge(other);
}

bool TruthValueWrap::isNullTv() const
{
    if (override isNullTv = this->get_override("isNullTv"))
        return isNullTv();

    return TruthValue::isNullTv();
}
bool TruthValueWrap::default_isNullTv() const
{
    return this->TruthValue::isNullTv();
}
<|endoftext|>


<commit_before>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * DataManReader.cpp
 *
 *  Created on: Feb 21, 2017
 *      Author: Jason Wang
 *              William F Godoy
 */

#include "DataManReader.h"
#include "DataManReader.tcc"

#include "adios2/ADIOSMacros.h"
#include "adios2/helper/adiosFunctions.h" //CSVToVector

namespace adios2
{

DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode,
                             MPI_Comm mpiComm)
: Engine("DataManReader", io, name, mode, mpiComm)
{
    m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n";
    Init();
}

DataManReader::~DataManReader()
{

    m_Listening = false;
    for (auto &t : m_DataThreads)
    {
        t.join();
    }
    for (auto &t : m_ControlThreads)
    {
        t.join();
    }
}

StepStatus DataManReader::BeginStep(StepMode stepMode,
                                    const float timeoutSeconds)
{

    StepStatus status;
    status = StepStatus::OK;

    status = StepStatus::EndOfStream;

    return status;
}

void DataManReader::ReadThread(std::shared_ptr<transportman::DataMan> man)
{

    if (m_UseFormat == "BP" || m_UseFormat == "bp")
    {
        format::BP3Deserializer deserializer(m_MPIComm, m_DebugMode);

        deserializer.InitParameters(m_IO.m_Parameters);

        while (m_Listening)
        {
            // TODO: This is absolutely not thread safe. If future
            // implementation needs multiple threads then this has to be
            // rewriten by manually removing variables one by one and it
            // has to be protected by mutex.

            std::shared_ptr<std::vector<char>> buffer = man->ReadWAN();
            if (buffer != nullptr)
            {
                if (buffer->size() > 0)
                {
                    deserializer.m_Data.Resize(buffer->size(),
                                               "in DataMan Streaming Listener");

                    std::memcpy(deserializer.m_Data.m_Buffer.data(),
                                buffer->data(), buffer->size());

                    m_MutexIO.lock();
                    m_IO.RemoveAllVariables();
                    m_IO.RemoveAllAttributes();
                    std::cout << "ParseMetadata ===\n";
                    deserializer.ParseMetadata(deserializer.m_Data, m_IO);
                    m_MutexIO.unlock();

                    const auto variablesInfo = m_IO.GetAvailableVariables();
                    for (const auto &variableInfoPair : variablesInfo)
                    {

                        std::string var = variableInfoPair.first;
                        std::string type = "null";

                        for (const auto &parameter : variableInfoPair.second)
                        {
                            if (parameter.first == "Type")
                            {
                                type = parameter.second;
                            }
                        }

                        if (type == "compound")
                        {
                            throw("Compound type is not supported yet.");
                        }

// for debug
/*
else if (type == GetType<float>())
{
adios2::Variable<float> *v =
m_IO.InquireVariable<float>(var);
deserializer.GetSyncVariableDataFromStream(
*v, deserializer.m_Data);
if (v->GetData() == nullptr)
{
throw("Data pointer obtained from BP "
"deserializer is anullptr");
}
else
{
for (auto &step : v->m_IndexStepBlockStarts)
{
std::shared_ptr<DataManVar> dmv =
std::make_shared<DataManVar>();
dmv->datatype = type;
dmv->shape = v->m_Shape;
dmv->start = v->m_Start;
dmv->count = v->m_Count;
dmv->data.resize(v->PayloadSize());
v->SetStepSelection({step.first - 1, 1});
deserializer.GetSyncVariableDataFromStream(
*v, deserializer.m_Data);
std::memcpy(dmv->data.data(), v->GetData(),
v->PayloadSize());
RunCallback(v->GetData(), "stream", var,
type, v->m_Shape);
m_VariableMap[step.first - 1][var] = dmv;
}
}
}
*/

#define declare_type(T)                                                        \
    else if (type == GetType<T>())                                             \
    {                                                                          \
        adios2::Variable<T> *v = m_IO.InquireVariable<T>(var);                 \
        deserializer.GetSyncVariableDataFromStream(*v, deserializer.m_Data);   \
        if (v->GetData() == nullptr)                                           \
        {                                                                      \
            throw("Data pointer obtained from BP "                             \
                  "deserializer is anullptr");                                 \
        }                                                                      \
        else                                                                   \
        {                                                                      \
            for (auto &step : v->m_IndexStepBlockStarts)                       \
            {                                                                  \
                std::shared_ptr<DataManVar> dmv =                              \
                    std::make_shared<DataManVar>();                            \
                dmv->datatype = type;                                          \
                dmv->shape = v->m_Shape;                                       \
                dmv->start = v->m_Start;                                       \
                dmv->count = v->m_Count;                                       \
                dmv->data.resize(v->PayloadSize());                            \
                v->SetStepSelection({step.first - 1, 1});                      \
                deserializer.GetSyncVariableDataFromStream(                    \
                    *v, deserializer.m_Data);                                  \
                std::memcpy(dmv->data.data(), v->GetData(), v->PayloadSize()); \
                RunCallback(v->GetData(), "stream", var, type, v->m_Shape);    \
                m_VariableMap[step.first - 1][var] = dmv;                      \
            }                                                                  \
        }                                                                      \
    }
                        ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type
                    }
                }
            }
        }
    }
}

void DataManReader::PerformGets() {}

void DataManReader::EndStep() { ++m_CurrentStep; }

// PRIVATE
bool DataManReader::GetBoolParameter(Params &params, std::string key,
                                     bool &value)
{
    auto itKey = params.find(key);
    if (itKey != params.end())
    {
        std::transform(itKey->second.begin(), itKey->second.end(),
                       itKey->second.begin(), ::tolower);
        if (itKey->second == "yes" || itKey->second == "true")
        {
            value = true;
            return true;
        }
        if (itKey->second == "no" || itKey->second == "false")
        {
            value = false;
            return true;
        }
    }
    return false;
}

bool DataManReader::GetStringParameter(Params &params, std::string key,
                                       std::string &value)
{
    auto it = params.find(key);
    if (it != params.end())
    {
        value = it->second;
        return true;
    }
    return false;
}

bool DataManReader::GetUIntParameter(Params &params, std::string key,
                                     unsigned int &value)
{
    auto it = params.find(key);
    if (it != params.end())
    {
        value = std::stoi(it->second);
        return true;
    }
    return false;
}

void DataManReader::Init()
{

    // register callbacks
    for (auto &j : m_IO.m_Operators)
    {
        if (j.ADIOSOperator.m_Type == "Signature2")
        {
            m_Callbacks.push_back(&j.ADIOSOperator);
        }
    }

    // get parameters
    GetUIntParameter(m_IO.m_Parameters, "NChannels", m_NChannels);
    GetStringParameter(m_IO.m_Parameters, "Format", m_UseFormat);

    // initialize transports

    m_DataMan = std::make_shared<transportman::DataMan>(m_MPIComm, m_DebugMode);

    size_t channels = m_IO.m_TransportsParameters.size();
    std::vector<std::string> names;
    for (size_t i = 0; i < channels; ++i)
    {
        names.push_back(m_Name + std::to_string(i));
        m_IO.m_TransportsParameters[i]["Name"] = std::to_string(i);
    }

    m_DataMan->OpenWANTransports(names, Mode::Read, m_IO.m_TransportsParameters,
                                 true);

    // start threads

    m_Listening = true;
    for (size_t i = 0; i < m_DataChannels; ++i)
    {
        m_DataThreads.emplace_back(
            std::thread(&DataManReader::ReadThread, this, m_DataMan));
    }
    for (size_t i = 0; i < m_ControlChannels; ++i)
    {
        m_ControlThreads.emplace_back(
            std::thread(&DataManReader::ReadThread, this, m_ControlMan));
    }
}

void DataManReader::RunCallback(void *buffer, std::string doid, std::string var,
                                std::string dtype, std::vector<size_t> shape)
{
    for (auto &i : m_Callbacks)
    {
        if (i != nullptr)
        {
            if (i->m_Type == "Signature2")
            {
                i->RunCallback2(buffer, doid, var, dtype, shape);
            }
        }
    }
}

#define declare_type(T)                                                        \
    void DataManReader::DoGetSync(Variable<T> &variable, T *data)              \
    {                                                                          \
        GetSyncCommon(variable, data);                                         \
    }                                                                          \
    void DataManReader::DoGetDeferred(Variable<T> &variable, T *data)          \
    {                                                                          \
        GetDeferredCommon(variable, data);                                     \
    }                                                                          \
    void DataManReader::DoGetDeferred(Variable<T> &variable, T &data)          \
    {                                                                          \
        GetDeferredCommon(variable, &data);                                    \
    }
ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type

void DataManReader::DoClose(const int transportIndex) {}

} // end namespace adios2
<commit_msg>Fixing Reader side for stream steps<commit_after>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * DataManReader.cpp
 *
 *  Created on: Feb 21, 2017
 *      Author: Jason Wang
 *              William F Godoy
 */

#include "DataManReader.h"
#include "DataManReader.tcc"

#include "adios2/ADIOSMacros.h"
#include "adios2/helper/adiosFunctions.h" //CSVToVector

namespace adios2
{

DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode,
                             MPI_Comm mpiComm)
: Engine("DataManReader", io, name, mode, mpiComm)
{
    m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n";
    Init();
}

DataManReader::~DataManReader()
{

    m_Listening = false;
    for (auto &t : m_DataThreads)
    {
        t.join();
    }
    for (auto &t : m_ControlThreads)
    {
        t.join();
    }
}

StepStatus DataManReader::BeginStep(StepMode stepMode,
                                    const float timeoutSeconds)
{

    StepStatus status;
    status = StepStatus::OK;

    status = StepStatus::EndOfStream;

    return status;
}

void DataManReader::ReadThread(std::shared_ptr<transportman::DataMan> man)
{

    if (m_UseFormat == "BP" || m_UseFormat == "bp")
    {
        format::BP3Deserializer deserializer(m_MPIComm, m_DebugMode);

        deserializer.InitParameters(m_IO.m_Parameters);

        while (m_Listening)
        {
            // TODO: This is absolutely not thread safe. If future
            // implementation needs multiple threads then this has to be
            // rewriten by manually removing variables one by one and it
            // has to be protected by mutex.

            std::shared_ptr<std::vector<char>> buffer = man->ReadWAN();
            if (buffer != nullptr)
            {
                if (buffer->size() > 0)
                {
                    deserializer.m_Data.Resize(buffer->size(),
                                               "in DataMan Streaming Listener");

                    std::memcpy(deserializer.m_Data.m_Buffer.data(),
                                buffer->data(), buffer->size());

                    m_MutexIO.lock();
                    m_IO.RemoveAllVariables();
                    m_IO.RemoveAllAttributes();
                    std::cout << "ParseMetadata ===\n";
                    deserializer.ParseMetadata(deserializer.m_Data, m_IO);
                    m_MutexIO.unlock();

                    const auto variablesInfo = m_IO.GetAvailableVariables();
                    for (const auto &variableInfoPair : variablesInfo)
                    {

                        std::string var = variableInfoPair.first;
                        std::string type = "null";

                        for (const auto &parameter : variableInfoPair.second)
                        {
                            if (parameter.first == "Type")
                            {
                                type = parameter.second;
                            }
                        }

                        if (type == "compound")
                        {
                            throw("Compound type is not supported yet.");
                        }

// for debug
#define declare_type(T)                                                        \
    else if (type == GetType<T>())                                             \
    {                                                                          \
        adios2::Variable<T> *v = m_IO.InquireVariable<T>(var);                 \
        if (v == nullptr)                                                      \
        {                                                                      \
            throw std::runtime_error("Data pointer obtained from BP "          \
                                     "deserializer is anullptr");              \
        }                                                                      \
        else                                                                   \
        {                                                                      \
            for (auto &step : v->m_IndexStepBlockStarts)                       \
            {                                                                  \
                std::shared_ptr<DataManVar> dmv =                              \
                    std::make_shared<DataManVar>();                            \
                dmv->datatype = type;                                          \
                dmv->shape = v->m_Shape;                                       \
                dmv->start = v->m_Start;                                       \
                dmv->count = v->m_Count;                                       \
                dmv->data.resize(v->PayloadSize());                            \
                v->SetStepSelection({step.first - 1, 1});                      \
                deserializer.GetSyncVariableDataFromStream(                    \
                    *v, deserializer.m_Data);                                  \
                std::memcpy(dmv->data.data(), v->GetData(), v->PayloadSize()); \
                RunCallback(v->GetData(), "stream", var, type, v->m_Shape);    \
                m_VariableMap[step.first - 1][var] = dmv;                      \
            }                                                                  \
        }                                                                      \
    }
                        ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type
                    }
                }
            }
        }
    }
}

void DataManReader::PerformGets() {}

void DataManReader::EndStep() { ++m_CurrentStep; }

// PRIVATE
bool DataManReader::GetBoolParameter(Params &params, std::string key,
                                     bool &value)
{
    auto itKey = params.find(key);
    if (itKey != params.end())
    {
        std::transform(itKey->second.begin(), itKey->second.end(),
                       itKey->second.begin(), ::tolower);
        if (itKey->second == "yes" || itKey->second == "true")
        {
            value = true;
            return true;
        }
        if (itKey->second == "no" || itKey->second == "false")
        {
            value = false;
            return true;
        }
    }
    return false;
}

bool DataManReader::GetStringParameter(Params &params, std::string key,
                                       std::string &value)
{
    auto it = params.find(key);
    if (it != params.end())
    {
        value = it->second;
        return true;
    }
    return false;
}

bool DataManReader::GetUIntParameter(Params &params, std::string key,
                                     unsigned int &value)
{
    auto it = params.find(key);
    if (it != params.end())
    {
        value = std::stoi(it->second);
        return true;
    }
    return false;
}

void DataManReader::Init()
{

    // register callbacks
    for (auto &j : m_IO.m_Operators)
    {
        if (j.ADIOSOperator.m_Type == "Signature2")
        {
            m_Callbacks.push_back(&j.ADIOSOperator);
        }
    }

    // get parameters
    GetUIntParameter(m_IO.m_Parameters, "NChannels", m_NChannels);
    GetStringParameter(m_IO.m_Parameters, "Format", m_UseFormat);

    // initialize transports

    m_DataMan = std::make_shared<transportman::DataMan>(m_MPIComm, m_DebugMode);

    size_t channels = m_IO.m_TransportsParameters.size();
    std::vector<std::string> names;
    for (size_t i = 0; i < channels; ++i)
    {
        names.push_back(m_Name + std::to_string(i));
        m_IO.m_TransportsParameters[i]["Name"] = std::to_string(i);
    }

    m_DataMan->OpenWANTransports(names, Mode::Read, m_IO.m_TransportsParameters,
                                 true);

    // start threads

    m_Listening = true;
    for (size_t i = 0; i < m_DataChannels; ++i)
    {
        m_DataThreads.emplace_back(
            std::thread(&DataManReader::ReadThread, this, m_DataMan));
    }
    for (size_t i = 0; i < m_ControlChannels; ++i)
    {
        m_ControlThreads.emplace_back(
            std::thread(&DataManReader::ReadThread, this, m_ControlMan));
    }
}

void DataManReader::RunCallback(void *buffer, std::string doid, std::string var,
                                std::string dtype, std::vector<size_t> shape)
{
    for (auto &i : m_Callbacks)
    {
        if (i != nullptr)
        {
            if (i->m_Type == "Signature2")
            {
                i->RunCallback2(buffer, doid, var, dtype, shape);
            }
        }
    }
}

#define declare_type(T)                                                        \
    void DataManReader::DoGetSync(Variable<T> &variable, T *data)              \
    {                                                                          \
        GetSyncCommon(variable, data);                                         \
    }                                                                          \
    void DataManReader::DoGetDeferred(Variable<T> &variable, T *data)          \
    {                                                                          \
        GetDeferredCommon(variable, data);                                     \
    }                                                                          \
    void DataManReader::DoGetDeferred(Variable<T> &variable, T &data)          \
    {                                                                          \
        GetDeferredCommon(variable, &data);                                    \
    }
ADIOS2_FOREACH_TYPE_1ARG(declare_type)
#undef declare_type

void DataManReader::DoClose(const int transportIndex) {}

} // end namespace adios2
<|endoftext|>


<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2014 VoltDB Inc.
 *
 * This file contains original code and/or modifications of original code.
 * Any modifications made by VoltDB Inc. are licensed under the following
 * terms and conditions:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */
/* Copyright (C) 2008 by H-Store Project
 * Brown University
 * Massachusetts Institute of Technology
 * Yale University
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <algorithm>
#include <vector>
#include "orderbyexecutor.h"
#include "common/debuglog.h"
#include "common/common.h"
#include "common/tabletuple.h"
#include "common/FatalException.hpp"
#include "execution/ProgressMonitorProxy.h"
#include "plannodes/orderbynode.h"
#include "plannodes/limitnode.h"
#include "storage/table.h"
#include "storage/temptable.h"
#include "storage/tableiterator.h"
#include "storage/tablefactory.h"

using namespace voltdb;
using namespace std;

bool
OrderByExecutor::p_init(AbstractPlanNode* abstract_node,
                        TempTableLimits* limits)
{
    VOLT_TRACE("init OrderBy Executor");

    OrderByPlanNode* node = dynamic_cast<OrderByPlanNode*>(abstract_node);
    assert(node);
    assert(node->getInputTables().size() == 1);

    assert(node->getChildren()[0] != NULL);

    //
    // Our output table should look exactly like out input table
    //
    node->
        setOutputTable(TableFactory::
                       getCopiedTempTable(node->databaseId(),
                                          node->getInputTables()[0]->name(),
                                          node->getInputTables()[0],
                                          limits));

    // pickup an inlined limit, if one exists
    limit_node =
        dynamic_cast<LimitPlanNode*>(node->
                                     getInlinePlanNode(PLAN_NODE_TYPE_LIMIT));

    return true;
}

class TupleComparer
{
public:
    TupleComparer(const vector<AbstractExpression*>& keys,
                  const vector<SortDirectionType>& dirs)
        : m_keys(keys), m_dirs(dirs), m_keyCount(keys.size())
    {
        assert(keys.size() == dirs.size());
    }

    bool operator()(TableTuple ta, TableTuple tb)
    {
        for (size_t i = 0; i < m_keyCount; ++i)
        {
            AbstractExpression* k = m_keys[i];
            SortDirectionType dir = m_dirs[i];
            int cmp = k->eval(&ta, NULL).compare(k->eval(&tb, NULL));
            if (dir == SORT_DIRECTION_TYPE_ASC)
            {
                if (cmp < 0) return true;
                if (cmp > 0) return false;
            }
            else if (dir == SORT_DIRECTION_TYPE_DESC)
            {
                if (cmp < 0) return false;
                if (cmp > 0) return true;
            }
            else
            {
                throw SerializableEEException(VOLT_EE_EXCEPTION_TYPE_EEEXCEPTION,
                                              "Attempted to sort using"
                                              " SORT_DIRECTION_TYPE_INVALID");
            }
        }
        return false; // ta == tb on these keys
    }

private:
    const vector<AbstractExpression*>& m_keys;
    const vector<SortDirectionType>& m_dirs;
    size_t m_keyCount;
};

bool
OrderByExecutor::p_execute(const NValueArray &params)
{
    OrderByPlanNode* node = dynamic_cast<OrderByPlanNode*>(m_abstractNode);
    assert(node);
    TempTable* output_table = dynamic_cast<TempTable*>(node->getOutputTable());
    assert(output_table);
    Table* input_table = node->getInputTables()[0];
    assert(input_table);

    //
    // OPTIMIZATION: NESTED LIMIT
    // How nice! We can also cut off our scanning with a nested limit!
    //
    int limit = -1;
    int offset = -1;
    if (limit_node != NULL)
    {
        limit_node->getLimitAndOffsetByReference(params, limit, offset);
    }

    VOLT_TRACE("Running OrderBy '%s'", m_abstractNode->debug().c_str());
    VOLT_TRACE("Input Table:\n '%s'", input_table->debug().c_str());
    TableIterator iterator = input_table->iterator();
    TableTuple tuple(input_table->schema());
    vector<TableTuple> xs;
    ProgressMonitorProxy pmp(m_engine, this);
    while (iterator.next(tuple))
    {
        pmp.countdownProgress();
        assert(tuple.isActive());
        xs.push_back(tuple);
    }
    VOLT_TRACE("\n***** Input Table PreSort:\n '%s'",
               input_table->debug().c_str());
    sort(xs.begin(), xs.end(), TupleComparer(node->getSortExpressions(),
                                             node->getSortDirections()));

    int tuple_ctr = 0;
    int tuple_skipped = 0;
    for (vector<TableTuple>::iterator it = xs.begin(); it != xs.end(); it++)
    {
        //
        // Check if has gone past the offset
        //
        if (tuple_skipped < offset) {
            tuple_skipped++;
            continue;
        }

        VOLT_TRACE("\n***** Input Table PostSort:\n '%s'",
                   input_table->debug().c_str());
        output_table->insertTupleNonVirtual(*it);
        pmp.countdownProgress();
        //
        // Check whether we have gone past our limit
        //
        if (limit >= 0 && ++tuple_ctr >= limit) {
            break;
        }
    }
    VOLT_TRACE("Result of OrderBy:\n '%s'", output_table->debug().c_str());

    return true;
}

OrderByExecutor::~OrderByExecutor() {
}
<commit_msg>Use partial sort algorithm instead of full sort if has inline limit node<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2014 VoltDB Inc.
 *
 * This file contains original code and/or modifications of original code.
 * Any modifications made by VoltDB Inc. are licensed under the following
 * terms and conditions:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */
/* Copyright (C) 2008 by H-Store Project
 * Brown University
 * Massachusetts Institute of Technology
 * Yale University
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <algorithm>
#include <vector>
#include "orderbyexecutor.h"
#include "common/debuglog.h"
#include "common/common.h"
#include "common/tabletuple.h"
#include "common/FatalException.hpp"
#include "execution/ProgressMonitorProxy.h"
#include "plannodes/orderbynode.h"
#include "plannodes/limitnode.h"
#include "storage/table.h"
#include "storage/temptable.h"
#include "storage/tableiterator.h"
#include "storage/tablefactory.h"

using namespace voltdb;
using namespace std;

bool
OrderByExecutor::p_init(AbstractPlanNode* abstract_node,
                        TempTableLimits* limits)
{
    VOLT_TRACE("init OrderBy Executor");

    OrderByPlanNode* node = dynamic_cast<OrderByPlanNode*>(abstract_node);
    assert(node);
    assert(node->getInputTables().size() == 1);

    assert(node->getChildren()[0] != NULL);

    //
    // Our output table should look exactly like out input table
    //
    node->
        setOutputTable(TableFactory::
                       getCopiedTempTable(node->databaseId(),
                                          node->getInputTables()[0]->name(),
                                          node->getInputTables()[0],
                                          limits));

    // pickup an inlined limit, if one exists
    limit_node =
        dynamic_cast<LimitPlanNode*>(node->
                                     getInlinePlanNode(PLAN_NODE_TYPE_LIMIT));

    return true;
}

class TupleComparer
{
public:
    TupleComparer(const vector<AbstractExpression*>& keys,
                  const vector<SortDirectionType>& dirs)
        : m_keys(keys), m_dirs(dirs), m_keyCount(keys.size())
    {
        assert(keys.size() == dirs.size());
    }

    bool operator()(TableTuple ta, TableTuple tb)
    {
        for (size_t i = 0; i < m_keyCount; ++i)
        {
            AbstractExpression* k = m_keys[i];
            SortDirectionType dir = m_dirs[i];
            int cmp = k->eval(&ta, NULL).compare(k->eval(&tb, NULL));
            if (dir == SORT_DIRECTION_TYPE_ASC)
            {
                if (cmp < 0) return true;
                if (cmp > 0) return false;
            }
            else if (dir == SORT_DIRECTION_TYPE_DESC)
            {
                if (cmp < 0) return false;
                if (cmp > 0) return true;
            }
            else
            {
                throw SerializableEEException(VOLT_EE_EXCEPTION_TYPE_EEEXCEPTION,
                                              "Attempted to sort using"
                                              " SORT_DIRECTION_TYPE_INVALID");
            }
        }
        return false; // ta == tb on these keys
    }

private:
    const vector<AbstractExpression*>& m_keys;
    const vector<SortDirectionType>& m_dirs;
    size_t m_keyCount;
};

bool
OrderByExecutor::p_execute(const NValueArray &params)
{
    OrderByPlanNode* node = dynamic_cast<OrderByPlanNode*>(m_abstractNode);
    assert(node);
    TempTable* output_table = dynamic_cast<TempTable*>(node->getOutputTable());
    assert(output_table);
    Table* input_table = node->getInputTables()[0];
    assert(input_table);

    //
    // OPTIMIZATION: NESTED LIMIT
    // How nice! We can also cut off our scanning with a nested limit!
    //
    int limit = -1;
    int offset = -1;
    if (limit_node != NULL)
    {
        limit_node->getLimitAndOffsetByReference(params, limit, offset);
    }

    VOLT_TRACE("Running OrderBy '%s'", m_abstractNode->debug().c_str());
    VOLT_TRACE("Input Table:\n '%s'", input_table->debug().c_str());
    TableIterator iterator = input_table->iterator();
    TableTuple tuple(input_table->schema());
    vector<TableTuple> xs;
    ProgressMonitorProxy pmp(m_engine, this);
    while (iterator.next(tuple))
    {
        pmp.countdownProgress();
        assert(tuple.isActive());
        xs.push_back(tuple);
    }
    VOLT_TRACE("\n***** Input Table PreSort:\n '%s'",
               input_table->debug().c_str());

    if (limit >= 0 && xs.begin() + limit < xs.end()) {
        // partial sort
        partial_sort(xs.begin(), xs.begin() + limit, xs.end(),
                TupleComparer(node->getSortExpressions(), node->getSortDirections()));
    } else {
        // full sort
        sort(xs.begin(), xs.end(),
                TupleComparer(node->getSortExpressions(), node->getSortDirections()));
    }

    int tuple_ctr = 0;
    int tuple_skipped = 0;
    for (vector<TableTuple>::iterator it = xs.begin(); it != xs.end(); it++)
    {
        //
        // Check if has gone past the offset
        //
        if (tuple_skipped < offset) {
            tuple_skipped++;
            continue;
        }

        VOLT_TRACE("\n***** Input Table PostSort:\n '%s'",
                   input_table->debug().c_str());
        output_table->insertTupleNonVirtual(*it);
        pmp.countdownProgress();
        //
        // Check whether we have gone past our limit
        //
        if (limit >= 0 && ++tuple_ctr >= limit) {
            break;
        }
    }
    VOLT_TRACE("Result of OrderBy:\n '%s'", output_table->debug().c_str());

    return true;
}

OrderByExecutor::~OrderByExecutor() {
}
<|endoftext|>


<commit_before>#include "MarchingCubesPipeline.h"

#include <gloperate/painter/AbstractViewportCapability.h>
#include <gloperate/painter/TypedRenderTargetCapability.h>
#include <gloperate/painter/PerspectiveProjectionCapability.h>
#include <gloperate/painter/AbstractCameraCapability.h>
#include <gloperate/painter/AbstractVirtualTimeCapability.h>
#include <gloperate/painter/AbstractTargetFramebufferCapability.h>
#include <gloperate/painter/InputCapability.h>
#include <gloperate/resources/ResourceManager.h>
#include <gloperate/navigation/CoordinateProvider.h>

#include "AddChunksStage.h"
#include "ManageChunksStage.h"
#include "RenderStage.h"
#include "TerrainTypeStage.h"
#include "TerrainModificationStage.h"

MarchingCubesPipeline::MarchingCubesPipeline()
: AbstractPipeline("MarchingCubes")
, viewport(nullptr)
, showGrid(false)
, showWireframe(false)
, freezeChunkLoading(false)
, modificationRadius(0.25f)
, useMipMap(false)
, userRotationVector1(glm::vec3(1, 0.3, 0.5))
, userRotationVector2(glm::vec3(0.1, 0.5, 0.3))
, userWarpFactor(3.4f)
, removeFloaters(false)
, targetFBO(nullptr)
, renderTargets(nullptr)
, useShadow(true)
, useOcclusion(true)
, userBaseTextureFilePath("data/marchingcubes/ground.png")
, useGroundTexture(true)
, userExtraTextureFilePath("data/marchingcubes/terrain_color.jpg")
, useStriationTexture(true)
, terrainType(TerrainType::Moon)
, userFragmentShaderFilePath("data/marchingcubes/marchingcubes.frag")
, userDensityGenererationShaderFilePath("data/marchingcubes/densitygeneration.vert")
{
    auto terrainTypeStage = new TerrainTypeStage();
    auto addChunksStage = new AddChunksStage();
    auto terrainModificationStage = new TerrainModificationStage();
    auto manageChunksStage = new ManageChunksStage();
    auto renderStage = new RenderStage();

    terrainTypeStage->terrainType = terrainType;
    terrainTypeStage->userBaseTextureFilePath = userBaseTextureFilePath;
    terrainTypeStage->userExtraTextureFilePath = userExtraTextureFilePath;
    terrainTypeStage->userFragmentShaderFilePath = userFragmentShaderFilePath;
    terrainTypeStage->userDensityGenererationShaderFilePath = userDensityGenererationShaderFilePath;
    terrainTypeStage->userRotationVector1 = userRotationVector1;
    terrainTypeStage->userRotationVector2 = userRotationVector2;
    terrainTypeStage->userWarpFactor = userWarpFactor;

    addChunksStage->camera = camera;
    addChunksStage->projection = projection;
    addChunksStage->freezeChunkLoading = freezeChunkLoading;

    terrainModificationStage->input = input;

    manageChunksStage->camera = camera;
    manageChunksStage->coordinateProvider = coordinateProvider;
    manageChunksStage->addPosition = terrainModificationStage->addPosition;
    manageChunksStage->removePosition = terrainModificationStage->removePosition;
    manageChunksStage->chunksToAdd = addChunksStage->chunksToAdd;
    manageChunksStage->rotationVector1 = terrainTypeStage->rotationVector1;
    manageChunksStage->rotationVector2 = terrainTypeStage->rotationVector2;
    manageChunksStage->warpFactor = terrainTypeStage->warpFactor;
    manageChunksStage->removeFloaters = removeFloaters;
    manageChunksStage->freezeChunkLoading = freezeChunkLoading;
    manageChunksStage->modificationRadius = modificationRadius;
    manageChunksStage->densityGenererationShaderFilePath = terrainTypeStage->densityGenererationShaderFilePath;

    renderStage->viewport = viewport;
    renderStage->camera = camera;
    renderStage->projection = projection;
    renderStage->targetFBO = targetFBO;
    renderStage->renderTargets = renderTargets;
    renderStage->showGrid = showGrid;
    renderStage->showWireframe = showWireframe;
    renderStage->useMipMap = useMipMap;
    renderStage->resourceManager = resourceManager;
    renderStage->useShadow = useShadow;
    renderStage->useOcclusion = useOcclusion;
    renderStage->baseTextureFilePath = terrainTypeStage->baseTextureFilePath;
    renderStage->useBaseTexture = useGroundTexture;
    renderStage->extraTextureFilePath = terrainTypeStage->extraTextureFilePath;
    renderStage->useExtraTexture = useStriationTexture;
    renderStage->fragmentShaderFilePath = terrainTypeStage->fragmentShaderTextureFilePath;
    renderStage->chunks = manageChunksStage->chunks;

    addStages(
        std::move(terrainTypeStage),
        std::move(addChunksStage),
        std::move(terrainModificationStage),
        std::move(manageChunksStage),
		std::move(renderStage)
	);
}

<commit_msg>Use mip mapping by default<commit_after>#include "MarchingCubesPipeline.h"

#include <gloperate/painter/AbstractViewportCapability.h>
#include <gloperate/painter/TypedRenderTargetCapability.h>
#include <gloperate/painter/PerspectiveProjectionCapability.h>
#include <gloperate/painter/AbstractCameraCapability.h>
#include <gloperate/painter/AbstractVirtualTimeCapability.h>
#include <gloperate/painter/AbstractTargetFramebufferCapability.h>
#include <gloperate/painter/InputCapability.h>
#include <gloperate/resources/ResourceManager.h>
#include <gloperate/navigation/CoordinateProvider.h>

#include "AddChunksStage.h"
#include "ManageChunksStage.h"
#include "RenderStage.h"
#include "TerrainTypeStage.h"
#include "TerrainModificationStage.h"

MarchingCubesPipeline::MarchingCubesPipeline()
: AbstractPipeline("MarchingCubes")
, viewport(nullptr)
, showGrid(false)
, showWireframe(false)
, freezeChunkLoading(false)
, modificationRadius(0.25f)
, useMipMap(true)
, userRotationVector1(glm::vec3(1, 0.3, 0.5))
, userRotationVector2(glm::vec3(0.1, 0.5, 0.3))
, userWarpFactor(3.4f)
, removeFloaters(false)
, targetFBO(nullptr)
, renderTargets(nullptr)
, useShadow(true)
, useOcclusion(true)
, userBaseTextureFilePath("data/marchingcubes/ground.png")
, useGroundTexture(true)
, userExtraTextureFilePath("data/marchingcubes/terrain_color.jpg")
, useStriationTexture(true)
, terrainType(TerrainType::Moon)
, userFragmentShaderFilePath("data/marchingcubes/marchingcubes.frag")
, userDensityGenererationShaderFilePath("data/marchingcubes/densitygeneration.vert")
{
    auto terrainTypeStage = new TerrainTypeStage();
    auto addChunksStage = new AddChunksStage();
    auto terrainModificationStage = new TerrainModificationStage();
    auto manageChunksStage = new ManageChunksStage();
    auto renderStage = new RenderStage();

    terrainTypeStage->terrainType = terrainType;
    terrainTypeStage->userBaseTextureFilePath = userBaseTextureFilePath;
    terrainTypeStage->userExtraTextureFilePath = userExtraTextureFilePath;
    terrainTypeStage->userFragmentShaderFilePath = userFragmentShaderFilePath;
    terrainTypeStage->userDensityGenererationShaderFilePath = userDensityGenererationShaderFilePath;
    terrainTypeStage->userRotationVector1 = userRotationVector1;
    terrainTypeStage->userRotationVector2 = userRotationVector2;
    terrainTypeStage->userWarpFactor = userWarpFactor;

    addChunksStage->camera = camera;
    addChunksStage->projection = projection;
    addChunksStage->freezeChunkLoading = freezeChunkLoading;

    terrainModificationStage->input = input;

    manageChunksStage->camera = camera;
    manageChunksStage->coordinateProvider = coordinateProvider;
    manageChunksStage->addPosition = terrainModificationStage->addPosition;
    manageChunksStage->removePosition = terrainModificationStage->removePosition;
    manageChunksStage->chunksToAdd = addChunksStage->chunksToAdd;
    manageChunksStage->rotationVector1 = terrainTypeStage->rotationVector1;
    manageChunksStage->rotationVector2 = terrainTypeStage->rotationVector2;
    manageChunksStage->warpFactor = terrainTypeStage->warpFactor;
    manageChunksStage->removeFloaters = removeFloaters;
    manageChunksStage->freezeChunkLoading = freezeChunkLoading;
    manageChunksStage->modificationRadius = modificationRadius;
    manageChunksStage->densityGenererationShaderFilePath = terrainTypeStage->densityGenererationShaderFilePath;

    renderStage->viewport = viewport;
    renderStage->camera = camera;
    renderStage->projection = projection;
    renderStage->targetFBO = targetFBO;
    renderStage->renderTargets = renderTargets;
    renderStage->showGrid = showGrid;
    renderStage->showWireframe = showWireframe;
    renderStage->useMipMap = useMipMap;
    renderStage->resourceManager = resourceManager;
    renderStage->useShadow = useShadow;
    renderStage->useOcclusion = useOcclusion;
    renderStage->baseTextureFilePath = terrainTypeStage->baseTextureFilePath;
    renderStage->useBaseTexture = useGroundTexture;
    renderStage->extraTextureFilePath = terrainTypeStage->extraTextureFilePath;
    renderStage->useExtraTexture = useStriationTexture;
    renderStage->fragmentShaderFilePath = terrainTypeStage->fragmentShaderTextureFilePath;
    renderStage->chunks = manageChunksStage->chunks;

    addStages(
        std::move(terrainTypeStage),
        std::move(addChunksStage),
        std::move(terrainModificationStage),
        std::move(manageChunksStage),
		std::move(renderStage)
	);
}

<|endoftext|>