<commit_before>/* Copyright 2021, The TensorFlow Federated Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
==============================================================================*/

#include "tensorflow_federated/cc/core/impl/executors/session_provider.h"

#include "absl/container/flat_hash_set.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/device_factory.h"
#include "tensorflow/core/framework/function.pb.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/graph/default_device.h"
#include "tensorflow/core/public/session_options.h"

namespace tensorflow_federated {

// A process level counter for SessionProvider creation. This ensures
// each session provider is creating unique containers for each session
// created in the process. This is necessary because the TesnorFlow
// ResourceManager shares resource objects across Sessions in the same process,
// but we need to ability to clear an individual Session's container after
// each call.
ABSL_CONST_INIT static absl::Mutex function_id_mutex(absl::kConstInit);
uint32_t GetNextFunctionId() {
  absl::MutexLock function_id_lock(&function_id_mutex);
  static uint32_t function_id = 0;
  return function_id++;
}

const tensorflow::SessionOptions& get_session_options() {
  // Creates default SessionOptions on first call, and re-uses them for all
  // future calls.
  static tensorflow::SessionOptions* session_options = []() {
    tensorflow::SessionOptions* options_pb = new tensorflow::SessionOptions();
    tensorflow::GraphOptions* graph_options_pb =
        options_pb->config.mutable_graph_options();
    // Disable JIT/runtime Grappler. TFF typically has short lived sessions,
    // meaning that the benefit from running grappler is generally not realized
    // and can be very expensive (anecdotally ~30% of CPU time).
    graph_options_pb->mutable_rewrite_options()->set_disable_meta_optimizer(
        true);
    graph_options_pb->mutable_optimizer_options()->set_opt_level(
        tensorflow::OptimizerOptions::L0);
    graph_options_pb->mutable_optimizer_options()->set_global_jit_level(
        tensorflow::OptimizerOptions::OFF);
    // Don't eagerly allocate all GPU memory in each session.
    options_pb->config.mutable_gpu_options()->set_allow_growth(true);
    // Let the session know the graph will not change.
    auto experimental_pb = options_pb->config.mutable_experimental();
    experimental_pb->set_optimize_for_static_graph(true);
    experimental_pb->set_disable_output_partition_graphs(true);
    options_pb->config.set_allow_soft_placement(true);
    return options_pb;
  }();
  return *session_options;
}

struct AcceleratorDevices {
  const int16_t num_gpus = 0;
  const int16_t num_tpus = 0;
};

const AcceleratorDevices& GetAcceleratorDevices() {
  static const AcceleratorDevices* accelerator_devices =
      []() -> AcceleratorDevices* {
    int16_t num_gpus = 0;
    int16_t num_tpus = 0;
    std::vector<std::string> devices;
    tensorflow::Status s =
        tensorflow::DeviceFactory::ListAllPhysicalDevices(&devices);
    if (!s.ok()) {
      LOG(ERROR) << "Error detecting physical devices, defaulting to CPU only: "
                 << s;
      return new AcceleratorDevices{0};
    }
    LOG_FIRST_N(INFO, 1) << "Found devices: [" << absl::StrJoin(devices, ",")
                         << "]";
    for (absl::string_view device : devices) {
      std::vector<absl::string_view> device_parts = absl::StrSplit(device, ':');
      if (device_parts.size() != 3) {
        LOG(ERROR) << "Unknown device name format: [" << device << "]";
        continue;
      }
      auto device_type = device_parts[1];
      if (device_type == tensorflow::DEVICE_GPU) {
        LOG_FIRST_N(INFO, 1) << "Found GPU device: [" << device << "]";
        ++num_gpus;
      } else if (device_type == tensorflow::DEVICE_TPU) {
        LOG_FIRST_N(INFO, 1) << "Found TPU device: [" << device << "]";
        ++num_tpus;
      } else {
        LOG_FIRST_N(INFO, 1) << "Skipping device: [" << device << "]";
      }
    }
    return new AcceleratorDevices{num_gpus, num_tpus};
  }();
  return *accelerator_devices;
}

void SetDevice(absl::string_view device, tensorflow::GraphDef* graph_def,
               const char* device_type) {
  for (tensorflow::NodeDef& node_pb : *graph_def->mutable_node()) {
    if (!node_pb.device().empty()) {
      VLOG(5) << "Skipping already placed node [" << node_pb.name() << "] ("
              << node_pb.op() << ") on " << node_pb.device();
      continue;
    } else if (tensorflow::KernelDefAvailable(device_type, node_pb)) {
      VLOG(5) << "Placing node [" << node_pb.name() << "] (" << node_pb.op()
              << ") on device [" << device << "]";
      node_pb.set_device(device.data(), device.size());
    } else {
      VLOG(5) << "Leaving node [" << node_pb.name() << "] alone.";
    }
  }
}

const absl::flat_hash_set<std::string>& GetOpsWithContainerAttr() {
  // Get the list of all ops that have an `attr` named `container`. We assume
  // this will be all ops that construct resources and allow us to configure
  // which container those resources will be placed in. This maybe fragile
  // because an op _might_ not expose the container attr, or might name it
  // something different.
  static auto* op_names = []() -> absl::flat_hash_set<std::string>* {
    std::vector<tensorflow::OpDef> ops;
    tensorflow::OpRegistry::Global()->GetRegisteredOps(&ops);
    auto op_has_container_attr =
        [](const tensorflow::OpDef& op_def_pb) -> bool {
      for (const auto& attr_def_pb : op_def_pb.attr()) {
        if (attr_def_pb.name() == "container") {
          return true;
        }
      }
      return false;
    };
    auto* op_names = new absl::flat_hash_set<std::string>;
    for (const auto& op_def_pb : ops) {
      if (op_has_container_attr(op_def_pb)) {
        op_names->emplace(op_def_pb.name());
      }
    }
    return op_names;
  }();
  return *op_names;
}

// Rewrites the `container` attr of any op that supports, creating a new
// `GraphDef` with altered `NodeDef`s.
tensorflow::GraphDef ReplaceContainers(
    const tensorflow::GraphDef& original_graph, const std::string& container) {
  // Go through the graph and explicitly set the `container` attr on any op that
  // supports a `container` attr.
  const auto& ops_with_container_attrs = GetOpsWithContainerAttr();
  tensorflow::GraphDef graph = original_graph;
  for (tensorflow::NodeDef& node_pb : *graph.mutable_node()) {
    if (ops_with_container_attrs.contains(node_pb.op())) {
      (*node_pb.mutable_attr())["container"].set_s(container);
    }
  }
  for (tensorflow::FunctionDef& function_pb :
       *graph.mutable_library()->mutable_function()) {
    for (tensorflow::NodeDef& node_pb : *function_pb.mutable_node_def()) {
      if (ops_with_container_attrs.contains(node_pb.op())) {
        (*node_pb.mutable_attr())["container"].set_s(container);
      }
    }
  }
  return graph;
}

SessionProvider::SessionProvider(tensorflow::GraphDef&& graph,
                                 absl::optional<uint16_t> max_active_sessions)
    : max_active_sessions_(max_active_sessions),
      active_sessions_(0),
      graph_(graph),
      function_id_(GetNextFunctionId()) {
  maybe_open_cpus_ = std::thread::hardware_concurrency();
}

absl::StatusOr<std::unique_ptr<tensorflow::Session>>
SessionProvider::CreateSession(const uint16_t session_id) {
  const std::string container = absl::StrCat(function_id_, "/", session_id);
  std::unique_ptr<tensorflow::Session> session;
  {
    tensorflow::Session* raw_session;
    tensorflow::Status status =
        tensorflow::NewSession(get_session_options(), &raw_session);
    if (!status.ok()) {
      return absl::InternalError(absl::StrCat(
          "Failed to create TensorFlow session: ", status.error_message()));
    }
    session.reset(raw_session);
  }
  tensorflow::GraphDef graph_def = ReplaceContainers(graph_, container);
  const AcceleratorDevices& devices = GetAcceleratorDevices();

  if (devices.num_gpus > 0 && devices.num_tpus > 0) {
    return absl::UnimplementedError(
        "Can't create a TensorFlow session for hardware with both GPUs and "
        "TPUs.");
  }

  if (devices.num_gpus > 0) {
    // If we have GPUs, round robin the session by explicitly setting the
    // `device` attr of the GPU-capable kernels.
    const uint16_t device_id = session_id % devices.num_gpus;
    const std::string& device =
        absl::StrCat("/device:", tensorflow::DEVICE_GPU, ":", device_id);
    VLOG(2) << "Pinning function [" << function_id_ << "] session ["
            << session_id << "] to device [" << device << "]";
    SetDevice(device, &graph_def, tensorflow::DEVICE_GPU);
  }
  if (devices.num_tpus > 0) {
    // If we have TPUs, round robin the session by explicitly setting the
    // `device` attr of the TPU-capable kernels.
    const uint16_t device_id = session_id % devices.num_tpus;
    const std::string& device =
        absl::StrCat("/device:", tensorflow::DEVICE_TPU, ":", device_id);
    VLOG(2) << "Pinning function [" << function_id_ << "] session ["
            << session_id << "] to device [" << device << "]";
    SetDevice(device, &graph_def, tensorflow::DEVICE_TPU);
  }
  auto status = session->Create(graph_def);
  if (!status.ok()) {
    LOG(ERROR) << status;
    for (absl::string_view line :
         absl::StrSplit(graph_def.Utf8DebugString(), '\n')) {
      LOG(ERROR) << line;
    }
    return absl::InternalError(absl::StrCat(
        "Failed to create graph in session: ", status.error_message()));
  }
  return std::move(session);
}

absl::StatusOr<SessionProvider::SessionWithResourceContainer>
SessionProvider::TakeSession() {
  lock_.LockWhen(
      absl::Condition(this, &SessionProvider::SessionOrCpuAvailable));
  active_sessions_++;
  if (!sessions_.empty()) {
    SessionProvider::SessionWithResourceContainer session(
        std::move(sessions_.back()));
    sessions_.pop_back();
    lock_.Unlock();
    return std::move(session);
  }
  maybe_open_cpus_--;
  // Build a container name based on the number of sessions created so that each
  // session gets its own container.
  const uint16_t session_id = session_creation_counter_++;
  lock_.Unlock();
  auto session = CreateSession(session_id);
  lock_.Lock();
  maybe_open_cpus_++;
  lock_.Unlock();
  if (session.ok()) {
    return SessionProvider::SessionWithResourceContainer{
        TFF_TRY(std::move(session)), function_id_, session_id};
  } else {
    return session.status();
  }
}

void SessionProvider::ReturnSession(
    SessionProvider::SessionWithResourceContainer&& session) {
  session.ClearResourceContainers();
  lock_.Lock();
  active_sessions_--;
  sessions_.emplace_back(std::move(session));
  lock_.Unlock();
}

}  // namespace tensorflow_federated
<commit_msg>Annotate ReduceDataset ops run on TPU _xla_compile_device_type='TPU' attribute.<commit_after>/* Copyright 2021, The TensorFlow Federated Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
==============================================================================*/

#include "tensorflow_federated/cc/core/impl/executors/session_provider.h"

#include "absl/container/flat_hash_set.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/synchronization/mutex.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/device_factory.h"
#include "tensorflow/core/framework/function.pb.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/graph/default_device.h"
#include "tensorflow/core/public/session_options.h"

namespace tensorflow_federated {

// A process level counter for SessionProvider creation. This ensures
// each session provider is creating unique containers for each session
// created in the process. This is necessary because the TesnorFlow
// ResourceManager shares resource objects across Sessions in the same process,
// but we need to ability to clear an individual Session's container after
// each call.
ABSL_CONST_INIT static absl::Mutex function_id_mutex(absl::kConstInit);
uint32_t GetNextFunctionId() {
  absl::MutexLock function_id_lock(&function_id_mutex);
  static uint32_t function_id = 0;
  return function_id++;
}

const tensorflow::SessionOptions& get_session_options() {
  // Creates default SessionOptions on first call, and re-uses them for all
  // future calls.
  static tensorflow::SessionOptions* session_options = []() {
    tensorflow::SessionOptions* options_pb = new tensorflow::SessionOptions();
    tensorflow::GraphOptions* graph_options_pb =
        options_pb->config.mutable_graph_options();
    // Disable JIT/runtime Grappler. TFF typically has short lived sessions,
    // meaning that the benefit from running grappler is generally not realized
    // and can be very expensive (anecdotally ~30% of CPU time).
    graph_options_pb->mutable_rewrite_options()->set_disable_meta_optimizer(
        true);
    graph_options_pb->mutable_optimizer_options()->set_opt_level(
        tensorflow::OptimizerOptions::L0);
    graph_options_pb->mutable_optimizer_options()->set_global_jit_level(
        tensorflow::OptimizerOptions::OFF);
    // Don't eagerly allocate all GPU memory in each session.
    options_pb->config.mutable_gpu_options()->set_allow_growth(true);
    // Let the session know the graph will not change.
    auto experimental_pb = options_pb->config.mutable_experimental();
    experimental_pb->set_optimize_for_static_graph(true);
    experimental_pb->set_disable_output_partition_graphs(true);
    options_pb->config.set_allow_soft_placement(true);
    return options_pb;
  }();
  return *session_options;
}

struct AcceleratorDevices {
  const int16_t num_gpus = 0;
  const int16_t num_tpus = 0;
};

const AcceleratorDevices& GetAcceleratorDevices() {
  static const AcceleratorDevices* accelerator_devices =
      []() -> AcceleratorDevices* {
    int16_t num_gpus = 0;
    int16_t num_tpus = 0;
    std::vector<std::string> devices;
    tensorflow::Status s =
        tensorflow::DeviceFactory::ListAllPhysicalDevices(&devices);
    if (!s.ok()) {
      LOG(ERROR) << "Error detecting physical devices, defaulting to CPU only: "
                 << s;
      return new AcceleratorDevices{0};
    }
    LOG_FIRST_N(INFO, 1) << "Found devices: [" << absl::StrJoin(devices, ",")
                         << "]";
    for (absl::string_view device : devices) {
      std::vector<absl::string_view> device_parts = absl::StrSplit(device, ':');
      if (device_parts.size() != 3) {
        LOG(ERROR) << "Unknown device name format: [" << device << "]";
        continue;
      }
      auto device_type = device_parts[1];
      if (device_type == tensorflow::DEVICE_GPU) {
        LOG_FIRST_N(INFO, 1) << "Found GPU device: [" << device << "]";
        ++num_gpus;
      } else if (device_type == tensorflow::DEVICE_TPU) {
        LOG_FIRST_N(INFO, 1) << "Found TPU device: [" << device << "]";
        ++num_tpus;
      } else {
        LOG_FIRST_N(INFO, 1) << "Skipping device: [" << device << "]";
      }
    }
    return new AcceleratorDevices{num_gpus, num_tpus};
  }();
  return *accelerator_devices;
}

void SetDevice(absl::string_view device, tensorflow::GraphDef* graph_def,
               const char* device_type) {
  for (tensorflow::NodeDef& node_pb : *graph_def->mutable_node()) {
    // Annotating ReduceDataset with _xla_compile_device_type will denote to
    // Tensorflow to decompose the op into dataset iteration and reduce_fn and
    // compile the reduce_fn.
    // TODO(b/233627338): Currently only supported for TPU.  Enable for CPU/GPU
    // once MLIR-based TF2XLA bridge supports CPU/GPU.
    if (node_pb.op() == "ReduceDataset" &&
        !(strcmp(device_type, tensorflow::DEVICE_TPU))) {
      (*node_pb.mutable_attr())["_xla_compile_device_type"].set_s(device_type);
    }
    if (!node_pb.device().empty()) {
      VLOG(5) << "Skipping already placed node [" << node_pb.name() << "] ("
              << node_pb.op() << ") on " << node_pb.device();
      continue;
    } else if (tensorflow::KernelDefAvailable(device_type, node_pb)) {
      VLOG(5) << "Placing node [" << node_pb.name() << "] (" << node_pb.op()
              << ") on device [" << device << "]";
      node_pb.set_device(device.data(), device.size());
    } else {
      VLOG(5) << "Leaving node [" << node_pb.name() << "] alone.";
    }
  }
}

const absl::flat_hash_set<std::string>& GetOpsWithContainerAttr() {
  // Get the list of all ops that have an `attr` named `container`. We assume
  // this will be all ops that construct resources and allow us to configure
  // which container those resources will be placed in. This maybe fragile
  // because an op _might_ not expose the container attr, or might name it
  // something different.
  static auto* op_names = []() -> absl::flat_hash_set<std::string>* {
    std::vector<tensorflow::OpDef> ops;
    tensorflow::OpRegistry::Global()->GetRegisteredOps(&ops);
    auto op_has_container_attr =
        [](const tensorflow::OpDef& op_def_pb) -> bool {
      for (const auto& attr_def_pb : op_def_pb.attr()) {
        if (attr_def_pb.name() == "container") {
          return true;
        }
      }
      return false;
    };
    auto* op_names = new absl::flat_hash_set<std::string>;
    for (const auto& op_def_pb : ops) {
      if (op_has_container_attr(op_def_pb)) {
        op_names->emplace(op_def_pb.name());
      }
    }
    return op_names;
  }();
  return *op_names;
}

// Rewrites the `container` attr of any op that supports, creating a new
// `GraphDef` with altered `NodeDef`s.
tensorflow::GraphDef ReplaceContainers(
    const tensorflow::GraphDef& original_graph, const std::string& container) {
  // Go through the graph and explicitly set the `container` attr on any op that
  // supports a `container` attr.
  const auto& ops_with_container_attrs = GetOpsWithContainerAttr();
  tensorflow::GraphDef graph = original_graph;
  for (tensorflow::NodeDef& node_pb : *graph.mutable_node()) {
    if (ops_with_container_attrs.contains(node_pb.op())) {
      (*node_pb.mutable_attr())["container"].set_s(container);
    }
  }
  for (tensorflow::FunctionDef& function_pb :
       *graph.mutable_library()->mutable_function()) {
    for (tensorflow::NodeDef& node_pb : *function_pb.mutable_node_def()) {
      if (ops_with_container_attrs.contains(node_pb.op())) {
        (*node_pb.mutable_attr())["container"].set_s(container);
      }
    }
  }
  return graph;
}

SessionProvider::SessionProvider(tensorflow::GraphDef&& graph,
                                 absl::optional<uint16_t> max_active_sessions)
    : max_active_sessions_(max_active_sessions),
      active_sessions_(0),
      graph_(graph),
      function_id_(GetNextFunctionId()) {
  maybe_open_cpus_ = std::thread::hardware_concurrency();
}

absl::StatusOr<std::unique_ptr<tensorflow::Session>>
SessionProvider::CreateSession(const uint16_t session_id) {
  const std::string container = absl::StrCat(function_id_, "/", session_id);
  std::unique_ptr<tensorflow::Session> session;
  {
    tensorflow::Session* raw_session;
    tensorflow::Status status =
        tensorflow::NewSession(get_session_options(), &raw_session);
    if (!status.ok()) {
      return absl::InternalError(absl::StrCat(
          "Failed to create TensorFlow session: ", status.error_message()));
    }
    session.reset(raw_session);
  }
  tensorflow::GraphDef graph_def = ReplaceContainers(graph_, container);
  const AcceleratorDevices& devices = GetAcceleratorDevices();

  if (devices.num_gpus > 0 && devices.num_tpus > 0) {
    return absl::UnimplementedError(
        "Can't create a TensorFlow session for hardware with both GPUs and "
        "TPUs.");
  }

  if (devices.num_gpus > 0) {
    // If we have GPUs, round robin the session by explicitly setting the
    // `device` attr of the GPU-capable kernels.
    const uint16_t device_id = session_id % devices.num_gpus;
    const std::string& device =
        absl::StrCat("/device:", tensorflow::DEVICE_GPU, ":", device_id);
    VLOG(2) << "Pinning function [" << function_id_ << "] session ["
            << session_id << "] to device [" << device << "]";
    SetDevice(device, &graph_def, tensorflow::DEVICE_GPU);
  }
  if (devices.num_tpus > 0) {
    // If we have TPUs, round robin the session by explicitly setting the
    // `device` attr of the TPU-capable kernels.
    const uint16_t device_id = session_id % devices.num_tpus;
    const std::string& device =
        absl::StrCat("/device:", tensorflow::DEVICE_TPU, ":", device_id);
    VLOG(2) << "Pinning function [" << function_id_ << "] session ["
            << session_id << "] to device [" << device << "]";
    SetDevice(device, &graph_def, tensorflow::DEVICE_TPU);
  }
  auto status = session->Create(graph_def);
  if (!status.ok()) {
    LOG(ERROR) << status;
    for (absl::string_view line :
         absl::StrSplit(graph_def.Utf8DebugString(), '\n')) {
      LOG(ERROR) << line;
    }
    return absl::InternalError(absl::StrCat(
        "Failed to create graph in session: ", status.error_message()));
  }
  return std::move(session);
}

absl::StatusOr<SessionProvider::SessionWithResourceContainer>
SessionProvider::TakeSession() {
  lock_.LockWhen(
      absl::Condition(this, &SessionProvider::SessionOrCpuAvailable));
  active_sessions_++;
  if (!sessions_.empty()) {
    SessionProvider::SessionWithResourceContainer session(
        std::move(sessions_.back()));
    sessions_.pop_back();
    lock_.Unlock();
    return std::move(session);
  }
  maybe_open_cpus_--;
  // Build a container name based on the number of sessions created so that each
  // session gets its own container.
  const uint16_t session_id = session_creation_counter_++;
  lock_.Unlock();
  auto session = CreateSession(session_id);
  lock_.Lock();
  maybe_open_cpus_++;
  lock_.Unlock();
  if (session.ok()) {
    return SessionProvider::SessionWithResourceContainer{
        TFF_TRY(std::move(session)), function_id_, session_id};
  } else {
    return session.status();
  }
}

void SessionProvider::ReturnSession(
    SessionProvider::SessionWithResourceContainer&& session) {
  session.ClearResourceContainers();
  lock_.Lock();
  active_sessions_--;
  sessions_.emplace_back(std::move(session));
  lock_.Unlock();
}

}  // namespace tensorflow_federated
<|endoftext|>


<commit_before>/*-
* Copyright 2009 Colin Percival, 2011 ArtForz, 2011 pooler, 2013 Balthazar
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* This file was originally written by Colin Percival as part of the Tarsnap
* online backup system.
*/

#include <stdlib.h>
#include <stdint.h>
#include <xmmintrin.h>

#include "scrypt_mine.h"
#include "pbkdf2.h"

#include "util.h"
#include "net.h"

#if defined (__x86_64__) || defined (__i386__)
#define SCRYPT_BUFFER_SIZE (131072 + 63)
extern "C" void scrypt_core(uint32_t *X, uint32_t *V);
#endif

void *scrypt_buffer_alloc() {
    return malloc(SCRYPT_BUFFER_SIZE);
}

void scrypt_buffer_free(void *scratchpad)
{
    free(scratchpad);
}

/* cpu and memory intensive function to transform a 80 byte buffer into a 32 byte output
scratchpad size needs to be at least 63 + (128 * r * p) + (256 * r + 64) + (128 * r * N) bytes
r = 1, p = 1, N = 1024
*/

static void scrypt(const void* input, size_t inputlen, uint32_t *res, void *scratchpad)
{
    uint32_t *V;
    uint32_t X[32];
    V = (uint32_t *)(((uintptr_t)(scratchpad) + 63) & ~ (uintptr_t)(63));

    PBKDF2_SHA256((const uint8_t*)input, inputlen, (const uint8_t*)input, sizeof(block_header), 1, (uint8_t *)X, 128);

    scrypt_core(X, V);

    PBKDF2_SHA256((const uint8_t*)input, inputlen, (uint8_t *)X, 128, 1, (uint8_t*)res, 32);
}

void scrypt_hash(const void* input, size_t inputlen, uint32_t *res, void *scratchpad)
{
    return scrypt(input, inputlen, res, scratchpad);
}<commit_msg>Remove SSE2<commit_after>/*-
* Copyright 2009 Colin Percival, 2011 ArtForz, 2011 pooler, 2013 Balthazar
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* This file was originally written by Colin Percival as part of the Tarsnap
* online backup system.
*/

#include <stdlib.h>
#include <stdint.h>

#include "scrypt_mine.h"
#include "pbkdf2.h"

#include "util.h"
#include "net.h"

#if defined (__x86_64__) || defined (__i386__)
#define SCRYPT_BUFFER_SIZE (131072 + 63)
extern "C" void scrypt_core(uint32_t *X, uint32_t *V);
#endif

void *scrypt_buffer_alloc() {
    return malloc(SCRYPT_BUFFER_SIZE);
}

void scrypt_buffer_free(void *scratchpad)
{
    free(scratchpad);
}

/* cpu and memory intensive function to transform a 80 byte buffer into a 32 byte output
scratchpad size needs to be at least 63 + (128 * r * p) + (256 * r + 64) + (128 * r * N) bytes
r = 1, p = 1, N = 1024
*/

static void scrypt(const void* input, size_t inputlen, uint32_t *res, void *scratchpad)
{
    uint32_t *V;
    uint32_t X[32];
    V = (uint32_t *)(((uintptr_t)(scratchpad) + 63) & ~ (uintptr_t)(63));

    PBKDF2_SHA256((const uint8_t*)input, inputlen, (const uint8_t*)input, sizeof(block_header), 1, (uint8_t *)X, 128);

    scrypt_core(X, V);

    PBKDF2_SHA256((const uint8_t*)input, inputlen, (uint8_t *)X, 128, 1, (uint8_t*)res, 32);
}

void scrypt_hash(const void* input, size_t inputlen, uint32_t *res, void *scratchpad)
{
    return scrypt(input, inputlen, res, scratchpad);
}<|endoftext|>


<commit_before>/*
 *	author: muggle wei <mugglewei@gmail.com>
 *
 *	Use of this source code is governed by the MIT license that can be
 *	found in the LICENSE file.
 */

#include "muggle_benchmark/muggle_benchmark.h"
#include <stdio.h>
#include <thread>
#include <chrono>

uint64_t *consumer_read_num = nullptr;

void fn_producer(
	muggle_ringbuffer_t *ring,
	muggle::BenchmarkConfig *config,
	muggle::LatencyBlock *blocks,
	muggle_atomic_int *consumer_ready,
	muggle_atomic_int consumer_cnt,
	uint64_t start_idx,
	uint64_t end_idx
)
{
	while (muggle_atomic_load(consumer_ready, muggle_memory_order_relaxed) != consumer_cnt);

	for (uint64_t i = 0; i < config->loop; ++i)
	{
		for (uint64_t j = start_idx; j < end_idx; ++j)
		{
			uint64_t idx = i * config->cnt_per_loop + j;
			memset(&blocks[idx], 0, sizeof(muggle::LatencyBlock));
			blocks[idx].idx = idx;

			timespec_get(&blocks[idx].ts[0], TIME_UTC);
			muggle::LatencyBlock *block = (muggle::LatencyBlock*)malloc(sizeof(muggle::LatencyBlock));
			timespec_get(&blocks[idx].ts[1], TIME_UTC);
			if (block == nullptr)
			{
				printf("failed allocate memory for idx: %llu\n", (unsigned long long)idx);
				continue;
			}

			block->idx = idx;
			muggle_ringbuffer_write(ring, block);
		}
		std::this_thread::sleep_for(std::chrono::milliseconds(config->loop_interval_ms));
	}
}

void fn_consumer(
	muggle_ringbuffer_t *ring,
	muggle_atomic_int *consumer_ready,
	muggle::LatencyBlock *blocks
)
{
	muggle_atomic_int consumer_idx = muggle_atomic_fetch_add(consumer_ready, 1, muggle_memory_order_relaxed);
	muggle_atomic_int pos = 0;
	uint64_t cnt = 0;
	while (1)
	{
		muggle::LatencyBlock *block = (muggle::LatencyBlock*)muggle_ringbuffer_read(ring, pos++);
		if (!block)
		{
			break;
		}

		uint64_t block_idx = block->idx;
		timespec_get(&blocks[block_idx].ts[2], TIME_UTC);
		free(block);
		timespec_get(&blocks[block_idx].ts[3], TIME_UTC);

		++cnt;
	}
	consumer_read_num[consumer_idx] = cnt;
}

void Benchmark_wr(FILE *fp, muggle::BenchmarkConfig &config, int cnt_producer, int cnt_consumer, int flag)
{
	uint64_t cnt = config.loop * config.cnt_per_loop;
	muggle::LatencyBlock *blocks = (muggle::LatencyBlock*)malloc(cnt * sizeof(muggle::LatencyBlock));
	muggle_atomic_int consumer_ready = 0;

	consumer_read_num = (uint64_t*)malloc(sizeof(uint64_t) * cnt_consumer);

	muggle_ringbuffer_t ring;
	muggle_ringbuffer_init(&ring, 1024 * 16, flag);

	std::vector<std::thread> producers;
	std::vector<std::thread> consumers;

	for (int i = 0; i < cnt_producer; ++i)
	{
		uint64_t start_idx = i * (config.cnt_per_loop / cnt_producer);
		uint64_t end_idx = (i + 1) * (config.cnt_per_loop / cnt_producer);
		if (i == cnt_producer - 1)
		{
			end_idx = config.cnt_per_loop;
		}

		producers.push_back(std::thread(fn_producer,
			&ring, &config, blocks, &consumer_ready, cnt_consumer, start_idx, end_idx
		));
	}
	for (int i = 0; i < cnt_consumer; ++i)
	{
		consumers.push_back(std::thread(fn_consumer,
			&ring, &consumer_ready, blocks
		));
	}

	for (auto &producer : producers)
	{
		producer.join();
	}

	muggle_ringbuffer_write(&ring, nullptr);

	for (auto &consumer : consumers)
	{
		consumer.join();
	}

	muggle_ringbuffer_destroy(&ring);

	char buf[128];

	for (int i = 0; i < cnt_consumer; ++i)
	{
		printf("%dw%dr consumer[%d] read %llu %s\n",
			cnt_producer, cnt_consumer, i, (unsigned long long)consumer_read_num[i],
			consumer_read_num[i] == config.cnt_per_loop * config.loop ? "" : "(message loss)");
	}
	free(consumer_read_num);

	snprintf(buf, sizeof(buf) - 1, "%dw%dr-sowr-alloc", cnt_producer, cnt_consumer);
	muggle::GenLatencyReportsBody(fp, &config, blocks, buf, cnt, 0, 1, 0);

	snprintf(buf, sizeof(buf) - 1, "%dw%dr-sowr-alloc-sorted", cnt_producer, cnt_consumer);
	muggle::GenLatencyReportsBody(fp, &config, blocks, buf, cnt, 0, 1, 1);

	snprintf(buf, sizeof(buf) - 1, "%dw%dr-sowr-free", cnt_producer, cnt_consumer);
	muggle::GenLatencyReportsBody(fp, &config, blocks, buf, cnt, 2, 3, 0);

	snprintf(buf, sizeof(buf) - 1, "%dw%dr-sowr-free-sorted", cnt_producer, cnt_consumer);
	muggle::GenLatencyReportsBody(fp, &config, blocks, buf, cnt, 2, 3, 1);

	free(blocks);
}

int main()
{
	muggle::BenchmarkConfig config;
	strncpy(config.name, "c_malloc_free", sizeof(config.name)-1);
	config.loop = 50;
	config.cnt_per_loop = 20000;
	config.loop_interval_ms = 10;
	config.report_step = 10;

	char file_name[128];
	snprintf(file_name, sizeof(file_name)-1, "benchmark_%s.csv", config.name);
	FILE *fp = fopen(file_name, "wb");
	if (fp == nullptr)
	{
		printf("failed open file: %s\n", file_name);
		exit(1);
	}

	muggle::GenLatencyReportsHead(fp, &config);

	int hc = (int)std::thread::hardware_concurrency();
	printf("hardware_concurrency: %d\n", hc);

	int flag = 0;

	// 1 writer, 1 reader
	Benchmark_wr(fp, config, 1, 1, flag);

	// hc write, 1 reader
	Benchmark_wr(fp, config, hc, 1, flag);

	// 2 * hc write, 1 reader
	Benchmark_wr(fp, config, 2 * hc, 1, flag);

	fclose(fp);
}
<commit_msg>remove benchmark_c_malloc_free<commit_after><|endoftext|>


<commit_before>/* test_main.cpp: テスト用プログラムのメイン。
   Copyright (c) 2011 Ishibashi Hironori

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to
   deal in the Software without restriction, including without limitation the
   rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
   sell copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
   IN THE SOFTWARE.
 */

#include <iostream>
#include <fstream>
#include <string>
#include <vector>

#include "misaki.h"

#include "misaki_debug.h"

// テスト用タイトルをプリントする。
void PrintTitle() {
  std::cout << "************" << std::endl;
  std::cout << "* Test Run *" << std::endl;
  std::cout << "************" << std::endl;
}

int main(int argc, char* argv[]) {
  // テスト用タイトルをプリント。
  PrintTitle();

  Misaki::Init();

  Misaki::ChessBoard* board = Misaki::ChessBoard::New();

  Misaki::Move move(Misaki::E2, Misaki::E4);
  board->TakeMove(move);

  int searching_time = 10;
  Misaki::TranspositionTable* table = Misaki::TranspositionTable::New();
  Misaki::EvalWeights weights;
  Misaki::Move best_move = board->GetBestMove(searching_time, *table, weights);

  board->TakeMove(best_move);

  std::cout << *board;

  delete board;

  return 0;
}
<commit_msg>test_main.cppを通常に変更<commit_after>/* test_main.cpp: テスト用プログラムのメイン。
   Copyright (c) 2011 Ishibashi Hironori

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to
   deal in the Software without restriction, including without limitation the
   rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
   sell copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
   IN THE SOFTWARE.
 */

#include <iostream>
#include <fstream>
#include <string>
#include <vector>

#include "misaki.h"

#include "misaki_debug.h"

// テスト用タイトルをプリントする。
void PrintTitle() {
  std::cout << "************" << std::endl;
  std::cout << "* Test Run *" << std::endl;
  std::cout << "************" << std::endl;
}

int main(int argc, char* argv[]) {
  // テスト用タイトルをプリント。
  PrintTitle();

  Misaki::Init();

  Misaki::ChessBoard* board = Misaki::ChessBoard::New();

  board->Test();

  delete board;

  return 0;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014, Red Hat.
 * Copyright (c) 2006, Ken McDonell.  All Rights Reserved.
 * Copyright (c) 2006-2007, Aconex.  All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */
#include <QtGui/QApplication>
#include <pcp/pmapi.h>
#include <pcp/impl.h>
#include "timelord.h"
#include "pmtime.h"

static pmOptions opts;
static pmLongOptions longopts[] = {
    PMAPI_OPTIONS_HEADER("Options"),
    PMOPT_GUIPORT,
    PMOPT_VERSION,
    PMOPT_HELP,
    PMAPI_OPTIONS_END
};

static void setupEnvironment(void)
{
    char *value;
    QString confirm = pmGetConfig("PCP_BIN_DIR");
    confirm.prepend("PCP_XCONFIRM_PROG=");
    confirm.append(QChar(__pmPathSeparator()));
    confirm.append("pmquery");
    if ((value = strdup((const char *)confirm.toAscii())) != NULL)
	putenv(value);
    if (getenv("PCP_STDERR") == NULL &&	// do not overwrite, for QA
	((value = strdup("PCP_STDERR=DISPLAY")) != NULL))
	putenv(value);

    QCoreApplication::setOrganizationName("PCP");
    QCoreApplication::setApplicationName("pmtime");
}

int main(int argc, char **argv)
{
    int		autoport = 0;

    QApplication a(argc, argv);
    setupEnvironment();

    opts.short_options = "D:p:V?";
    opts.long_options = longopts;
    pmGetOptions(argc, argv, &opts);
    if (opts.errors || opts.optind != argc) {
	pmUsageMessage(&opts);
	exit(1);
    }

    if (!opts.guiport) {
	char	*endnum, *envstr;

	autoport = 1;
	if ((envstr = getenv("PMTIME_PORT")) == NULL) {
	    opts.guiport = PmTime::BasePort;
	} else {
	    opts.guiport = strtol(envstr, &endnum, 0);
	    if (*endnum != '\0' || opts.guiport < 0) {
		pmprintf(
		    "%s: PMTIME_PORT must be a numeric port number (not %s)\n",
			pmProgname, envstr);
		pmflush();
		exit(1);
	    }
	}
    }

    console = new Console;
    TimeLord tl(&a);
    do {
	if (tl.listen(QHostAddress::LocalHost, opts.guiport))
	    break;
	opts.guiport++;
    } while (autoport && (opts.guiport >= 0));

    if (!opts.guiport || tl.isListening() == false) {
	if (!autoport)
	    pmprintf("%s: cannot find an available port\n", pmProgname);
	else
	    pmprintf("%s: cannot connect to requested port (%d)\n",
		    pmProgname, opts.guiport);
	pmflush();
	exit(1);
    } else if (autoport) {	/* write to stdout for client */
	char	name[32];
	int	c = snprintf(name, sizeof(name), "port=%u\n", opts.guiport);
	if (write(fileno(stdout), name, c + 1) < 0) {
	    if (errno != EPIPE) {
		pmprintf("%s: cannot write port for client: %s\n",
		    pmProgname, strerror(errno));
		pmflush();
	    }
	    exit(1);
	}
    }

    PmTimeLive hc;
    PmTimeArch ac;
    tl.setContext(&hc, &ac);

    hc.init();
    if (!pmDebug) hc.disableConsole();
    else hc.popup(1);

    ac.init();
    if (!pmDebug) ac.disableConsole();
    else ac.popup(1);

    a.exec();
    return 0;
}
<commit_msg>pmtime: coverity-detected return code check fixes<commit_after>/*
 * Copyright (c) 2014, Red Hat.
 * Copyright (c) 2006, Ken McDonell.  All Rights Reserved.
 * Copyright (c) 2006-2007, Aconex.  All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */
#include <QtGui/QApplication>
#include <pcp/pmapi.h>
#include <pcp/impl.h>
#include "timelord.h"
#include "pmtime.h"

static pmOptions opts;
static pmLongOptions longopts[] = {
    PMAPI_OPTIONS_HEADER("Options"),
    PMOPT_GUIPORT,
    PMOPT_VERSION,
    PMOPT_HELP,
    PMAPI_OPTIONS_END
};

static void setupEnvironment(void)
{
    char *value;
    QString confirm = pmGetConfig("PCP_BIN_DIR");
    confirm.prepend("PCP_XCONFIRM_PROG=");
    confirm.append(QChar(__pmPathSeparator()));
    confirm.append("pmquery");
    if ((value = strdup((const char *)confirm.toAscii())) != NULL)
	putenv(value);
    if (getenv("PCP_STDERR") == NULL &&	// do not overwrite, for QA
	((value = strdup("PCP_STDERR=DISPLAY")) != NULL))
	putenv(value);

    QCoreApplication::setOrganizationName("PCP");
    QCoreApplication::setApplicationName("pmtime");
}

int main(int argc, char **argv)
{
    int		autoport = 0;

    QApplication a(argc, argv);
    setupEnvironment();

    opts.short_options = "D:p:V?";
    opts.long_options = longopts;
    (void)pmGetOptions(argc, argv, &opts);
    if (opts.errors || opts.optind != argc) {
	pmUsageMessage(&opts);
	exit(1);
    }

    if (!opts.guiport) {
	char	*endnum, *envstr;

	autoport = 1;
	if ((envstr = getenv("PMTIME_PORT")) == NULL) {
	    opts.guiport = PmTime::BasePort;
	} else {
	    opts.guiport = strtol(envstr, &endnum, 0);
	    if (*endnum != '\0' || opts.guiport < 0) {
		pmprintf(
		    "%s: PMTIME_PORT must be a numeric port number (not %s)\n",
			pmProgname, envstr);
		pmflush();
		exit(1);
	    }
	}
    }

    console = new Console;
    TimeLord tl(&a);
    do {
	if (tl.listen(QHostAddress::LocalHost, opts.guiport))
	    break;
	opts.guiport++;
    } while (autoport && (opts.guiport >= 0));

    if (!opts.guiport || tl.isListening() == false) {
	if (!autoport)
	    pmprintf("%s: cannot find an available port\n", pmProgname);
	else
	    pmprintf("%s: cannot connect to requested port (%d)\n",
		    pmProgname, opts.guiport);
	pmflush();
	exit(1);
    } else if (autoport) {	/* write to stdout for client */
	char	name[32];
	int	c = snprintf(name, sizeof(name), "port=%u\n", opts.guiport);
	if (write(fileno(stdout), name, c + 1) < 0) {
	    if (errno != EPIPE) {
		pmprintf("%s: cannot write port for client: %s\n",
		    pmProgname, strerror(errno));
		pmflush();
	    }
	    exit(1);
	}
    }

    PmTimeLive hc;
    PmTimeArch ac;
    tl.setContext(&hc, &ac);

    hc.init();
    if (!pmDebug) hc.disableConsole();
    else hc.popup(1);

    ac.init();
    if (!pmDebug) ac.disableConsole();
    else ac.popup(1);

    a.exec();
    return 0;
}
<|endoftext|>


<commit_before>

#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#define DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
#define DOCTEST_CONFIG_USE_STD_HEADERS
#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
#define DOCTEST_CONFIG_NO_EXCEPTIONS
#define DOCTEST_CONFIG_NO_WINDOWS_SEH
#define DOCTEST_CONFIG_NO_POSIX_SIGNALS
// #define DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS

#include <doctest.h>

using std::cout;
using std::endl;

struct can_Message_t {
    uint32_t id    = 0x000;  // 11-bit max is 0x7ff, 29-bit max is 0x1FFFFFFF
    bool isExt     = false;
    bool rtr       = false;
    uint8_t len    = 8;
    uint8_t buf[8] = {0, 0, 0, 0, 0, 0, 0, 0};
};

struct can_Signal_t {
    const uint8_t startBit;
    const uint8_t length;
    const bool isIntel;
    const float factor;
    const float offset;
};

// Fetch a specific signal from the message
template <typename T>
T can_getSignal(can_Message_t msg, const uint8_t startBit, const uint8_t length, const bool isIntel, const float factor, const float offset) {
    uint64_t tempVal = 0;
    uint64_t mask    = (1ULL << length) - 1;

    if (isIntel) {
        std::memcpy(&tempVal, msg.buf, sizeof(tempVal));
        tempVal = (tempVal >> startBit) & mask;
    } else {
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
        std::memcpy(&tempVal, msg.buf, sizeof(tempVal));
        tempVal = (tempVal >> (64 - startBit - length)) & mask;
    }

    T retVal;
    std::memcpy(&retVal, &tempVal, sizeof(T));
    return static_cast<T>((retVal * factor) + offset);
}

template <typename T>
void can_setSignal(can_Message_t& msg, const T& val, const uint8_t startBit, const uint8_t length, const bool isIntel, const float factor, const float offset) {
    T scaledVal        = (val - offset) / factor;
    uint64_t valAsBits = 0;
    std::memcpy(&valAsBits, &scaledVal, sizeof(scaledVal));

    uint64_t mask = (1ULL << length) - 1;

    if (isIntel) {
        uint64_t data = 0;
        std::memcpy(&data, msg.buf, sizeof(data));

        data &= ~(mask << startBit);
        data |= valAsBits << startBit;

        std::memcpy(msg.buf, &data, sizeof(data));
    } else {
        uint64_t data = 0;
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
        std::memcpy(&data, msg.buf, sizeof(data));

        data &= ~(mask << (64 - startBit - length));
        data |= valAsBits << (64 - startBit - length);

        std::memcpy(msg.buf, &data, sizeof(data));
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
    }
}

template <typename T>
T can_getSignal(can_Message_t msg, const can_Signal_t& signal) {
    return can_getSignal<T>(msg, signal.startBit, signal.length, signal.isIntel, signal.factor, signal.offset);
}

template <typename T>
void can_setSignal(can_Message_t& msg, const T& val, const can_Signal_t& signal) {
    can_setSignal(msg, val, signal.startBit, signal.length, signal.isIntel, signal.factor, signal.offset);
}

TEST_CASE("fake") {
    cout << endl;
}

TEST_SUITE("CAN Functions") {
    TEST_CASE("reverse") {
        can_Message_t rxmsg;
        rxmsg.id    = 0x000;
        rxmsg.isExt = false;
        rxmsg.len   = 8;

        rxmsg.buf[0] = 0x12;
        rxmsg.buf[1] = 0x34;

        std::reverse(std::begin(rxmsg.buf), std::end(rxmsg.buf));
        CHECK(rxmsg.buf[0] == 0x00);
        CHECK(rxmsg.buf[6] == 0x34);
        CHECK(rxmsg.buf[7] == 0x12);
    }

    TEST_CASE("getSignal") {
        can_Message_t rxmsg;

        auto val = 0x1234;
        std::memcpy(rxmsg.buf, &val, sizeof(val));

        val = can_getSignal<uint16_t>(rxmsg, 0, 16, true, 1, 0);
        CHECK(val == 0x1234);

        val = can_getSignal<uint16_t>(rxmsg, 0, 16, false, 1, 0);
        CHECK(val == 0x3412);

        float myFloat = 1234.6789f;
        std::memcpy(rxmsg.buf, &myFloat, sizeof(myFloat));
        auto floatVal = can_getSignal<float>(rxmsg, 0, 32, true, 1, 0);
        CHECK(floatVal == 1234.6789f);
    }

    TEST_CASE("setSignal") {
        can_Message_t txmsg;

        can_setSignal<uint16_t>(txmsg, 0x1234, 0, 16, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);

        can_setSignal<uint16_t>(txmsg, 0xABCD, 16, 16, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);

        can_setSignal<float>(txmsg, 1234.5678f, 32, 32, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);
        CHECK(can_getSignal<float>(txmsg, 32, 32, true, 1.0f, 0.0f));

        can_setSignal<uint16_t>(txmsg, 0x1234, 0, 16, false, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, false, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);
        CHECK(can_getSignal<float>(txmsg, 32, 32, true, 1.0f, 0.0f));

        can_setSignal<float>(txmsg, 234981.0f, 12, 32, false, 2.0f, 1.1f);
        CHECK(can_getSignal<float>(txmsg, 12, 32, false, 2.0f, 1.1f) == 234981.0f);
    }
}<commit_msg>Add CAN tests for enums<commit_after>

#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#define DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
#define DOCTEST_CONFIG_USE_STD_HEADERS
#define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS
#define DOCTEST_CONFIG_NO_EXCEPTIONS
#define DOCTEST_CONFIG_NO_WINDOWS_SEH
#define DOCTEST_CONFIG_NO_POSIX_SIGNALS
// #define DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS

#include <doctest.h>

using std::cout;
using std::endl;

struct can_Message_t {
    uint32_t id    = 0x000;  // 11-bit max is 0x7ff, 29-bit max is 0x1FFFFFFF
    bool isExt     = false;
    bool rtr       = false;
    uint8_t len    = 8;
    uint8_t buf[8] = {0, 0, 0, 0, 0, 0, 0, 0};
};

struct can_Signal_t {
    const uint8_t startBit;
    const uint8_t length;
    const bool isIntel;
    const float factor;
    const float offset;
};

enum InputMode_t {
    INPUT_MODE_INACTIVE,
    INPUT_MODE_PASSTHROUGH,
    INPUT_MODE_VEL_RAMP,
    INPUT_MODE_POS_FILTER,
    INPUT_MODE_MIX_CHANNELS,
    INPUT_MODE_TRAP_TRAJ,
};

// Fetch a specific signal from the message
template <typename T>
T can_getSignal(can_Message_t msg, const uint8_t startBit, const uint8_t length, const bool isIntel, const float factor, const float offset) {
    uint64_t tempVal = 0;
    uint64_t mask    = (1ULL << length) - 1;

    if (isIntel) {
        std::memcpy(&tempVal, msg.buf, sizeof(tempVal));
        tempVal = (tempVal >> startBit) & mask;
    } else {
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
        std::memcpy(&tempVal, msg.buf, sizeof(tempVal));
        tempVal = (tempVal >> (64 - startBit - length)) & mask;
    }

    T retVal;
    std::memcpy(&retVal, &tempVal, sizeof(T));
    return static_cast<T>((retVal * factor) + offset);
}

template <typename T>
void can_setSignal(can_Message_t& msg, const T& val, const uint8_t startBit, const uint8_t length, const bool isIntel, const float factor, const float offset) {
    T scaledVal        = (val - offset) / factor;
    uint64_t valAsBits = 0;
    std::memcpy(&valAsBits, &scaledVal, sizeof(scaledVal));

    uint64_t mask = (1ULL << length) - 1;

    if (isIntel) {
        uint64_t data = 0;
        std::memcpy(&data, msg.buf, sizeof(data));

        data &= ~(mask << startBit);
        data |= valAsBits << startBit;

        std::memcpy(msg.buf, &data, sizeof(data));
    } else {
        uint64_t data = 0;
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
        std::memcpy(&data, msg.buf, sizeof(data));

        data &= ~(mask << (64 - startBit - length));
        data |= valAsBits << (64 - startBit - length);

        std::memcpy(msg.buf, &data, sizeof(data));
        std::reverse(std::begin(msg.buf), std::end(msg.buf));
    }
}

template <typename T>
T can_getSignal(can_Message_t msg, const can_Signal_t& signal) {
    return can_getSignal<T>(msg, signal.startBit, signal.length, signal.isIntel, signal.factor, signal.offset);
}

template <typename T>
void can_setSignal(can_Message_t& msg, const T& val, const can_Signal_t& signal) {
    can_setSignal(msg, val, signal.startBit, signal.length, signal.isIntel, signal.factor, signal.offset);
}

TEST_CASE("fake") {
    cout << endl;
}

TEST_SUITE("CAN Functions") {
    TEST_CASE("reverse") {
        can_Message_t rxmsg;
        rxmsg.id    = 0x000;
        rxmsg.isExt = false;
        rxmsg.len   = 8;

        rxmsg.buf[0] = 0x12;
        rxmsg.buf[1] = 0x34;

        std::reverse(std::begin(rxmsg.buf), std::end(rxmsg.buf));
        CHECK(rxmsg.buf[0] == 0x00);
        CHECK(rxmsg.buf[6] == 0x34);
        CHECK(rxmsg.buf[7] == 0x12);
    }

    TEST_CASE("getSignal") {
        can_Message_t rxmsg;

        auto val = 0x1234;
        std::memcpy(rxmsg.buf, &val, sizeof(val));

        val = can_getSignal<uint16_t>(rxmsg, 0, 16, true, 1, 0);
        CHECK(val == 0x1234);

        val = can_getSignal<uint16_t>(rxmsg, 0, 16, false, 1, 0);
        CHECK(val == 0x3412);

        float myFloat = 1234.6789f;
        std::memcpy(rxmsg.buf, &myFloat, sizeof(myFloat));
        auto floatVal = can_getSignal<float>(rxmsg, 0, 32, true, 1, 0);
        CHECK(floatVal == 1234.6789f);
    }

    TEST_CASE("setSignal") {
        can_Message_t txmsg;

        can_setSignal<uint16_t>(txmsg, 0x1234, 0, 16, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);

        can_setSignal<uint16_t>(txmsg, 0xABCD, 16, 16, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);

        can_setSignal<float>(txmsg, 1234.5678f, 32, 32, true, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, true, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);
        CHECK(can_getSignal<float>(txmsg, 32, 32, true, 1.0f, 0.0f));

        can_setSignal<uint16_t>(txmsg, 0x1234, 0, 16, false, 1.0f, 0.0f);
        CHECK(can_getSignal<uint16_t>(txmsg, 0, 16, false, 1.0f, 0.0f) == 0x1234);
        CHECK(can_getSignal<uint16_t>(txmsg, 16, 16, true, 1.0f, 0.0f) == 0xABCD);
        CHECK(can_getSignal<float>(txmsg, 32, 32, true, 1.0f, 0.0f));

        can_setSignal<float>(txmsg, 234981.0f, 12, 32, false, 2.0f, 1.1f);
        CHECK(can_getSignal<float>(txmsg, 12, 32, false, 2.0f, 1.1f) == 234981.0f);
    }

    TEST_CASE("getSignal enums") {
        can_Message_t rxmsg;
        rxmsg.buf[0] = INPUT_MODE_MIX_CHANNELS;
        rxmsg.buf[1] = INPUT_MODE_PASSTHROUGH;
        CHECK(can_getSignal<InputMode_t>(rxmsg, 0, 8, true, 1, 0) == INPUT_MODE_MIX_CHANNELS);
        CHECK(can_getSignal<InputMode_t>(rxmsg, 8, 8, true, 1, 0) == INPUT_MODE_PASSTHROUGH);
    }
}<|endoftext|>


<commit_before>#ifndef FIXED_ALLOCATION_PATTERN_HPP
#define FIXED_ALLOCATION_PATTERN_HPP

#include <type_traits>
#include "fixed/impl/fixed_def.hpp"
#include "fixed/impl/aligned_allocation_pattern.hpp"
#include "fixed/impl/constexpr_allocation_pattern.hpp"

namespace fixed
{
	namespace _impl
	{
		template <bool B,
			template <typename, fixed::_impl::container_size_type> typename T,
			template <typename, fixed::_impl::container_size_type> typename F,
			typename VALUE, fixed::_impl::container_size_type SIZE>
		struct allocator_dispatch
		{
			typedef typename T<VALUE, SIZE> type;
		};

		template <
			template <typename, fixed::_impl::container_size_type> typename T,
			template <typename, fixed::_impl::container_size_type> typename F,
			typename VALUE, fixed::_impl::container_size_type SIZE>
		struct allocator_dispatch<false, T, F, VALUE, SIZE>
		{
			 typedef typename F<VALUE, SIZE> type;
		};
	}

	template <typename T, fixed::_impl::container_size_type SIZE>
	using stack_allocator = typename _impl::allocator_dispatch<
		astd::is_trivial_v<T>,
		fixed::_impl::constexpr_stack_allocator,
		fixed::_impl::aligned_stack_allocator,
		T, SIZE>::type;

	template <typename T, fixed::_impl::container_size_type SIZE>
	using heap_allocator = fixed::_impl::aligned_heap_allocator<T, SIZE>;
}

#endif //!FIXED_ALLOCATION_PATTERN_HPP<commit_msg>fix error: expected nested-name-specifier<commit_after>#ifndef FIXED_ALLOCATION_PATTERN_HPP
#define FIXED_ALLOCATION_PATTERN_HPP

#include <type_traits>
#include "fixed/impl/fixed_def.hpp"
#include "fixed/impl/aligned_allocation_pattern.hpp"
#include "fixed/impl/constexpr_allocation_pattern.hpp"

namespace fixed
{
	namespace _impl
	{
		template <bool B,
			template <typename, fixed::_impl::container_size_type> typename T,
			template <typename, fixed::_impl::container_size_type> typename F,
			typename VALUE, fixed::_impl::container_size_type SIZE>
		struct allocator_dispatch
		{
			typedef T<VALUE, SIZE> type;
		};

		template <
			template <typename, fixed::_impl::container_size_type> typename T,
			template <typename, fixed::_impl::container_size_type> typename F,
			typename VALUE, fixed::_impl::container_size_type SIZE>
		struct allocator_dispatch<false, T, F, VALUE, SIZE>
		{
			 typedef F<VALUE, SIZE> type;
		};
	}

	template <typename T, fixed::_impl::container_size_type SIZE>
	using stack_allocator = typename _impl::allocator_dispatch<
		astd::is_trivial_v<T>,
		fixed::_impl::constexpr_stack_allocator,
		fixed::_impl::aligned_stack_allocator,
		T, SIZE>::type;

	template <typename T, fixed::_impl::container_size_type SIZE>
	using heap_allocator = fixed::_impl::aligned_heap_allocator<T, SIZE>;
}

#endif //!FIXED_ALLOCATION_PATTERN_HPP<|endoftext|>


<commit_before>#ifndef MAE_READER_HPP_
#define MAE_READER_HPP_

#include <cerrno>
#include <cstdio>
#include <map>
#include <stdexcept>
#include <string>

#include <boost/dynamic_bitset.hpp>
#include <utility>

#include "Buffer.hpp"
#include "MaeBlock.hpp"
#include "MaeParserConfig.hpp"

#define MAEPARSER_EXCEPTION_BUFFER_SIZE 256

namespace schrodinger
{

namespace mae
{

/**
 * Parse (and throw away) a comment of the form '# comment #'.
 */
void comment(Buffer& buffer);

/**
 * Parse (and throw away) zero or more characters of whitespace including \t,
 * \r, \n and ' ', along with any embedded comments.
 */
EXPORT_MAEPARSER void whitespace(Buffer& buffer);

/**
 * Parse a triple colon or raise an exception.
 */
void triple_colon(Buffer& buffer);

/**
 * Parse the specific character requested. Return true if successful, false if
 * not.
 */
bool character(char c, Buffer& buffer);

/**
 * Parse the specific character requested. Return true if successful, false if
 * not. Update any save points if buffer reload is required.
 */
bool character(char c, Buffer& buffer, char*& save);

/**
 * Parse a full (b|i|r|s)_<author>_<name> property key.
 *
 * Return NULL if a starting character of ':' is found (the beginning of the
 * ':::' property name terminator).
 *
 * Raise a read_exception in any other situation.
 */
std::shared_ptr<std::string> property_key(Buffer& buffer);

/**
 * Read through the opening '{' of a named or unnamed outer block.
 *
 * Set the name of the block in the provided argument; an empty string
 * if unnamed. If EOF is reached, return false, otherwise return true.
 */
EXPORT_MAEPARSER std::string outer_block_beginning(Buffer& buffer);

template <typename T> T parse_value(Buffer& buffer);

class EXPORT_MAEPARSER read_exception : public std::exception
{
  private:
    char m_msg[MAEPARSER_EXCEPTION_BUFFER_SIZE];

    void format(size_t line_number, size_t column, const char* msg);

  public:
    read_exception(const Buffer& buffer, const char* msg)
    {
        format(buffer.line_number, buffer.getColumn(), msg);
    }

    read_exception(size_t line_number, size_t column, const char* msg)
    {
        format(line_number, column, msg);
    }

    const char* what() const noexcept override { return m_msg; }
};

/**
 * A pure virtual base class for parsers. Allows us to store these in
 * collections, so we can do things like easily invoke different parsers on
 * each element while parsing a row of values.
 */
class EXPORT_MAEPARSER Parser
{
  public:
    virtual void parse(Buffer& buffer) = 0;
    virtual ~Parser() = default;
    ;
};

class EXPORT_MAEPARSER IndexedBlockParser
{
    std::vector<std::string> m_property_names;

  public:
    virtual ~IndexedBlockParser() = default;
    ;

    virtual void parse(const std::string& name, size_t size,
                       Buffer& buffer) = 0;

    virtual std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() = 0;
};

class EXPORT_MAEPARSER IndexedBlockBuffer
{
  private:
    std::vector<std::string> m_property_names;
    std::string m_name;
    TokenBufferList m_tokens_list;
    size_t m_rows;

  public:
    IndexedBlockBuffer(std::string name, size_t rows)
        : m_property_names(), m_name(std::move(name)), m_rows(rows)
    {
    }

    virtual ~IndexedBlockBuffer() = default;

    void addPropertyName(std::string&& name)
    {
        m_property_names.push_back(name);
    }

    /**
     * Parse the indexed block values, store them in a linked list of buffers.
     */
    virtual void parse(Buffer& buffer);

    /**
     * Parse a value.
     */
    void value(Buffer& buffer);

    void getData(size_t ix, const char** data, size_t* const len) const
    {
        m_tokens_list.getData(ix, data, len);
    }

    std::string getName() const { return m_name; }

    size_t size() const { return m_rows; }

    IndexedBlock* getIndexedBlock();
};

class EXPORT_MAEPARSER BufferedIndexedBlockParser : public IndexedBlockParser
{
  private:
    std::shared_ptr<BufferedIndexedBlockMap> m_indexed_block_map;

  public:
    BufferedIndexedBlockParser();

    std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() override;

    void parse(const std::string& name, size_t size, Buffer& buffer) override;
};

class EXPORT_MAEPARSER DirectIndexedBlockParser : public IndexedBlockParser
{
    std::shared_ptr<IndexedBlockMap> m_indexed_block_map;

  public:
    void parse(const std::string& name, size_t size, Buffer& buffer) override;

    std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() override;
};

class EXPORT_MAEPARSER IndexedValueParser : public Parser
{
  public:
    virtual void addToIndexedBlock(IndexedBlock* block) = 0;
};

template <typename T> class IndexedValueCollector : public IndexedValueParser
{
  public:
    std::string m_name;
    std::vector<T> m_values;
    boost::dynamic_bitset<>* m_is_null;

  public:
    explicit IndexedValueCollector(std::string name, size_t size)
        : m_name(std::move(name)), m_values(), m_is_null(nullptr)
    {
        m_values.reserve(size);
        m_is_null = nullptr;
    }

    ~IndexedValueCollector() override
    {
        if (m_is_null) {
            delete m_is_null;
        }
    }

    void parse(Buffer& buffer) override
    {
        if (buffer.current >= buffer.end) {
            if (!buffer.load()) {
                throw read_exception(buffer, "Unexpected EOF.");
            }
        }
        if (*buffer.current == '<') {
            char* save = buffer.current;
            ++buffer.current;
            if (buffer.current >= buffer.end) {
                if (!buffer.load(save)) {
                    throw read_exception(buffer, "Unexpected EOF.");
                }
            }
            // TODO: not sure, but I assume that unquoted strings like
            // <foo> are allowed as values. Ugh. This requires saving the
            // starting point of '<' in case we need to back up.
            if (*buffer.current != '>') {
                // Back up and parse as a normal value.
                --buffer.current;
            } else {
                ++buffer.current;
                if (m_is_null == nullptr) {
                    m_is_null =
                        new boost::dynamic_bitset<>(m_values.capacity());
                }
                m_is_null->set(m_values.size());
                m_values.push_back(T());
                return;
            }
        }
        m_values.push_back(parse_value<T>(buffer));
    }

    void addToIndexedBlock(IndexedBlock* block) override
    {
        auto ptr = std::shared_ptr<IndexedProperty<T>>(
            new IndexedProperty<T>(m_values, m_is_null));
        block->setProperty<T>(m_name, ptr);
        m_is_null = nullptr;
    }
};

class EXPORT_MAEPARSER MaeParser
{
  protected:
    Buffer m_buffer;
    std::shared_ptr<std::istream> m_stream;

    virtual IndexedBlockParser* getIndexedBlockParser()
    {
        return new BufferedIndexedBlockParser();
    }

  public:
    explicit MaeParser(const std::shared_ptr<std::istream>& stream,
                       size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : m_buffer(*stream, buffer_size), m_stream(stream)
    {
        m_buffer.load();
    }

    explicit MaeParser(FILE* file,
                       size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : m_buffer(file, buffer_size)
    {
        if (file == nullptr) {
            std::string msg("Bad file argument");
            if (errno != 0) {
                msg += ": ";
                msg += strerror(errno);
            } else {
                msg += ".";
            }
            throw std::runtime_error(msg);
        }
        m_buffer.load();
    }

    // TODO: finish big three (four)
    virtual ~MaeParser() = default;

    std::shared_ptr<Block> blockBody(const std::string& name);

    IndexedBlock* indexedBlock(const std::string& name, size_t size);

    // TODO: private
    IndexedBlockBuffer* indexedBlockBuffer(const std::string& name,
                                           size_t size);

    std::shared_ptr<Block> outerBlock();

    /**
     * Read a block name or a closing '}'. The argument 'indexed' is set to
     * a positive integer value indicating the number of rows, or zero if
     * the block is not indexed.
     *
     * Return the block name or NULL if the closing '}' was found.
     */
    std::string blockBeginning(int* indexed);

    std::shared_ptr<std::string> property();

    /**
     * Read a list of properties, ending at the name/value separator.
     * Populate the provided std::vector of std::string shared pointers.
     */
    void properties(std::vector<std::shared_ptr<std::string>>* property_names);

    /**
     * Read (and throw away) any whitespace.
     */
    void whitespace() { schrodinger::mae::whitespace(m_buffer); }
};

class EXPORT_MAEPARSER DirectMaeParser : public MaeParser
{
  public:
    explicit DirectMaeParser(const std::shared_ptr<std::istream>& stream,
                             size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : MaeParser(stream, buffer_size)
    {
    }

    explicit DirectMaeParser(FILE* file,
                             size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : MaeParser(file, buffer_size)
    {
    }

  private:
    IndexedBlockParser* getIndexedBlockParser() override
    {
        return new DirectIndexedBlockParser();
    }
};

} // end namespace mae

} // end namespace schrodinger

#endif
<commit_msg>workaround VS2012 noexcept<commit_after>#ifndef MAE_READER_HPP_
#define MAE_READER_HPP_

// Visual Studio versions prior to 2015 don't support noexcept
#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023026
#define NOEXCEPT
#else
#define NOEXCEPT noexcept
#endif

#include <cerrno>
#include <cstdio>
#include <map>
#include <stdexcept>
#include <string>

#include <boost/dynamic_bitset.hpp>
#include <utility>

#include "Buffer.hpp"
#include "MaeBlock.hpp"
#include "MaeParserConfig.hpp"

#define MAEPARSER_EXCEPTION_BUFFER_SIZE 256

namespace schrodinger
{

namespace mae
{

/**
 * Parse (and throw away) a comment of the form '# comment #'.
 */
void comment(Buffer& buffer);

/**
 * Parse (and throw away) zero or more characters of whitespace including \t,
 * \r, \n and ' ', along with any embedded comments.
 */
EXPORT_MAEPARSER void whitespace(Buffer& buffer);

/**
 * Parse a triple colon or raise an exception.
 */
void triple_colon(Buffer& buffer);

/**
 * Parse the specific character requested. Return true if successful, false if
 * not.
 */
bool character(char c, Buffer& buffer);

/**
 * Parse the specific character requested. Return true if successful, false if
 * not. Update any save points if buffer reload is required.
 */
bool character(char c, Buffer& buffer, char*& save);

/**
 * Parse a full (b|i|r|s)_<author>_<name> property key.
 *
 * Return NULL if a starting character of ':' is found (the beginning of the
 * ':::' property name terminator).
 *
 * Raise a read_exception in any other situation.
 */
std::shared_ptr<std::string> property_key(Buffer& buffer);

/**
 * Read through the opening '{' of a named or unnamed outer block.
 *
 * Set the name of the block in the provided argument; an empty string
 * if unnamed. If EOF is reached, return false, otherwise return true.
 */
EXPORT_MAEPARSER std::string outer_block_beginning(Buffer& buffer);

template <typename T> T parse_value(Buffer& buffer);

class EXPORT_MAEPARSER read_exception : public std::exception
{
  private:
    char m_msg[MAEPARSER_EXCEPTION_BUFFER_SIZE];

    void format(size_t line_number, size_t column, const char* msg);

  public:
    read_exception(const Buffer& buffer, const char* msg)
    {
        format(buffer.line_number, buffer.getColumn(), msg);
    }

    read_exception(size_t line_number, size_t column, const char* msg)
    {
        format(line_number, column, msg);
    }

    const char* what() const NOEXCEPT override { return m_msg; }
};

/**
 * A pure virtual base class for parsers. Allows us to store these in
 * collections, so we can do things like easily invoke different parsers on
 * each element while parsing a row of values.
 */
class EXPORT_MAEPARSER Parser
{
  public:
    virtual void parse(Buffer& buffer) = 0;
    virtual ~Parser() = default;
};

class EXPORT_MAEPARSER IndexedBlockParser
{
    std::vector<std::string> m_property_names;

  public:
    virtual ~IndexedBlockParser() = default;

    virtual void parse(const std::string& name, size_t size,
                       Buffer& buffer) = 0;

    virtual std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() = 0;
};

class EXPORT_MAEPARSER IndexedBlockBuffer
{
  private:
    std::vector<std::string> m_property_names;
    std::string m_name;
    TokenBufferList m_tokens_list;
    size_t m_rows;

  public:
    IndexedBlockBuffer(std::string name, size_t rows)
        : m_property_names(), m_name(std::move(name)), m_rows(rows)
    {
    }

    virtual ~IndexedBlockBuffer() = default;

    void addPropertyName(std::string&& name)
    {
        m_property_names.push_back(name);
    }

    /**
     * Parse the indexed block values, store them in a linked list of buffers.
     */
    virtual void parse(Buffer& buffer);

    /**
     * Parse a value.
     */
    void value(Buffer& buffer);

    void getData(size_t ix, const char** data, size_t* const len) const
    {
        m_tokens_list.getData(ix, data, len);
    }

    std::string getName() const { return m_name; }

    size_t size() const { return m_rows; }

    IndexedBlock* getIndexedBlock();
};

class EXPORT_MAEPARSER BufferedIndexedBlockParser : public IndexedBlockParser
{
  private:
    std::shared_ptr<BufferedIndexedBlockMap> m_indexed_block_map;

  public:
    BufferedIndexedBlockParser();

    std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() override;

    void parse(const std::string& name, size_t size, Buffer& buffer) override;
};

class EXPORT_MAEPARSER DirectIndexedBlockParser : public IndexedBlockParser
{
    std::shared_ptr<IndexedBlockMap> m_indexed_block_map;

  public:
    void parse(const std::string& name, size_t size, Buffer& buffer) override;

    std::shared_ptr<IndexedBlockMapI> getIndexedBlockMap() override;
};

class EXPORT_MAEPARSER IndexedValueParser : public Parser
{
  public:
    virtual void addToIndexedBlock(IndexedBlock* block) = 0;
};

template <typename T> class IndexedValueCollector : public IndexedValueParser
{
  public:
    std::string m_name;
    std::vector<T> m_values;
    boost::dynamic_bitset<>* m_is_null;

  public:
    explicit IndexedValueCollector(std::string name, size_t size)
        : m_name(std::move(name)), m_values(), m_is_null(nullptr)
    {
        m_values.reserve(size);
        m_is_null = nullptr;
    }

    ~IndexedValueCollector() override
    {
        if (m_is_null) {
            delete m_is_null;
        }
    }

    void parse(Buffer& buffer) override
    {
        if (buffer.current >= buffer.end) {
            if (!buffer.load()) {
                throw read_exception(buffer, "Unexpected EOF.");
            }
        }
        if (*buffer.current == '<') {
            char* save = buffer.current;
            ++buffer.current;
            if (buffer.current >= buffer.end) {
                if (!buffer.load(save)) {
                    throw read_exception(buffer, "Unexpected EOF.");
                }
            }
            // TODO: not sure, but I assume that unquoted strings like
            // <foo> are allowed as values. Ugh. This requires saving the
            // starting point of '<' in case we need to back up.
            if (*buffer.current != '>') {
                // Back up and parse as a normal value.
                --buffer.current;
            } else {
                ++buffer.current;
                if (m_is_null == nullptr) {
                    m_is_null =
                        new boost::dynamic_bitset<>(m_values.capacity());
                }
                m_is_null->set(m_values.size());
                m_values.push_back(T());
                return;
            }
        }
        m_values.push_back(parse_value<T>(buffer));
    }

    void addToIndexedBlock(IndexedBlock* block) override
    {
        auto ptr = std::shared_ptr<IndexedProperty<T>>(
            new IndexedProperty<T>(m_values, m_is_null));
        block->setProperty<T>(m_name, ptr);
        m_is_null = nullptr;
    }
};

class EXPORT_MAEPARSER MaeParser
{
  protected:
    Buffer m_buffer;
    std::shared_ptr<std::istream> m_stream;

    virtual IndexedBlockParser* getIndexedBlockParser()
    {
        return new BufferedIndexedBlockParser();
    }

  public:
    explicit MaeParser(const std::shared_ptr<std::istream>& stream,
                       size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : m_buffer(*stream, buffer_size), m_stream(stream)
    {
        m_buffer.load();
    }

    explicit MaeParser(FILE* file,
                       size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : m_buffer(file, buffer_size)
    {
        if (file == nullptr) {
            std::string msg("Bad file argument");
            if (errno != 0) {
                msg += ": ";
                msg += strerror(errno);
            } else {
                msg += ".";
            }
            throw std::runtime_error(msg);
        }
        m_buffer.load();
    }

    // TODO: finish big three (four)
    virtual ~MaeParser() = default;

    std::shared_ptr<Block> blockBody(const std::string& name);

    IndexedBlock* indexedBlock(const std::string& name, size_t size);

    // TODO: private
    IndexedBlockBuffer* indexedBlockBuffer(const std::string& name,
                                           size_t size);

    std::shared_ptr<Block> outerBlock();

    /**
     * Read a block name or a closing '}'. The argument 'indexed' is set to
     * a positive integer value indicating the number of rows, or zero if
     * the block is not indexed.
     *
     * Return the block name or NULL if the closing '}' was found.
     */
    std::string blockBeginning(int* indexed);

    std::shared_ptr<std::string> property();

    /**
     * Read a list of properties, ending at the name/value separator.
     * Populate the provided std::vector of std::string shared pointers.
     */
    void properties(std::vector<std::shared_ptr<std::string>>* property_names);

    /**
     * Read (and throw away) any whitespace.
     */
    void whitespace() { schrodinger::mae::whitespace(m_buffer); }
};

class EXPORT_MAEPARSER DirectMaeParser : public MaeParser
{
  public:
    explicit DirectMaeParser(const std::shared_ptr<std::istream>& stream,
                             size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : MaeParser(stream, buffer_size)
    {
    }

    explicit DirectMaeParser(FILE* file,
                             size_t buffer_size = BufferLoader::DEFAULT_SIZE)
        : MaeParser(file, buffer_size)
    {
    }

  private:
    IndexedBlockParser* getIndexedBlockParser() override
    {
        return new DirectIndexedBlockParser();
    }
};

} // end namespace mae

} // end namespace schrodinger

#endif
<|endoftext|>


<commit_before>#define LOG_PREFIX "openreil"

#include<stdlib.h>
#include<string.h>
#include<stdint.h>
#include<assert.h>
#include<string>
#include<vector>
#include<unordered_map>
#include<libopenreil.h>

#include"utils.h"
#include"expr.h"
#include"vm.h"
#include"state.h"
#include"arch/openreil/openreil.h"

using namespace openreil;

static std::vector<reil_inst_t> instlist;
static const char *REGNAME[] = {
    "R_EAX",
    "R_EBX",
    "R_ECX",
    "R_EDX",
    "R_EDI",
    "R_ESI",
    "R_EBP",
    "R_ESP",

    "R_EFLAGS",
    "R_DFLAG",

    "R_CF",
    "R_PF",
    "R_AF",
    "R_ZF",
    "R_SF",
    "R_OF",
};
static const unsigned int REGSIZE[] = {
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    1,
    1,
    1,
    1,
    1,
    1,
};
static const unsigned int REILSIZE[] = {
    1,
    8,
    16,
    32,
    64,
};

Context::Context(const char *_exe_path)
    : symx::Context(256,64),container_path("."),exe_path(_exe_path)
{}

VirtualMachine* Context::create_vm() {
    VirtualMachine *vm = new VirtualMachine();
    const char *argv[] = {exe_path,NULL};
    if(vm->create(container_path,exe_path,argv)) {
	err("virtual machine create failed\n");
    }
    return vm;
}
int Context::destroy_vm(symx::VirtualMachine *vm) {
    vm->destroy();
    delete vm;
    return 0;
}
uint64_t VirtualMachine::event_get_pc() const {
    assert(com_mem->evt == VMCOM_EVT_EXECUTE);
    return com_mem->context.pc;
}
symx::refSnapshot VirtualMachine::event_suspend() {
    int i;
    uint64_t reg[REGIDX_END];

    if(set_state(SUSPEND)) {
	return nullptr;
    }

    for(i = 0;i < REGIDX_EFLAGS;i++) {
	reg[i] = com_mem->context.reg[i];
    }

    reg[REGIDX_EFLAGS] = com_mem->context.flag;
    reg[REGIDX_DFLAG] = (com_mem->context.flag >> 10) & 0x1;

    reg[REGIDX_CF] = (com_mem->context.flag >> 0) & 0x1;
    reg[REGIDX_PF] = (com_mem->context.flag >> 2) & 0x1;
    reg[REGIDX_AF] = (com_mem->context.flag >> 4) & 0x1;
    reg[REGIDX_ZF] = (com_mem->context.flag >> 6) & 0x1;
    reg[REGIDX_SF] = (com_mem->context.flag >> 7) & 0x1;
    reg[REGIDX_OF] = (com_mem->context.flag >> 11) & 0x1;

    return ref<Snapshot>(this,reg);
}
int VirtualMachine::mem_read(uint8_t *buf,uint64_t pos,size_t len) {
    assert(len < sizeof(com_mem->membuf.buf));

    com_mem->membuf.pos = (uint32_t)pos;
    com_mem->membuf.len = len;
    event_send(VMCOM_EVT_READMEM);
    event_wait();

    memcpy(buf,com_mem->membuf.buf,len);
    return 0;
}

Snapshot::Snapshot(VirtualMachine *_vm,const uint64_t *_reg)
    : symx::Snapshot(CS_ARCH_X86,CS_MODE_32),vm(_vm)
{
    int i;
    for(i = 0;i < REGIDX_END;i++) {
	reg.push_back(symx::BytVec::create_imm(REGSIZE[i],_reg[i]));
    }
}
int Snapshot::mem_read(uint8_t *buf,uint64_t pos,size_t len) const {
    return vm->mem_read(buf,pos,len);
}

#define MAX_ARG_STR 50
const char *inst_op[] = {
    "NONE", "UNK", "JCC",
    "STR", "STM", "LDM",
    "ADD", "SUB", "NEG", "MUL", "DIV", "MOD", "SMUL", "SDIV", "SMOD",
    "SHL", "SHR", "AND", "OR", "XOR", "NOT",
    "EQ", "LT"
};
int arg_size[] = { 1, 8, 16, 32, 64 };
char *arg_print(reil_arg_t *arg,char *arg_str) {
    memset(arg_str, 0, MAX_ARG_STR);
    switch (arg->type) {
    case A_NONE:
        break;
    case A_REG:
    case A_TEMP:
        snprintf(
		arg_str,
		MAX_ARG_STR - 1,
		"%s:%d",
		arg->name,
		arg_size[arg->size]);
        break;
    case A_CONST:
        snprintf(
		arg_str,
		MAX_ARG_STR - 1,
		"%llx:%d",
		arg->val,
		arg_size[arg->size]);
        break;
    }
    return arg_str;
}
void inst_print(const std::vector<reil_inst_t>::iterator &inst) {
    char arg_str[3][MAX_ARG_STR];
    arg_print(&inst->a, arg_str[0]);
    arg_print(&inst->b, arg_str[1]);
    arg_print(&inst->c, arg_str[2]);
    dbg("%.8llx.%.2x %7s %16s, %16s, %16s\n",
	    inst->raw_info.addr,
	    inst->inum,
	    inst_op[inst->op],
	    arg_str[0],
	    arg_str[1],
	    arg_str[2]);
}
int Snapshot::inst_handler(reil_inst_t *inst,void *ctx) {
    instlist.push_back(*inst);
    return 0;
}
symx::refBlock Snapshot::translate(
	uint8_t *code,
	const symx::ProgCtr &pc,
	size_t len
) const {
    int i;
    uint64_t rawpc = pc.rawpc;
    reil_t reil;
    int translated;

    symx::refExpr mem;
    std::vector<symx::refExpr> reglist;
    std::unordered_map<std::string,symx::refExpr> regmap;
    symx::refExpr exra,exrb,exrc;
    std::vector<symx::ProgCtr> nextpc;

    //initialize openreil, translate to reil IR
    reil = reil_init(ARCH_X86,inst_handler,(void*)&translated);
    instlist.clear();
    reil_translate(reil,rawpc,code,len);
    reil_close(reil);

    //initialize dangle memory, register
    mem = symx::BytMem::create_dangle(-1);
    for(i = 0;i < REGIDX_END;i++) {
	auto reg = symx::BytVec::create_dangle(REGSIZE[i],i);
	reglist.push_back(reg);
	regmap[REGNAME[i]] = reg;
    }

    for(auto ins = instlist.begin();ins != instlist.end();ins++) {
	inst_print(ins);
	switch(ins->op) {
	    case I_STR:
		translate_set_arg(
			&regmap,
			ins->c,
			translate_get_arg(regmap,ins->a));
		break;
	    case I_ADD:
		exra = translate_get_arg(regmap,ins->a);
		exrb = translate_get_arg(regmap,ins->b);
		translate_set_arg(
			&regmap,
			ins->c,
			symx::expr_add(exra,exrb));
		break;
	    default:
		err("unsupport unstruction %d\n",ins->op);
		break;
	}
    }

    instlist.clear();

    return ref<symx::Block>(nextpc);
}
symx::refExpr Snapshot::translate_get_arg(
	const std::unordered_map<std::string,symx::refExpr> &regmap,
	const reil_arg_t &arg) const {

    assert(arg.type == A_REG || arg.type == A_TEMP || arg.type == A_CONST);

    if(arg.type == A_REG || arg.type == A_TEMP) {
	auto it = regmap.find(arg.name);
	assert(it != regmap.end());
	assert(it->second->size == REILSIZE[arg.size]);
	return it->second;
    } else {
	return symx::BytVec::create_imm(REILSIZE[arg.size],arg.val);
    }
}
int Snapshot::translate_set_arg(
	std::unordered_map<std::string,symx::refExpr> *regmap,
	const reil_arg_t &arg,
	const symx::refExpr &value) const {

    assert(arg.type == A_REG || arg.type == A_TEMP);

    regmap->insert(std::make_pair(arg.name,value));
    return 0;
}

<commit_msg>Add many instructions support<commit_after>#define LOG_PREFIX "openreil"

#include<stdlib.h>
#include<string.h>
#include<stdint.h>
#include<assert.h>
#include<string>
#include<vector>
#include<unordered_map>
#include<libopenreil.h>

#include"utils.h"
#include"expr.h"
#include"vm.h"
#include"state.h"
#include"arch/openreil/openreil.h"

using namespace openreil;

static std::vector<reil_inst_t> instlist;
static const char *REGNAME[] = {
    "R_EAX",
    "R_EBX",
    "R_ECX",
    "R_EDX",
    "R_EDI",
    "R_ESI",
    "R_EBP",
    "R_ESP",

    "R_EFLAGS",
    "R_DFLAG",

    "R_CF",
    "R_PF",
    "R_AF",
    "R_ZF",
    "R_SF",
    "R_OF",
};
static const unsigned int REGSIZE[] = {
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    32,
    1,
    1,
    1,
    1,
    1,
    1,
};
static const unsigned int REILSIZE[] = {
    1,
    8,
    16,
    32,
    64,
};
static symx::refExpr IMMFALSE = symx::BytVec::create_imm(1,0x0);
static symx::refExpr IMMTRUE = symx::BytVec::create_imm(1,0x1);

Context::Context(const char *_exe_path)
    : symx::Context(256,64),container_path("."),exe_path(_exe_path)
{}

VirtualMachine* Context::create_vm() {
    VirtualMachine *vm = new VirtualMachine();
    const char *argv[] = {exe_path,NULL};
    if(vm->create(container_path,exe_path,argv)) {
	err("virtual machine create failed\n");
    }
    return vm;
}
int Context::destroy_vm(symx::VirtualMachine *vm) {
    vm->destroy();
    delete vm;
    return 0;
}
uint64_t VirtualMachine::event_get_pc() const {
    assert(com_mem->evt == VMCOM_EVT_EXECUTE);
    return com_mem->context.pc;
}
symx::refSnapshot VirtualMachine::event_suspend() {
    int i;
    uint64_t reg[REGIDX_END];

    if(set_state(SUSPEND)) {
	return nullptr;
    }

    for(i = 0;i < REGIDX_EFLAGS;i++) {
	reg[i] = com_mem->context.reg[i];
    }

    reg[REGIDX_EFLAGS] = com_mem->context.flag;
    reg[REGIDX_DFLAG] = (com_mem->context.flag >> 10) & 0x1;

    reg[REGIDX_CF] = (com_mem->context.flag >> 0) & 0x1;
    reg[REGIDX_PF] = (com_mem->context.flag >> 2) & 0x1;
    reg[REGIDX_AF] = (com_mem->context.flag >> 4) & 0x1;
    reg[REGIDX_ZF] = (com_mem->context.flag >> 6) & 0x1;
    reg[REGIDX_SF] = (com_mem->context.flag >> 7) & 0x1;
    reg[REGIDX_OF] = (com_mem->context.flag >> 11) & 0x1;

    return ref<Snapshot>(this,reg);
}
int VirtualMachine::mem_read(uint8_t *buf,uint64_t pos,size_t len) {
    assert(len < sizeof(com_mem->membuf.buf));

    com_mem->membuf.pos = (uint32_t)pos;
    com_mem->membuf.len = len;
    event_send(VMCOM_EVT_READMEM);
    event_wait();

    memcpy(buf,com_mem->membuf.buf,len);
    return 0;
}

Snapshot::Snapshot(VirtualMachine *_vm,const uint64_t *_reg)
    : symx::Snapshot(CS_ARCH_X86,CS_MODE_32),vm(_vm)
{
    int i;
    for(i = 0;i < REGIDX_END;i++) {
	reg.push_back(symx::BytVec::create_imm(REGSIZE[i],_reg[i]));
    }
}
int Snapshot::mem_read(uint8_t *buf,uint64_t pos,size_t len) const {
    return vm->mem_read(buf,pos,len);
}

#define MAX_ARG_STR 50
const char *inst_op[] = {
    "NONE", "UNK", "JCC",
    "STR", "STM", "LDM",
    "ADD", "SUB", "NEG", "MUL", "DIV", "MOD", "SMUL", "SDIV", "SMOD",
    "SHL", "SHR", "AND", "OR", "XOR", "NOT",
    "EQ", "LT"
};
int arg_size[] = { 1, 8, 16, 32, 64 };
char *arg_print(reil_arg_t *arg,char *arg_str) {
    memset(arg_str, 0, MAX_ARG_STR);
    switch (arg->type) {
    case A_NONE:
        break;
    case A_REG:
    case A_TEMP:
        snprintf(
		arg_str,
		MAX_ARG_STR - 1,
		"%s:%d",
		arg->name,
		arg_size[arg->size]);
        break;
    case A_CONST:
        snprintf(
		arg_str,
		MAX_ARG_STR - 1,
		"%llx:%d",
		arg->val,
		arg_size[arg->size]);
        break;
    }
    return arg_str;
}
void inst_print(const std::vector<reil_inst_t>::iterator &inst) {
    char arg_str[3][MAX_ARG_STR];
    arg_print(&inst->a, arg_str[0]);
    arg_print(&inst->b, arg_str[1]);
    arg_print(&inst->c, arg_str[2]);
    dbg("%.8llx.%.2x %7s %16s, %16s, %16s\n",
	    inst->raw_info.addr,
	    inst->inum,
	    inst_op[inst->op],
	    arg_str[0],
	    arg_str[1],
	    arg_str[2]);
}
int Snapshot::inst_handler(reil_inst_t *inst,void *ctx) {
    instlist.push_back(*inst);
    return 0;
}
symx::refBlock Snapshot::translate(
	uint8_t *code,
	const symx::ProgCtr &pc,
	size_t len
) const {
    int i;
    uint64_t rawpc = pc.rawpc;
    reil_t reil;
    int translated;

    symx::refExpr mem;
    std::vector<symx::refExpr> reglist;
    std::unordered_map<std::string,symx::refExpr> regmap;
    symx::refExpr xra,xrb,xrc;
    std::vector<symx::ProgCtr> nextpc;

    //initialize openreil, translate to reil IR
    reil = reil_init(ARCH_X86,inst_handler,(void*)&translated);
    instlist.clear();
    reil_translate(reil,rawpc,code,len);
    reil_close(reil);

    //initialize dangle memory, register
    mem = symx::BytMem::create_dangle(-1);
    for(i = 0;i < REGIDX_END;i++) {
	auto reg = symx::BytVec::create_dangle(REGSIZE[i],i);
	reglist.push_back(reg);
	regmap[REGNAME[i]] = reg;
    }

    for(auto ins = instlist.begin();ins != instlist.end();ins++) {
	inst_print(ins);
	switch(ins->op) {
	    case I_STR:
		translate_set_arg(
			&regmap,
			ins->c,
			translate_get_arg(regmap,ins->a));
		break;

	    case I_ADD:
	    case I_SUB:
	    case I_MUL:
	    case I_AND:
	    case I_OR:
	    case I_XOR:
	    case I_SHR:
	    case I_SHL:
		xra = translate_get_arg(regmap,ins->a);
		xrb = translate_get_arg(regmap,ins->b);
		switch(ins->op) {
		    case I_ADD:
			xrc = symx::expr_add(xra,xrb);
			break;
		    case I_SUB:
			xrc = symx::expr_sub(xra,xrb);
			break;
		    case I_MUL:
			xrc = symx::expr_mul(xra,xrb);
			break;
		    case I_AND:
			xrc = symx::expr_and(xra,xrb);
			break;
		    case I_OR:
			xrc = symx::expr_or(xra,xrb);
			break;
		    case I_XOR:
			xrc = symx::expr_xor(xra,xrb);
			break;
		    case I_SHR:
			xrc = symx::expr_lshr(xra,xrb);
			break;
		    case I_SHL:
			xrc = symx::expr_shl(xra,xrb);
			break;
		    default:
			err("unexpected\n");
			break;
		}
		translate_set_arg(&regmap,ins->c,xrc);
		break;

	    case I_NEG:
	    case I_NOT:
		xra = translate_get_arg(regmap,ins->a);
		switch(ins->op) {
		    case I_NEG:
			xrc = symx::expr_neg(xra);
			break;
		    case I_NOT:
			xrc = symx::expr_not(xra);
			break;
		    default:
			err("unexpected\n");
			break;
		}
		translate_set_arg(&regmap,ins->c,xrc);
		break;

	    case I_EQ:
	    case I_LT:
		xra = translate_get_arg(regmap,ins->a);
		xrb = translate_get_arg(regmap,ins->b);
		switch(ins->op) {
		    case I_EQ:
			xrc = symx::expr_ite(
				symx::cond_eq(xra,xrb),
				IMMTRUE,
				IMMFALSE);
			break;
		    case I_LT:
			xrc = symx::expr_ite(
				symx::cond_ul(xra,xrb),
				IMMTRUE,
				IMMFALSE);
			break;
		    default:
			err("unexpected\n");
			break;
		}
		translate_set_arg(&regmap,ins->c,xrc);
		break;

	    default:
		err("unsupport unstruction %d\n",ins->op);
		break;
	}
    }

    instlist.clear();

    return ref<symx::Block>(nextpc);
}
symx::refExpr Snapshot::translate_get_arg(
	const std::unordered_map<std::string,symx::refExpr> &regmap,
	const reil_arg_t &arg) const {

    assert(arg.type == A_REG || arg.type == A_TEMP || arg.type == A_CONST);

    if(arg.type == A_REG || arg.type == A_TEMP) {
	auto it = regmap.find(arg.name);
	assert(it != regmap.end());
	assert(it->second->size == REILSIZE[arg.size]);
	return it->second;
    } else {
	return symx::BytVec::create_imm(REILSIZE[arg.size],arg.val);
    }
}
int Snapshot::translate_set_arg(
	std::unordered_map<std::string,symx::refExpr> *regmap,
	const reil_arg_t &arg,
	const symx::refExpr &value) const {
    symx::refExpr xrt;
    unsigned int size = REILSIZE[arg.size];

    assert(arg.type == A_REG || arg.type == A_TEMP);

    if(size == value->size) {
	xrt = value;
    } else if(size < value->size) {
	xrt = symx::expr_extract(value,0,size);
    } else if(size > value->size) {
	xrt = symx::expr_zext(value,size);
    }
    regmap->insert(std::make_pair(arg.name,xrt));
    return 0;
}

<|endoftext|>


<commit_before>/*
 * Copyright 2017 deepstreamHub GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <algorithm>
#include <stdexcept>
#include <iterator>

#include <buffer.hpp>
#include <event.hpp>
#include <message_builder.hpp>

#include <cassert>

namespace deepstream
{

Event::Event(const SendFn& send) :
	send_(send)
{
	assert( send_ );
}

void Event::subscribe(const Name& name, const SubscribeFnRef& p_f)
{
	assert( p_f );

	if( name.empty() )
		throw std::invalid_argument( "Empty event subscription pattern" );

	auto ret = subscriber_map_.equal_range(name);
	SubscriberMap::iterator first = ret.first;
	SubscriberMap::iterator last = ret.second;

	if( first != last )
	{
		assert( first != subscriber_map_.end() );
		assert( first->first == name );
		assert( std::distance(first, last) == 1 );

		// avoid inserting duplicates
		SubscriberList& subscribers = first->second;
		assert(  !subscribers.empty() );

		SubscriberList::const_iterator it =
			std::find( subscribers.cbegin(), subscribers.cend(), p_f );

		if( it == subscribers.cend() )
			subscribers.push_back(p_f);

		return;
	}

	subscriber_map_[name].push_back(p_f);

	MessageBuilder message(Topic::EVENT, Action::SUBSCRIBE);
	message.add_argument( name );
	send_(message);
}

void Event::unsubscribe(const Name& name)
{
	SubscriberMap::iterator it = subscriber_map_.find(name);

	if( it == subscriber_map_.end() )
		return;

	subscriber_map_.erase(it);

	MessageBuilder message(Topic::EVENT, Action::UNSUBSCRIBE);
	message.add_argument(name);
	send_(message);
}

void Event::unsubscribe(const Name& name, const SubscribeFnRef& p_f)
{
	SubscriberMap::iterator it = subscriber_map_.find(name);

	if( it == subscriber_map_.end() )
		return;

	SubscriberList subscribers = it->second;
	SubscriberList::iterator ju =
		std::find( subscribers.begin(), subscribers.end(), p_f );

	if( ju == subscribers.end() )
		return;

	subscribers.erase(ju);

	if( subscribers.empty() )
		unsubscribe(name);
}

void Event::listen(const std::string& pattern, const ListenFnRef& p_f)
{
	ListenerMap::iterator it = listener_map_.find(pattern);

	if( it != listener_map_.end() )
		return;

	listener_map_[pattern] = p_f;

	MessageBuilder message(Topic::EVENT, Action::LISTEN);
	message.add_argument(pattern);
	send_(message);
}

void Event::unlisten(const std::string& pattern)
{
	ListenerMap::iterator it = listener_map_.find(pattern);

	if( it == listener_map_.end() )
		return;

	listener_map_.erase(it);

	MessageBuilder message(Topic::EVENT, Action::UNLISTEN);
	message.add_argument(pattern);
	send_(message);
}

}
<commit_msg>Event: fix a bug<commit_after>/*
 * Copyright 2017 deepstreamHub GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <algorithm>
#include <stdexcept>
#include <iterator>

#include <buffer.hpp>
#include <event.hpp>
#include <message_builder.hpp>

#include <cassert>

namespace deepstream
{

Event::Event(const SendFn& send) :
	send_(send)
{
	assert( send_ );
}

void Event::subscribe(const Name& name, const SubscribeFnRef& p_f)
{
	assert( p_f );

	if( name.empty() )
		throw std::invalid_argument( "Empty event subscription pattern" );

	auto ret = subscriber_map_.equal_range(name);
	SubscriberMap::iterator first = ret.first;
	SubscriberMap::iterator last = ret.second;

	if( first != last )
	{
		assert( first != subscriber_map_.end() );
		assert( first->first == name );
		assert( std::distance(first, last) == 1 );

		// avoid inserting duplicates
		SubscriberList& subscribers = first->second;
		assert( !subscribers.empty() );

		SubscriberList::const_iterator it =
			std::find( subscribers.cbegin(), subscribers.cend(), p_f );

		if( it == subscribers.cend() )
			subscribers.push_back(p_f);

		return;
	}

	subscriber_map_[name].push_back(p_f);

	MessageBuilder message(Topic::EVENT, Action::SUBSCRIBE);
	message.add_argument( name );
	send_(message);
}

void Event::unsubscribe(const Name& name)
{
	SubscriberMap::iterator it = subscriber_map_.find(name);

	if( it == subscriber_map_.end() )
		return;

	subscriber_map_.erase(it);

	MessageBuilder message(Topic::EVENT, Action::UNSUBSCRIBE);
	message.add_argument(name);
	send_(message);
}

void Event::unsubscribe(const Name& name, const SubscribeFnRef& p_f)
{
	SubscriberMap::iterator it = subscriber_map_.find(name);

	if( it == subscriber_map_.end() )
		return;

	SubscriberList& subscribers = it->second;
	SubscriberList::iterator ju =
		std::find( subscribers.begin(), subscribers.end(), p_f );

	if( ju == subscribers.end() )
		return;

	subscribers.erase(ju);

	if( subscribers.empty() )
		unsubscribe(name);
}

void Event::listen(const std::string& pattern, const ListenFnRef& p_f)
{
	ListenerMap::iterator it = listener_map_.find(pattern);

	if( it != listener_map_.end() )
		return;

	listener_map_[pattern] = p_f;

	MessageBuilder message(Topic::EVENT, Action::LISTEN);
	message.add_argument(pattern);
	send_(message);
}

void Event::unlisten(const std::string& pattern)
{
	ListenerMap::iterator it = listener_map_.find(pattern);

	if( it == listener_map_.end() )
		return;

	listener_map_.erase(it);

	MessageBuilder message(Topic::EVENT, Action::UNLISTEN);
	message.add_argument(pattern);
	send_(message);
}

}
<|endoftext|>