<commit_before>/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// Defines the XlaCompileOnDemandOp.

#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
#include "tensorflow/compiler/jit/xla_device.h"
#include "tensorflow/compiler/jit/xla_launch_util.h"
#include "tensorflow/compiler/tf2xla/xla_compiler.h"
#include "tensorflow/compiler/tf2xla/xla_op_registry.h"

namespace tensorflow {

namespace {
std::map<int, OptionalTensor> GetVariables(OpKernelContext* ctx) {
  std::map<int, OptionalTensor> variables;
  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
    if (ctx->input(i).dtype() == DT_RESOURCE) {
      Var* variable = nullptr;
      ResourceHandle handle = HandleFromInput(ctx, i);
      OptionalTensor& optional = variables[i];
      optional.name = handle.name();
      if (LookupResource(ctx, handle, &variable).ok()) {
        tf_shared_lock lock(*variable->mu());
        optional.present = true;
        optional.value = *variable->tensor();
      }
    }
  }
  return variables;
}
}  // namespace

Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx,
                                 const XlaDevice::Metadata& metadata,
                                 const XlaCompiler::CompilationResult* result,
                                 xla::LocalExecutable* executable) {
  std::map<int, OptionalTensor> variables = GetVariables(ctx);

  xla::LocalClient* client = metadata.client();

  // Builds an XLA allocator for the device.
  XlaComputationLaunchContext launch_context(
      client, client->backend().memory_allocator(), true);

  launch_context.PopulateInputs(ctx, result, variables);

  se::Stream* stream =
      ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr;
  TF_RET_CHECK(stream);

  VLOG(2) << "Executing computation: " << name();
  for (const xla::ShapedBuffer* arg : launch_context.arguments()) {
    VLOG(2) << name() << ": " << *arg;
  }
  xla::ExecutableRunOptions run_options;
  run_options.set_stream(stream);
  run_options.set_allocator(client->backend().memory_allocator());
  run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device());
  run_options.set_rng_seed(ctx->step_id());

  xla::StatusOr<xla::ScopedShapedBuffer> run_result;
  {
    // TODO(b/110383871): fix concurrency problems and remove this mutex.
    static mutex* mu = new mutex;
    mutex_lock lock(*mu);

    run_result = executable->Run(launch_context.arguments(), run_options);
  }
  TF_RETURN_IF_ERROR(run_result.status());

  launch_context.PopulateOutputs(ctx, result, run_result.ConsumeValueOrDie());
  return Status::OK();
}

bool XlaCompileOnDemandOp::MustArgumentBeConstant(const OpKernel* op_kernel,
                                                  int64 argument_idx) {
  // TODO(jmolloy): This could be expensive, so memoize.
  auto* constant_inputs = tensorflow::XlaOpRegistry::CompileTimeConstantInputs(
      op_kernel->def().op());
  CHECK(constant_inputs);
  std::set<int64> constant_input_indices;
  for (const auto& name : *constant_inputs) {
    int start, stop;
    TF_CHECK_OK(op_kernel->InputRange(name, &start, &stop));
    for (int i = start; i < stop; ++i) {
      constant_input_indices.insert(i);
    }
  }
  return constant_input_indices.count(argument_idx) > 0;
}

bool XlaCompileOnDemandOp::ShouldArgumentBeConstant(const OpKernel* op_kernel,
                                                    int64 argument_idx) {
  // Right now we only create kConstant arguments when absolutely required, but
  // there may be benefit in eagerly constant-folding a larger subset of
  // arguments in the future.
  return MustArgumentBeConstant(op_kernel, argument_idx);
}

Status XlaCompileOnDemandOp::Compile(
    OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
    const XlaCompiler::CompilationResult** result,
    xla::LocalExecutable** executable) {
  std::map<int, Tensor> constant_arguments;
  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
    const Tensor& device_tensor = ctx->input(i);
    if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) {
      if (xla_tensor->has_host_tensor() &&
          ShouldArgumentBeConstant(&ctx->op_kernel(), i)) {
        constant_arguments[i] = xla_tensor->host_tensor();
      }
    }
    if (constant_arguments.count(i) == 0 &&
        MustArgumentBeConstant(&ctx->op_kernel(), i)) {
      // Slow path; the argument is not available as a host constant so we must
      // fetch it synchronously.
      Tensor host_tensor;
      AllocatorAttributes attrs;
      attrs.set_on_host(true);
      TF_RETURN_IF_ERROR(ctx->allocate_temp(
          device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs));
      Notification n;
      ctx->op_device_context()->CopyDeviceTensorToCPU(
          &device_tensor, "ConstantArgument",
          reinterpret_cast<Device*>(ctx->device()), &host_tensor,
          [&](Status status) { n.Notify(); });
      n.WaitForNotification();
      constant_arguments[i] = host_tensor;
    }
  }

  // We store information about the JIT-compiled XLA computation
  // in the ResourceMgr.
  ResourceMgr* rm = ctx->resource_manager();
  CHECK(rm);

  XlaCompilationCache* cache;
  TF_RETURN_IF_ERROR(rm->LookupOrCreate<XlaCompilationCache>(
      rm->default_container(), "xla_cache", &cache,
      [&](XlaCompilationCache** cache) {
        *cache = new XlaCompilationCache(metadata.client(),
                                         metadata.jit_device_type());
        return Status::OK();
      }));
  // Hold the reference to the JIT during evaluation. (We could probably
  // free it sooner because the ResourceMgr will retain a reference, but
  // this is more obviously correct.)
  core::ScopedUnref cache_ref(cache);

  XlaCompiler::Options options;
  options.device_type = metadata.jit_device_type();
  options.client = metadata.client();
  options.flib_def =
      new FunctionLibraryDefinition(OpRegistry::Global(), FunctionDefLibrary{});
  options.shape_representation_fn = metadata.shape_representation_fn();

  XlaCompiler::CompileOptions compile_options;
  compile_options.is_entry_computation = true;

  std::map<int, OptionalTensor> variable_args = GetVariables(ctx);
  return cache->CompileSingleOp(options, constant_arguments, variable_args, ctx,
                                result, executable, &compile_options);
}

void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) {
  const XlaCompiler::CompilationResult* result;
  xla::LocalExecutable* executable;
  const XlaDevice::Metadata* metadata;
  OP_REQUIRES_OK(ctx, XlaDevice::GetMetadata(ctx, &metadata));
  OP_REQUIRES_OK(ctx, Compile(ctx, *metadata, &result, &executable));
  OP_REQUIRES_OK(ctx, Run(ctx, *metadata, result, executable));
}

}  // namespace tensorflow
<commit_msg>Internal change<commit_after>/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// Defines the XlaCompileOnDemandOp.

#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
#include "tensorflow/compiler/jit/xla_device.h"
#include "tensorflow/compiler/jit/xla_launch_util.h"
#include "tensorflow/compiler/tf2xla/xla_compiler.h"
#include "tensorflow/compiler/tf2xla/xla_op_registry.h"

namespace tensorflow {

namespace {
std::map<int, OptionalTensor> GetVariables(OpKernelContext* ctx) {
  std::map<int, OptionalTensor> variables;
  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
    if (ctx->input(i).dtype() == DT_RESOURCE) {
      Var* variable = nullptr;
      ResourceHandle handle = HandleFromInput(ctx, i);
      OptionalTensor& optional = variables[i];
      optional.name = handle.name();
      if (LookupResource(ctx, handle, &variable).ok()) {
        tf_shared_lock lock(*variable->mu());
        optional.present = true;
        optional.value = *variable->tensor();
      }
    }
  }
  return variables;
}
}  // namespace

Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx,
                                 const XlaDevice::Metadata& metadata,
                                 const XlaCompiler::CompilationResult* result,
                                 xla::LocalExecutable* executable) {
  std::map<int, OptionalTensor> variables = GetVariables(ctx);

  xla::LocalClient* client = metadata.client();

  // Builds an XLA allocator for the device.
  XlaComputationLaunchContext launch_context(
      client, client->backend().memory_allocator(), true);

  launch_context.PopulateInputs(ctx, result, variables);

  se::Stream* stream =
      ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr;
  TF_RET_CHECK(stream);

  VLOG(2) << "Executing computation: " << name();
  for (const xla::ShapedBuffer* arg : launch_context.arguments()) {
    VLOG(2) << name() << ": " << *arg;
  }
  xla::ExecutableRunOptions run_options;
  run_options.set_stream(stream);
  run_options.set_allocator(client->backend().memory_allocator());
  run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device());
  run_options.set_rng_seed(ctx->step_id());

  xla::StatusOr<xla::ScopedShapedBuffer> run_result =
      executable->Run(launch_context.arguments(), run_options);
  TF_RETURN_IF_ERROR(run_result.status());

  launch_context.PopulateOutputs(ctx, result, run_result.ConsumeValueOrDie());
  return Status::OK();
}

bool XlaCompileOnDemandOp::MustArgumentBeConstant(const OpKernel* op_kernel,
                                                  int64 argument_idx) {
  // TODO(jmolloy): This could be expensive, so memoize.
  auto* constant_inputs = tensorflow::XlaOpRegistry::CompileTimeConstantInputs(
      op_kernel->def().op());
  CHECK(constant_inputs);
  std::set<int64> constant_input_indices;
  for (const auto& name : *constant_inputs) {
    int start, stop;
    TF_CHECK_OK(op_kernel->InputRange(name, &start, &stop));
    for (int i = start; i < stop; ++i) {
      constant_input_indices.insert(i);
    }
  }
  return constant_input_indices.count(argument_idx) > 0;
}

bool XlaCompileOnDemandOp::ShouldArgumentBeConstant(const OpKernel* op_kernel,
                                                    int64 argument_idx) {
  // Right now we only create kConstant arguments when absolutely required, but
  // there may be benefit in eagerly constant-folding a larger subset of
  // arguments in the future.
  return MustArgumentBeConstant(op_kernel, argument_idx);
}

Status XlaCompileOnDemandOp::Compile(
    OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
    const XlaCompiler::CompilationResult** result,
    xla::LocalExecutable** executable) {
  std::map<int, Tensor> constant_arguments;
  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
    const Tensor& device_tensor = ctx->input(i);
    if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) {
      if (xla_tensor->has_host_tensor() &&
          ShouldArgumentBeConstant(&ctx->op_kernel(), i)) {
        constant_arguments[i] = xla_tensor->host_tensor();
      }
    }
    if (constant_arguments.count(i) == 0 &&
        MustArgumentBeConstant(&ctx->op_kernel(), i)) {
      // Slow path; the argument is not available as a host constant so we must
      // fetch it synchronously.
      Tensor host_tensor;
      AllocatorAttributes attrs;
      attrs.set_on_host(true);
      TF_RETURN_IF_ERROR(ctx->allocate_temp(
          device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs));
      Notification n;
      ctx->op_device_context()->CopyDeviceTensorToCPU(
          &device_tensor, "ConstantArgument",
          reinterpret_cast<Device*>(ctx->device()), &host_tensor,
          [&](Status status) { n.Notify(); });
      n.WaitForNotification();
      constant_arguments[i] = host_tensor;
    }
  }

  // We store information about the JIT-compiled XLA computation
  // in the ResourceMgr.
  ResourceMgr* rm = ctx->resource_manager();
  CHECK(rm);

  XlaCompilationCache* cache;
  TF_RETURN_IF_ERROR(rm->LookupOrCreate<XlaCompilationCache>(
      rm->default_container(), "xla_cache", &cache,
      [&](XlaCompilationCache** cache) {
        *cache = new XlaCompilationCache(metadata.client(),
                                         metadata.jit_device_type());
        return Status::OK();
      }));
  // Hold the reference to the JIT during evaluation. (We could probably
  // free it sooner because the ResourceMgr will retain a reference, but
  // this is more obviously correct.)
  core::ScopedUnref cache_ref(cache);

  XlaCompiler::Options options;
  options.device_type = metadata.jit_device_type();
  options.client = metadata.client();
  options.flib_def =
      new FunctionLibraryDefinition(OpRegistry::Global(), FunctionDefLibrary{});
  options.shape_representation_fn = metadata.shape_representation_fn();

  XlaCompiler::CompileOptions compile_options;
  compile_options.is_entry_computation = true;

  std::map<int, OptionalTensor> variable_args = GetVariables(ctx);
  return cache->CompileSingleOp(options, constant_arguments, variable_args, ctx,
                                result, executable, &compile_options);
}

void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) {
  const XlaCompiler::CompilationResult* result;
  xla::LocalExecutable* executable;
  const XlaDevice::Metadata* metadata;
  OP_REQUIRES_OK(ctx, XlaDevice::GetMetadata(ctx, &metadata));
  OP_REQUIRES_OK(ctx, Compile(ctx, *metadata, &result, &executable));
  OP_REQUIRES_OK(ctx, Run(ctx, *metadata, result, executable));
}

}  // namespace tensorflow
<|endoftext|>


<commit_before>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <sstream>

#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Signals.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Main.h"
#include "llvm/TableGen/Record.h"
#include "llvm/TableGen/TableGenBackend.h"
#include "mlir/TableGen/Operator.h"  // TF:local_config_mlir

using llvm::dyn_cast;
using llvm::LessRecord;
using llvm::raw_ostream;
using llvm::Record;
using llvm::RecordKeeper;
using llvm::StringRef;
using mlir::tblgen::Operator;

// Returns the builder function name for the given op definition.
// E.g., AddOp -> CreateAddOp
static inline std::string GetOperatorBuilderName(StringRef op_name) {
  return "Create" + op_name.str();
}

static std::string GetConversionFunction(
    mlir::tblgen::NamedAttribute named_attr) {
  auto storage_type = named_attr.attr.getStorageType();
  // For some attribute types we have a general conversion, so use that.
  if (storage_type.endswith("IntegerAttr") ||
      storage_type.endswith("FloatAttr")) {
    return "Convert" + named_attr.attr.getReturnType().str();
  }
  return "Convert_" + named_attr.name.str();
}

using ArgumentName = string;
using ArgumentDeclaration = string;
using Argument = std::pair<ArgumentName, ArgumentDeclaration>;
using ArgumentList = std::vector<Argument>;

static std::string BuildOperator(const Operator& op) {
  std::stringstream os;
  StringRef op_name = op.getCppClassName();
  std::string xla_op_name = op_name.drop_back(2).str();

  // Signature.
  os << "static xla::XlaOp " << GetOperatorBuilderName(op_name)
     << "(mlir::XLA::" << op_name.str() << " xla_op, "
     << "llvm::DenseMap<mlir::Value*, xla::XlaOp>* "
        "value_lowering) {\n";

  os << "  auto& value_map = *value_lowering;\n"
     << "  auto result = xla_op.getResult();\n";

  // Invoke the conversion function for each attribute.
  for (const auto& named_attr : op.getAttributes()) {
    os << "  auto " << named_attr.name.str() << " = "
       << GetConversionFunction(named_attr) << "("
       << "xla_op." << named_attr.name.str() << "());\n";
  }

  // Assumes that the client builder method names closely follow the op names
  // in the dialect. For e.g., AddOp -> xla::Add method.
  os << "  auto xla_result = xla::" << xla_op_name << "(";

  int num_operands = op.getNumOperands();
  if (num_operands == 1) {
    os << "value_map[xla_op.getOperand()]";
  } else {
    for (auto i = 0; i < num_operands; i++) {
      os << "value_map[xla_op.getOperand(" << i << ")]";
      if (i != num_operands - 1) {
        os << ", ";
      }
    }
  }

  for (const auto& named_attr : op.getAttributes()) {
    os << ", Unwrap(" << named_attr.name.str() << ")";
  }

  os << ");\n";

  os << "  value_map[result] = xla_result;\n";
  os << "  return xla_result;\n";
  os << "}\n\n";
  return os.str();
}

// For each XLA op, emits a builder function that constructs the XLA op using
// the HLO client builder.
static void EmitOperatorBuilders(const RecordKeeper& record_keeper,
                                 const std::vector<Record*>& defs,
                                 raw_ostream* ostream) {
  raw_ostream& os = *ostream;

  for (const auto* def : defs) {
    // Skip operations that have a custom converter.
    if (def->getValueAsBit("hasCustomHLOConverter")) continue;

    Operator op(def);
    os << BuildOperator(op);
  }
}

// Emits a builder function that returns the XlaOp object given a
// mlir::Operation.
//
// The signature of the function is:
//
//   llvm::Optional<xla::XlaOp>
//   mlir::CreateXlaOperator(
//       mlir::Operation* op,
//       llvm::DenseMap<mlir::Value*, xla::XlaOp>
//       *value_lowering);
static void EmitBuilder(const std::vector<Record*>& defs,
                        raw_ostream* ostream) {
  raw_ostream& os = *ostream;

  // Signature
  os << "llvm::Optional<xla::XlaOp>\n"
        "mlir::CreateXlaOperator(mlir::Operation* op, "
        "llvm::DenseMap<mlir::Value*, xla::XlaOp> "
        "*value_lowering) {\n";

  for (const auto* def : defs) {
    // Skip operations that have a custom converter.
    if (def->getValueAsBit("hasCustomHLOConverter")) continue;

    StringRef op_name = def->getName().drop_front(4);

    // Try to cast to each op and call the corresponding op builder.
    os << "  if (auto xla_op = llvm::dyn_cast<mlir::XLA::" << op_name
       << ">(op))\n     return " << GetOperatorBuilderName(op_name)
       << "(xla_op, value_lowering);\n";
  }

  os << "  return llvm::None;\n"
        "}\n";
}

// The function below has a non-constant reference as that is required by LLVM's
// TableGenMain.
// NOLINTNEXTLINE
static bool OperatorWritersMain(raw_ostream& os, RecordKeeper& records) {
  emitSourceFileHeader("MLIR XLA Builders", os);

  // Retrieve all the definitions derived from XLA_Op and sort by record name.
  std::vector<Record*> defs = records.getAllDerivedDefinitions("XLA_Op");
  llvm::sort(defs, LessRecord());

  for (const auto* def : defs) {
    // XLA ops in the .td file are expected to follow the naming convention:
    // XLA_<OpName>Op.
    // The generated XLA op C++ class should be XLA::<OpName>Op.
    if (!def->getName().startswith("XLA_"))
      PrintFatalError(def->getLoc(),
                      "unexpected op name format: 'XLA_' prefix missing");
    if (!def->getName().endswith("Op"))
      PrintFatalError(def->getLoc(),
                      "unexpected op name format: 'Op' suffix missing");
  }

  EmitOperatorBuilders(records, defs, &os);
  os << "\n\n";
  EmitBuilder(defs, &os);

  return false;
}

int main(int argc, char** argv) {
  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
  llvm::PrettyStackTraceProgram X(argc, argv);

  llvm::llvm_shutdown_obj Y;
  llvm::cl::ParseCommandLineOptions(argc, argv);
  return TableGenMain(argv[0], &OperatorWritersMain);
}
<commit_msg>Add missing namespace specification for string.<commit_after>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <sstream>

#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Signals.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Main.h"
#include "llvm/TableGen/Record.h"
#include "llvm/TableGen/TableGenBackend.h"
#include "mlir/TableGen/Operator.h"  // TF:local_config_mlir

using llvm::dyn_cast;
using llvm::LessRecord;
using llvm::raw_ostream;
using llvm::Record;
using llvm::RecordKeeper;
using llvm::StringRef;
using mlir::tblgen::Operator;

// Returns the builder function name for the given op definition.
// E.g., AddOp -> CreateAddOp
static inline std::string GetOperatorBuilderName(StringRef op_name) {
  return "Create" + op_name.str();
}

static std::string GetConversionFunction(
    mlir::tblgen::NamedAttribute named_attr) {
  auto storage_type = named_attr.attr.getStorageType();
  // For some attribute types we have a general conversion, so use that.
  if (storage_type.endswith("IntegerAttr") ||
      storage_type.endswith("FloatAttr")) {
    return "Convert" + named_attr.attr.getReturnType().str();
  }
  return "Convert_" + named_attr.name.str();
}

using ArgumentName = std::string;
using ArgumentDeclaration = std::string;
using Argument = std::pair<ArgumentName, ArgumentDeclaration>;
using ArgumentList = std::vector<Argument>;

static std::string BuildOperator(const Operator& op) {
  std::stringstream os;
  StringRef op_name = op.getCppClassName();
  std::string xla_op_name = op_name.drop_back(2).str();

  // Signature.
  os << "static xla::XlaOp " << GetOperatorBuilderName(op_name)
     << "(mlir::XLA::" << op_name.str() << " xla_op, "
     << "llvm::DenseMap<mlir::Value*, xla::XlaOp>* "
        "value_lowering) {\n";

  os << "  auto& value_map = *value_lowering;\n"
     << "  auto result = xla_op.getResult();\n";

  // Invoke the conversion function for each attribute.
  for (const auto& named_attr : op.getAttributes()) {
    os << "  auto " << named_attr.name.str() << " = "
       << GetConversionFunction(named_attr) << "("
       << "xla_op." << named_attr.name.str() << "());\n";
  }

  // Assumes that the client builder method names closely follow the op names
  // in the dialect. For e.g., AddOp -> xla::Add method.
  os << "  auto xla_result = xla::" << xla_op_name << "(";

  int num_operands = op.getNumOperands();
  if (num_operands == 1) {
    os << "value_map[xla_op.getOperand()]";
  } else {
    for (auto i = 0; i < num_operands; i++) {
      os << "value_map[xla_op.getOperand(" << i << ")]";
      if (i != num_operands - 1) {
        os << ", ";
      }
    }
  }

  for (const auto& named_attr : op.getAttributes()) {
    os << ", Unwrap(" << named_attr.name.str() << ")";
  }

  os << ");\n";

  os << "  value_map[result] = xla_result;\n";
  os << "  return xla_result;\n";
  os << "}\n\n";
  return os.str();
}

// For each XLA op, emits a builder function that constructs the XLA op using
// the HLO client builder.
static void EmitOperatorBuilders(const RecordKeeper& record_keeper,
                                 const std::vector<Record*>& defs,
                                 raw_ostream* ostream) {
  raw_ostream& os = *ostream;

  for (const auto* def : defs) {
    // Skip operations that have a custom converter.
    if (def->getValueAsBit("hasCustomHLOConverter")) continue;

    Operator op(def);
    os << BuildOperator(op);
  }
}

// Emits a builder function that returns the XlaOp object given a
// mlir::Operation.
//
// The signature of the function is:
//
//   llvm::Optional<xla::XlaOp>
//   mlir::CreateXlaOperator(
//       mlir::Operation* op,
//       llvm::DenseMap<mlir::Value*, xla::XlaOp>
//       *value_lowering);
static void EmitBuilder(const std::vector<Record*>& defs,
                        raw_ostream* ostream) {
  raw_ostream& os = *ostream;

  // Signature
  os << "llvm::Optional<xla::XlaOp>\n"
        "mlir::CreateXlaOperator(mlir::Operation* op, "
        "llvm::DenseMap<mlir::Value*, xla::XlaOp> "
        "*value_lowering) {\n";

  for (const auto* def : defs) {
    // Skip operations that have a custom converter.
    if (def->getValueAsBit("hasCustomHLOConverter")) continue;

    StringRef op_name = def->getName().drop_front(4);

    // Try to cast to each op and call the corresponding op builder.
    os << "  if (auto xla_op = llvm::dyn_cast<mlir::XLA::" << op_name
       << ">(op))\n     return " << GetOperatorBuilderName(op_name)
       << "(xla_op, value_lowering);\n";
  }

  os << "  return llvm::None;\n"
        "}\n";
}

// The function below has a non-constant reference as that is required by LLVM's
// TableGenMain.
// NOLINTNEXTLINE
static bool OperatorWritersMain(raw_ostream& os, RecordKeeper& records) {
  emitSourceFileHeader("MLIR XLA Builders", os);

  // Retrieve all the definitions derived from XLA_Op and sort by record name.
  std::vector<Record*> defs = records.getAllDerivedDefinitions("XLA_Op");
  llvm::sort(defs, LessRecord());

  for (const auto* def : defs) {
    // XLA ops in the .td file are expected to follow the naming convention:
    // XLA_<OpName>Op.
    // The generated XLA op C++ class should be XLA::<OpName>Op.
    if (!def->getName().startswith("XLA_"))
      PrintFatalError(def->getLoc(),
                      "unexpected op name format: 'XLA_' prefix missing");
    if (!def->getName().endswith("Op"))
      PrintFatalError(def->getLoc(),
                      "unexpected op name format: 'Op' suffix missing");
  }

  EmitOperatorBuilders(records, defs, &os);
  os << "\n\n";
  EmitBuilder(defs, &os);

  return false;
}

int main(int argc, char** argv) {
  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
  llvm::PrettyStackTraceProgram X(argc, argv);

  llvm::llvm_shutdown_obj Y;
  llvm::cl::ParseCommandLineOptions(argc, argv);
  return TableGenMain(argv[0], &OperatorWritersMain);
}
<|endoftext|>


<commit_before>/*##############################################################################

    Copyright (C) 2011 HPCC Systems.

    All rights reserved. This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
############################################################################## */

#ifndef __JTHREAD__
#define __JTHREAD__

#include "jexpdef.hpp"
#include "jiface.hpp"
#include "jmutex.hpp"
#include "jexcept.hpp"
#include "jhash.hpp"

#ifdef _WIN32
#define DEFAULT_THREAD_PRIORITY THREAD_PRIORITY_NORMAL
#else
// no thread priority handling?
#endif

interface jlib_decl IThread : public IInterface
{
    virtual void start() = 0;
    virtual int run() = 0;
};

interface jlib_decl IThreadName
{
    virtual const char *get()=0;
};

extern jlib_decl void addThreadExceptionHandler(IExceptionHandler *handler);
extern jlib_decl void removeThreadExceptionHandler(IExceptionHandler *handler);
extern jlib_decl void enableThreadSEH();
extern jlib_decl void disableThreadSEH();

extern jlib_decl unsigned threadLogID();  // for use in logging

class jlib_decl Thread : public CInterface, public IThread
{
private:
    ThreadId threadid;
    unsigned short stacksize; // in 4K blocks
    char prioritydelta;
    char nicelevel;

    bool alive;
    unsigned tidlog;
#ifdef _WIN32
    HANDLE hThread;
    static unsigned WINAPI _threadmain(LPVOID v);
#else
    static void *_threadmain(void *v);
#endif
    virtual int begin();
    void init(const char *name);
    void handleException(IException *e);
    void adjustNiceLevel();

protected:
    struct cThreadName: implements IThreadName
    {
        char *threadname;
        const char *get() { return threadname; }
    } cthreadname;
    IThreadName *ithreadname;
public:
#ifndef _WIN32
    Semaphore suspend;
    Semaphore starting;
#endif
    Semaphore stopped;

    IMPLEMENT_IINTERFACE;

    Thread(const char *_name) { init(_name); }
    Thread() { init(NULL); }
    ~Thread();

    void adjustPriority(char delta);
    void setNice(char nicelevel);
    void setStackSize(size32_t size);               // required stack size in bytes - called before start() (obviously)
    const char *getName() { const char *ret = ithreadname?ithreadname->get():NULL; return ret?ret:"unknown"; }
    bool isAlive() { return alive; }
    bool join(unsigned timeout=INFINITE);

    virtual void start();
    virtual void startRelease();

    StringBuffer &getInfo(StringBuffer &str) { str.appendf("%8"I64F"X %6"I64F"d %u: %s",(__int64)threadid,(__int64)threadid,tidlog,getName()); return str; }
    const char *getLogInfo(int &thandle,unsigned &tid) {
#ifdef _WIN32
        thandle = (int)(memsize_t)hThread;
#elif defined __FreeBSD__ || defined __APPLE__
        thandle = (int)(memsize_t)threadid;
#else
        thandle = threadid;
#endif
        tid = tidlog;
        return getName();
    }

    // run method not implemented - concrete derived classes must do so
    static  void setDefaultStackSize(size32_t size);    // NB under windows requires linker setting (/stack:)

    IThreadName *queryThreadName() { return ithreadname; }
    void setThreadName(IThreadName *name) { ithreadname = name; }

};

interface IThreaded
{
    virtual void main() = 0;
};

// utility class, useful for containing a thread
class CThreaded : public Thread
{
    IThreaded *owner;
public:
    inline CThreaded(const char *name) : Thread(name) { owner = NULL; }
    inline void init(IThreaded *_owner) { owner = _owner; start(); }
    virtual int run() { owner->main(); return 1; }
};

// Asynchronous 'for' utility class
// see HRPCUTIL.CPP for example of usage

class jlib_decl CAsyncFor
{
public:
    void For(unsigned num,unsigned maxatonce,bool abortFollowingException=false,bool shuffled=false);
    virtual void Do(unsigned idx=0)=0;
};

// Thread local storage - use MAKETHREADLOCALIINTERFACE macro to get a thread local type

template <class CLASS, class CLASSINIT = CLASS, class MAP = MapBetween<ThreadId, ThreadId, CLASS, CLASSINIT> >
class ThreadLocalOf : public MAP
{
public:
    CLASS * query()
    {
        CLASS * find = threadMap.getValue(GetCurrentThreadId());
        if(find) return find;
        threadMap.setValue(GetCurrentThreadId(), CLASSINIT());
        return threadMap.getValue(GetCurrentThreadId());
    }
    operator CLASS & ()
    {
        return *query();
    }
private:
    MAP threadMap;
};

#define MAKETHREADLOCALIINTERFACE(C, CI, NAME)                                                      \
typedef ThreadLocalOf<C, CI> NAME

// ---------------------------------------------------------------------------
// Thread Pools
// ---------------------------------------------------------------------------

interface IPooledThread: extends IInterface                 // base class for deriving pooled thread (alternative to Thread)
{
public:
       virtual void init(void *param)=0;                    // called before main started (from within start)
       virtual void main()=0;                               // where threads code goes (param is passed from start)
       virtual bool stop()=0;                               // called to cause main to return, returns false if request rejected
       virtual bool canReuse()=0;                           // return true if object can be re-used (after stopped), otherwise released
};

interface IThreadFactory: extends IInterface                // factory for creating new pooled instances (called when pool empty)
{
       virtual IPooledThread *createNew()=0;
};

typedef IIteratorOf<IPooledThread> IPooledThreadIterator;
typedef unsigned PooledThreadHandle;

interface IThreadPool : extends IInterface
{
        virtual PooledThreadHandle start(void *param)=0;    // starts a new thread reuses stopped pool entries
        virtual PooledThreadHandle start(void *param,const char *name)=0;   // starts a new thread reuses stopped pool entries
        virtual PooledThreadHandle start(void *param,const char *name,unsigned timeout)=0;  // starts a new thread reuses stopped pool entries, throws exception if can't start within timeout
        virtual bool stop(PooledThreadHandle handle)=0;     // initiates stop on specified thread (may return false)
        virtual bool stopAll(bool tryall=false)=0;          // initiates stop on all threads, if tryall continues even if one or more fails
        virtual bool join(PooledThreadHandle handle,unsigned timeout=INFINITE)=0;
                                                            // waits for a single thread to terminate
        virtual bool joinAll(bool del=true,unsigned timeout=INFINITE)=0;    // waits for all threads in thread pool to terminate
                                                                            // if del true frees all pooled threads
        virtual IPooledThreadIterator *running()=0;                 // return an iterator for all currently running threads
        virtual unsigned runningCount()=0;                  // number of currently running threads
        virtual PooledThreadHandle startNoBlock(void *param)=0; // starts a new thread if it can do so without blocking, else throws exception
        virtual PooledThreadHandle startNoBlock(void *param,const char *name)=0;    // starts a new thread if it can do so without blocking, else throws exception
};

extern jlib_decl IThreadPool *createThreadPool(
                                const char *poolname,       // trace name of pool
                                IThreadFactory *factory,    // factory for creating new thread instances
                                IExceptionHandler *exceptionHandler=NULL, // optional exception handler
                                unsigned defaultmax=50,     // maximum number of threads before starts blocking
                                unsigned delay=1000,        // maximum delay on each block
                                unsigned stacksize=0,       // stack size (bytes) 0 is default
                                unsigned timeoutOnRelease=INFINITE, // maximum time waited for thread to terminate on releasing pool
                                unsigned targetpoolsize=0           // target maximum size of pool (default same as defaultmax)
                             );

extern jlib_decl StringBuffer &getThreadList(StringBuffer &str);
extern jlib_decl unsigned getThreadCount();
extern jlib_decl StringBuffer &getThreadName(int thandle,unsigned logtid,StringBuffer &name); // either thandle or tid should be 0

// Simple pipe process support
interface ISimpleReadStream;

interface IPipeProcess: extends IInterface
{
    virtual bool run(const char *title,const char *prog, const char *dir,
                       bool hasinput,bool hasoutput,bool haserror=false,
                       size32_t stderrbufsize=0) = 0;               // set to non-zero to automatically buffer stderror output
    virtual bool hasInput() = 0;                                    // i.e. can write to pipe
    virtual size32_t write(size32_t sz, const void *buffer) = 0;    // write pipe process standard output
    virtual bool hasOutput() = 0;                                   // i.e. can read from pipe
    virtual size32_t read(size32_t sz, void *buffer) = 0;           // read from pipe process standard output
    virtual ISimpleReadStream *getOutputStream() = 0;               // read from pipe process standard output
    virtual bool hasError() = 0;                                    // i.e. can read from pipe stderr
    virtual size32_t readError(size32_t sz, void *buffer) = 0;      // read from pipe process standard error
    virtual ISimpleReadStream *getErrorStream() = 0;                // read from pipe process standard error

    virtual unsigned wait() = 0;                                    // returns return code
    virtual unsigned wait(unsigned timeoutms, bool &timedout) = 0;  // sets timedout to true if times out
    virtual void closeInput() = 0;                                  // indicate finished input to pipe
    virtual void closeOutput() = 0;                                 // indicate finished reading from pipe (generally called automatically)
    virtual void closeError() = 0;                                  // indicate finished reading from pipe stderr
    virtual void abort() = 0;
    virtual void notifyTerminated(HANDLE pid,unsigned retcode) = 0; // internal
    virtual HANDLE getProcessHandle() = 0;                          // used to auto kill
};

extern jlib_decl IPipeProcess *createPipeProcess(const char *allowedprograms=NULL);

//--------------------------------------------------------

interface IWorkQueueItem: extends IInterface
{
    virtual void execute()=0;
};

interface IWorkQueueThread: extends IInterface
{
    virtual void post(IWorkQueueItem *item)=0;  // takes ownership of item
    virtual void wait()=0;
    virtual unsigned pending()=0;
};

// Simple lightweight async worker queue
// internally thread persists for specified time waiting before self destroying
extern jlib_decl IWorkQueueThread *createWorkQueueThread(unsigned persisttime=1000*60);

#endif
<commit_msg>Implementation of a persist thread<commit_after>/*##############################################################################

    Copyright (C) 2011 HPCC Systems.

    All rights reserved. This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
############################################################################## */

#ifndef __JTHREAD__
#define __JTHREAD__

#include "jexpdef.hpp"
#include "jiface.hpp"
#include "jmutex.hpp"
#include "jexcept.hpp"
#include "jhash.hpp"

#ifdef _WIN32
#define DEFAULT_THREAD_PRIORITY THREAD_PRIORITY_NORMAL
#else
// no thread priority handling?
#endif

interface jlib_decl IThread : public IInterface
{
    virtual void start() = 0;
    virtual int run() = 0;
};

interface jlib_decl IThreadName
{
    virtual const char *get()=0;
};

extern jlib_decl void addThreadExceptionHandler(IExceptionHandler *handler);
extern jlib_decl void removeThreadExceptionHandler(IExceptionHandler *handler);
extern jlib_decl void enableThreadSEH();
extern jlib_decl void disableThreadSEH();

extern jlib_decl unsigned threadLogID();  // for use in logging

class jlib_decl Thread : public CInterface, public IThread
{
private:
    ThreadId threadid;
    unsigned short stacksize; // in 4K blocks
    char prioritydelta;
    char nicelevel;

    bool alive;
    unsigned tidlog;
#ifdef _WIN32
    HANDLE hThread;
    static unsigned WINAPI _threadmain(LPVOID v);
#else
    static void *_threadmain(void *v);
#endif
    virtual int begin();
    void init(const char *name);
    void handleException(IException *e);
    void adjustNiceLevel();

protected:
    struct cThreadName: implements IThreadName
    {
        char *threadname;
        const char *get() { return threadname; }
    } cthreadname;
    IThreadName *ithreadname;
public:
#ifndef _WIN32
    Semaphore suspend;
    Semaphore starting;
#endif
    Semaphore stopped;

    IMPLEMENT_IINTERFACE;

    Thread(const char *_name) { init(_name); }
    Thread() { init(NULL); }
    ~Thread();

    void adjustPriority(char delta);
    void setNice(char nicelevel);
    void setStackSize(size32_t size);               // required stack size in bytes - called before start() (obviously)
    const char *getName() { const char *ret = ithreadname?ithreadname->get():NULL; return ret?ret:"unknown"; }
    bool isAlive() { return alive; }
    bool join(unsigned timeout=INFINITE);

    virtual void start();
    virtual void startRelease();

    StringBuffer &getInfo(StringBuffer &str) { str.appendf("%8"I64F"X %6"I64F"d %u: %s",(__int64)threadid,(__int64)threadid,tidlog,getName()); return str; }
    const char *getLogInfo(int &thandle,unsigned &tid) {
#ifdef _WIN32
        thandle = (int)(memsize_t)hThread;
#elif defined __FreeBSD__ || defined __APPLE__
        thandle = (int)(memsize_t)threadid;
#else
        thandle = threadid;
#endif
        tid = tidlog;
        return getName();
    }

    // run method not implemented - concrete derived classes must do so
    static  void setDefaultStackSize(size32_t size);    // NB under windows requires linker setting (/stack:)

    IThreadName *queryThreadName() { return ithreadname; }
    void setThreadName(IThreadName *name) { ithreadname = name; }

};

interface IThreaded
{
    virtual void main() = 0;
};

// utility class, useful for containing a thread
class CThreaded : public Thread
{
    IThreaded *owner;
public:
    inline CThreaded(const char *name, IThreaded *_owner) : Thread(name), owner(_owner) { }
    inline CThreaded(const char *name) : Thread(name) { owner = NULL; }
    inline void init(IThreaded *_owner) { owner = _owner; start(); }
    virtual int run() { owner->main(); return 1; }
};

// Similar to above, but the underlying thread always remains running. This can make repeated start + join's significantly quicker
class CThreadedPersistent : public CInterface
{
    class CAThread : public Thread
    {
        CThreadedPersistent &owner;
    public:
        CAThread(CThreadedPersistent &_owner, const char *name) : Thread(name), owner(_owner) { }
        virtual int run() { owner.main(); return 1; }
    } athread;
    IThreaded *owner;
    bool stopped, running, joinWaiting;
    Semaphore sem, joinSem;
    SpinLock spin;

    void stop()
    {
        stopped = true;
        sem.signal();
    }
    void main()
    {
        loop
        {
            sem.wait();
            if (stopped)
                break;
            owner->main();

            spin.enter();
            running = false;
            if (joinWaiting)
            {
                spin.leave();
                joinWaiting = false;
                joinSem.signal();
            }
            else
                spin.leave();
        }
    }
public:
    inline CThreadedPersistent(const char *name, IThreaded *_owner) : athread(*this, name), owner(_owner)
    {
        stopped = false;
        running = false;
        joinWaiting = false;
        athread.start();
    }
    inline ~CThreadedPersistent()
    {
        join(INFINITE);
        stop();
        athread.join();
    }
    inline void start()
    {
        assertex(!stopped);
        SpinBlock b(spin);
        if (running)
            throw MakeStringException(0, "CThreadedPersistent(%s) already running", athread.queryThreadName()->get());
        running = true;
        sem.signal();
    }
    inline bool join(unsigned timeout=INFINITE)
    {
        assertex(!stopped);
        spin.enter();
        if (running)
        {
            joinWaiting = true;
            spin.leave();
            if (!joinSem.wait(timeout))
            {
                SpinBlock b(spin);
                joinWaiting = false;
                return false;
            }
        }
        else
            spin.leave();
        return true;
    }
};

// Asynchronous 'for' utility class
// see HRPCUTIL.CPP for example of usage

class jlib_decl CAsyncFor
{
public:
    void For(unsigned num,unsigned maxatonce,bool abortFollowingException=false,bool shuffled=false);
    virtual void Do(unsigned idx=0)=0;
};

// Thread local storage - use MAKETHREADLOCALIINTERFACE macro to get a thread local type

template <class CLASS, class CLASSINIT = CLASS, class MAP = MapBetween<ThreadId, ThreadId, CLASS, CLASSINIT> >
class ThreadLocalOf : public MAP
{
public:
    CLASS * query()
    {
        CLASS * find = threadMap.getValue(GetCurrentThreadId());
        if(find) return find;
        threadMap.setValue(GetCurrentThreadId(), CLASSINIT());
        return threadMap.getValue(GetCurrentThreadId());
    }
    operator CLASS & ()
    {
        return *query();
    }
private:
    MAP threadMap;
};

#define MAKETHREADLOCALIINTERFACE(C, CI, NAME)                                                      \
typedef ThreadLocalOf<C, CI> NAME

// ---------------------------------------------------------------------------
// Thread Pools
// ---------------------------------------------------------------------------

interface IPooledThread: extends IInterface                 // base class for deriving pooled thread (alternative to Thread)
{
public:
       virtual void init(void *param)=0;                    // called before main started (from within start)
       virtual void main()=0;                               // where threads code goes (param is passed from start)
       virtual bool stop()=0;                               // called to cause main to return, returns false if request rejected
       virtual bool canReuse()=0;                           // return true if object can be re-used (after stopped), otherwise released
};

interface IThreadFactory: extends IInterface                // factory for creating new pooled instances (called when pool empty)
{
       virtual IPooledThread *createNew()=0;
};

typedef IIteratorOf<IPooledThread> IPooledThreadIterator;
typedef unsigned PooledThreadHandle;

interface IThreadPool : extends IInterface
{
        virtual PooledThreadHandle start(void *param)=0;    // starts a new thread reuses stopped pool entries
        virtual PooledThreadHandle start(void *param,const char *name)=0;   // starts a new thread reuses stopped pool entries
        virtual PooledThreadHandle start(void *param,const char *name,unsigned timeout)=0;  // starts a new thread reuses stopped pool entries, throws exception if can't start within timeout
        virtual bool stop(PooledThreadHandle handle)=0;     // initiates stop on specified thread (may return false)
        virtual bool stopAll(bool tryall=false)=0;          // initiates stop on all threads, if tryall continues even if one or more fails
        virtual bool join(PooledThreadHandle handle,unsigned timeout=INFINITE)=0;
                                                            // waits for a single thread to terminate
        virtual bool joinAll(bool del=true,unsigned timeout=INFINITE)=0;    // waits for all threads in thread pool to terminate
                                                                            // if del true frees all pooled threads
        virtual IPooledThreadIterator *running()=0;                 // return an iterator for all currently running threads
        virtual unsigned runningCount()=0;                  // number of currently running threads
        virtual PooledThreadHandle startNoBlock(void *param)=0; // starts a new thread if it can do so without blocking, else throws exception
        virtual PooledThreadHandle startNoBlock(void *param,const char *name)=0;    // starts a new thread if it can do so without blocking, else throws exception
};

extern jlib_decl IThreadPool *createThreadPool(
                                const char *poolname,       // trace name of pool
                                IThreadFactory *factory,    // factory for creating new thread instances
                                IExceptionHandler *exceptionHandler=NULL, // optional exception handler
                                unsigned defaultmax=50,     // maximum number of threads before starts blocking
                                unsigned delay=1000,        // maximum delay on each block
                                unsigned stacksize=0,       // stack size (bytes) 0 is default
                                unsigned timeoutOnRelease=INFINITE, // maximum time waited for thread to terminate on releasing pool
                                unsigned targetpoolsize=0           // target maximum size of pool (default same as defaultmax)
                             );

extern jlib_decl StringBuffer &getThreadList(StringBuffer &str);
extern jlib_decl unsigned getThreadCount();
extern jlib_decl StringBuffer &getThreadName(int thandle,unsigned logtid,StringBuffer &name); // either thandle or tid should be 0

// Simple pipe process support
interface ISimpleReadStream;

interface IPipeProcess: extends IInterface
{
    virtual bool run(const char *title,const char *prog, const char *dir,
                       bool hasinput,bool hasoutput,bool haserror=false,
                       size32_t stderrbufsize=0) = 0;               // set to non-zero to automatically buffer stderror output
    virtual bool hasInput() = 0;                                    // i.e. can write to pipe
    virtual size32_t write(size32_t sz, const void *buffer) = 0;    // write pipe process standard output
    virtual bool hasOutput() = 0;                                   // i.e. can read from pipe
    virtual size32_t read(size32_t sz, void *buffer) = 0;           // read from pipe process standard output
    virtual ISimpleReadStream *getOutputStream() = 0;               // read from pipe process standard output
    virtual bool hasError() = 0;                                    // i.e. can read from pipe stderr
    virtual size32_t readError(size32_t sz, void *buffer) = 0;      // read from pipe process standard error
    virtual ISimpleReadStream *getErrorStream() = 0;                // read from pipe process standard error

    virtual unsigned wait() = 0;                                    // returns return code
    virtual unsigned wait(unsigned timeoutms, bool &timedout) = 0;  // sets timedout to true if times out
    virtual void closeInput() = 0;                                  // indicate finished input to pipe
    virtual void closeOutput() = 0;                                 // indicate finished reading from pipe (generally called automatically)
    virtual void closeError() = 0;                                  // indicate finished reading from pipe stderr
    virtual void abort() = 0;
    virtual void notifyTerminated(HANDLE pid,unsigned retcode) = 0; // internal
    virtual HANDLE getProcessHandle() = 0;                          // used to auto kill
};

extern jlib_decl IPipeProcess *createPipeProcess(const char *allowedprograms=NULL);

//--------------------------------------------------------

interface IWorkQueueItem: extends IInterface
{
    virtual void execute()=0;
};

interface IWorkQueueThread: extends IInterface
{
    virtual void post(IWorkQueueItem *item)=0;  // takes ownership of item
    virtual void wait()=0;
    virtual unsigned pending()=0;
};

// Simple lightweight async worker queue
// internally thread persists for specified time waiting before self destroying
extern jlib_decl IWorkQueueThread *createWorkQueueThread(unsigned persisttime=1000*60);

#endif
<|endoftext|>


<commit_before>//
//  ex10_32.cpp
//  Exercise 10.32
//
//  Created by pezy on 12/13/14.
//  Copyright (c) 2014 pezy. All rights reserved.
//
//  Rewrite the bookstore problem from 1.6 (p. 24) using a vector to hold the transactions
//  and various algorithms to do the processing.
//  Use sort with your compareIsbn function from 10.3.1 (p. 387) to arrange the transactions in order,
//  and then use find and accumulate to do the sum.

#include <iostream>
#include <vector>
#include <algorithm>
#include <iterator>
#include <numeric>
#include "../include/Sales_item.h"

int main()
{
    std::istream_iterator<Sales_item> in_iter(std::cin), in_eof;
    std::vector<Sales_item> vec;

    while (in_iter != in_eof)
        vec.push_back(*in_iter++);
    sort(vec.begin(), vec.end(), compareIsbn);
    for (auto beg = vec.cbegin(), end = beg; beg != vec.cend(); beg = end) {
        end = find_if(beg, vec.cend(), [beg](const Sales_item &item){return item.isbn() != beg->isbn();});
        std::cout << std::accumulate(beg, end, Sales_item(beg->isbn())) << std::endl;
    }
}<commit_msg>fixed ReadingLab/Discussion-for-Cpp#12<commit_after>//
//  ex10_32.cpp
//  Exercise 10.32
//
//  Created by pezy on 12/13/14.
//  Copyright (c) 2014 pezy. All rights reserved.
//
//  Rewrite the bookstore problem from 1.6 (p. 24) using a vector to hold the transactions
//  and various algorithms to do the processing.
//  Use sort with your compareIsbn function from 10.3.1 (p. 387) to arrange the transactions in order,
//  and then use find and accumulate to do the sum.

#include <iostream>
#include <vector>
#include <algorithm>
#include <iterator>
#include <numeric>
#include "../include/Sales_item.h"

int main()
{
    std::istream_iterator<Sales_item> in_iter(std::cin), in_eof;
    std::vector<Sales_item> vec;

    while (in_iter != in_eof)
        vec.push_back(*in_iter++);
    sort(vec.begin(), vec.end(), [](Sales_item const& lhs, Sales_item const& rhs){ return lhs.isbn() < rhs.isbn(); });
    for (auto beg = vec.cbegin(), end = beg; beg != vec.cend(); beg = end) {
        end = find_if(beg, vec.cend(), [beg](const Sales_item &item){return item.isbn() != beg->isbn();});
        std::cout << std::accumulate(beg, end, Sales_item(beg->isbn())) << std::endl;
    }
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

/* When running tests on remote machines, the framework takes a round-robin pick
 * of a port within certain range. There is no need to recycle ports.
 */
#include "src/core/lib/iomgr/port.h"
#include "test/core/util/test_config.h"
#if defined(GRPC_PORT_ISOLATED_RUNTIME)

#include "test/core/util/port.h"

#define LOWER_PORT 49152
static int s_allocated_port = LOWER_PORT;

int grpc_pick_unused_port_or_die(void) {
  int allocated_port = s_allocated_port++;
  if (s_allocated_port == 65536) {
    s_allocated_port = LOWER_PORT;
  }

  return allocated_port;
}

void grpc_recycle_unused_port(int port) { (void)port; }

#endif /* GRPC_PORT_ISOLATED_RUNTIME */
<commit_msg>Pick a random starting port<commit_after>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

/* When running tests on remote machines, the framework takes a round-robin pick
 * of a port within certain range. There is no need to recycle ports.
 */
#include <grpc/support/time.h>
#include <stdlib.h>
#include "src/core/lib/iomgr/port.h"
#include "test/core/util/test_config.h"
#if defined(GRPC_PORT_ISOLATED_RUNTIME)

#include "test/core/util/port.h"

#define MIN_PORT 49152
#define MAX_PORT 65536

int get_random_starting_port() {
  srand(gpr_now(GPR_CLOCK_REALTIME).tv_nsec);
  return rand() % (MAX_PORT - MIN_PORT + 1) + MIN_PORT;
}

static int s_allocated_port = get_random_starting_port();

int grpc_pick_unused_port_or_die(void) {
  int allocated_port = s_allocated_port++;
  if (s_allocated_port == MAX_PORT) {
    s_allocated_port = MIN_PORT;
  }

  return allocated_port;
}

void grpc_recycle_unused_port(int port) { (void)port; }

#endif /* GRPC_PORT_ISOLATED_RUNTIME */
<|endoftext|>


<commit_before>#include "helpers.hpp"

/// Convert non-standard taxon to a standard taxon (per taxon)
uint32_t correctTaxan(uint32_t tempTax)
{
	string tempRank=rankMap[tempTax];

	if (tempTax==0)
	{
		//cout << "The taxon = "  << tempTax << "\n";
		return 0;
	}
	else if (tempRank == "")
	{
		undefined++;
		cout << "The taxon = "  << tempTax << " is undefined in the database \n";
		return 0;
	}
	if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
	{
		while(1)
		{
			tempTax=parentMap[tempTax];
			tempRank=rankMap[tempTax];
			if(tempRank=="root" || tempRank=="superkingdom" || tempRank == "kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
			{
				break;
			}
		}
	}

	return tempTax;
}

/// Convert non-standard taxon to a standard taxon (per vector)
void correctTaxa(vector<readsInfo> &seq)
{
	uint32_t tempTax;
	string tempRank="";

	for (uint32_t i=0 ; i<seq.size() ;i++)
	{

		for (uint32_t j=0 ; j<seq[i].annotation.size(); j++)
		{

			tempTax = seq[i].annotation[j];
			tempRank = rankMap[tempTax];

			if (tempTax==0)
			{
				continue;
			}
			else if (tempRank == "")
			{
				undefined++;
				seq[i].annotation[j] =0;
				cout << "The taxonomy on line= " << i << "  which is= " << seq[i].annotation[j] << " is not availbe \n";
				continue;
			}

			if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
			{
				while(1)
				{
					tempTax=parentMap[tempTax];
					tempRank=rankMap[tempTax];
					if(tempRank=="root" || tempRank=="superkingdom" || tempRank == "kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
					{
						seq[i].annotation[j] = tempTax;
						break;
					}
				}
			}
		}
	}
}

/// For an array of taxa: Convert non-standard taxon to a standard taxon
void correctTaxa_vecTaxa(vector<uint32_t> &InputTaxa)
{
	uint32_t tempTax;
	string tempRank="";

	for (uint32_t i=0 ; i<InputTaxa.size() ;i++)
	{

			tempTax = InputTaxa[i];
			tempRank = rankMap[tempTax];

			if (tempTax==0)
			{
				continue;
			}
			else if (tempRank == "")
			{
				undefined++;
				InputTaxa[i] =0;
				cout << "The taxonomy on line= " << i << "  which is= " << InputTaxa[i] << " is not availbe \n";
				continue;
			}

			if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
			{
				while(1)
				{
					tempTax=parentMap[tempTax];
					tempRank=rankMap[tempTax];
					if(tempRank=="root" || tempRank=="superkingdom" || tempRank=="kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
					{
						InputTaxa[i] = tempTax;
						break;
					}
				}
			}
	}
}

/// Get LCA
uint32_t lca(uint32_t a, uint32_t b)
{
	if (a == 0 || b == 0)
		return a ? a : b;

	set<uint32_t> a_path;
	while (a > 0)
	{
		a_path.insert(a);
		a = standardMap[a];
	}
	while (b > 0)
	{
		if (a_path.count(b) > 0)
			return b;
		b = standardMap[b];
	}
	return 1;
}

/// Get all ancestry of a taxon
set<uint32_t> get_ancestry(uint32_t taxon)
{
	set<uint32_t> path;
	path.clear();
	while (taxon > 0)
	{
		path.insert(taxon);
		taxon = standardMap[taxon];
	}
	return path;
}

/// Return the taxon of the highest weighted Root-to-Taxon path and passes WEVOTE threshold.
uint32_t resolve_tree(map<uint32_t, uint32_t> &hit_counts, uint32_t numToolsReported, uint32_t minNumAgreed)
{
	set<uint32_t> max_taxa;
	uint32_t max_taxon = 0, max_score = 0;
	map<uint32_t, uint32_t>::iterator it = hit_counts.begin();
	uint32_t Threshold=floor(0.5*(double)numToolsReported);

	if((minNumAgreed-1) > Threshold)
	Threshold=(minNumAgreed-1);

	while (it != hit_counts.end())
	{
		if(it->second <= Threshold)
		{
			it++;
			continue;
		}

		uint32_t taxon = it->first;
		uint32_t node = taxon;
		uint32_t score = 0;
		while (node > 0)
		{
			score += hit_counts[node];
			node = standardMap[node];
		}

		if (score > max_score)
		{
			max_taxa.clear();
			max_score = score;
			max_taxon = taxon;
		}
		else if (score == max_score)
		{
			if (max_taxa.empty())
				max_taxa.insert(max_taxon);

			max_taxa.insert(taxon);
		}

		it++;
	}

	if (! max_taxa.empty())
	{
		set<uint32_t>::iterator sit = max_taxa.begin();
		max_taxon = *sit;

		for (sit++; sit != max_taxa.end(); sit++)
		max_taxon = lca(max_taxon, *sit);
	}
	return max_taxon;
}

<commit_msg>solve the overflow bug<commit_after>#include "helpers.hpp"

/// Convert non-standard taxon to a standard taxon (per taxon)
uint32_t correctTaxan(uint32_t tempTax)
{
	string tempRank=rankMap[tempTax];

	if (tempTax==0)
	{
		//cout << "The taxon = "  << tempTax << "\n";
		return 0;
	}
	else if (tempRank == "")
	{
		undefined++;
		cout << "The taxon = "  << tempTax << " is undefined in the database \n";
		return 0;
	}
	if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
	{
		while(1)
		{
			tempTax=parentMap[tempTax];
			tempRank=rankMap[tempTax];
			if(tempRank=="root" || tempRank=="superkingdom" || tempRank == "kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
			{
				break;
			}
		}
	}

	return tempTax;
}

/// Convert non-standard taxon to a standard taxon (per vector)
void correctTaxa(vector<readsInfo> &seq)
{
	uint32_t tempTax;
	string tempRank="";

	for (uint32_t i=0 ; i<seq.size() ;i++)
	{

		for (uint32_t j=0 ; j<seq[i].annotation.size(); j++)
		{

			tempTax = seq[i].annotation[j];
			tempRank = rankMap[tempTax];

			if (tempTax==0)
			{
				continue;
			}
			else if (tempRank == "")
			{
				undefined++;
				seq[i].annotation[j] =0;
				cout << "The taxonomy on line= " << i << "  which is= " << seq[i].annotation[j] << " is not availbe \n";
				continue;
			}

			if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
			{
				while(1)
				{
					tempTax=parentMap[tempTax];
					tempRank=rankMap[tempTax];
					if(tempRank=="root" || tempRank=="superkingdom" || tempRank == "kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
					{
						seq[i].annotation[j] = tempTax;
						break;
					}
				}
			}
		}
	}
}

/// For an array of taxa: Convert non-standard taxon to a standard taxon
void correctTaxa_vecTaxa(vector<uint32_t> &InputTaxa)
{
	uint32_t tempTax;
	string tempRank="";

	for (uint32_t i=0 ; i<InputTaxa.size() ;i++)
	{

			tempTax = InputTaxa[i];
			tempRank = rankMap[tempTax];

			if (tempTax==0)
			{
				continue;
			}
			else if (tempRank == "")
			{
				undefined++;
				InputTaxa[i] =0;
				cout << "The taxonomy on line= " << i << "  which is= " << InputTaxa[i] << " is not availbe \n";
				continue;
			}

			if (!( tempRank=="root" || tempRank == "superkingdom" || tempRank == "kingdom" || tempRank== "phylum" || tempRank == "class" || tempRank == "order" || tempRank == "family" || tempRank == "genus" || tempRank == "species"))
			{
				while(1)
				{
					tempTax=parentMap[tempTax];
					tempRank=rankMap[tempTax];
					if(tempRank=="root" || tempRank=="superkingdom" || tempRank=="kingdom" || tempRank=="phylum" || tempRank=="class" || tempRank=="order" || tempRank=="family" || tempRank=="genus" || tempRank=="species")
					{
						InputTaxa[i] = tempTax;
						break;
					}
				}
			}
	}
}

/// Get LCA
uint32_t lca(uint32_t a, uint32_t b)
{
	if (a == 0 || b == 0)
		return a ? a : b;

	set<uint32_t> a_path;
	while (a > 0)
	{
		a_path.insert(a);
		a = standardMap[a];
	}
	while (b > 0)
	{
		if (a_path.count(b) > 0)
			return b;
		b = standardMap[b];
	}
	return 1;
}

/// Get all ancestry of a taxon
set<uint32_t> get_ancestry(uint32_t taxon)
{
	set<uint32_t> path;
	path.clear();
	while (taxon > 0)
	{
		path.insert(taxon);
		taxon = standardMap[taxon];
	}
	return path;
}

/// Return the taxon of the highest weighted Root-to-Taxon path and passes WEVOTE threshold.
uint32_t resolve_tree(map<uint32_t, uint32_t> &hit_counts, uint32_t numToolsReported, uint32_t minNumAgreed)
{
	set<uint32_t> max_taxa;
	uint32_t max_taxon = 0, max_score = 0;
	map<uint32_t, uint32_t>::iterator it = hit_counts.begin();
	uint32_t Threshold=floor(0.5*(double)numToolsReported);

	if(minNumAgreed > Threshold)
	Threshold=(minNumAgreed-1);

	while (it != hit_counts.end())
	{
		if(it->second <= Threshold)
		{
			it++;
			continue;
		}

		uint32_t taxon = it->first;
		uint32_t node = taxon;
		uint32_t score = 0;
		while (node > 0)
		{
			score += hit_counts[node];
			node = standardMap[node];
		}

		if (score > max_score)
		{
			max_taxa.clear();
			max_score = score;
			max_taxon = taxon;
		}
		else if (score == max_score)
		{
			if (max_taxa.empty())
				max_taxa.insert(max_taxon);

			max_taxa.insert(taxon);
		}

		it++;
	}

	if (! max_taxa.empty())
	{
		set<uint32_t>::iterator sit = max_taxa.begin();
		max_taxon = *sit;

		for (sit++; sit != max_taxa.end(); sit++)
		max_taxon = lca(max_taxon, *sit);
	}
	return max_taxon;
}

<|endoftext|>


<commit_before>/*
 * Copyright (c) 2017 Darren Smith
 *
 * wampcc is free software; you can redistribute it and/or modify
 * it under the terms of the MIT license. See LICENSE for details.
 */

#include "test_common.h"

using namespace wampcc;
using namespace std;

// logging control
#ifdef TLOG
#undef TLOG
#define TLOG(X)
#endif

void test_WS_destroyed_after_kernel(int port)
{
  TLOG("----- test_WS_destroyed_after_kernel -----");

  callback_status = callback_status_t::not_invoked;

  shared_ptr<wamp_session> ws_outer;
  {
    unique_ptr<kernel> the_kernel( new kernel({}, logger::nolog() ) );

    /* attempt to connect the socket */
    TLOG("attemping socket connection ...");
    unique_ptr<tcp_socket> sock (new tcp_socket(the_kernel.get()));
    auto fut = sock->connect("127.0.0.1", port);
    auto connect_status = fut.wait_for(chrono::milliseconds(100));

    if (connect_status == future_status::timeout)
    {
      throw runtime_error("unexpected -- should have connected");
    }
    TLOG("got socket connection");

    /* attempt to create a session */
    TLOG("attemping session creation");
    shared_ptr<wamp_session> session = wamp_session::create<rawsocket_protocol>(
      the_kernel.get(),
      std::move(sock),
      session_cb, {});
    TLOG("got session");

    TLOG("assigning session to outer scope (causes wamp_session destruction after kernel destruction)");
    ws_outer = session;
    TLOG("exiting scope (will trigger kernel, io_loop, ev_loop destruction)");
  }
  TLOG("scope complete");
  TLOG("triggering ~wamp_session...");
  ws_outer.reset();
  TLOG("complete");

  // In this test, the kernel is deleted before the wamp_session is deleted (due
  // to the outer shared_ptr holding on to it). This ordering means that the
  // wamp_session is still available during the wamp_session callback.
  assert(callback_status == callback_status_t::close_with_sp);

  TLOG("test success");
}

int main(int argc, char** argv)
{
  try
  {
    throw std::runtime_error("TEST STILL UNDER DEVELOPMENT");

    int starting_port_number = 24000;
    int loops = 500;

    if (argc>1)
      starting_port_number = atoi(argv[1]);

    // share a common internal_server
    for (int i = 0; i < loops; i++)
    {
      // build a client
      std::unique_ptr<internal_server> iserver(new internal_server());
      int port = iserver->start(starting_port_number++);

      // add connections
      //add_client_connections(port, 10);

      // drop the client
      iserver.reset();

    }

    return 0;
  }
  catch (exception& e)
  {
    cout << e.what() << endl;
    return 1;
  }

}
<commit_msg>remove compiler warning<commit_after>/*
 * Copyright (c) 2017 Darren Smith
 *
 * wampcc is free software; you can redistribute it and/or modify
 * it under the terms of the MIT license. See LICENSE for details.
 */

#include "test_common.h"

using namespace wampcc;
using namespace std;

// logging control
#ifdef TLOG
#undef TLOG
#define TLOG(X)
#endif

void test_WS_destroyed_after_kernel(int port)
{
  TLOG("----- test_WS_destroyed_after_kernel -----");

  callback_status = callback_status_t::not_invoked;

  shared_ptr<wamp_session> ws_outer;
  {
    unique_ptr<kernel> the_kernel( new kernel({}, logger::nolog() ) );

    /* attempt to connect the socket */
    TLOG("attemping socket connection ...");
    unique_ptr<tcp_socket> sock (new tcp_socket(the_kernel.get()));
    auto fut = sock->connect("127.0.0.1", port);
    auto connect_status = fut.wait_for(chrono::milliseconds(100));

    if (connect_status == future_status::timeout)
    {
      throw runtime_error("unexpected -- should have connected");
    }
    TLOG("got socket connection");

    /* attempt to create a session */
    TLOG("attemping session creation");
    shared_ptr<wamp_session> session = wamp_session::create<rawsocket_protocol>(
      the_kernel.get(),
      std::move(sock),
      session_cb, {});
    TLOG("got session");

    TLOG("assigning session to outer scope (causes wamp_session destruction after kernel destruction)");
    ws_outer = session;
    TLOG("exiting scope (will trigger kernel, io_loop, ev_loop destruction)");
  }
  TLOG("scope complete");
  TLOG("triggering ~wamp_session...");
  ws_outer.reset();
  TLOG("complete");

  // In this test, the kernel is deleted before the wamp_session is deleted (due
  // to the outer shared_ptr holding on to it). This ordering means that the
  // wamp_session is still available during the wamp_session callback.
  assert(callback_status == callback_status_t::close_with_sp);

  TLOG("test success");
}

int main(int argc, char** argv)
{
  try
  {
    throw std::runtime_error("TEST STILL UNDER DEVELOPMENT");

    int starting_port_number = 24000;
    int loops = 500;

    if (argc>1)
      starting_port_number = atoi(argv[1]);

    // share a common internal_server
    for (int i = 0; i < loops; i++)
    {
      // build a client
      std::unique_ptr<internal_server> iserver(new internal_server());
      iserver->start(starting_port_number++);

      // add connections
      //add_client_connections(port, 10);

      // drop the client
      iserver.reset();

    }

    return 0;
  }
  catch (exception& e)
  {
    cout << e.what() << endl;
    return 1;
  }

}
<|endoftext|>


<commit_before>/**
 * @file    Column_test.cpp
 * @ingroup tests
 * @brief   Test of a SQLiteCpp Column.
 *
 * Copyright (c) 2012-2016 Sebastien Rombauts (sebastien.rombauts@gmail.com)
 *
 * Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt
 * or copy at http://opensource.org/licenses/MIT)
 */

#include <SQLiteCpp/Database.h>
#include <SQLiteCpp/Statement.h>
#include <SQLiteCpp/Column.h>

#include <sqlite3.h> // for sqlite3_int64

#include <gtest/gtest.h>

#include <cstdio>
#include <stdint.h>

TEST(Column, basis) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(SQLite::OK, db.getErrorCode());
    EXPECT_EQ(SQLite::OK, db.getExtendedErrorCode());

    // Create a new table
    EXPECT_EQ(0, db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, msg TEXT, int INTEGER, double REAL, binary BLOB, empty TEXT)"));
    EXPECT_TRUE(db.tableExists("test"));
    EXPECT_TRUE(db.tableExists(std::string("test")));
    EXPECT_EQ(0, db.getLastInsertRowid());

    // Create a first row (autoid: 1) with all kind of data and a null value
    SQLite::Statement   insert(db, "INSERT INTO test VALUES (NULL, \"first\", -123, 0.123, ?, NULL)");
    // Bind the blob value to the first parameter of the SQL query
    const char  buffer[] = {'b', 'l', '\0', 'b'}; // "bl\0b" : 4 char, with a null byte inside
    const int   size = sizeof(buffer); // size = 4
    const void* blob = &buffer;
    insert.bind(1, blob, size);
    // Execute the one-step query to insert the row
    EXPECT_EQ(1, insert.exec());
    EXPECT_EQ(1, db.getLastInsertRowid());
    EXPECT_EQ(1, db.getTotalChanges());

    EXPECT_THROW(insert.exec(), SQLite::Exception); // exec() shall throw as it needs to be reseted

    // Compile a SQL query
    SQLite::Statement   query(db, "SELECT * FROM test");
    EXPECT_STREQ("SELECT * FROM test", query.getQuery().c_str());
    EXPECT_EQ(6, query.getColumnCount ());
    query.executeStep();
    EXPECT_TRUE (query.isOk());
    EXPECT_FALSE(query.isDone());

    // validates every variant of cast operators, and conversions of types
    {
        const sqlite3_int64 id1     = query.getColumn(0); // operator int64_t()
        const int64_t       id2     = query.getColumn(0); // operator int64_t()
        const long long     id3     = query.getColumn(0); // operator int64_t()
        const long          id4     = query.getColumn(0); // operator int64_t() or long() depending on compiler/architecture
        const unsigned int  uint1   = query.getColumn(0); // operator uint32_t()
        const uint32_t      uint2   = query.getColumn(0); // operator uint32_t()
        const char*         ptxt    = query.getColumn(1); // operator const char*()
        const std::string   msg     = query.getColumn(1); // operator std::string() (or const char* with MSVC)
        const int           integer = query.getColumn(2); // operator int()
        const double        real    = query.getColumn(3); // operator double()
        const void*         pblob   = query.getColumn(4); // operator void*()
        const std::string   sblob   = query.getColumn(4); // operator std::string() (or const char* with MSVC)
        const void*         pempty  = query.getColumn(5); // operator void*()
        EXPECT_EQ(1,            id1);
        EXPECT_EQ(1,            id2);
        EXPECT_EQ(1,            id3);
        EXPECT_EQ(1,            id4);
        EXPECT_EQ(1,            uint1);
        EXPECT_EQ(1,            uint2);
        EXPECT_STREQ("first",   ptxt);
        EXPECT_EQ("first",      msg);
        EXPECT_EQ(-123,         integer);
        EXPECT_EQ(0.123,        real);
        EXPECT_EQ(0,            memcmp("bl\0b", pblob, size));
        EXPECT_EQ(size,         sblob.size());
        EXPECT_EQ(0,            memcmp("bl\0b", &sblob[0], size));
        EXPECT_EQ(NULL,         pempty);
    }

    // validates every variant of explicit getters
    {
        int64_t             id      = query.getColumn(0).getInt64();
        const unsigned int  uint1   = query.getColumn(0).getUInt();
        const uint32_t      uint2   = query.getColumn(0).getUInt();
        const char*         ptxt    = query.getColumn(1).getText();
        const std::string   msg1    = query.getColumn(1).getText();
        const std::string   msg2    = query.getColumn(1).getString();
        const int           integer = query.getColumn(2).getInt();
        const double        real    = query.getColumn(3).getDouble();
        const void*         pblob   = query.getColumn(4).getBlob();
        const std::string   sblob   = query.getColumn(4).getString();
        EXPECT_EQ(1,            id);
        EXPECT_EQ(1,            uint1);
        EXPECT_EQ(1,            uint2);
        EXPECT_STREQ("first",   ptxt);
        EXPECT_EQ("first",      msg1);
        EXPECT_EQ("first",      msg2);
        EXPECT_EQ(-123,         integer);
        EXPECT_EQ(0.123,        real);
        EXPECT_EQ(0,            memcmp("bl\0b", pblob, 4));
        EXPECT_EQ(0,            memcmp("bl\0b", &sblob[0], 4));
    }

    // Validate getBytes(), getType(), isInteger(), isNull()...
    EXPECT_EQ(SQLite::INTEGER,  query.getColumn(0).getType());
    EXPECT_EQ(true,             query.getColumn(0).isInteger());
    EXPECT_EQ(false,            query.getColumn(0).isFloat());
    EXPECT_EQ(false,            query.getColumn(0).isText());
    EXPECT_EQ(false,            query.getColumn(0).isBlob());
    EXPECT_EQ(false,            query.getColumn(0).isNull());
    EXPECT_STREQ("1",           query.getColumn(0).getText());  // convert to string
    EXPECT_EQ(1,                query.getColumn(0).getBytes()); // size of the string "1" without the null terminator
    EXPECT_EQ(SQLite::TEXT,     query.getColumn(1).getType());
    EXPECT_EQ(false,            query.getColumn(1).isInteger());
    EXPECT_EQ(false,            query.getColumn(1).isFloat());
    EXPECT_EQ(true,             query.getColumn(1).isText());
    EXPECT_EQ(false,            query.getColumn(1).isBlob());
    EXPECT_EQ(false,            query.getColumn(1).isNull());
    EXPECT_STREQ("first",       query.getColumn(1).getText());  // convert to string
    EXPECT_EQ(5,                query.getColumn(1).getBytes()); // size of the string "first"
    EXPECT_EQ(SQLite::INTEGER,  query.getColumn(2).getType());
    EXPECT_EQ(true,             query.getColumn(2).isInteger());
    EXPECT_EQ(false,            query.getColumn(2).isFloat());
    EXPECT_EQ(false,            query.getColumn(2).isText());
    EXPECT_EQ(false,            query.getColumn(2).isBlob());
    EXPECT_EQ(false,            query.getColumn(2).isNull());
    EXPECT_STREQ("-123",        query.getColumn(2).getText());  // convert to string
    EXPECT_EQ(4,                query.getColumn(2).getBytes()); // size of the string "-123"
    EXPECT_EQ(SQLite::FLOAT,    query.getColumn(3).getType());
    EXPECT_EQ(false,            query.getColumn(3).isInteger());
    EXPECT_EQ(true,             query.getColumn(3).isFloat());
    EXPECT_EQ(false,            query.getColumn(3).isText());
    EXPECT_EQ(false,            query.getColumn(3).isBlob());
    EXPECT_EQ(false,            query.getColumn(3).isNull());
    EXPECT_STREQ("0.123",       query.getColumn(3).getText());  // convert to string
    EXPECT_EQ(5,                query.getColumn(3).getBytes()); // size of the string "0.123"
    EXPECT_EQ(SQLite::BLOB,     query.getColumn(4).getType());
    EXPECT_EQ(false,            query.getColumn(4).isInteger());
    EXPECT_EQ(false,            query.getColumn(4).isFloat());
    EXPECT_EQ(false,            query.getColumn(4).isText());
    EXPECT_EQ(true,             query.getColumn(4).isBlob());
    EXPECT_EQ(false,            query.getColumn(4).isNull());
    EXPECT_STREQ("bl\0b",       query.getColumn(4).getText());  // convert to string
    EXPECT_EQ(4,                query.getColumn(4).getBytes()); // size of the blob "bl\0b" with the null char
    EXPECT_EQ(SQLite::Null,     query.getColumn(5).getType());
    EXPECT_EQ(false,            query.getColumn(5).isInteger());
    EXPECT_EQ(false,            query.getColumn(5).isFloat());
    EXPECT_EQ(false,            query.getColumn(5).isText());
    EXPECT_EQ(false,            query.getColumn(5).isBlob());
    EXPECT_EQ(true,             query.getColumn(5).isNull());
    EXPECT_STREQ("",            query.getColumn(5).getText());  // convert to string
    EXPECT_EQ(0,                query.getColumn(5).getBytes()); // size of the string "" without the null terminator

    // Use intermediate Column objects (this is not the recommended way to use the API)
    {
        const SQLite::Column id = query.getColumn(0);
        EXPECT_EQ(1, id.getInt64());
        const SQLite::Column msg = query.getColumn(1);
        EXPECT_EQ("first", msg.getString());
        const SQLite::Column integer = query.getColumn(2);
        EXPECT_EQ(-123, integer.getInt());
        const SQLite::Column dbl = query.getColumn(3);
        EXPECT_EQ(0.123, dbl.getDouble());
    }
}

TEST(Column, getName) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(0, db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, msg TEXT)"));
    EXPECT_EQ(1, db.exec("INSERT INTO test VALUES (NULL, \"first\")"));

    // Compile a SQL query, using the "id" column name as-is, but aliasing the "msg" column with new name "value"
    SQLite::Statement   query(db, "SELECT id, msg as value FROM test");
    query.executeStep();

    // Show how to get the aliased names of the result columns.
    const std::string name0 = query.getColumn(0).getName();
    const std::string name1 = query.getColumn(1).getName();
    EXPECT_EQ("id",     name0);
    EXPECT_EQ("value",  name1);

#ifdef SQLITE_ENABLE_COLUMN_METADATA
    // Show how to get origin names of the table columns from which theses result columns come from.
    // Requires the SQLITE_ENABLE_COLUMN_METADATA preprocessor macro to be
    // also defined at compile times of the SQLite library itself.
    const std::string oname0 = query.getColumn(0).getOriginName();
    const std::string oname1 = query.getColumn(1).getOriginName();
    EXPECT_EQ("id",     oname0);
    EXPECT_EQ("msg",    oname1);
#endif
}

TEST(Column, stream) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(0, db.exec("CREATE TABLE test (msg TEXT)"));
    SQLite::Statement insert(db, "INSERT INTO test VALUES (?)");

    // content to test
    const char str_[] = "stringwith\0embedded";
    std::string str(str_, sizeof(str_)-1);

    insert.bind(1, str);
    // Execute the one-step query to insert the row
    EXPECT_EQ(1, insert.exec());
    EXPECT_EQ(1, db.getTotalChanges());

    SQLite::Statement query(db, "SELECT * FROM test");
    query.executeStep();
    std::stringstream ss;
    auto col = query.getColumn(0);
    ss << query.getColumn(0);
    std::string content = ss.str();
    EXPECT_EQ(content, str);
}
<commit_msg>Remove unused variable<commit_after>/**
 * @file    Column_test.cpp
 * @ingroup tests
 * @brief   Test of a SQLiteCpp Column.
 *
 * Copyright (c) 2012-2016 Sebastien Rombauts (sebastien.rombauts@gmail.com)
 *
 * Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt
 * or copy at http://opensource.org/licenses/MIT)
 */

#include <SQLiteCpp/Database.h>
#include <SQLiteCpp/Statement.h>
#include <SQLiteCpp/Column.h>

#include <sqlite3.h> // for sqlite3_int64

#include <gtest/gtest.h>

#include <cstdio>
#include <stdint.h>

TEST(Column, basis) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(SQLite::OK, db.getErrorCode());
    EXPECT_EQ(SQLite::OK, db.getExtendedErrorCode());

    // Create a new table
    EXPECT_EQ(0, db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, msg TEXT, int INTEGER, double REAL, binary BLOB, empty TEXT)"));
    EXPECT_TRUE(db.tableExists("test"));
    EXPECT_TRUE(db.tableExists(std::string("test")));
    EXPECT_EQ(0, db.getLastInsertRowid());

    // Create a first row (autoid: 1) with all kind of data and a null value
    SQLite::Statement   insert(db, "INSERT INTO test VALUES (NULL, \"first\", -123, 0.123, ?, NULL)");
    // Bind the blob value to the first parameter of the SQL query
    const char  buffer[] = {'b', 'l', '\0', 'b'}; // "bl\0b" : 4 char, with a null byte inside
    const int   size = sizeof(buffer); // size = 4
    const void* blob = &buffer;
    insert.bind(1, blob, size);
    // Execute the one-step query to insert the row
    EXPECT_EQ(1, insert.exec());
    EXPECT_EQ(1, db.getLastInsertRowid());
    EXPECT_EQ(1, db.getTotalChanges());

    EXPECT_THROW(insert.exec(), SQLite::Exception); // exec() shall throw as it needs to be reseted

    // Compile a SQL query
    SQLite::Statement   query(db, "SELECT * FROM test");
    EXPECT_STREQ("SELECT * FROM test", query.getQuery().c_str());
    EXPECT_EQ(6, query.getColumnCount ());
    query.executeStep();
    EXPECT_TRUE (query.isOk());
    EXPECT_FALSE(query.isDone());

    // validates every variant of cast operators, and conversions of types
    {
        const sqlite3_int64 id1     = query.getColumn(0); // operator int64_t()
        const int64_t       id2     = query.getColumn(0); // operator int64_t()
        const long long     id3     = query.getColumn(0); // operator int64_t()
        const long          id4     = query.getColumn(0); // operator int64_t() or long() depending on compiler/architecture
        const unsigned int  uint1   = query.getColumn(0); // operator uint32_t()
        const uint32_t      uint2   = query.getColumn(0); // operator uint32_t()
        const char*         ptxt    = query.getColumn(1); // operator const char*()
        const std::string   msg     = query.getColumn(1); // operator std::string() (or const char* with MSVC)
        const int           integer = query.getColumn(2); // operator int()
        const double        real    = query.getColumn(3); // operator double()
        const void*         pblob   = query.getColumn(4); // operator void*()
        const std::string   sblob   = query.getColumn(4); // operator std::string() (or const char* with MSVC)
        const void*         pempty  = query.getColumn(5); // operator void*()
        EXPECT_EQ(1,            id1);
        EXPECT_EQ(1,            id2);
        EXPECT_EQ(1,            id3);
        EXPECT_EQ(1,            id4);
        EXPECT_EQ(1,            uint1);
        EXPECT_EQ(1,            uint2);
        EXPECT_STREQ("first",   ptxt);
        EXPECT_EQ("first",      msg);
        EXPECT_EQ(-123,         integer);
        EXPECT_EQ(0.123,        real);
        EXPECT_EQ(0,            memcmp("bl\0b", pblob, size));
        EXPECT_EQ(size,         sblob.size());
        EXPECT_EQ(0,            memcmp("bl\0b", &sblob[0], size));
        EXPECT_EQ(NULL,         pempty);
    }

    // validates every variant of explicit getters
    {
        int64_t             id      = query.getColumn(0).getInt64();
        const unsigned int  uint1   = query.getColumn(0).getUInt();
        const uint32_t      uint2   = query.getColumn(0).getUInt();
        const char*         ptxt    = query.getColumn(1).getText();
        const std::string   msg1    = query.getColumn(1).getText();
        const std::string   msg2    = query.getColumn(1).getString();
        const int           integer = query.getColumn(2).getInt();
        const double        real    = query.getColumn(3).getDouble();
        const void*         pblob   = query.getColumn(4).getBlob();
        const std::string   sblob   = query.getColumn(4).getString();
        EXPECT_EQ(1,            id);
        EXPECT_EQ(1,            uint1);
        EXPECT_EQ(1,            uint2);
        EXPECT_STREQ("first",   ptxt);
        EXPECT_EQ("first",      msg1);
        EXPECT_EQ("first",      msg2);
        EXPECT_EQ(-123,         integer);
        EXPECT_EQ(0.123,        real);
        EXPECT_EQ(0,            memcmp("bl\0b", pblob, 4));
        EXPECT_EQ(0,            memcmp("bl\0b", &sblob[0], 4));
    }

    // Validate getBytes(), getType(), isInteger(), isNull()...
    EXPECT_EQ(SQLite::INTEGER,  query.getColumn(0).getType());
    EXPECT_EQ(true,             query.getColumn(0).isInteger());
    EXPECT_EQ(false,            query.getColumn(0).isFloat());
    EXPECT_EQ(false,            query.getColumn(0).isText());
    EXPECT_EQ(false,            query.getColumn(0).isBlob());
    EXPECT_EQ(false,            query.getColumn(0).isNull());
    EXPECT_STREQ("1",           query.getColumn(0).getText());  // convert to string
    EXPECT_EQ(1,                query.getColumn(0).getBytes()); // size of the string "1" without the null terminator
    EXPECT_EQ(SQLite::TEXT,     query.getColumn(1).getType());
    EXPECT_EQ(false,            query.getColumn(1).isInteger());
    EXPECT_EQ(false,            query.getColumn(1).isFloat());
    EXPECT_EQ(true,             query.getColumn(1).isText());
    EXPECT_EQ(false,            query.getColumn(1).isBlob());
    EXPECT_EQ(false,            query.getColumn(1).isNull());
    EXPECT_STREQ("first",       query.getColumn(1).getText());  // convert to string
    EXPECT_EQ(5,                query.getColumn(1).getBytes()); // size of the string "first"
    EXPECT_EQ(SQLite::INTEGER,  query.getColumn(2).getType());
    EXPECT_EQ(true,             query.getColumn(2).isInteger());
    EXPECT_EQ(false,            query.getColumn(2).isFloat());
    EXPECT_EQ(false,            query.getColumn(2).isText());
    EXPECT_EQ(false,            query.getColumn(2).isBlob());
    EXPECT_EQ(false,            query.getColumn(2).isNull());
    EXPECT_STREQ("-123",        query.getColumn(2).getText());  // convert to string
    EXPECT_EQ(4,                query.getColumn(2).getBytes()); // size of the string "-123"
    EXPECT_EQ(SQLite::FLOAT,    query.getColumn(3).getType());
    EXPECT_EQ(false,            query.getColumn(3).isInteger());
    EXPECT_EQ(true,             query.getColumn(3).isFloat());
    EXPECT_EQ(false,            query.getColumn(3).isText());
    EXPECT_EQ(false,            query.getColumn(3).isBlob());
    EXPECT_EQ(false,            query.getColumn(3).isNull());
    EXPECT_STREQ("0.123",       query.getColumn(3).getText());  // convert to string
    EXPECT_EQ(5,                query.getColumn(3).getBytes()); // size of the string "0.123"
    EXPECT_EQ(SQLite::BLOB,     query.getColumn(4).getType());
    EXPECT_EQ(false,            query.getColumn(4).isInteger());
    EXPECT_EQ(false,            query.getColumn(4).isFloat());
    EXPECT_EQ(false,            query.getColumn(4).isText());
    EXPECT_EQ(true,             query.getColumn(4).isBlob());
    EXPECT_EQ(false,            query.getColumn(4).isNull());
    EXPECT_STREQ("bl\0b",       query.getColumn(4).getText());  // convert to string
    EXPECT_EQ(4,                query.getColumn(4).getBytes()); // size of the blob "bl\0b" with the null char
    EXPECT_EQ(SQLite::Null,     query.getColumn(5).getType());
    EXPECT_EQ(false,            query.getColumn(5).isInteger());
    EXPECT_EQ(false,            query.getColumn(5).isFloat());
    EXPECT_EQ(false,            query.getColumn(5).isText());
    EXPECT_EQ(false,            query.getColumn(5).isBlob());
    EXPECT_EQ(true,             query.getColumn(5).isNull());
    EXPECT_STREQ("",            query.getColumn(5).getText());  // convert to string
    EXPECT_EQ(0,                query.getColumn(5).getBytes()); // size of the string "" without the null terminator

    // Use intermediate Column objects (this is not the recommended way to use the API)
    {
        const SQLite::Column id = query.getColumn(0);
        EXPECT_EQ(1, id.getInt64());
        const SQLite::Column msg = query.getColumn(1);
        EXPECT_EQ("first", msg.getString());
        const SQLite::Column integer = query.getColumn(2);
        EXPECT_EQ(-123, integer.getInt());
        const SQLite::Column dbl = query.getColumn(3);
        EXPECT_EQ(0.123, dbl.getDouble());
    }
}

TEST(Column, getName) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(0, db.exec("CREATE TABLE test (id INTEGER PRIMARY KEY, msg TEXT)"));
    EXPECT_EQ(1, db.exec("INSERT INTO test VALUES (NULL, \"first\")"));

    // Compile a SQL query, using the "id" column name as-is, but aliasing the "msg" column with new name "value"
    SQLite::Statement   query(db, "SELECT id, msg as value FROM test");
    query.executeStep();

    // Show how to get the aliased names of the result columns.
    const std::string name0 = query.getColumn(0).getName();
    const std::string name1 = query.getColumn(1).getName();
    EXPECT_EQ("id",     name0);
    EXPECT_EQ("value",  name1);

#ifdef SQLITE_ENABLE_COLUMN_METADATA
    // Show how to get origin names of the table columns from which theses result columns come from.
    // Requires the SQLITE_ENABLE_COLUMN_METADATA preprocessor macro to be
    // also defined at compile times of the SQLite library itself.
    const std::string oname0 = query.getColumn(0).getOriginName();
    const std::string oname1 = query.getColumn(1).getOriginName();
    EXPECT_EQ("id",     oname0);
    EXPECT_EQ("msg",    oname1);
#endif
}

TEST(Column, stream) {
    // Create a new database
    SQLite::Database db(":memory:", SQLite::OPEN_READWRITE|SQLite::OPEN_CREATE);
    EXPECT_EQ(0, db.exec("CREATE TABLE test (msg TEXT)"));
    SQLite::Statement insert(db, "INSERT INTO test VALUES (?)");

    // content to test
    const char str_[] = "stringwith\0embedded";
    std::string str(str_, sizeof(str_)-1);

    insert.bind(1, str);
    // Execute the one-step query to insert the row
    EXPECT_EQ(1, insert.exec());
    EXPECT_EQ(1, db.getTotalChanges());

    SQLite::Statement query(db, "SELECT * FROM test");
    query.executeStep();
    std::stringstream ss;
    ss << query.getColumn(0);
    std::string content = ss.str();
    EXPECT_EQ(content, str);
}
<|endoftext|>