<commit_before>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "absl/container/flat_hash_map.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/op_requires.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/types.h"

namespace tensorflow {

template <class T>
using BatchedMap = std::vector<absl::flat_hash_map<int64_t, T>>;

namespace {
// TODO(momernick): Extend this function to work with outputs of rank > 2.
template <class T>
Status OutputSparse(const BatchedMap<T>& per_batch_counts, int num_values,
                    bool is_1d, OpKernelContext* context) {
  int total_values = 0;
  int num_batches = per_batch_counts.size();
  for (const auto& per_batch_count : per_batch_counts) {
    total_values += per_batch_count.size();
  }

  Tensor* indices;
  int inner_dim = is_1d ? 1 : 2;
  TF_RETURN_IF_ERROR(context->allocate_output(
      0, TensorShape({total_values, inner_dim}), &indices));

  Tensor* values;
  TF_RETURN_IF_ERROR(
      context->allocate_output(1, TensorShape({total_values}), &values));

  auto output_indices = indices->matrix<int64_t>();
  auto output_values = values->flat<T>();
  int64_t value_loc = 0;
  for (int b = 0; b < num_batches; ++b) {
    const auto& per_batch_count = per_batch_counts[b];
    std::vector<std::pair<int, T>> pairs(per_batch_count.begin(),
                                         per_batch_count.end());
    std::sort(pairs.begin(), pairs.end());
    for (const auto& x : pairs) {
      if (is_1d) {
        output_indices(value_loc, 0) = x.first;
      } else {
        output_indices(value_loc, 0) = b;
        output_indices(value_loc, 1) = x.first;
      }
      output_values(value_loc) = x.second;
      ++value_loc;
    }
  }
  Tensor* dense_shape;
  if (is_1d) {
    TF_RETURN_IF_ERROR(
        context->allocate_output(2, TensorShape({1}), &dense_shape));
    dense_shape->flat<int64_t>().data()[0] = num_values;
  } else {
    TF_RETURN_IF_ERROR(
        context->allocate_output(2, TensorShape({2}), &dense_shape));
    dense_shape->flat<int64_t>().data()[0] = num_batches;
    dense_shape->flat<int64_t>().data()[1] = num_values;
  }

  return Status::OK();
}

int GetOutputSize(int max_seen, int max_length, int min_length) {
  return max_length > 0 ? max_length : std::max((max_seen + 1), min_length);
}

}  // namespace

template <class T, class W>
class DenseCount : public OpKernel {
 public:
  explicit DenseCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& data = context->input(0);
    const Tensor& weights = context->input(1);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context,
                TensorShapeUtils::IsVector(data.shape()) ||
                    TensorShapeUtils::IsMatrix(data.shape()),
                errors::InvalidArgument(
                    "Input must be a 1 or 2-dimensional tensor. Got: ",
                    data.shape().DebugString()));

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == data.shape(),
          errors::InvalidArgument(
              "Weights and data must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; data shape: ", data.shape().DebugString()));
    }

    bool is_1d = TensorShapeUtils::IsVector(data.shape());
    int negative_valued_axis = -1;
    int num_batch_dimensions = (data.shape().dims() + negative_valued_axis);

    int num_batch_elements = 1;
    for (int i = 0; i < num_batch_dimensions; ++i) {
      OP_REQUIRES(context, data.shape().dim_size(i) != 0,
                  errors::InvalidArgument(
                      "Invalid input: Shapes dimension cannot be 0."));
      num_batch_elements *= data.shape().dim_size(i);
    }
    int num_value_elements = data.shape().num_elements() / num_batch_elements;
    auto per_batch_counts = BatchedMap<W>(num_batch_elements);

    T max_value = 0;

    const auto data_values = data.flat<T>();
    const auto weight_values = weights.flat<W>();
    int i = 0;
    for (int b = 0; b < num_batch_elements; ++b) {
      for (int v = 0; v < num_value_elements; ++v) {
        const auto& value = data_values(i);
        if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
          if (binary_output_) {
            per_batch_counts[b][value] = 1;
          } else if (use_weights) {
            per_batch_counts[b][value] += weight_values(i);
          } else {
            per_batch_counts[b][value]++;
          }
          if (value > max_value) {
            max_value = value;
          }
        }
        ++i;
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
};

template <class T, class W>
class SparseCount : public OpKernel {
 public:
  explicit SparseCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& indices = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& shape = context->input(2);
    const Tensor& weights = context->input(3);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(indices.shape()),
                errors::InvalidArgument(
                    "Input indices must be a 2-dimensional tensor. Got: ",
                    indices.shape().DebugString()));
    OP_REQUIRES(context, TensorShapeUtils::IsVector(values.shape()),
                errors::InvalidArgument("Input values must be a vector. Got: ",
                                        values.shape().DebugString()));
    OP_REQUIRES(context, TensorShapeUtils::IsVector(shape.shape()),
                errors::InvalidArgument("Input shape must be a vector. Got: ",
                                        shape.shape().DebugString()));
    OP_REQUIRES(context,
                values.shape().dim_size(0) == indices.shape().dim_size(0),
                errors::InvalidArgument(
                    "Number of values must match first dimension of indices.",
                    "Got ", values.shape().dim_size(0),
                    " values, indices shape: ", indices.shape().DebugString()));
    OP_REQUIRES(
        context, shape.shape().dim_size(0) == indices.shape().dim_size(1),
        errors::InvalidArgument(
            "Number of dimensions must match second dimension of indices.",
            "Got ", shape.shape().dim_size(0),
            " dimensions, indices shape: ", indices.shape().DebugString()));
    OP_REQUIRES(context, shape.NumElements() > 0,
                errors::InvalidArgument(
                    "The shape argument requires at least one element."));
    // Validate indices: each index must be valid for the corresponding
    // dimension. This could be possibly done better.
    const auto indices_values = indices.matrix<int64_t>();
    const auto shape_vector = shape.vec<int64_t>();
    int num_values = values.NumElements();  // same as first dim of indices
    int rank = indices.shape().dim_size(1);
    for (int i = 0; i < num_values; ++i) {
      for (int j = 0; j < rank; ++j) {
        OP_REQUIRES(
            context,
            indices_values(i, j) >= 0 && indices_values(i, j) < shape_vector(j),
            errors::InvalidArgument(
                "Invalid index value at ", i, ": dimension ", j, " has value ",
                indices_values(i, j), " which is not in [0, ", shape_vector(j),
                ") (as given by dense shape ", shape.DebugString()));
      }
    }

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    bool is_1d = shape.NumElements() == 1;
    int num_batches = is_1d ? 1 : shape_vector(0);

    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();

    auto per_batch_counts = BatchedMap<W>(num_batches);

    T max_value = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      int batch = is_1d ? 0 : indices_values(idx, 0);
      if (batch >= num_batches) {
        OP_REQUIRES(context, batch < num_batches,
                    errors::InvalidArgument(
                        "Indices value along the first dimension must be ",
                        "lower than the first index of the shape.", "Got ",
                        batch, " as batch and ", num_batches,
                        " as the first dimension of the shape."));
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch][value] += weight_values(idx);
        } else {
          per_batch_counts[batch][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
  bool validate_;
};

template <class T, class W>
class RaggedCount : public OpKernel {
 public:
  explicit RaggedCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& splits = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& weights = context->input(2);
    bool use_weights = weights.NumElements() > 0;
    bool is_1d = false;

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    const auto splits_values = splits.flat<int64_t>();
    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();
    int num_batches = splits.NumElements() - 1;
    int num_values = values.NumElements();

    OP_REQUIRES(
        context, num_batches > 0,
        errors::InvalidArgument(
            "Must provide at least 2 elements for the splits argument"));
    OP_REQUIRES(context, splits_values(0) == 0,
                errors::InvalidArgument("Splits must start with 0, not with ",
                                        splits_values(0)));
    OP_REQUIRES(context, splits_values(num_batches) == num_values,
                errors::InvalidArgument(
                    "Splits must end with the number of values, got ",
                    splits_values(num_batches), " instead of ", num_values));

    auto per_batch_counts = BatchedMap<W>(num_batches);
    T max_value = 0;
    int batch_idx = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      while (idx >= splits_values(batch_idx)) {
        batch_idx++;
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch_idx - 1][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch_idx - 1][value] += weight_values(idx);
        } else {
          per_batch_counts[batch_idx - 1][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
  bool validate_;
};

#define REGISTER_W(W_TYPE) \
  REGISTER(int32, W_TYPE)  \
  REGISTER(int64_t, W_TYPE)

#define REGISTER(I_TYPE, W_TYPE)                                     \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("DenseCountSparseOutput")             \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          DenseCount<I_TYPE, W_TYPE>)                \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("SparseCountSparseOutput")            \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          SparseCount<I_TYPE, W_TYPE>)               \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("RaggedCountSparseOutput")            \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          RaggedCount<I_TYPE, W_TYPE>)

TF_CALL_INTEGRAL_TYPES(REGISTER_W);
TF_CALL_float(REGISTER_W);
TF_CALL_double(REGISTER_W);

#undef REGISTER_W
#undef REGISTER

}  // namespace tensorflow
<commit_msg>Prevent crash due to integer overflow followed by allocating negative sized array.<commit_after>/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <limits>

#include "absl/container/flat_hash_map.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/op_requires.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/types.h"

namespace tensorflow {

// Don't allocate too large `BatchedMap<T>` objects
static int kMaxBatches = std::numeric_limits<int>::max();

template <class T>
using BatchedMap = std::vector<absl::flat_hash_map<int64_t, T>>;

namespace {
// TODO(momernick): Extend this function to work with outputs of rank > 2.
template <class T>
Status OutputSparse(const BatchedMap<T>& per_batch_counts, int num_values,
                    bool is_1d, OpKernelContext* context) {
  int total_values = 0;
  int num_batches = per_batch_counts.size();
  for (const auto& per_batch_count : per_batch_counts) {
    total_values += per_batch_count.size();
  }

  Tensor* indices;
  int inner_dim = is_1d ? 1 : 2;
  TF_RETURN_IF_ERROR(context->allocate_output(
      0, TensorShape({total_values, inner_dim}), &indices));

  Tensor* values;
  TF_RETURN_IF_ERROR(
      context->allocate_output(1, TensorShape({total_values}), &values));

  auto output_indices = indices->matrix<int64_t>();
  auto output_values = values->flat<T>();
  int64_t value_loc = 0;
  for (int b = 0; b < num_batches; ++b) {
    const auto& per_batch_count = per_batch_counts[b];
    std::vector<std::pair<int, T>> pairs(per_batch_count.begin(),
                                         per_batch_count.end());
    std::sort(pairs.begin(), pairs.end());
    for (const auto& x : pairs) {
      if (is_1d) {
        output_indices(value_loc, 0) = x.first;
      } else {
        output_indices(value_loc, 0) = b;
        output_indices(value_loc, 1) = x.first;
      }
      output_values(value_loc) = x.second;
      ++value_loc;
    }
  }
  Tensor* dense_shape;
  if (is_1d) {
    TF_RETURN_IF_ERROR(
        context->allocate_output(2, TensorShape({1}), &dense_shape));
    dense_shape->flat<int64_t>().data()[0] = num_values;
  } else {
    TF_RETURN_IF_ERROR(
        context->allocate_output(2, TensorShape({2}), &dense_shape));
    dense_shape->flat<int64_t>().data()[0] = num_batches;
    dense_shape->flat<int64_t>().data()[1] = num_values;
  }

  return Status::OK();
}

int GetOutputSize(int max_seen, int max_length, int min_length) {
  return max_length > 0 ? max_length : std::max((max_seen + 1), min_length);
}

}  // namespace

template <class T, class W>
class DenseCount : public OpKernel {
 public:
  explicit DenseCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& data = context->input(0);
    const Tensor& weights = context->input(1);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context,
                TensorShapeUtils::IsVector(data.shape()) ||
                    TensorShapeUtils::IsMatrix(data.shape()),
                errors::InvalidArgument(
                    "Input must be a 1 or 2-dimensional tensor. Got: ",
                    data.shape().DebugString()));

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == data.shape(),
          errors::InvalidArgument(
              "Weights and data must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; data shape: ", data.shape().DebugString()));
    }

    bool is_1d = TensorShapeUtils::IsVector(data.shape());
    int negative_valued_axis = -1;
    int num_batch_dimensions = (data.shape().dims() + negative_valued_axis);

    int num_batch_elements = 1;
    for (int i = 0; i < num_batch_dimensions; ++i) {
      OP_REQUIRES(context, data.shape().dim_size(i) != 0,
                  errors::InvalidArgument(
                      "Invalid input: Shapes dimension cannot be 0."));
      num_batch_elements *= data.shape().dim_size(i);
    }
    int num_value_elements = data.shape().num_elements() / num_batch_elements;
    auto per_batch_counts = BatchedMap<W>(num_batch_elements);

    T max_value = 0;

    const auto data_values = data.flat<T>();
    const auto weight_values = weights.flat<W>();
    int i = 0;
    for (int b = 0; b < num_batch_elements; ++b) {
      for (int v = 0; v < num_value_elements; ++v) {
        const auto& value = data_values(i);
        if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
          if (binary_output_) {
            per_batch_counts[b][value] = 1;
          } else if (use_weights) {
            per_batch_counts[b][value] += weight_values(i);
          } else {
            per_batch_counts[b][value]++;
          }
          if (value > max_value) {
            max_value = value;
          }
        }
        ++i;
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
};

template <class T, class W>
class SparseCount : public OpKernel {
 public:
  explicit SparseCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& indices = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& shape = context->input(2);
    const Tensor& weights = context->input(3);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(indices.shape()),
                errors::InvalidArgument(
                    "Input indices must be a 2-dimensional tensor. Got: ",
                    indices.shape().DebugString()));
    OP_REQUIRES(context, TensorShapeUtils::IsVector(values.shape()),
                errors::InvalidArgument("Input values must be a vector. Got: ",
                                        values.shape().DebugString()));
    OP_REQUIRES(context, TensorShapeUtils::IsVector(shape.shape()),
                errors::InvalidArgument("Input shape must be a vector. Got: ",
                                        shape.shape().DebugString()));
    OP_REQUIRES(context,
                values.shape().dim_size(0) == indices.shape().dim_size(0),
                errors::InvalidArgument(
                    "Number of values must match first dimension of indices.",
                    "Got ", values.shape().dim_size(0),
                    " values, indices shape: ", indices.shape().DebugString()));
    OP_REQUIRES(
        context, shape.shape().dim_size(0) == indices.shape().dim_size(1),
        errors::InvalidArgument(
            "Number of dimensions must match second dimension of indices.",
            "Got ", shape.shape().dim_size(0),
            " dimensions, indices shape: ", indices.shape().DebugString()));
    OP_REQUIRES(context, shape.NumElements() > 0,
                errors::InvalidArgument(
                    "The shape argument requires at least one element."));
    // Validate indices: each index must be valid for the corresponding
    // dimension. This could be possibly done better.
    const auto indices_values = indices.matrix<int64_t>();
    const auto shape_vector = shape.vec<int64_t>();
    int num_values = values.NumElements();  // same as first dim of indices
    int rank = indices.shape().dim_size(1);
    for (int i = 0; i < num_values; ++i) {
      for (int j = 0; j < rank; ++j) {
        OP_REQUIRES(
            context,
            indices_values(i, j) >= 0 && indices_values(i, j) < shape_vector(j),
            errors::InvalidArgument(
                "Invalid index value at ", i, ": dimension ", j, " has value ",
                indices_values(i, j), " which is not in [0, ", shape_vector(j),
                ") (as given by dense shape ", shape.DebugString()));
      }
    }

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    bool is_1d = shape.NumElements() == 1;
    int num_batches = is_1d ? 1 : shape_vector(0);
    OP_REQUIRES(
        context, 0 < num_batches && num_batches < kMaxBatches,
        errors::InvalidArgument("Cannot allocate ", num_batches,
                                " batches, is the dense shape too wide?"));

    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();

    auto per_batch_counts = BatchedMap<W>(num_batches);

    T max_value = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      int batch = is_1d ? 0 : indices_values(idx, 0);
      if (batch >= num_batches) {
        OP_REQUIRES(context, batch < num_batches,
                    errors::InvalidArgument(
                        "Indices value along the first dimension must be ",
                        "lower than the first index of the shape.", "Got ",
                        batch, " as batch and ", num_batches,
                        " as the first dimension of the shape."));
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch][value] += weight_values(idx);
        } else {
          per_batch_counts[batch][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
  bool validate_;
};

template <class T, class W>
class RaggedCount : public OpKernel {
 public:
  explicit RaggedCount(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("minlength", &minlength_));
    OP_REQUIRES_OK(context, context->GetAttr("maxlength", &maxlength_));
    OP_REQUIRES_OK(context, context->GetAttr("binary_output", &binary_output_));
  }

  void Compute(OpKernelContext* context) override {
    const Tensor& splits = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& weights = context->input(2);
    bool use_weights = weights.NumElements() > 0;
    bool is_1d = false;

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    const auto splits_values = splits.flat<int64_t>();
    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();
    int num_batches = splits.NumElements() - 1;
    int num_values = values.NumElements();

    OP_REQUIRES(
        context, num_batches > 0,
        errors::InvalidArgument(
            "Must provide at least 2 elements for the splits argument"));
    OP_REQUIRES(context, splits_values(0) == 0,
                errors::InvalidArgument("Splits must start with 0, not with ",
                                        splits_values(0)));
    OP_REQUIRES(context, splits_values(num_batches) == num_values,
                errors::InvalidArgument(
                    "Splits must end with the number of values, got ",
                    splits_values(num_batches), " instead of ", num_values));

    auto per_batch_counts = BatchedMap<W>(num_batches);
    T max_value = 0;
    int batch_idx = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      while (idx >= splits_values(batch_idx)) {
        batch_idx++;
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch_idx - 1][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch_idx - 1][value] += weight_values(idx);
        } else {
          per_batch_counts[batch_idx - 1][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

 private:
  int maxlength_;
  int minlength_;
  bool binary_output_;
  bool validate_;
};

#define REGISTER_W(W_TYPE) \
  REGISTER(int32, W_TYPE)  \
  REGISTER(int64_t, W_TYPE)

#define REGISTER(I_TYPE, W_TYPE)                                     \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("DenseCountSparseOutput")             \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          DenseCount<I_TYPE, W_TYPE>)                \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("SparseCountSparseOutput")            \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          SparseCount<I_TYPE, W_TYPE>)               \
                                                                     \
  REGISTER_KERNEL_BUILDER(Name("RaggedCountSparseOutput")            \
                              .TypeConstraint<I_TYPE>("T")           \
                              .TypeConstraint<W_TYPE>("output_type") \
                              .Device(DEVICE_CPU),                   \
                          RaggedCount<I_TYPE, W_TYPE>)

TF_CALL_INTEGRAL_TYPES(REGISTER_W);
TF_CALL_float(REGISTER_W);
TF_CALL_double(REGISTER_W);

#undef REGISTER_W
#undef REGISTER

}  // namespace tensorflow
<|endoftext|>


<commit_before>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <cstddef>
#include <cstdlib>
#include <string>

#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/bounds_check.h"
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/string_util.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/bcast.h"

namespace tensorflow {

// Position/length can be 32 or 64-bit integers
template <typename T>
class SubstrOp : public OpKernel {
 public:
  explicit SubstrOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
    string unit;
    OP_REQUIRES_OK(ctx, ctx->GetAttr("unit", &unit));
    OP_REQUIRES_OK(ctx, ParseCharUnit(unit, &unit_));
  }

  void Compute(OpKernelContext* context) override {
    // Get inputs
    const Tensor& input_tensor = context->input(0);
    const Tensor& pos_tensor = context->input(1);
    const Tensor& len_tensor = context->input(2);
    const TensorShape& input_shape = input_tensor.shape();
    const TensorShape& pos_shape = pos_tensor.shape();
    const TensorShape& len_shape = len_tensor.shape();
    OP_REQUIRES(context, (pos_shape == len_shape),
                errors::InvalidArgument(
                    "pos and len should have the same shape, got: ",
                    pos_shape.DebugString(), " vs. ", len_shape.DebugString()));

    bool is_scalar = TensorShapeUtils::IsScalar(pos_shape);

    if (is_scalar || input_shape == pos_shape) {
      // pos/len are either scalar or match the shape of input_tensor
      // Do not need to do broadcasting

      // Reshape input
      auto input = input_tensor.flat<tstring>();
      // Allocate output
      Tensor* output_tensor = nullptr;
      OP_REQUIRES_OK(context,
                     context->allocate_output("output", input_tensor.shape(),
                                              &output_tensor));
      auto output = output_tensor->flat<tstring>();
      if (is_scalar) {
        // Perform Op with scalar pos/len
        const T pos =
            tensorflow::internal::SubtleMustCopy(pos_tensor.scalar<T>()());
        const T len =
            tensorflow::internal::SubtleMustCopy(len_tensor.scalar<T>()());
        for (size_t i = 0; i < input_tensor.NumElements(); ++i) {
          StringPiece in(input(i));
          T byte_pos = pos;
          T byte_len = len;
          switch (unit_) {
            case CharUnit::UTF8_CHAR:
              OP_REQUIRES(
                  context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string at index ", i));
              break;
            case CharUnit::BYTE:
              byte_pos = AdjustedPosIndex(byte_pos, in);
              OP_REQUIRES(
                  context, FastBoundsCheck(byte_pos, in.size() + 1),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string b'", in, "' at index ", i));
          }
          StringPiece sub_in = in.substr(byte_pos, byte_len);
          output(i).assign(sub_in.data(), sub_in.size());
        }
      } else {
        // Perform Op element-wise with tensor pos/len
        auto pos_flat = pos_tensor.flat<T>();
        auto len_flat = len_tensor.flat<T>();
        for (size_t i = 0; i < input_tensor.NumElements(); ++i) {
          StringPiece in(input(i));
          const T pos = tensorflow::internal::SubtleMustCopy(pos_flat(i));
          const T len = tensorflow::internal::SubtleMustCopy(len_flat(i));
          T byte_pos = pos;
          T byte_len = len;
          switch (unit_) {
            case CharUnit::UTF8_CHAR:
              OP_REQUIRES(
                  context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string at index ", i));
              break;
            case CharUnit::BYTE:
              byte_pos = AdjustedPosIndex(byte_pos, in);
              OP_REQUIRES(
                  context, FastBoundsCheck(byte_pos, in.size() + 1),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string b'", in, "' at index ", i));
          }
          StringPiece sub_in = in.substr(byte_pos, byte_len);
          output(i).assign(sub_in.data(), sub_in.size());
        }
      }
    } else {
      // Perform op with broadcasting
      // TODO: Use ternary broadcasting for once available in Eigen. Current
      //       implementation iterates through broadcasted ops element-wise;
      //       this should be parallelized.

      // Create BCast helper with shape of input and pos/len
      BCast bcast(BCast::FromShape(input_shape), BCast::FromShape(pos_shape));
      OP_REQUIRES(context, bcast.IsValid(),
                  errors::InvalidArgument(
                      "Incompatible shapes: ", input_shape.DebugString(),
                      " vs. ", pos_shape.DebugString()));
      TensorShape output_shape = BCast::ToShape(bcast.result_shape());
      int ndims = output_shape.dims();
      Tensor* output_tensor = nullptr;
      OP_REQUIRES_OK(context, context->allocate_output("output", output_shape,
                                                       &output_tensor));
      switch (ndims) {
        case 1: {
          // Reshape tensors according to BCast results
          auto input = input_tensor.shaped<tstring, 1>(bcast.x_reshape());
          auto output = output_tensor->shaped<tstring, 1>(bcast.result_shape());
          auto pos_shaped = pos_tensor.shaped<T, 1>(bcast.y_reshape());
          auto len_shaped = len_tensor.shaped<T, 1>(bcast.y_reshape());

          // Allocate temporary buffer for broadcasted input tensor
          Tensor input_buffer;
          OP_REQUIRES_OK(context, context->allocate_temp(
                                      DT_STRING, output_shape, &input_buffer));
          TTypes<tstring, 1>::Tensor input_bcast =
              input_buffer.shaped<tstring, 1>(bcast.result_shape());
          input_bcast =
              input.broadcast(BCast::ToIndexArray<1>(bcast.x_bcast()));

          // Allocate temporary buffer for broadcasted position tensor
          Tensor pos_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &pos_buffer));
          typename TTypes<T, 1>::Tensor pos_bcast(
              pos_buffer.shaped<T, 1>(bcast.result_shape()));
          pos_bcast =
              pos_shaped.broadcast(BCast::ToIndexArray<1>(bcast.y_bcast()));

          // Allocate temporary buffer for broadcasted length tensor
          Tensor len_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &len_buffer));
          typename TTypes<T, 1>::Tensor len_bcast(
              len_buffer.shaped<T, 1>(bcast.result_shape()));
          len_bcast =
              len_shaped.broadcast(BCast::ToIndexArray<1>(bcast.y_bcast()));

          // Iterate through broadcasted tensors and perform substr
          for (int i = 0; i < output_shape.dim_size(0); ++i) {
            StringPiece in(input_bcast(i));
            const T pos = tensorflow::internal::SubtleMustCopy(pos_bcast(i));
            const T len = tensorflow::internal::SubtleMustCopy(len_bcast(i));
            T byte_pos = pos;
            T byte_len = len;
            switch (unit_) {
              case CharUnit::UTF8_CHAR:
                OP_REQUIRES(
                    context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                    errors::InvalidArgument("pos ", pos, " out of range for ",
                                            "string at index ", i));
                break;
              case CharUnit::BYTE:
                byte_pos = AdjustedPosIndex(byte_pos, in);
                OP_REQUIRES(
                    context,
                    FastBoundsCheck(byte_pos, input_bcast(i).size() + 1),
                    errors::InvalidArgument("pos ", pos, " out of range for ",
                                            "string b'", in, "' at index ", i));
            }
            StringPiece sub_in = in.substr(byte_pos, byte_len);
            output(i).assign(sub_in.data(), sub_in.size());
          }
          break;
        }
        case 2: {
          // Reshape tensors according to BCast results
          auto input = input_tensor.shaped<tstring, 2>(bcast.x_reshape());
          auto output = output_tensor->shaped<tstring, 2>(bcast.result_shape());
          auto pos_shaped = pos_tensor.shaped<T, 2>(bcast.y_reshape());
          auto len_shaped = len_tensor.shaped<T, 2>(bcast.y_reshape());

          // Allocate temporary buffer for broadcasted input tensor
          Tensor input_buffer;
          OP_REQUIRES_OK(context, context->allocate_temp(
                                      DT_STRING, output_shape, &input_buffer));
          TTypes<tstring, 2>::Tensor input_bcast =
              input_buffer.shaped<tstring, 2>(bcast.result_shape());
          input_bcast =
              input.broadcast(BCast::ToIndexArray<2>(bcast.x_bcast()));

          // Allocate temporary buffer for broadcasted position tensor
          Tensor pos_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &pos_buffer));
          typename TTypes<T, 2>::Tensor pos_bcast(
              pos_buffer.shaped<T, 2>(bcast.result_shape()));
          pos_bcast =
              pos_shaped.broadcast(BCast::ToIndexArray<2>(bcast.y_bcast()));

          // Allocate temporary buffer for broadcasted length tensor
          Tensor len_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &len_buffer));
          typename TTypes<T, 2>::Tensor len_bcast(
              len_buffer.shaped<T, 2>(bcast.result_shape()));
          len_bcast =
              len_shaped.broadcast(BCast::ToIndexArray<2>(bcast.y_bcast()));

          // Iterate through broadcasted tensors and perform substr
          for (int i = 0; i < output_shape.dim_size(0); ++i) {
            for (int j = 0; j < output_shape.dim_size(1); ++j) {
              StringPiece in(input_bcast(i, j));
              const T pos =
                  tensorflow::internal::SubtleMustCopy(pos_bcast(i, j));
              const T len =
                  tensorflow::internal::SubtleMustCopy(len_bcast(i, j));
              T byte_pos = pos;
              T byte_len = len;
              switch (unit_) {
                case CharUnit::UTF8_CHAR:
                  OP_REQUIRES(
                      context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                      errors::InvalidArgument("pos ", pos, " out of range for ",
                                              "string at index ", i));
                  break;
                case CharUnit::BYTE:
                  byte_pos = AdjustedPosIndex(byte_pos, in);
                  OP_REQUIRES(
                      context, FastBoundsCheck(byte_pos, in.size() + 1),
                      errors::InvalidArgument("pos ", pos, " out of range for ",
                                              "string b'", in, "' at index (",
                                              i, ", ", j, ")"));
              }
              StringPiece sub_in = in.substr(byte_pos, byte_len);
              output(i, j).assign(sub_in.data(), sub_in.size());
            }
          }
          break;
        }
        default: {
          context->SetStatus(errors::Unimplemented(
              "Substr broadcast not implemented for ", ndims, " dimensions"));
        }
      }
    }
  }

 private:
  // This adjusts the requested position. Note it does not perform any bound
  // checks.
  static inline T AdjustedPosIndex(const T pos_requested, const StringPiece s) {
    if (pos_requested < 0) {
      return s.size() + pos_requested;
    }
    return pos_requested;
  }

  // Return true if successful; otherwise, return false if the `pos` argument
  // is out of range in the string.
  static inline bool UpdatePosAndLenForUtf8(const StringPiece in, T* pos,
                                            T* len) {
    if (*pos >= 0) {
      return UpdatePositivePosAndLenForUtf8(in, *pos, *len, pos, len);
    } else {
      return UpdateNegativePosAndLenForUtf8(in, *pos, *len, pos, len);
    }
  }

  static bool UpdatePositivePosAndLenForUtf8(const StringPiece in, const T pos,
                                             const T len, T* char_pos,
                                             T* char_len) {
    *char_pos = 0;
    // Determine byte position of the substring start.
    if (!ForwardNUTF8CharPositions(in, pos, char_pos)) {
      return false;
    }
    // Determine position of the end of the substring.
    // The length will be capped at the end of the string, and we ignore whether
    // the string had enough characters to handle it or not.
    *char_len = *char_pos;
    ForwardNUTF8CharPositions(in, len, char_len);
    // The length in bytes is the position end of the substring less the start.
    *char_len = *char_len - *char_pos;
    return true;
  }

  // This function expects a negative position relative to the end of the
  // string, but will update the character position to a positive number
  // relative to the beginning of the string.
  static bool UpdateNegativePosAndLenForUtf8(const StringPiece in, const T pos,
                                             const T len, T* char_pos,
                                             T* char_len) {
    // Initially treat the length as position of the end of the substring.
    *char_len = in.size();
    // This is the number of character to skip from the end of the string to
    // arrive at the position where the substring should end.
    T utf8_chars_to_skip = -pos - len;
    if (utf8_chars_to_skip < 0) {
      utf8_chars_to_skip = 0;
    }
    // Find the byte position where the substring should end using the computed
    // number of characters to skip.
    if (!BackNUTF8CharPositions(in, utf8_chars_to_skip, char_len)) {
      return false;
    }
    // Next, determine where the substring should begin. The number of chars to
    // skip is the requested position minus the chars we've previously skipped.
    *char_pos = *char_len;
    if (!BackNUTF8CharPositions(in, -pos - utf8_chars_to_skip, char_pos)) {
      return false;
    }
    // The length in bytes is the position end of the substring less the start.
    *char_len = *char_len - *char_pos;
    return true;
  }

  CharUnit unit_ = CharUnit::BYTE;
};

#define REGISTER_SUBSTR(type)                                      \
  REGISTER_KERNEL_BUILDER(                                         \
      Name("Substr").Device(DEVICE_CPU).TypeConstraint<type>("T"), \
      SubstrOp<type>);
REGISTER_SUBSTR(int32);
REGISTER_SUBSTR(int64);
}  // namespace tensorflow
<commit_msg>Avoid string copies during Substr broadcasting.<commit_after>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <cstddef>
#include <cstdlib>
#include <string>

#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/bounds_check.h"
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/string_util.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/bcast.h"

namespace tensorflow {

// Position/length can be 32 or 64-bit integers
template <typename T>
class SubstrOp : public OpKernel {
 public:
  explicit SubstrOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
    string unit;
    OP_REQUIRES_OK(ctx, ctx->GetAttr("unit", &unit));
    OP_REQUIRES_OK(ctx, ParseCharUnit(unit, &unit_));
  }

  void Compute(OpKernelContext* context) override {
    // Get inputs
    const Tensor& input_tensor = context->input(0);
    const Tensor& pos_tensor = context->input(1);
    const Tensor& len_tensor = context->input(2);
    const TensorShape& input_shape = input_tensor.shape();
    const TensorShape& pos_shape = pos_tensor.shape();
    const TensorShape& len_shape = len_tensor.shape();
    OP_REQUIRES(context, (pos_shape == len_shape),
                errors::InvalidArgument(
                    "pos and len should have the same shape, got: ",
                    pos_shape.DebugString(), " vs. ", len_shape.DebugString()));

    bool is_scalar = TensorShapeUtils::IsScalar(pos_shape);

    if (is_scalar || input_shape == pos_shape) {
      // pos/len are either scalar or match the shape of input_tensor
      // Do not need to do broadcasting

      // Reshape input
      auto input = input_tensor.flat<tstring>();
      // Allocate output
      Tensor* output_tensor = nullptr;
      OP_REQUIRES_OK(context,
                     context->allocate_output("output", input_tensor.shape(),
                                              &output_tensor));
      auto output = output_tensor->flat<tstring>();
      if (is_scalar) {
        // Perform Op with scalar pos/len
        const T pos =
            tensorflow::internal::SubtleMustCopy(pos_tensor.scalar<T>()());
        const T len =
            tensorflow::internal::SubtleMustCopy(len_tensor.scalar<T>()());
        for (size_t i = 0; i < input_tensor.NumElements(); ++i) {
          StringPiece in(input(i));
          T byte_pos = pos;
          T byte_len = len;
          switch (unit_) {
            case CharUnit::UTF8_CHAR:
              OP_REQUIRES(
                  context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string at index ", i));
              break;
            case CharUnit::BYTE:
              byte_pos = AdjustedPosIndex(byte_pos, in);
              OP_REQUIRES(
                  context, FastBoundsCheck(byte_pos, in.size() + 1),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string b'", in, "' at index ", i));
          }
          StringPiece sub_in = in.substr(byte_pos, byte_len);
          output(i).assign(sub_in.data(), sub_in.size());
        }
      } else {
        // Perform Op element-wise with tensor pos/len
        auto pos_flat = pos_tensor.flat<T>();
        auto len_flat = len_tensor.flat<T>();
        for (size_t i = 0; i < input_tensor.NumElements(); ++i) {
          StringPiece in(input(i));
          const T pos = tensorflow::internal::SubtleMustCopy(pos_flat(i));
          const T len = tensorflow::internal::SubtleMustCopy(len_flat(i));
          T byte_pos = pos;
          T byte_len = len;
          switch (unit_) {
            case CharUnit::UTF8_CHAR:
              OP_REQUIRES(
                  context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string at index ", i));
              break;
            case CharUnit::BYTE:
              byte_pos = AdjustedPosIndex(byte_pos, in);
              OP_REQUIRES(
                  context, FastBoundsCheck(byte_pos, in.size() + 1),
                  errors::InvalidArgument("pos ", pos, " out of range for ",
                                          "string b'", in, "' at index ", i));
          }
          StringPiece sub_in = in.substr(byte_pos, byte_len);
          output(i).assign(sub_in.data(), sub_in.size());
        }
      }
    } else {
      // Perform op with broadcasting
      // TODO: Use ternary broadcasting for once available in Eigen. Current
      //       implementation iterates through broadcasted ops element-wise;
      //       this should be parallelized.

      // Create BCast helper with shape of input and pos/len
      BCast bcast(BCast::FromShape(input_shape), BCast::FromShape(pos_shape));
      OP_REQUIRES(context, bcast.IsValid(),
                  errors::InvalidArgument(
                      "Incompatible shapes: ", input_shape.DebugString(),
                      " vs. ", pos_shape.DebugString()));
      TensorShape output_shape = BCast::ToShape(bcast.result_shape());
      int ndims = output_shape.dims();
      Tensor* output_tensor = nullptr;
      OP_REQUIRES_OK(context, context->allocate_output("output", output_shape,
                                                       &output_tensor));
      switch (ndims) {
        case 1: {
          // Reshape tensors according to BCast results
          auto input = input_tensor.shaped<tstring, 1>(bcast.x_reshape());
          auto output = output_tensor->shaped<tstring, 1>(bcast.result_shape());
          auto pos_shaped = pos_tensor.shaped<T, 1>(bcast.y_reshape());
          auto len_shaped = len_tensor.shaped<T, 1>(bcast.y_reshape());

          // Allocate temporary buffer for broadcasted position tensor
          Tensor pos_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &pos_buffer));
          typename TTypes<T, 1>::Tensor pos_bcast(
              pos_buffer.shaped<T, 1>(bcast.result_shape()));
          pos_bcast =
              pos_shaped.broadcast(BCast::ToIndexArray<1>(bcast.y_bcast()));

          // Allocate temporary buffer for broadcasted length tensor
          Tensor len_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &len_buffer));
          typename TTypes<T, 1>::Tensor len_bcast(
              len_buffer.shaped<T, 1>(bcast.result_shape()));
          len_bcast =
              len_shaped.broadcast(BCast::ToIndexArray<1>(bcast.y_bcast()));

          // Iterate through broadcasted tensors and perform substr
          for (int i = 0; i < output_shape.dim_size(0); ++i) {
            StringPiece in(input(input.dimension(0) > 1 ? i : 0));
            const T pos = tensorflow::internal::SubtleMustCopy(pos_bcast(i));
            const T len = tensorflow::internal::SubtleMustCopy(len_bcast(i));
            T byte_pos = pos;
            T byte_len = len;
            switch (unit_) {
              case CharUnit::UTF8_CHAR:
                OP_REQUIRES(
                    context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                    errors::InvalidArgument("pos ", pos, " out of range for ",
                                            "string at index ", i));
                break;
              case CharUnit::BYTE:
                byte_pos = AdjustedPosIndex(byte_pos, in);
                OP_REQUIRES(
                    context, FastBoundsCheck(byte_pos, in.size() + 1),
                    errors::InvalidArgument("pos ", pos, " out of range for ",
                                            "string b'", in, "' at index ", i));
            }
            StringPiece sub_in = in.substr(byte_pos, byte_len);
            output(i).assign(sub_in.data(), sub_in.size());
          }
          break;
        }
        case 2: {
          // Reshape tensors according to BCast results
          auto input = input_tensor.shaped<tstring, 2>(bcast.x_reshape());
          auto output = output_tensor->shaped<tstring, 2>(bcast.result_shape());
          auto pos_shaped = pos_tensor.shaped<T, 2>(bcast.y_reshape());
          auto len_shaped = len_tensor.shaped<T, 2>(bcast.y_reshape());

          // Allocate temporary buffer for broadcasted position tensor
          Tensor pos_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &pos_buffer));
          typename TTypes<T, 2>::Tensor pos_bcast(
              pos_buffer.shaped<T, 2>(bcast.result_shape()));
          pos_bcast =
              pos_shaped.broadcast(BCast::ToIndexArray<2>(bcast.y_bcast()));

          // Allocate temporary buffer for broadcasted length tensor
          Tensor len_buffer;
          OP_REQUIRES_OK(context,
                         context->allocate_temp(DataTypeToEnum<T>::v(),
                                                output_shape, &len_buffer));
          typename TTypes<T, 2>::Tensor len_bcast(
              len_buffer.shaped<T, 2>(bcast.result_shape()));
          len_bcast =
              len_shaped.broadcast(BCast::ToIndexArray<2>(bcast.y_bcast()));

          // Iterate through broadcasted tensors and perform substr
          for (int i = 0; i < output_shape.dim_size(0); ++i) {
            for (int j = 0; j < output_shape.dim_size(1); ++j) {
              StringPiece in(input(input.dimension(0) > 1 ? i : 0,
                                   input.dimension(1) > 1 ? j : 0));
              const T pos =
                  tensorflow::internal::SubtleMustCopy(pos_bcast(i, j));
              const T len =
                  tensorflow::internal::SubtleMustCopy(len_bcast(i, j));
              T byte_pos = pos;
              T byte_len = len;
              switch (unit_) {
                case CharUnit::UTF8_CHAR:
                  OP_REQUIRES(
                      context, UpdatePosAndLenForUtf8(in, &byte_pos, &byte_len),
                      errors::InvalidArgument("pos ", pos, " out of range for ",
                                              "string at index ", i));
                  break;
                case CharUnit::BYTE:
                  byte_pos = AdjustedPosIndex(byte_pos, in);
                  OP_REQUIRES(
                      context, FastBoundsCheck(byte_pos, in.size() + 1),
                      errors::InvalidArgument("pos ", pos, " out of range for ",
                                              "string b'", in, "' at index (",
                                              i, ", ", j, ")"));
              }
              StringPiece sub_in = in.substr(byte_pos, byte_len);
              output(i, j).assign(sub_in.data(), sub_in.size());
            }
          }
          break;
        }
        default: {
          context->SetStatus(errors::Unimplemented(
              "Substr broadcast not implemented for ", ndims, " dimensions"));
        }
      }
    }
  }

 private:
  // This adjusts the requested position. Note it does not perform any bound
  // checks.
  static inline T AdjustedPosIndex(const T pos_requested, const StringPiece s) {
    if (pos_requested < 0) {
      return s.size() + pos_requested;
    }
    return pos_requested;
  }

  // Return true if successful; otherwise, return false if the `pos` argument
  // is out of range in the string.
  static inline bool UpdatePosAndLenForUtf8(const StringPiece in, T* pos,
                                            T* len) {
    if (*pos >= 0) {
      return UpdatePositivePosAndLenForUtf8(in, *pos, *len, pos, len);
    } else {
      return UpdateNegativePosAndLenForUtf8(in, *pos, *len, pos, len);
    }
  }

  static bool UpdatePositivePosAndLenForUtf8(const StringPiece in, const T pos,
                                             const T len, T* char_pos,
                                             T* char_len) {
    *char_pos = 0;
    // Determine byte position of the substring start.
    if (!ForwardNUTF8CharPositions(in, pos, char_pos)) {
      return false;
    }
    // Determine position of the end of the substring.
    // The length will be capped at the end of the string, and we ignore whether
    // the string had enough characters to handle it or not.
    *char_len = *char_pos;
    ForwardNUTF8CharPositions(in, len, char_len);
    // The length in bytes is the position end of the substring less the start.
    *char_len = *char_len - *char_pos;
    return true;
  }

  // This function expects a negative position relative to the end of the
  // string, but will update the character position to a positive number
  // relative to the beginning of the string.
  static bool UpdateNegativePosAndLenForUtf8(const StringPiece in, const T pos,
                                             const T len, T* char_pos,
                                             T* char_len) {
    // Initially treat the length as position of the end of the substring.
    *char_len = in.size();
    // This is the number of character to skip from the end of the string to
    // arrive at the position where the substring should end.
    T utf8_chars_to_skip = -pos - len;
    if (utf8_chars_to_skip < 0) {
      utf8_chars_to_skip = 0;
    }
    // Find the byte position where the substring should end using the computed
    // number of characters to skip.
    if (!BackNUTF8CharPositions(in, utf8_chars_to_skip, char_len)) {
      return false;
    }
    // Next, determine where the substring should begin. The number of chars to
    // skip is the requested position minus the chars we've previously skipped.
    *char_pos = *char_len;
    if (!BackNUTF8CharPositions(in, -pos - utf8_chars_to_skip, char_pos)) {
      return false;
    }
    // The length in bytes is the position end of the substring less the start.
    *char_len = *char_len - *char_pos;
    return true;
  }

  CharUnit unit_ = CharUnit::BYTE;
};

#define REGISTER_SUBSTR(type)                                      \
  REGISTER_KERNEL_BUILDER(                                         \
      Name("Substr").Device(DEVICE_CPU).TypeConstraint<type>("T"), \
      SubstrOp<type>);
REGISTER_SUBSTR(int32);
REGISTER_SUBSTR(int64);
}  // namespace tensorflow
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2017 OFFIS Institute for Information Technology
 *                          Oldenburg, Germany
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "tvs/tracing.h"

#include "producer.h"

#include "epoch.h"

#include <chrono>
#include <iostream>
#include <thread>

/// provide a way to print out the enum value
std::ostream&
operator<<(std::ostream& out, producer::events const& e)
{
// clang-format off
#define _MAP(event) {producer::events::event,  #event}
  // clang-format on

  static const std::map<producer::events, const char*> cached = {
    _MAP(idle), _MAP(event_one), _MAP(event_two),
  };

#undef _MAP

  return out << cached.at(e);
}

producer::producer()
  : base_type("producer")
  , writer_("my_event_writer", tracing::STREAM_CREATE)
  , start_(get_epoch())
{
}

void
producer::run()
{
  std::thread t1(&producer::loop, this);
  t1.join();
}

producer::events
producer::wait_for_event()
{
  auto evt = events::event_two;

  if ((rand() % 100) < 50) {
    evt = events::event_one;
  }

  auto duration = std::chrono::microseconds(rand() % 1000000);
  std::this_thread::sleep_for(duration);

  return evt;
}

sysx::units::time_type
producer::tstamp()
{
  using namespace std::chrono;

  // use nanoseconds since this day 0:00 UTC

  // get the current time elapsed since instantiating this obj.  Ideally we
  // would use the time elapsed since some absolute point in time (e.g. unix
  // epoch start) , but then the uint64 overflows since we are using a higher
  // resolution.
  auto now = duration_cast<nanoseconds>(clock_type::now() - start_);

  auto dur = (1.0 * now.count()) * sysx::si::nanoseconds;

  return dur;
}

void
producer::loop()
{
  tracing::time_type now;

  auto evt_wr = tracing::timed_var(writer_.writer());
  int loops = 10;

  while (loops-- > 0) {

    // block until next event arrives
    auto evt = wait_for_event();
    std::set<events> new_evts{ evt };

    // How long did we sleep?
    tracing::timed_duration dur = tstamp() - tracing::timed_duration(now);

    TVS_TIMED_BLOCK(dur) { evt_wr = new_evts; }

    writer_.commit(dur);
    now += tracing::time_type(dur);
  }
}
<commit_msg>speed up test case<commit_after>/*
 * Copyright (c) 2017 OFFIS Institute for Information Technology
 *                          Oldenburg, Germany
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "tvs/tracing.h"

#include "producer.h"

#include "epoch.h"

#include <chrono>
#include <iostream>
#include <thread>

/// provide a way to print out the enum value
std::ostream&
operator<<(std::ostream& out, producer::events const& e)
{
// clang-format off
#define _MAP(event) {producer::events::event,  #event}
  // clang-format on

  static const std::map<producer::events, const char*> cached = {
    _MAP(idle), _MAP(event_one), _MAP(event_two),
  };

#undef _MAP

  return out << cached.at(e);
}

producer::producer()
  : base_type("producer")
  , writer_("my_event_writer", tracing::STREAM_CREATE)
  , start_(get_epoch())
{
}

void
producer::run()
{
  std::thread t1(&producer::loop, this);
  t1.join();
}

producer::events
producer::wait_for_event()
{
  auto evt = events::event_two;

  if ((rand() % 100) < 50) {
    evt = events::event_one;
  }

  auto duration = std::chrono::microseconds(rand() % 50000);
  std::this_thread::sleep_for(duration);

  return evt;
}

sysx::units::time_type
producer::tstamp()
{
  using namespace std::chrono;

  // use nanoseconds since this day 0:00 UTC

  // get the current time elapsed since instantiating this obj.  Ideally we
  // would use the time elapsed since some absolute point in time (e.g. unix
  // epoch start) , but then the uint64 overflows since we are using a higher
  // resolution.
  auto now = duration_cast<nanoseconds>(clock_type::now() - start_);

  auto dur = (1.0 * now.count()) * sysx::si::nanoseconds;

  return dur;
}

void
producer::loop()
{
  tracing::time_type now;

  auto evt_wr = tracing::timed_var(writer_.writer());
  int loops = 10;

  while (loops-- > 0) {

    // block until next event arrives
    auto evt = wait_for_event();
    std::set<events> new_evts{ evt };

    // How long did we sleep?
    tracing::timed_duration dur = tstamp() - tracing::timed_duration(now);

    TVS_TIMED_BLOCK(dur) { evt_wr = new_evts; }

    writer_.commit(dur);
    now += tracing::time_type(dur);
  }
}
<|endoftext|>


<commit_before>/*
 *
 *  Copyright 2013 Mario Alviano, Carmine Dodaro, Wolfgang Faber, Nicola Leone, Francesco Ricca, and Marco Sirianni.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

#include "Variable.h"
#include "Clause.h"
#include "Literal.h"
#include "heuristics/factories/HeuristicCounterFactoryForLiteral.h"
#include "heuristics/visitors/HeuristicVisitor.h"
#include "Solver.h"

ostream& operator<<(
    ostream& out,
    const Variable& var )
{
    out << var.name;
    return out;
}

Variable::~Variable()
{
    if( heuristicCounters[ 0 ] )
        delete heuristicCounters[ 0 ];

    if( heuristicCounters[ 1 ] )
        delete heuristicCounters[ 1 ];
}

void
Variable::setHeuristicCounterForLiterals(
    HeuristicCounterFactoryForLiteral* heuristicCounterFactoryForLiteral )
{
    heuristicCounters[ 0 ] = heuristicCounterFactoryForLiteral->createHeuristicCounter();
    heuristicCounters[ 1 ] = heuristicCounterFactoryForLiteral->createHeuristicCounter();
}

void
Variable::onLearning(
    LearningStrategy* strategy )
{
    //The implicant can be NULL if the literal is a choice.
    if( implicant != NULL )
    {
        implicant->onLearning( strategy );
    }
}

void
Variable::visitForHeuristic(
    HeuristicVisitor* heuristicVisitor )
{
    assert( "Heuristic visitor has not been set." && heuristicVisitor != NULL );
    if( isUndefined() )
        heuristicVisitor->onNavigatingVariable( this );
}

void
Variable::unitPropagation(
    Solver& solver,
    Literal literal,
    unsigned int sign )
{
    assert( "Variable to propagate is Undefined." && !this->isUndefined() );
    assert( "A literal must be true or false." && sign <= 1 );

    watchedLists[ sign ].startIteration();
    while( watchedLists[ sign ].hasNext() && !solver.conflictDetected() )
    {
        Clause* clause = watchedLists[ sign ].next();
        trace( solving, 3, "Considering clause %s.\n", clause->clauseToCharStar() );
        clause->onLiteralFalse( literal, solver );
    }
}
<commit_msg>Fixed a compilation error.<commit_after>/*
 *
 *  Copyright 2013 Mario Alviano, Carmine Dodaro, Wolfgang Faber, Nicola Leone, Francesco Ricca, and Marco Sirianni.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

#include "Variable.h"
#include "Clause.h"
#include "Literal.h"
#include "heuristics/factories/HeuristicCounterFactoryForLiteral.h"
#include "heuristics/HeuristicVisitor.h"
#include "Solver.h"

ostream& operator<<(
    ostream& out,
    const Variable& var )
{
    out << var.name;
    return out;
}

Variable::~Variable()
{
    if( heuristicCounters[ 0 ] )
        delete heuristicCounters[ 0 ];

    if( heuristicCounters[ 1 ] )
        delete heuristicCounters[ 1 ];
}

void
Variable::setHeuristicCounterForLiterals(
    HeuristicCounterFactoryForLiteral* heuristicCounterFactoryForLiteral )
{
    heuristicCounters[ 0 ] = heuristicCounterFactoryForLiteral->createHeuristicCounter();
    heuristicCounters[ 1 ] = heuristicCounterFactoryForLiteral->createHeuristicCounter();
}

void
Variable::onLearning(
    LearningStrategy* strategy )
{
    //The implicant can be NULL if the literal is a choice.
    if( implicant != NULL )
    {
        implicant->onLearning( strategy );
    }
}

void
Variable::visitForHeuristic(
    HeuristicVisitor* heuristicVisitor )
{
    assert( "Heuristic visitor has not been set." && heuristicVisitor != NULL );
    if( isUndefined() )
        heuristicVisitor->onNavigatingVariable( this );
}

void
Variable::unitPropagation(
    Solver& solver,
    Literal literal,
    unsigned int sign )
{
    assert( "Variable to propagate is Undefined." && !this->isUndefined() );
    assert( "A literal must be true or false." && sign <= 1 );

    watchedLists[ sign ].startIteration();
    while( watchedLists[ sign ].hasNext() && !solver.conflictDetected() )
    {
        Clause* clause = watchedLists[ sign ].next();
        trace( solving, 3, "Considering clause %s.\n", clause->clauseToCharStar() );
        clause->onLiteralFalse( literal, solver );
    }
}
<|endoftext|>


<commit_before>
#include "View.h"
#include "Log.h"
#include "Tcl.h"
#include "Index.h"

#include "UbuntuMono.ttf.h"

#include <vector>
#include <tigr.h>
#include <stb_truetype.h>

namespace view {

  static const tcl::Variable FONT_SIZE("view_fontSize", 16);
  static const tcl::Variable BLINK_RATE("view_cursorBlinkRate", 400);

  struct Glyph
  {
    //SDL_Surface * surface = nullptr;
    Tigr * surface = nullptr;
    int x = 0;
    int y = 0;
  };

  struct Cell
  {
    int ch = 0;
    uint16_t fg = COLOR_DEFAULT;
    uint16_t bg = COLOR_DEFAULT;
  };

  struct Color
  {
    uint8_t r;
    uint8_t g;
    uint8_t b;
  };

  //static SDL_Window * _window = nullptr;
  static Tigr * _window = nullptr;

  static stbtt_fontinfo _font;
  static int _fontBaseline;
  static int _fontLineHeight;
  static int _fontAdvance;
  static float _fontScale;

  static int _width = 0;
  static int _height = 0;
  static Index _cursor;
  static int _cursorBlinkTimeout = 0;
  static int _cursorBlinkVisible = true;
  static uint16_t _clearForeground = COLOR_TEXT;
  static uint16_t _clearBackground = COLOR_BACKGROUND;

  static Color BACKGROUND_COLOR = {39, 40, 34};
  static Color TEXT_COLOR = {248, 248, 242};

  static std::vector<Glyph> _glyphCache;
  static std::vector<Cell> _cells;

  static std::vector<Event> _eventQueue;

  static bool initializeFont();
  static void initGlyph(int ch);

  bool init(int preferredWidth, int preferredHeight, const char * title)
  {
    if (!initializeFont())
      return false;

    _window = tigrWindow(_fontAdvance * preferredWidth, _fontLineHeight * preferredHeight, title, TIGR_RETINA | TIGR_AUTO);
    if (_window == NULL)
    {
      logError("Could not create window");
      return false;
    }

    tigrUpdate(_window);

    _width = _window->w / _fontAdvance;
    _height = _window->h / _fontLineHeight;

    _cells.resize(_width * _height);
    return true;
  }

  void shutdown()
  {
    tigrFree(_window);
  }

  void setCursor(int x, int y)
  {
    _cursor.x = x;
    _cursor.y = y;
  }

  void hideCursor()
  {
    _cursor.x = -1;
    _cursor.y = -1;
  }

  void setClearAttributes(uint16_t fg, uint16_t bg)
  {
    _clearForeground = fg;
    _clearBackground = bg;
  }

  void clear()
  {
    for (uint32_t i = 0; i < _cells.size(); ++i)
    {
      Cell & cell = _cells[i];
      cell.ch = ' ';
      cell.fg = _clearForeground;
      cell.bg = _clearBackground;
    }
  }

  void changeCell(int x, int y, uint32_t ch, uint16_t fg, uint16_t bg)
  {
    if (x < 0 || x >= _width || y < 0 || y >= _height)
      return;

    int idx = (y * _width) + x;
    Cell & cell = _cells[idx];
    cell.ch = ch;
    cell.fg = fg;
    cell.bg = bg;
  }

  int width()
  {
    return _width;
  }

  int height()
  {
    return _height;
  }

  inline TPixel colorFromEnum(uint16_t color)
  {
    switch (color & 0x00FF)
    {
      case COLOR_BACKGROUND:  return tigrRGB(33, 37, 43);
      case COLOR_PANEL:       return tigrRGB(40, 44, 52);
      case COLOR_HIGHLIGHT:   return tigrRGB(82, 139, 255);
      case COLOR_TEXT:        return tigrRGB(174, 182, 195);
      case COLOR_SELECTION:   return tigrRGB(44, 50, 60);
      case COLOR_WHITE:       return tigrRGB(255, 255, 255);
      default:                return tigrRGB(255, 255, 255);
    }
  }

  void present()
  {
    tigrFill(_window, 0, 0, _window->w, _window->h, colorFromEnum(COLOR_BACKGROUND));

    int xPos = 0;
    int yPos = 0;

    for (int y = 0; y < _height; ++y)
    {
      for (int x = 0; x < _width; ++x)
      {
        Cell & cell = _cells[y * _width + x];

        if (cell.ch >= _glyphCache.size() || _glyphCache[cell.ch].surface == nullptr)
          initGlyph(cell.ch);

        Glyph & glyph = _glyphCache[cell.ch];

        if (cell.bg != COLOR_DEFAULT)
        {
          tigrFill(_window, xPos, yPos, _fontAdvance, _fontLineHeight, colorFromEnum(cell.bg));
        }

        //if (cell.fg & COLOR_REVERSE)
        //  textColor = tigrRGB(BACKGROUND_COLOR.r, BACKGROUND_COLOR.g, BACKGROUND_COLOR.b);

        tigrBlitTint(_window, glyph.surface, xPos + glyph.x, yPos + _fontBaseline + glyph.y, 0, 0, _fontAdvance, _fontLineHeight, colorFromEnum(cell.fg));

        xPos += _fontAdvance;
      }

      xPos = 0;
      yPos += _fontLineHeight;
    }

    if (_cursor.x >= 0 && _cursor.y >= 0 && _cursorBlinkVisible)
    {
      tigrLine(_window, _fontAdvance * _cursor.x, _fontLineHeight * _cursor.y,
                        _fontAdvance * _cursor.x, _fontLineHeight * (_cursor.y + 1),
                        colorFromEnum(COLOR_HIGHLIGHT));
    }
  }

  inline Keys handleCtrl(uint32_t symb)
  {
    switch (symb)
    {
      case 'a': return KEY_CTRL_A;
      case 'b': return KEY_CTRL_B;
      case 'c': return KEY_CTRL_C;
      case 'd': return KEY_CTRL_D;
      case 'e': return KEY_CTRL_E;
      case 'f': return KEY_CTRL_F;
      case 'g': return KEY_CTRL_G;
      case 'h': return KEY_CTRL_H;
      case 'i': return KEY_CTRL_I;
      case 'j': return KEY_CTRL_J;
      case 'k': return KEY_CTRL_K;
      case 'l': return KEY_CTRL_L;
      case 'm': return KEY_CTRL_M;
      case 'n': return KEY_CTRL_N;
      case 'o': return KEY_CTRL_O;
      case 'p': return KEY_CTRL_P;
      case 'q': return KEY_CTRL_Q;
      case 'r': return KEY_CTRL_R;
      case 's': return KEY_CTRL_S;
      case 't': return KEY_CTRL_T;
      case 'u': return KEY_CTRL_U;
      case 'v': return KEY_CTRL_V;
      case 'w': return KEY_CTRL_W;
      case 'x': return KEY_CTRL_X;
      case 'y': return KEY_CTRL_Y;
      case 'z': return KEY_CTRL_Z;
      case '2': return KEY_CTRL_2;
      case '3': return KEY_CTRL_3;
      case '4': return KEY_CTRL_4;
      case '5': return KEY_CTRL_5;
      case '6': return KEY_CTRL_6;
      case '7': return KEY_CTRL_7;
      case '8': return KEY_CTRL_8;
    }
    return KEY_NONE;
  }

  struct TigrToKey
  {
    int tigr;
    Keys key;
  };

  static const TigrToKey KEYS[] = {
    { TK_F1, KEY_F1 },
    { TK_F2, KEY_F2 },
    { TK_F3, KEY_F3 },
    { TK_F4, KEY_F4 },
    { TK_F5, KEY_F5 },
    { TK_F6, KEY_F6 },
    { TK_F7, KEY_F7 },
    { TK_F8, KEY_F8 },
    { TK_F9, KEY_F9 },
    { TK_F10, KEY_F10 },
    { TK_F11, KEY_F11 },
    { TK_F12, KEY_F12 },
    { TK_INSERT, KEY_INSERT },
    { TK_DELETE, KEY_DELETE },
    { TK_HOME, KEY_HOME },
    { TK_END, KEY_END },
    { TK_PAGEUP, KEY_PGUP },
    { TK_PAGEDN, KEY_PGDN },
    { TK_UP, KEY_ARROW_UP },
    { TK_DOWN, KEY_ARROW_DOWN },
    { TK_LEFT, KEY_ARROW_LEFT },
    { TK_RIGHT, KEY_ARROW_RIGHT },
    { TK_BACKSPACE, KEY_BACKSPACE },
    { TK_TAB, KEY_TAB },
    { TK_RETURN, KEY_ENTER },
    { TK_ESCAPE, KEY_ESC },
    { 0, KEY_NONE },
  };

  bool peekEvent(Event * event, int timeout)
  {
    //SDL_Event sdlEvent;

    if (_eventQueue.size() > 0)
    {
      *event = _eventQueue.front();
      _eventQueue.erase(_eventQueue.begin());
      return true;
    }

    int timeLeft = timeout;
    while (timeLeft > 0)
    {
      float startTime = tigrTime();

      int lastW = _window->w;
      int lastH = _window->h;
      tigrUpdate(_window);

      // Did the window size change?
      if (lastW != _window->w || lastH != _window->h)
      {
        _width = _window->w / _fontAdvance;
        _height = _window->h / _fontLineHeight;

        if (_width == 0) _width = 1;
        if (_height == 0) _height = 1;

        _cells.resize(_width * _height);

        Event e = {EVENT_RESIZE, KEY_NONE, 0};
        _eventQueue.push_back(e);
      }

      // Did we close the window?
      if (tigrClosed(_window) > 0)
      {
        Event e = {EVENT_QUIT, KEY_NONE, 0};
        _eventQueue.push_back(e);
      }

      // Keyboard input?
      if (tigrKeyHeld(_window, TK_RCONTROL) || tigrKeyHeld(_window, TK_LCONTROL))
      {
        Keys key = handleCtrl(tigrReadChar(_window));
        if (key != KEY_NONE)
          _eventQueue.push_back(Event {EVENT_KEY, key, 0});
      }
      else
      {
        uint32_t ch = tigrReadChar(_window);
        if (ch >  0 && ch != 10 && ch != 27 && ch != 8 && ch != 9)
        {
          Event e = {EVENT_KEY, KEY_NONE, ch};
          logInfo("Ch: ", (int)ch);
          _eventQueue.push_back(e);
        }

        for (int i = 0; KEYS[i].key != KEY_NONE; ++i)
        {
          if (tigrKeyDown(_window, KEYS[i].tigr))
            _eventQueue.push_back(Event {EVENT_KEY, KEYS[i].key, 0});
        }
      }

      float endTime = tigrTime();
      int dt = (endTime - startTime) * 1000;
      timeLeft -= dt;
    }

    /*
    while (timeLeft > 0)
    {
      if (SDL_PollEvent(&sdlEvent))
      {
        event->type = EVENT_NONE;
        event->key = KEY_NONE;
        event->ch = 0;

        switch (sdlEvent.type)
        {
          case SDL_QUIT:
            event->type = EVENT_QUIT;
            break;

          case SDL_WINDOWEVENT:
            {
              if (sdlEvent.window.event == SDL_WINDOWEVENT_RESIZED)
              {
                _width = sdlEvent.window.data1 / _fontAdvance;
                _height = sdlEvent.window.data2 / _fontLineHeight;

                if (_width == 0) _width = 1;
                if (_height == 0) _height = 1;

                _cells.resize(_width * _height);

                event->type = EVENT_RESIZE;
              }
            }
            break;

          case SDL_KEYDOWN:
            {
              event->type = EVENT_KEY;
              event->key = toKeys(sdlEvent.key.keysym);

              if (event->key == 0)
              {
                event->type = EVENT_NONE;
                event->key = KEY_NONE;
              }
            }
            break;

          case SDL_TEXTINPUT:
            {
              uint32_t text[4];

              if (str::toUTF32(std::string(sdlEvent.text.text), text, 4) > 0)
              {
                event->type = EVENT_KEY;
                event->ch = text[0];
              }
            }
            break;
        }
      }
      else
      {
        SDL_Delay(10);
        timeLeft -= 10;

        _cursorBlinkTimeout += 10;
        if (_cursorBlinkTimeout > BLINK_RATE.toInt())
        {
          _cursorBlinkVisible = !_cursorBlinkVisible;
          _cursorBlinkTimeout = 0;
        }

        present();
      }
    }

    if (event->type != EVENT_NONE)
    {
    // Always reset the cursor blink rate if we press a key
    if (event->type == EVENT_KEY)
    {
    _cursorBlinkTimeout = 0;
    _cursorBlinkVisible = true;
  }

  return true;
}
*/

    return false;
  }

  static void initGlyph(int ch)
  {
    Glyph glyph;
    int width, height;

    unsigned char * pixels = stbtt_GetCodepointBitmap(&_font, _fontScale, _fontScale, ch, &width, &height, &glyph.x, &glyph.y);

    glyph.surface = tigrBitmap(width, height);

    TPixel * p = glyph.surface->pix;

    for (int y = 0; y < height; ++y)
      for (int x = 0; x < width; ++x)
      {
        const int idx = y * width + x;
        p[idx] = tigrRGBA(255, 255, 255, pixels[idx]);
      }

    stbtt_FreeBitmap(pixels, nullptr);

    if (ch >= _glyphCache.size())
      _glyphCache.resize(ch + 1);

    _glyphCache[ch] = glyph;
  }

  static bool initializeFont()
  {
    if (stbtt_InitFont(&_font, (unsigned char *)UbuntuMono, 0) == 0)
    {
      logError("Could not initialize the font");
      return false;
    }

    // Font size
    int ascent, decent, lineGap, advance;

    _fontScale = stbtt_ScaleForPixelHeight(&_font, FONT_SIZE.toInt());

    stbtt_GetFontVMetrics(&_font, &ascent, &decent, &lineGap);
    stbtt_GetCodepointHMetrics(&_font, '0', &advance, nullptr);

    _fontBaseline = ascent * _fontScale;
    _fontLineHeight = (ascent - decent + lineGap) * _fontScale;
    _fontAdvance = advance * _fontScale;

    // Initialize some default glyphs
    const Str DEFAULT_GLYPHS(" 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ,.-;:()=+-*/!\"'#$%&{[]}<>|~");
    for (auto ch : DEFAULT_GLYPHS)
      initGlyph(ch);

    return true;
  }

}
<commit_msg>Added some padding to the line height<commit_after>
#include "View.h"
#include "Log.h"
#include "Tcl.h"
#include "Index.h"

#include "UbuntuMono.ttf.h"

#include <vector>
#include <tigr.h>
#include <stb_truetype.h>

namespace view {

  static const tcl::Variable FONT_SIZE("view_fontSize", 16);
  static const tcl::Variable BLINK_RATE("view_cursorBlinkRate", 400);

  struct Glyph
  {
    //SDL_Surface * surface = nullptr;
    Tigr * surface = nullptr;
    int x = 0;
    int y = 0;
  };

  struct Cell
  {
    int ch = 0;
    uint16_t fg = COLOR_DEFAULT;
    uint16_t bg = COLOR_DEFAULT;
  };

  struct Color
  {
    uint8_t r;
    uint8_t g;
    uint8_t b;
  };

  //static SDL_Window * _window = nullptr;
  static Tigr * _window = nullptr;

  static stbtt_fontinfo _font;
  static int _fontBaseline;
  static int _fontLineHeight;
  static int _fontLinePadding = 2;
  static int _fontAdvance;
  static float _fontScale;

  static int _width = 0;
  static int _height = 0;
  static Index _cursor;
  static int _cursorBlinkTimeout = 0;
  static int _cursorBlinkVisible = true;
  static uint16_t _clearForeground = COLOR_TEXT;
  static uint16_t _clearBackground = COLOR_BACKGROUND;

  static Color BACKGROUND_COLOR = {39, 40, 34};
  static Color TEXT_COLOR = {248, 248, 242};

  static std::vector<Glyph> _glyphCache;
  static std::vector<Cell> _cells;

  static std::vector<Event> _eventQueue;

  static bool initializeFont();
  static void initGlyph(int ch);

  bool init(int preferredWidth, int preferredHeight, const char * title)
  {
    if (!initializeFont())
      return false;

    _window = tigrWindow(_fontAdvance * preferredWidth, _fontLineHeight * preferredHeight, title, TIGR_RETINA | TIGR_AUTO);
    if (_window == NULL)
    {
      logError("Could not create window");
      return false;
    }

    tigrUpdate(_window);

    _width = _window->w / _fontAdvance;
    _height = _window->h / _fontLineHeight;

    _cells.resize(_width * _height);
    return true;
  }

  void shutdown()
  {
    tigrFree(_window);
  }

  void setCursor(int x, int y)
  {
    _cursor.x = x;
    _cursor.y = y;
  }

  void hideCursor()
  {
    _cursor.x = -1;
    _cursor.y = -1;
  }

  void setClearAttributes(uint16_t fg, uint16_t bg)
  {
    _clearForeground = fg;
    _clearBackground = bg;
  }

  void clear()
  {
    for (uint32_t i = 0; i < _cells.size(); ++i)
    {
      Cell & cell = _cells[i];
      cell.ch = ' ';
      cell.fg = _clearForeground;
      cell.bg = _clearBackground;
    }
  }

  void changeCell(int x, int y, uint32_t ch, uint16_t fg, uint16_t bg)
  {
    if (x < 0 || x >= _width || y < 0 || y >= _height)
      return;

    int idx = (y * _width) + x;
    Cell & cell = _cells[idx];
    cell.ch = ch;
    cell.fg = fg;
    cell.bg = bg;
  }

  int width()
  {
    return _width;
  }

  int height()
  {
    return _height;
  }

  inline TPixel colorFromEnum(uint16_t color)
  {
    switch (color & 0x00FF)
    {
      case COLOR_BACKGROUND:  return tigrRGB(33, 37, 43);
      case COLOR_PANEL:       return tigrRGB(40, 44, 52);
      case COLOR_HIGHLIGHT:   return tigrRGB(82, 139, 255);
      case COLOR_TEXT:        return tigrRGB(174, 182, 195);
      case COLOR_SELECTION:   return tigrRGB(44, 50, 60);
      case COLOR_WHITE:       return tigrRGB(255, 255, 255);
      default:                return tigrRGB(255, 255, 255);
    }
  }

  void present()
  {
    tigrFill(_window, 0, 0, _window->w, _window->h, colorFromEnum(COLOR_BACKGROUND));

    int xPos = 0;
    int yPos = 0;

    for (int y = 0; y < _height; ++y)
    {
      for (int x = 0; x < _width; ++x)
      {
        Cell & cell = _cells[y * _width + x];

        if (cell.ch >= _glyphCache.size() || _glyphCache[cell.ch].surface == nullptr)
          initGlyph(cell.ch);

        Glyph & glyph = _glyphCache[cell.ch];

        if (cell.bg != COLOR_DEFAULT)
        {
          tigrFill(_window, xPos, yPos, _fontAdvance, _fontLineHeight, colorFromEnum(cell.bg));
        }

        //if (cell.fg & COLOR_REVERSE)
        //  textColor = tigrRGB(BACKGROUND_COLOR.r, BACKGROUND_COLOR.g, BACKGROUND_COLOR.b);

        tigrBlitTint(_window, glyph.surface, xPos + glyph.x, yPos + _fontBaseline + glyph.y + (_fontLinePadding / 2), 0, 0, _fontAdvance, _fontLineHeight, colorFromEnum(cell.fg));

        xPos += _fontAdvance;
      }

      xPos = 0;
      yPos += _fontLineHeight;
    }

    if (_cursor.x >= 0 && _cursor.y >= 0 && _cursorBlinkVisible)
    {
      tigrLine(_window, _fontAdvance * _cursor.x, _fontLineHeight * _cursor.y,
                        _fontAdvance * _cursor.x, _fontLineHeight * (_cursor.y + 1),
                        colorFromEnum(COLOR_HIGHLIGHT));
    }
  }

  inline Keys handleCtrl(uint32_t symb)
  {
    switch (symb)
    {
      case 'a': return KEY_CTRL_A;
      case 'b': return KEY_CTRL_B;
      case 'c': return KEY_CTRL_C;
      case 'd': return KEY_CTRL_D;
      case 'e': return KEY_CTRL_E;
      case 'f': return KEY_CTRL_F;
      case 'g': return KEY_CTRL_G;
      case 'h': return KEY_CTRL_H;
      case 'i': return KEY_CTRL_I;
      case 'j': return KEY_CTRL_J;
      case 'k': return KEY_CTRL_K;
      case 'l': return KEY_CTRL_L;
      case 'm': return KEY_CTRL_M;
      case 'n': return KEY_CTRL_N;
      case 'o': return KEY_CTRL_O;
      case 'p': return KEY_CTRL_P;
      case 'q': return KEY_CTRL_Q;
      case 'r': return KEY_CTRL_R;
      case 's': return KEY_CTRL_S;
      case 't': return KEY_CTRL_T;
      case 'u': return KEY_CTRL_U;
      case 'v': return KEY_CTRL_V;
      case 'w': return KEY_CTRL_W;
      case 'x': return KEY_CTRL_X;
      case 'y': return KEY_CTRL_Y;
      case 'z': return KEY_CTRL_Z;
      case '2': return KEY_CTRL_2;
      case '3': return KEY_CTRL_3;
      case '4': return KEY_CTRL_4;
      case '5': return KEY_CTRL_5;
      case '6': return KEY_CTRL_6;
      case '7': return KEY_CTRL_7;
      case '8': return KEY_CTRL_8;
    }
    return KEY_NONE;
  }

  struct TigrToKey
  {
    int tigr;
    Keys key;
  };

  static const TigrToKey KEYS[] = {
    { TK_F1, KEY_F1 },
    { TK_F2, KEY_F2 },
    { TK_F3, KEY_F3 },
    { TK_F4, KEY_F4 },
    { TK_F5, KEY_F5 },
    { TK_F6, KEY_F6 },
    { TK_F7, KEY_F7 },
    { TK_F8, KEY_F8 },
    { TK_F9, KEY_F9 },
    { TK_F10, KEY_F10 },
    { TK_F11, KEY_F11 },
    { TK_F12, KEY_F12 },
    { TK_INSERT, KEY_INSERT },
    { TK_DELETE, KEY_DELETE },
    { TK_HOME, KEY_HOME },
    { TK_END, KEY_END },
    { TK_PAGEUP, KEY_PGUP },
    { TK_PAGEDN, KEY_PGDN },
    { TK_UP, KEY_ARROW_UP },
    { TK_DOWN, KEY_ARROW_DOWN },
    { TK_LEFT, KEY_ARROW_LEFT },
    { TK_RIGHT, KEY_ARROW_RIGHT },
    { TK_BACKSPACE, KEY_BACKSPACE },
    { TK_TAB, KEY_TAB },
    { TK_RETURN, KEY_ENTER },
    { TK_ESCAPE, KEY_ESC },
    { 0, KEY_NONE },
  };

  bool peekEvent(Event * event, int timeout)
  {
    //SDL_Event sdlEvent;

    if (_eventQueue.size() > 0)
    {
      *event = _eventQueue.front();
      _eventQueue.erase(_eventQueue.begin());
      return true;
    }

    int timeLeft = timeout;
    while (timeLeft > 0)
    {
      float startTime = tigrTime();

      int lastW = _window->w;
      int lastH = _window->h;
      tigrUpdate(_window);

      // Did the window size change?
      if (lastW != _window->w || lastH != _window->h)
      {
        _width = _window->w / _fontAdvance;
        _height = _window->h / _fontLineHeight;

        if (_width == 0) _width = 1;
        if (_height == 0) _height = 1;

        _cells.resize(_width * _height);

        Event e = {EVENT_RESIZE, KEY_NONE, 0};
        _eventQueue.push_back(e);
      }

      // Did we close the window?
      if (tigrClosed(_window) > 0)
      {
        Event e = {EVENT_QUIT, KEY_NONE, 0};
        _eventQueue.push_back(e);
      }

      // Keyboard input?
      if (tigrKeyHeld(_window, TK_RCONTROL) || tigrKeyHeld(_window, TK_LCONTROL))
      {
        Keys key = handleCtrl(tigrReadChar(_window));
        if (key != KEY_NONE)
          _eventQueue.push_back(Event {EVENT_KEY, key, 0});
      }
      else
      {
        uint32_t ch = tigrReadChar(_window);
        if (ch >  0 && ch != 10 && ch != 27 && ch != 8 && ch != 9)
        {
          Event e = {EVENT_KEY, KEY_NONE, ch};
          logInfo("Ch: ", (int)ch);
          _eventQueue.push_back(e);
        }

        for (int i = 0; KEYS[i].key != KEY_NONE; ++i)
        {
          if (tigrKeyDown(_window, KEYS[i].tigr))
            _eventQueue.push_back(Event {EVENT_KEY, KEYS[i].key, 0});
        }
      }

      float endTime = tigrTime();
      int dt = (endTime - startTime) * 1000;
      timeLeft -= dt;
    }

    /*
    while (timeLeft > 0)
    {
      if (SDL_PollEvent(&sdlEvent))
      {
        event->type = EVENT_NONE;
        event->key = KEY_NONE;
        event->ch = 0;

        switch (sdlEvent.type)
        {
          case SDL_QUIT:
            event->type = EVENT_QUIT;
            break;

          case SDL_WINDOWEVENT:
            {
              if (sdlEvent.window.event == SDL_WINDOWEVENT_RESIZED)
              {
                _width = sdlEvent.window.data1 / _fontAdvance;
                _height = sdlEvent.window.data2 / _fontLineHeight;

                if (_width == 0) _width = 1;
                if (_height == 0) _height = 1;

                _cells.resize(_width * _height);

                event->type = EVENT_RESIZE;
              }
            }
            break;

          case SDL_KEYDOWN:
            {
              event->type = EVENT_KEY;
              event->key = toKeys(sdlEvent.key.keysym);

              if (event->key == 0)
              {
                event->type = EVENT_NONE;
                event->key = KEY_NONE;
              }
            }
            break;

          case SDL_TEXTINPUT:
            {
              uint32_t text[4];

              if (str::toUTF32(std::string(sdlEvent.text.text), text, 4) > 0)
              {
                event->type = EVENT_KEY;
                event->ch = text[0];
              }
            }
            break;
        }
      }
      else
      {
        SDL_Delay(10);
        timeLeft -= 10;

        _cursorBlinkTimeout += 10;
        if (_cursorBlinkTimeout > BLINK_RATE.toInt())
        {
          _cursorBlinkVisible = !_cursorBlinkVisible;
          _cursorBlinkTimeout = 0;
        }

        present();
      }
    }

    if (event->type != EVENT_NONE)
    {
    // Always reset the cursor blink rate if we press a key
    if (event->type == EVENT_KEY)
    {
    _cursorBlinkTimeout = 0;
    _cursorBlinkVisible = true;
  }

  return true;
}
*/

    return false;
  }

  static void initGlyph(int ch)
  {
    Glyph glyph;
    int width, height;

    unsigned char * pixels = stbtt_GetCodepointBitmap(&_font, _fontScale, _fontScale, ch, &width, &height, &glyph.x, &glyph.y);

    glyph.surface = tigrBitmap(width, height);

    TPixel * p = glyph.surface->pix;

    for (int y = 0; y < height; ++y)
      for (int x = 0; x < width; ++x)
      {
        const int idx = y * width + x;
        p[idx] = tigrRGBA(255, 255, 255, pixels[idx]);
      }

    stbtt_FreeBitmap(pixels, nullptr);

    if (ch >= _glyphCache.size())
      _glyphCache.resize(ch + 1);

    _glyphCache[ch] = glyph;
  }

  static bool initializeFont()
  {
    if (stbtt_InitFont(&_font, (unsigned char *)UbuntuMono, 0) == 0)
    {
      logError("Could not initialize the font");
      return false;
    }

    // Font size
    int ascent, decent, lineGap, advance;

    _fontScale = stbtt_ScaleForPixelHeight(&_font, FONT_SIZE.toInt());

    stbtt_GetFontVMetrics(&_font, &ascent, &decent, &lineGap);
    stbtt_GetCodepointHMetrics(&_font, '0', &advance, nullptr);

    _fontBaseline = ascent * _fontScale;
    _fontLineHeight = (ascent - decent + lineGap) * _fontScale + _fontLinePadding;
    _fontAdvance = advance * _fontScale;

    // Initialize some default glyphs
    const Str DEFAULT_GLYPHS(" 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ,.-;:()=+-*/!\"'#$%&{[]}<>|~");
    for (auto ch : DEFAULT_GLYPHS)
      initGlyph(ch);

    return true;
  }

}
<|endoftext|>