<commit_before>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// See docs in ../ops/sdca_ops.cc.

#define EIGEN_USE_THREADS

#include <stdint.h>
#include <atomic>
#include <limits>
#include <memory>
#include <new>
#include <string>
#include <vector>

#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_def_builder.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/hinge-loss.h"
#include "tensorflow/core/kernels/logistic-loss.h"
#include "tensorflow/core/kernels/loss.h"
#include "tensorflow/core/kernels/sdca_internal.h"
#include "tensorflow/core/kernels/smooth-hinge-loss.h"
#include "tensorflow/core/kernels/squared-loss.h"
#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/fingerprint.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/work_sharder.h"

namespace tensorflow {

namespace {

using sdca::Regularizations;
using sdca::Example;
using sdca::Examples;
using sdca::ExampleStatistics;
using sdca::ModelWeights;

struct ComputeOptions {
  ComputeOptions(OpKernelConstruction* const context) {
    string loss_type;
    OP_REQUIRES_OK(context, context->GetAttr("loss_type", &loss_type));
    if (loss_type == "logistic_loss") {
      loss_updater.reset(new LogisticLossUpdater);
    } else if (loss_type == "squared_loss") {
      loss_updater.reset(new SquaredLossUpdater);
    } else if (loss_type == "hinge_loss") {
      loss_updater.reset(new HingeLossUpdater);
    } else if (loss_type == "smooth_hinge_loss") {
      loss_updater.reset(new SmoothHingeLossUpdater);
    } else {
      OP_REQUIRES(context, false, errors::InvalidArgument(
                                      "Unsupported loss type: ", loss_type));
    }
    OP_REQUIRES_OK(context, context->GetAttr("adaptative", &adaptative));
    OP_REQUIRES_OK(
        context, context->GetAttr("num_sparse_features", &num_sparse_features));
    OP_REQUIRES_OK(context, context->GetAttr("num_sparse_features_with_values",
                                             &num_sparse_features_with_values));
    OP_REQUIRES_OK(context,
                   context->GetAttr("num_dense_features", &num_dense_features));
    OP_REQUIRES(
        context, num_sparse_features + num_dense_features > 0,
        errors::InvalidArgument("Requires at least one feature to train."));

    OP_REQUIRES(context, static_cast<int64>(num_sparse_features) +
                                 static_cast<int64>(num_dense_features) <=
                             std::numeric_limits<int>::max(),
                errors::InvalidArgument(
                    strings::Printf("Too many feature groups: %lld > %d",
                                    static_cast<int64>(num_sparse_features) +
                                        static_cast<int64>(num_dense_features),
                                    std::numeric_limits<int>::max())));
    OP_REQUIRES_OK(
        context, context->GetAttr("num_loss_partitions", &num_loss_partitions));
    OP_REQUIRES_OK(context, context->GetAttr("num_inner_iterations",
                                             &num_inner_iterations));
    OP_REQUIRES_OK(context, regularizations.Initialize(context));
  }

  std::unique_ptr<DualLossUpdater> loss_updater;
  int num_sparse_features = 0;
  int num_sparse_features_with_values = 0;
  int num_dense_features = 0;
  int num_inner_iterations = 0;
  int num_loss_partitions = 0;
  bool adaptative = false;
  Regularizations regularizations;
};

// TODO(shengx): The helper classes/methods are changed to support multiclass
// SDCA, which lead to changes within this function. Need to revisit the
// convergence once the multiclass SDCA is in.
void DoCompute(const ComputeOptions& options, OpKernelContext* const context) {
  ModelWeights model_weights;
  OP_REQUIRES_OK(context, model_weights.Initialize(context));

  Examples examples;
  OP_REQUIRES_OK(
      context,
      examples.Initialize(context, model_weights, options.num_sparse_features,
                          options.num_sparse_features_with_values,
                          options.num_dense_features));

  const Tensor* example_state_data_t;
  OP_REQUIRES_OK(context,
                 context->input("example_state_data", &example_state_data_t));
  TensorShape expected_example_state_shape({examples.num_examples(), 4});
  OP_REQUIRES(context,
              example_state_data_t->shape() == expected_example_state_shape,
              errors::InvalidArgument(
                  "Expected shape ", expected_example_state_shape.DebugString(),
                  " for example_state_data, got ",
                  example_state_data_t->shape().DebugString()));

  Tensor mutable_example_state_data_t(*example_state_data_t);
  auto example_state_data = mutable_example_state_data_t.matrix<float>();
  context->set_output("out_example_state_data", mutable_example_state_data_t);

  if (options.adaptative) {
    OP_REQUIRES_OK(context,
                   examples.SampleAdaptativeProbabilities(
                       options.num_loss_partitions, options.regularizations,
                       model_weights, example_state_data, options.loss_updater,
                       /*num_weight_vectors =*/1));
  }

  mutex mu;
  Status train_step_status GUARDED_BY(mu);
  std::atomic<std::int64_t> atomic_index(-1);
  auto train_step = [&](const int64 begin, const int64 end) {
    // The static_cast here is safe since begin and end can be at most
    // num_examples which is an int.
    for (int id = static_cast<int>(begin); id < end; ++id) {
      const int64 example_index =
          examples.sampled_index(++atomic_index, options.adaptative);
      const Example& example = examples.example(example_index);
      const float dual = example_state_data(example_index, 0);
      const float example_weight = example.example_weight();
      float example_label = example.example_label();
      const Status conversion_status =
          options.loss_updater->ConvertLabel(&example_label);
      if (!conversion_status.ok()) {
        mutex_lock l(mu);
        train_step_status = conversion_status;
        // Return from this worker thread - the calling thread is
        // responsible for checking context status and returning on error.
        return;
      }

      // Compute wx, example norm weighted by regularization, dual loss,
      // primal loss.
      // For binary SDCA, num_weight_vectors should be one.
      const ExampleStatistics example_statistics =
          example.ComputeWxAndWeightedExampleNorm(
              options.num_loss_partitions, model_weights,
              options.regularizations, 1 /* num_weight_vectors */);

      const double new_dual = options.loss_updater->ComputeUpdatedDual(
          options.num_loss_partitions, example_label, example_weight, dual,
          example_statistics.wx[0], example_statistics.normalized_squared_norm);

      // Compute new weights.
      const double normalized_bounded_dual_delta =
          (new_dual - dual) * example_weight /
          options.regularizations.symmetric_l2();
      model_weights.UpdateDeltaWeights(
          context->eigen_cpu_device(), example,
          std::vector<double>{normalized_bounded_dual_delta});

      // Update example data.
      example_state_data(example_index, 0) = new_dual;
      example_state_data(example_index, 1) =
          options.loss_updater->ComputePrimalLoss(
              example_statistics.prev_wx[0], example_label, example_weight);
      example_state_data(example_index, 2) =
          options.loss_updater->ComputeDualLoss(dual, example_label,
                                                example_weight);
      example_state_data(example_index, 3) = example_weight;
    }
  };
  // TODO(sibyl-Aix6ihai): Tune this properly based on sparsity of the data,
  // number of cpus, and cost per example.
  const int64 kCostPerUnit = examples.num_features();
  const DeviceBase::CpuWorkerThreads& worker_threads =
      *context->device()->tensorflow_cpu_worker_threads();

  Shard(worker_threads.num_threads, worker_threads.workers,
        examples.num_examples(), kCostPerUnit, train_step);
  OP_REQUIRES_OK(context, train_step_status);
}

}  // namespace

class SdcaOptimizer : public OpKernel {
 public:
  explicit SdcaOptimizer(OpKernelConstruction* const context)
      : OpKernel(context), options_(context) {}

  void Compute(OpKernelContext* const context) override {
    DoCompute(options_, context);
  }

 private:
  // TODO(sibyl-Aix6ihai): We could use the type-constraint on loss_type, and
  // template the entire class to avoid the virtual table lookup penalty in
  // the inner loop.
  ComputeOptions options_;
};
REGISTER_KERNEL_BUILDER(Name("SdcaOptimizer").Device(DEVICE_CPU),
                        SdcaOptimizer);

class SdcaShrinkL1 : public OpKernel {
 public:
  explicit SdcaShrinkL1(OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, regularizations_.Initialize(context));
  }

  void Compute(OpKernelContext* const context) override {
    OpMutableInputList weights_inputs;
    OP_REQUIRES_OK(context,
                   context->mutable_input_list("weights", &weights_inputs));

    auto do_work = [&](const int64 begin, const int64 end) {
      for (int i = begin; i < end; ++i) {
        auto prox_w = weights_inputs.at(i, /*lock_held=*/true).flat<float>();
        prox_w.device(context->eigen_cpu_device()) =
            regularizations_.EigenShrinkVector(prox_w);
      }
    };

    if (weights_inputs.size() > 0) {
      int64 num_weights = 0;
      for (int i = 0; i < weights_inputs.size(); ++i) {
        num_weights += weights_inputs.at(i, /*lock_held=*/true).NumElements();
      }
      // TODO(sibyl-Aix6ihai): Tune this value.
      const int64 kCostPerUnit = (num_weights * 50) / weights_inputs.size();
      const DeviceBase::CpuWorkerThreads& worker_threads =
          *context->device()->tensorflow_cpu_worker_threads();
      Shard(worker_threads.num_threads, worker_threads.workers,
            weights_inputs.size(), kCostPerUnit, do_work);
    }
  }

 private:
  Regularizations regularizations_;
};
REGISTER_KERNEL_BUILDER(Name("SdcaShrinkL1").Device(DEVICE_CPU), SdcaShrinkL1);

// Computes platform independent, compact and unique (with very high
// probability) representation of an example id. It shouldn't be put in
// persistent storage, as its implementation may change in the future.
//
// The current probability of at least one collision for 1B example_ids is
// approximately 10^-21 (ie 2^60 / 2^129).
class SdcaFprint : public OpKernel {
 public:
  explicit SdcaFprint(OpKernelConstruction* const context)
      : OpKernel(context) {}

  void Compute(OpKernelContext* const context) override {
    const Tensor& input = context->input(0);
    OP_REQUIRES(context, TensorShapeUtils::IsVector(input.shape()),
                errors::InvalidArgument("Input must be a vector, got shape ",
                                        input.shape().DebugString()));
    Tensor* out;
    const int64 num_elements = input.NumElements();
    OP_REQUIRES_OK(context, context->allocate_output(
                                0, TensorShape({num_elements, 2}), &out));

    const auto in_values = input.flat<string>();
    auto out_values = out->matrix<int64>();

    for (int64 i = 0; i < num_elements; ++i) {
      const Fprint128 fprint = Fingerprint128(in_values(i));
      // Never return 0 or 1 as the first value of the hash to allow these to
      // safely be used as sentinel values (e.g. dense hash table empty key).
      out_values(i, 0) = TF_PREDICT_TRUE(fprint.low64 >= 2)
                             ? fprint.low64
                             : fprint.low64 + ~static_cast<uint64>(1);
      out_values(i, 1) = fprint.high64;
    }
  }
};
REGISTER_KERNEL_BUILDER(Name("SdcaFprint").Device(DEVICE_CPU), SdcaFprint);

}  // namespace tensorflow
<commit_msg>Fixed the signature of the Sdca*::Compute() methods Change: 141203556<commit_after>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

// See docs in ../ops/sdca_ops.cc.

#define EIGEN_USE_THREADS

#include <stdint.h>
#include <atomic>
#include <limits>
#include <memory>
#include <new>
#include <string>
#include <vector>

#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_def_builder.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/hinge-loss.h"
#include "tensorflow/core/kernels/logistic-loss.h"
#include "tensorflow/core/kernels/loss.h"
#include "tensorflow/core/kernels/sdca_internal.h"
#include "tensorflow/core/kernels/smooth-hinge-loss.h"
#include "tensorflow/core/kernels/squared-loss.h"
#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/fingerprint.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/work_sharder.h"

namespace tensorflow {

namespace {

using sdca::Regularizations;
using sdca::Example;
using sdca::Examples;
using sdca::ExampleStatistics;
using sdca::ModelWeights;

struct ComputeOptions {
  ComputeOptions(OpKernelConstruction* const context) {
    string loss_type;
    OP_REQUIRES_OK(context, context->GetAttr("loss_type", &loss_type));
    if (loss_type == "logistic_loss") {
      loss_updater.reset(new LogisticLossUpdater);
    } else if (loss_type == "squared_loss") {
      loss_updater.reset(new SquaredLossUpdater);
    } else if (loss_type == "hinge_loss") {
      loss_updater.reset(new HingeLossUpdater);
    } else if (loss_type == "smooth_hinge_loss") {
      loss_updater.reset(new SmoothHingeLossUpdater);
    } else {
      OP_REQUIRES(context, false, errors::InvalidArgument(
                                      "Unsupported loss type: ", loss_type));
    }
    OP_REQUIRES_OK(context, context->GetAttr("adaptative", &adaptative));
    OP_REQUIRES_OK(
        context, context->GetAttr("num_sparse_features", &num_sparse_features));
    OP_REQUIRES_OK(context, context->GetAttr("num_sparse_features_with_values",
                                             &num_sparse_features_with_values));
    OP_REQUIRES_OK(context,
                   context->GetAttr("num_dense_features", &num_dense_features));
    OP_REQUIRES(
        context, num_sparse_features + num_dense_features > 0,
        errors::InvalidArgument("Requires at least one feature to train."));

    OP_REQUIRES(context, static_cast<int64>(num_sparse_features) +
                                 static_cast<int64>(num_dense_features) <=
                             std::numeric_limits<int>::max(),
                errors::InvalidArgument(
                    strings::Printf("Too many feature groups: %lld > %d",
                                    static_cast<int64>(num_sparse_features) +
                                        static_cast<int64>(num_dense_features),
                                    std::numeric_limits<int>::max())));
    OP_REQUIRES_OK(
        context, context->GetAttr("num_loss_partitions", &num_loss_partitions));
    OP_REQUIRES_OK(context, context->GetAttr("num_inner_iterations",
                                             &num_inner_iterations));
    OP_REQUIRES_OK(context, regularizations.Initialize(context));
  }

  std::unique_ptr<DualLossUpdater> loss_updater;
  int num_sparse_features = 0;
  int num_sparse_features_with_values = 0;
  int num_dense_features = 0;
  int num_inner_iterations = 0;
  int num_loss_partitions = 0;
  bool adaptative = false;
  Regularizations regularizations;
};

// TODO(shengx): The helper classes/methods are changed to support multiclass
// SDCA, which lead to changes within this function. Need to revisit the
// convergence once the multiclass SDCA is in.
void DoCompute(const ComputeOptions& options, OpKernelContext* const context) {
  ModelWeights model_weights;
  OP_REQUIRES_OK(context, model_weights.Initialize(context));

  Examples examples;
  OP_REQUIRES_OK(
      context,
      examples.Initialize(context, model_weights, options.num_sparse_features,
                          options.num_sparse_features_with_values,
                          options.num_dense_features));

  const Tensor* example_state_data_t;
  OP_REQUIRES_OK(context,
                 context->input("example_state_data", &example_state_data_t));
  TensorShape expected_example_state_shape({examples.num_examples(), 4});
  OP_REQUIRES(context,
              example_state_data_t->shape() == expected_example_state_shape,
              errors::InvalidArgument(
                  "Expected shape ", expected_example_state_shape.DebugString(),
                  " for example_state_data, got ",
                  example_state_data_t->shape().DebugString()));

  Tensor mutable_example_state_data_t(*example_state_data_t);
  auto example_state_data = mutable_example_state_data_t.matrix<float>();
  context->set_output("out_example_state_data", mutable_example_state_data_t);

  if (options.adaptative) {
    OP_REQUIRES_OK(context,
                   examples.SampleAdaptativeProbabilities(
                       options.num_loss_partitions, options.regularizations,
                       model_weights, example_state_data, options.loss_updater,
                       /*num_weight_vectors =*/1));
  }

  mutex mu;
  Status train_step_status GUARDED_BY(mu);
  std::atomic<std::int64_t> atomic_index(-1);
  auto train_step = [&](const int64 begin, const int64 end) {
    // The static_cast here is safe since begin and end can be at most
    // num_examples which is an int.
    for (int id = static_cast<int>(begin); id < end; ++id) {
      const int64 example_index =
          examples.sampled_index(++atomic_index, options.adaptative);
      const Example& example = examples.example(example_index);
      const float dual = example_state_data(example_index, 0);
      const float example_weight = example.example_weight();
      float example_label = example.example_label();
      const Status conversion_status =
          options.loss_updater->ConvertLabel(&example_label);
      if (!conversion_status.ok()) {
        mutex_lock l(mu);
        train_step_status = conversion_status;
        // Return from this worker thread - the calling thread is
        // responsible for checking context status and returning on error.
        return;
      }

      // Compute wx, example norm weighted by regularization, dual loss,
      // primal loss.
      // For binary SDCA, num_weight_vectors should be one.
      const ExampleStatistics example_statistics =
          example.ComputeWxAndWeightedExampleNorm(
              options.num_loss_partitions, model_weights,
              options.regularizations, 1 /* num_weight_vectors */);

      const double new_dual = options.loss_updater->ComputeUpdatedDual(
          options.num_loss_partitions, example_label, example_weight, dual,
          example_statistics.wx[0], example_statistics.normalized_squared_norm);

      // Compute new weights.
      const double normalized_bounded_dual_delta =
          (new_dual - dual) * example_weight /
          options.regularizations.symmetric_l2();
      model_weights.UpdateDeltaWeights(
          context->eigen_cpu_device(), example,
          std::vector<double>{normalized_bounded_dual_delta});

      // Update example data.
      example_state_data(example_index, 0) = new_dual;
      example_state_data(example_index, 1) =
          options.loss_updater->ComputePrimalLoss(
              example_statistics.prev_wx[0], example_label, example_weight);
      example_state_data(example_index, 2) =
          options.loss_updater->ComputeDualLoss(dual, example_label,
                                                example_weight);
      example_state_data(example_index, 3) = example_weight;
    }
  };
  // TODO(sibyl-Aix6ihai): Tune this properly based on sparsity of the data,
  // number of cpus, and cost per example.
  const int64 kCostPerUnit = examples.num_features();
  const DeviceBase::CpuWorkerThreads& worker_threads =
      *context->device()->tensorflow_cpu_worker_threads();

  Shard(worker_threads.num_threads, worker_threads.workers,
        examples.num_examples(), kCostPerUnit, train_step);
  OP_REQUIRES_OK(context, train_step_status);
}

}  // namespace

class SdcaOptimizer : public OpKernel {
 public:
  explicit SdcaOptimizer(OpKernelConstruction* const context)
      : OpKernel(context), options_(context) {}

  void Compute(OpKernelContext* context) override {
    DoCompute(options_, context);
  }

 private:
  // TODO(sibyl-Aix6ihai): We could use the type-constraint on loss_type, and
  // template the entire class to avoid the virtual table lookup penalty in
  // the inner loop.
  ComputeOptions options_;
};
REGISTER_KERNEL_BUILDER(Name("SdcaOptimizer").Device(DEVICE_CPU),
                        SdcaOptimizer);

class SdcaShrinkL1 : public OpKernel {
 public:
  explicit SdcaShrinkL1(OpKernelConstruction* const context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, regularizations_.Initialize(context));
  }

  void Compute(OpKernelContext* context) override {
    OpMutableInputList weights_inputs;
    OP_REQUIRES_OK(context,
                   context->mutable_input_list("weights", &weights_inputs));

    auto do_work = [&](const int64 begin, const int64 end) {
      for (int i = begin; i < end; ++i) {
        auto prox_w = weights_inputs.at(i, /*lock_held=*/true).flat<float>();
        prox_w.device(context->eigen_cpu_device()) =
            regularizations_.EigenShrinkVector(prox_w);
      }
    };

    if (weights_inputs.size() > 0) {
      int64 num_weights = 0;
      for (int i = 0; i < weights_inputs.size(); ++i) {
        num_weights += weights_inputs.at(i, /*lock_held=*/true).NumElements();
      }
      // TODO(sibyl-Aix6ihai): Tune this value.
      const int64 kCostPerUnit = (num_weights * 50) / weights_inputs.size();
      const DeviceBase::CpuWorkerThreads& worker_threads =
          *context->device()->tensorflow_cpu_worker_threads();
      Shard(worker_threads.num_threads, worker_threads.workers,
            weights_inputs.size(), kCostPerUnit, do_work);
    }
  }

 private:
  Regularizations regularizations_;
};
REGISTER_KERNEL_BUILDER(Name("SdcaShrinkL1").Device(DEVICE_CPU), SdcaShrinkL1);

// Computes platform independent, compact and unique (with very high
// probability) representation of an example id. It shouldn't be put in
// persistent storage, as its implementation may change in the future.
//
// The current probability of at least one collision for 1B example_ids is
// approximately 10^-21 (ie 2^60 / 2^129).
class SdcaFprint : public OpKernel {
 public:
  explicit SdcaFprint(OpKernelConstruction* const context)
      : OpKernel(context) {}

  void Compute(OpKernelContext* context) override {
    const Tensor& input = context->input(0);
    OP_REQUIRES(context, TensorShapeUtils::IsVector(input.shape()),
                errors::InvalidArgument("Input must be a vector, got shape ",
                                        input.shape().DebugString()));
    Tensor* out;
    const int64 num_elements = input.NumElements();
    OP_REQUIRES_OK(context, context->allocate_output(
                                0, TensorShape({num_elements, 2}), &out));

    const auto in_values = input.flat<string>();
    auto out_values = out->matrix<int64>();

    for (int64 i = 0; i < num_elements; ++i) {
      const Fprint128 fprint = Fingerprint128(in_values(i));
      // Never return 0 or 1 as the first value of the hash to allow these to
      // safely be used as sentinel values (e.g. dense hash table empty key).
      out_values(i, 0) = TF_PREDICT_TRUE(fprint.low64 >= 2)
                             ? fprint.low64
                             : fprint.low64 + ~static_cast<uint64>(1);
      out_values(i, 1) = fprint.high64;
    }
  }
};
REGISTER_KERNEL_BUILDER(Name("SdcaFprint").Device(DEVICE_CPU), SdcaFprint);

}  // namespace tensorflow
<|endoftext|>


<commit_before>#include "qmapboxgl.h"
#include "qmapboxgl_p.h"

#include <QMouseEvent>
#include <QOpenGLContext>
#include <QWheelEvent>

#include <mbgl/map/map.hpp>
#include <mbgl/platform/gl.hpp>
#include <mbgl/storage/default_file_source.hpp>

QMapboxGL::QMapboxGL(QWindow *parent)
    : QOpenGLWindow(QOpenGLWindow::NoPartialUpdate, parent)
    , d_ptr(new QMapboxGLPrivate(this))
{
    QSurfaceFormat format;
    format.setRedBufferSize(8);
    format.setGreenBufferSize(8);
    format.setBlueBufferSize(8);
    format.setStencilBufferSize(8);
    format.setAlphaBufferSize(8);
    format.setDepthBufferSize(16);

    setFormat(format);
}

QMapboxGL::~QMapboxGL()
{
    delete d_ptr;
}

void QMapboxGL::setAccessToken(const QString &token)
{
    d_ptr->map.setAccessToken(token.toUtf8().constData());
}

void QMapboxGL::setStyleJSON(const QString &style)
{
    d_ptr->map.setStyleJSON(style.toUtf8().constData());
}

double QMapboxGL::latitude() const
{
    return d_ptr->map.getLatLng().latitude;
}

void QMapboxGL::setLatitude(double latitude)
{
    d_ptr->map.setLatLng({latitude, longitude()});
}

double QMapboxGL::longitude() const
{
    return d_ptr->map.getLatLng().longitude;
}

void QMapboxGL::setLongitude(double longitude)
{
    d_ptr->map.setLatLng({latitude(), longitude});
}

double QMapboxGL::zoom() const
{
    return d_ptr->map.getZoom();
}

void QMapboxGL::setZoom(double zoom)
{
    d_ptr->map.setZoom(zoom);
}

void QMapboxGL::mousePressEvent(QMouseEvent *ev)
{
    if (!(ev->buttons() & Qt::LeftButton)) {
        return;
    }

    d_ptr->lastX = ev->x();
    d_ptr->lastY = ev->y();

    ev->accept();
}

void QMapboxGL::mouseMoveEvent(QMouseEvent *ev)
{
    if (!(ev->buttons() & Qt::LeftButton)) {
        return;
    }

    int dx = ev->x() - d_ptr->lastX;
    int dy = ev->y() - d_ptr->lastY;

    d_ptr->lastX = ev->x();
    d_ptr->lastY = ev->y();

    if (dx || dy) {
        d_ptr->map.moveBy(dx, dy);
    }

    ev->accept();
}

void QMapboxGL::wheelEvent(QWheelEvent *ev)
{
    QPoint numDegrees = ev->angleDelta();

    if (numDegrees.y() > 0) {
        d_ptr->map.scaleBy(1.10, ev->x(), ev->y());
    } else {
        d_ptr->map.scaleBy(0.90, ev->x(), ev->y());
    }

    ev->accept();
}

void QMapboxGL::resizeGL(int w, int h)
{
    // FIXME: Find out why sprites don't get rendered
    // correctly if I use 1.0 here.
    d_ptr->map.resize(w, h, 1.00001);
}

QMapboxGLPrivate::QMapboxGLPrivate(QMapboxGL *q)
    : fileSource(nullptr)
    , map(*this, fileSource)
    , q_ptr(q)
{
}

QMapboxGLPrivate::~QMapboxGLPrivate()
{
}

void QMapboxGLPrivate::deactivate()
{
    context->doneCurrent();
}

void QMapboxGLPrivate::invalidate(std::function<void()> renderCallback)
{
    if (!q_ptr->isExposed()) {
        return;
    }

    if (!context) {
        context = new QOpenGLContext();
        context->setFormat(q_ptr->requestedFormat());
        context->create();
        context->makeCurrent(q_ptr);

        mbgl::gl::InitializeExtensions([](const char *name) {
            QOpenGLContext *thisContext = QOpenGLContext::currentContext();
            return thisContext->getProcAddress(name);
        });
    }

    renderCallback();

    context->swapBuffers(q_ptr);
}
<commit_msg>Updated Mapboxgl API usage to ios-v0.3.2<commit_after>#include "qmapboxgl.h"
#include "qmapboxgl_p.h"

#include <QMouseEvent>
#include <QOpenGLContext>
#include <QWheelEvent>

#include <mbgl/map/map.hpp>
#include <mbgl/platform/gl.hpp>
#include <mbgl/storage/default_file_source.hpp>

QMapboxGL::QMapboxGL(QWindow *parent)
    : QOpenGLWindow(QOpenGLWindow::NoPartialUpdate, parent)
    , d_ptr(new QMapboxGLPrivate(this))
{
    QSurfaceFormat format;
    format.setRedBufferSize(8);
    format.setGreenBufferSize(8);
    format.setBlueBufferSize(8);
    format.setStencilBufferSize(8);
    format.setAlphaBufferSize(8);
    format.setDepthBufferSize(16);

    setFormat(format);
}

QMapboxGL::~QMapboxGL()
{
    delete d_ptr;
}

void QMapboxGL::setAccessToken(const QString &token)
{
    d_ptr->fileSource.setAccessToken(token.toUtf8().constData());
}

void QMapboxGL::setStyleJSON(const QString &style)
{
    d_ptr->map.setStyleJSON(style.toUtf8().constData());
}

double QMapboxGL::latitude() const
{
    return d_ptr->map.getLatLng().latitude;
}

void QMapboxGL::setLatitude(double latitude)
{
    d_ptr->map.setLatLng({latitude, longitude()});
}

double QMapboxGL::longitude() const
{
    return d_ptr->map.getLatLng().longitude;
}

void QMapboxGL::setLongitude(double longitude)
{
    d_ptr->map.setLatLng({latitude(), longitude});
}

double QMapboxGL::zoom() const
{
    return d_ptr->map.getZoom();
}

void QMapboxGL::setZoom(double zoom)
{
    d_ptr->map.setZoom(zoom);
}

void QMapboxGL::mousePressEvent(QMouseEvent *ev)
{
    if (!(ev->buttons() & Qt::LeftButton)) {
        return;
    }

    d_ptr->lastX = ev->x();
    d_ptr->lastY = ev->y();

    ev->accept();
}

void QMapboxGL::mouseMoveEvent(QMouseEvent *ev)
{
    if (!(ev->buttons() & Qt::LeftButton)) {
        return;
    }

    int dx = ev->x() - d_ptr->lastX;
    int dy = ev->y() - d_ptr->lastY;

    d_ptr->lastX = ev->x();
    d_ptr->lastY = ev->y();

    if (dx || dy) {
        d_ptr->map.moveBy(dx, dy);
    }

    ev->accept();
}

void QMapboxGL::wheelEvent(QWheelEvent *ev)
{
    QPoint numDegrees = ev->angleDelta();

    if (numDegrees.y() > 0) {
        d_ptr->map.scaleBy(1.10, ev->x(), ev->y());
    } else {
        d_ptr->map.scaleBy(0.90, ev->x(), ev->y());
    }

    ev->accept();
}

void QMapboxGL::resizeGL(int w, int h)
{
    // FIXME: Find out why sprites don't get rendered
    // correctly if I use 1.0 here.
    d_ptr->map.resize(w, h, 1.00001);
}

QMapboxGLPrivate::QMapboxGLPrivate(QMapboxGL *q)
    : fileSource(nullptr)
    , map(*this, fileSource)
    , q_ptr(q)
{
}

QMapboxGLPrivate::~QMapboxGLPrivate()
{
}

void QMapboxGLPrivate::deactivate()
{
    context->doneCurrent();
}

void QMapboxGLPrivate::invalidate(std::function<void()> renderCallback)
{
    if (!q_ptr->isExposed()) {
        return;
    }

    if (!context) {
        context = new QOpenGLContext();
        context->setFormat(q_ptr->requestedFormat());
        context->create();
        context->makeCurrent(q_ptr);

        mbgl::gl::InitializeExtensions([](const char *name) {
            QOpenGLContext *thisContext = QOpenGLContext::currentContext();
            return thisContext->getProcAddress(name);
        });
    }

    renderCallback();

    context->swapBuffers(q_ptr);
}
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>

#include "dbn/rbm.hpp"
#include "dbn/layer.hpp"
#include "dbn/conf.hpp"

#include "mnist/mnist_reader.hpp"

#include "utils.hpp"

int main(int argc, char* argv[]){
    auto reconstruction = false;
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "sample"){
            reconstruction = true;
        }

        if(command == "load"){
            load = true;
        }
    }

    dbn::rbm<dbn::layer<
        dbn::conf<true, 50, true, false, true, dbn::Type::SIGMOID, dbn::Type::RLU>,
        28 * 28, 100>> rbm;

    auto training_images = mnist::read_training_images<std::vector, vector, double>();

    binarize_each(training_images);

    if(load){
        std::ifstream is("rbm-1.dat", std::ofstream::binary);
        rbm.load(is);
    } else {
        rbm.train(training_images, 10);

        std::ofstream os("rbm-1.dat", std::ofstream::binary);
        rbm.store(os);
    }

    if(reconstruction){
        auto test_images = mnist::read_test_images<std::vector, vector, double>();
        binarize_each(test_images);

        for(size_t t = 0; t < 10; ++t){
            auto& image = test_images[666 + t];

            std::cout << "Source image" << std::endl;
            for(size_t i = 0; i < 28; ++i){
                for(size_t j = 0; j < 28; ++j){
                    std::cout << static_cast<size_t>(image[i * 28 + j]) << " ";
                }
                std::cout << std::endl;
            }

            rbm.reconstruct(image);

            std::cout << "Reconstructed image" << std::endl;
            rbm.display_visible_units(28);
        }
    }

    return 0;
}
<commit_msg>Replace RLU with nRLU<commit_after>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>

#include "dbn/rbm.hpp"
#include "dbn/layer.hpp"
#include "dbn/conf.hpp"

#include "mnist/mnist_reader.hpp"

#include "utils.hpp"

int main(int argc, char* argv[]){
    auto reconstruction = false;
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "sample"){
            reconstruction = true;
        }

        if(command == "load"){
            load = true;
        }
    }

    dbn::rbm<dbn::layer<
        dbn::conf<true, 50, true, false, true, dbn::Type::SIGMOID, dbn::Type::NRLU>,
        28 * 28, 100>> rbm;

    auto training_images = mnist::read_training_images<std::vector, vector, double>();

    binarize_each(training_images);

    if(load){
        std::ifstream is("rbm-1.dat", std::ofstream::binary);
        rbm.load(is);
    } else {
        rbm.train(training_images, 10);

        std::ofstream os("rbm-1.dat", std::ofstream::binary);
        rbm.store(os);
    }

    if(reconstruction){
        auto test_images = mnist::read_test_images<std::vector, vector, double>();
        binarize_each(test_images);

        for(size_t t = 0; t < 10; ++t){
            auto& image = test_images[666 + t];

            std::cout << "Source image" << std::endl;
            for(size_t i = 0; i < 28; ++i){
                for(size_t j = 0; j < 28; ++j){
                    std::cout << static_cast<size_t>(image[i * 28 + j]) << " ";
                }
                std::cout << std::endl;
            }

            rbm.reconstruct(image);

            std::cout << "Reconstructed image" << std::endl;
            rbm.display_visible_units(28);
        }
    }

    return 0;
}
<|endoftext|>


<commit_before>#include <stdlib.h>
#include <limits.h>
#include "redis_conn.h"
#include "pink_define.h"
#include "worker_thread.h"
#include "xdebug.h"

#include <string>

namespace pink {
static bool IsHexDigit(char ch) {
  return (ch>='0' && ch<='9') || (ch>='a' && ch<='f') || (ch>='A' && ch<'F');
}

static int HexDigitToInt32(char ch) {
  if (ch <= '9' && ch >= '0') {
    return ch-'0';
  } else if (ch <= 'F' && ch >= 'A') {
    return ch-'A';
  } else if (ch <= 'f' && ch >= 'a') {
    return ch-'a';
  } else {
    return 0;
  }
}
static int split2args(const std::string& req_buf, RedisCmdArgsType& argv) {
  const char *p = req_buf.data();
  std::string arg;

  while(1) {
    /* skip blanks */
    while(*p && isspace(*p)) p++;
    if (*p) {
      /* get a token */
      int inq=0;  /* set to 1 if we are in "quotes" */
      int insq=0; /* set to 1 if we are in 'single quotes' */
      int done=0;

      arg.clear();
      while(!done) {
        if (inq) {
          if (*p == '\\' && *(p+1) == 'x' &&
              IsHexDigit(*(p+2)) &&
              IsHexDigit(*(p+3))) {
            unsigned char byte = HexDigitToInt32(*(p+2))*16 + HexDigitToInt32(*(p+3));
            arg.append(1, byte);
            p += 3;
          } else if (*p == '\\' && *(p+1)) {
            char c;

            p++;
            switch(*p) {
              case 'n': c = '\n'; break;
              case 'r': c = '\r'; break;
              case 't': c = '\t'; break;
              case 'b': c = '\b'; break;
              case 'a': c = '\a'; break;
              default: c = *p; break;
            }
            arg.append(1, c);
          } else if (*p == '"') {
            /* closing quote must be followed by a space or
            * nothing at all. */
            if (*(p+1) && !isspace(*(p+1))) {
              argv.clear();
              return -1;
            };
            done=1;
          } else if (!*p) {
            /* unterminated quotes */
            argv.clear();
            return -1;
          } else {
            arg.append(1, *p);
          }
        } else if (insq) {
          if (*p == '\\' && *(p+1) == '\'') {
            p++;
            arg.append(1, '\'');
          } else if (*p == '\'') {
            /* closing quote must be followed by a space or
            * nothing at all. */
            if (*(p+1) && !isspace(*(p+1))) {
              argv.clear();
              return -1;
            }
            done=1;
          } else if (!*p) {
            /* unterminated quotes */
            argv.clear();
            return -1;
          } else {
            arg.append(1, *p);
          }
        } else {
          switch(*p) {
            case ' ':
            case '\n':
            case '\r':
            case '\t':
            case '\0':
              done=1;
            break;
            case '"':
              inq=1;
            break;
            case '\'':
              insq=1;
            break;
            default:
              //current = sdscatlen(current,p,1);
              arg.append(1, *p);
              break;
          }
        }
        if (*p) p++;
      }
      argv.push_back(arg);
    } else {
      return 0;
    }
  }
}

RedisConn::RedisConn(const int fd, const std::string &ip_port) :
  PinkConn(fd, ip_port),
  wbuf_size_(REDIS_MAX_MESSAGE),
  last_read_pos_(-1),
  next_parse_pos_(0),
  req_type_(0),
  multibulk_len_(0),
  bulk_len_(-1),
  is_find_sep_(true),
  is_overtake_(false),
  wbuf_pos_(0),
  wbuf_len_(0)
{
  rbuf_ = (char *)malloc(sizeof(char) * REDIS_MAX_MESSAGE);
  wbuf_ = (char *)malloc(sizeof(char) * REDIS_MAX_MESSAGE);
}

RedisConn::~RedisConn()
{
  free(wbuf_);
  free(rbuf_);
}

ReadStatus RedisConn::ProcessInlineBuffer() {
  int32_t pos, ret;
  pos = FindNextSeparators();
  if (pos == -1 && (last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
    return kFullError;
  }
  if (pos == -1 && (last_read_pos_ + 1) % REDIS_MAX_MESSAGE != next_parse_pos_) {
    return kReadHalf;
  }
  std::string req_buf;
  if (pos > next_parse_pos_) {
    req_buf = std::string(rbuf_ + next_parse_pos_, pos - next_parse_pos_-1);
  } else if (pos == 0) {
    req_buf = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_ - 1);
  } else {
    req_buf = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_) + std::string(rbuf_, pos - 1);
  }
  argv_.clear();
  ret = split2args(req_buf, argv_);
  next_parse_pos_ = (pos+1)%REDIS_MAX_MESSAGE;
  if (ret == -1) {
    return kParseError;
  }

  if ((last_read_pos_+1)%REDIS_MAX_MESSAGE == next_parse_pos_) {
    is_overtake_ = true;
  }
  return kReadAll;
}

ReadStatus RedisConn::ProcessMultibulkBuffer() {
  int32_t pos;
  if (multibulk_len_ == 0) {
    /* The client should have been reset */;

    pos = FindNextSeparators();
    if (pos != -1) {
        if (GetNextNum(pos, &multibulk_len_) != 0) {
            //Protocol error: invalid multibulk length
            return kParseError;
        }
      next_parse_pos_ = (pos + 1) % REDIS_MAX_MESSAGE;
      argv_.clear();
      if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
        return kReadHalf;
      }
    } else {
      if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
        return kFullError; /*FULL_ERROR*/
      } else {
        return kReadHalf; /*HALF*/
      }
    }
  }
  std::string tmp;
  while (!is_overtake_ && multibulk_len_) {
    if (bulk_len_ == -1) {
      pos = FindNextSeparators();
      if (pos != -1) {
        if (rbuf_[next_parse_pos_] != '$') {
           return kParseError;//PARSE_ERROR
        }

        if (GetNextNum(pos, &bulk_len_) != 0) {
            //Protocol error: invalid bulk length
            return kParseError;
        }
        next_parse_pos_ = (pos + 1) % REDIS_MAX_MESSAGE;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kReadHalf;
        }
      } else {
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kFullError; /*FULL_ERROR*/
        } else {
          return kReadHalf; /*HALF*/
        }
      }
    }
    if (next_parse_pos_ <= last_read_pos_) {
      if (last_read_pos_ - next_parse_pos_ + 1 < bulk_len_ + 2) {
        break;
      } else {
        argv_.push_back(std::string(rbuf_ + next_parse_pos_, bulk_len_));
        next_parse_pos_ = (next_parse_pos_ + (bulk_len_ + 2)) % REDIS_MAX_MESSAGE;
        bulk_len_ = -1;
        multibulk_len_--;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          is_overtake_ = true;
        }
      }
    } else {
      if (REDIS_MAX_MESSAGE - next_parse_pos_  + last_read_pos_ + 1 < bulk_len_ + 2) {
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kFullError;
        }
        break;
      } else {
        if (REDIS_MAX_MESSAGE - next_parse_pos_ >= bulk_len_) {
          argv_.push_back(std::string(rbuf_ + next_parse_pos_, bulk_len_));
        } else {
          tmp = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_) + std::string(rbuf_, bulk_len_ - (REDIS_MAX_MESSAGE - next_parse_pos_));
          argv_.push_back(tmp);
        }
        next_parse_pos_ = bulk_len_ - (REDIS_MAX_MESSAGE - next_parse_pos_) + 2;
        bulk_len_ = -1;
        multibulk_len_--;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          is_overtake_ = true;
        }
      }
    }
  }

  if (multibulk_len_ == 0) {
    return kReadAll; /*OK*/
  } else {
    return kReadHalf; /*HALF*/
  }
}

void RedisConn::ResetClient() {
    argv_.clear();
    req_type_ = 0;
    multibulk_len_ = 0;
    bulk_len_ = -1;
}

bool RedisConn::ExpandWbuf() {
  if (wbuf_size_ >= REDIS_MAX_MESSAGE * 32) {
    return false;
  }
  wbuf_size_ *= 2;
  wbuf_ = (char*)realloc(wbuf_, wbuf_size_);
  return true;
}

ReadStatus RedisConn::ProcessInputBuffer() {
  ReadStatus ret;
  while (!is_overtake_) {
    if (!req_type_) {
      if (rbuf_[next_parse_pos_] == '*') {
        req_type_ = REDIS_REQ_MULTIBULK;
      } else {
        req_type_ = REDIS_REQ_INLINE;
      }
    }

    if (req_type_ == REDIS_REQ_INLINE) {
      ret = ProcessInlineBuffer();
      if (ret != kReadAll) {
        return ret;
      }
    } else if (req_type_ == REDIS_REQ_MULTIBULK) {
      ret = ProcessMultibulkBuffer();
      if (ret != kReadAll/*OK*/) { //FULL_ERROR || HALF || PARSE_ERROR
        return ret;
      }
    } else {
      //Unknown requeset type;
      return kParseError;
    }

    if (argv_.size() == 0) {
      ResetClient();
    } else {
      DealMessage();
    }
  }
  req_type_ = 0;
  next_parse_pos_ = 0;
  last_read_pos_ = -1;
  return kReadAll;/*OK*/
}

ReadStatus RedisConn::GetRequest()
{
  ssize_t nread = 0;
  int32_t next_read_pos = (last_read_pos_ + 1) % REDIS_MAX_MESSAGE;
  int32_t read_len = 0;
  if (next_read_pos == next_parse_pos_ && !is_find_sep_) {
    //too big message, close client;
    //err_msg_ = "-ERR: Protocol error: too big mbulk count string\r\n";
    return kParseError;
  } else if (next_read_pos >= next_parse_pos_) {
    read_len = REDIS_IOBUF_LEN < (REDIS_MAX_MESSAGE - next_read_pos) ? REDIS_IOBUF_LEN : (REDIS_MAX_MESSAGE - next_read_pos);
  } else if (next_read_pos < next_parse_pos_) {
    read_len = next_parse_pos_ - next_read_pos;
  }

  nread = read(fd(), rbuf_ + next_read_pos, read_len);
  if (nread == -1) {
    if (errno == EAGAIN) {
      nread = 0;
      return kReadHalf; //HALF
    } else {
      // error happened, close client
      return kReadError;
    }
  } else if (nread == 0) {
    // client closed, close client
    return kReadClose;
  }

  if (nread) {
    last_read_pos_ = (last_read_pos_ + nread) % REDIS_MAX_MESSAGE;
    is_overtake_ = false;
  }
  ReadStatus ret = ProcessInputBuffer();
  if (ret == kFullError/*FULL_ERROR*/) {
    is_find_sep_ = false;
  }
  return ret; //OK || HALF || FULL_ERROR || PARSE_ERROR
}

WriteStatus RedisConn::SendReply()
{
  ssize_t nwritten = 0;
  while (wbuf_len_ > 0) {
    nwritten = write(fd(), wbuf_ + wbuf_pos_, wbuf_len_ - wbuf_pos_);
    if (nwritten <= 0) {
      break;
    }
    wbuf_pos_ += nwritten;
    if (wbuf_pos_ == wbuf_len_) {
      wbuf_len_ = 0;
      wbuf_pos_ = 0;
    }
  }
  if (nwritten == -1) {
    if (errno == EAGAIN) {
      return kWriteHalf;
    } else {
      // Here we should close the connection
      return kWriteError;
    }
  }
  if (wbuf_len_ == 0) {
    return kWriteAll;
  } else {
    return kWriteHalf;
  }
}

int32_t RedisConn::FindNextSeparators() {
  int pos;
  if (next_parse_pos_ <= last_read_pos_) {
    pos = next_parse_pos_;
  } else {
    pos = 0;
  }
  while (pos <= last_read_pos_) {
    if (rbuf_[pos] == '\n') {
      return pos;
    }
    pos++;
  }
  return -1;
}

int32_t RedisConn::GetNextNum(int32_t pos, int32_t *value) {
  std::string tmp;
  if (pos > next_parse_pos_) {
    // [next_parse_pos_ + 1, pos - next_parse_pos_- 2]
    tmp = std::string(rbuf_ + next_parse_pos_ + 1, pos - next_parse_pos_ - 2);
  } else {
    if (pos != 0) {
      tmp = std::string(rbuf_ + ((next_parse_pos_ + 1) % REDIS_MAX_MESSAGE), REDIS_MAX_MESSAGE - next_parse_pos_ - 1) + std::string(rbuf_, pos - 1);
    } else {
      tmp = std::string(rbuf_ + ((next_parse_pos_ + 1) % REDIS_MAX_MESSAGE), REDIS_MAX_MESSAGE - next_parse_pos_ - 1);
    }
    // [next_parse_pos_ + 1, REDIS_MAX_MESSAGE - next_parse_pos_ - 1] + [0, pos - 1]
  }

  char* end;
  errno = 0;
  long num = strtol(tmp.c_str(), &end, 10);
  if ((num == 0 && errno == EINVAL) ||
          ((num == LONG_MAX || num == LONG_MIN) && errno == ERANGE)) {
    return -1;
  }

  if (value) *value = static_cast<int32_t>(num);
  return 0;
}
}
<commit_msg>bugfix: reset wbuf_pos_ = 0 when expandwbuf error<commit_after>#include <stdlib.h>
#include <limits.h>
#include "redis_conn.h"
#include "pink_define.h"
#include "worker_thread.h"
#include "xdebug.h"

#include <string>

namespace pink {
static bool IsHexDigit(char ch) {
  return (ch>='0' && ch<='9') || (ch>='a' && ch<='f') || (ch>='A' && ch<'F');
}

static int HexDigitToInt32(char ch) {
  if (ch <= '9' && ch >= '0') {
    return ch-'0';
  } else if (ch <= 'F' && ch >= 'A') {
    return ch-'A';
  } else if (ch <= 'f' && ch >= 'a') {
    return ch-'a';
  } else {
    return 0;
  }
}
static int split2args(const std::string& req_buf, RedisCmdArgsType& argv) {
  const char *p = req_buf.data();
  std::string arg;

  while(1) {
    /* skip blanks */
    while(*p && isspace(*p)) p++;
    if (*p) {
      /* get a token */
      int inq=0;  /* set to 1 if we are in "quotes" */
      int insq=0; /* set to 1 if we are in 'single quotes' */
      int done=0;

      arg.clear();
      while(!done) {
        if (inq) {
          if (*p == '\\' && *(p+1) == 'x' &&
              IsHexDigit(*(p+2)) &&
              IsHexDigit(*(p+3))) {
            unsigned char byte = HexDigitToInt32(*(p+2))*16 + HexDigitToInt32(*(p+3));
            arg.append(1, byte);
            p += 3;
          } else if (*p == '\\' && *(p+1)) {
            char c;

            p++;
            switch(*p) {
              case 'n': c = '\n'; break;
              case 'r': c = '\r'; break;
              case 't': c = '\t'; break;
              case 'b': c = '\b'; break;
              case 'a': c = '\a'; break;
              default: c = *p; break;
            }
            arg.append(1, c);
          } else if (*p == '"') {
            /* closing quote must be followed by a space or
            * nothing at all. */
            if (*(p+1) && !isspace(*(p+1))) {
              argv.clear();
              return -1;
            };
            done=1;
          } else if (!*p) {
            /* unterminated quotes */
            argv.clear();
            return -1;
          } else {
            arg.append(1, *p);
          }
        } else if (insq) {
          if (*p == '\\' && *(p+1) == '\'') {
            p++;
            arg.append(1, '\'');
          } else if (*p == '\'') {
            /* closing quote must be followed by a space or
            * nothing at all. */
            if (*(p+1) && !isspace(*(p+1))) {
              argv.clear();
              return -1;
            }
            done=1;
          } else if (!*p) {
            /* unterminated quotes */
            argv.clear();
            return -1;
          } else {
            arg.append(1, *p);
          }
        } else {
          switch(*p) {
            case ' ':
            case '\n':
            case '\r':
            case '\t':
            case '\0':
              done=1;
            break;
            case '"':
              inq=1;
            break;
            case '\'':
              insq=1;
            break;
            default:
              //current = sdscatlen(current,p,1);
              arg.append(1, *p);
              break;
          }
        }
        if (*p) p++;
      }
      argv.push_back(arg);
    } else {
      return 0;
    }
  }
}

RedisConn::RedisConn(const int fd, const std::string &ip_port) :
  PinkConn(fd, ip_port),
  wbuf_size_(REDIS_MAX_MESSAGE),
  last_read_pos_(-1),
  next_parse_pos_(0),
  req_type_(0),
  multibulk_len_(0),
  bulk_len_(-1),
  is_find_sep_(true),
  is_overtake_(false),
  wbuf_pos_(0),
  wbuf_len_(0)
{
  rbuf_ = (char *)malloc(sizeof(char) * REDIS_MAX_MESSAGE);
  wbuf_ = (char *)malloc(sizeof(char) * REDIS_MAX_MESSAGE);
}

RedisConn::~RedisConn()
{
  free(wbuf_);
  free(rbuf_);
}

ReadStatus RedisConn::ProcessInlineBuffer() {
  int32_t pos, ret;
  pos = FindNextSeparators();
  if (pos == -1 && (last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
    return kFullError;
  }
  if (pos == -1 && (last_read_pos_ + 1) % REDIS_MAX_MESSAGE != next_parse_pos_) {
    return kReadHalf;
  }
  std::string req_buf;
  if (pos > next_parse_pos_) {
    req_buf = std::string(rbuf_ + next_parse_pos_, pos - next_parse_pos_-1);
  } else if (pos == 0) {
    req_buf = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_ - 1);
  } else {
    req_buf = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_) + std::string(rbuf_, pos - 1);
  }
  argv_.clear();
  ret = split2args(req_buf, argv_);
  next_parse_pos_ = (pos+1)%REDIS_MAX_MESSAGE;
  if (ret == -1) {
    return kParseError;
  }

  if ((last_read_pos_+1)%REDIS_MAX_MESSAGE == next_parse_pos_) {
    is_overtake_ = true;
  }
  return kReadAll;
}

ReadStatus RedisConn::ProcessMultibulkBuffer() {
  int32_t pos;
  if (multibulk_len_ == 0) {
    /* The client should have been reset */;

    pos = FindNextSeparators();
    if (pos != -1) {
        if (GetNextNum(pos, &multibulk_len_) != 0) {
            //Protocol error: invalid multibulk length
            return kParseError;
        }
      next_parse_pos_ = (pos + 1) % REDIS_MAX_MESSAGE;
      argv_.clear();
      if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
        return kReadHalf;
      }
    } else {
      if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
        return kFullError; /*FULL_ERROR*/
      } else {
        return kReadHalf; /*HALF*/
      }
    }
  }
  std::string tmp;
  while (!is_overtake_ && multibulk_len_) {
    if (bulk_len_ == -1) {
      pos = FindNextSeparators();
      if (pos != -1) {
        if (rbuf_[next_parse_pos_] != '$') {
           return kParseError;//PARSE_ERROR
        }

        if (GetNextNum(pos, &bulk_len_) != 0) {
            //Protocol error: invalid bulk length
            return kParseError;
        }
        next_parse_pos_ = (pos + 1) % REDIS_MAX_MESSAGE;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kReadHalf;
        }
      } else {
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kFullError; /*FULL_ERROR*/
        } else {
          return kReadHalf; /*HALF*/
        }
      }
    }
    if (next_parse_pos_ <= last_read_pos_) {
      if (last_read_pos_ - next_parse_pos_ + 1 < bulk_len_ + 2) {
        break;
      } else {
        argv_.push_back(std::string(rbuf_ + next_parse_pos_, bulk_len_));
        next_parse_pos_ = (next_parse_pos_ + (bulk_len_ + 2)) % REDIS_MAX_MESSAGE;
        bulk_len_ = -1;
        multibulk_len_--;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          is_overtake_ = true;
        }
      }
    } else {
      if (REDIS_MAX_MESSAGE - next_parse_pos_  + last_read_pos_ + 1 < bulk_len_ + 2) {
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          return kFullError;
        }
        break;
      } else {
        if (REDIS_MAX_MESSAGE - next_parse_pos_ >= bulk_len_) {
          argv_.push_back(std::string(rbuf_ + next_parse_pos_, bulk_len_));
        } else {
          tmp = std::string(rbuf_ + next_parse_pos_, REDIS_MAX_MESSAGE - next_parse_pos_) + std::string(rbuf_, bulk_len_ - (REDIS_MAX_MESSAGE - next_parse_pos_));
          argv_.push_back(tmp);
        }
        next_parse_pos_ = bulk_len_ - (REDIS_MAX_MESSAGE - next_parse_pos_) + 2;
        bulk_len_ = -1;
        multibulk_len_--;
        if ((last_read_pos_ + 1) % REDIS_MAX_MESSAGE == next_parse_pos_) {
          is_overtake_ = true;
        }
      }
    }
  }

  if (multibulk_len_ == 0) {
    return kReadAll; /*OK*/
  } else {
    return kReadHalf; /*HALF*/
  }
}

void RedisConn::ResetClient() {
    argv_.clear();
    req_type_ = 0;
    multibulk_len_ = 0;
    bulk_len_ = -1;
}

bool RedisConn::ExpandWbuf() {
  if (wbuf_size_ >= REDIS_MAX_MESSAGE * 32) {
    wbuf_pos_ = 0;
    return false;
  }
  wbuf_size_ *= 2;
  wbuf_ = (char*)realloc(wbuf_, wbuf_size_);
  return true;
}

ReadStatus RedisConn::ProcessInputBuffer() {
  ReadStatus ret;
  while (!is_overtake_) {
    if (!req_type_) {
      if (rbuf_[next_parse_pos_] == '*') {
        req_type_ = REDIS_REQ_MULTIBULK;
      } else {
        req_type_ = REDIS_REQ_INLINE;
      }
    }

    if (req_type_ == REDIS_REQ_INLINE) {
      ret = ProcessInlineBuffer();
      if (ret != kReadAll) {
        return ret;
      }
    } else if (req_type_ == REDIS_REQ_MULTIBULK) {
      ret = ProcessMultibulkBuffer();
      if (ret != kReadAll/*OK*/) { //FULL_ERROR || HALF || PARSE_ERROR
        return ret;
      }
    } else {
      //Unknown requeset type;
      return kParseError;
    }

    if (argv_.size() == 0) {
      ResetClient();
    } else {
      DealMessage();
    }
  }
  req_type_ = 0;
  next_parse_pos_ = 0;
  last_read_pos_ = -1;
  return kReadAll;/*OK*/
}

ReadStatus RedisConn::GetRequest()
{
  ssize_t nread = 0;
  int32_t next_read_pos = (last_read_pos_ + 1) % REDIS_MAX_MESSAGE;
  int32_t read_len = 0;
  if (next_read_pos == next_parse_pos_ && !is_find_sep_) {
    //too big message, close client;
    //err_msg_ = "-ERR: Protocol error: too big mbulk count string\r\n";
    return kParseError;
  } else if (next_read_pos >= next_parse_pos_) {
    read_len = REDIS_IOBUF_LEN < (REDIS_MAX_MESSAGE - next_read_pos) ? REDIS_IOBUF_LEN : (REDIS_MAX_MESSAGE - next_read_pos);
  } else if (next_read_pos < next_parse_pos_) {
    read_len = next_parse_pos_ - next_read_pos;
  }

  nread = read(fd(), rbuf_ + next_read_pos, read_len);
  if (nread == -1) {
    if (errno == EAGAIN) {
      nread = 0;
      return kReadHalf; //HALF
    } else {
      // error happened, close client
      return kReadError;
    }
  } else if (nread == 0) {
    // client closed, close client
    return kReadClose;
  }

  if (nread) {
    last_read_pos_ = (last_read_pos_ + nread) % REDIS_MAX_MESSAGE;
    is_overtake_ = false;
  }
  ReadStatus ret = ProcessInputBuffer();
  if (ret == kFullError/*FULL_ERROR*/) {
    is_find_sep_ = false;
  }
  return ret; //OK || HALF || FULL_ERROR || PARSE_ERROR
}

WriteStatus RedisConn::SendReply()
{
  ssize_t nwritten = 0;
  while (wbuf_len_ > 0) {
    nwritten = write(fd(), wbuf_ + wbuf_pos_, wbuf_len_ - wbuf_pos_);
    if (nwritten <= 0) {
      break;
    }
    wbuf_pos_ += nwritten;
    if (wbuf_pos_ == wbuf_len_) {
      wbuf_len_ = 0;
      wbuf_pos_ = 0;
    }
  }
  if (nwritten == -1) {
    if (errno == EAGAIN) {
      return kWriteHalf;
    } else {
      // Here we should close the connection
      return kWriteError;
    }
  }
  if (wbuf_len_ == 0) {
    return kWriteAll;
  } else {
    return kWriteHalf;
  }
}

int32_t RedisConn::FindNextSeparators() {
  int pos;
  if (next_parse_pos_ <= last_read_pos_) {
    pos = next_parse_pos_;
  } else {
    pos = 0;
  }
  while (pos <= last_read_pos_) {
    if (rbuf_[pos] == '\n') {
      return pos;
    }
    pos++;
  }
  return -1;
}

int32_t RedisConn::GetNextNum(int32_t pos, int32_t *value) {
  std::string tmp;
  if (pos > next_parse_pos_) {
    // [next_parse_pos_ + 1, pos - next_parse_pos_- 2]
    tmp = std::string(rbuf_ + next_parse_pos_ + 1, pos - next_parse_pos_ - 2);
  } else {
    if (pos != 0) {
      tmp = std::string(rbuf_ + ((next_parse_pos_ + 1) % REDIS_MAX_MESSAGE), REDIS_MAX_MESSAGE - next_parse_pos_ - 1) + std::string(rbuf_, pos - 1);
    } else {
      tmp = std::string(rbuf_ + ((next_parse_pos_ + 1) % REDIS_MAX_MESSAGE), REDIS_MAX_MESSAGE - next_parse_pos_ - 1);
    }
    // [next_parse_pos_ + 1, REDIS_MAX_MESSAGE - next_parse_pos_ - 1] + [0, pos - 1]
  }

  char* end;
  errno = 0;
  long num = strtol(tmp.c_str(), &end, 10);
  if ((num == 0 && errno == EINVAL) ||
          ((num == LONG_MAX || num == LONG_MIN) && errno == ERANGE)) {
    return -1;
  }

  if (value) *value = static_cast<int32_t>(num);
  return 0;
}
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 1994-2019 Leonid Yuriev <leo@yuriev.ru>.
 *  https://github.com/leo-yuriev/erthink
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#ifdef _MSC_VER
#define _USE_MATH_DEFINES
#endif

#include "erthink_d2a.h"
#include "erthink_defs.h"
#include "testing.h"

#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4710) /* function not inlined */
#endif
#include <cfloat> // for FLT_MAX, etc
#include <cmath>  // for M_PI, etc
#ifdef _MSC_VER
#pragma warning(pop)
#endif

__hot __dll_export __noinline char *_d2a(const double value, char *ptr) {
  return erthink::d2a(value, ptr);
}

//------------------------------------------------------------------------------

static void probe_d2a(char (&buffer)[23 + 1], const double value) {
  char *d2a_end = _d2a(value, buffer);
  ASSERT_LT(buffer, d2a_end);
  ASSERT_GT(erthink::array_end(buffer), d2a_end);
  *d2a_end = '\0';

  char *strtod_end = nullptr;
  double probe = strtod(buffer, &strtod_end);
  EXPECT_EQ(d2a_end, strtod_end);
  EXPECT_EQ(value, probe);
}

TEST(d2a, trivia) {
  char buffer[23 + 1];
  char *end = _d2a(0, buffer);
  EXPECT_EQ(1, end - buffer);
  EXPECT_EQ(buffer[0], '0');

  probe_d2a(buffer, 0.0);
  probe_d2a(buffer, 1.0);
  probe_d2a(buffer, 2.0);
  probe_d2a(buffer, 3.0);
  probe_d2a(buffer, -0.0);
  probe_d2a(buffer, -1.0);
  probe_d2a(buffer, -2.0);
  probe_d2a(buffer, -3.0);
  probe_d2a(buffer, M_PI);
  probe_d2a(buffer, -M_PI);

  probe_d2a(buffer, INT32_MIN);
  probe_d2a(buffer, INT32_MAX);
  probe_d2a(buffer, UINT16_MAX);
  probe_d2a(buffer, UINT32_MAX);
  probe_d2a(buffer, FLT_MAX);
  probe_d2a(buffer, -FLT_MAX);
  probe_d2a(buffer, FLT_MAX);
  probe_d2a(buffer, -FLT_MAX);
  probe_d2a(buffer, FLT_MIN);
  probe_d2a(buffer, -FLT_MIN);
  probe_d2a(buffer, FLT_MAX * M_PI);
  probe_d2a(buffer, -FLT_MAX * M_PI);
  probe_d2a(buffer, FLT_MIN * M_PI);
  probe_d2a(buffer, -FLT_MIN * M_PI);

  probe_d2a(buffer, DBL_MAX);
  probe_d2a(buffer, -DBL_MAX);
  probe_d2a(buffer, DBL_MIN);
  probe_d2a(buffer, -DBL_MIN);
  probe_d2a(buffer, DBL_MAX / M_PI);
  probe_d2a(buffer, -DBL_MAX / M_PI);
  probe_d2a(buffer, DBL_MIN * M_PI);
  probe_d2a(buffer, -DBL_MIN * M_PI);
}

static bool probe_d2a(uint64_t u64, char (&buffer)[23 + 1]) {
  erthink::grisu::casting_union casting(u64);
  switch (std::fpclassify(casting.f)) {
  case FP_NAN:
  case FP_INFINITE:
    return false;
  default:
    probe_d2a(buffer, casting.f);
  }

  const float f32 = static_cast<float>(casting.f);
  switch (std::fpclassify(f32)) {
  case FP_NAN:
  case FP_INFINITE:
    return false;
  default:
    probe_d2a(buffer, f32);
    return true;
  }
}

TEST(d2a, stairwell) {
  char buffer[23 + 1];
  probe_d2a(UINT64_C(4989988387303176888), buffer);
  probe_d2a(UINT64_C(4895412794877399892), buffer);
  probe_d2a(UINT64_C(13717964465041107931), buffer);
  probe_d2a(UINT64_C(13416223289762161370), buffer);
  probe_d2a(UINT64_C(13434237688651515774), buffer);
  probe_d2a(UINT64_C(5008002785836588600), buffer);
  probe_d2a(UINT64_C(4210865651786747166), buffer);
  probe_d2a(UINT64_C(14231374822717078073), buffer);
  probe_d2a(UINT64_C(13434237688189056622), buffer);
  probe_d2a(UINT64_C(13717964465155820979), buffer);
  probe_d2a(UINT64_C(4237887249171175423), buffer);
  probe_d2a(UINT64_C(13632396072180810313), buffer);

  const double up = 1.1283791670955125739 /* 2/sqrt(pi) */;
  for (double value = DBL_MIN * up; value < DBL_MAX / up; value *= up) {
    probe_d2a(buffer, value);
    const float f32 = static_cast<float>(value);
    if (!std::isinf(f32))
      probe_d2a(buffer, f32);
  }

  const double down = 0.91893853320467274178 /* ln(sqrt(2pi)) */;
  for (double value = DBL_MAX * down; value > DBL_MIN / down; value *= down) {
    probe_d2a(buffer, value);
    const float f32 = static_cast<float>(value);
    if (!std::isinf(f32))
      probe_d2a(buffer, f32);
  }

  for (uint64_t mantissa = erthink::grisu::IEEE754_DOUBLE_MANTISSA_MASK;
       mantissa != 0; mantissa >>= 1) {
    for (uint64_t offset = 1; offset < mantissa; offset <<= 1) {
      for (uint64_t exp = 0;
           exp <= erthink::grisu::IEEE754_DOUBLE_EXPONENT_MASK;
           exp += erthink::grisu::IEEE754_DOUBLE_IMPLICIT_LEAD) {
        probe_d2a((mantissa + offset) ^ exp, buffer);
        probe_d2a((mantissa - offset) ^ exp, buffer);
      }
    }
  }
}

TEST(d2a, random3e6) {
  char buffer[23 + 1];
  uint64_t prng(uint64_t(time(0)));
  SCOPED_TRACE("PGNG seed=" + std::to_string(prng));
  for (int i = 0; i < 3333333;) {
    i += probe_d2a(prng, buffer);
    prng *= UINT64_C(6364136223846793005);
    prng += UINT64_C(1442695040888963407);
  }
}

//------------------------------------------------------------------------------

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<commit_msg>erthink: d2a-test: adds SHOW_LAST_DIGIT_ROUNDING_INACCURACY option.<commit_after>/*
 *  Copyright (c) 1994-2019 Leonid Yuriev <leo@yuriev.ru>.
 *  https://github.com/leo-yuriev/erthink
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#ifdef _MSC_VER
#define _USE_MATH_DEFINES
#endif

#include "erthink_d2a.h"
#include "erthink_defs.h"
#include "testing.h"

#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4710) /* function not inlined */
#endif
#include <cfloat> // for FLT_MAX, etc
#include <cmath>  // for M_PI, etc
#ifdef _MSC_VER
#pragma warning(pop)
#endif

__hot __dll_export __noinline char *_d2a(const double value, char *ptr) {
  return erthink::d2a(value, ptr);
}

//------------------------------------------------------------------------------

// #define SHOW_LAST_DIGIT_ROUNDING_INACCURACY
#ifdef SHOW_LAST_DIGIT_ROUNDING_INACCURACY
static std::tuple<bool, int, int> mantissa_str_diff(const char *a,
                                                    const char *b) {
  int i = 0, j = 0;
  for (;;) {
    if (a[i] == '.')
      i++;
    const bool a_end = (a[i] == 'e' || a[i] == 'E' || a[i] == '\0');

    if (b[j] == '.')
      j++;
    const bool b_end = (b[j] == 'e' || b[j] == 'E' || b[j] == '\0');

    if (a_end && b_end)
      break;

    const char a_digit = a_end ? '0' : a[i];
    const char b_digit = b_end ? '0' : b[j];
    if (a_digit != b_digit)
      return std::make_tuple(true, i, j);

    i += !a_end;
    j += !b_end;
  }
  return std::make_tuple(false, 0, 0);
}
#endif /* SHOW_LAST_DIGIT_ROUNDING_INACCURACY */

static void probe_d2a(char (&buffer)[23 + 1], const double value) {
  char *d2a_end = _d2a(value, buffer);
  ASSERT_LT(buffer, d2a_end);
  ASSERT_GT(erthink::array_end(buffer), d2a_end);
  *d2a_end = '\0';

  char *strtod_end = nullptr;
  double probe = strtod(buffer, &strtod_end);
  EXPECT_EQ(d2a_end, strtod_end);
  EXPECT_EQ(value, probe);

#ifdef SHOW_LAST_DIGIT_ROUNDING_INACCURACY
  int i = 0;
  const char *s = buffer;
  for (;;) {
    if (*s == '-' || *s == '.')
      s++;
    else if (s[i] >= '0' && s[i] <= '9')
      i++;
    else
      break;
  }
  char print_buffer[32];
  snprintf(print_buffer, sizeof(print_buffer), "%.*e", i - 1, value);
  const auto diff = mantissa_str_diff(buffer, print_buffer);
  if (std::get<0>(diff)) {
    printf("d2a:%s <> printf:%s\n"
           "%*c%*c\n",
           buffer, print_buffer, std::get<1>(diff) + 5, '^',
           std::get<2>(diff) + 16, '^');
    fflush(nullptr);
  }
#endif /* SHOW_LAST_DIGIT_ROUNDING_INACCURACY */
}

TEST(d2a, trivia) {
  char buffer[23 + 1];
  char *end = _d2a(0, buffer);
  EXPECT_EQ(1, end - buffer);
  EXPECT_EQ(buffer[0], '0');

  probe_d2a(buffer, 0.0);
  probe_d2a(buffer, 1.0);
  probe_d2a(buffer, 2.0);
  probe_d2a(buffer, 3.0);
  probe_d2a(buffer, -0.0);
  probe_d2a(buffer, -1.0);
  probe_d2a(buffer, -2.0);
  probe_d2a(buffer, -3.0);
  probe_d2a(buffer, M_PI);
  probe_d2a(buffer, -M_PI);

  probe_d2a(buffer, INT32_MIN);
  probe_d2a(buffer, INT32_MAX);
  probe_d2a(buffer, UINT16_MAX);
  probe_d2a(buffer, UINT32_MAX);
  probe_d2a(buffer, FLT_MAX);
  probe_d2a(buffer, -FLT_MAX);
  probe_d2a(buffer, FLT_MAX);
  probe_d2a(buffer, -FLT_MAX);
  probe_d2a(buffer, FLT_MIN);
  probe_d2a(buffer, -FLT_MIN);
  probe_d2a(buffer, FLT_MAX * M_PI);
  probe_d2a(buffer, -FLT_MAX * M_PI);
  probe_d2a(buffer, FLT_MIN * M_PI);
  probe_d2a(buffer, -FLT_MIN * M_PI);

  probe_d2a(buffer, DBL_MAX);
  probe_d2a(buffer, -DBL_MAX);
  probe_d2a(buffer, DBL_MIN);
  probe_d2a(buffer, -DBL_MIN);
  probe_d2a(buffer, DBL_MAX / M_PI);
  probe_d2a(buffer, -DBL_MAX / M_PI);
  probe_d2a(buffer, DBL_MIN * M_PI);
  probe_d2a(buffer, -DBL_MIN * M_PI);
}

static bool probe_d2a(uint64_t u64, char (&buffer)[23 + 1]) {
  erthink::grisu::casting_union casting(u64);
  switch (std::fpclassify(casting.f)) {
  case FP_NAN:
  case FP_INFINITE:
    return false;
  default:
    probe_d2a(buffer, casting.f);
  }

  const float f32 = static_cast<float>(casting.f);
  switch (std::fpclassify(f32)) {
  case FP_NAN:
  case FP_INFINITE:
    return false;
  default:
    probe_d2a(buffer, f32);
    return true;
  }
}

TEST(d2a, stairwell) {
  char buffer[23 + 1];
  probe_d2a(UINT64_C(4989988387303176888), buffer);
  probe_d2a(UINT64_C(4895412794877399892), buffer);
  probe_d2a(UINT64_C(13717964465041107931), buffer);
  probe_d2a(UINT64_C(13416223289762161370), buffer);
  probe_d2a(UINT64_C(13434237688651515774), buffer);
  probe_d2a(UINT64_C(5008002785836588600), buffer);
  probe_d2a(UINT64_C(4210865651786747166), buffer);
  probe_d2a(UINT64_C(14231374822717078073), buffer);
  probe_d2a(UINT64_C(13434237688189056622), buffer);
  probe_d2a(UINT64_C(13717964465155820979), buffer);
  probe_d2a(UINT64_C(4237887249171175423), buffer);
  probe_d2a(UINT64_C(13632396072180810313), buffer);

  const double up = 1.1283791670955125739 /* 2/sqrt(pi) */;
  for (double value = DBL_MIN * up; value < DBL_MAX / up; value *= up) {
    probe_d2a(buffer, value);
    const float f32 = static_cast<float>(value);
    if (!std::isinf(f32))
      probe_d2a(buffer, f32);
  }

  const double down = 0.91893853320467274178 /* ln(sqrt(2pi)) */;
  for (double value = DBL_MAX * down; value > DBL_MIN / down; value *= down) {
    probe_d2a(buffer, value);
    const float f32 = static_cast<float>(value);
    if (!std::isinf(f32))
      probe_d2a(buffer, f32);
  }

  for (uint64_t mantissa = erthink::grisu::IEEE754_DOUBLE_MANTISSA_MASK;
       mantissa != 0; mantissa >>= 1) {
    for (uint64_t offset = 1; offset < mantissa; offset <<= 1) {
      for (uint64_t exp = 0;
           exp <= erthink::grisu::IEEE754_DOUBLE_EXPONENT_MASK;
           exp += erthink::grisu::IEEE754_DOUBLE_IMPLICIT_LEAD) {
        probe_d2a((mantissa + offset) ^ exp, buffer);
        probe_d2a((mantissa - offset) ^ exp, buffer);
      }
    }
  }
}

TEST(d2a, random3e6) {
  char buffer[23 + 1];
  uint64_t prng(uint64_t(time(0)));
  SCOPED_TRACE("PGNG seed=" + std::to_string(prng));
  for (int i = 0; i < 3333333;) {
    i += probe_d2a(prng, buffer);
    prng *= UINT64_C(6364136223846793005);
    prng += UINT64_C(1442695040888963407);
  }
}

//------------------------------------------------------------------------------

int main(int argc, char **argv) {
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2015 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <deque>

#include "catch.hpp"

#include "dll/dbn.hpp"

#include "dll/binarize_layer.hpp"
#include "dll/normalize_layer.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

TEST_CASE( "dbn/mnist_1", "dbn::simple" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

TEST_CASE( "dbn/mnist_2", "dbn::containers" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>();

    REQUIRE(!dataset.training_images.empty());
    dataset.training_images.resize(200);
    dataset.training_labels.resize(200);

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 5);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 5, 50);

    REQUIRE(error < 5e-2);
}

TEST_CASE( "dbn/mnist_3", "dbn::labels" ) {
    auto dataset = mnist::read_dataset<std::vector, std::vector, double>();

    REQUIRE(!dataset.training_images.empty());
    dataset.training_images.resize(1000);
    dataset.training_labels.resize(1000);

    mnist::binarize_dataset(dataset);

    typedef dll::dbn_desc<
        dll::dbn_label_layers<
        dll::rbm_desc<28 * 28, 200, dll::batch_size<50>, dll::init_weights, dll::momentum>::rbm_t,
        dll::rbm_desc<200, 300, dll::batch_size<50>, dll::momentum>::rbm_t,
        dll::rbm_desc<310, 500, dll::batch_size<50>, dll::momentum>::rbm_t>>::dbn_t dbn_simple_t;

    auto dbn = std::make_unique<dbn_simple_t>();

    dbn->train_with_labels(dataset.training_images, dataset.training_labels, 10, 10);

    auto error = dll::test_set(dbn, dataset.training_images, dataset.training_labels, dll::label_predictor());
    std::cout << "test_error:" << error << std::endl;
    REQUIRE(error < 0.3);
}

TEST_CASE( "dbn/mnist_6", "dbn::cg_gaussian" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 200, dll::momentum, dll::batch_size<25>, dll::visible<dll::unit_type::GAUSSIAN>>::rbm_t,
        dll::rbm_desc<200, 500, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<500, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>
    >::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>(1000);

    REQUIRE(!dataset.training_images.empty());

    mnist::normalize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

//This test should not perform well, but should not fail
TEST_CASE( "dbn/mnist_8", "dbn::cg_relu" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::RELU>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>(200);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(std::isfinite(error));

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;
}

TEST_CASE( "dbn/mnist_15", "dbn::parallel" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::parallel, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::parallel, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::parallel, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

TEST_CASE( "dbn/mnist_16", "dbn::fast" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<5>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<5>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<5>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(25);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 5);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 2, 5);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

//{{{ Performance debugging tests

TEST_CASE( "dbn/mnist_101", "dbn::slow_parallel" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 300, dll::momentum, dll::parallel, dll::batch_size<24>, dll::init_weights>::rbm_t,
        dll::rbm_desc<300, 1000, dll::momentum, dll::parallel, dll::batch_size<24>>::rbm_t,
        dll::rbm_desc<1000, 10, dll::momentum, dll::parallel, dll::batch_size<24>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(2000);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
}

TEST_CASE( "dbn/mnist_102", "[dbn][bench][slow]" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 300, dll::momentum, dll::batch_size<24>, dll::init_weights>::rbm_t,
        dll::rbm_desc<300, 1000, dll::momentum, dll::batch_size<24>>::rbm_t,
        dll::rbm_desc<1000, 10, dll::momentum, dll::batch_size<24>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(2000);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
}

//}}}

TEST_CASE( "dbn/mnist_17", "dbn::memory" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>
    , dll::memory>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(
        dataset.training_images.begin(), dataset.training_images.end(),
        dataset.training_labels.begin(), dataset.training_labels.end(),
        10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}
<commit_msg>Add test for prepare_one_output<commit_after>//=======================================================================
// Copyright (c) 2014-2015 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <deque>

#include "catch.hpp"

#include "dll/dbn.hpp"

#include "dll/binarize_layer.hpp"
#include "dll/normalize_layer.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

TEST_CASE( "dbn/mnist_1", "dbn::simple" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

TEST_CASE( "dbn/mnist_2", "dbn::containers" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>();

    REQUIRE(!dataset.training_images.empty());
    dataset.training_images.resize(200);
    dataset.training_labels.resize(200);

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 5);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 5, 50);

    REQUIRE(error < 5e-2);
}

TEST_CASE( "dbn/mnist_3", "dbn::labels" ) {
    auto dataset = mnist::read_dataset<std::vector, std::vector, double>();

    REQUIRE(!dataset.training_images.empty());
    dataset.training_images.resize(1000);
    dataset.training_labels.resize(1000);

    mnist::binarize_dataset(dataset);

    typedef dll::dbn_desc<
        dll::dbn_label_layers<
        dll::rbm_desc<28 * 28, 200, dll::batch_size<50>, dll::init_weights, dll::momentum>::rbm_t,
        dll::rbm_desc<200, 300, dll::batch_size<50>, dll::momentum>::rbm_t,
        dll::rbm_desc<310, 500, dll::batch_size<50>, dll::momentum>::rbm_t>>::dbn_t dbn_simple_t;

    auto dbn = std::make_unique<dbn_simple_t>();

    dbn->train_with_labels(dataset.training_images, dataset.training_labels, 10, 10);

    auto error = dll::test_set(dbn, dataset.training_images, dataset.training_labels, dll::label_predictor());
    std::cout << "test_error:" << error << std::endl;
    REQUIRE(error < 0.3);
}

TEST_CASE( "dbn/mnist_6", "dbn::cg_gaussian" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 200, dll::momentum, dll::batch_size<25>, dll::visible<dll::unit_type::GAUSSIAN>>::rbm_t,
        dll::rbm_desc<200, 500, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<500, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>
    >::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>(1000);

    REQUIRE(!dataset.training_images.empty());

    mnist::normalize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

//This test should not perform well, but should not fail
TEST_CASE( "dbn/mnist_8", "dbn::cg_relu" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::RELU>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::deque, double>(200);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(std::isfinite(error));

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;
}

TEST_CASE( "dbn/mnist_15", "dbn::parallel" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::parallel, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::parallel, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::parallel, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

TEST_CASE( "dbn/mnist_16", "dbn::fast" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<5>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<5>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<5>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(25);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 5);
    auto error = dbn->fine_tune(dataset.training_images, dataset.training_labels, 2, 5);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);
}

//{{{ Performance debugging tests

TEST_CASE( "dbn/mnist_101", "dbn::slow_parallel" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 300, dll::momentum, dll::parallel, dll::batch_size<24>, dll::init_weights>::rbm_t,
        dll::rbm_desc<300, 1000, dll::momentum, dll::parallel, dll::batch_size<24>>::rbm_t,
        dll::rbm_desc<1000, 10, dll::momentum, dll::parallel, dll::batch_size<24>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(2000);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
}

TEST_CASE( "dbn/mnist_102", "[dbn][bench][slow]" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 300, dll::momentum, dll::batch_size<24>, dll::init_weights>::rbm_t,
        dll::rbm_desc<300, 1000, dll::momentum, dll::batch_size<24>>::rbm_t,
        dll::rbm_desc<1000, 10, dll::momentum, dll::batch_size<24>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(2000);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
}

//}}}

TEST_CASE( "dbn/mnist_17", "dbn::memory" ) {
    typedef dll::dbn_desc<
        dll::dbn_layers<
        dll::rbm_desc<28 * 28, 100, dll::momentum, dll::batch_size<25>, dll::init_weights>::rbm_t,
        dll::rbm_desc<100, 200, dll::momentum, dll::batch_size<25>>::rbm_t,
        dll::rbm_desc<200, 10, dll::momentum, dll::batch_size<25>, dll::hidden<dll::unit_type::SOFTMAX>>::rbm_t>
    , dll::memory>::dbn_t dbn_t;

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(500);

    REQUIRE(!dataset.training_images.empty());

    mnist::binarize_dataset(dataset);

    auto dbn = std::make_unique<dbn_t>();

    dbn->pretrain(dataset.training_images, 20);
    auto error = dbn->fine_tune(
        dataset.training_images.begin(), dataset.training_images.end(),
        dataset.training_labels.begin(), dataset.training_labels.end(),
        10, 50);

    REQUIRE(error < 5e-2);

    auto test_error = dll::test_set(dbn, dataset.test_images, dataset.test_labels, dll::predictor());

    std::cout << "test_error:" << test_error << std::endl;

    REQUIRE(test_error < 0.2);

    //Mostly here to ensure compilation
    auto out = dbn->prepare_one_output();
    REQUIRE(out.size() > 0);
}
<|endoftext|>