<commit_before>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/kernels/data/fixed_length_record_dataset_op.h"

#include "tensorflow/core/kernels/data/dataset_test_base.h"

namespace tensorflow {
namespace data {
namespace {

constexpr char kNodeName[] = "fixed_length_record_dataset";
constexpr int kOpVersion = 2;

class FixedLengthRecordDatasetParams : public DatasetParams {
 public:
  FixedLengthRecordDatasetParams(const std::vector<tstring>& filenames,
                                 int64 header_bytes, int64 record_bytes,
                                 int64 footer_bytes, int64 buffer_size,
                                 CompressionType compression_type,
                                 string node_name)
      : DatasetParams({DT_STRING}, {PartialTensorShape({})},
                      std::move(node_name)),
        filenames_(filenames),
        header_bytes_(header_bytes),
        record_bytes_(record_bytes),
        footer_bytes_(footer_bytes),
        buffer_size_(buffer_size),
        compression_type_(compression_type) {
    op_version_ = 2;
  }

  std::vector<Tensor> GetInputTensors() const override {
    int num_files = filenames_.size();
    return {
        CreateTensor<tstring>(TensorShape({num_files}), filenames_),
        CreateTensor<int64>(TensorShape({}), {header_bytes_}),
        CreateTensor<int64>(TensorShape({}), {record_bytes_}),
        CreateTensor<int64>(TensorShape({}), {footer_bytes_}),
        CreateTensor<int64>(TensorShape({}), {buffer_size_}),
        CreateTensor<tstring>(TensorShape({}), {ToString(compression_type_)})};
  }

  Status GetInputNames(std::vector<string>* input_names) const override {
    input_names->clear();
    *input_names = {FixedLengthRecordDatasetOp::kFileNames,
                    FixedLengthRecordDatasetOp::kHeaderBytes,
                    FixedLengthRecordDatasetOp::kRecordBytes,
                    FixedLengthRecordDatasetOp::kFooterBytes,
                    FixedLengthRecordDatasetOp::kBufferSize,
                    FixedLengthRecordDatasetOp::kCompressionType};
    return Status::OK();
  }

  Status GetAttributes(AttributeVector* attr_vector) const override {
    *attr_vector = {};
    return Status::OK();
  }

  string dataset_type() const override {
    return FixedLengthRecordDatasetOp::kDatasetType;
  }

 private:
  std::vector<tstring> filenames_;
  int64 header_bytes_;
  int64 record_bytes_;
  int64 footer_bytes_;
  int64 buffer_size_;
  CompressionType compression_type_;
};

class FixedLengthRecordDatasetOpTest : public DatasetOpsTestBase {};

Status CreateTestFiles(const std::vector<tstring>& filenames,
                       const std::vector<string>& contents,
                       CompressionType compression_type) {
  if (filenames.size() != contents.size()) {
    return tensorflow::errors::InvalidArgument(
        "The number of files does not match with the contents");
  }
  if (compression_type == CompressionType::UNCOMPRESSED) {
    for (int i = 0; i < filenames.size(); ++i) {
      TF_RETURN_IF_ERROR(WriteDataToFile(filenames[i], contents[i].data()));
    }
  } else {
    CompressionParams params;
    params.output_buffer_size = 10;
    params.compression_type = compression_type;
    for (int i = 0; i < filenames.size(); ++i) {
      TF_RETURN_IF_ERROR(
          WriteDataToFile(filenames[i], contents[i].data(), params));
    }
  }
  return Status::OK();
}

// Test case 1: multiple fixed-length record files with ZLIB compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams1() {
  std::vector<tstring> filenames = {
      absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_1"),
      absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_2")};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::ZLIB;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }

  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

// Test case 2: multiple fixed-length record files with GZIP compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams2() {
  std::vector<tstring> filenames = {
      absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"),
      absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::GZIP;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }
  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

// Test case 3: multiple fixed-length record files without compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams3() {
  std::vector<tstring> filenames = {
      absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED_1"),
      absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED_2")};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::UNCOMPRESSED;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }
  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

std::vector<GetNextTestCase<FixedLengthRecordDatasetParams>>
GetNextTestCases() {
  return {
      {/*dataset_params=*/FixedLengthRecordDatasetParams1(),
       /*expected_outputs=*/
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams2(),
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams3(),
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})}};
}

ITERATOR_GET_NEXT_TEST_P(FixedLengthRecordDatasetOpTest,
                         FixedLengthRecordDatasetParams, GetNextTestCases())

TEST_F(FixedLengthRecordDatasetOpTest, DatasetNodeName) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetNodeName(dataset_params.node_name()));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetTypeString) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  name_utils::OpNameParams params;
  params.op_version = kOpVersion;
  TF_ASSERT_OK(CheckDatasetTypeString(
      name_utils::OpName(FixedLengthRecordDatasetOp::kDatasetType, params)));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetOutputDtypes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetOutputDtypes({DT_STRING}));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetOutputShapes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetOutputShapes({PartialTensorShape({})}));
}

TEST_F(FixedLengthRecordDatasetOpTest, Cardinality) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetCardinality(kUnknownCardinality));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorOutputDtypes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckIteratorOutputDtypes({DT_STRING}));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorOutputShapes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckIteratorOutputShapes({PartialTensorShape({})}));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorPrefix) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  name_utils::IteratorPrefixParams iterator_prefix_params;
  iterator_prefix_params.op_version = kOpVersion;
  TF_ASSERT_OK(CheckIteratorPrefix(name_utils::IteratorPrefix(
      FixedLengthRecordDatasetOp::kDatasetType,
      dataset_params.iterator_prefix(), iterator_prefix_params)));
}

std::vector<IteratorSaveAndRestoreTestCase<FixedLengthRecordDatasetParams>>
IteratorSaveAndRestoreTestCases() {
  return {
      {/*dataset_params=*/FixedLengthRecordDatasetParams1(),
       /*breakpoints=*/{0, 2, 6},
       /*expected_outputs=*/
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams2(),
       /*breakpoints=*/{0, 2, 6},
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams3(),
       /*breakpoints=*/{0, 2, 6},
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})}};
}

ITERATOR_SAVE_AND_RESTORE_TEST_P(FixedLengthRecordDatasetOpTest,
                                 FixedLengthRecordDatasetParams,
                                 IteratorSaveAndRestoreTestCases())

}  // namespace
}  // namespace data
}  // namespace tensorflow
<commit_msg>Use Env::LocalTempFilename for a temp filename.<commit_after>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/kernels/data/fixed_length_record_dataset_op.h"

#include "tensorflow/core/kernels/data/dataset_test_base.h"

namespace tensorflow {
namespace data {
namespace {

constexpr char kNodeName[] = "fixed_length_record_dataset";
constexpr int kOpVersion = 2;

tstring LocalTempFilename() {
  std::string path;
  CHECK(Env::Default()->LocalTempFilename(&path));
  return tstring(path);
}

class FixedLengthRecordDatasetParams : public DatasetParams {
 public:
  FixedLengthRecordDatasetParams(const std::vector<tstring>& filenames,
                                 int64 header_bytes, int64 record_bytes,
                                 int64 footer_bytes, int64 buffer_size,
                                 CompressionType compression_type,
                                 string node_name)
      : DatasetParams({DT_STRING}, {PartialTensorShape({})},
                      std::move(node_name)),
        filenames_(filenames),
        header_bytes_(header_bytes),
        record_bytes_(record_bytes),
        footer_bytes_(footer_bytes),
        buffer_size_(buffer_size),
        compression_type_(compression_type) {
    op_version_ = 2;
  }

  std::vector<Tensor> GetInputTensors() const override {
    int num_files = filenames_.size();
    return {
        CreateTensor<tstring>(TensorShape({num_files}), filenames_),
        CreateTensor<int64>(TensorShape({}), {header_bytes_}),
        CreateTensor<int64>(TensorShape({}), {record_bytes_}),
        CreateTensor<int64>(TensorShape({}), {footer_bytes_}),
        CreateTensor<int64>(TensorShape({}), {buffer_size_}),
        CreateTensor<tstring>(TensorShape({}), {ToString(compression_type_)})};
  }

  Status GetInputNames(std::vector<string>* input_names) const override {
    input_names->clear();
    *input_names = {FixedLengthRecordDatasetOp::kFileNames,
                    FixedLengthRecordDatasetOp::kHeaderBytes,
                    FixedLengthRecordDatasetOp::kRecordBytes,
                    FixedLengthRecordDatasetOp::kFooterBytes,
                    FixedLengthRecordDatasetOp::kBufferSize,
                    FixedLengthRecordDatasetOp::kCompressionType};
    return Status::OK();
  }

  Status GetAttributes(AttributeVector* attr_vector) const override {
    *attr_vector = {};
    return Status::OK();
  }

  string dataset_type() const override {
    return FixedLengthRecordDatasetOp::kDatasetType;
  }

 private:
  std::vector<tstring> filenames_;
  int64 header_bytes_;
  int64 record_bytes_;
  int64 footer_bytes_;
  int64 buffer_size_;
  CompressionType compression_type_;
};

class FixedLengthRecordDatasetOpTest : public DatasetOpsTestBase {};

Status CreateTestFiles(const std::vector<tstring>& filenames,
                       const std::vector<string>& contents,
                       CompressionType compression_type) {
  if (filenames.size() != contents.size()) {
    return tensorflow::errors::InvalidArgument(
        "The number of files does not match with the contents");
  }
  if (compression_type == CompressionType::UNCOMPRESSED) {
    for (int i = 0; i < filenames.size(); ++i) {
      TF_RETURN_IF_ERROR(WriteDataToFile(filenames[i], contents[i].data()));
    }
  } else {
    CompressionParams params;
    params.output_buffer_size = 10;
    params.compression_type = compression_type;
    for (int i = 0; i < filenames.size(); ++i) {
      TF_RETURN_IF_ERROR(
          WriteDataToFile(filenames[i], contents[i].data(), params));
    }
  }
  return Status::OK();
}

// Test case 1: multiple fixed-length record files with ZLIB compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams1() {
  std::vector<tstring> filenames = {LocalTempFilename(), LocalTempFilename()};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::ZLIB;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }

  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

// Test case 2: multiple fixed-length record files with GZIP compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams2() {
  std::vector<tstring> filenames = {LocalTempFilename(), LocalTempFilename()};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::GZIP;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }
  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

// Test case 3: multiple fixed-length record files without compression.
FixedLengthRecordDatasetParams FixedLengthRecordDatasetParams3() {
  std::vector<tstring> filenames = {LocalTempFilename(), LocalTempFilename()};
  std::vector<string> contents = {
      absl::StrCat("HHHHH", "111", "222", "333", "FF"),
      absl::StrCat("HHHHH", "aaa", "bbb", "FF")};
  CompressionType compression_type = CompressionType::UNCOMPRESSED;
  if (!CreateTestFiles(filenames, contents, compression_type).ok()) {
    VLOG(WARNING) << "Failed to create the test files: "
                  << absl::StrJoin(filenames, ", ");
  }
  return FixedLengthRecordDatasetParams(filenames,
                                        /*header_bytes=*/5,
                                        /*record_bytes=*/3,
                                        /*footer_bytes=*/2,
                                        /*buffer_size=*/10,
                                        /*compression_type=*/compression_type,
                                        /*node_name=*/kNodeName);
}

std::vector<GetNextTestCase<FixedLengthRecordDatasetParams>>
GetNextTestCases() {
  return {
      {/*dataset_params=*/FixedLengthRecordDatasetParams1(),
       /*expected_outputs=*/
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams2(),
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams3(),
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})}};
}

ITERATOR_GET_NEXT_TEST_P(FixedLengthRecordDatasetOpTest,
                         FixedLengthRecordDatasetParams, GetNextTestCases())

TEST_F(FixedLengthRecordDatasetOpTest, DatasetNodeName) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetNodeName(dataset_params.node_name()));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetTypeString) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  name_utils::OpNameParams params;
  params.op_version = kOpVersion;
  TF_ASSERT_OK(CheckDatasetTypeString(
      name_utils::OpName(FixedLengthRecordDatasetOp::kDatasetType, params)));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetOutputDtypes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetOutputDtypes({DT_STRING}));
}

TEST_F(FixedLengthRecordDatasetOpTest, DatasetOutputShapes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetOutputShapes({PartialTensorShape({})}));
}

TEST_F(FixedLengthRecordDatasetOpTest, Cardinality) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckDatasetCardinality(kUnknownCardinality));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorOutputDtypes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckIteratorOutputDtypes({DT_STRING}));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorOutputShapes) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  TF_ASSERT_OK(CheckIteratorOutputShapes({PartialTensorShape({})}));
}

TEST_F(FixedLengthRecordDatasetOpTest, IteratorPrefix) {
  auto dataset_params = FixedLengthRecordDatasetParams1();
  TF_ASSERT_OK(Initialize(dataset_params));
  name_utils::IteratorPrefixParams iterator_prefix_params;
  iterator_prefix_params.op_version = kOpVersion;
  TF_ASSERT_OK(CheckIteratorPrefix(name_utils::IteratorPrefix(
      FixedLengthRecordDatasetOp::kDatasetType,
      dataset_params.iterator_prefix(), iterator_prefix_params)));
}

std::vector<IteratorSaveAndRestoreTestCase<FixedLengthRecordDatasetParams>>
IteratorSaveAndRestoreTestCases() {
  return {
      {/*dataset_params=*/FixedLengthRecordDatasetParams1(),
       /*breakpoints=*/{0, 2, 6},
       /*expected_outputs=*/
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams2(),
       /*breakpoints=*/{0, 2, 6},
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})},
      {/*dataset_params=*/FixedLengthRecordDatasetParams3(),
       /*breakpoints=*/{0, 2, 6},
       CreateTensors<tstring>(TensorShape({}),
                              {{"111"}, {"222"}, {"333"}, {"aaa"}, {"bbb"}})}};
}

ITERATOR_SAVE_AND_RESTORE_TEST_P(FixedLengthRecordDatasetOpTest,
                                 FixedLengthRecordDatasetParams,
                                 IteratorSaveAndRestoreTestCases())

}  // namespace
}  // namespace data
}  // namespace tensorflow
<|endoftext|>


<commit_before>
#include <cstdio>
#include <functional>
#include <map>
#include <queue>
#include <string>
#include <vector>
#include <utility>

#include "Eigen/Dense"
#include "math/math.h"
#include "nn/nn_analysis.h"
#include "nn/dnn.h"
#include "nn/dnn2.h"
#include "nn/dnn3.h"
#include "rng/well_1024.h"
#include "rng/normal.h"
#include "util/timer.h"
#include "util/mnist.h"

using Eigen::Matrix;
using Eigen::Dynamic;
using nn::NN_Analyzer;
using nn::DNN;
using nn::DNN2;
using nn::DNN3;
using rng::Well_1024;
using rng::Random;
using rng::Normal;
using std::greater;
using std::map;
using std::multimap;
using std::pair;
using std::priority_queue;
using std::string;
using std::vector;
using util::MNIST;
using util::Timer;

int main ( int argc, char *argv[] ) {

// -------------------------------------------------------------
//  Begin Configuration Parameters

    const int size_in  =  784;
    const int size_h1  =  392;
    const int size_h2  =    0;
    const int size_h3  =    0;
    const int size_h4  =    0;
    const int size_out =   10;
    const int size_mini_batch = 100;
    const int max_num_mini_batch = -1;
    const int size_validation = 1000;
    const float lambda = 0.0e-0;
    const float learning_rate_rbm = 0.05;
    const float learning_rate_bp = 0.1;
    const float momentum_rbm = 0.50;
    const float momentum_bp = 0.5;
    const int rbm_epochs = 10;
    const int bp_epochs = 20;
    int depth_rbm = -1;
    int depth_bp = -1;
    int replicas = 1;
    const int seed = 17439073;
    //const int seed = 508203299;

    const string train_data_file   = "data/train-images-idx3-ubyte";
    const string train_labels_file = "data/train-labels-idx1-ubyte";
    const string test_data_file    = "data/t10k-images-idx3-ubyte";
    const string test_labels_file  = "data/t10k-labels-idx1-ubyte";

    printf("#size_in: %d\n",size_in);
    printf("#size_h1: %d\n",size_h1);
    printf("#size_h2: %d\n",size_h2);
    printf("#size_h3: %d\n",size_h3);
    printf("#size_h4: %d\n",size_h4);
    printf("#size_out: %d\n",size_out);
    printf("#size_mini_batch: %d\n",size_mini_batch);
    printf("#max_num_mini_batch: %d\n",max_num_mini_batch);
    printf("#size_validation: %d\n",size_validation);
    printf("#lambda: %f\n",lambda);
    printf("#learning rate BP: %f\n",learning_rate_bp);
    printf("#learning rate RBM: %f\n",learning_rate_rbm);
    printf("#momentum BP: %f\n",momentum_bp);
    printf("#momentum RBM: %f\n",momentum_rbm);
    printf("#BP epoch: %d\n",bp_epochs);
    printf("#RBM epoch: %d\n",rbm_epochs);
    printf("#depth BP: %d\n",depth_bp);
    printf("#depth RBM: %d\n",depth_rbm);
    printf("#number of data distortions: %d\n",replicas);
    printf("#seed: %d\n",seed);

//  End Configuration Parameters
// -------------------------------------------------------------

    // read in the data

    fprintf(stderr,"Creating data...\n");
    MNIST<float> mnist_data( train_data_file, train_labels_file,
                             test_data_file, test_labels_file );

    // create the mini-batches used for training

    Well_1024 rand(seed);

    vector<Matrix<float,Dynamic,Dynamic>> mini_batch_inputs;
    vector<Matrix<float,Dynamic,Dynamic>> mini_batch_outputs;

    mnist_data.get_mini_batches( size_mini_batch, replicas,
                                 max_num_mini_batch, rand,
                                 mini_batch_inputs, mini_batch_outputs );

    int num_mini_batch = mini_batch_inputs.size();
    fprintf(stderr,"Number of mini batches: %d\n",num_mini_batch);
    vector<int> schedule;
    for ( int i = 0; i < num_mini_batch; ++i ) schedule.push_back(i);

    rng::shuffle( schedule.begin(), schedule.end(), rand );

    // get a subset of the test data for testing

    Matrix<float,Dynamic,Dynamic> val_inputs;
    Matrix<float,Dynamic,Dynamic> val_outputs;

    int num_validation = size_validation;
    mnist_data.get_validation_set( num_validation, rand,
                                 val_inputs, val_outputs );

    // create the network

    vector<unsigned> layers;
    layers.push_back( size_in );
    layers.push_back( size_h1 );
    if ( size_h2 > 0 ) layers.push_back( size_h2 );
    if ( size_h3 > 0 ) layers.push_back( size_h3 );
    if ( size_h4 > 0 ) layers.push_back( size_h4 );
    layers.push_back( size_out );

    fprintf(stderr,"Creating network...\n");
    DNN<math::logistic<float>> dnn( layers );
    //DNN<math::tanh<float>> dnn( layers );
    //DNN2<float> dnn( size_in, size_h1, size_out );
    //DNN3<float> dnn( size_in, size_h1, size_h2, size_out );

    fprintf(stdout,"#Number of neuron layers: %d\n",
                                         dnn.get_num_neuron_layers());
    fprintf(stdout,"#Number of weight layers: %d\n",
                                         dnn.get_num_weight_layers());

    if ( depth_bp == -1 ) {
        depth_bp = dnn.get_num_weight_layers();
    }
    if ( depth_rbm == -1 ) {
        depth_rbm = dnn.get_num_weight_layers() - 1;
    }

    Normal normal( &rand );
    fprintf(stderr,"initialize network...\n");
    dnn.init( normal );
    //dnn.init( rand );

    double loss = 0.0;
    double val_loss = 0.0;

    // perform the RBM learning for the first depth_rbm number of layers
    // (count from the first layer)
    fprintf(stderr,"start RBM learning ...\n");

    int mb_count = 0;

    vector<Matrix<float,Dynamic,Dynamic>> rbm_mini_batches;

    for ( int drbm = 0; drbm < depth_rbm; ++drbm ) {

        fprintf(stderr,"sampling ...\n");
        dnn.sample_states( drbm, rand, mini_batch_inputs, rbm_mini_batches );

        double var_lr = learning_rate_rbm;
        double var_mom = momentum_rbm;

        fprintf(stdout,"#weights: %d\n",drbm);
        for ( int step = 0; step < rbm_epochs*num_mini_batch; ++step ) {
            int mb = schedule[mb_count];

            loss = dnn.cd1( drbm, var_lr, var_mom, rand, rbm_mini_batches[mb] );
            fprintf(stdout,"%5d  %f \n", step, loss);

            var_mom *= 1.0003;
            if ( var_mom > 0.90 ) var_mom = 0.90;
            //var_lr *= 0.9993;
            //if ( var_lr < 0.01 ) var_lr = 0.01;

            ++mb_count;
            if ( mb_count == num_mini_batch ) {
                mb_count = 0;
                rng::shuffle( schedule.begin(), schedule.end(), rand );
            }

        }
        fprintf(stdout,"\n#weights: %d\n",drbm);

    }
/*
*/

    fprintf(stderr,"start BP learning ...\n");

    mb_count = 0;
    double var_lr = learning_rate_bp;
    double var_mom = momentum_bp;
    //dnn.reset_mom();
    for ( int step = 0; step < bp_epochs*num_mini_batch; ++step ) {

        int mb = schedule[mb_count];

        // perform the BP learning for the depth_bp number of layers (counting
        // from the last layer)

        loss = dnn.back_prop( var_lr, var_mom, lambda, depth_bp,
                                  mini_batch_inputs[mb],
                                  mini_batch_outputs[mb] );

        if ( (step % 500 ) == 0 ) {
            val_loss = dnn.loss( val_inputs, val_outputs, 0.0 );
        }
        fprintf(stdout,"%5d  %f  %f\n", step, loss, val_loss);

        ++mb_count;
        if ( mb_count == num_mini_batch ) {
            mb_count = 0;
            rng::shuffle( schedule.begin(), schedule.end(), rand );
        }

        var_mom *= 1.0003;
        if ( var_mom > 0.95 ) var_mom = 0.95;
//        var_lr *= 0.9999;
 //       if ( var_lr < 0.001 ) var_lr = 0.001;

    }

    // Determine how well the network performs on all the test data.

    fprintf(stderr,"start Testing phase ...\n");

    num_validation = -1;
    mnist_data.get_validation_set( num_validation, rand,
                                 val_inputs, val_outputs );

    val_loss = dnn.loss( val_inputs, val_outputs, 0.0 );
    fprintf(stdout,"#final val: loss %f\n", val_loss);

    Matrix<float,Dynamic,Dynamic> pred_outputs;

    dnn.forward_prop( val_inputs, pred_outputs );

    vector<double> val_error =
           NN_Analyzer::Error_Analyis( pred_outputs, val_outputs );
    for (size_t r = 0; r < val_error.size(); ++r ) {
        fprintf(stdout,"#%2zd %f\n", r, val_error[r]);
    }

    fprintf(stdout,"#final error: %f\n", (1.0-val_error[val_error.size()-1]));

}
<commit_msg>minor correction in main.cc<commit_after>
#include <cstdio>
#include <functional>
#include <map>
#include <queue>
#include <string>
#include <vector>
#include <utility>

#include "Eigen/Dense"
#include "math/math.h"
#include "nn/nn_analysis.h"
#include "nn/dnn.h"
#include "nn/dnn2.h"
#include "nn/dnn3.h"
#include "rng/well_1024.h"
#include "rng/normal.h"
#include "util/timer.h"
#include "util/mnist.h"

using Eigen::Matrix;
using Eigen::Dynamic;
using nn::NN_Analyzer;
using nn::DNN;
using nn::DNN2;
using nn::DNN3;
using rng::Well_1024;
using rng::Random;
using rng::Normal;
using std::greater;
using std::map;
using std::multimap;
using std::pair;
using std::priority_queue;
using std::string;
using std::vector;
using util::MNIST;
using util::Timer;

int main ( int argc, char *argv[] ) {

// -------------------------------------------------------------
//  Begin Configuration Parameters

    const int size_in  =  784;
    const int size_h1  =  392;
    const int size_h2  =    0;
    const int size_h3  =    0;
    const int size_h4  =    0;
    const int size_out =   10;
    const int size_mini_batch = 100;
    const int max_num_mini_batch = -1;
    const int size_validation = 1000;
    const float lambda = 0.0e-0;
    const float learning_rate_rbm = 0.05;
    const float learning_rate_bp = 0.5;
    const float momentum_rbm = 0.50;
    const float momentum_bp = 0.5;
    const int rbm_epochs = 10;
    const int bp_epochs = 20;
    int depth_rbm = -1;
    int depth_bp = -1;
    int replicas = 1;
    const int seed = 17439073;
    //const int seed = 508203299;

    const string train_data_file   = "data/train-images-idx3-ubyte";
    const string train_labels_file = "data/train-labels-idx1-ubyte";
    const string test_data_file    = "data/t10k-images-idx3-ubyte";
    const string test_labels_file  = "data/t10k-labels-idx1-ubyte";

    printf("#size_in: %d\n",size_in);
    printf("#size_h1: %d\n",size_h1);
    printf("#size_h2: %d\n",size_h2);
    printf("#size_h3: %d\n",size_h3);
    printf("#size_h4: %d\n",size_h4);
    printf("#size_out: %d\n",size_out);
    printf("#size_mini_batch: %d\n",size_mini_batch);
    printf("#max_num_mini_batch: %d\n",max_num_mini_batch);
    printf("#size_validation: %d\n",size_validation);
    printf("#lambda: %f\n",lambda);
    printf("#learning rate BP: %f\n",learning_rate_bp);
    printf("#learning rate RBM: %f\n",learning_rate_rbm);
    printf("#momentum BP: %f\n",momentum_bp);
    printf("#momentum RBM: %f\n",momentum_rbm);
    printf("#BP epoch: %d\n",bp_epochs);
    printf("#RBM epoch: %d\n",rbm_epochs);
    printf("#depth BP: %d\n",depth_bp);
    printf("#depth RBM: %d\n",depth_rbm);
    printf("#number of data distortions: %d\n",replicas);
    printf("#seed: %d\n",seed);

//  End Configuration Parameters
// -------------------------------------------------------------

    // read in the data

    fprintf(stderr,"Creating data...\n");
    MNIST<float> mnist_data( train_data_file, train_labels_file,
                             test_data_file, test_labels_file );

    // create the mini-batches used for training

    Well_1024 rand(seed);

    vector<Matrix<float,Dynamic,Dynamic>> mini_batch_inputs;
    vector<Matrix<float,Dynamic,Dynamic>> mini_batch_outputs;

    mnist_data.get_mini_batches( size_mini_batch, replicas,
                                 max_num_mini_batch, rand,
                                 mini_batch_inputs, mini_batch_outputs );

    int num_mini_batch = mini_batch_inputs.size();
    fprintf(stderr,"Number of mini batches: %d\n",num_mini_batch);
    vector<int> schedule;
    for ( int i = 0; i < num_mini_batch; ++i ) schedule.push_back(i);

    rng::shuffle( schedule.begin(), schedule.end(), rand );

    // get a subset of the test data for testing

    Matrix<float,Dynamic,Dynamic> val_inputs;
    Matrix<float,Dynamic,Dynamic> val_outputs;

    int num_validation = size_validation;
    mnist_data.get_validation_set( num_validation, rand,
                                 val_inputs, val_outputs );

    // create the network

    vector<unsigned> layers;
    layers.push_back( size_in );
    layers.push_back( size_h1 );
    if ( size_h2 > 0 ) layers.push_back( size_h2 );
    if ( size_h3 > 0 ) layers.push_back( size_h3 );
    if ( size_h4 > 0 ) layers.push_back( size_h4 );
    layers.push_back( size_out );

    fprintf(stderr,"Creating network...\n");
    DNN<math::logistic<float>> dnn( layers );
    //DNN<math::tanh<float>> dnn( layers );
    //DNN2<float> dnn( size_in, size_h1, size_out );
    //DNN3<float> dnn( size_in, size_h1, size_h2, size_out );

    fprintf(stdout,"#Number of neuron layers: %d\n",
                                         dnn.get_num_neuron_layers());
    fprintf(stdout,"#Number of weight layers: %d\n",
                                         dnn.get_num_weight_layers());

    if ( depth_bp == -1 ) {
        depth_bp = dnn.get_num_weight_layers();
    }
    if ( depth_rbm == -1 ) {
        depth_rbm = dnn.get_num_weight_layers() - 1;
    }

    Normal normal( &rand );
    fprintf(stderr,"initialize network...\n");
    dnn.init( normal );
    //dnn.init( rand );

    double loss = 0.0;
    double val_loss = 0.0;

    // perform the RBM learning for the first depth_rbm number of layers
    // (count from the first layer)
    fprintf(stderr,"start RBM learning ...\n");

    int mb_count = 0;

    vector<Matrix<float,Dynamic,Dynamic>> rbm_mini_batches;

    for ( int drbm = 0; drbm < depth_rbm; ++drbm ) {

        fprintf(stderr,"sampling ...\n");
        dnn.sample_states( drbm, rand, mini_batch_inputs, rbm_mini_batches );

        double var_lr = learning_rate_rbm;
        double var_mom = momentum_rbm;

        fprintf(stdout,"#weights: %d\n",drbm);
        for ( int step = 0; step < rbm_epochs*num_mini_batch; ++step ) {
            int mb = schedule[mb_count];

            loss = dnn.cd1( drbm, var_lr, var_mom, rand, rbm_mini_batches[mb] );
            fprintf(stdout,"%5d  %f \n", step, loss);

            var_mom *= 1.0003;
            if ( var_mom > 0.90 ) var_mom = 0.90;
            //var_lr *= 0.9993;
            //if ( var_lr < 0.01 ) var_lr = 0.01;

            ++mb_count;
            if ( mb_count == num_mini_batch ) {
                mb_count = 0;
                rng::shuffle( schedule.begin(), schedule.end(), rand );
            }

        }
        fprintf(stdout,"\n#weights: %d\n",drbm);

    }
/*
*/

    fprintf(stderr,"start BP learning ...\n");

    mb_count = 0;
    double var_lr = learning_rate_bp;
    double var_mom = momentum_bp;
    //dnn.reset_mom();
    for ( int step = 0; step < bp_epochs*num_mini_batch; ++step ) {

        int mb = schedule[mb_count];

        // perform the BP learning for the depth_bp number of layers (counting
        // from the last layer)

        loss = dnn.back_prop( var_lr, var_mom, lambda, depth_bp,
                                  mini_batch_inputs[mb],
                                  mini_batch_outputs[mb] );

        if ( (step % 500 ) == 0 ) {
            val_loss = dnn.loss( val_inputs, val_outputs, 0.0 );
        }
        fprintf(stdout,"%5d  %f  %f\n", step, loss, val_loss);

        ++mb_count;
        if ( mb_count == num_mini_batch ) {
            mb_count = 0;
            rng::shuffle( schedule.begin(), schedule.end(), rand );
        }

        var_mom *= 1.0003;
        if ( var_mom > 0.95 ) var_mom = 0.95;
//        var_lr *= 0.9999;
 //       if ( var_lr < 0.001 ) var_lr = 0.001;

    }

    // Determine how well the network performs on all the test data.

    fprintf(stderr,"start Testing phase ...\n");

    num_validation = -1;
    mnist_data.get_validation_set( num_validation, rand,
                                 val_inputs, val_outputs );

    val_loss = dnn.loss( val_inputs, val_outputs, 0.0 );
    fprintf(stdout,"#final val: loss %f\n", val_loss);

    Matrix<float,Dynamic,Dynamic> pred_outputs;

    dnn.forward_prop( val_inputs, pred_outputs );

    vector<double> val_error =
           NN_Analyzer::Error_Analyis( pred_outputs, val_outputs );
    for (size_t r = 0; r < val_error.size(); ++r ) {
        fprintf(stdout,"#%2zd %f\n", r, val_error[r]);
    }

    fprintf(stdout,"#final error: %f\n", (1.0-val_error[val_error.size()-1]));

}
<|endoftext|>


<commit_before>// Copyright Hugh Perkins (hughperkins at gmail), Josef Moudrik 2015
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

#include "DeepCL.h"
#include "loss/SoftMaxLayer.h"
#ifdef _WIN32
#include <stdio.h>
#include <fcntl.h>
#include <io.h>
#endif // _WIN32

using namespace std;

/* [[[cog
    # These are used in the later cog sections in this file:
    options = [
        {'name': 'gpuIndex', 'type': 'int', 'description': 'gpu device index; default value is gpu if present, cpu otw.', 'default': -1, 'ispublicapi': True},

        {'name': 'weightsFile', 'type': 'string', 'description': 'file to read weights from', 'default': 'weights.dat', 'ispublicapi': True},
        # removing loadondemand for now, let's always load exactly one batch at a time for now
        # ('loadOnDemand', 'int', 'load data on demand [1|0]', 0, [0,1], True},
        {'name': 'batchSize', 'type': 'int', 'description': 'batch size', 'default': 128, 'ispublicapi': True},

        # lets go with pipe for now, and then somehow shoehorn files in later?
        {'name': 'inputFile',  'type': 'string', 'description': 'file to read inputs from, if empty, read stdin (default)', 'default': ''},
        {'name': 'outputFile', 'type': 'string', 'description': 'file to write outputs to, if empty, write to stdout', 'default': ''},
        {'name': 'outputLayer', 'type': 'int', 'description': 'layer to write output from, default -1 means: last layer', 'default': -1},
        {'name': 'writeLabels', 'type': 'int', 'description': 'write integer labels, instead of probabilities etc (default 0)', 'default': 0},
        {'name': 'outputFormat', 'type': 'string', 'description': 'output format [binary|text]', 'default': 'text'}
    ]
*///]]]
// [[[end]]]

class Config {
public:
    /* [[[cog
        cog.outl('// generated using cog:')
        for option in options:
            cog.outl( option['type'] + ' ' + option['name'] + ';')
    */// ]]]
    // generated using cog:
    int gpuIndex;
    string weightsFile;
    int batchSize;
    string inputFile;
    string outputFile;
    int outputLayer;
    int writeLabels;
    string outputFormat;
    // [[[end]]]

    Config() {
        /* [[[cog
            cog.outl('// generated using cog:')
            for option in options:
                defaultString = ''
                default = option['default']
                type = option['type']
                if type == 'string':
                    defaultString = '"' + default + '"'
                elif type == 'int':
                    defaultString = str(default)
                elif type == 'float':
                    defaultString = str(default)
                    if '.' not in defaultString:
                        defaultString += '.0'
                    defaultString += 'f'
                cog.outl( option['name'] + ' = ' + defaultString + ';')
        */// ]]]
        // generated using cog:
        gpuIndex = -1;
        weightsFile = "weights.dat";
        batchSize = 128;
        inputFile = "";
        outputFile = "";
        outputLayer = -1;
        writeLabels = 0;
        outputFormat = "text";
        // [[[end]]]
    }
};

void go(Config config) {
    bool verbose = true;
    if( config.outputFile == "" ) {
        verbose = false;
    }

    int N = -1;
    int numPlanes;
    int imageSize;
    int imageSizeCheck;
    if( config.inputFile == "" ) {
        int dims[3];
        cin.read( reinterpret_cast< char * >( dims ), 3 * 4l );
        numPlanes = dims[0];
        imageSize = dims[1];
        imageSizeCheck = dims[2];
        if( imageSize != imageSizeCheck ) {
            throw std::runtime_error( "imageSize doesnt match imageSizeCheck, image not square" );
        }
    } else {
        GenericLoader::getDimensions( config.inputFile, &N, &numPlanes, &imageSize );
        if( verbose ) cout << "N " << N << " planes " << numPlanes << " size " << imageSize << endl;
    }

    const long inputCubeSize = numPlanes * imageSize * imageSize ;

    //
    // ## Set up the Network
    //

    EasyCL *cl = 0;
    if( config.gpuIndex >= 0 ) {
        cl = EasyCL::createForIndexedGpu( config.gpuIndex, verbose );
    } else {
        cl = EasyCL::createForFirstGpuOtherwiseCpu( verbose );
    }

    NeuralNet *net;
    net = new NeuralNet(cl);

    // just use the default for net creation, weights are overriden from the weightsFile
    WeightsInitializer *weightsInitializer = new OriginalInitializer();

    if( config.weightsFile == "" ) {
        cout << "weightsFile not specified" << endl;
        return;
    }

    string netDef;
    if ( !WeightsPersister::loadConfigString( config.weightsFile, netDef ) ){
        cout << "Cannot load network definition from weightsFile." << endl;
        return;
    }
//    cout << "net def from weights file: " << netDef << endl;

    net->addLayer( InputLayerMaker::instance()->numPlanes(numPlanes)->imageSize(imageSize) );
    net->addLayer( NormalizationLayerMaker::instance()->translate( 0.0f )->scale( 1.0f ) ); // This will be read from weights file

    if( !NetdefToNet::createNetFromNetdef( net, netDef, weightsInitializer ) ) {
        return;
    }

    // ignored int and float, s.t. we can use loadWeights
    int ignI;
    float ignF;

    // weights file contains normalization layer parameters as 'weights' now.  We should probably rename weights to parameters
    // sooner or later ,but anyway, tehcnically, works for onw
    if( !WeightsPersister::loadWeights( config.weightsFile, string("netDef=")+netDef, net, &ignI, &ignI, &ignF, &ignI, &ignF ) ){
        cout << "Cannot load network weights from weightsFile." << endl;
        return;
    }

    if( verbose ) {
        net->print();
    }
    net->setBatchSize(config.batchSize);
    if( verbose ) cout << "batchSize: " << config.batchSize << endl;

    //
    // ## All is set up now
    //

    float *inputData = new float[ inputCubeSize * config.batchSize];

    int *labels = new int[config.batchSize];
    int n = 0;
    bool more = true;
    ostream *outFile = 0;
    if( config.outputFile == "" ) {
        outFile = &cout;
    } else {
        if( config.outputFormat == "text" ) {
            outFile = new ofstream( config.outputFile, ios::out );
        } else if( config.outputFormat == "binary" ) {
            outFile = new ofstream( config.outputFile, ios::out | std::ios::binary );
        } else {
            throw runtime_error( "outputFormat " + config.outputFormat + " not recognized" );
        }
    }
    if( config.outputLayer == -1 ) {
        config.outputLayer = net->getNumLayers() - 1;
    }
    if( config.inputFile == "" ) {
        #ifdef _WIN32
        // refs:
        // http://www.thecodingforums.com/threads/binary-output-to-stdout-in-windows.317367/
        // http://www.cplusplus.com/forum/windows/77812/
        _setmode( _fileno( stdout ), _O_BINARY );
        #endif
        cin.read( reinterpret_cast< char * >( inputData ), inputCubeSize * config.batchSize * 4l );
        more = !cin.eof();
    } else {
        // pass 0 for labels, and this will cause GenericLoader to simply not try to load any labels
        // now, after modifying GenericLoader to have this new behavior
        GenericLoader::load( config.inputFile, inputData, 0, n, config.batchSize );
    }
    while( more ) {
        // no point in forwarding through all, so forward through each, one by one
        if( config.outputLayer < 0 || config.outputLayer > net->getNumLayers() ) {
            throw runtime_error( "outputLayer should be the layer number of one of the layers in the network" );
        }
        dynamic_cast<InputLayer *>( net->getLayer(0) )->in( inputData );
        for( int layerId = 0; layerId <= config.outputLayer; layerId++ ) {
            StatefulTimer::setPrefix("layer" + toString(layerId) + " " );
            net->getLayer( layerId )->forward();
            StatefulTimer::setPrefix("" );
        }

        if( !config.writeLabels ) {
            if( config.outputFormat == "text" ) {
                float const*output = net->getLayer( config.outputLayer )->getOutput();
                const int numFields = net->getLayer( config.outputLayer )->getOutputCubeSize();
                for( int i = 0; i < config.batchSize; i++ ) {
                    for( int f = 0; f < numFields; f++ ) {
                        if( f > 0 ) {
                            *outFile << " ";
                        }
                        *outFile << output[ i * numFields + f ];
                    }
                    *outFile << "\n";
                }
            } else {
                outFile->write( reinterpret_cast<const char *>(net->getOutput()), net->getOutputSize() * 4 * config.batchSize);
            }
        } else {
            SoftMaxLayer *softMaxLayer = dynamic_cast< SoftMaxLayer *>(net->getLayer( config.outputLayer ) );
            if( softMaxLayer == 0 ) {
                cout << "must choose softmaxlayer, if want to output labels" << endl;
                return;
            }
            softMaxLayer->getLabels(labels);
            if( config.outputFormat == "text" ) {
                for( int i = 0; i < config.batchSize; i++ ) {
                    *outFile << labels[i] << "\n";
                }
            } else {
                outFile->write( reinterpret_cast< char * >( labels ), config.batchSize * 4l );
            }
            outFile->flush();
        }
        n += config.batchSize;
        if( config.inputFile == "" ) {
            cin.read( reinterpret_cast< char * >( inputData ), inputCubeSize * config.batchSize * 4l );
            more = !cin.eof();
        } else {
            if( n + config.batchSize < N ) {
                GenericLoader::load( config.inputFile, inputData, 0, n, config.batchSize );
            } else {
                more = false;
                if( n != N ) {
                    cout << "breaking prematurely, since file is not an exact multiple of batchsize, and we didnt handle this yet" << endl;
                }
            }
        }
    }
    if( config.outputFile != "" ) {
        delete outFile;
    }

    delete[] inputData;
    delete[] labels;
    delete weightsInitializer;
    delete net;
    delete cl;
}

void printUsage( char *argv[], Config config ) {
    cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
    cout << endl;
    cout << "Possible key=value pairs:" << endl;
    /* [[[cog
        cog.outl('// generated using cog:')
        cog.outl('cout << "public api, shouldnt change within major version:" << endl;')
        for option in options:
            name = option['name']
            description = option['description']
            if 'ispublicapi' in option and option['ispublicapi']:
                cog.outl( 'cout << "    ' + name.lower() + '=[' + description + '] (" << config.' + name + ' << ")" << endl;')
        cog.outl('cout << "" << endl; ')
        cog.outl('cout << "unstable, might change within major version:" << endl; ')
        for option in options:
            if 'ispublicapi' not in option or not option['ispublicapi']:
                name = option['name']
                description = option['description']
                cog.outl( 'cout << "    ' + name.lower() + '=[' + description + '] (" << config.' + name + ' << ")" << endl;')
    *///]]]
    // generated using cog:
    cout << "public api, shouldnt change within major version:" << endl;
    cout << "    gpuindex=[gpu device index; default value is gpu if present, cpu otw.] (" << config.gpuIndex << ")" << endl;
    cout << "    weightsfile=[file to read weights from] (" << config.weightsFile << ")" << endl;
    cout << "    batchsize=[batch size] (" << config.batchSize << ")" << endl;
    cout << "" << endl;
    cout << "unstable, might change within major version:" << endl;
    cout << "    inputfile=[file to read inputs from, if empty, read stdin (default)] (" << config.inputFile << ")" << endl;
    cout << "    outputfile=[file to write outputs to, if empty, write to stdout] (" << config.outputFile << ")" << endl;
    cout << "    outputlayer=[layer to write output from, default -1 means: last layer] (" << config.outputLayer << ")" << endl;
    cout << "    writelabels=[write integer labels, instead of probabilities etc (default 0)] (" << config.writeLabels << ")" << endl;
    cout << "    outputformat=[output format [binary|text]] (" << config.outputFormat << ")" << endl;
    // [[[end]]]
}

int main( int argc, char *argv[] ) {
    Config config;
    if( argc == 2 && ( string(argv[1]) == "--help" || string(argv[1]) == "--?" || string(argv[1]) == "-?" || string(argv[1]) == "-h" ) ) {
        printUsage( argv, config );
    }
    for( int i = 1; i < argc; i++ ) {
        vector<string> splitkeyval = split( argv[i], "=" );
        if( splitkeyval.size() != 2 ) {
          cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
          exit(1);
        } else {
            string key = splitkeyval[0];
            string value = splitkeyval[1];
//            cout << "key [" << key << "]" << endl;
            /* [[[cog
                cog.outl('// generated using cog:')
                cog.outl('if( false ) {')
                for option in options:
                    name = option['name']
                    type = option['type']
                    cog.outl( '} else if( key == "' + name.lower() + '" ) {')
                    converter = '';
                    if type == 'int':
                        converter = 'atoi';
                    elif type == 'float':
                        converter = 'atof';
                    cog.outl( '    config.' + name + ' = ' + converter + '(value);')
            */// ]]]
            // generated using cog:
            if( false ) {
            } else if( key == "gpuindex" ) {
                config.gpuIndex = atoi(value);
            } else if( key == "weightsfile" ) {
                config.weightsFile = (value);
            } else if( key == "batchsize" ) {
                config.batchSize = atoi(value);
            } else if( key == "inputfile" ) {
                config.inputFile = (value);
            } else if( key == "outputfile" ) {
                config.outputFile = (value);
            } else if( key == "outputlayer" ) {
                config.outputLayer = atoi(value);
            } else if( key == "writelabels" ) {
                config.writeLabels = atoi(value);
            } else if( key == "outputformat" ) {
                config.outputFormat = (value);
            // [[[end]]]
            } else {
                cout << endl;
                cout << "Error: key '" << key << "' not recognised" << endl;
                cout << endl;
                printUsage( argv, config );
                cout << endl;
                return -1;
            }
        }
    }
    if( config.outputFormat != "text" && config.outputFormat != "binary" ) {
        cout << endl;
        cout << "outputformat must be 'text' or 'binary'" << endl;
        cout << endl;
        return -1;
    }
    try {
        go( config );
    } catch( runtime_error e ) {
        cout << "Something went wrong: " << e.what() << endl;
        return -1;
    }
}

<commit_msg>predict: extend debug messages, print inputFile and outputFile names.<commit_after>// Copyright Hugh Perkins (hughperkins at gmail), Josef Moudrik 2015
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

#include "DeepCL.h"
#include "loss/SoftMaxLayer.h"
#ifdef _WIN32
#include <stdio.h>
#include <fcntl.h>
#include <io.h>
#endif // _WIN32

using namespace std;

/* [[[cog
    # These are used in the later cog sections in this file:
    options = [
        {'name': 'gpuIndex', 'type': 'int', 'description': 'gpu device index; default value is gpu if present, cpu otw.', 'default': -1, 'ispublicapi': True},

        {'name': 'weightsFile', 'type': 'string', 'description': 'file to read weights from', 'default': 'weights.dat', 'ispublicapi': True},
        # removing loadondemand for now, let's always load exactly one batch at a time for now
        # ('loadOnDemand', 'int', 'load data on demand [1|0]', 0, [0,1], True},
        {'name': 'batchSize', 'type': 'int', 'description': 'batch size', 'default': 128, 'ispublicapi': True},

        # lets go with pipe for now, and then somehow shoehorn files in later?
        {'name': 'inputFile',  'type': 'string', 'description': 'file to read inputs from, if empty, read stdin (default)', 'default': ''},
        {'name': 'outputFile', 'type': 'string', 'description': 'file to write outputs to, if empty, write to stdout', 'default': ''},
        {'name': 'outputLayer', 'type': 'int', 'description': 'layer to write output from, default -1 means: last layer', 'default': -1},
        {'name': 'writeLabels', 'type': 'int', 'description': 'write integer labels, instead of probabilities etc (default 0)', 'default': 0},
        {'name': 'outputFormat', 'type': 'string', 'description': 'output format [binary|text]', 'default': 'text'}
    ]
*///]]]
// [[[end]]]

class Config {
public:
    /* [[[cog
        cog.outl('// generated using cog:')
        for option in options:
            cog.outl( option['type'] + ' ' + option['name'] + ';')
    */// ]]]
    // generated using cog:
    int gpuIndex;
    string weightsFile;
    int batchSize;
    string inputFile;
    string outputFile;
    int outputLayer;
    int writeLabels;
    string outputFormat;
    // [[[end]]]

    Config() {
        /* [[[cog
            cog.outl('// generated using cog:')
            for option in options:
                defaultString = ''
                default = option['default']
                type = option['type']
                if type == 'string':
                    defaultString = '"' + default + '"'
                elif type == 'int':
                    defaultString = str(default)
                elif type == 'float':
                    defaultString = str(default)
                    if '.' not in defaultString:
                        defaultString += '.0'
                    defaultString += 'f'
                cog.outl( option['name'] + ' = ' + defaultString + ';')
        */// ]]]
        // generated using cog:
        gpuIndex = -1;
        weightsFile = "weights.dat";
        batchSize = 128;
        inputFile = "";
        outputFile = "";
        outputLayer = -1;
        writeLabels = 0;
        outputFormat = "text";
        // [[[end]]]
    }
};

void go(Config config) {
    bool verbose = true;
    if( config.outputFile == "" ) {
        verbose = false;
    }

    int N = -1;
    int numPlanes;
    int imageSize;
    int imageSizeCheck;
    if( config.inputFile == "" ) {
        int dims[3];
        cin.read( reinterpret_cast< char * >( dims ), 3 * 4l );
        numPlanes = dims[0];
        imageSize = dims[1];
        imageSizeCheck = dims[2];
        if( imageSize != imageSizeCheck ) {
            throw std::runtime_error( "imageSize doesnt match imageSizeCheck, image not square" );
        }
    } else {
        GenericLoader::getDimensions( config.inputFile, &N, &numPlanes, &imageSize );
        if( verbose ) cout << "N " << N << " planes " << numPlanes << " size " << imageSize << endl;
    }

    const long inputCubeSize = numPlanes * imageSize * imageSize ;

    //
    // ## Set up the Network
    //

    EasyCL *cl = 0;
    if( config.gpuIndex >= 0 ) {
        cl = EasyCL::createForIndexedGpu( config.gpuIndex, verbose );
    } else {
        cl = EasyCL::createForFirstGpuOtherwiseCpu( verbose );
    }

    NeuralNet *net;
    net = new NeuralNet(cl);

    // just use the default for net creation, weights are overriden from the weightsFile
    WeightsInitializer *weightsInitializer = new OriginalInitializer();

    if( config.weightsFile == "" ) {
        cout << "weightsFile not specified" << endl;
        return;
    }

    string netDef;
    if ( !WeightsPersister::loadConfigString( config.weightsFile, netDef ) ){
        cout << "Cannot load network definition from weightsFile." << endl;
        return;
    }
//    cout << "net def from weights file: " << netDef << endl;

    net->addLayer( InputLayerMaker::instance()->numPlanes(numPlanes)->imageSize(imageSize) );
    net->addLayer( NormalizationLayerMaker::instance()->translate( 0.0f )->scale( 1.0f ) ); // This will be read from weights file

    if( !NetdefToNet::createNetFromNetdef( net, netDef, weightsInitializer ) ) {
        return;
    }

    // ignored int and float, s.t. we can use loadWeights
    int ignI;
    float ignF;

    // weights file contains normalization layer parameters as 'weights' now.  We should probably rename weights to parameters
    // sooner or later ,but anyway, tehcnically, works for onw
    if( !WeightsPersister::loadWeights( config.weightsFile, string("netDef=")+netDef, net, &ignI, &ignI, &ignF, &ignI, &ignF ) ){
        cout << "Cannot load network weights from weightsFile." << endl;
        return;
    }

    if( verbose ) {
        net->print();
    }
    net->setBatchSize(config.batchSize);
    if( verbose ) cout << "batchSize: " << config.batchSize << endl;

    //
    // ## All is set up now
    //

    float *inputData = new float[ inputCubeSize * config.batchSize];

    int *labels = new int[config.batchSize];
    int n = 0;
    bool more = true;
    ostream *outFile = 0;
    if( verbose ) cout << "outputFile: '" << config.outputFile << "'"<< endl;
    if( config.outputFile == "" ) {
        outFile = &cout;
    } else {
        if( config.outputFormat == "text" ) {
            outFile = new ofstream( config.outputFile, ios::out );
        } else if( config.outputFormat == "binary" ) {
            outFile = new ofstream( config.outputFile, ios::out | std::ios::binary );
        } else {
            throw runtime_error( "outputFormat " + config.outputFormat + " not recognized" );
        }
    }
    if( config.outputLayer == -1 ) {
        config.outputLayer = net->getNumLayers() - 1;
    }
    if( verbose ) cout << "inputFile: '" << config.inputFile << "'"<< endl;
    if( config.inputFile == "" ) {
        #ifdef _WIN32
        // refs:
        // http://www.thecodingforums.com/threads/binary-output-to-stdout-in-windows.317367/
        // http://www.cplusplus.com/forum/windows/77812/
        _setmode( _fileno( stdout ), _O_BINARY );
        #endif
        cin.read( reinterpret_cast< char * >( inputData ), inputCubeSize * config.batchSize * 4l );
        more = !cin.eof();
    } else {
        // pass 0 for labels, and this will cause GenericLoader to simply not try to load any labels
        // now, after modifying GenericLoader to have this new behavior
        GenericLoader::load( config.inputFile, inputData, 0, n, config.batchSize );
    }
    while( more ) {
        // no point in forwarding through all, so forward through each, one by one
        if( config.outputLayer < 0 || config.outputLayer > net->getNumLayers() ) {
            throw runtime_error( "outputLayer should be the layer number of one of the layers in the network" );
        }
        dynamic_cast<InputLayer *>( net->getLayer(0) )->in( inputData );
        for( int layerId = 0; layerId <= config.outputLayer; layerId++ ) {
            StatefulTimer::setPrefix("layer" + toString(layerId) + " " );
            net->getLayer( layerId )->forward();
            StatefulTimer::setPrefix("" );
        }

        if( !config.writeLabels ) {
            if( config.outputFormat == "text" ) {
                float const*output = net->getLayer( config.outputLayer )->getOutput();
                const int numFields = net->getLayer( config.outputLayer )->getOutputCubeSize();
                for( int i = 0; i < config.batchSize; i++ ) {
                    for( int f = 0; f < numFields; f++ ) {
                        if( f > 0 ) {
                            *outFile << " ";
                        }
                        *outFile << output[ i * numFields + f ];
                    }
                    *outFile << "\n";
                }
            } else {
                outFile->write( reinterpret_cast<const char *>(net->getOutput()), net->getOutputSize() * 4 * config.batchSize);
            }
        } else {
            SoftMaxLayer *softMaxLayer = dynamic_cast< SoftMaxLayer *>(net->getLayer( config.outputLayer ) );
            if( softMaxLayer == 0 ) {
                cout << "must choose softmaxlayer, if want to output labels" << endl;
                return;
            }
            softMaxLayer->getLabels(labels);
            if( config.outputFormat == "text" ) {
                for( int i = 0; i < config.batchSize; i++ ) {
                    *outFile << labels[i] << "\n";
                }
            } else {
                outFile->write( reinterpret_cast< char * >( labels ), config.batchSize * 4l );
            }
            outFile->flush();
        }
        n += config.batchSize;
        if( config.inputFile == "" ) {
            cin.read( reinterpret_cast< char * >( inputData ), inputCubeSize * config.batchSize * 4l );
            more = !cin.eof();
        } else {
            if( n + config.batchSize < N ) {
                GenericLoader::load( config.inputFile, inputData, 0, n, config.batchSize );
            } else {
                more = false;
                if( n != N ) {
                    cout << "breaking prematurely, since file is not an exact multiple of batchsize, and we didnt handle this yet" << endl;
                }
            }
        }
    }
    if( config.outputFile != "" ) {
        delete outFile;
    }

    delete[] inputData;
    delete[] labels;
    delete weightsInitializer;
    delete net;
    delete cl;
}

void printUsage( char *argv[], Config config ) {
    cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
    cout << endl;
    cout << "Possible key=value pairs:" << endl;
    /* [[[cog
        cog.outl('// generated using cog:')
        cog.outl('cout << "public api, shouldnt change within major version:" << endl;')
        for option in options:
            name = option['name']
            description = option['description']
            if 'ispublicapi' in option and option['ispublicapi']:
                cog.outl( 'cout << "    ' + name.lower() + '=[' + description + '] (" << config.' + name + ' << ")" << endl;')
        cog.outl('cout << "" << endl; ')
        cog.outl('cout << "unstable, might change within major version:" << endl; ')
        for option in options:
            if 'ispublicapi' not in option or not option['ispublicapi']:
                name = option['name']
                description = option['description']
                cog.outl( 'cout << "    ' + name.lower() + '=[' + description + '] (" << config.' + name + ' << ")" << endl;')
    *///]]]
    // generated using cog:
    cout << "public api, shouldnt change within major version:" << endl;
    cout << "    gpuindex=[gpu device index; default value is gpu if present, cpu otw.] (" << config.gpuIndex << ")" << endl;
    cout << "    weightsfile=[file to read weights from] (" << config.weightsFile << ")" << endl;
    cout << "    batchsize=[batch size] (" << config.batchSize << ")" << endl;
    cout << "" << endl;
    cout << "unstable, might change within major version:" << endl;
    cout << "    inputfile=[file to read inputs from, if empty, read stdin (default)] (" << config.inputFile << ")" << endl;
    cout << "    outputfile=[file to write outputs to, if empty, write to stdout] (" << config.outputFile << ")" << endl;
    cout << "    outputlayer=[layer to write output from, default -1 means: last layer] (" << config.outputLayer << ")" << endl;
    cout << "    writelabels=[write integer labels, instead of probabilities etc (default 0)] (" << config.writeLabels << ")" << endl;
    cout << "    outputformat=[output format [binary|text]] (" << config.outputFormat << ")" << endl;
    // [[[end]]]
}

int main( int argc, char *argv[] ) {
    Config config;
    if( argc == 2 && ( string(argv[1]) == "--help" || string(argv[1]) == "--?" || string(argv[1]) == "-?" || string(argv[1]) == "-h" ) ) {
        printUsage( argv, config );
    }
    for( int i = 1; i < argc; i++ ) {
        vector<string> splitkeyval = split( argv[i], "=" );
        if( splitkeyval.size() != 2 ) {
          cout << "Usage: " << argv[0] << " [key]=[value] [[key]=[value]] ..." << endl;
          exit(1);
        } else {
            string key = splitkeyval[0];
            string value = splitkeyval[1];
//            cout << "key [" << key << "]" << endl;
            /* [[[cog
                cog.outl('// generated using cog:')
                cog.outl('if( false ) {')
                for option in options:
                    name = option['name']
                    type = option['type']
                    cog.outl( '} else if( key == "' + name.lower() + '" ) {')
                    converter = '';
                    if type == 'int':
                        converter = 'atoi';
                    elif type == 'float':
                        converter = 'atof';
                    cog.outl( '    config.' + name + ' = ' + converter + '(value);')
            */// ]]]
            // generated using cog:
            if( false ) {
            } else if( key == "gpuindex" ) {
                config.gpuIndex = atoi(value);
            } else if( key == "weightsfile" ) {
                config.weightsFile = (value);
            } else if( key == "batchsize" ) {
                config.batchSize = atoi(value);
            } else if( key == "inputfile" ) {
                config.inputFile = (value);
            } else if( key == "outputfile" ) {
                config.outputFile = (value);
            } else if( key == "outputlayer" ) {
                config.outputLayer = atoi(value);
            } else if( key == "writelabels" ) {
                config.writeLabels = atoi(value);
            } else if( key == "outputformat" ) {
                config.outputFormat = (value);
            // [[[end]]]
            } else {
                cout << endl;
                cout << "Error: key '" << key << "' not recognised" << endl;
                cout << endl;
                printUsage( argv, config );
                cout << endl;
                return -1;
            }
        }
    }
    if( config.outputFormat != "text" && config.outputFormat != "binary" ) {
        cout << endl;
        cout << "outputformat must be 'text' or 'binary'" << endl;
        cout << endl;
        return -1;
    }
    try {
        go( config );
    } catch( runtime_error e ) {
        cout << "Something went wrong: " << e.what() << endl;
        return -1;
    }
}

<|endoftext|>


<commit_before>/**
 * Copyright (c) 2011-2015 libbitcoin developers (see AUTHORS)
 *
 * This file is part of libbitcoin.
 *
 * libbitcoin is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License with
 * additional permissions to the one published by the Free Software
 * Foundation, either version 3 of the License, or (at your option)
 * any later version. For more information see LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
#include <bitcoin/bitcoin/utility/threadpool.hpp>

#include <new>
#include <thread>
#include <bitcoin/bitcoin/network/asio.hpp>
#include <bitcoin/bitcoin/utility/thread.hpp>

namespace libbitcoin {

threadpool::threadpool(size_t number_threads, thread_priority priority)
{
    spawn(number_threads, priority);
}

threadpool::~threadpool()
{
    shutdown();
}

void threadpool::spawn(size_t number_threads, thread_priority priority)
{
    for (size_t i = 0; i < number_threads; ++i)
        spawn_once(priority);
}

void threadpool::spawn_once(thread_priority priority)
{
    // Work prevents the service from running out of work and terminating.
    if (!work_)
        work_ = std::make_unique<asio::service::work>(service_);

    const auto action = [this, priority]
    {
        set_thread_priority(priority);
        service_.run();
    };

    threads_.push_back(std::thread(action));
}

void threadpool::stop()
{
    // TODO: don't call this on shutdown, instead signal the application.
    // Signals the service to stop. Calls will return immediately without
    // invoking any handlers.
    service_.stop();
}

void threadpool::shutdown()
{
    // This does not terminate outstanding calls, it just allows service to
    // terminate once all outstanding calls are complete.
    work_ = nullptr;
}

void threadpool::join()
{
    // TODO: join the threadpool on destruct?
    for (auto& thread: threads_)
        if (thread.joinable())
            thread.join();
}

asio::service& threadpool::service()
{
    return service_;
}

const asio::service& threadpool::service() const
{
    return service_;
}

} // namespace libbitcoin

<commit_msg>Add missing include.<commit_after>/**
 * Copyright (c) 2011-2015 libbitcoin developers (see AUTHORS)
 *
 * This file is part of libbitcoin.
 *
 * libbitcoin is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License with
 * additional permissions to the one published by the Free Software
 * Foundation, either version 3 of the License, or (at your option)
 * any later version. For more information see LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
#include <bitcoin/bitcoin/utility/threadpool.hpp>

#include <memory>
#include <new>
#include <thread>
#include <bitcoin/bitcoin/network/asio.hpp>
#include <bitcoin/bitcoin/utility/thread.hpp>

namespace libbitcoin {

threadpool::threadpool(size_t number_threads, thread_priority priority)
{
    spawn(number_threads, priority);
}

threadpool::~threadpool()
{
    shutdown();
}

void threadpool::spawn(size_t number_threads, thread_priority priority)
{
    for (size_t i = 0; i < number_threads; ++i)
        spawn_once(priority);
}

void threadpool::spawn_once(thread_priority priority)
{
    // Work prevents the service from running out of work and terminating.
    if (!work_)
        work_ = std::make_unique<asio::service::work>(service_);

    const auto action = [this, priority]
    {
        set_thread_priority(priority);
        service_.run();
    };

    threads_.push_back(std::thread(action));
}

void threadpool::stop()
{
    // TODO: don't call this on shutdown, instead signal the application.
    // Signals the service to stop. Calls will return immediately without
    // invoking any handlers.
    service_.stop();
}

void threadpool::shutdown()
{
    // This does not terminate outstanding calls, it just allows service to
    // terminate once all outstanding calls are complete.
    work_ = nullptr;
}

void threadpool::join()
{
    // TODO: join the threadpool on destruct?
    for (auto& thread: threads_)
        if (thread.joinable())
            thread.join();
}

asio::service& threadpool::service()
{
    return service_;
}

const asio::service& threadpool::service() const
{
    return service_;
}

} // namespace libbitcoin

<|endoftext|>


<commit_before>#ifndef __NODE_MAPNIK_IMAGE_H__
#define __NODE_MAPNIK_IMAGE_H__

#include <v8.h>
#include <node.h>
#include <node_object_wrap.h>
#include <mapnik/graphics.hpp>
#include <boost/shared_ptr.hpp>

using namespace v8;
using namespace node;

typedef boost::shared_ptr<mapnik::image_32> image_ptr;

class Image: public node::ObjectWrap {
  public:
    static Persistent<FunctionTemplate> constructor;
    static void Initialize(Handle<Object> target);
    static Handle<Value> New(const Arguments &args);
    static Handle<Value> toString(const Arguments &args);
    static Handle<Value> open(const Arguments &args);

    Image(unsigned int width, unsigned int height);
    Image(image_ptr this_);

  private:
    ~Image();
    image_ptr this_;
};

#endif
<commit_msg>expose shared pointer on mapnik.Image<commit_after>#ifndef __NODE_MAPNIK_IMAGE_H__
#define __NODE_MAPNIK_IMAGE_H__

#include <v8.h>
#include <node.h>
#include <node_object_wrap.h>
#include <mapnik/graphics.hpp>
#include <boost/shared_ptr.hpp>

using namespace v8;
using namespace node;

typedef boost::shared_ptr<mapnik::image_32> image_ptr;

class Image: public node::ObjectWrap {
  public:
    static Persistent<FunctionTemplate> constructor;
    static void Initialize(Handle<Object> target);
    static Handle<Value> New(const Arguments &args);
    static Handle<Value> toString(const Arguments &args);
    static Handle<Value> open(const Arguments &args);

    Image(unsigned int width, unsigned int height);
    Image(image_ptr this_);
    inline image_ptr get() { return this_; }

  private:
    ~Image();
    image_ptr this_;
};

#endif
<|endoftext|>