<commit_before>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <map>
#include <string>
#include <vector>
#include <utility>

#include <gtest/gtest.h>
#include <pficommon/lang/shared_ptr.h>
#include <pficommon/lang/cast.h>
#include "../common/jsonconfig.hpp"
#include "nearest_neighbor_base.hpp"
#include "nearest_neighbor_factory.hpp"

using std::map;
using std::string;
using std::vector;
using pfi::lang::shared_ptr;

namespace jubatus {
namespace core {
namespace nearest_neighbor {

namespace {

// utility to write map<string, string> value in an expression
class make_config_type {
 public:
  make_config_type& operator()(const string& key, const string& value) {
    config_.insert(make_pair(key, value));
    return *this;
  }

  map<string, string> operator()() {
    map<string, string> ret;
    ret.swap(config_);
    return ret;
  }

 private:
  map<string, string> config_;
} make_config;

}  // namespace

class nearest_neighbor_test
    : public ::testing::TestWithParam<map<string, string> > {
 protected:
  void SetUp() {
    try {
      map<string, string> param = GetParam();
      string name = param["nearest_neighbor:name"];
      param.erase("nearest_neighbor:name");
      using pfi::text::json::json;
      json config_js(new pfi::text::json::json_object);
      for (map<string, string>::iterator it = param.begin();
           it != param.end(); ++it) {
        config_js.add(it->first, json(new pfi::text::json::json_integer(
            pfi::lang::lexical_cast<int>(it->second))));
      }

      using common::jsonconfig::config;
      using pfi::text::json::json;

      table_.reset(new table::column_table);
      nn_ = create_nearest_neighbor(
          name, config(config_js, ""), table_, "localhost");
    } catch (common::jsonconfig::cast_check_error& e) {
      std::cout << "In Setup():" <<e.what() << '\n';
      vector<pfi::lang::shared_ptr<common::jsonconfig::config_error> > v =
          e.errors();
      for (size_t i = 0; i < v.size(); ++i) {
        std::cout << v[i]->what() << '\n';
      }
      throw;
    }
  }

  table::column_table* get_table() {
    return table_.get();
  }
  nearest_neighbor_base* get_nn() {
    return nn_.get();
  }

 private:
  shared_ptr<table::column_table> table_;
  shared_ptr<nearest_neighbor_base> nn_;
};

TEST_P(nearest_neighbor_test, type) {
  map<string, string> param = GetParam();
  const string name = param["nearest_neighbor:name"];
  nearest_neighbor_base* nn = get_nn();
  EXPECT_EQ(name, nn->type());
}

TEST_P(nearest_neighbor_test, empty_get_all_row_ids) {
  nearest_neighbor_base* nn = get_nn();

  vector<string> ids;
  nn->get_all_row_ids(ids);
  EXPECT_TRUE(ids.empty());
}

TEST_P(nearest_neighbor_test, get_all_row_ids) {
  nearest_neighbor_base* nn = get_nn();

  vector<string> expect;
  expect.push_back("jubatus");
  expect.push_back("kyun");
  expect.push_back("hah hah");

  for (int count = 0; count < 2; ++count) {  // duplicated set_row
    for (size_t i = 0; i < expect.size(); ++i) {
      nn->set_row(expect[i], common::sfv_t());
    }
  }

  vector<string> actual;
  nn->get_all_row_ids(actual);

  EXPECT_EQ(expect, actual);
}

TEST_P(nearest_neighbor_test, empty_neighbor_row) {
  nearest_neighbor_base* nn = get_nn();

  vector<std::pair<string, float> > ids;
  nn->neighbor_row("", ids, 1);
  EXPECT_TRUE(ids.empty());

  nn->neighbor_row(common::sfv_t(), ids, 1);
  EXPECT_TRUE(ids.empty());
}

// TODO(beam2d): Write approximated test of neighbor_row().

const map<string, string> configs[] = {
  make_config("nearest_neighbor:name", "lsh")("hash_num", "64")(),
  make_config("nearest_neighbor:name", "minhash")("hash_num", "64")(),
  make_config(
      "nearest_neighbor:name", "euclid_lsh")(
      "hash_num", "64")()
};

INSTANTIATE_TEST_CASE_P(
    lsh_test,
    nearest_neighbor_test,
    ::testing::ValuesIn(configs));

}  // namespace nearest_neighbor
}  // namespace core
}  // namespace jubatus
<commit_msg>Removed an unnecessary using declaration.<commit_after>// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2011 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include <map>
#include <string>
#include <vector>
#include <utility>

#include <gtest/gtest.h>
#include <pficommon/lang/shared_ptr.h>
#include <pficommon/lang/cast.h>
#include "../common/jsonconfig.hpp"
#include "nearest_neighbor_base.hpp"
#include "nearest_neighbor_factory.hpp"

using std::map;
using std::string;
using std::vector;
using pfi::lang::shared_ptr;

namespace jubatus {
namespace core {
namespace nearest_neighbor {

namespace {

// utility to write map<string, string> value in an expression
class make_config_type {
 public:
  make_config_type& operator()(const string& key, const string& value) {
    config_.insert(make_pair(key, value));
    return *this;
  }

  map<string, string> operator()() {
    map<string, string> ret;
    ret.swap(config_);
    return ret;
  }

 private:
  map<string, string> config_;
} make_config;

}  // namespace

class nearest_neighbor_test
    : public ::testing::TestWithParam<map<string, string> > {
 protected:
  void SetUp() {
    try {
      map<string, string> param = GetParam();
      string name = param["nearest_neighbor:name"];
      param.erase("nearest_neighbor:name");
      using pfi::text::json::json;
      json config_js(new pfi::text::json::json_object);
      for (map<string, string>::iterator it = param.begin();
           it != param.end(); ++it) {
        config_js.add(it->first, json(new pfi::text::json::json_integer(
            pfi::lang::lexical_cast<int>(it->second))));
      }

      using common::jsonconfig::config;

      table_.reset(new table::column_table);
      nn_ = create_nearest_neighbor(
          name, config(config_js, ""), table_, "localhost");
    } catch (common::jsonconfig::cast_check_error& e) {
      std::cout << "In Setup():" <<e.what() << '\n';
      vector<pfi::lang::shared_ptr<common::jsonconfig::config_error> > v =
          e.errors();
      for (size_t i = 0; i < v.size(); ++i) {
        std::cout << v[i]->what() << '\n';
      }
      throw;
    }
  }

  table::column_table* get_table() {
    return table_.get();
  }
  nearest_neighbor_base* get_nn() {
    return nn_.get();
  }

 private:
  shared_ptr<table::column_table> table_;
  shared_ptr<nearest_neighbor_base> nn_;
};

TEST_P(nearest_neighbor_test, type) {
  map<string, string> param = GetParam();
  const string name = param["nearest_neighbor:name"];
  nearest_neighbor_base* nn = get_nn();
  EXPECT_EQ(name, nn->type());
}

TEST_P(nearest_neighbor_test, empty_get_all_row_ids) {
  nearest_neighbor_base* nn = get_nn();

  vector<string> ids;
  nn->get_all_row_ids(ids);
  EXPECT_TRUE(ids.empty());
}

TEST_P(nearest_neighbor_test, get_all_row_ids) {
  nearest_neighbor_base* nn = get_nn();

  vector<string> expect;
  expect.push_back("jubatus");
  expect.push_back("kyun");
  expect.push_back("hah hah");

  for (int count = 0; count < 2; ++count) {  // duplicated set_row
    for (size_t i = 0; i < expect.size(); ++i) {
      nn->set_row(expect[i], common::sfv_t());
    }
  }

  vector<string> actual;
  nn->get_all_row_ids(actual);

  EXPECT_EQ(expect, actual);
}

TEST_P(nearest_neighbor_test, empty_neighbor_row) {
  nearest_neighbor_base* nn = get_nn();

  vector<std::pair<string, float> > ids;
  nn->neighbor_row("", ids, 1);
  EXPECT_TRUE(ids.empty());

  nn->neighbor_row(common::sfv_t(), ids, 1);
  EXPECT_TRUE(ids.empty());
}

// TODO(beam2d): Write approximated test of neighbor_row().

const map<string, string> configs[] = {
  make_config("nearest_neighbor:name", "lsh")("hash_num", "64")(),
  make_config("nearest_neighbor:name", "minhash")("hash_num", "64")(),
  make_config(
      "nearest_neighbor:name", "euclid_lsh")(
      "hash_num", "64")()
};

INSTANTIATE_TEST_CASE_P(
    lsh_test,
    nearest_neighbor_test,
    ::testing::ValuesIn(configs));

}  // namespace nearest_neighbor
}  // namespace core
}  // namespace jubatus
<|endoftext|>


<commit_before>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow_lite_support/cc/task/text/universal_sentence_encoder_qa.h"

#include <algorithm>
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/container/flat_hash_map.h"  // from @com_google_absl
#include "absl/status/status.h"  // from @com_google_absl
#include "tensorflow_lite_support/cc/port/statusor.h"
#include "tensorflow_lite_support/cc/task/core/base_task_api.h"
#include "tensorflow_lite_support/cc/task/core/task_api_factory.h"
#include "tensorflow_lite_support/cc/task/core/task_utils.h"
#include "tensorflow_lite_support/cc/task/core/tflite_engine.h"
#include "tensorflow_lite_support/cc/task/text/proto/retrieval.pb.h"

namespace tflite {
namespace task {
namespace text {

using ::absl::Status;
using ::absl::StatusCode;
using internal::QAInput;
using internal::QAOutput;
using ::tflite::support::StatusOr;
using ::tflite::support::TfLiteSupportStatus;
using ::tflite::task::core::FindTensorIndexByName;
using ::tflite::task::core::PopulateTensor;
using ::tflite::task::core::PopulateVectorToRepeated;
using ::tflite::task::core::TaskAPIFactory;
using FeatureVector = UniversalSentenceEncoderQA::FeatureVector;

namespace {
constexpr char kQueryTextMetadataName[] = "inp_text";
constexpr char kResponseTextMetadataName[] = "res_text";
constexpr char kResponseContextMetadataName[] = "res_context";
constexpr char kQueryEncodingMetadataName[] = "query_encoding";
constexpr char kResponseEncodingMetadataName[] = "response_encoding";

constexpr char kQueryTextTensorName[] = "ParseExample/ParseExampleV2:1";
constexpr char kResponseTextTensorName[] = "ParseExample/ParseExampleV2:2";
constexpr char kResponseContextTensorName[] = "ParseExample/ParseExampleV2:3";
constexpr char kQueryEncodingTensorName[] = "Final/EncodeQuery/mul";
constexpr char kResponseEncodingTensorName[] = "Final/EncodeResult/mul";

constexpr int kDefaultInputTensorIndices[3] = {0, 1, 2};
constexpr int kDefaultOutputTensorIndices[2] = {0, 1};

// Sanity check for options to ensure required fields.
absl::Status SanityCheckOptions(const RetrievalOptions& options) {
  if (!options.has_base_options()) {
    return CreateStatusWithPayload(StatusCode::kInvalidArgument,
                                   "Missing mandatory `base_options` field",
                                   TfLiteSupportStatus::kInvalidArgumentError);
  }
  return absl::OkStatus();
}

// Copy vector from model output.
inline absl::Status CopyVector(const TfLiteTensor* src, FeatureVector* target) {
  return PopulateVectorToRepeated(src, target->mutable_value_float());
}

// Dot product of two vectors. Returns error status if size is mismatched.
template <class TCollection, class T = float>
tflite::support::StatusOr<T> Dot(const TCollection& a, const TCollection& b) {
  if (a.size() != b.size()) {
    return Status(
        StatusCode::kInvalidArgument,
        absl::StrFormat("mismatched vector size %d != %d", a.size(), b.size()));
  }
  auto dist = T();
  for (size_t i = 0; i < a.size(); ++i) {
    dist += T(a[i]) * T(b[i]);
  }
  return dist;
}
}  // namespace

namespace internal {
struct QAInput {
  std::string query_text;
  std::string response_text;
  std::string response_context;
};

struct QAOutput {
  // Directly populate from raw tensor pointers to avoid extra copy.
  const TfLiteTensor* query_encoding;     // not owned.
  const TfLiteTensor* response_encoding;  // not owned.
};
}  // namespace internal

constexpr int UniversalSentenceEncoderQA::kFinalEmbeddingSize;

StatusOr<RetrievalOutput> UniversalSentenceEncoderQA::Retrieve(
    const RetrievalInput& input) {
  if (input.query_text().empty()) {
    return Status(StatusCode::kInvalidArgument, "query text cannot be empty.");
  }
  if (input.responses().empty()) {
    return Status(StatusCode::kInvalidArgument, "responses cannot be empty.");
  }

  RetrievalOutput output;
  // Run inference.
  // (1) Query is only encoded for once.
  // (2) If responses are raw text, run model to get encoded vectors; otherwise,
  //     the encoded vector is kept from the input when given.
  for (size_t i = 0; i < input.responses_size(); ++i) {
    const auto& resp = input.responses(i);

    if (resp.has_raw_text()) {
      // If response is in th raw text, encode both query and response.
      const auto out = Run(input.query_text(), resp.raw_text().text(),
                           resp.raw_text().context());

      // Only encode query for the first time.
      if (i == 0) {
        RETURN_IF_ERROR(
            CopyVector(out.query_encoding, output.mutable_query_encoding()));
      }

      // For each answer, set the response result.
      auto r = output.mutable_response_results()->Add();
      RETURN_IF_ERROR(CopyVector(out.response_encoding, r->mutable_encoding()));
    } else {
      // If response is already encoded, encode query only and keep response
      // encoding.

      // Only encode query for the first time.
      if (i == 0) {
        const auto& q = EncodeQuery(input.query_text());
        *output.mutable_query_encoding() = q.value();
      }

      // For each answer, set the response result from text_encoding
      auto r = output.mutable_response_results()->Add();
      *r->mutable_encoding() = resp.text_encoding();
    }
  }

  // Calculate scores.
  for (size_t i = 0; i < output.response_results_size(); ++i) {
    auto* r = output.mutable_response_results(i);
    // TODO(tianlin): For a large size of results, it is more efficient to use
    // matrix multiplication.
    const auto& score = Similarity(output.query_encoding(), r->encoding());
    if (!score.ok()) {
      return score.status();
    }
    r->set_score(score.value());
  }
  return output;
}

StatusOr<FeatureVector> UniversalSentenceEncoderQA::EncodeQuery(
    absl::string_view query_text) {
  if (query_text.empty()) {
    return Status(StatusCode::kInvalidArgument, "query text cannot be empty.");
  }

  const auto& output = Run(query_text, "", "");
  FeatureVector v;
  RETURN_IF_ERROR(CopyVector(output.query_encoding, &v));
  return v;
}

StatusOr<FeatureVector> UniversalSentenceEncoderQA::EncodeResponse(
    absl::string_view response_text, absl::string_view response_context) {
  if (response_text.empty() && response_context.empty()) {
    return Status(
        StatusCode::kInvalidArgument,
        "either response text or context should be set to non-empty.");
  }

  const auto& output = Run("", response_text, response_context);
  FeatureVector v;
  RETURN_IF_ERROR(CopyVector(output.response_encoding, &v));
  return v;
}

StatusOr<float> UniversalSentenceEncoderQA::Similarity(const FeatureVector& a,
                                                       const FeatureVector& b) {
  const auto& av = a.value_float();
  const auto& bv = b.value_float();
  return Dot(av, bv);
}

std::vector<size_t> UniversalSentenceEncoderQA::Top(
    const RetrievalOutput& output, size_t k) {
  // Ensure k in [0, total_size).
  // If k == 0, it means that all outputs are ranked.
  if (k == 0) {
    k = output.response_results_size();
  } else {
    k = std::min(k, size_t(output.response_results_size()));
  }

  std::vector<size_t> pos(output.response_results_size());
  for (size_t i = 0; i < output.response_results_size(); ++i) {
    pos[i] = i;
  }
  const auto greater_score = [&output](size_t i, size_t j) {
    return output.response_results(i).score() >
           output.response_results(j).score();
  };
  std::partial_sort(pos.begin(), pos.begin() + k, pos.end(), greater_score);

  // Return sorted.
  return std::vector<size_t>(pos.begin(), pos.begin() + k);
}

Status UniversalSentenceEncoderQA::Preprocess(
    const std::vector<TfLiteTensor*>& input_tensors, const QAInput& input) {
  RETURN_IF_ERROR(
      PopulateTensor(input.query_text, input_tensors[input_indices_[0]]));
  RETURN_IF_ERROR(
      PopulateTensor(input.response_context, input_tensors[input_indices_[1]]));
  RETURN_IF_ERROR(
      PopulateTensor(input.response_text, input_tensors[input_indices_[2]]));

  return absl::OkStatus();
}

StatusOr<QAOutput> UniversalSentenceEncoderQA::Postprocess(
    const std::vector<const TfLiteTensor*>& output_tensors,
    const QAInput& /*input*/) {
  QAOutput output;
  output.query_encoding = output_tensors[output_indices_[0]];
  output.response_encoding = output_tensors[output_indices_[1]];
  return output;
}

internal::QAOutput UniversalSentenceEncoderQA::Run(
    absl::string_view query_text, absl::string_view response_text,
    absl::string_view response_context) {
  QAInput input;
  input.query_text = query_text;
  input.response_text = response_text;
  input.response_context = response_context;
  return Infer(input).value();
}

absl::Status UniversalSentenceEncoderQA::Init(
    std::unique_ptr<RetrievalOptions> options) {
  options_ = std::move(options);

  auto input_tensors = GetInputTensors();
  if (input_tensors.size() < 3) {
    return CreateStatusWithPayload(
        StatusCode::kInvalidArgument,
        "The number of input tensors should be at least 3, including query text"
        " input, response context input, response text input.",
        TfLiteSupportStatus::kInvalidArgumentError);
  }

  auto* input_tensor_metadatas =
      GetMetadataExtractor()->GetInputTensorMetadata();
  input_indices_ = {
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kQueryTextMetadataName, kQueryTextTensorName),
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kResponseContextMetadataName,
                            kResponseContextTensorName),
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kResponseTextMetadataName,
                            kResponseTextTensorName)};
  if (std::find(input_indices_.begin(), input_indices_.end(), -1) !=
      input_indices_.end()) {
    // Use the default index if any input tensor is not found.
    input_indices_ = std::vector<int>(std::begin(kDefaultInputTensorIndices),
                                      std::end(kDefaultInputTensorIndices));
  }

  auto output_tensors = GetOutputTensors();
  if (output_tensors.size() < 2) {
    return CreateStatusWithPayload(
        StatusCode::kInvalidArgument,
        "The number of output tensors should be at least 2, including query "
        "encoding output, response encoding output.",
        TfLiteSupportStatus::kInvalidArgumentError);
  }

  auto* output_tensor_metadatas =
      GetMetadataExtractor()->GetOutputTensorMetadata();
  output_indices_ = {
      FindTensorIndexByName(output_tensors, output_tensor_metadatas,
                            kQueryEncodingMetadataName,
                            kQueryEncodingTensorName),
      FindTensorIndexByName(output_tensors, output_tensor_metadatas,
                            kResponseEncodingMetadataName,
                            kResponseEncodingTensorName)};

  if (std::find(output_indices_.begin(), output_indices_.end(), -1) !=
      output_indices_.end()) {
    // Use the default index if any output tensor is not found.
    output_indices_ = std::vector<int>(std::begin(kDefaultOutputTensorIndices),
                                       std::end(kDefaultOutputTensorIndices));
  }

  return absl::OkStatus();
}  // namespace retrieval

StatusOr<std::unique_ptr<UniversalSentenceEncoderQA>>
UniversalSentenceEncoderQA::CreateFromOption(
    const RetrievalOptions& options,
    std::unique_ptr<tflite::OpResolver> resolver) {
  RETURN_IF_ERROR(SanityCheckOptions(options));

  // Copy options to ensure the ExternalFile outlives the duration of this
  // created object.
  auto options_copy = absl::make_unique<RetrievalOptions>(options);

  ASSIGN_OR_RETURN(
      auto encoder,
      TaskAPIFactory::CreateFromBaseOptions<UniversalSentenceEncoderQA>(
          &options_copy->base_options(), std::move(resolver)));

  RETURN_IF_ERROR(encoder->Init(std::move(options_copy)));
  return encoder;
}

}  // namespace text
}  // namespace task
}  // namespace tflite
<commit_msg>Internal change<commit_after>/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow_lite_support/cc/task/text/universal_sentence_encoder_qa.h"

#include <algorithm>
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/container/flat_hash_map.h"  // from @com_google_absl
#include "absl/status/status.h"  // from @com_google_absl
#include "tensorflow_lite_support/cc/port/statusor.h"
#include "tensorflow_lite_support/cc/task/core/base_task_api.h"
#include "tensorflow_lite_support/cc/task/core/task_api_factory.h"
#include "tensorflow_lite_support/cc/task/core/task_utils.h"
#include "tensorflow_lite_support/cc/task/core/tflite_engine.h"
#include "tensorflow_lite_support/cc/task/text/proto/retrieval.pb.h"

namespace tflite {
namespace task {
namespace text {

using ::absl::Status;
using ::absl::StatusCode;
using internal::QAInput;
using internal::QAOutput;
using ::tflite::support::StatusOr;
using ::tflite::support::TfLiteSupportStatus;
using ::tflite::task::core::FindTensorIndexByName;
using ::tflite::task::core::PopulateTensor;
using ::tflite::task::core::PopulateVectorToRepeated;
using ::tflite::task::core::TaskAPIFactory;
using FeatureVector = UniversalSentenceEncoderQA::FeatureVector;

namespace {
constexpr char kQueryTextMetadataName[] = "inp_text";
constexpr char kResponseContextMetadataName[] = "res_context";
constexpr char kResponseTextMetadataName[] = "res_text";
constexpr char kQueryEncodingMetadataName[] = "query_encoding";
constexpr char kResponseEncodingMetadataName[] = "response_encoding";

constexpr char kQueryTextTensorName[] = "ParseExample/ParseExampleV2:1";
constexpr char kResponseContextTensorName[] = "ParseExample/ParseExampleV2:2";
constexpr char kResponseTextTensorName[] = "ParseExample/ParseExampleV2:3";
constexpr char kQueryEncodingTensorName[] = "Final/EncodeQuery/mul";
constexpr char kResponseEncodingTensorName[] = "Final/EncodeResult/mul";

constexpr int kDefaultInputTensorIndices[3] = {0, 1, 2};
constexpr int kDefaultOutputTensorIndices[2] = {0, 1};

// Sanity check for options to ensure required fields.
absl::Status SanityCheckOptions(const RetrievalOptions& options) {
  if (!options.has_base_options()) {
    return CreateStatusWithPayload(StatusCode::kInvalidArgument,
                                   "Missing mandatory `base_options` field",
                                   TfLiteSupportStatus::kInvalidArgumentError);
  }
  return absl::OkStatus();
}

// Copy vector from model output.
inline absl::Status CopyVector(const TfLiteTensor* src, FeatureVector* target) {
  return PopulateVectorToRepeated(src, target->mutable_value_float());
}

// Dot product of two vectors. Returns error status if size is mismatched.
template <class TCollection, class T = float>
tflite::support::StatusOr<T> Dot(const TCollection& a, const TCollection& b) {
  if (a.size() != b.size()) {
    return Status(
        StatusCode::kInvalidArgument,
        absl::StrFormat("mismatched vector size %d != %d", a.size(), b.size()));
  }
  auto dist = T();
  for (size_t i = 0; i < a.size(); ++i) {
    dist += T(a[i]) * T(b[i]);
  }
  return dist;
}
}  // namespace

namespace internal {
struct QAInput {
  std::string query_text;
  std::string response_text;
  std::string response_context;
};

struct QAOutput {
  // Directly populate from raw tensor pointers to avoid extra copy.
  const TfLiteTensor* query_encoding;     // not owned.
  const TfLiteTensor* response_encoding;  // not owned.
};
}  // namespace internal

constexpr int UniversalSentenceEncoderQA::kFinalEmbeddingSize;

StatusOr<RetrievalOutput> UniversalSentenceEncoderQA::Retrieve(
    const RetrievalInput& input) {
  if (input.query_text().empty()) {
    return Status(StatusCode::kInvalidArgument, "query text cannot be empty.");
  }
  if (input.responses().empty()) {
    return Status(StatusCode::kInvalidArgument, "responses cannot be empty.");
  }

  RetrievalOutput output;
  // Run inference.
  // (1) Query is only encoded for once.
  // (2) If responses are raw text, run model to get encoded vectors; otherwise,
  //     the encoded vector is kept from the input when given.
  for (size_t i = 0; i < input.responses_size(); ++i) {
    const auto& resp = input.responses(i);

    if (resp.has_raw_text()) {
      // If response is in th raw text, encode both query and response.
      const auto out = Run(input.query_text(), resp.raw_text().text(),
                           resp.raw_text().context());

      // Only encode query for the first time.
      if (i == 0) {
        RETURN_IF_ERROR(
            CopyVector(out.query_encoding, output.mutable_query_encoding()));
      }

      // For each answer, set the response result.
      auto r = output.mutable_response_results()->Add();
      RETURN_IF_ERROR(CopyVector(out.response_encoding, r->mutable_encoding()));
    } else {
      // If response is already encoded, encode query only and keep response
      // encoding.

      // Only encode query for the first time.
      if (i == 0) {
        const auto& q = EncodeQuery(input.query_text());
        *output.mutable_query_encoding() = q.value();
      }

      // For each answer, set the response result from text_encoding
      auto r = output.mutable_response_results()->Add();
      *r->mutable_encoding() = resp.text_encoding();
    }
  }

  // Calculate scores.
  for (size_t i = 0; i < output.response_results_size(); ++i) {
    auto* r = output.mutable_response_results(i);
    // TODO(tianlin): For a large size of results, it is more efficient to use
    // matrix multiplication.
    const auto& score = Similarity(output.query_encoding(), r->encoding());
    if (!score.ok()) {
      return score.status();
    }
    r->set_score(score.value());
  }
  return output;
}

StatusOr<FeatureVector> UniversalSentenceEncoderQA::EncodeQuery(
    absl::string_view query_text) {
  if (query_text.empty()) {
    return Status(StatusCode::kInvalidArgument, "query text cannot be empty.");
  }

  const auto& output = Run(query_text, "", "");
  FeatureVector v;
  RETURN_IF_ERROR(CopyVector(output.query_encoding, &v));
  return v;
}

StatusOr<FeatureVector> UniversalSentenceEncoderQA::EncodeResponse(
    absl::string_view response_text, absl::string_view response_context) {
  if (response_text.empty() && response_context.empty()) {
    return Status(
        StatusCode::kInvalidArgument,
        "either response text or context should be set to non-empty.");
  }

  const auto& output = Run("", response_text, response_context);
  FeatureVector v;
  RETURN_IF_ERROR(CopyVector(output.response_encoding, &v));
  return v;
}

StatusOr<float> UniversalSentenceEncoderQA::Similarity(const FeatureVector& a,
                                                       const FeatureVector& b) {
  const auto& av = a.value_float();
  const auto& bv = b.value_float();
  return Dot(av, bv);
}

std::vector<size_t> UniversalSentenceEncoderQA::Top(
    const RetrievalOutput& output, size_t k) {
  // Ensure k in [0, total_size).
  // If k == 0, it means that all outputs are ranked.
  if (k == 0) {
    k = output.response_results_size();
  } else {
    k = std::min(k, size_t(output.response_results_size()));
  }

  std::vector<size_t> pos(output.response_results_size());
  for (size_t i = 0; i < output.response_results_size(); ++i) {
    pos[i] = i;
  }
  const auto greater_score = [&output](size_t i, size_t j) {
    return output.response_results(i).score() >
           output.response_results(j).score();
  };
  std::partial_sort(pos.begin(), pos.begin() + k, pos.end(), greater_score);

  // Return sorted.
  return std::vector<size_t>(pos.begin(), pos.begin() + k);
}

Status UniversalSentenceEncoderQA::Preprocess(
    const std::vector<TfLiteTensor*>& input_tensors, const QAInput& input) {
  RETURN_IF_ERROR(
      PopulateTensor(input.query_text, input_tensors[input_indices_[0]]));
  RETURN_IF_ERROR(
      PopulateTensor(input.response_context, input_tensors[input_indices_[1]]));
  RETURN_IF_ERROR(
      PopulateTensor(input.response_text, input_tensors[input_indices_[2]]));

  return absl::OkStatus();
}

StatusOr<QAOutput> UniversalSentenceEncoderQA::Postprocess(
    const std::vector<const TfLiteTensor*>& output_tensors,
    const QAInput& /*input*/) {
  QAOutput output;
  output.query_encoding = output_tensors[output_indices_[0]];
  output.response_encoding = output_tensors[output_indices_[1]];
  return output;
}

internal::QAOutput UniversalSentenceEncoderQA::Run(
    absl::string_view query_text, absl::string_view response_text,
    absl::string_view response_context) {
  QAInput input;
  input.query_text = query_text;
  input.response_text = response_text;
  input.response_context = response_context;
  return Infer(input).value();
}

absl::Status UniversalSentenceEncoderQA::Init(
    std::unique_ptr<RetrievalOptions> options) {
  options_ = std::move(options);

  auto input_tensors = GetInputTensors();
  if (input_tensors.size() < 3) {
    return CreateStatusWithPayload(
        StatusCode::kInvalidArgument,
        "The number of input tensors should be at least 3, including query text"
        " input, response context input, response text input.",
        TfLiteSupportStatus::kInvalidArgumentError);
  }

  auto* input_tensor_metadatas =
      GetMetadataExtractor()->GetInputTensorMetadata();
  input_indices_ = {
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kQueryTextMetadataName, kQueryTextTensorName),
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kResponseContextMetadataName,
                            kResponseContextTensorName),
      FindTensorIndexByName(input_tensors, input_tensor_metadatas,
                            kResponseTextMetadataName,
                            kResponseTextTensorName)};
  if (std::find(input_indices_.begin(), input_indices_.end(), -1) !=
      input_indices_.end()) {
    // Use the default index if any input tensor is not found.
    input_indices_ = std::vector<int>(std::begin(kDefaultInputTensorIndices),
                                      std::end(kDefaultInputTensorIndices));
  }

  auto output_tensors = GetOutputTensors();
  if (output_tensors.size() < 2) {
    return CreateStatusWithPayload(
        StatusCode::kInvalidArgument,
        "The number of output tensors should be at least 2, including query "
        "encoding output, response encoding output.",
        TfLiteSupportStatus::kInvalidArgumentError);
  }

  auto* output_tensor_metadatas =
      GetMetadataExtractor()->GetOutputTensorMetadata();
  output_indices_ = {
      FindTensorIndexByName(output_tensors, output_tensor_metadatas,
                            kQueryEncodingMetadataName,
                            kQueryEncodingTensorName),
      FindTensorIndexByName(output_tensors, output_tensor_metadatas,
                            kResponseEncodingMetadataName,
                            kResponseEncodingTensorName)};

  if (std::find(output_indices_.begin(), output_indices_.end(), -1) !=
      output_indices_.end()) {
    // Use the default index if any output tensor is not found.
    output_indices_ = std::vector<int>(std::begin(kDefaultOutputTensorIndices),
                                       std::end(kDefaultOutputTensorIndices));
  }

  return absl::OkStatus();
}  // namespace retrieval

StatusOr<std::unique_ptr<UniversalSentenceEncoderQA>>
UniversalSentenceEncoderQA::CreateFromOption(
    const RetrievalOptions& options,
    std::unique_ptr<tflite::OpResolver> resolver) {
  RETURN_IF_ERROR(SanityCheckOptions(options));

  // Copy options to ensure the ExternalFile outlives the duration of this
  // created object.
  auto options_copy = absl::make_unique<RetrievalOptions>(options);

  ASSIGN_OR_RETURN(
      auto encoder,
      TaskAPIFactory::CreateFromBaseOptions<UniversalSentenceEncoderQA>(
          &options_copy->base_options(), std::move(resolver)));

  RETURN_IF_ERROR(encoder->Init(std::move(options_copy)));
  return encoder;
}

}  // namespace text
}  // namespace task
}  // namespace tflite
<|endoftext|>


<commit_before>/**
 * Sorting integers - quick sort method.
 */
#include <vector>
#include <algorithm>
#include <iostream>

using namespace std;

namespace sort {
	void counting(vector<int> &v)
	{
		vector<int> tmp;
		unsigned int i, j;

		for (i = 0; i < v.size(); ++i)
		{
			if (tmp.size() < v[i])
			{
				tmp.resize(v[i] + 1);
			}
			tmp[v[i]]++;
		}

		v.clear();

		for (i = 0; i < tmp.size(); ++i)
		{
			for (j = 0; j < tmp[i]; ++j)
			{
				v.push_back(i);
			}
		}
	}
}<commit_msg>fix doc<commit_after>/**
 * Sorting integers - counting sort method.
 */
#include <vector>
#include <algorithm>
#include <iostream>

using namespace std;

namespace sort {
	void counting(vector<int> &v)
	{
		vector<int> tmp;
		unsigned int i, j;

		for (i = 0; i < v.size(); ++i)
		{
			if (tmp.size() < v[i])
			{
				tmp.resize(v[i] + 1);
			}
			tmp[v[i]]++;
		}

		v.clear();

		for (i = 0; i < tmp.size(); ++i)
		{
			for (j = 0; j < tmp[i]; ++j)
			{
				v.push_back(i);
			}
		}
	}
}<|endoftext|>


<commit_before>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
The MIT License

Copyright (c) 2008, 2009 Flusspferd contributors (see "CONTRIBUTORS" or
                                       http://flusspferd.org/contributors.txt)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "flusspferd/object.hpp"
#include "flusspferd/property_iterator.hpp"
#include "flusspferd/function.hpp"
#include "flusspferd/exception.hpp"
#include "flusspferd/local_root_scope.hpp"
#include "flusspferd/root.hpp"
#include "flusspferd/arguments.hpp"
#include "flusspferd/string.hpp"
#include "flusspferd/native_object_base.hpp"
#include "flusspferd/spidermonkey/init.hpp"
#include "flusspferd/spidermonkey/value.hpp"
#include "flusspferd/spidermonkey/object.hpp"
#include <js/jsapi.h>

using namespace flusspferd;

object::object() : Impl::object_impl(0) { }
object::~object() { }

void object::seal(bool deep) {
  if (is_null())
    throw exception("Could not seal object (object is null)");
  if (!JS_SealObject(Impl::current_context(), get(), deep))
    throw exception("Could not seal object");
}

object object::parent() {
  if (is_null())
    throw exception("Could not get object parent (object is null)");
  return Impl::wrap_object(JS_GetParent(Impl::current_context(), get()));
}

object object::prototype() {
  if (is_null())
    throw exception("Could not get object prototype (object is null)");
  return Impl::wrap_object(JS_GetPrototype(Impl::current_context(), get()));
}

void object::set_parent(object const &o) {
  if (is_null())
    throw exception("Could not set parent (object is null)");
  if (!JS_SetParent(Impl::current_context(), get(), o.get_const()))
    throw exception("Could not set object parent");
}

void object::set_prototype(object const &o) {
  if (is_null())
    throw exception("Could not set object prototype (object is null)");
  if (!JS_SetPrototype(Impl::current_context(), get(), o.get_const()))
    throw exception("Could not set object prototype");
}

value object::set_property(char const *name, value const &v_) {
  if (is_null())
    throw exception("Could not set property (object is null)");
  value v = v_;
  if (!JS_SetProperty(Impl::current_context(), get(), name,
                      Impl::get_jsvalp(v)))
    throw exception("Could not set property");
  return v;
}

value object::set_property(std::string const &name, value const &v) {
  return set_property(name.c_str(), v);
}

value object::set_property(value const &id, value const &v_) {
  if (is_null())
    throw exception("Could not set property (object is null)");
  local_root_scope scope;
  value v = v_;
  string name = id.to_string();
  if (!JS_SetPropertyById(Impl::current_context(), get(),
                          Impl::get_jsid(id),
                          Impl::get_jsvalp(v)))
    throw exception("Could not set property");
  return v;
}

value object::get_property(char const *name) const {
  if (is_null())
    throw exception("Could not get property (object is null)");
  value result;
  if (!JS_GetProperty(Impl::current_context(), get_const(),
                      name, Impl::get_jsvalp(result)))
    throw exception("Could not get property");
  return result;
}

value object::get_property(std::string const &name) const {
  return get_property(name.c_str());
}

value object::get_property(value const &id) const {
  if (is_null())
    throw exception("Could not get property (object is null)");
  value result;
  local_root_scope scope;
  string name = id.to_string();
  if (!JS_GetPropertyById(Impl::current_context(), get_const(),
                          Impl::get_jsid(id),
                          Impl::get_jsvalp(result)))
    throw exception("Could not get property");
  return result;
}

bool object::has_property(char const *name) const {
  if (is_null())
    throw exception("Could not check property (object is null)");
  JSBool foundp;
  if (!JS_HasProperty(Impl::current_context(), get_const(), name,
                      &foundp))
    throw exception("Could not check property");
  return foundp;
}

bool object::has_property(std::string const &name) const {
  return has_property(name.c_str());
}

bool object::has_property(value const &id) const {
  if (is_null())
    throw exception("Could not check property (object is null)");
  local_root_scope scope;
  string name = id.to_string();
  JSBool foundp;
  if (!JS_HasPropertyById(Impl::current_context(), get_const(),
                          Impl::get_jsid(id),
                          &foundp))
    throw exception("Could not check property");
  return foundp;
}

bool object::has_own_property(char const *name_) const {
  local_root_scope scope;
  string name(name_);
  return has_own_property(name);
}

bool object::has_own_property(std::string const &name_) const {
  local_root_scope scope;
  string name(name_);
  return has_own_property(name);
}

bool object::has_own_property(value const &id) const {
  if (is_null())
    throw exception("Could not check property (object is null)");

  JSBool has;
  string name = id.to_string();
  if (!JS_AlreadyHasOwnPropertyById(Impl::current_context(), get_const(),
                                    Impl::get_jsid(id), &has))
  {
    throw exception("Unable to check for own property");
  }
  return has;
}

void object::delete_property(char const *name) {
  if (is_null())
    throw exception("Could not delete property (object is null)");
  if (!JS_DeleteProperty(Impl::current_context(), get(), name))
    throw exception("Could not delete property");
}

void object::delete_property(std::string const &name) {
  delete_property(name.c_str());
}

void object::delete_property(value const &id) {
  if (is_null())
    throw exception("Could not delete property (object is null)");
  local_root_scope scope;
  string name = id.to_string();
  jsval dummy;
  if (!JS_DeletePropertyById2(Impl::current_context(), get(),
                              Impl::get_jsid(id), &dummy))
    throw exception("Could not delete property");
}

property_iterator object::begin() const {
  if (is_null())
    throw exception("Could not create iterator for null object");
  return property_iterator(*this);
}

property_iterator object::end() const {
  if (is_null())
    throw exception("Could not create iterator for null object");
  return property_iterator();
}

namespace {
  /// common function of define_property
  void extract_attributes(
    property_attributes const &attrs,
    JSObject *&getter_o,
    JSObject *&setter_o,
    unsigned &sm_flags)
  {
    function getter;
    if (attrs.getter) getter = attrs.getter.get();
    function setter;
    if (attrs.setter) setter = attrs.setter.get();

    getter_o = Impl::get_object(getter);
    setter_o = Impl::get_object(setter);

    unsigned const flags = attrs.flags;
    if (~flags & dont_enumerate) sm_flags |= JSPROP_ENUMERATE;
    if (flags & read_only_property) sm_flags |= JSPROP_READONLY;
    if (flags & permanent_property) sm_flags |= JSPROP_PERMANENT;
    if (flags & shared_property) sm_flags |= JSPROP_SHARED;

    if (getter_o) sm_flags |= JSPROP_GETTER;
    if (setter_o) sm_flags |= JSPROP_SETTER;
  }
}

void object::define_property(
  value const &id,
  value const &init_value,
  property_attributes const &attrs)
{
  if (is_null())
    throw exception("Could not define property (object is null)");

  root_value id_r(id);
  root_value v(init_value);

  JSObject *getter_o = 0x0;
  JSObject *setter_o = 0x0;
  unsigned sm_flags = 0;
  extract_attributes(attrs, getter_o, setter_o, sm_flags);

  if (!JS_DefinePropertyById(Impl::current_context(),
                             get_const(),
                             Impl::get_jsid(id),
                             Impl::get_jsval(v),
                             *(JSPropertyOp*) &getter_o,
                             *(JSPropertyOp*) &setter_o,
                             sm_flags))
    throw exception("Could not define property");
}

void object::define_property(
  string const &name,
  value const &init_value,
  property_attributes const &attrs)
{
  if (is_null())
    throw exception("Could not define property (object is null)");

  root_string name_r(name);
  root_value v(init_value);

  JSObject *getter_o = 0x0;
  JSObject *setter_o = 0x0;
  unsigned sm_flags = 0;
  extract_attributes(attrs, getter_o, setter_o, sm_flags);

  if (!JS_DefineUCProperty(Impl::current_context(),
                           get_const(),
                           name.data(), name.length(),
                           Impl::get_jsval(v),
                           *(JSPropertyOp*) &getter_o,
                           *(JSPropertyOp*) &setter_o,
                           sm_flags))
    throw exception("Could not define property");
}

bool object::is_null() const {
  return !get_const();
}

value object::apply(object const &fn, arguments const &arg_) {
  if (is_null())
    throw exception("Could not apply function (object is null)");

  if (fn.is_null())
    throw exception("Could not apply function (function is null)");

  value fnv(fn);
  root_value result((value()));

  arguments arg(arg_);

  JSContext *cx = Impl::current_context();

  JSBool status = JS_CallFunctionValue(
      cx,
      get(),
      Impl::get_jsval(fnv),
      arg.size(),
      Impl::get_arguments(arg),
      Impl::get_jsvalp(result));

  if (!status) {
    if (JS_IsExceptionPending(cx))
      throw exception("Could not call function");
    else
      throw js_quit();
  }

  return result;
}

value object::call(char const *fn, arguments const &arg_) {
  if (is_null())
    throw exception("Could not call function (object is null)");

  root_value result((value()));

  arguments arg(arg_);

  JSContext *cx = Impl::current_context();

  JSBool status = JS_CallFunctionName(
      cx,
      get(),
      fn,
      arg.size(),
      Impl::get_arguments(arg),
      Impl::get_jsvalp(result));

  if (!status) {
    if (JS_IsExceptionPending(cx))
      throw exception("Could not call function");
    else
      throw js_quit();
  }

  return result;
}

value object::call(std::string const &fn, arguments const &arg) {
  return call(fn.c_str(), arg);
}

value object::call(object o, arguments const &arg) {
  return o.apply(*this, arg);
}

value object::call(arguments const &arg) {
  return call(global(), arg);
}

bool object::is_array() const {
  if (is_null())
    return false;
  return JS_IsArrayObject(Impl::current_context(), get_const());
}

bool object::is_generator() const {
  if (is_null())
    return false;

  JSContext *ctx = Impl::current_context();
  (void)ctx;

  // There seems to be no way with the SM API to get the standard Generator
  // JSClass back. SO make a best effort guess
  JSClass *our_class = JS_GET_CLASS(ctx, get_const());

  return strcmp(our_class->name, "Generator") == 0
      && get_property("next").is_function();
}

namespace {
  property_attributes
  set_property_attributes(
    uintN sm_flags, void *getter_op, void *setter_op)
  {
    property_attributes attrs;
    if (~sm_flags & JSPROP_ENUMERATE)
      attrs.flags = attrs.flags | dont_enumerate;
    if (sm_flags & JSPROP_PERMANENT)
      attrs.flags = attrs.flags | permanent_property;
    if (sm_flags & JSPROP_READONLY)
      attrs.flags = attrs.flags | read_only_property;
    if (sm_flags & JSPROP_SHARED)
      attrs.flags = attrs.flags | shared_property;

    if (getter_op) {
      if (sm_flags & JSPROP_GETTER) {
        attrs.getter = Impl::wrap_object((JSObject*)getter_op);
      } else {
        // What do i set attrs.getter to here....?
      }
    }

    if (setter_op) {
      if (sm_flags & JSPROP_SETTER) {
        attrs.setter = Impl::wrap_object((JSObject*)setter_op);
      } else {
        // What do i set attrs.setter to here....?
      }
    }
    return attrs;
  }
}

boost::optional<property_attributes>
object::get_property_attributes(string const &name) const {
  if (is_null())
    throw exception("Could not get property attributes (object is null)");

  flusspferd::root_string name_r(name);

  JSBool found;
  void *getter_op, *setter_op;
  uintN sm_flags;
  JSBool success = JS_GetUCPropertyAttrsGetterAndSetter(
          Impl::current_context(), get_const(), name.data(), name.length(),
          &sm_flags, &found,
          (JSPropertyOp*)&getter_op, (JSPropertyOp*)&setter_op);

  if (!success)
    throw exception("Could not query property attributes");

  if (!found)
    return boost::none;

  return set_property_attributes(sm_flags, getter_op, setter_op);
}

boost::optional<property_attributes>
object::get_property_attributes(value const &id) const {
  if (is_null())
    throw exception("Could not get property attributes (object is null)");

  flusspferd::root_value id_r(id);

  JSBool found;
  void *getter_op, *setter_op;
  uintN sm_flags;
  JSBool success = JS_GetPropertyAttrsGetterAndSetterById(
          Impl::current_context(), get_const(), Impl::get_jsid(id),
          &sm_flags, &found,
          (JSPropertyOp*)&getter_op, (JSPropertyOp*)&setter_op);

  if (!success)
    throw exception("Could not query property attributes");

  if (!found)
    return boost::none;

  return set_property_attributes(sm_flags, getter_op, setter_op);
}
<commit_msg>core: harden object::*<commit_after>// vim:ts=2:sw=2:expandtab:autoindent:filetype=cpp:
/*
The MIT License

Copyright (c) 2008, 2009 Flusspferd contributors (see "CONTRIBUTORS" or
                                       http://flusspferd.org/contributors.txt)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#include "flusspferd/object.hpp"
#include "flusspferd/property_iterator.hpp"
#include "flusspferd/function.hpp"
#include "flusspferd/exception.hpp"
#include "flusspferd/local_root_scope.hpp"
#include "flusspferd/root.hpp"
#include "flusspferd/arguments.hpp"
#include "flusspferd/string.hpp"
#include "flusspferd/native_object_base.hpp"
#include "flusspferd/spidermonkey/init.hpp"
#include "flusspferd/spidermonkey/value.hpp"
#include "flusspferd/spidermonkey/object.hpp"
#include <js/jsapi.h>

using namespace flusspferd;

object::object() : Impl::object_impl(0) { }
object::~object() { }

void object::seal(bool deep) {
  if (is_null())
    throw exception("Could not seal object (object is null)");
  if (!JS_SealObject(Impl::current_context(), get(), deep))
    throw exception("Could not seal object");
}

object object::parent() {
  if (is_null())
    throw exception("Could not get object parent (object is null)");
  return Impl::wrap_object(JS_GetParent(Impl::current_context(), get()));
}

object object::prototype() {
  if (is_null())
    throw exception("Could not get object prototype (object is null)");
  return Impl::wrap_object(JS_GetPrototype(Impl::current_context(), get()));
}

void object::set_parent(object const &o) {
  if (is_null())
    throw exception("Could not set parent (object is null)");
  if (!JS_SetParent(Impl::current_context(), get(), o.get_const()))
    throw exception("Could not set object parent");
}

void object::set_prototype(object const &o) {
  if (is_null())
    throw exception("Could not set object prototype (object is null)");
  if (!JS_SetPrototype(Impl::current_context(), get(), o.get_const()))
    throw exception("Could not set object prototype");
}

value object::set_property(char const *name, value const &v_) {
  if (is_null())
    throw exception("Could not set property (object is null)");
  root_value v(v_);
  if (!JS_SetProperty(Impl::current_context(), get(), name,
                      Impl::get_jsvalp(v)))
    throw exception("Could not set property");
  return v;
}

value object::set_property(std::string const &name, value const &v) {
  return set_property(name.c_str(), v);
}

value object::set_property(value const &id_, value const &v_) {
  if (is_null())
    throw exception("Could not set property (object is null)");
  root_value id(id_);
  root_value v(v_);
  if (!JS_SetPropertyById(Impl::current_context(), get(),
                          Impl::get_jsid(id),
                          Impl::get_jsvalp(v)))
    throw exception("Could not set property");
  return v;
}

value object::get_property(char const *name) const {
  if (is_null())
    throw exception("Could not get property (object is null)");
  root_value result;
  if (!JS_GetProperty(Impl::current_context(), get_const(),
                      name, Impl::get_jsvalp(result)))
    throw exception("Could not get property");
  return result;
}

value object::get_property(std::string const &name) const {
  return get_property(name.c_str());
}

value object::get_property(value const &id_) const {
  if (is_null())
    throw exception("Could not get property (object is null)");
  root_value result;
  root_value id(id_);
  if (!JS_GetPropertyById(Impl::current_context(), get_const(),
                          Impl::get_jsid(id),
                          Impl::get_jsvalp(result)))
    throw exception("Could not get property");
  return result;
}

bool object::has_property(char const *name) const {
  if (is_null())
    throw exception("Could not check property (object is null)");
  JSBool foundp;
  if (!JS_HasProperty(Impl::current_context(), get_const(), name,
                      &foundp))
    throw exception("Could not check property");
  return foundp;
}

bool object::has_property(std::string const &name) const {
  return has_property(name.c_str());
}

bool object::has_property(value const &id_) const {
  if (is_null())
    throw exception("Could not check property (object is null)");
  root_value id(id_);
  JSBool foundp;
  if (!JS_HasPropertyById(Impl::current_context(), get_const(),
                          Impl::get_jsid(id),
                          &foundp))
    throw exception("Could not check property");
  return foundp;
}

bool object::has_own_property(char const *name_) const {
  root_string name(name_);
  return has_own_property(name);
}

bool object::has_own_property(std::string const &name_) const {
  root_string name(name_);
  return has_own_property(name);
}

bool object::has_own_property(value const &id) const {
  if (is_null())
    throw exception("Could not check property (object is null)");

  JSBool has;
  string name = id.to_string();
  if (!JS_AlreadyHasOwnPropertyById(Impl::current_context(), get_const(),
                                    Impl::get_jsid(id), &has))
  {
    throw exception("Unable to check for own property");
  }
  return has;
}

void object::delete_property(char const *name) {
  if (is_null())
    throw exception("Could not delete property (object is null)");
  if (!JS_DeleteProperty(Impl::current_context(), get(), name))
    throw exception("Could not delete property");
}

void object::delete_property(std::string const &name) {
  delete_property(name.c_str());
}

void object::delete_property(value const &id_) {
  if (is_null())
    throw exception("Could not delete property (object is null)");
  root_value id(id_);
  jsval dummy;
  if (!JS_DeletePropertyById2(Impl::current_context(), get(),
                              Impl::get_jsid(id), &dummy))
    throw exception("Could not delete property");
}

property_iterator object::begin() const {
  if (is_null())
    throw exception("Could not create iterator for null object");
  return property_iterator(*this);
}

property_iterator object::end() const {
  if (is_null())
    throw exception("Could not create iterator for null object");
  return property_iterator();
}

namespace {
  /// common function of define_property
  void extract_attributes(
    property_attributes const &attrs,
    JSObject *&getter_o,
    JSObject *&setter_o,
    unsigned &sm_flags)
  {
    function getter;
    if (attrs.getter) getter = attrs.getter.get();
    function setter;
    if (attrs.setter) setter = attrs.setter.get();

    getter_o = Impl::get_object(getter);
    setter_o = Impl::get_object(setter);

    unsigned const flags = attrs.flags;
    if (~flags & dont_enumerate) sm_flags |= JSPROP_ENUMERATE;
    if (flags & read_only_property) sm_flags |= JSPROP_READONLY;
    if (flags & permanent_property) sm_flags |= JSPROP_PERMANENT;
    if (flags & shared_property) sm_flags |= JSPROP_SHARED;

    if (getter_o) sm_flags |= JSPROP_GETTER;
    if (setter_o) sm_flags |= JSPROP_SETTER;
  }
}

void object::define_property(
  value const &id,
  value const &init_value,
  property_attributes const &attrs)
{
  if (is_null())
    throw exception("Could not define property (object is null)");

  root_value id_r(id);
  root_value v(init_value);

  JSObject *getter_o = 0x0;
  JSObject *setter_o = 0x0;
  unsigned sm_flags = 0;
  extract_attributes(attrs, getter_o, setter_o, sm_flags);

  if (!JS_DefinePropertyById(Impl::current_context(),
                             get_const(),
                             Impl::get_jsid(id),
                             Impl::get_jsval(v),
                             *(JSPropertyOp*) &getter_o,
                             *(JSPropertyOp*) &setter_o,
                             sm_flags))
    throw exception("Could not define property");
}

void object::define_property(
  string const &name,
  value const &init_value,
  property_attributes const &attrs)
{
  if (is_null())
    throw exception("Could not define property (object is null)");

  root_string name_r(name);
  root_value v(init_value);

  JSObject *getter_o = 0x0;
  JSObject *setter_o = 0x0;
  unsigned sm_flags = 0;
  extract_attributes(attrs, getter_o, setter_o, sm_flags);

  if (!JS_DefineUCProperty(Impl::current_context(),
                           get_const(),
                           name.data(), name.length(),
                           Impl::get_jsval(v),
                           *(JSPropertyOp*) &getter_o,
                           *(JSPropertyOp*) &setter_o,
                           sm_flags))
    throw exception("Could not define property");
}

bool object::is_null() const {
  return !get_const();
}

value object::apply(object const &fn, arguments const &arg_) {
  if (is_null())
    throw exception("Could not apply function (object is null)");

  if (fn.is_null())
    throw exception("Could not apply function (function is null)");

  value fnv(fn);
  root_value result((value()));

  arguments arg(arg_);

  JSContext *cx = Impl::current_context();

  JSBool status = JS_CallFunctionValue(
      cx,
      get(),
      Impl::get_jsval(fnv),
      arg.size(),
      Impl::get_arguments(arg),
      Impl::get_jsvalp(result));

  if (!status) {
    if (JS_IsExceptionPending(cx))
      throw exception("Could not call function");
    else
      throw js_quit();
  }

  return result;
}

value object::call(char const *fn, arguments const &arg_) {
  if (is_null())
    throw exception("Could not call function (object is null)");

  root_value result((value()));

  arguments arg(arg_);

  JSContext *cx = Impl::current_context();

  JSBool status = JS_CallFunctionName(
      cx,
      get(),
      fn,
      arg.size(),
      Impl::get_arguments(arg),
      Impl::get_jsvalp(result));

  if (!status) {
    if (JS_IsExceptionPending(cx))
      throw exception("Could not call function");
    else
      throw js_quit();
  }

  return result;
}

value object::call(std::string const &fn, arguments const &arg) {
  return call(fn.c_str(), arg);
}

value object::call(object o, arguments const &arg) {
  return o.apply(*this, arg);
}

value object::call(arguments const &arg) {
  return call(global(), arg);
}

bool object::is_array() const {
  if (is_null())
    return false;
  return JS_IsArrayObject(Impl::current_context(), get_const());
}

bool object::is_generator() const {
  if (is_null())
    return false;

  JSContext *ctx = Impl::current_context();
  (void)ctx;

  // There seems to be no way with the SM API to get the standard Generator
  // JSClass back. SO make a best effort guess
  JSClass *our_class = JS_GET_CLASS(ctx, get_const());

  return strcmp(our_class->name, "Generator") == 0
      && get_property("next").is_function();
}

namespace {
  property_attributes
  set_property_attributes(
    uintN sm_flags, void *getter_op, void *setter_op)
  {
    property_attributes attrs;
    if (~sm_flags & JSPROP_ENUMERATE)
      attrs.flags = attrs.flags | dont_enumerate;
    if (sm_flags & JSPROP_PERMANENT)
      attrs.flags = attrs.flags | permanent_property;
    if (sm_flags & JSPROP_READONLY)
      attrs.flags = attrs.flags | read_only_property;
    if (sm_flags & JSPROP_SHARED)
      attrs.flags = attrs.flags | shared_property;

    if (getter_op) {
      if (sm_flags & JSPROP_GETTER) {
        attrs.getter = Impl::wrap_object((JSObject*)getter_op);
      } else {
        // What do i set attrs.getter to here....?
      }
    }

    if (setter_op) {
      if (sm_flags & JSPROP_SETTER) {
        attrs.setter = Impl::wrap_object((JSObject*)setter_op);
      } else {
        // What do i set attrs.setter to here....?
      }
    }
    return attrs;
  }
}

boost::optional<property_attributes>
object::get_property_attributes(string const &name) const {
  if (is_null())
    throw exception("Could not get property attributes (object is null)");

  flusspferd::root_string name_r(name);

  JSBool found;
  void *getter_op, *setter_op;
  uintN sm_flags;
  JSBool success = JS_GetUCPropertyAttrsGetterAndSetter(
          Impl::current_context(), get_const(), name.data(), name.length(),
          &sm_flags, &found,
          (JSPropertyOp*)&getter_op, (JSPropertyOp*)&setter_op);

  if (!success)
    throw exception("Could not query property attributes");

  if (!found)
    return boost::none;

  return set_property_attributes(sm_flags, getter_op, setter_op);
}

boost::optional<property_attributes>
object::get_property_attributes(value const &id) const {
  if (is_null())
    throw exception("Could not get property attributes (object is null)");

  flusspferd::root_value id_r(id);

  JSBool found;
  void *getter_op, *setter_op;
  uintN sm_flags;
  JSBool success = JS_GetPropertyAttrsGetterAndSetterById(
          Impl::current_context(), get_const(), Impl::get_jsid(id),
          &sm_flags, &found,
          (JSPropertyOp*)&getter_op, (JSPropertyOp*)&setter_op);

  if (!success)
    throw exception("Could not query property attributes");

  if (!found)
    return boost::none;

  return set_property_attributes(sm_flags, getter_op, setter_op);
}
<|endoftext|>


<commit_before>#ifndef STAN_LANG_GENERATOR_HPP
#define STAN_LANG_GENERATOR_HPP

// TODO(carpenter): move into AST
#include <stan/lang/generator/has_lb.hpp>
#include <stan/lang/generator/has_lub.hpp>
#include <stan/lang/generator/has_ub.hpp>
#include <stan/lang/generator/has_only_int_args.hpp>

// TODO(carpenter): move into general utilities
#include <stan/lang/generator/to_string.hpp>

// utilities
#include <stan/lang/generator/constants.hpp>
#include <stan/lang/generator/fun_scalar_type.hpp>

// visitor classes for tests
#include <stan/lang/generator/is_numbered_statement_vis.hpp>

// visitor classes for generation
#include <stan/lang/generator/constrained_param_names_visgen.hpp>
#include <stan/lang/generator/dump_member_var_visgen.hpp>
#include <stan/lang/generator/expression_visgen.hpp>
#include <stan/lang/generator/printable_visgen.hpp>
#include <stan/lang/generator/idx_visgen.hpp>
#include <stan/lang/generator/idx_user_visgen.hpp>
#include <stan/lang/generator/init_local_var_visgen.hpp>
#include <stan/lang/generator/init_vars_visgen.hpp>
#include <stan/lang/generator/init_visgen.hpp>
#include <stan/lang/generator/local_var_decl_visgen.hpp>
#include <stan/lang/generator/local_var_init_nan_visgen.hpp>
#include <stan/lang/generator/member_var_decl_visgen.hpp>
#include <stan/lang/generator/set_param_ranges_visgen.hpp>
#include <stan/lang/generator/statement_visgen.hpp>
#include <stan/lang/generator/unconstrained_param_names_visgen.hpp>
#include <stan/lang/generator/validate_var_decl_visgen.hpp>
#include <stan/lang/generator/validate_transformed_params_visgen.hpp>
#include <stan/lang/generator/var_resizing_visgen.hpp>
#include <stan/lang/generator/var_size_validating_visgen.hpp>
#include <stan/lang/generator/visgen.hpp>
#include <stan/lang/generator/write_array_visgen.hpp>
#include <stan/lang/generator/write_array_vars_visgen.hpp>
#include <stan/lang/generator/write_dims_visgen.hpp>
#include <stan/lang/generator/write_param_names_visgen.hpp>

// generation functions, starts from generate_cpp
#include <stan/lang/generator/generate_arg_decl.hpp>
#include <stan/lang/generator/generate_array_var_type.hpp>
#include <stan/lang/generator/generate_array_builder_adds.hpp>
#include <stan/lang/generator/generate_bare_type.hpp>
#include <stan/lang/generator/generate_catch_throw_located.hpp>
#include <stan/lang/generator/generate_class_decl.hpp>
#include <stan/lang/generator/generate_class_decl_end.hpp>
#include <stan/lang/generator/generate_comment.hpp>
#include <stan/lang/generator/generate_constrained_param_names_method.hpp>
#include <stan/lang/generator/generate_constructor.hpp>
#include <stan/lang/generator/generate_cpp.hpp>
#include <stan/lang/generator/generate_destructor.hpp>
#include <stan/lang/generator/generate_dims_method.hpp>
#include <stan/lang/generator/generate_eigen_index_expression.hpp>
#include <stan/lang/generator/generate_expression.hpp>
#include <stan/lang/generator/generate_function.hpp>
#include <stan/lang/generator/generate_functions.hpp>
#include <stan/lang/generator/generate_function_arguments.hpp>
#include <stan/lang/generator/generate_function_body.hpp>
#include <stan/lang/generator/generate_function_functor.hpp>
#include <stan/lang/generator/generate_function_inline_return_type.hpp>
#include <stan/lang/generator/generate_function_template_parameters.hpp>
#include <stan/lang/generator/generate_functor_arguments.hpp>
#include <stan/lang/generator/generate_globals.hpp>
#include <stan/lang/generator/generate_idx.hpp>
#include <stan/lang/generator/generate_idxs.hpp>
#include <stan/lang/generator/generate_idxs_user.hpp>
#include <stan/lang/generator/generate_indent.hpp>
#include <stan/lang/generator/generate_include.hpp>
#include <stan/lang/generator/generate_includes.hpp>
#include <stan/lang/generator/generate_indexed_expr.hpp>
#include <stan/lang/generator/generate_indexed_expr_user.hpp>
#include <stan/lang/generator/generate_init_method.hpp>
#include <stan/lang/generator/generate_initializer.hpp>
#include <stan/lang/generator/generate_initialization.hpp>
#include <stan/lang/generator/generate_local_var_decls.hpp>
#include <stan/lang/generator/generate_local_var_inits.hpp>
//#include <stan/lang/generator/generate_located_statement.hpp>
//#include <stan/lang/generator/generate_located_statements.hpp>
#include <stan/lang/generator/generate_log_prob.hpp>
#include <stan/lang/generator/generate_member_var_decls.hpp>
#include <stan/lang/generator/generate_member_var_decls_all.hpp>
#include <stan/lang/generator/generate_member_var_inits.hpp>
#include <stan/lang/generator/generate_model_name_method.hpp>
#include <stan/lang/generator/generate_model_typedef.hpp>
#include <stan/lang/generator/generate_namespace_end.hpp>
#include <stan/lang/generator/generate_namespace_start.hpp>
#include <stan/lang/generator/generate_param_names_method.hpp>
#include <stan/lang/generator/generate_printable.hpp>
#include <stan/lang/generator/generate_private_decl.hpp>
#include <stan/lang/generator/generate_program_reader_fun.hpp>
#include <stan/lang/generator/generate_propto_default_function.hpp>
#include <stan/lang/generator/generate_propto_default_function_body.hpp>
#include <stan/lang/generator/generate_public_decl.hpp>
#include <stan/lang/generator/generate_quoted_expression.hpp>
#include <stan/lang/generator/generate_quoted_string.hpp>
#include <stan/lang/generator/generate_real_var_type.hpp>
#include <stan/lang/generator/generate_set_param_ranges.hpp>
#include <stan/lang/generator/generate_statement.hpp>
#include <stan/lang/generator/generate_type.hpp>
#include <stan/lang/generator/generate_typedef.hpp>
#include <stan/lang/generator/generate_typedefs.hpp>
#include <stan/lang/generator/generate_try.hpp>
#include <stan/lang/generator/generate_unconstrained_param_names_method.hpp>
#include <stan/lang/generator/generate_using.hpp>
#include <stan/lang/generator/generate_using_namespace.hpp>
#include <stan/lang/generator/generate_usings.hpp>
#include <stan/lang/generator/generate_validate_context_size.hpp>
#include <stan/lang/generator/generate_validate_positive.hpp>
#include <stan/lang/generator/generate_validate_transformed_params.hpp>
#include <stan/lang/generator/generate_validate_var_decl.hpp>
#include <stan/lang/generator/generate_validate_var_decls.hpp>
#include <stan/lang/generator/generate_var_resizing.hpp>
#include <stan/lang/generator/generate_version_comment.hpp>
#include <stan/lang/generator/generate_void_statement.hpp>
#include <stan/lang/generator/generate_write_array_method.hpp>

#endif
<commit_msg>remove generate_located_statment functions<commit_after>#ifndef STAN_LANG_GENERATOR_HPP
#define STAN_LANG_GENERATOR_HPP

// TODO(carpenter): move into AST
#include <stan/lang/generator/has_lb.hpp>
#include <stan/lang/generator/has_lub.hpp>
#include <stan/lang/generator/has_ub.hpp>
#include <stan/lang/generator/has_only_int_args.hpp>

// TODO(carpenter): move into general utilities
#include <stan/lang/generator/to_string.hpp>

// utilities
#include <stan/lang/generator/constants.hpp>
#include <stan/lang/generator/fun_scalar_type.hpp>

// visitor classes for tests
#include <stan/lang/generator/is_numbered_statement_vis.hpp>

// visitor classes for generation
#include <stan/lang/generator/constrained_param_names_visgen.hpp>
#include <stan/lang/generator/dump_member_var_visgen.hpp>
#include <stan/lang/generator/expression_visgen.hpp>
#include <stan/lang/generator/printable_visgen.hpp>
#include <stan/lang/generator/idx_visgen.hpp>
#include <stan/lang/generator/idx_user_visgen.hpp>
#include <stan/lang/generator/init_local_var_visgen.hpp>
#include <stan/lang/generator/init_vars_visgen.hpp>
#include <stan/lang/generator/init_visgen.hpp>
#include <stan/lang/generator/local_var_decl_visgen.hpp>
#include <stan/lang/generator/local_var_init_nan_visgen.hpp>
#include <stan/lang/generator/member_var_decl_visgen.hpp>
#include <stan/lang/generator/set_param_ranges_visgen.hpp>
#include <stan/lang/generator/statement_visgen.hpp>
#include <stan/lang/generator/unconstrained_param_names_visgen.hpp>
#include <stan/lang/generator/validate_var_decl_visgen.hpp>
#include <stan/lang/generator/validate_transformed_params_visgen.hpp>
#include <stan/lang/generator/var_resizing_visgen.hpp>
#include <stan/lang/generator/var_size_validating_visgen.hpp>
#include <stan/lang/generator/visgen.hpp>
#include <stan/lang/generator/write_array_visgen.hpp>
#include <stan/lang/generator/write_array_vars_visgen.hpp>
#include <stan/lang/generator/write_dims_visgen.hpp>
#include <stan/lang/generator/write_param_names_visgen.hpp>

// generation functions, starts from generate_cpp
#include <stan/lang/generator/generate_arg_decl.hpp>
#include <stan/lang/generator/generate_array_var_type.hpp>
#include <stan/lang/generator/generate_array_builder_adds.hpp>
#include <stan/lang/generator/generate_bare_type.hpp>
#include <stan/lang/generator/generate_catch_throw_located.hpp>
#include <stan/lang/generator/generate_class_decl.hpp>
#include <stan/lang/generator/generate_class_decl_end.hpp>
#include <stan/lang/generator/generate_comment.hpp>
#include <stan/lang/generator/generate_constrained_param_names_method.hpp>
#include <stan/lang/generator/generate_constructor.hpp>
#include <stan/lang/generator/generate_cpp.hpp>
#include <stan/lang/generator/generate_destructor.hpp>
#include <stan/lang/generator/generate_dims_method.hpp>
#include <stan/lang/generator/generate_eigen_index_expression.hpp>
#include <stan/lang/generator/generate_expression.hpp>
#include <stan/lang/generator/generate_function.hpp>
#include <stan/lang/generator/generate_functions.hpp>
#include <stan/lang/generator/generate_function_arguments.hpp>
#include <stan/lang/generator/generate_function_body.hpp>
#include <stan/lang/generator/generate_function_functor.hpp>
#include <stan/lang/generator/generate_function_inline_return_type.hpp>
#include <stan/lang/generator/generate_function_template_parameters.hpp>
#include <stan/lang/generator/generate_functor_arguments.hpp>
#include <stan/lang/generator/generate_globals.hpp>
#include <stan/lang/generator/generate_idx.hpp>
#include <stan/lang/generator/generate_idxs.hpp>
#include <stan/lang/generator/generate_idxs_user.hpp>
#include <stan/lang/generator/generate_indent.hpp>
#include <stan/lang/generator/generate_include.hpp>
#include <stan/lang/generator/generate_includes.hpp>
#include <stan/lang/generator/generate_indexed_expr.hpp>
#include <stan/lang/generator/generate_indexed_expr_user.hpp>
#include <stan/lang/generator/generate_init_method.hpp>
#include <stan/lang/generator/generate_initializer.hpp>
#include <stan/lang/generator/generate_initialization.hpp>
#include <stan/lang/generator/generate_local_var_decls.hpp>
#include <stan/lang/generator/generate_local_var_inits.hpp>
#include <stan/lang/generator/generate_log_prob.hpp>
#include <stan/lang/generator/generate_member_var_decls.hpp>
#include <stan/lang/generator/generate_member_var_decls_all.hpp>
#include <stan/lang/generator/generate_member_var_inits.hpp>
#include <stan/lang/generator/generate_model_name_method.hpp>
#include <stan/lang/generator/generate_model_typedef.hpp>
#include <stan/lang/generator/generate_namespace_end.hpp>
#include <stan/lang/generator/generate_namespace_start.hpp>
#include <stan/lang/generator/generate_param_names_method.hpp>
#include <stan/lang/generator/generate_printable.hpp>
#include <stan/lang/generator/generate_private_decl.hpp>
#include <stan/lang/generator/generate_program_reader_fun.hpp>
#include <stan/lang/generator/generate_propto_default_function.hpp>
#include <stan/lang/generator/generate_propto_default_function_body.hpp>
#include <stan/lang/generator/generate_public_decl.hpp>
#include <stan/lang/generator/generate_quoted_expression.hpp>
#include <stan/lang/generator/generate_quoted_string.hpp>
#include <stan/lang/generator/generate_real_var_type.hpp>
#include <stan/lang/generator/generate_set_param_ranges.hpp>
#include <stan/lang/generator/generate_statement.hpp>
#include <stan/lang/generator/generate_type.hpp>
#include <stan/lang/generator/generate_typedef.hpp>
#include <stan/lang/generator/generate_typedefs.hpp>
#include <stan/lang/generator/generate_try.hpp>
#include <stan/lang/generator/generate_unconstrained_param_names_method.hpp>
#include <stan/lang/generator/generate_using.hpp>
#include <stan/lang/generator/generate_using_namespace.hpp>
#include <stan/lang/generator/generate_usings.hpp>
#include <stan/lang/generator/generate_validate_context_size.hpp>
#include <stan/lang/generator/generate_validate_positive.hpp>
#include <stan/lang/generator/generate_validate_transformed_params.hpp>
#include <stan/lang/generator/generate_validate_var_decl.hpp>
#include <stan/lang/generator/generate_validate_var_decls.hpp>
#include <stan/lang/generator/generate_var_resizing.hpp>
#include <stan/lang/generator/generate_version_comment.hpp>
#include <stan/lang/generator/generate_void_statement.hpp>
#include <stan/lang/generator/generate_write_array_method.hpp>

#endif
<|endoftext|>


<commit_before>// Copyright (c) 2014-2018 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chain.h>
#include <util/system.h>
#include <test/test_chaincoin.h>

#include <vector>

#include <boost/test/unit_test.hpp>

#define SKIPLIST_LENGTH 300000

BOOST_FIXTURE_TEST_SUITE(skiplist_tests, BasicTestingSetup)

BOOST_AUTO_TEST_CASE(skiplist_test)
{
    std::vector<CBlockIndex> vIndex(SKIPLIST_LENGTH);

    for (int i=0; i<SKIPLIST_LENGTH; i++) {
        vIndex[i].nHeight = i;
        vIndex[i].pprev = (i == 0) ? nullptr : &vIndex[i - 1];
        vIndex[i].BuildSkip();
    }

    for (int i=0; i<SKIPLIST_LENGTH; i++) {
        if (i > 0) {
            BOOST_CHECK(vIndex[i].pskip == &vIndex[vIndex[i].pskip->nHeight]);
            BOOST_CHECK(vIndex[i].pskip->nHeight < i);
        } else {
            BOOST_CHECK(vIndex[i].pskip == nullptr);
        }
    }

    for (int i=0; i < 1000; i++) {
        int from = InsecureRandRange(SKIPLIST_LENGTH - 1);
        int to = InsecureRandRange(from + 1);

        BOOST_CHECK(vIndex[SKIPLIST_LENGTH - 1].GetAncestor(from) == &vIndex[from]);
        BOOST_CHECK(vIndex[from].GetAncestor(to) == &vIndex[to]);
        BOOST_CHECK(vIndex[from].GetAncestor(0) == vIndex.data());
    }
}

BOOST_AUTO_TEST_CASE(getlocator_test)
{
    // Build a main chain 100000 blocks long.
    std::vector<uint256> vHashMain(100000);
    std::vector<CBlockIndex> vBlocksMain(100000);
    for (unsigned int i=0; i<vBlocksMain.size(); i++) {
        vHashMain[i] = ArithToUint256(i); // Set the hash equal to the height, so we can quickly check the distances.
        vBlocksMain[i].nHeight = i;
        vBlocksMain[i].pprev = i ? &vBlocksMain[i - 1] : nullptr;
        vBlocksMain[i].phashBlock = &vHashMain[i];
        vBlocksMain[i].BuildSkip();
        BOOST_CHECK_EQUAL((int)UintToArith256(vBlocksMain[i].GetBlockHash()).GetLow64(), vBlocksMain[i].nHeight);
        BOOST_CHECK(vBlocksMain[i].pprev == nullptr || vBlocksMain[i].nHeight == vBlocksMain[i].pprev->nHeight + 1);
    }

    // Build a branch that splits off at block 49999, 50000 blocks long.
    std::vector<uint256> vHashSide(50000);
    std::vector<CBlockIndex> vBlocksSide(50000);
    for (unsigned int i=0; i<vBlocksSide.size(); i++) {
        vHashSide[i] = ArithToUint256(i + 50000 + (arith_uint256(1) << 128)); // Add 1<<128 to the hashes, so GetLow64() still returns the height.
        vBlocksSide[i].nHeight = i + 50000;
        vBlocksSide[i].pprev = i ? &vBlocksSide[i - 1] : (vBlocksMain.data()+49999);
        vBlocksSide[i].phashBlock = &vHashSide[i];
        vBlocksSide[i].BuildSkip();
        BOOST_CHECK_EQUAL((int)UintToArith256(vBlocksSide[i].GetBlockHash()).GetLow64(), vBlocksSide[i].nHeight);
        BOOST_CHECK(vBlocksSide[i].pprev == nullptr || vBlocksSide[i].nHeight == vBlocksSide[i].pprev->nHeight + 1);
    }

    // Build a CChain for the main branch.
    CChain chain;
    chain.SetTip(&vBlocksMain.back());

    // Test 100 random starting points for locators.
    for (int n=0; n<100; n++) {
        int r = InsecureRandRange(150000);
        CBlockIndex* tip = (r < 100000) ? &vBlocksMain[r] : &vBlocksSide[r - 100000];
        CBlockLocator locator = chain.GetLocator(tip);

        // The first result must be the block itself, the last one must be genesis.
        BOOST_CHECK(locator.vHave.front() == tip->GetBlockHash());
        BOOST_CHECK(locator.vHave.back() == vBlocksMain[0].GetBlockHash());

        // Entries 1 through 11 (inclusive) go back one step each.
        for (unsigned int i = 1; i < 12 && i < locator.vHave.size() - 1; i++) {
            BOOST_CHECK_EQUAL(UintToArith256(locator.vHave[i]).GetLow64(), tip->nHeight - i);
        }

        // The further ones (excluding the last one) go back with exponential steps.
        unsigned int dist = 2;
        for (unsigned int i = 12; i < locator.vHave.size() - 1; i++) {
            BOOST_CHECK_EQUAL(UintToArith256(locator.vHave[i - 1]).GetLow64() - UintToArith256(locator.vHave[i]).GetLow64(), dist);
            dist *= 2;
        }
    }
}

BOOST_AUTO_TEST_CASE(findearliestatleast_test)
{
    std::vector<uint256> vHashMain(100000);
    std::vector<CBlockIndex> vBlocksMain(100000);
    for (unsigned int i=0; i<vBlocksMain.size(); i++) {
        vHashMain[i] = ArithToUint256(i); // Set the hash equal to the height
        vBlocksMain[i].nHeight = i;
        vBlocksMain[i].pprev = i ? &vBlocksMain[i - 1] : nullptr;
        vBlocksMain[i].phashBlock = &vHashMain[i];
        vBlocksMain[i].BuildSkip();
        if (i < 10) {
            vBlocksMain[i].nTime = i;
            vBlocksMain[i].nTimeMax = i;
        } else {
            // randomly choose something in the range [MTP, MTP*2]
            int64_t medianTimePast = vBlocksMain[i].GetMedianTimePast();
            int r = InsecureRandRange(medianTimePast);
            vBlocksMain[i].nTime = r + medianTimePast;
            vBlocksMain[i].nTimeMax = std::max(vBlocksMain[i].nTime, vBlocksMain[i-1].nTimeMax);
        }
    }
    // Check that we set nTimeMax up correctly.
    unsigned int curTimeMax = 0;
    for (unsigned int i=0; i<vBlocksMain.size(); ++i) {
        curTimeMax = std::max(curTimeMax, vBlocksMain[i].nTime);
        BOOST_CHECK(curTimeMax == vBlocksMain[i].nTimeMax);
    }

    // Build a CChain for the main branch.
    CChain chain;
    chain.SetTip(&vBlocksMain.back());

    // Verify that FindEarliestAtLeast is correct.
    for (unsigned int i=0; i<10000; ++i) {
        // Pick a random element in vBlocksMain.
        int r = InsecureRandRange(vBlocksMain.size());
        int64_t test_time = vBlocksMain[r].nTime;
        CBlockIndex *ret = chain.FindEarliestAtLeast(test_time);
        BOOST_CHECK(ret->nTimeMax >= test_time);
        BOOST_CHECK((ret->pprev==nullptr) || ret->pprev->nTimeMax < test_time);
        BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret);
    }
}

BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test)
{
    std::list<CBlockIndex> blocks;
    for (const unsigned int timeMax : {100, 100, 100, 200, 200, 200, 300, 300, 300}) {
        CBlockIndex* prev = blocks.empty() ? nullptr : &blocks.back();
        blocks.emplace_back();
        blocks.back().nHeight = prev ? prev->nHeight + 1 : 0;
        blocks.back().pprev = prev;
        blocks.back().BuildSkip();
        blocks.back().nTimeMax = timeMax;
    }

    CChain chain;
    chain.SetTip(&blocks.back());

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6);
    BOOST_CHECK(!chain.FindEarliestAtLeast(350));

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0);

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min())->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::min())->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1)->nHeight, 0);
    BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max()));
    BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max()));
    BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1));
}

BOOST_AUTO_TEST_SUITE_END()
<commit_msg>Remove duplicated code<commit_after>// Copyright (c) 2014-2018 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chain.h>
#include <util/system.h>
#include <test/test_chaincoin.h>

#include <vector>

#include <boost/test/unit_test.hpp>

#define SKIPLIST_LENGTH 300000

BOOST_FIXTURE_TEST_SUITE(skiplist_tests, BasicTestingSetup)

BOOST_AUTO_TEST_CASE(skiplist_test)
{
    std::vector<CBlockIndex> vIndex(SKIPLIST_LENGTH);

    for (int i=0; i<SKIPLIST_LENGTH; i++) {
        vIndex[i].nHeight = i;
        vIndex[i].pprev = (i == 0) ? nullptr : &vIndex[i - 1];
        vIndex[i].BuildSkip();
    }

    for (int i=0; i<SKIPLIST_LENGTH; i++) {
        if (i > 0) {
            BOOST_CHECK(vIndex[i].pskip == &vIndex[vIndex[i].pskip->nHeight]);
            BOOST_CHECK(vIndex[i].pskip->nHeight < i);
        } else {
            BOOST_CHECK(vIndex[i].pskip == nullptr);
        }
    }

    for (int i=0; i < 1000; i++) {
        int from = InsecureRandRange(SKIPLIST_LENGTH - 1);
        int to = InsecureRandRange(from + 1);

        BOOST_CHECK(vIndex[SKIPLIST_LENGTH - 1].GetAncestor(from) == &vIndex[from]);
        BOOST_CHECK(vIndex[from].GetAncestor(to) == &vIndex[to]);
        BOOST_CHECK(vIndex[from].GetAncestor(0) == vIndex.data());
    }
}

BOOST_AUTO_TEST_CASE(getlocator_test)
{
    // Build a main chain 100000 blocks long.
    std::vector<uint256> vHashMain(100000);
    std::vector<CBlockIndex> vBlocksMain(100000);
    for (unsigned int i=0; i<vBlocksMain.size(); i++) {
        vHashMain[i] = ArithToUint256(i); // Set the hash equal to the height, so we can quickly check the distances.
        vBlocksMain[i].nHeight = i;
        vBlocksMain[i].pprev = i ? &vBlocksMain[i - 1] : nullptr;
        vBlocksMain[i].phashBlock = &vHashMain[i];
        vBlocksMain[i].BuildSkip();
        BOOST_CHECK_EQUAL((int)UintToArith256(vBlocksMain[i].GetBlockHash()).GetLow64(), vBlocksMain[i].nHeight);
        BOOST_CHECK(vBlocksMain[i].pprev == nullptr || vBlocksMain[i].nHeight == vBlocksMain[i].pprev->nHeight + 1);
    }

    // Build a branch that splits off at block 49999, 50000 blocks long.
    std::vector<uint256> vHashSide(50000);
    std::vector<CBlockIndex> vBlocksSide(50000);
    for (unsigned int i=0; i<vBlocksSide.size(); i++) {
        vHashSide[i] = ArithToUint256(i + 50000 + (arith_uint256(1) << 128)); // Add 1<<128 to the hashes, so GetLow64() still returns the height.
        vBlocksSide[i].nHeight = i + 50000;
        vBlocksSide[i].pprev = i ? &vBlocksSide[i - 1] : (vBlocksMain.data()+49999);
        vBlocksSide[i].phashBlock = &vHashSide[i];
        vBlocksSide[i].BuildSkip();
        BOOST_CHECK_EQUAL((int)UintToArith256(vBlocksSide[i].GetBlockHash()).GetLow64(), vBlocksSide[i].nHeight);
        BOOST_CHECK(vBlocksSide[i].pprev == nullptr || vBlocksSide[i].nHeight == vBlocksSide[i].pprev->nHeight + 1);
    }

    // Build a CChain for the main branch.
    CChain chain;
    chain.SetTip(&vBlocksMain.back());

    // Test 100 random starting points for locators.
    for (int n=0; n<100; n++) {
        int r = InsecureRandRange(150000);
        CBlockIndex* tip = (r < 100000) ? &vBlocksMain[r] : &vBlocksSide[r - 100000];
        CBlockLocator locator = chain.GetLocator(tip);

        // The first result must be the block itself, the last one must be genesis.
        BOOST_CHECK(locator.vHave.front() == tip->GetBlockHash());
        BOOST_CHECK(locator.vHave.back() == vBlocksMain[0].GetBlockHash());

        // Entries 1 through 11 (inclusive) go back one step each.
        for (unsigned int i = 1; i < 12 && i < locator.vHave.size() - 1; i++) {
            BOOST_CHECK_EQUAL(UintToArith256(locator.vHave[i]).GetLow64(), tip->nHeight - i);
        }

        // The further ones (excluding the last one) go back with exponential steps.
        unsigned int dist = 2;
        for (unsigned int i = 12; i < locator.vHave.size() - 1; i++) {
            BOOST_CHECK_EQUAL(UintToArith256(locator.vHave[i - 1]).GetLow64() - UintToArith256(locator.vHave[i]).GetLow64(), dist);
            dist *= 2;
        }
    }
}

BOOST_AUTO_TEST_CASE(findearliestatleast_test)
{
    std::vector<uint256> vHashMain(100000);
    std::vector<CBlockIndex> vBlocksMain(100000);
    for (unsigned int i=0; i<vBlocksMain.size(); i++) {
        vHashMain[i] = ArithToUint256(i); // Set the hash equal to the height
        vBlocksMain[i].nHeight = i;
        vBlocksMain[i].pprev = i ? &vBlocksMain[i - 1] : nullptr;
        vBlocksMain[i].phashBlock = &vHashMain[i];
        vBlocksMain[i].BuildSkip();
        if (i < 10) {
            vBlocksMain[i].nTime = i;
            vBlocksMain[i].nTimeMax = i;
        } else {
            // randomly choose something in the range [MTP, MTP*2]
            int64_t medianTimePast = vBlocksMain[i].GetMedianTimePast();
            int r = InsecureRandRange(medianTimePast);
            vBlocksMain[i].nTime = r + medianTimePast;
            vBlocksMain[i].nTimeMax = std::max(vBlocksMain[i].nTime, vBlocksMain[i-1].nTimeMax);
        }
    }
    // Check that we set nTimeMax up correctly.
    unsigned int curTimeMax = 0;
    for (unsigned int i=0; i<vBlocksMain.size(); ++i) {
        curTimeMax = std::max(curTimeMax, vBlocksMain[i].nTime);
        BOOST_CHECK(curTimeMax == vBlocksMain[i].nTimeMax);
    }

    // Build a CChain for the main branch.
    CChain chain;
    chain.SetTip(&vBlocksMain.back());

    // Verify that FindEarliestAtLeast is correct.
    for (unsigned int i=0; i<10000; ++i) {
        // Pick a random element in vBlocksMain.
        int r = InsecureRandRange(vBlocksMain.size());
        int64_t test_time = vBlocksMain[r].nTime;
        CBlockIndex *ret = chain.FindEarliestAtLeast(test_time);
        BOOST_CHECK(ret->nTimeMax >= test_time);
        BOOST_CHECK((ret->pprev==nullptr) || ret->pprev->nTimeMax < test_time);
        BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret);
    }
}

BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test)
{
    std::list<CBlockIndex> blocks;
    for (const unsigned int timeMax : {100, 100, 100, 200, 200, 200, 300, 300, 300}) {
        CBlockIndex* prev = blocks.empty() ? nullptr : &blocks.back();
        blocks.emplace_back();
        blocks.back().nHeight = prev ? prev->nHeight + 1 : 0;
        blocks.back().pprev = prev;
        blocks.back().BuildSkip();
        blocks.back().nTimeMax = timeMax;
    }

    CChain chain;
    chain.SetTip(&blocks.back());

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6);
    BOOST_CHECK(!chain.FindEarliestAtLeast(350));

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0);

    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min())->nHeight, 0);
    BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1)->nHeight, 0);
    BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max()));
    BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max()));
    BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1));
}

BOOST_AUTO_TEST_SUITE_END()
<|endoftext|>