<commit_before>/*
 * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
#pragma once

#include <graphene/chain/protocol/asset.hpp>
#include <graphene/chain/protocol/types.hpp>
#include <graphene/db/generic_index.hpp>
#include <graphene/db/object.hpp>

#include <boost/multi_index/composite_key.hpp>

namespace graphene { namespace chain {

using namespace graphene::db;

/**
 *  @brief an offer to sell a amount of a asset at a specified exchange rate by a certain time
 *  @ingroup object
 *  @ingroup protocol
 *  @ingroup market
 *
 *  This limit_order_objects are indexed by @ref expiration and is automatically deleted on the first block after expiration.
 */
class limit_order_object : public abstract_object<limit_order_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = limit_order_object_type;

      time_point_sec   expiration;
      account_id_type  seller;
      share_type       for_sale; ///< asset id is sell_price.base.asset_id
      price            sell_price;
      share_type       deferred_fee;

      pair<asset_id_type,asset_id_type> get_market()const
      {
         auto tmp = std::make_pair( sell_price.base.asset_id, sell_price.quote.asset_id );
         if( tmp.first > tmp.second ) std::swap( tmp.first, tmp.second );
         return tmp;
      }

      asset amount_for_sale()const   { return asset( for_sale, sell_price.base.asset_id ); }
      asset amount_to_receive()const { return amount_for_sale() * sell_price; }
};

struct by_id;
struct by_price;
struct by_expiration;
struct by_account;
typedef multi_index_container<
   limit_order_object,
   indexed_by<
      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_expiration>,
         composite_key< limit_order_object,
            member< limit_order_object, time_point_sec, &limit_order_object::expiration>,
            member< object, object_id_type, &object::id>
         >
      >,
      ordered_unique< tag<by_price>,
         composite_key< limit_order_object,
            member< limit_order_object, price, &limit_order_object::sell_price>,
            member< object, object_id_type, &object::id>
         >,
         composite_key_compare< std::greater<price>, std::less<object_id_type> >
      >,
      ordered_unique< tag<by_account>,
         composite_key< limit_order_object,
            member<limit_order_object, account_id_type, &limit_order_object::seller>,
            member<object, object_id_type, &object::id>
         >
      >
   >
> limit_order_multi_index_type;

typedef generic_index<limit_order_object, limit_order_multi_index_type> limit_order_index;

/**
 * @class call_order_object
 * @brief tracks debt and call price information
 *
 * There should only be one call_order_object per asset pair per account and
 * they will all have the same call price.
 */
class call_order_object : public abstract_object<call_order_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = call_order_object_type;

      asset get_collateral()const { return asset( collateral, call_price.base.asset_id ); }
      asset get_debt()const { return asset( debt, debt_type() ); }
      asset amount_to_receive()const { return get_debt(); }
      asset_id_type debt_type()const { return call_price.quote.asset_id; }
      price collateralization()const { return get_collateral() / get_debt(); }

      account_id_type  borrower;
      share_type       collateral;  ///< call_price.base.asset_id, access via get_collateral
      share_type       debt;        ///< call_price.quote.asset_id, access via get_collateral
      price            call_price;  ///< Debt / Collateral
};

/**
 *  @brief tracks bitassets scheduled for force settlement at some point in the future.
 *
 *  On the @ref settlement_date the @ref balance will be converted to the collateral asset
 *  and paid to @ref owner and then this object will be deleted.
 */
class force_settlement_object : public abstract_object<force_settlement_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = force_settlement_object_type;

      account_id_type   owner;
      asset             balance;
      time_point_sec    settlement_date;

      asset_id_type settlement_asset_id()const
      { return balance.asset_id; }
};

struct by_collateral;
struct by_account;
struct by_price;
typedef multi_index_container<
   call_order_object,
   indexed_by<
      ordered_unique< tag<by_id>,
         member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_price>,
         composite_key< call_order_object,
            member< call_order_object, price, &call_order_object::call_price>,
            member< object, object_id_type, &object::id>
         >,
         composite_key_compare< std::less<price>, std::less<object_id_type> >
      >,
      ordered_unique< tag<by_account>,
         composite_key< call_order_object,
            member< call_order_object, account_id_type, &call_order_object::borrower >,
            const_mem_fun< call_order_object, asset_id_type, &call_order_object::debt_type>
         >
      >,
      ordered_unique< tag<by_collateral>,
         composite_key< call_order_object,
            const_mem_fun< call_order_object, price, &call_order_object::collateralization >,
            member< object, object_id_type, &object::id >
         >
      >
   >
> call_order_multi_index_type;

struct by_expiration;
typedef multi_index_container<
   force_settlement_object,
   indexed_by<
      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_account>,
         composite_key< force_settlement_object,
            member<force_settlement_object, account_id_type, &force_settlement_object::owner>,
            member< object, object_id_type, &object::id >
         >
      >,
      ordered_unique< tag<by_expiration>,
         composite_key< force_settlement_object,
            const_mem_fun<force_settlement_object, asset_id_type, &force_settlement_object::settlement_asset_id>,
            member<force_settlement_object, time_point_sec, &force_settlement_object::settlement_date>,
            member< object, object_id_type, &object::id >
         >
      >
   >
> force_settlement_object_multi_index_type;

typedef generic_index<call_order_object, call_order_multi_index_type>                      call_order_index;
typedef generic_index<force_settlement_object, force_settlement_object_multi_index_type>   force_settlement_index;

} } // graphene::chain

FC_REFLECT_DERIVED( graphene::chain::limit_order_object,
                    (graphene::db::object),
                    (expiration)(seller)(for_sale)(sell_price)(deferred_fee)
                  )

FC_REFLECT_DERIVED( graphene::chain::call_order_object, (graphene::db::object),
                    (borrower)(collateral)(debt)(call_price) )

FC_REFLECT_DERIVED( graphene::chain::force_settlement_object,
                    (graphene::db::object),
                    (owner)(balance)(settlement_date)
                  )
<commit_msg>add get_market() function to call_order_object<commit_after>/*
 * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
#pragma once

#include <graphene/chain/protocol/asset.hpp>
#include <graphene/chain/protocol/types.hpp>
#include <graphene/db/generic_index.hpp>
#include <graphene/db/object.hpp>

#include <boost/multi_index/composite_key.hpp>

namespace graphene { namespace chain {

using namespace graphene::db;

/**
 *  @brief an offer to sell a amount of a asset at a specified exchange rate by a certain time
 *  @ingroup object
 *  @ingroup protocol
 *  @ingroup market
 *
 *  This limit_order_objects are indexed by @ref expiration and is automatically deleted on the first block after expiration.
 */
class limit_order_object : public abstract_object<limit_order_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = limit_order_object_type;

      time_point_sec   expiration;
      account_id_type  seller;
      share_type       for_sale; ///< asset id is sell_price.base.asset_id
      price            sell_price;
      share_type       deferred_fee;

      pair<asset_id_type,asset_id_type> get_market()const
      {
         auto tmp = std::make_pair( sell_price.base.asset_id, sell_price.quote.asset_id );
         if( tmp.first > tmp.second ) std::swap( tmp.first, tmp.second );
         return tmp;
      }

      asset amount_for_sale()const   { return asset( for_sale, sell_price.base.asset_id ); }
      asset amount_to_receive()const { return amount_for_sale() * sell_price; }
};

struct by_id;
struct by_price;
struct by_expiration;
struct by_account;
typedef multi_index_container<
   limit_order_object,
   indexed_by<
      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_expiration>,
         composite_key< limit_order_object,
            member< limit_order_object, time_point_sec, &limit_order_object::expiration>,
            member< object, object_id_type, &object::id>
         >
      >,
      ordered_unique< tag<by_price>,
         composite_key< limit_order_object,
            member< limit_order_object, price, &limit_order_object::sell_price>,
            member< object, object_id_type, &object::id>
         >,
         composite_key_compare< std::greater<price>, std::less<object_id_type> >
      >,
      ordered_unique< tag<by_account>,
         composite_key< limit_order_object,
            member<limit_order_object, account_id_type, &limit_order_object::seller>,
            member<object, object_id_type, &object::id>
         >
      >
   >
> limit_order_multi_index_type;

typedef generic_index<limit_order_object, limit_order_multi_index_type> limit_order_index;

/**
 * @class call_order_object
 * @brief tracks debt and call price information
 *
 * There should only be one call_order_object per asset pair per account and
 * they will all have the same call price.
 */
class call_order_object : public abstract_object<call_order_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = call_order_object_type;

      asset get_collateral()const { return asset( collateral, call_price.base.asset_id ); }
      asset get_debt()const { return asset( debt, debt_type() ); }
      asset amount_to_receive()const { return get_debt(); }
      asset_id_type debt_type()const { return call_price.quote.asset_id; }
      price collateralization()const { return get_collateral() / get_debt(); }

      account_id_type  borrower;
      share_type       collateral;  ///< call_price.base.asset_id, access via get_collateral
      share_type       debt;        ///< call_price.quote.asset_id, access via get_collateral
      price            call_price;  ///< Debt / Collateral

      pair<asset_id_type,asset_id_type> get_market()const
      {
         auto tmp = std::make_pair( call_price.base.asset_id, call_price.quote.asset_id );
         if( tmp.first > tmp.second ) std::swap( tmp.first, tmp.second );
         return tmp;
      }
};

/**
 *  @brief tracks bitassets scheduled for force settlement at some point in the future.
 *
 *  On the @ref settlement_date the @ref balance will be converted to the collateral asset
 *  and paid to @ref owner and then this object will be deleted.
 */
class force_settlement_object : public abstract_object<force_settlement_object>
{
   public:
      static const uint8_t space_id = protocol_ids;
      static const uint8_t type_id  = force_settlement_object_type;

      account_id_type   owner;
      asset             balance;
      time_point_sec    settlement_date;

      asset_id_type settlement_asset_id()const
      { return balance.asset_id; }
};

struct by_collateral;
struct by_account;
struct by_price;
typedef multi_index_container<
   call_order_object,
   indexed_by<
      ordered_unique< tag<by_id>,
         member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_price>,
         composite_key< call_order_object,
            member< call_order_object, price, &call_order_object::call_price>,
            member< object, object_id_type, &object::id>
         >,
         composite_key_compare< std::less<price>, std::less<object_id_type> >
      >,
      ordered_unique< tag<by_account>,
         composite_key< call_order_object,
            member< call_order_object, account_id_type, &call_order_object::borrower >,
            const_mem_fun< call_order_object, asset_id_type, &call_order_object::debt_type>
         >
      >,
      ordered_unique< tag<by_collateral>,
         composite_key< call_order_object,
            const_mem_fun< call_order_object, price, &call_order_object::collateralization >,
            member< object, object_id_type, &object::id >
         >
      >
   >
> call_order_multi_index_type;

struct by_expiration;
typedef multi_index_container<
   force_settlement_object,
   indexed_by<
      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
      ordered_unique< tag<by_account>,
         composite_key< force_settlement_object,
            member<force_settlement_object, account_id_type, &force_settlement_object::owner>,
            member< object, object_id_type, &object::id >
         >
      >,
      ordered_unique< tag<by_expiration>,
         composite_key< force_settlement_object,
            const_mem_fun<force_settlement_object, asset_id_type, &force_settlement_object::settlement_asset_id>,
            member<force_settlement_object, time_point_sec, &force_settlement_object::settlement_date>,
            member< object, object_id_type, &object::id >
         >
      >
   >
> force_settlement_object_multi_index_type;

typedef generic_index<call_order_object, call_order_multi_index_type>                      call_order_index;
typedef generic_index<force_settlement_object, force_settlement_object_multi_index_type>   force_settlement_index;

} } // graphene::chain

FC_REFLECT_DERIVED( graphene::chain::limit_order_object,
                    (graphene::db::object),
                    (expiration)(seller)(for_sale)(sell_price)(deferred_fee)
                  )

FC_REFLECT_DERIVED( graphene::chain::call_order_object, (graphene::db::object),
                    (borrower)(collateral)(debt)(call_price) )

FC_REFLECT_DERIVED( graphene::chain::force_settlement_object,
                    (graphene::db::object),
                    (owner)(balance)(settlement_date)
                  )
<|endoftext|>


<commit_before>#ifndef STAN_MATH_PRIM_MAT_FUNCTOR_MAP_RECT_CONCURRENT_HPP
#define STAN_MATH_PRIM_MAT_FUNCTOR_MAP_RECT_CONCURRENT_HPP

#include <stan/math/prim/mat/fun/typedefs.hpp>

#include <stan/math/prim/mat/functor/map_rect_reduce.hpp>
#include <stan/math/prim/mat/functor/map_rect_combine.hpp>

#include <vector>
#include <thread>
#include <future>
#include <cstdlib>

namespace stan {
namespace math {
namespace internal {

/**
 * Get number of threads to use for num_jobs jobs. The function uses
 * the environment variable STAN_NUM_THREADS and follows these
 * conventions:
 *
 * - STAN_NUM_THREADS is not defined or is not a number => num_threads=1
 * - STAN_NUM_THREADS is positive => num_threads is set to the
 *   specified number
 * - STAN_NUM_THREADS is set to -1 => num_threads is the number of
 *   available cores on the machine
 * - STAN_NUM_THREADS < -1 => num_threads is 1
 *
 * Should num_threads exceed the number of jobs, then num_threads will
 * be set equal to the number of jobs.
 *
 * @param num_jobs number of jobs
 * @return number of threads to use
 */
inline int get_num_threads(int num_jobs) {
  int num_threads = 1;
#ifdef STAN_THREADS
  const char* env_stan_num_threads = std::getenv("STAN_NUM_THREADS");
  if (env_stan_num_threads != nullptr) {
    const int env_num_threads = std::atoi(env_stan_num_threads);
    if (env_num_threads > 0)
      num_threads = env_num_threads;
    else if (env_num_threads == -1)
      num_threads = std::thread::hardware_concurrency();
    // anything else will use 1 thread.
  }
  if (num_threads > num_jobs)
    num_threads = num_jobs;
#endif
  return num_threads;
}

template <int call_id, typename F, typename T_shared_param,
          typename T_job_param>
Eigen::Matrix<typename stan::return_type<T_shared_param, T_job_param>::type,
              Eigen::Dynamic, 1>
map_rect_concurrent(
    const Eigen::Matrix<T_shared_param, Eigen::Dynamic, 1>& shared_params,
    const std::vector<Eigen::Matrix<T_job_param, Eigen::Dynamic, 1>>&
        job_params,
    const std::vector<std::vector<double>>& x_r,
    const std::vector<std::vector<int>>& x_i, std::ostream* msgs = nullptr) {
  typedef map_rect_reduce<F, T_shared_param, T_job_param> ReduceF;
  typedef map_rect_combine<F, T_shared_param, T_job_param> CombineF;

  const int num_jobs = job_params.size();
  const vector_d shared_params_dbl = value_of(shared_params);
  std::vector<std::future<std::vector<matrix_d>>> futures;

  auto execute_chunk = [&](int start, int size) -> std::vector<matrix_d> {
    const int end = start + size;
    std::vector<matrix_d> chunk_f_out;
    chunk_f_out.reserve(size);
    for (int i = start; i != end; i++)
      chunk_f_out.push_back(ReduceF()(
          shared_params_dbl, value_of(job_params[i]), x_r[i], x_i[i], msgs));
    return chunk_f_out;
  };

  int num_threads = get_num_threads(num_jobs);
  int num_jobs_per_thread = num_jobs / num_threads;
  futures.emplace_back(
      std::async(std::launch::deferred, execute_chunk, 0, num_jobs_per_thread));

#ifdef STAN_THREADS
  if (num_threads > 1) {
    const int num_big_threads
        = (num_jobs - num_jobs_per_thread) % (num_threads - 1);
    const int first_big_thread = num_threads - num_big_threads;
    for (int i = 1, job_start = num_jobs_per_thread, job_size = 0;
         i < num_threads; ++i, job_start += job_size) {
      job_size = i >= first_big_thread ? num_jobs_per_thread + 1
                                       : num_jobs_per_thread;
      futures.emplace_back(
          std::async(std::launch::async, execute_chunk, job_start, job_size));
    }
  }
#endif

  // collect results
  std::vector<int> world_f_out;
  world_f_out.reserve(num_jobs);
  matrix_d world_output(0, 0);

  for (int i = 0, offset = 0; i < futures.size(); ++i) {
    const std::vector<matrix_d>& chunk_result = futures[i].get();
    if (i == 0)
      world_output.resize(chunk_result[0].rows(),
                          num_jobs * chunk_result[0].cols());

    for (const auto& job_result : chunk_result) {
      const int num_job_outputs = job_result.cols();
      world_f_out.push_back(num_job_outputs);

      if (world_output.cols() < offset + num_job_outputs)
        world_output.conservativeResize(Eigen::NoChange,
                                        2 * (offset + num_job_outputs));

      world_output.block(0, offset, world_output.rows(), num_job_outputs)
          = job_result;

      offset += num_job_outputs;
    }
  }
  CombineF combine(shared_params, job_params);
  return combine(world_output, world_f_out);
}

}  // namespace internal
}  // namespace math
}  // namespace stan

#endif
<commit_msg>fix size_t / int issues<commit_after>#ifndef STAN_MATH_PRIM_MAT_FUNCTOR_MAP_RECT_CONCURRENT_HPP
#define STAN_MATH_PRIM_MAT_FUNCTOR_MAP_RECT_CONCURRENT_HPP

#include <stan/math/prim/mat/fun/typedefs.hpp>

#include <stan/math/prim/mat/functor/map_rect_reduce.hpp>
#include <stan/math/prim/mat/functor/map_rect_combine.hpp>

#include <vector>
#include <thread>
#include <future>
#include <cstdlib>

namespace stan {
namespace math {
namespace internal {

/**
 * Get number of threads to use for num_jobs jobs. The function uses
 * the environment variable STAN_NUM_THREADS and follows these
 * conventions:
 *
 * - STAN_NUM_THREADS is not defined or is not a number => num_threads=1
 * - STAN_NUM_THREADS is positive => num_threads is set to the
 *   specified number
 * - STAN_NUM_THREADS is set to -1 => num_threads is the number of
 *   available cores on the machine
 * - STAN_NUM_THREADS < -1 => num_threads is 1
 *
 * Should num_threads exceed the number of jobs, then num_threads will
 * be set equal to the number of jobs.
 *
 * @param num_jobs number of jobs
 * @return number of threads to use
 */
inline int get_num_threads(int num_jobs) {
  int num_threads = 1;
#ifdef STAN_THREADS
  const char* env_stan_num_threads = std::getenv("STAN_NUM_THREADS");
  if (env_stan_num_threads != nullptr) {
    const int env_num_threads = std::atoi(env_stan_num_threads);
    if (env_num_threads > 0)
      num_threads = env_num_threads;
    else if (env_num_threads == -1)
      num_threads = std::thread::hardware_concurrency();
    // anything else will use 1 thread.
  }
  if (num_threads > num_jobs)
    num_threads = num_jobs;
#endif
  return num_threads;
}

template <int call_id, typename F, typename T_shared_param,
          typename T_job_param>
Eigen::Matrix<typename stan::return_type<T_shared_param, T_job_param>::type,
              Eigen::Dynamic, 1>
map_rect_concurrent(
    const Eigen::Matrix<T_shared_param, Eigen::Dynamic, 1>& shared_params,
    const std::vector<Eigen::Matrix<T_job_param, Eigen::Dynamic, 1>>&
        job_params,
    const std::vector<std::vector<double>>& x_r,
    const std::vector<std::vector<int>>& x_i, std::ostream* msgs = nullptr) {
  typedef map_rect_reduce<F, T_shared_param, T_job_param> ReduceF;
  typedef map_rect_combine<F, T_shared_param, T_job_param> CombineF;

  const int num_jobs = job_params.size();
  const vector_d shared_params_dbl = value_of(shared_params);
  std::vector<std::future<std::vector<matrix_d>>> futures;

  auto execute_chunk = [&](int start, int size) -> std::vector<matrix_d> {
    const int end = start + size;
    std::vector<matrix_d> chunk_f_out;
    chunk_f_out.reserve(size);
    for (int i = start; i != end; i++)
      chunk_f_out.push_back(ReduceF()(
          shared_params_dbl, value_of(job_params[i]), x_r[i], x_i[i], msgs));
    return chunk_f_out;
  };

  int num_threads = get_num_threads(num_jobs);
  int num_jobs_per_thread = num_jobs / num_threads;
  futures.emplace_back(
      std::async(std::launch::deferred, execute_chunk, 0, num_jobs_per_thread));

#ifdef STAN_THREADS
  if (num_threads > 1) {
    const int num_big_threads
        = (num_jobs - num_jobs_per_thread) % (num_threads - 1);
    const int first_big_thread = num_threads - num_big_threads;
    for (int i = 1, job_start = num_jobs_per_thread, job_size = 0;
         i < num_threads; ++i, job_start += job_size) {
      job_size = i >= first_big_thread ? num_jobs_per_thread + 1
                                       : num_jobs_per_thread;
      futures.emplace_back(
          std::async(std::launch::async, execute_chunk, job_start, job_size));
    }
  }
#endif

  // collect results
  std::vector<int> world_f_out;
  world_f_out.reserve(num_jobs);
  matrix_d world_output(0, 0);

  int offset = 0;
  for (std::size_t i = 0; i < futures.size(); ++i) {
    const std::vector<matrix_d>& chunk_result = futures[i].get();
    if (i == 0)
      world_output.resize(chunk_result[0].rows(),
                          num_jobs * chunk_result[0].cols());

    for (const auto& job_result : chunk_result) {
      const int num_job_outputs = job_result.cols();
      world_f_out.push_back(num_job_outputs);

      if (world_output.cols() < offset + num_job_outputs)
        world_output.conservativeResize(Eigen::NoChange,
                                        2 * (offset + num_job_outputs));

      world_output.block(0, offset, world_output.rows(), num_job_outputs)
          = job_result;

      offset += num_job_outputs;
    }
  }
  CombineF combine(shared_params, job_params);
  return combine(world_output, world_f_out);
}

}  // namespace internal
}  // namespace math
}  // namespace stan

#endif
<|endoftext|>


<commit_before>#include <stdio.h> 				//׼ͷļ
#include <stdlib.h>				//׼ͷļ

#define LIST_INIT_SIZE 100		//̬ʵֵԱĳʼС
#define LISTINCREMENT 10		//ÿζ̬ӵĴС

#define OVERFLOW -1		//Խ
#define FALSE 0			//ʧ
#define OK 1			//ɹ

//̬ʵֵԱݽṹ
typedef struct{
	int *elem;		//ÿԱӦĶ̬׵ַַ
	int length; 	//Աǰĳ
	int listsize;		//ԱĴС
}SqList;

//ʼһյԱ
int InitList_Sq(SqList &L){

	//̬
	L.elem=(int*)malloc(LIST_INIT_SIZE*sizeof(int));

	if (!L.elem) return OVERFLOW; 				//ʧ

	L.length=0;									//ձĳΪ0
	L.listsize=LIST_INIT_SIZE; 						//ʼ洢

	return OK;									//سʼɹ
}//InitList_Sq 㷨 2.3

//˳Lеiλ֮ǰµԪe
int ListInsert_Sq(SqList &L,int i, int e){

	//жiֵǷϷ 1<=i<=L.length+1
	if (i<1||i>L.length+1) return OVERFLOW;

	//ǰ洢ռӷ
	if (L.length>=L.listsize){

		int * newbase=(int *)realloc(L.elem,(L.listsize+LISTINCREMENT)*sizeof(int));

		if (!newbase) return OVERFLOW;		//ʧ

		L.elem=newbase;						//µĵַ
		L.listsize+=LISTINCREMENT; 			//ԱĴС
	}

	int *q=&(L.elem[i-1]);		//Ҫλ
	for (int *p=&(L.elem[L.length-1]); p>=q; p--){
		*(p+1)=*p;				//λüԪغһλ
	}

	*q=e;			//e
	++L.length;		//+1

	return OK;
}

//ԱLɾiԪأeֵ
int ListDelete_Sq(SqList &L,int i,int &e){

	if (i<1||i>L.length) return OVERFLOW;		// iȡֵΧΪ 1<=i<=L.length

	int *p=&(L.elem[i-1]);			//pΪɾԪ

	e=*p;							//eرɾԪ

	int *q=&(L.elem[L.length-1]); 	//βԪ  //  int *q=L.elem+L.length

	for (++p; p<=q; ++p){

		*(p-1)=*p;					//iԪԺԪͳͳǰƶһ ܹƶ n-i
	}

	--L.length;						//-1

	return OK;
}

//ԪرȽϺ
int compare(int p,int e){
	return p==e;
} //Compare

//Աвe compareԪصλ δҵ򷵻0
int LocateElem_Sq(SqList &L,int e,int (* compare)(int,int)){

	int i=1; 			//һԪصλ

	int *p=L.elem;		//һԪصָ

	while (i<=L.length&&!(*compare)(*p++,e)) ++i;		//ɨԱ  һ  ܺã

	if (i<=L.length) return i;			//ڷ λ
	else return FALSE;					//ڷʧ
} //LocateElem_Sq

int main(){

	//һ ʵֵԱ
	SqList L;

	//ʼԱ
	InitList_Sq(L);

	//Բ20
	for (int i=0; i<20; i++){

		ListInsert_Sq(L,i+1,i);

		printf("%d %d \n",L.elem[i],L.length);
	}

	// ԱɾһԪ λ1
	int e;
	ListDelete_Sq(L,1,e);

	//ɾԪ
	printf("%d\n",e);

	//ӦɾԱ
	for (int i=0; i<L.length; i++){

		printf("%d %d\n",L.elem[i],i+1);
	}

	//Աв eԪ
	int i=LocateElem_Sq(L,12,compare);

	//ҽ
	printf("%d\n",i);
}
<commit_msg>add-linear-list-merge-la-lb-to-lc<commit_after>#include <stdio.h> 				//׼ͷļ
#include <stdlib.h>				//׼ͷļ

#define LIST_INIT_SIZE 100		//̬ʵֵԱĳʼС
#define LISTINCREMENT 10		//ÿζ̬ӵĴС

#define OVERFLOW -1		//Խ
#define FALSE 0			//ʧ
#define OK 1			//ɹ

//̬ʵֵԱݽṹ
typedef struct{
	int *elem;		//ÿԱӦĶ̬׵ַַ
	int length; 	//Աǰĳ
	int listsize;		//ԱĴС
}SqList;

//ʼһյԱ
int InitList_Sq(SqList &L){

	//̬
	L.elem=(int*)malloc(LIST_INIT_SIZE*sizeof(int));

	if (!L.elem) return OVERFLOW; 				//ʧ

	L.length=0;									//ձĳΪ0
	L.listsize=LIST_INIT_SIZE; 						//ʼ洢

	return OK;									//سʼɹ
}//InitList_Sq 㷨 2.3

//˳Lеiλ֮ǰµԪe
int ListInsert_Sq(SqList &L,int i, int e){

	//жiֵǷϷ 1<=i<=L.length+1
	if (i<1||i>L.length+1) return OVERFLOW;

	//ǰ洢ռӷ
	if (L.length>=L.listsize){

		int * newbase=(int *)realloc(L.elem,(L.listsize+LISTINCREMENT)*sizeof(int));

		if (!newbase) return OVERFLOW;		//ʧ

		L.elem=newbase;						//µĵַ
		L.listsize+=LISTINCREMENT; 			//ԱĴС
	}

	int *q=&(L.elem[i-1]);		//Ҫλ
	for (int *p=&(L.elem[L.length-1]); p>=q; p--){
		*(p+1)=*p;				//λüԪغһλ
	}

	*q=e;			//e
	++L.length;		//+1

	return OK;
}

//ԱLɾiԪأeֵ
int ListDelete_Sq(SqList &L,int i,int &e){

	if (i<1||i>L.length) return OVERFLOW;		// iȡֵΧΪ 1<=i<=L.length

	int *p=&(L.elem[i-1]);			//pΪɾԪ

	e=*p;							//eرɾԪ

	int *q=&(L.elem[L.length-1]); 	//βԪ  //  int *q=L.elem+L.length

	for (++p; p<=q; ++p){

		*(p-1)=*p;					//iԪԺԪͳͳǰƶһ ܹƶ n-i
	}

	--L.length;						//-1

	return OK;
}

//ԪرȽϺ
int compare(int p,int e){
	return p==e;
} //Compare

//Աвe compareԪصλ δҵ򷵻0
int LocateElem_Sq(SqList &L,int e,int (* compare)(int,int)){

	int i=1; 			//һԪصλ

	int *p=L.elem;		//һԪصָ

	while (i<=L.length&&!(*compare)(*p++,e)) ++i;		//ɨԱ  һ  ܺã

	if (i<=L.length) return i;			//ڷ λ
	else return FALSE;					//ڷʧ
} //LocateElem_Sq

//ǵݼԱ a b 鲢  Աc ع鲢ĳ
int MergeList_Sq(SqList La,SqList Lb,SqList &Lc){

	//Ĺ鲢ĳ
	Lc.length=Lc.listsize=La.length+Lb.length;

	Lc.elem=(int *)malloc(Lc.listsize*sizeof(int));  	//

	if (!Lc.elem) return OVERFLOW;			//ʧ   0

	int *qa=La.elem;	// ԱaԪصַ
	int *qb=Lb.elem;	// ԱbԪصַ
	int *qc=Lc.elem;	// Աc׵ַ

	int *pa=&(La.elem[La.length-1]);	//Աa βַ
	int *pb=&(Lb.elem[Lb.length-1]);	//Աb βַ

	//ʼ鲢
	while (qa<=pa&&qb<=pb){

		if (*qa<=*qb) *qc++=*qa++;
		else *qc++=*qb++;
	}

	//Ҫ鲢  ֱ
	while (qa<=pa) *qc++=*qa++;
	while (qb<=pb) *qc++=*qb++;

	return Lc.length;		//س
} //MergeList_Sq

int main(){

	//һ ʵֵԱ
	SqList L;

	//ʼԱ
	InitList_Sq(L);

	//Բ20
	for (int i=0; i<20; i++){

		ListInsert_Sq(L,i+1,i);

		printf("%d %d \n",L.elem[i],L.length);
	}

	// ԱɾһԪ λ1
	int e;
	ListDelete_Sq(L,1,e);

	//ɾԪ
	printf("%d\n",e);

	//ӦɾԱ
	for (int i=0; i<L.length; i++){

		printf("%d %d\n",L.elem[i],i+1);
	}

	//Աв eԪ
	int i=LocateElem_Sq(L,12,compare);

	//ҽ
	printf("%d\n",i);

	//鲢 ԱĲ
	SqList La;		//Աa
	SqList Lb;		//Աb
	SqList Lc;		//Աc --ڱ鲢

	//ʼ  Աa b
	InitList_Sq(La);
	InitList_Sq(Lb);

	//ʼǵݼ
	for (int i=0; i<20; i++){

		ListInsert_Sq(La,i+1,i*2);
		ListInsert_Sq(Lb,i+1,i*3);
	}

	// ӡ  a b
	for (int i=0; i<20; i++) printf("%d ",La.elem[i]);
	printf("\n");
	for (int i=0; i<20; i++) printf("%d ",Lb.elem[i]);
	printf("\n");

	//ʼϲ
	MergeList_Sq(La,Lb,Lc);

	//ϲĽ
	for (int i=0; i<Lc.length; i++){

		printf("%d--> %d\n",i+1,Lc.elem[i]);
	}
}
<|endoftext|>


<commit_before>
/*
    pbrt source code Copyright(c) 1998-2010 Matt Pharr and Greg Humphreys.

    This file is part of pbrt.

    pbrt is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.  Note that the text contents of
    the book "Physically Based Rendering" are *not* licensed under the
    GNU GPL.

    pbrt is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

// materials/measured.cpp*
#include "stdafx.h"
#include "materials/measured.h"
#include "paramset.h"
#include "floatfile.h"

// MeasuredMaterial Method Definitions
static map<string, float *> loadedRegularHalfangle;
static map<string, KdTree<IrregIsotropicBRDFSample> *> loadedThetaPhi;
MeasuredMaterial::MeasuredMaterial(const string &filename,
      Reference<Texture<float> > bump) {
    bumpMap = bump;
    const char *suffix = strrchr(filename.c_str(), '.');
    regularHalfangleData = NULL;
    thetaPhiData = NULL;
    if (!suffix)
        Error("No suffix in measured BRDF filename \"%s\".  "
              "Can't determine file type (.brdf / .merl)", filename.c_str());
    if (!strcmp(suffix, ".brdf") || !strcmp(suffix, ".BRDF")) {
        // Load $(\theta, \phi)$ measured BRDF data
        if (loadedThetaPhi.find(filename) != loadedThetaPhi.end()) {
            thetaPhiData = loadedThetaPhi[filename];
            return;
        }

        vector<float> values;
        if (!ReadFloatFile(filename.c_str(), &values))
            Error("Unable to read BRDF data from file \"%s\"", filename.c_str());

        uint32_t pos = 0;
        int numWls = int(values[pos++]);
        if ((values.size() - 1 - numWls) % (4 + numWls) != 0)
            Error("Excess or insufficient data in theta, phi BRDF file \"%s\"",
                  filename.c_str());
        vector<float> wls;
        for (int i = 0; i < numWls; ++i)
            wls.push_back(values[pos++]);

        BBox bbox;
        vector<IrregIsotropicBRDFSample> samples;
        while (pos < values.size()) {
            float thetai = values[pos++];
            float phii = values[pos++];
            float thetao = values[pos++];
            float phio = values[pos++];
            Vector wo = SphericalDirection(sinf(thetao), cosf(thetao), phio);
            Vector wi = SphericalDirection(sinf(thetai), cosf(thetai), phii);
            Spectrum s = Spectrum::FromSampled(&wls[0], &values[pos], numWls);
            pos += numWls;
            Point p = BRDFRemap(wo, wi);
            samples.push_back(IrregIsotropicBRDFSample(p, s));
            bbox = Union(bbox, p);
        }
        loadedThetaPhi[filename] = thetaPhiData = new KdTree<IrregIsotropicBRDFSample>(samples);
    }
    else {
        // Load RegularHalfangle BRDF Data
        nThetaH = 90;
        nThetaD = 90;
        nPhiD = 180;

        if (loadedRegularHalfangle.find(filename) != loadedRegularHalfangle.end()) {
            regularHalfangleData = loadedRegularHalfangle[filename];
            return;
        }

        FILE *f = fopen(filename.c_str(), "rb");
        if (!f) {
            Error("Unable to open BRDF data file \"%s\"", filename.c_str());
            return;
        }
        int dims[3];
        if (fread(dims, sizeof(int), 3, f) != 3) {
            Error("Premature end-of-file in measured BRDF data file \"%s\"",
                  filename.c_str());
            fclose(f);
            return;
        }
        uint32_t n = dims[0] * dims[1] * dims[2];
        if (n != nThetaH * nThetaD * nPhiD)  {
            Error("Dimensions don't match\n");
            fclose(f);
            return;
        }

        regularHalfangleData = new float[3*n];
        const uint32_t chunkSize = 2*nPhiD;
        double *tmp = ALLOCA(double, chunkSize);
        uint32_t nChunks = n / chunkSize;
        Assert((n % chunkSize) == 0);
        float scales[3] = { 1.f/1500.f, 1.15f/1500.f, 1.66f/1500.f };
        for (int c = 0; c < 3; ++c) {
            int offset = 0;
            for (uint32_t i = 0; i < nChunks; ++i) {
                if (fread(tmp, sizeof(double), chunkSize, f) != chunkSize) {
                    Error("Premature end-of-file in measured BRDF data file \"%s\"",
                          filename.c_str());
                    delete[] regularHalfangleData;
                    regularHalfangleData = NULL;
                    fclose(f);
                    return;
                }
                for (uint32_t j = 0; j < chunkSize; ++j)
                    regularHalfangleData[3 * offset++ + c] = max(0., tmp[j] * scales[c]);
            }
        }

        loadedRegularHalfangle[filename] = regularHalfangleData;
        fclose(f);
    }
}

BSDF *MeasuredMaterial::GetBSDF(const DifferentialGeometry &dgGeom,
                                const DifferentialGeometry &dgShading,
                                MemoryArena &arena) const {
    // Allocate _BSDF_, possibly doing bump mapping with _bumpMap_
    DifferentialGeometry dgs;
    if (bumpMap)
        Bump(bumpMap, dgGeom, dgShading, &dgs);
    else
        dgs = dgShading;
    BSDF *bsdf = BSDF_ALLOC(arena, BSDF)(dgs, dgGeom.nn);
    if (regularHalfangleData)
        bsdf->Add(BSDF_ALLOC(arena, RegularHalfangleBRDF)
            (regularHalfangleData, nThetaH, nThetaD, nPhiD));
    else if (thetaPhiData)
        bsdf->Add(BSDF_ALLOC(arena, IrregIsotropicBRDF)(thetaPhiData));
    return bsdf;
}

MeasuredMaterial *CreateMeasuredMaterial(const Transform &xform,
        const TextureParams &mp) {
    Reference<Texture<float> > bumpMap = mp.GetFloatTexture("bumpmap", 0.f);
    return new MeasuredMaterial(mp.FindFilename("filename"), bumpMap);
}

<commit_msg>Add documentation of BRDF file formats in comment in source code.<commit_after>
/*
    pbrt source code Copyright(c) 1998-2010 Matt Pharr and Greg Humphreys.

    This file is part of pbrt.

    pbrt is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.  Note that the text contents of
    the book "Physically Based Rendering" are *not* licensed under the
    GNU GPL.

    pbrt is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

// materials/measured.cpp*
#include "stdafx.h"
#include "materials/measured.h"
#include "paramset.h"
#include "floatfile.h"

/*
  File format descriptions:

  -- Irregularly Sampled Isotropic BRDF --

  This is the format of the BRDFs in the scenes/brdfs/ folder of the pbrt
  distribution.  This is a simple text format of numbers in a particular
  format; the hash character # is used to denote a comment that continues
  to the end of the current line.

  The first number in the file gives the number of wavelengths at which the
  reflection data was measured, numWls.  This is followed by numWls values
  that give the frequency in nm of these wavelengths.  Each BRDF
  measurement is represented by 4+numWls values.  The first two give the
  (theta,phi) angles of the incident illumination direction, the next two
  give (theta,phi) for the measured reflection direction, and the following
  numWls give the spectral coefficients for the measurement at the
  wavelength specified at the start of the file.

  -- Regular Half-Angle BRDF --
  This is the file format used in the MERL BRDF database; see http://merl.com/brdf.

  This file format is a binary format, with numbers encoded in low-endian
  form.  It represents a regular 3D tabularization of BRDF samples in RGB
  color where the dimensions indexed over are (delta phi, delta theta,
  sqrt(theta_h)).  Here, theta_h is the angle between the halfangle vector
  and the normal, and delta theta and delta phi are the offset in theta and
  phi of one of the two directions.  (Note that the offset would be the
  same for the other direction, since it's from the half-angle vector.)

  The starts with three 32-bit integers, giving the resolution of the
  overall table.  It then containes a number of samples equal to the
  product of those three integers, times 3 (for RGB).  Samples are laid out
  with delta phi the minor index, then delta theta, then sqrt(theta_h) as
  the major index.

  In the file each sample should be scaled by RGB(1500,1500,1500/1.6) of
  the original measurement.  (In order words, the sample values are scaled
  by the inverse of that as they are read in.
*/

// MeasuredMaterial Method Definitions
static map<string, float *> loadedRegularHalfangle;
static map<string, KdTree<IrregIsotropicBRDFSample> *> loadedThetaPhi;
MeasuredMaterial::MeasuredMaterial(const string &filename,
      Reference<Texture<float> > bump) {
    bumpMap = bump;
    const char *suffix = strrchr(filename.c_str(), '.');
    regularHalfangleData = NULL;
    thetaPhiData = NULL;
    if (!suffix)
        Error("No suffix in measured BRDF filename \"%s\".  "
              "Can't determine file type (.brdf / .merl)", filename.c_str());
    if (!strcmp(suffix, ".brdf") || !strcmp(suffix, ".BRDF")) {
        // Load $(\theta, \phi)$ measured BRDF data
        if (loadedThetaPhi.find(filename) != loadedThetaPhi.end()) {
            thetaPhiData = loadedThetaPhi[filename];
            return;
        }

        vector<float> values;
        if (!ReadFloatFile(filename.c_str(), &values))
            Error("Unable to read BRDF data from file \"%s\"", filename.c_str());

        uint32_t pos = 0;
        int numWls = int(values[pos++]);
        if ((values.size() - 1 - numWls) % (4 + numWls) != 0)
            Error("Excess or insufficient data in theta, phi BRDF file \"%s\"",
                  filename.c_str());
        vector<float> wls;
        for (int i = 0; i < numWls; ++i)
            wls.push_back(values[pos++]);

        BBox bbox;
        vector<IrregIsotropicBRDFSample> samples;
        while (pos < values.size()) {
            float thetai = values[pos++];
            float phii = values[pos++];
            float thetao = values[pos++];
            float phio = values[pos++];
            Vector wo = SphericalDirection(sinf(thetao), cosf(thetao), phio);
            Vector wi = SphericalDirection(sinf(thetai), cosf(thetai), phii);
            Spectrum s = Spectrum::FromSampled(&wls[0], &values[pos], numWls);
            pos += numWls;
            Point p = BRDFRemap(wo, wi);
            samples.push_back(IrregIsotropicBRDFSample(p, s));
            bbox = Union(bbox, p);
        }
        loadedThetaPhi[filename] = thetaPhiData = new KdTree<IrregIsotropicBRDFSample>(samples);
    }
    else {
        // Load RegularHalfangle BRDF Data
        nThetaH = 90;
        nThetaD = 90;
        nPhiD = 180;

        if (loadedRegularHalfangle.find(filename) != loadedRegularHalfangle.end()) {
            regularHalfangleData = loadedRegularHalfangle[filename];
            return;
        }

        FILE *f = fopen(filename.c_str(), "rb");
        if (!f) {
            Error("Unable to open BRDF data file \"%s\"", filename.c_str());
            return;
        }
        int dims[3];
        if (fread(dims, sizeof(int), 3, f) != 3) {
            Error("Premature end-of-file in measured BRDF data file \"%s\"",
                  filename.c_str());
            fclose(f);
            return;
        }
        uint32_t n = dims[0] * dims[1] * dims[2];
        if (n != nThetaH * nThetaD * nPhiD)  {
            Error("Dimensions don't match\n");
            fclose(f);
            return;
        }

        regularHalfangleData = new float[3*n];
        const uint32_t chunkSize = 2*nPhiD;
        double *tmp = ALLOCA(double, chunkSize);
        uint32_t nChunks = n / chunkSize;
        Assert((n % chunkSize) == 0);
        float scales[3] = { 1.f/1500.f, 1.15f/1500.f, 1.66f/1500.f };
        for (int c = 0; c < 3; ++c) {
            int offset = 0;
            for (uint32_t i = 0; i < nChunks; ++i) {
                if (fread(tmp, sizeof(double), chunkSize, f) != chunkSize) {
                    Error("Premature end-of-file in measured BRDF data file \"%s\"",
                          filename.c_str());
                    delete[] regularHalfangleData;
                    regularHalfangleData = NULL;
                    fclose(f);
                    return;
                }
                for (uint32_t j = 0; j < chunkSize; ++j)
                    regularHalfangleData[3 * offset++ + c] = max(0., tmp[j] * scales[c]);
            }
        }

        loadedRegularHalfangle[filename] = regularHalfangleData;
        fclose(f);
    }
}

BSDF *MeasuredMaterial::GetBSDF(const DifferentialGeometry &dgGeom,
                                const DifferentialGeometry &dgShading,
                                MemoryArena &arena) const {
    // Allocate _BSDF_, possibly doing bump mapping with _bumpMap_
    DifferentialGeometry dgs;
    if (bumpMap)
        Bump(bumpMap, dgGeom, dgShading, &dgs);
    else
        dgs = dgShading;
    BSDF *bsdf = BSDF_ALLOC(arena, BSDF)(dgs, dgGeom.nn);
    if (regularHalfangleData)
        bsdf->Add(BSDF_ALLOC(arena, RegularHalfangleBRDF)
            (regularHalfangleData, nThetaH, nThetaD, nPhiD));
    else if (thetaPhiData)
        bsdf->Add(BSDF_ALLOC(arena, IrregIsotropicBRDF)(thetaPhiData));
    return bsdf;
}

MeasuredMaterial *CreateMeasuredMaterial(const Transform &xform,
        const TextureParams &mp) {
    Reference<Texture<float> > bumpMap = mp.GetFloatTexture("bumpmap", 0.f);
    return new MeasuredMaterial(mp.FindFilename("filename"), bumpMap);
}

<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    // REVIEW(an): this is useless, you never use the inheritance.  But, you
    // actually SHOULD use it! for all member fields and methods that are in
    // common. But NOT for virtual functions. Remove the virtuals. Find out what
    // functions the methods below have in common and make them functions of the
    // superclass.
    class InputLineIterator
    {
    public:
        static const bool debug = false;
        const size_t read_size = 2 * 1024 * 1024;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            auto my_start_and_end = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            size_t my_start = std::get<0>(my_start_and_end);
            my_end_ = std::get<1>(my_start_and_end);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                c_file_ = OpenFile(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = lseek(c_file_, my_start - files_[current_file_].second, SEEK_CUR);
            buffer_.Reserve(Base::read_size);
            ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
                    for (auto it = current_; it != buffer_.end(); it++) {
                        if (THRILL_UNLIKELY(*it == '\n')) {
                            current_ = it + 1;
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        std::string Next() {
            std::string ret;
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        size_t strlen = it - current_;
                        current_ = it + 1;
                        LOG << "returning string";
                        return ret.append(it - strlen, it);
                    }
                }
                ret.append(current_, buffer_.end());
                current_ = buffer_.begin();
                ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    close(c_file_);
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = OpenFile(files_[current_file_].first);
                        ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() + files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (ret.length()) {
                        LOG << "end - returning string of length" << ret.length();
                        return ret;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_; // REVIEW(an): use a const & to the vector
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        int c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        std::string Next() {
            std::string ret;
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        size_t strlen = it - current_;
                        current_ = it + 1;
						if (ret.size()) {
							return ret.append(it - strlen, it);
						} else {
							return std::string(it - strlen, it);
						}
                    }
                }
                ret.append(current_, buffer_.end());
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    c_file_.close();
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                        // current_ = files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (ret.length()) {
                        LOG << "end - returning string of length" << ret.length();
                        return ret;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                offset_ += buffer_.size();
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    c_file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        offset_ = 0;

                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(c_file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<commit_msg>reduce allocations<commit_after>/*******************************************************************************
 * thrill/api/read_lines.hpp
 *
 * Part of Project Thrill.
 *
 * Copyright (C) 2015 Alexander Noe <aleexnoe@gmail.com>
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef THRILL_API_READ_LINES_HEADER
#define THRILL_API_READ_LINES_HEADER

#include <thrill/api/dia.hpp>
#include <thrill/api/source_node.hpp>
#include <thrill/common/defines.hpp>
#include <thrill/common/logger.hpp>
#include <thrill/common/math.hpp>
#include <thrill/common/string.hpp>
#include <thrill/core/file_io.hpp>
#include <thrill/net/buffer_builder.hpp>

#include <string>
#include <utility>
#include <vector>

namespace thrill {
namespace api {

//! \addtogroup api Interface
//! \{

/*!
 * A DIANode which performs a line-based Read operation. Read reads a file from
 * the file system and emits it as a DIA.
 */
class ReadLinesNode : public SourceNode<std::string>
{
public:
    using Super = SourceNode<std::string>;
    using Super::context_;

    using FileSizePair = std::pair<std::string, size_t>;

    static const bool debug = false;

    /*!
     * Constructor for a ReadLinesNode. Sets the Context
     * and file path.
     *
     * \param ctx Reference to Context, which holds references to data and
     * network.
     *
     * \param path Path of the input file(s)
     */
    ReadLinesNode(Context& ctx,
                  const std::string& path,
                  StatsNode* stats_node)
        : Super(ctx, { }, stats_node),
          path_(path)
    {
		LOG << "Opening read notes for " << path_;

        filesize_prefix_ = core::GlobFileSizePrefixSum(path_);

        for (auto file : filesize_prefix_) {
            if (core::IsCompressed(file.first)) {
                contains_compressed_file_ = true;
                break;
            }
        }
    }

    void PushData() final {
        if (contains_compressed_file_) {
            InputLineIteratorCompressed it = InputLineIteratorCompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
        else {
            InputLineIteratorUncompressed it = InputLineIteratorUncompressed(
                filesize_prefix_, context_.my_rank(), context_.num_workers());

            // Hook Read
            while (it.HasNext()) {
                this->PushItem(it.Next());
            }
        }
    }

    void Dispose() final { }

    /*!
     * Produces an 'empty' function stack, which only contains the identity
     * emitter function.
     *
     * \return Empty function stack
     */
    auto ProduceStack() {
        return FunctionStack<std::string>();
    }

private:
    //! True, if at least one input file is compressed.
    bool contains_compressed_file_ = false;
    //! Path of the input file.
    std::string path_;

    std::vector<std::pair<std::string, size_t> > filesize_prefix_;

    // REVIEW(an): this is useless, you never use the inheritance.  But, you
    // actually SHOULD use it! for all member fields and methods that are in
    // common. But NOT for virtual functions. Remove the virtuals. Find out what
    // functions the methods below have in common and make them functions of the
    // superclass.
    class InputLineIterator
    {
    public:
        static const bool debug = false;
        const size_t read_size = 2 * 1024 * 1024;

        virtual ~InputLineIterator() { }
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorUncompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorUncompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            auto my_start_and_end = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            size_t my_start = std::get<0>(my_start_and_end);
            my_end_ = std::get<1>(my_start_and_end);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }
            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                c_file_ = OpenFile(files_[current_file_].first);
            }
            else {
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                return;
            }

            // find offset in current file:
            // offset = start - sum of previous file sizes
            offset_ = lseek(c_file_, my_start - files_[current_file_].second, SEEK_CUR);
            buffer_.Reserve(Base::read_size);
            ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();

            if (offset_ != 0) {
                bool found_n = false;

                // find next newline, discard all previous data as previous worker already covers it
                while (!found_n) {
                    for (auto it = current_; it != buffer_.end(); it++) {
                        if (THRILL_UNLIKELY(*it == '\n')) {
                            current_ = it + 1;
                            found_n = true;
                            break;
                        }
                    }
                    // no newline found: read new data into buffer_builder
                    if (!found_n) {
                        current_ = buffer_.begin();
                        offset_ += buffer_.size();
                        buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        // EOF = newline per definition
                        if (!buffer_size) {
                            found_n = true;
                        }
                        buffer_.set_size(buffer_size);
                    }
                }
                assert(*(current_ - 1) == '\n' || !buffer_size);
            }
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        std::string Next() {
            std::string ret;
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        size_t strlen = it - current_;
                        current_ = it + 1;
                        LOG << "returning string";
						if (ret.size()) {
							return ret.append(it - strlen, it);
						} else {
							return std::string(it - strlen, it);
						}
                    }
                }
                ret.append(current_, buffer_.end());
                current_ = buffer_.begin();
                ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    close(c_file_);
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = OpenFile(files_[current_file_].first);
                        ssize_t buffer_size = read(c_file_, buffer_.data(), Base::read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin() + files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (ret.length()) {
                        LOG << "end - returning string of length" << ret.length();
                        return ret;
                    }
                }
            }
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            size_t global_index = offset_ + current_ - buffer_.begin() + files_[current_file_].second;
            return global_index < my_end_ ||
				(global_index == my_end_ &&
				 files_[current_file_ + 1].second - files_[current_file_].second >
				 offset_ + (current_ - buffer_.begin()));
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! Open file and return file handle
        //! \param path Path to open
        int OpenFile(const std::string& path) {
            return open(path.c_str(), O_RDONLY);
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_; // REVIEW(an): use a const & to the vector
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        int c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };

    //! InputLineIterator gives you access to lines of a file
    class InputLineIteratorCompressed : public InputLineIterator
    {
    public:
        using Base = InputLineIterator;

        //! Creates an instance of iterator that reads file line based
        InputLineIteratorCompressed(
            std::vector<FileSizePair> files,
            size_t my_id,
            size_t num_workers)
            : files_(files),
              my_id_(my_id),
              num_workers_(num_workers) {

            input_size_ = files[NumFiles()].second;

            // Go to start of 'local part'.
            size_t my_start;
            std::tie(my_start, my_end_) = common::CalculateLocalRange(input_size_, num_workers_, my_id_);

            while (files_[current_file_ + 1].second <= my_start) {
                current_file_++;
            }

            for (size_t file_nr = current_file_; file_nr < NumFiles(); file_nr++) {
                if (files[file_nr + 1].second == my_end_) {
                    break;
                }
                if (files[file_nr + 1].second > my_end_) {
                    my_end_ = files_[file_nr].second;
                    break;
                }
            }

            if (my_start < my_end_) {
                LOG << "Opening file " << current_file_;
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
            }
            else {
                // No local files, set buffer size to 2, so HasNext() does not try to read
                LOG << "my_start : " << my_start << " my_end_: " << my_end_;
                buffer_.Reserve(2);
                buffer_.set_size(2);
				current_ = buffer_.begin();
                return;
            }
            buffer_.Reserve(read_size);
            ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
            buffer_.set_size(buffer_size);
			current_ = buffer_.begin();
        }

        //! returns the next element if one exists
        //!
        //! does no checks whether a next element exists!
        std::string Next() {
            std::string ret;
            while (true) {
                for (auto it = current_; it != buffer_.end(); it++) {
                    if (THRILL_UNLIKELY(*it == '\n')) {
                        size_t strlen = it - current_;
                        current_ = it + 1;
						if (ret.size()) {
							return ret.append(it - strlen, it);
						} else {
							return std::string(it - strlen, it);
						}
                    }
                }
                ret.append(current_, buffer_.end());
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                offset_ += buffer_.size();
                if (buffer_size) {
                    buffer_.set_size(buffer_size);
                }
                else {
                    LOG << "Opening new file!";
                    c_file_.close();
                    current_file_++;
                    offset_ = 0;

                    if (current_file_ < NumFiles()) {
                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                        buffer_.set_size(buffer_size);
                    }
                    else {
                        current_ = buffer_.begin();
                        // current_ = files_[current_file_].second - files_[current_file_ - 1].second;
                    }

                    if (ret.length()) {
                        LOG << "end - returning string of length" << ret.length();
                        return ret;
                    }
                }
            }
        }

        size_t NumFiles() {
            return files_.size() - 1;
        }

        //! returns true, if an element is available in local part
        bool HasNext() {
            // if block is fully read, read next block. needs to be done here
            // as HasNext() has to know if file is finished
            //         v-- no new line at end ||   v-- newline at end of file
            if (current_ >= buffer_.end() || (current_ + 1 >= buffer_.end() && *current_ == '\n')) {
                LOG << "New buffer in HasNext()";
                current_ = buffer_.begin();
                ssize_t buffer_size = c_file_.read(buffer_.data(), read_size);
                offset_ += buffer_.size();
                if (buffer_size > 1 || (buffer_size == 1 && buffer_[0] != '\n')) {
                    buffer_.set_size(buffer_size);
                    return true;
                }
                else {
                    LOG << "Opening new file in HasNext()";
                    // already at last file
                    if (current_file_ >= NumFiles() - 1) {
                        return false;
                    }
                    c_file_.close();
                    // if (this worker reads at least one more file)
                    if (my_end_ > files_[current_file_ + 1].second) {
                        current_file_++;
                        offset_ = 0;

                        c_file_ = core::SysFile::OpenForRead(files_[current_file_].first);
                        buffer_.set_size(c_file_.read(buffer_.data(), read_size));
                        return true;
                    }
                    else {
                        return false;
                    }
                }
            }
            else {
				return files_[current_file_].second < my_end_;
            }
        }

    private:
        //! Input files with size prefixsum.
        std::vector<FileSizePair> files_;
        //! Index of current file in files_
        size_t current_file_ = 0;
        //! File handle to files_[current_file_]
        core::SysFile c_file_;
        //! Offset of current block in c_file_.
        size_t offset_ = 0;
        //! Start of next element in current buffer.
		unsigned char* current_;
        //! (exclusive) end of local block
        size_t my_end_;
        //! Byte buffer to create line-std::strings
        net::BufferBuilder buffer_;
        //! Worker ID
        size_t my_id_;
        //! total number of workers
        size_t num_workers_;
        //! Size of all files combined (in bytes)
        size_t input_size_;
    };
};

DIARef<std::string> ReadLines(Context& ctx, std::string filepath) {

    StatsNode* stats_node = ctx.stats_graph().AddNode(
        "ReadLines", DIANodeType::DOP);

    auto shared_node =
        std::make_shared<ReadLinesNode>(
            ctx, filepath, stats_node);

    auto read_stack = shared_node->ProduceStack();

    return DIARef<std::string, decltype(read_stack)>(
        shared_node, read_stack, { stats_node });
}

//! \}

} // namespace api
} // namespace thrill

#endif // !THRILL_API_READ_LINES_HEADER

/******************************************************************************/
<|endoftext|>