<commit_before>/* The MIT License (MIT)

Copyright (c) 2015 Gabriel Corona

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#ifndef DNSFWD_DNSFWD_HPP
#define DNSFWD_DNSFWD_HPP

#include <syslog.h>

#include <arpa/inet.h>

#include <cstdint>

#include <iostream>
#include <memory>
#include <vector>
#include <array>
#include <string>
#include <chrono>

#include <boost/bind.hpp>

#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>

#include <boost/system/error_code.hpp>

#include <boost/intrusive/list.hpp>
#include <boost/intrusive/list_hook.hpp>
#include <boost/intrusive/set.hpp>
#include <boost/intrusive/set_hook.hpp>

#include <boost/asio/io_service.hpp>
#include <boost/asio/placeholders.hpp>
#include <boost/asio/ip/udp.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/generic/datagram_protocol.hpp>

#define LOG(k) if(::dnsfwd::loglevel >= LOG_ ## k)\
  std::cerr << ::dnsfwd::logformat[(LOG_ ## k)]

namespace dnsfwd {

struct endpoint;
class message;
class server;
class client;
class service;

struct order_message_by_server_id;
struct order_message_by_client_id;

const size_t MIN_MESSAGE_SIZE = 12;
extern int loglevel;
extern const char** logformat;

struct deleter {
  template<class T>
  void operator()(T* p) const
  {
    delete p;
  }
};

struct endpoint {
  std::string name;
  std::string port;

  boost::asio::ip::udp::endpoint udp_endpoint(
    boost::asio::io_service& service, const char* default_port) const;
};

struct config {
  std::vector<std::string> args;
  std::vector<endpoint> bind_udp;
  std::vector<endpoint> connect_tcp;
#ifdef USE_SYSTEMD
  std::size_t listen_fds = 0;
#endif
};

void setup_config(dnsfwd::config& config, int argc, char** argv);

class message {
public:
  message() : buffer_(1024), server_(nullptr)
  {
  }
  message(message &) = delete;
  message& operator=(message &) = delete;

  std::vector<char> buffer_;
  std::uint16_t size_;
  std::uint16_t network_size_;
  std::uint16_t client_id_;
  std::uint16_t server_id_;
  std::chrono::steady_clock::time_point timestamp_;
  boost::asio::generic::datagram_protocol::endpoint endpoint_;
  dnsfwd::server* server_;

public:
  std::uint16_t id() const
  {
    std::uint16_t res;
    std::memcpy(&res, buffer_.data(), sizeof(res));
    return res;
  }
  void id(std::uint16_t id)
  {
    std::memcpy(buffer_.data(), &id, sizeof(id));
  }
  std::array<boost::asio::const_buffer, 2> vc_buffer()
  {
    network_size_ = htons(size_);
    return {{
      boost::asio::buffer( &network_size_, sizeof(network_size_) ),
      boost::asio::buffer( buffer_.data(), size_ )
    }};
  }
  bool operator==(message const& that) const {
    return this==&that;
  }
private:
  boost::intrusive::set_member_hook<> by_client_id_hook_;
  boost::intrusive::list_member_hook<> queue_hook_;
public:
  typedef boost::intrusive::member_hook<
    message,
    boost::intrusive::set_member_hook<>,
    &message::by_client_id_hook_
  > ByClientIdOptions;
  typedef boost::intrusive::member_hook<
    message,
    boost::intrusive::list_member_hook<>,
    &message::queue_hook_
  > QueueOptions;
};

struct order_message_by_server_id {
  bool operator()(message const& a, message const& b) const
  {
    return a.server_id_ < b.server_id_;
  }
  bool operator()(message const& a, std::uint16_t id) const
  {
    return a.server_id_ < id;
  }
  bool operator()(std::uint16_t id, message const& b) const
  {
    return id < b.server_id_;
  }
};
struct order_message_by_client_id {
  bool operator()(message const& a, message const& b) const
  {
    return a.client_id_ < b.client_id_;
  }
  bool operator()(message const& a, std::uint16_t id) const
  {
    return a.client_id_ < id;
  }
  bool operator()(std::uint16_t id, message const& b) const
  {
    return id < b.client_id_;
  }
};

class server {
public:
  server(boost::asio::io_service& io_service, service& service,
    dnsfwd::endpoint const& udp_endpoint);
#ifdef USE_SYSTEMD
  server(boost::asio::io_service& io_service, service& service, int socket);
#endif
public:
  void send_response(
    std::vector<char> response,
    boost::asio::generic::datagram_protocol::endpoint endpoint);
private:
  void start_receive();
  void on_message(const boost::system::error_code& error, std::size_t size);
  void response_sent(std::vector<char>& response,
    const boost::system::error_code& error, std::size_t size);
private:
  service* service_;
  boost::asio::generic::datagram_protocol::socket socket_;
  std::unique_ptr<message> context_;
};

class client
  : public std::enable_shared_from_this<client> {
public:
  client(boost::asio::io_service& io_service, service& service);
  ~client();
  bool add_request(std::unique_ptr<message>& context);
  std::uint16_t random_client_id();
  void connect();
private:
  void start_receive();
  void on_message_size(const boost::system::error_code& error, std::size_t size);
  void on_message(const boost::system::error_code& error, std::size_t size);
  void reset();
  void send();
  void on_send(const boost::system::error_code& error, std::size_t bytes_transferred);
  void clear(std::chrono::steady_clock::time_point time);
private:
  typedef boost::intrusive::set<
    message,
    message::ByClientIdOptions,
    boost::intrusive::compare<order_message_by_client_id>
  > by_client_id_type;

  typedef boost::intrusive::list<
    message, message::QueueOptions, boost::intrusive::cache_last<true>
  > queue_type;

  boost::asio::io_service* io_service_;
  service* service_;
  boost::asio::ip::tcp::socket socket_;
  by_client_id_type by_client_id_;
  queue_type queue_;
  std::unique_ptr<message> context_;

  std::uint16_t response_size_;
  std::vector<char> buffer_;
};

class service {
public:
  service(boost::asio::io_service& io_service, dnsfwd::config config);
  void add_request(std::unique_ptr<message>& context);
  std::uint16_t random_id()
  {
    return (std::uint16_t) random_();
  }
  // TODO, remote this
  std::vector<endpoint> const& tcp_connect_endpoints()
  {
    return config_.connect_tcp;
  }
  std::vector<endpoint> const& udp_listen_endpoints()
  {
    return config_.bind_udp;
  }
  std::unique_ptr<message> unqueue();
  void unregister(std::shared_ptr<client> client);
  std::chrono::seconds time_to_live() const
  {
    return std::chrono::seconds(60);
  }
private:

  typedef boost::intrusive::list<
    message, message::QueueOptions, boost::intrusive::cache_last<true>
  > queue_type;

  boost::asio::io_service* io_service_;
  dnsfwd::config config_;
  std::vector<std::unique_ptr<server>> servers_;
  std::shared_ptr<client> client_;
  boost::random::mt11213b random_;
  queue_type queue_;
};

}

#endif
<commit_msg>Make listen_fds an int<commit_after>/* The MIT License (MIT)

Copyright (c) 2015 Gabriel Corona

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

#ifndef DNSFWD_DNSFWD_HPP
#define DNSFWD_DNSFWD_HPP

#include <syslog.h>

#include <arpa/inet.h>

#include <cstdint>

#include <iostream>
#include <memory>
#include <vector>
#include <array>
#include <string>
#include <chrono>

#include <boost/bind.hpp>

#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>

#include <boost/system/error_code.hpp>

#include <boost/intrusive/list.hpp>
#include <boost/intrusive/list_hook.hpp>
#include <boost/intrusive/set.hpp>
#include <boost/intrusive/set_hook.hpp>

#include <boost/asio/io_service.hpp>
#include <boost/asio/placeholders.hpp>
#include <boost/asio/ip/udp.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/generic/datagram_protocol.hpp>

#define LOG(k) if(::dnsfwd::loglevel >= LOG_ ## k)\
  std::cerr << ::dnsfwd::logformat[(LOG_ ## k)]

namespace dnsfwd {

struct endpoint;
class message;
class server;
class client;
class service;

struct order_message_by_server_id;
struct order_message_by_client_id;

const size_t MIN_MESSAGE_SIZE = 12;
extern int loglevel;
extern const char** logformat;

struct deleter {
  template<class T>
  void operator()(T* p) const
  {
    delete p;
  }
};

struct endpoint {
  std::string name;
  std::string port;

  boost::asio::ip::udp::endpoint udp_endpoint(
    boost::asio::io_service& service, const char* default_port) const;
};

struct config {
  std::vector<std::string> args;
  std::vector<endpoint> bind_udp;
  std::vector<endpoint> connect_tcp;
#ifdef USE_SYSTEMD
  int listen_fds = 0;
#endif
};

void setup_config(dnsfwd::config& config, int argc, char** argv);

class message {
public:
  message() : buffer_(1024), server_(nullptr)
  {
  }
  message(message &) = delete;
  message& operator=(message &) = delete;

  std::vector<char> buffer_;
  std::uint16_t size_;
  std::uint16_t network_size_;
  std::uint16_t client_id_;
  std::uint16_t server_id_;
  std::chrono::steady_clock::time_point timestamp_;
  boost::asio::generic::datagram_protocol::endpoint endpoint_;
  dnsfwd::server* server_;

public:
  std::uint16_t id() const
  {
    std::uint16_t res;
    std::memcpy(&res, buffer_.data(), sizeof(res));
    return res;
  }
  void id(std::uint16_t id)
  {
    std::memcpy(buffer_.data(), &id, sizeof(id));
  }
  std::array<boost::asio::const_buffer, 2> vc_buffer()
  {
    network_size_ = htons(size_);
    return {{
      boost::asio::buffer( &network_size_, sizeof(network_size_) ),
      boost::asio::buffer( buffer_.data(), size_ )
    }};
  }
  bool operator==(message const& that) const {
    return this==&that;
  }
private:
  boost::intrusive::set_member_hook<> by_client_id_hook_;
  boost::intrusive::list_member_hook<> queue_hook_;
public:
  typedef boost::intrusive::member_hook<
    message,
    boost::intrusive::set_member_hook<>,
    &message::by_client_id_hook_
  > ByClientIdOptions;
  typedef boost::intrusive::member_hook<
    message,
    boost::intrusive::list_member_hook<>,
    &message::queue_hook_
  > QueueOptions;
};

struct order_message_by_server_id {
  bool operator()(message const& a, message const& b) const
  {
    return a.server_id_ < b.server_id_;
  }
  bool operator()(message const& a, std::uint16_t id) const
  {
    return a.server_id_ < id;
  }
  bool operator()(std::uint16_t id, message const& b) const
  {
    return id < b.server_id_;
  }
};
struct order_message_by_client_id {
  bool operator()(message const& a, message const& b) const
  {
    return a.client_id_ < b.client_id_;
  }
  bool operator()(message const& a, std::uint16_t id) const
  {
    return a.client_id_ < id;
  }
  bool operator()(std::uint16_t id, message const& b) const
  {
    return id < b.client_id_;
  }
};

class server {
public:
  server(boost::asio::io_service& io_service, service& service,
    dnsfwd::endpoint const& udp_endpoint);
#ifdef USE_SYSTEMD
  server(boost::asio::io_service& io_service, service& service, int socket);
#endif
public:
  void send_response(
    std::vector<char> response,
    boost::asio::generic::datagram_protocol::endpoint endpoint);
private:
  void start_receive();
  void on_message(const boost::system::error_code& error, std::size_t size);
  void response_sent(std::vector<char>& response,
    const boost::system::error_code& error, std::size_t size);
private:
  service* service_;
  boost::asio::generic::datagram_protocol::socket socket_;
  std::unique_ptr<message> context_;
};

class client
  : public std::enable_shared_from_this<client> {
public:
  client(boost::asio::io_service& io_service, service& service);
  ~client();
  bool add_request(std::unique_ptr<message>& context);
  std::uint16_t random_client_id();
  void connect();
private:
  void start_receive();
  void on_message_size(const boost::system::error_code& error, std::size_t size);
  void on_message(const boost::system::error_code& error, std::size_t size);
  void reset();
  void send();
  void on_send(const boost::system::error_code& error, std::size_t bytes_transferred);
  void clear(std::chrono::steady_clock::time_point time);
private:
  typedef boost::intrusive::set<
    message,
    message::ByClientIdOptions,
    boost::intrusive::compare<order_message_by_client_id>
  > by_client_id_type;

  typedef boost::intrusive::list<
    message, message::QueueOptions, boost::intrusive::cache_last<true>
  > queue_type;

  boost::asio::io_service* io_service_;
  service* service_;
  boost::asio::ip::tcp::socket socket_;
  by_client_id_type by_client_id_;
  queue_type queue_;
  std::unique_ptr<message> context_;

  std::uint16_t response_size_;
  std::vector<char> buffer_;
};

class service {
public:
  service(boost::asio::io_service& io_service, dnsfwd::config config);
  void add_request(std::unique_ptr<message>& context);
  std::uint16_t random_id()
  {
    return (std::uint16_t) random_();
  }
  // TODO, remote this
  std::vector<endpoint> const& tcp_connect_endpoints()
  {
    return config_.connect_tcp;
  }
  std::vector<endpoint> const& udp_listen_endpoints()
  {
    return config_.bind_udp;
  }
  std::unique_ptr<message> unqueue();
  void unregister(std::shared_ptr<client> client);
  std::chrono::seconds time_to_live() const
  {
    return std::chrono::seconds(60);
  }
private:

  typedef boost::intrusive::list<
    message, message::QueueOptions, boost::intrusive::cache_last<true>
  > queue_type;

  boost::asio::io_service* io_service_;
  dnsfwd::config config_;
  std::vector<std::unique_ptr<server>> servers_;
  std::shared_ptr<client> client_;
  boost::random::mt11213b random_;
  queue_type queue_;
};

}

#endif
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2020 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "server.hh"

#include <map>
#include <ranges>
#include <seastar/core/sleep.hh>
#include <seastar/core/future-util.hh>
#include <seastar/core/coroutine.hh>
#include <seastar/core/pipe.hh>

#include "fsm.hh"
#include "log.hh"

using namespace std::chrono_literals;

namespace raft {

class server_impl : public rpc_server, public server {
public:
    explicit server_impl(server_id uuid, std::unique_ptr<rpc> rpc,
        std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
        seastar::shared_ptr<failure_detector> failure_detector);

    server_impl(server_impl&&) = delete;

    ~server_impl() {}

    // rpc_server interface
    void append_entries(server_id from, append_request_recv append_request) override;
    void append_entries_reply(server_id from, append_reply reply) override;
    void request_vote(server_id from, vote_request vote_request) override;
    void request_vote_reply(server_id from, vote_reply vote_reply) override;

    // server interface
    future<> add_entry(command command, wait_type type);
    future<> apply_snapshot(server_id from, install_snapshot snp) override;
    future<> add_server(server_id id, bytes node_info, clock_type::duration timeout) override;
    future<> remove_server(server_id id, clock_type::duration timeout) override;
    future<> start() override;
    future<> abort() override;
    term_t get_current_term() const override;
    future<> read_barrier() override;
    void make_me_leader() override;
private:
    std::unique_ptr<rpc> _rpc;
    std::unique_ptr<state_machine> _state_machine;
    std::unique_ptr<storage> _storage;
    seastar::shared_ptr<failure_detector> _failure_detector;
    // Protocol deterministic finite-state machine
    std::unique_ptr<fsm> _fsm;
    // id of this server
    server_id _id;
    seastar::timer<lowres_clock> _ticker;

    seastar::pipe<std::vector<log_entry_ptr>> _apply_entries = seastar::pipe<std::vector<log_entry_ptr>>(10);

    struct op_status {
        term_t term; // term the entry was added with
        promise<> done; // notify when done here
    };

    // Entries that have a waiter that needs to be notified when the
    // respective entry is known to be committed.
    std::map<index_t, op_status> _awaited_commits;

    // Entries that have a waiter that needs to be notified after
    // the respective entry is applied.
    std::map<index_t, op_status> _awaited_applies;

    // Called to commit entries (on a leader or otherwise).
    void notify_waiters(std::map<index_t, op_status>& waiters, const std::vector<log_entry_ptr>& entries);

    // This fiber process fsm output, by doing the following steps in that order:
    //  - persists current term and voter
    //  - persists unstable log entries on disk.
    //  - sends out messages
    future<> io_fiber(index_t stable_idx);

    // This fiber runs in the background and applies committed entries.
    future<> applier_fiber();

    template <typename T> future<> add_entry_internal(T command, wait_type type);
    template <typename Message> future<> send_message(server_id id, Message m);

    // Apply a dummy entry. Dummy entry is not propagated to the
    // state machine, but waiting for it to be "applied" ensures
    // all previous entries are applied as well.
    // Resolves when the entry is committed.
    // The function has to be called on the leader, throws otherwise
    // May fail because of an internal error or because the leader
    // has changed and the entry was replaced by another one,
    // submitted to the new leader.
    future<> apply_dummy_entry();

    future<> _applier_status = make_ready_future<>();
    future<> _io_status = make_ready_future<>();

    friend std::ostream& operator<<(std::ostream& os, const server_impl& s);
};

server_impl::server_impl(server_id uuid, std::unique_ptr<rpc> rpc,
        std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
        seastar::shared_ptr<failure_detector> failure_detector) :
                    _rpc(std::move(rpc)), _state_machine(std::move(state_machine)),
                    _storage(std::move(storage)), _failure_detector(failure_detector),
                    _id(uuid) {
    set_rpc_server(_rpc.get());
}

future<> server_impl::start() {
    auto [term, vote] = co_await _storage->load_term_and_vote();
    auto snapshot  = co_await _storage->load_snapshot();
    auto log_entries = co_await _storage->load_log();
    auto log = raft::log(std::move(snapshot), std::move(log_entries));
    index_t stable_idx = log.stable_idx();
    _fsm = std::make_unique<fsm>(_id, term, vote, std::move(log), *_failure_detector);
    assert(_fsm->get_current_term() != term_t(0));

    // start fiber to persist entries added to in-memory log
    _io_status = io_fiber(stable_idx);
    // start fiber to apply committed entries
    _applier_status = applier_fiber();

    _ticker.arm_periodic(100ms);
    _ticker.set_callback([this] {
        _fsm->tick();
    });

    co_return;
}

template <typename T>
future<> server_impl::add_entry_internal(T command, wait_type type) {
    logger.trace("An entry is submitted on a leader");

    // lock access to the raft log while it is been updated

    // @todo: ensure the reference to the entry is stable between
    // yields, before removing _log_lock.
    const log_entry& e = _fsm->add_entry(std::move(command));

    auto& container = type == wait_type::committed ? _awaited_commits : _awaited_applies;

    // This will track the commit/apply status of the entry
    auto [it, inserted] = container.emplace(e.idx, op_status{e.term, promise<>()});
    assert(inserted);
    return it->second.done.get_future();
}

future<> server_impl::add_entry(command command, wait_type type) {
    return add_entry_internal(std::move(command), type);
}

future<> server_impl::apply_dummy_entry() {
    return add_entry_internal(log_entry::dummy(), wait_type::applied);
}
void server_impl::append_entries(server_id from, append_request_recv append_request) {
    _fsm->step(from, std::move(append_request));
}

void server_impl::append_entries_reply(server_id from, append_reply reply) {
    _fsm->step(from, std::move(reply));
}

void server_impl::request_vote(server_id from, vote_request vote_request) {
    _fsm->step(from, std::move(vote_request));
}

void server_impl::request_vote_reply(server_id from, vote_reply vote_reply) {
    _fsm->step(from, std::move(vote_reply));
}

void server_impl::notify_waiters(std::map<index_t, op_status>& waiters,
        const std::vector<log_entry_ptr>& entries) {
    index_t commit_idx = entries.back()->idx;
    index_t first_idx = entries.front()->idx;

    while (waiters.size() != 0) {
        auto it = waiters.begin();
        if (it->first > commit_idx) {
            break;
        }
        auto [entry_idx, status] = std::move(*it);

        // if there is a waiter entry with an index smaller than first entry
        // it means that notification is out of order which is prohinited
        assert(entry_idx >= first_idx);

        waiters.erase(it);
        if (status.term == entries[entry_idx - first_idx]->term) {
            status.done.set_value();
        } else {
            // term does not match which means that between the entry was submitted
            // and committed there was a leadership change and the entry was replaced.
            status.done.set_exception(dropped_entry());
        }
    }
}

template <typename Message>
future<> server_impl::send_message(server_id id, Message m) {
    return std::visit([this, id] (auto&& m) {
        using T = std::decay_t<decltype(m)>;
        if constexpr (std::is_same_v<T, append_reply>) {
            return _rpc->send_append_entries_reply(id, m);
        } else if constexpr (std::is_same_v<T, append_request_send>) {
            return _rpc->send_append_entries(id, m);
        } else if constexpr (std::is_same_v<T, vote_request>) {
            return _rpc->send_vote_request(id, m);
        } else if constexpr (std::is_same_v<T, vote_reply>) {
            return _rpc->send_vote_reply(id, m);
        } else {
            static_assert(!sizeof(Message*), "not all message types are handled");
            return make_ready_future<>();
        }
    }, std::move(m));
}

future<> server_impl::io_fiber(index_t last_stable) {
    logger.trace("[{}] io_fiber start", _id);
    try {
        while (true) {
            auto batch = co_await _fsm->poll_output();

            if (batch.term != term_t{}) {
                // Current term and vote are always persisted
                // together. A vote may change independently of
                // term, but it's safe to update both in this
                // case.
                co_await _storage->store_term_and_vote(batch.term, batch.vote);
            }

            if (batch.log_entries.size()) {
                auto& entries = batch.log_entries;

                if (last_stable >= entries[0]->idx) {
                    co_await _storage->truncate_log(entries[0]->idx);
                }

                // Combine saving and truncating into one call?
                // will require storage to keep track of last idx
                co_await _storage->store_log_entries(entries);

                last_stable = (*entries.crbegin())->idx;
            }

            if (batch.messages.size()) {
                // after entries are persisted we can send messages
                co_await seastar::parallel_for_each(std::move(batch.messages), [this] (std::pair<server_id, rpc_message>& message) {
                    return send_message(message.first, std::move(message.second));
                });
            }

            // process committed entries
            if (batch.committed.size()) {
                notify_waiters(_awaited_commits, batch.committed);
                co_await _apply_entries.writer.write(std::move(batch.committed));
            }
        }
    } catch (seastar::broken_condition_variable&) {
        // log fiber is stopped explicitly.
    } catch (...) {
        logger.error("[{}] io fiber stopped because of the error: {}", _id, std::current_exception());
    }
    co_return;
}

future<> server_impl::apply_snapshot(server_id from, install_snapshot snp) {
    return make_ready_future<>();
}

future<> server_impl::applier_fiber() {
    logger.trace("applier_fiber start");
    try {
        while (true) {
            auto opt_batch = co_await _apply_entries.reader.read();
            if (!opt_batch) {
                // EOF
                break;
            }
            std::vector<command_cref> commands;
            commands.reserve(opt_batch->size());

            std::ranges::copy(
                    *opt_batch |
                    std::views::filter([] (log_entry_ptr& entry) { return std::holds_alternative<command>(entry->data); }) |
                    std::views::transform([] (log_entry_ptr& entry) { return std::cref(std::get<command>(entry->data)); }),
                    std::back_inserter(commands));

            co_await _state_machine->apply(std::move(commands));
            notify_waiters(_awaited_applies, *opt_batch);
        }
    } catch (...) {
        logger.error("applier fiber {} stopped because of the error: {}", _id, std::current_exception());
    }
    co_return;
}

term_t server_impl::get_current_term() const {
    return _fsm->get_current_term();
}

future<> server_impl::read_barrier() {
    if (_fsm->can_read()) {
        co_return;
    }

    co_await apply_dummy_entry();
    co_return;
}

future<> server_impl::abort() {
    logger.trace("abort() called");
    _fsm->stop();
    {
        // there is not explicit close for the pipe!
        auto tmp = std::move(_apply_entries.writer);
    }
    for (auto& ac: _awaited_commits) {
        ac.second.done.set_exception(stopped_error());
    }
    for (auto& aa: _awaited_applies) {
        aa.second.done.set_exception(stopped_error());
    }
    _awaited_commits.clear();
    _awaited_applies.clear();
    _ticker.cancel();

    return seastar::when_all_succeed(std::move(_io_status), std::move(_applier_status),
            _rpc->abort(), _state_machine->abort(), _storage->abort()).discard_result();
}

future<> server_impl::add_server(server_id id, bytes node_info, clock_type::duration timeout) {
    return make_ready_future<>();
}

// Removes a server from the cluster. If the server is not a member
// of the cluster does nothing. Can be called on a leader only
// otherwise throws not_a_leader.
// Cannot be called until previous add/remove server completes
// otherwise conf_change_in_progress exception is returned.
future<> server_impl::remove_server(server_id id, clock_type::duration timeout) {
    return make_ready_future<>();
}

void server_impl::make_me_leader() {
    _fsm->become_leader();
}

std::unique_ptr<server> create_server(server_id uuid, std::unique_ptr<rpc> rpc,
    std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
    seastar::shared_ptr<failure_detector> failure_detector) {
    return std::make_unique<raft::server_impl>(uuid, std::move(rpc), std::move(state_machine),
        std::move(storage), failure_detector);
}

std::ostream& operator<<(std::ostream& os, const server_impl& s) {
    os << "[id: " << s._id << ", fsm (" << s._fsm << ")]\n";
    return os;
}

} // end of namespace raft
<commit_msg>raft: Forbid server address 0 as it has special meaning<commit_after>/*
 * Copyright (C) 2020 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "server.hh"

#include <map>
#include <ranges>
#include <seastar/core/sleep.hh>
#include <seastar/core/future-util.hh>
#include <seastar/core/coroutine.hh>
#include <seastar/core/pipe.hh>

#include "fsm.hh"
#include "log.hh"

using namespace std::chrono_literals;

namespace raft {

class server_impl : public rpc_server, public server {
public:
    explicit server_impl(server_id uuid, std::unique_ptr<rpc> rpc,
        std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
        seastar::shared_ptr<failure_detector> failure_detector);

    server_impl(server_impl&&) = delete;

    ~server_impl() {}

    // rpc_server interface
    void append_entries(server_id from, append_request_recv append_request) override;
    void append_entries_reply(server_id from, append_reply reply) override;
    void request_vote(server_id from, vote_request vote_request) override;
    void request_vote_reply(server_id from, vote_reply vote_reply) override;

    // server interface
    future<> add_entry(command command, wait_type type);
    future<> apply_snapshot(server_id from, install_snapshot snp) override;
    future<> add_server(server_id id, bytes node_info, clock_type::duration timeout) override;
    future<> remove_server(server_id id, clock_type::duration timeout) override;
    future<> start() override;
    future<> abort() override;
    term_t get_current_term() const override;
    future<> read_barrier() override;
    void make_me_leader() override;
private:
    std::unique_ptr<rpc> _rpc;
    std::unique_ptr<state_machine> _state_machine;
    std::unique_ptr<storage> _storage;
    seastar::shared_ptr<failure_detector> _failure_detector;
    // Protocol deterministic finite-state machine
    std::unique_ptr<fsm> _fsm;
    // id of this server
    server_id _id;
    seastar::timer<lowres_clock> _ticker;

    seastar::pipe<std::vector<log_entry_ptr>> _apply_entries = seastar::pipe<std::vector<log_entry_ptr>>(10);

    struct op_status {
        term_t term; // term the entry was added with
        promise<> done; // notify when done here
    };

    // Entries that have a waiter that needs to be notified when the
    // respective entry is known to be committed.
    std::map<index_t, op_status> _awaited_commits;

    // Entries that have a waiter that needs to be notified after
    // the respective entry is applied.
    std::map<index_t, op_status> _awaited_applies;

    // Called to commit entries (on a leader or otherwise).
    void notify_waiters(std::map<index_t, op_status>& waiters, const std::vector<log_entry_ptr>& entries);

    // This fiber process fsm output, by doing the following steps in that order:
    //  - persists current term and voter
    //  - persists unstable log entries on disk.
    //  - sends out messages
    future<> io_fiber(index_t stable_idx);

    // This fiber runs in the background and applies committed entries.
    future<> applier_fiber();

    template <typename T> future<> add_entry_internal(T command, wait_type type);
    template <typename Message> future<> send_message(server_id id, Message m);

    // Apply a dummy entry. Dummy entry is not propagated to the
    // state machine, but waiting for it to be "applied" ensures
    // all previous entries are applied as well.
    // Resolves when the entry is committed.
    // The function has to be called on the leader, throws otherwise
    // May fail because of an internal error or because the leader
    // has changed and the entry was replaced by another one,
    // submitted to the new leader.
    future<> apply_dummy_entry();

    future<> _applier_status = make_ready_future<>();
    future<> _io_status = make_ready_future<>();

    friend std::ostream& operator<<(std::ostream& os, const server_impl& s);
};

server_impl::server_impl(server_id uuid, std::unique_ptr<rpc> rpc,
        std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
        seastar::shared_ptr<failure_detector> failure_detector) :
                    _rpc(std::move(rpc)), _state_machine(std::move(state_machine)),
                    _storage(std::move(storage)), _failure_detector(failure_detector),
                    _id(uuid) {
    set_rpc_server(_rpc.get());
}

future<> server_impl::start() {
    auto [term, vote] = co_await _storage->load_term_and_vote();
    auto snapshot  = co_await _storage->load_snapshot();
    auto log_entries = co_await _storage->load_log();
    auto log = raft::log(std::move(snapshot), std::move(log_entries));
    index_t stable_idx = log.stable_idx();
    _fsm = std::make_unique<fsm>(_id, term, vote, std::move(log), *_failure_detector);
    assert(_fsm->get_current_term() != term_t(0));

    // start fiber to persist entries added to in-memory log
    _io_status = io_fiber(stable_idx);
    // start fiber to apply committed entries
    _applier_status = applier_fiber();

    _ticker.arm_periodic(100ms);
    _ticker.set_callback([this] {
        _fsm->tick();
    });

    co_return;
}

template <typename T>
future<> server_impl::add_entry_internal(T command, wait_type type) {
    logger.trace("An entry is submitted on a leader");

    // lock access to the raft log while it is been updated

    // @todo: ensure the reference to the entry is stable between
    // yields, before removing _log_lock.
    const log_entry& e = _fsm->add_entry(std::move(command));

    auto& container = type == wait_type::committed ? _awaited_commits : _awaited_applies;

    // This will track the commit/apply status of the entry
    auto [it, inserted] = container.emplace(e.idx, op_status{e.term, promise<>()});
    assert(inserted);
    return it->second.done.get_future();
}

future<> server_impl::add_entry(command command, wait_type type) {
    return add_entry_internal(std::move(command), type);
}

future<> server_impl::apply_dummy_entry() {
    return add_entry_internal(log_entry::dummy(), wait_type::applied);
}
void server_impl::append_entries(server_id from, append_request_recv append_request) {
    _fsm->step(from, std::move(append_request));
}

void server_impl::append_entries_reply(server_id from, append_reply reply) {
    _fsm->step(from, std::move(reply));
}

void server_impl::request_vote(server_id from, vote_request vote_request) {
    _fsm->step(from, std::move(vote_request));
}

void server_impl::request_vote_reply(server_id from, vote_reply vote_reply) {
    _fsm->step(from, std::move(vote_reply));
}

void server_impl::notify_waiters(std::map<index_t, op_status>& waiters,
        const std::vector<log_entry_ptr>& entries) {
    index_t commit_idx = entries.back()->idx;
    index_t first_idx = entries.front()->idx;

    while (waiters.size() != 0) {
        auto it = waiters.begin();
        if (it->first > commit_idx) {
            break;
        }
        auto [entry_idx, status] = std::move(*it);

        // if there is a waiter entry with an index smaller than first entry
        // it means that notification is out of order which is prohinited
        assert(entry_idx >= first_idx);

        waiters.erase(it);
        if (status.term == entries[entry_idx - first_idx]->term) {
            status.done.set_value();
        } else {
            // term does not match which means that between the entry was submitted
            // and committed there was a leadership change and the entry was replaced.
            status.done.set_exception(dropped_entry());
        }
    }
}

template <typename Message>
future<> server_impl::send_message(server_id id, Message m) {
    return std::visit([this, id] (auto&& m) {
        using T = std::decay_t<decltype(m)>;
        if constexpr (std::is_same_v<T, append_reply>) {
            return _rpc->send_append_entries_reply(id, m);
        } else if constexpr (std::is_same_v<T, append_request_send>) {
            return _rpc->send_append_entries(id, m);
        } else if constexpr (std::is_same_v<T, vote_request>) {
            return _rpc->send_vote_request(id, m);
        } else if constexpr (std::is_same_v<T, vote_reply>) {
            return _rpc->send_vote_reply(id, m);
        } else {
            static_assert(!sizeof(Message*), "not all message types are handled");
            return make_ready_future<>();
        }
    }, std::move(m));
}

future<> server_impl::io_fiber(index_t last_stable) {
    logger.trace("[{}] io_fiber start", _id);
    try {
        while (true) {
            auto batch = co_await _fsm->poll_output();

            if (batch.term != term_t{}) {
                // Current term and vote are always persisted
                // together. A vote may change independently of
                // term, but it's safe to update both in this
                // case.
                co_await _storage->store_term_and_vote(batch.term, batch.vote);
            }

            if (batch.log_entries.size()) {
                auto& entries = batch.log_entries;

                if (last_stable >= entries[0]->idx) {
                    co_await _storage->truncate_log(entries[0]->idx);
                }

                // Combine saving and truncating into one call?
                // will require storage to keep track of last idx
                co_await _storage->store_log_entries(entries);

                last_stable = (*entries.crbegin())->idx;
            }

            if (batch.messages.size()) {
                // after entries are persisted we can send messages
                co_await seastar::parallel_for_each(std::move(batch.messages), [this] (std::pair<server_id, rpc_message>& message) {
                    return send_message(message.first, std::move(message.second));
                });
            }

            // process committed entries
            if (batch.committed.size()) {
                notify_waiters(_awaited_commits, batch.committed);
                co_await _apply_entries.writer.write(std::move(batch.committed));
            }
        }
    } catch (seastar::broken_condition_variable&) {
        // log fiber is stopped explicitly.
    } catch (...) {
        logger.error("[{}] io fiber stopped because of the error: {}", _id, std::current_exception());
    }
    co_return;
}

future<> server_impl::apply_snapshot(server_id from, install_snapshot snp) {
    return make_ready_future<>();
}

future<> server_impl::applier_fiber() {
    logger.trace("applier_fiber start");
    try {
        while (true) {
            auto opt_batch = co_await _apply_entries.reader.read();
            if (!opt_batch) {
                // EOF
                break;
            }
            std::vector<command_cref> commands;
            commands.reserve(opt_batch->size());

            std::ranges::copy(
                    *opt_batch |
                    std::views::filter([] (log_entry_ptr& entry) { return std::holds_alternative<command>(entry->data); }) |
                    std::views::transform([] (log_entry_ptr& entry) { return std::cref(std::get<command>(entry->data)); }),
                    std::back_inserter(commands));

            co_await _state_machine->apply(std::move(commands));
            notify_waiters(_awaited_applies, *opt_batch);
        }
    } catch (...) {
        logger.error("applier fiber {} stopped because of the error: {}", _id, std::current_exception());
    }
    co_return;
}

term_t server_impl::get_current_term() const {
    return _fsm->get_current_term();
}

future<> server_impl::read_barrier() {
    if (_fsm->can_read()) {
        co_return;
    }

    co_await apply_dummy_entry();
    co_return;
}

future<> server_impl::abort() {
    logger.trace("abort() called");
    _fsm->stop();
    {
        // there is not explicit close for the pipe!
        auto tmp = std::move(_apply_entries.writer);
    }
    for (auto& ac: _awaited_commits) {
        ac.second.done.set_exception(stopped_error());
    }
    for (auto& aa: _awaited_applies) {
        aa.second.done.set_exception(stopped_error());
    }
    _awaited_commits.clear();
    _awaited_applies.clear();
    _ticker.cancel();

    return seastar::when_all_succeed(std::move(_io_status), std::move(_applier_status),
            _rpc->abort(), _state_machine->abort(), _storage->abort()).discard_result();
}

future<> server_impl::add_server(server_id id, bytes node_info, clock_type::duration timeout) {
    return make_ready_future<>();
}

// Removes a server from the cluster. If the server is not a member
// of the cluster does nothing. Can be called on a leader only
// otherwise throws not_a_leader.
// Cannot be called until previous add/remove server completes
// otherwise conf_change_in_progress exception is returned.
future<> server_impl::remove_server(server_id id, clock_type::duration timeout) {
    return make_ready_future<>();
}

void server_impl::make_me_leader() {
    _fsm->become_leader();
}

std::unique_ptr<server> create_server(server_id uuid, std::unique_ptr<rpc> rpc,
    std::unique_ptr<state_machine> state_machine, std::unique_ptr<storage> storage,
    seastar::shared_ptr<failure_detector> failure_detector) {
    assert(uuid != raft::server_id{utils::UUID(0, 0)});
    return std::make_unique<raft::server_impl>(uuid, std::move(rpc), std::move(state_machine),
        std::move(storage), failure_detector);
}

std::ostream& operator<<(std::ostream& os, const server_impl& s) {
    os << "[id: " << s._id << ", fsm (" << s._fsm << ")]\n";
    return os;
}

} // end of namespace raft
<|endoftext|>


<commit_before>#include <cstdlib>
#include <complex>
#include <iostream>
#include <mpl/mpl.hpp>

template<typename T1, typename T2>
std::ostream & operator<<(std::ostream &out, const std::pair<T1, T2> &p) {
  out << p.first << ' ' << p.second;
  return out;
}

template<typename F, typename T, std::size_t n>
class apply_n {
  F &f;
public:
  apply_n(F &f) : f(f) {
  }
  void operator()(const T &x) {
    apply_n<F, T, n-1> next(f);
    next(x);
    f(std::get<n-1>(x));
  }
};

template<typename F, typename T>
struct apply_n<F, T, 1> {
  F &f;
public:
  apply_n(F &f) : f(f) {
  }
  void operator()(const T &x) {
    f(std::get<0>(x));
  }
};

template<typename F, typename... Args>
void apply(const std::tuple<Args...> &t, F &f) {
  apply_n<F, std::tuple<Args...>, std::tuple_size<std::tuple<Args...> >::value> app(f);
  app(t);
}

class print_element {
  std::ostream &out;
public:
  print_element(std::ostream &out) : out(out) {
  }
  template<typename T>
  void operator()(const T &x) const {
    out << x << ' ';
  }
};

template<typename... Ts>
std::ostream & operator<<(std::ostream &out, const std::tuple<Ts...> &t) {
  print_element f(out);
  apply<print_element>(t, f);
  return out;
}

template<typename T>
void send(const mpl::communicator &comm, const T &x) {
  comm.send(x, 1);
}

template<typename T>
void recv(const mpl::communicator &comm) {
  T x;
  comm.recv(x, 0);
  std::cout << "x = " << std::boolalpha << x << '\n';
}

int main() {
  const mpl::communicator &comm_world=mpl::environment::comm_world();
  if (comm_world.size()<2)
    comm_world.abort(EXIT_FAILURE);
  if (comm_world.rank()==0) {
    char t1='A';                               send(comm_world, t1);
    signed char t2='B';                        send(comm_world, t2);
    unsigned char t3='C';                      send(comm_world, t3);
    signed short int t4=-1;                    send(comm_world, t4);
    unsigned short int t5=1;                   send(comm_world, t5);
    signed int t6=-10;                         send(comm_world, t6);
    unsigned int t7=10;                        send(comm_world, t7);
    signed long int t8=-100;                   send(comm_world, t8);
    unsigned long int t9=100;                  send(comm_world, t9);
    signed long long int t10=-1000;            send(comm_world, t10);
    unsigned long long int t11=1000;           send(comm_world, t11);
    bool t12=true;                             send(comm_world, t12);
    float t13=1.2345;                          send(comm_world, t13);
    double t14=2.3456;                         send(comm_world, t14);
    long double t15=3.4567;                    send(comm_world, t15);
    std::complex<float> t16(1.2, -1.2);        send(comm_world, t16);
    std::complex<double> t17(2.3, -2.3);       send(comm_world, t17);
    std::complex<long double> t18(3.4, -3.4);  send(comm_world, t18);
    std::pair<int, double> t19(-2, 0.1234);    send(comm_world, t19);
    std::tuple<int, std::complex<double> > t20(-2, 0.1234);   send(comm_world, t20);
  }
  if (comm_world.rank()==1) {
    recv<char>(comm_world);
    recv<signed char>(comm_world);
    recv<unsigned char>(comm_world);
    recv<signed short int>(comm_world);
    recv<unsigned short int>(comm_world);
    recv<signed int>(comm_world);
    recv<unsigned int>(comm_world);
    recv<signed long int>(comm_world);
    recv<unsigned long int>(comm_world);
    recv<signed long long int>(comm_world);
    recv<unsigned long long int>(comm_world);
    recv<bool>(comm_world);
    recv<float>(comm_world);
    recv<double>(comm_world);
    recv<long double>(comm_world);
    recv<std::complex<float> >(comm_world);
    recv<std::complex<double> >(comm_world);
    recv<std::complex<long double> >(comm_world);
    recv<std::pair<int, double> >(comm_world);
    recv<std::tuple<int, std::complex<double> > >(comm_world);
  }
  return EXIT_SUCCESS;
}
<commit_msg>comments for standard_types<commit_after>#include <cstdlib>
#include <complex>
#include <iostream>
#include <tuple>
#include <utility>
#include <mpl/mpl.hpp>

// print elements of a pair
template<typename T1, typename T2>
std::ostream & operator<<(std::ostream &out, const std::pair<T1, T2> &p) {
  return out << '(' << p.first << ',' << p.second << ')';
}

// print elements of a tuple
template<typename ch, typename tr, typename tuple, std::size_t... is>
void print_tuple_impl(std::basic_ostream<ch, tr>& os,
                      const tuple & t,
                      std::index_sequence<is...> is_) {
  auto print_element = [&] (auto i, auto a) -> void {
    os << a << (i+1<is_.size() ? "," : "");
  };
  std::initializer_list<int> { (print_element(is, std::get<is>(t)), 0)... };
}

template<typename ch, typename tr, typename... args>
decltype(auto) operator<<(std::basic_ostream<ch, tr>& os,
                          const std::tuple<args...>& t) {
  os << '(';
  print_tuple_impl(os, t, std::index_sequence_for<args...>{});
  return os << ')';
}

// send some item of a standard type
template<typename T>
void send(const mpl::communicator &comm, const T &x) {
  comm.send(x, 1);
}

// receive some item of a standard type
template<typename T>
void recv(const mpl::communicator &comm) {
  T x;
  comm.recv(x, 0);
  std::cout << "x = " << std::boolalpha << x << '\n';
}

int main() {
  const mpl::communicator &comm_world=mpl::environment::comm_world();
  // run the program with two or more processes
  if (comm_world.size()<2)
    comm_world.abort(EXIT_FAILURE);
  // process 0 sends
  if (comm_world.rank()==0) {
    char t1='A';                               send(comm_world, t1);
    signed char t2='B';                        send(comm_world, t2);
    unsigned char t3='C';                      send(comm_world, t3);
    signed short int t4=-1;                    send(comm_world, t4);
    unsigned short int t5=1;                   send(comm_world, t5);
    signed int t6=-10;                         send(comm_world, t6);
    unsigned int t7=10;                        send(comm_world, t7);
    signed long int t8=-100;                   send(comm_world, t8);
    unsigned long int t9=100;                  send(comm_world, t9);
    signed long long int t10=-1000;            send(comm_world, t10);
    unsigned long long int t11=1000;           send(comm_world, t11);
    bool t12=true;                             send(comm_world, t12);
    float t13=1.2345;                          send(comm_world, t13);
    double t14=2.3456;                         send(comm_world, t14);
    long double t15=3.4567;                    send(comm_world, t15);
    std::complex<float> t16(1.2, -1.2);        send(comm_world, t16);
    std::complex<double> t17(2.3, -2.3);       send(comm_world, t17);
    std::complex<long double> t18(3.4, -3.4);  send(comm_world, t18);
    std::pair<int, double> t19(-2, 0.1234);    send(comm_world, t19);
    std::tuple<int, std::complex<double> > t20(-2, 0.1234);   send(comm_world, t20);
  }
  // process 1 recieves
  if (comm_world.rank()==1) {
    recv<char>(comm_world);
    recv<signed char>(comm_world);
    recv<unsigned char>(comm_world);
    recv<signed short int>(comm_world);
    recv<unsigned short int>(comm_world);
    recv<signed int>(comm_world);
    recv<unsigned int>(comm_world);
    recv<signed long int>(comm_world);
    recv<unsigned long int>(comm_world);
    recv<signed long long int>(comm_world);
    recv<unsigned long long int>(comm_world);
    recv<bool>(comm_world);
    recv<float>(comm_world);
    recv<double>(comm_world);
    recv<long double>(comm_world);
    recv<std::complex<float>>(comm_world);
    recv<std::complex<double>>(comm_world);
    recv<std::complex<long double>>(comm_world);
    recv<std::pair<int, double>>(comm_world);
    recv<std::tuple<int, std::complex<double>> >(comm_world);
  }
  return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>//=============================================================================================================
/**
* @file     main.cpp
* @author   Jana Kiesel <jana.kiesel@tu-ilmenau.de>
*           Christoph Dinh <chdinh@nmr.mgh.harvard.edu>;
*           Matti Hamalainen <msh@nmr.mgh.harvard.edu>
* @version  1.0
* @date     November, 2015
* @section  LICENSE
*
* Copyright (C) 2015, Jana Kiesel, Christoph Dinh and Matti Hamalainen. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that
* the following conditions are met:
*     * Redistributions of source code must retain the above copyright notice, this list of conditions and the
*       following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
*       the following disclaimer in the documentation and/or other materials provided with the distribution.
*     * Neither the name of MNE-CPP authors nor the names of its contributors may be used
*       to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*
* @brief    Example to test the warp class
*
*/

//*************************************************************************************************************
//=============================================================================================================
// INCLUDES
//=============================================================================================================

//#include <fstream>
#include <iostream>
#include <cstdlib>
#include <time.h>

#include <utils/warp.h>
#include <fiff/fiff.h>
#include <mne/mne_bem.h>

#include <QFile>

//*************************************************************************************************************
//=============================================================================================================
// QT INCLUDES
//=============================================================================================================

#include <QtCore/QCoreApplication>

//*************************************************************************************************************
//=============================================================================================================
// USED NAMESPACES
//=============================================================================================================

using namespace UTILSLIB;
using namespace FIFFLIB;

using namespace std;

//*************************************************************************************************************
//=============================================================================================================
// MAIN
//=============================================================================================================

//=============================================================================================================
/**
* The function main marks the entry point of the program.
* By default, main has the storage class extern.
*
* @param [in] argc (argument count) is an integer that indicates how many arguments were entered on the command line when the program was started.
* @param [in] argv (argument vector) is an array of pointers to arrays of character objects. The array objects are null-terminated strings, representing the arguments that were entered on the command line when the program was started.
* @return the value that was set to exit() (which is 0 if exit() is called via quit()).
*/

#define MAXBUFSIZE  ((int) 1e6)

int main(int argc, char *argv[])
{
    QCoreApplication a(argc, argv);

//    Matrix<double,6,3> sLm;
//    sLm <<  1,1,1,
//            2,1,1,
//            3,1,1,
//            1,2,1,
//            2,2,1,
//            3,2,1;
//    std::cout << "Here is the matrix sLm:" << std::endl << sLm << std::endl;

//    Matrix<double,6,3> dLm;
//    dLm <<  1,1,2,
//            2,1,3,
//            3,1,2,
//            1,2,2,
//            2,2,3,
//            3,2,2;

//    Matrix<double,16,3> sVert;
//    sVert <<1.0,0.5,1,
//            1.5,0.5,1,
//            2.0,0.5,1,
//            2.3,0.5,1,
//            2.7,0.5,1,
//            3.0,0.5,1,
//            3.5,0.5,1,
//            4.0,0.5,1,
//            1.0,1,1,
//            1.5,1,1,
//            2.0,1,1,
//            2.3,1,1,
//            2.7,1,1,
//            3.0,1,1,
//            3.5,1,1,
//            4.0,1,1;;

    //
    // Read Electrode Positions from fiff
    //
    QFile t_fileRaw("./MNE-sample-data/MEG/sample/sample_audvis_raw.fif");
    FiffRawData raw(t_fileRaw);
    QList<FiffChInfo> ChannelInfo= raw.info.chs;
    QList<QVector<double>> tempElecInfo;
    QList<QString> ElecChanName;
    for (int i=0; i<ChannelInfo.length();i++)
    {
        if (ChannelInfo[i].kind==2)
        {
            QVector<double> temp;
            ElecChanName.append(ChannelInfo[i].ch_name);
            temp.append(ChannelInfo[i].loc(0,0));
            temp.append(ChannelInfo[i].loc(1,0));
            temp.append(ChannelInfo[i].loc(2,0));
            tempElecInfo.append(temp);
        }
    }
    MatrixXd ElecPos(tempElecInfo.length(),3);
    for (int i=0; i<tempElecInfo.length();i++)
    {
        ElecPos(i,0)=tempElecInfo[i][0];
        ElecPos(i,1)=tempElecInfo[i][1];
        ElecPos(i,2)=tempElecInfo[i][2];
    }

//    std::cout << "Here is the matrix ElecPos:" << std::endl << ElecPos << std::endl;

    //
    // read electrode positions from Database
    //
//    QString electrodeFileName= "./MNE-sample-data/MriDatabase/FirstYear/Sources/Electrodes/AVG3-0Months_10-5_Electrodes.txt";
//    Matrix<double, Dynamic ,3> ElecPos = test.readsLm(electrodeFileName);
//    std::cout << "Here is the matrix ElecPos:" << std::endl << ElecPos << std::endl;

    //
    //Read BEM from fiff
    //
    QFile t_fileBem("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem.fif");
    MNELIB::MNEBem t_Bem (t_fileBem) ;

    //
    // prepare warp
    //
    MatrixXd sLm=ElecPos;
    MatrixXd random(ElecPos.rows(),3);
    srand(time(NULL));
    for (int i=0;i<ElecPos.rows();i++)
    {

        for (int j=0; j<3;j++)
        {
            random(i,j)=rand();
            random(i,j)/=RAND_MAX;
            random(i,j)*=0.06;
            random(i,j)-=0.03;
            //            random(i,j)=0.005*(rand()/RAND_MAX);
        }
    }
//    std::cout << "Here are the first row of the matrix random:" << std::endl << random.topRows(9) << std::endl;
    MatrixXd dLm=sLm+random;
    MNELIB::MNEBemSurface skin=t_Bem[0];
    MatrixXd sVert=skin.rr.cast<double>();

    std::cout << "Here are the first row of the matrix skin.rr bevor warp:" << std::endl << skin.rr.topRows(9) << std::endl;

    //
    // calculate Warp
    //
    Warp test;
    MatrixXd wVert(sVert.rows(),3);
    wVert = test.calculate(sLm, dLm, sVert);

    //
    // WRITE NEW VERTICES BACK TO BEM
    //
    skin.rr=wVert.cast<float>();
    skin.addVertexNormals();

    std::cout << "Here are the first row of the matrix skin.rr after warp:" << std::endl << skin.rr.topRows(9) << std::endl;
//    std::cout << "Here is the first row of the final matrix skin.tris:" << std::endl << skin.tris.topRows(9) << std::endl;
//    std::cout << "Here is the last row of the final matrix skin.tris:" << std::endl << skin.tris.bottomRows(1) << std::endl;

    MNELIB::MNEBem t_BemWarpedA;
    t_BemWarpedA<<skin;
    QFile t_fileBemWarped("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem-warped.fif");
    t_BemWarpedA.write(t_fileBemWarped);
    t_fileBemWarped.close();

    MNELIB::MNEBem t_BemWarpedB (t_fileBemWarped) ;
    MNELIB::MNEBemSurface skinWarped=t_BemWarpedB[0];
    return a.exec();
}
<commit_msg>[LIB-49] read electrode Data and transformation from fif files<commit_after>//=============================================================================================================
/**
* @file     main.cpp
* @author   Jana Kiesel <jana.kiesel@tu-ilmenau.de>
*           Christoph Dinh <chdinh@nmr.mgh.harvard.edu>;
*           Matti Hamalainen <msh@nmr.mgh.harvard.edu>
* @version  1.0
* @date     November, 2015
* @section  LICENSE
*
* Copyright (C) 2015, Jana Kiesel, Christoph Dinh and Matti Hamalainen. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that
* the following conditions are met:
*     * Redistributions of source code must retain the above copyright notice, this list of conditions and the
*       following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
*       the following disclaimer in the documentation and/or other materials provided with the distribution.
*     * Neither the name of MNE-CPP authors nor the names of its contributors may be used
*       to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*
* @brief    Example to test the warp class
*
*/

//*************************************************************************************************************
//=============================================================================================================
// INCLUDES
//=============================================================================================================

//#include <fstream>
#include <iostream>
#include <cstdlib>
#include <time.h>

#include <utils/warp.h>
#include <fiff/fiff.h>
#include <mne/mne_bem.h>

#include <QFile>

//*************************************************************************************************************
//=============================================================================================================
// QT INCLUDES
//=============================================================================================================

#include <QtCore/QCoreApplication>

//*************************************************************************************************************
//=============================================================================================================
// USED NAMESPACES
//=============================================================================================================

using namespace UTILSLIB;
using namespace FIFFLIB;

using namespace std;

//*************************************************************************************************************
//=============================================================================================================
// MAIN
//=============================================================================================================

//=============================================================================================================
/**
* The function main marks the entry point of the program.
* By default, main has the storage class extern.
*
* @param [in] argc (argument count) is an integer that indicates how many arguments were entered on the command line when the program was started.
* @param [in] argv (argument vector) is an array of pointers to arrays of character objects. The array objects are null-terminated strings, representing the arguments that were entered on the command line when the program was started.
* @return the value that was set to exit() (which is 0 if exit() is called via quit()).
*/

#define MAXBUFSIZE  ((int) 1e6)

int main(int argc, char *argv[])
{
    QCoreApplication a(argc, argv);

//    Matrix<double,6,3> sLm;
//    sLm <<  1,1,1,
//            2,1,1,
//            3,1,1,
//            1,2,1,
//            2,2,1,
//            3,2,1;
//    std::cout << "Here is the matrix sLm:" << std::endl << sLm << std::endl;

//    Matrix<double,6,3> dLm;
//    dLm <<  1,1,2,
//            2,1,3,
//            3,1,2,
//            1,2,2,
//            2,2,3,
//            3,2,2;

//    Matrix<double,16,3> sVert;
//    sVert <<1.0,0.5,1,
//            1.5,0.5,1,
//            2.0,0.5,1,
//            2.3,0.5,1,
//            2.7,0.5,1,
//            3.0,0.5,1,
//            3.5,0.5,1,
//            4.0,0.5,1,
//            1.0,1,1,
//            1.5,1,1,
//            2.0,1,1,
//            2.3,1,1,
//            2.7,1,1,
//            3.0,1,1,
//            3.5,1,1,
//            4.0,1,1;;

    //
    // Read Electrode Positions from fiff
    //
//    QFile t_fileRaw("./MNE-sample-data/MEG/sample/sample_audvis_raw.fif");
//    FiffRawData raw(t_fileRaw);
//    QList<FiffChInfo> ChannelInfo= raw.info.chs;

    QFile t_fileElec("./MNE-sample-data/warping/AVG4-0Years_GSN128.fif");
    FiffRawData elec(t_fileElec);
    QList<FiffChInfo> ChannelInfo= elec.info.chs;
    QList<QVector<double>> tempElecInfo;
    QList<QString> ElecChanName;
    for (int i=0; i<ChannelInfo.length();i++)
    {
        if (ChannelInfo[i].kind==2)
        {
            QVector<double> temp;
            ElecChanName.append(ChannelInfo[i].ch_name);
            temp.append(ChannelInfo[i].loc(0,0));
            temp.append(ChannelInfo[i].loc(1,0));
            temp.append(ChannelInfo[i].loc(2,0));
            tempElecInfo.append(temp);
        }
    }
    MatrixXd ElecPos(tempElecInfo.length(),3);
    for (int i=0; i<tempElecInfo.length();i++)
    {
        ElecPos(i,0)=tempElecInfo[i][0];
        ElecPos(i,1)=tempElecInfo[i][1];
        ElecPos(i,2)=tempElecInfo[i][2];
    }

//    std::cout << "Here is the matrix ElecPos:" << std::endl << ElecPos << std::endl;

    //
    // read electrode positions from Database
    //
//    QString electrodeFileName= "./MNE-sample-data/MriDatabase/FirstYear/Sources/Electrodes/AVG3-0Months_10-5_Electrodes.txt";
//    Matrix<double, Dynamic ,3> ElecPos = test.readsLm(electrodeFileName);
//    std::cout << "Here is the matrix ElecPos:" << std::endl << ElecPos << std::endl;

    //
    //Read BEM from fiff
    //
    QFile t_fileBem("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem.fif");
    MNELIB::MNEBem t_Bem (t_fileBem) ;

    //
    // prepare warp
    //
    MatrixXd sLm=ElecPos;
    MatrixXd random(ElecPos.rows(),3);
    srand(time(NULL));
    for (int i=0;i<ElecPos.rows();i++)
    {

        for (int j=0; j<3;j++)
        {
            random(i,j)=rand();
            random(i,j)/=RAND_MAX;
            random(i,j)*=0.06;
            random(i,j)-=0.03;
            //            random(i,j)=0.005*(rand()/RAND_MAX);
        }
    }
//    std::cout << "Here are the first row of the matrix random:" << std::endl << random.topRows(9) << std::endl;
    MatrixXd dLm=sLm+random;
    MNELIB::MNEBemSurface skin=t_Bem[0];
    MatrixXd sVert=skin.rr.cast<double>();

    std::cout << "Here are the first row of the matrix skin.rr bevor warp:" << std::endl << skin.rr.topRows(9) << std::endl;

    //
    // calculate Warp
    //
    Warp test;
    MatrixXd wVert(sVert.rows(),3);
    wVert = test.calculate(sLm, dLm, sVert);

    //
    // WRITE NEW VERTICES BACK TO BEM
    //
    skin.rr=wVert.cast<float>();
    skin.addVertexNormals();

    std::cout << "Here are the first row of the matrix skin.rr after warp:" << std::endl << skin.rr.topRows(9) << std::endl;
//    std::cout << "Here is the first row of the final matrix skin.tris:" << std::endl << skin.tris.topRows(9) << std::endl;
//    std::cout << "Here is the last row of the final matrix skin.tris:" << std::endl << skin.tris.bottomRows(1) << std::endl;

    MNELIB::MNEBem t_BemWarpedA;
    t_BemWarpedA<<skin;
    QFile t_fileBemWarped("./MNE-sample-data/subjects/sample/bem/sample-5120-5120-5120-bem-warped.fif");
    t_BemWarpedA.write(t_fileBemWarped);
    t_fileBemWarped.close();

    MNELIB::MNEBem t_BemWarpedB (t_fileBemWarped) ;
    MNELIB::MNEBemSurface skinWarped=t_BemWarpedB[0];
    return a.exec();
}
<|endoftext|>


<commit_before>#pragma once
//=====================================================================//
/*! @file
    @brief  CSV クラス
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2018 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/RX/blob/master/LICENSE
*/
//=====================================================================//
#include <vector>
#include <string>
#include <boost/format.hpp>

#include "utils/string_utils.hpp"

namespace app {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  CSV 出力クラス
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	class csv {

		typedef std::vector<utils::strings> CELL;

		CELL	cell_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
		*/
		//-----------------------------------------------------------------//
		csv() { }

		//-----------------------------------------------------------------//
		/*!
			@brief  作成
			@param[in]	columns	列
			@param[in]	rows	行
		*/
		//-----------------------------------------------------------------//
		void create(uint32_t columns, uint32_t rows)
		{
			cell_.clear();
			for(uint32_t r = 0; r < rows; ++r) {
				utils::strings line;
				for(uint32_t c = 0; c < columns; ++c) {
					line.push_back("");
				}
				cell_.push_back(line);
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  クリア
		*/
		//-----------------------------------------------------------------//
		void clear()
		{
			cell_.clear();
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  設定
			@param[in]	columns	列
			@param[in]	rows	行
			@param[in]	str		文字列
			@return 設定不可なら「false」
		*/
		//-----------------------------------------------------------------//
		bool set(uint32_t columns, uint32_t rows, const std::string& str)
		{
			if(cell_.size() <= rows) return false;

			utils::strings& ss = cell_[rows];
			if(ss.size() <= columns) return false;
			ss[columns] = str;
			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  セーブ
			@param[in]	path	セーブ・パス
			@return 成功なら「true」
		*/
		//-----------------------------------------------------------------//
		bool save(const std::string& path)
		{
			utils::file_io fio;
			if(!fio.open(path, "wb")) {
				return false;
			}

			for(uint32_t r = 0; r < cell_.size(); ++r) {
				utils::strings& line = cell_[r];
				for(uint32_t c = 0; c < line.size(); ++c) {
					const std::string& s = line[c];
					fio.put(s);
					if(c < (line.size() - 1)) fio.put(",");
					else if(c == (line.size() - 1)) fio.put("\n");
				}
			}
			fio.close();

			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  ロード
			@param[in]	path	ロード・パス
			@return 成功なら「true」
		*/
		//-----------------------------------------------------------------//
		bool load(const std::string& path)
		{
			utils::file_io fio;
			if(!fio.open(path, "rb")) {
				return false;
			}

			uint32_t lno = 0;
			while(fio.eof()) {
				auto line = fio.get_line();
				if(line.empty()) continue;

				auto ss = utils::split_text(line, ",");
				uint32_t n = 0;
				for(auto s : ss) {
					set(n, lno, s);
					++n;
				}
				++lno;
			}
			fio.close();

			return true;
		}
	};
}
<commit_msg>update: analize min/max/ave<commit_after>#pragma once
//=====================================================================//
/*! @file
    @brief  CSV クラス
    @author 平松邦仁 (hira@rvf-rc45.net)
	@copyright	Copyright (C) 2018 Kunihito Hiramatsu @n
				Released under the MIT license @n
				https://github.com/hirakuni45/RX/blob/master/LICENSE
*/
//=====================================================================//
#include <vector>
#include <string>
#include <boost/format.hpp>

#include "utils/string_utils.hpp"

namespace app {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  CSV 出力クラス
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	class csv {

		typedef std::vector<utils::strings> CELL;

		CELL	cell_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
		*/
		//-----------------------------------------------------------------//
		csv() { }

		//-----------------------------------------------------------------//
		/*!
			@brief  作成
			@param[in]	columns	列
			@param[in]	rows	行
		*/
		//-----------------------------------------------------------------//
		void create(uint32_t columns, uint32_t rows)
		{
			cell_.clear();
			for(uint32_t r = 0; r < rows; ++r) {
				utils::strings line;
				for(uint32_t c = 0; c < columns; ++c) {
					line.push_back("");
				}
				cell_.push_back(line);
			}
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  クリア
		*/
		//-----------------------------------------------------------------//
		void clear()
		{
			cell_.clear();
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  アイテムの設定
			@param[in]	columns	列
			@param[in]	rows	行
			@param[in]	str		文字列
			@return 設定不可なら「false」
		*/
		//-----------------------------------------------------------------//
		bool set(uint32_t columns, uint32_t rows, const std::string& str)
		{
			if(cell_.size() <= rows) return false;

			utils::strings& ss = cell_[rows];
			if(ss.size() <= columns) return false;
			ss[columns] = str;
			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  アイテムの取得
			@param[in]	columns	列
			@param[in]	rows	行
			@return 文字列
		*/
		//-----------------------------------------------------------------//
		std::string get(uint32_t columns, uint32_t rows) const
		{
			if(cell_.size() <= rows) return "";

			const utils::strings& ss = cell_[rows];
			if(ss.size() <= columns) return "";
			return ss[columns];
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  セーブ
			@param[in]	path	セーブ・パス
			@return 成功なら「true」
		*/
		//-----------------------------------------------------------------//
		bool save(const std::string& path)
		{
			utils::file_io fio;
			if(!fio.open(path, "wb")) {
				return false;
			}

			for(uint32_t r = 0; r < cell_.size(); ++r) {
				utils::strings& line = cell_[r];
				for(uint32_t c = 0; c < line.size(); ++c) {
					const std::string& s = line[c];
					fio.put(s);
					if(c < (line.size() - 1)) fio.put(",");
					else if(c == (line.size() - 1)) fio.put("\n");
				}
			}
			fio.close();

			return true;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  ロード
			@param[in]	path	ロード・パス
			@return 成功なら「true」
		*/
		//-----------------------------------------------------------------//
		bool load(const std::string& path)
		{
			utils::file_io fio;
			if(!fio.open(path, "rb")) {
				return false;
			}

			uint32_t lno = 0;
			while(!fio.eof()) {
				auto line = fio.get_line();
				if(line.empty()) continue;
				auto ss = utils::split_text(line, ",");
				uint32_t n = 0;
				for(auto s : ss) {
					set(n, lno, s);
					++n;
				}
				++lno;
			}
			fio.close();

			return true;
		}
	};
}
<|endoftext|>


<commit_before>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef CONCURRENCY_SIGNAL_HPP_
#define CONCURRENCY_SIGNAL_HPP_

#include "concurrency/pubsub.hpp"
#include "utils.hpp"

/* A `signal_t` is a boolean variable, combined with a way to be notified if
that boolean variable becomes true. Typically you will construct a concrete
subclass of `signal_t`, then pass a pointer to the underlying `signal_t` to
another object which will read from or listen to it.

To check if a `signal_t` has already been pulsed, call `is_pulsed()`
on it. To be notified when it gets pulsed, construct a
`signal_t::subscription_t` subclass and reset(...) it with the signal to
watch. The callback will be called when the signal is pulsed. If the
signal is already pulsed at the time you construct the
`signal_t::subscription_t`, then the callback will be called
immediately.

`signal_t` is not thread-safe.

Although you may be tempted to, please do not add a method that "unpulses" a
`signal_t`. Part of the definition of a `signal_t` is that it does not return to
the unpulsed state after being pulsed, and some things may depend on that
property. If you want something like that, maybe you should look at something
other than `signal_t`. */

class signal_t : public home_thread_mixin_t {
public:
    /* True if somebody has called `pulse()`. */
    bool is_pulsed() const {
        assert_thread();
        return pulsed;
    }

    /* Wrapper around a `publisher_t<signal_t::subscription_t>::subscription_t`
    */

    class subscription_t : public home_thread_mixin_t {
    public:
        subscription_t() : subs(this) { }
        virtual ~subscription_t() { }

        void reset(signal_t *s = NULL) {
            if (s) {
                mutex_assertion_t::acq_t acq(&s->lock);
                if (s->is_pulsed()) {
                    subs.subscriber->run();
                } else {
                    subs.reset(s->publisher_controller.get_publisher());
                }
            } else {
                subs.reset(NULL);
            }
        }

        virtual void run() = 0;
    private:
        publisher_t<subscription_t *>::subscription_t subs;
        DISABLE_COPYING(subscription_t);
    };

    /* The coro that calls `wait_lazily_ordered()` will be pushed onto the event
    queue when the signal is pulsed, but will not wake up immediately. Unless
    you really need the ordering guarantee, you should call
    `wait_lazily_unordered()`. */
    void wait_lazily_ordered() const;

    /* The coro that calls `wait_lazily_unordered()` will be notified soon after
    the signal has been pulsed, but not immediately. */
    void wait_lazily_unordered() const;

    /* `wait()` is a deprecated synonym for `wait_lazily_ordered()`. */
    void wait() const {
        wait_lazily_ordered();
    }

    void rethread(threadnum_t new_thread) {
        real_home_thread = new_thread;
        publisher_controller.rethread(new_thread);
        lock.rethread(new_thread);
    }

protected:
    signal_t() : pulsed(false) { }
    ~signal_t() { }

    void pulse() THROWS_NOTHING {
        assert_thread();
        mutex_assertion_t::acq_t acq(&lock);
        rassert(!is_pulsed());
        pulsed = true;
        publisher_controller.publish(&signal_t::call);
    }

private:
    static void call(subscription_t *subscription) THROWS_NOTHING {
        subscription->run();
    }

    bool pulsed;
    publisher_controller_t<subscription_t *> publisher_controller;
    mutex_assertion_t lock;
    DISABLE_COPYING(signal_t);
};

#endif /* CONCURRENCY_SIGNAL_HPP_ */
<commit_msg>Removed unused home_thread_mixin_t base class of signal_t::subscription_t.<commit_after>// Copyright 2010-2012 RethinkDB, all rights reserved.
#ifndef CONCURRENCY_SIGNAL_HPP_
#define CONCURRENCY_SIGNAL_HPP_

#include "concurrency/pubsub.hpp"
#include "utils.hpp"

/* A `signal_t` is a boolean variable, combined with a way to be notified if
that boolean variable becomes true. Typically you will construct a concrete
subclass of `signal_t`, then pass a pointer to the underlying `signal_t` to
another object which will read from or listen to it.

To check if a `signal_t` has already been pulsed, call `is_pulsed()`
on it. To be notified when it gets pulsed, construct a
`signal_t::subscription_t` subclass and reset(...) it with the signal to
watch. The callback will be called when the signal is pulsed. If the
signal is already pulsed at the time you construct the
`signal_t::subscription_t`, then the callback will be called
immediately.

`signal_t` is not thread-safe.

Although you may be tempted to, please do not add a method that "unpulses" a
`signal_t`. Part of the definition of a `signal_t` is that it does not return to
the unpulsed state after being pulsed, and some things may depend on that
property. If you want something like that, maybe you should look at something
other than `signal_t`. */

class signal_t : public home_thread_mixin_t {
public:
    /* True if somebody has called `pulse()`. */
    bool is_pulsed() const {
        assert_thread();
        return pulsed;
    }

    /* Wrapper around a `publisher_t<signal_t::subscription_t>::subscription_t`
    */

    class subscription_t {
    public:
        subscription_t() : subs(this) { }
        virtual ~subscription_t() { }

        void reset(signal_t *s = NULL) {
            if (s) {
                mutex_assertion_t::acq_t acq(&s->lock);
                if (s->is_pulsed()) {
                    subs.subscriber->run();
                } else {
                    subs.reset(s->publisher_controller.get_publisher());
                }
            } else {
                subs.reset(NULL);
            }
        }

        virtual void run() = 0;
    private:
        publisher_t<subscription_t *>::subscription_t subs;
        DISABLE_COPYING(subscription_t);
    };

    /* The coro that calls `wait_lazily_ordered()` will be pushed onto the event
    queue when the signal is pulsed, but will not wake up immediately. Unless
    you really need the ordering guarantee, you should call
    `wait_lazily_unordered()`. */
    void wait_lazily_ordered() const;

    /* The coro that calls `wait_lazily_unordered()` will be notified soon after
    the signal has been pulsed, but not immediately. */
    void wait_lazily_unordered() const;

    /* `wait()` is a deprecated synonym for `wait_lazily_ordered()`. */
    void wait() const {
        wait_lazily_ordered();
    }

    void rethread(threadnum_t new_thread) {
        real_home_thread = new_thread;
        publisher_controller.rethread(new_thread);
        lock.rethread(new_thread);
    }

protected:
    signal_t() : pulsed(false) { }
    ~signal_t() { }

    void pulse() THROWS_NOTHING {
        assert_thread();
        mutex_assertion_t::acq_t acq(&lock);
        rassert(!is_pulsed());
        pulsed = true;
        publisher_controller.publish(&signal_t::call);
    }

private:
    static void call(subscription_t *subscription) THROWS_NOTHING {
        subscription->run();
    }

    bool pulsed;
    publisher_controller_t<subscription_t *> publisher_controller;
    mutex_assertion_t lock;
    DISABLE_COPYING(signal_t);
};

#endif /* CONCURRENCY_SIGNAL_HPP_ */
<|endoftext|>