<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Modified by ScyllaDB
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "log.hh"
#include "streaming/stream_detail.hh"
#include "streaming/stream_transfer_task.hh"
#include "streaming/stream_session.hh"
#include "streaming/stream_manager.hh"
#include "streaming/stream_reason.hh"
#include "streaming/stream_mutation_fragments_cmd.hh"
#include "mutation_reader.hh"
#include "flat_mutation_reader.hh"
#include "frozen_mutation.hh"
#include "mutation.hh"
#include "message/messaging_service.hh"
#include "range.hh"
#include "dht/i_partitioner.hh"
#include "dht/sharder.hh"
#include "service/priority_manager.hh"
#include <boost/range/irange.hpp>
#include <boost/icl/interval.hpp>
#include <boost/icl/interval_set.hpp>
#include "sstables/sstables.hh"
#include "database.hh"
#include "gms/feature_service.hh"

namespace streaming {

extern logging::logger sslog;

stream_transfer_task::stream_transfer_task(shared_ptr<stream_session> session, UUID cf_id, dht::token_range_vector ranges, long total_size)
    : stream_task(session, cf_id)
    , _ranges(std::move(ranges))
    , _total_size(total_size) {
}

stream_transfer_task::~stream_transfer_task() = default;

struct send_info {
    database& db;
    utils::UUID plan_id;
    utils::UUID cf_id;
    netw::messaging_service::msg_addr id;
    uint32_t dst_cpu_id;
    stream_reason reason;
    size_t mutations_nr{0};
    semaphore mutations_done{0};
    bool error_logged = false;
    column_family& cf;
    dht::token_range_vector ranges;
    dht::partition_range_vector prs;
    flat_mutation_reader reader;
    send_info(database& db_, utils::UUID plan_id_, utils::UUID cf_id_,
              dht::token_range_vector ranges_, netw::messaging_service::msg_addr id_,
              uint32_t dst_cpu_id_, stream_reason reason_)
        : db(db_)
        , plan_id(plan_id_)
        , cf_id(cf_id_)
        , id(id_)
        , dst_cpu_id(dst_cpu_id_)
        , reason(reason_)
        , cf(db.find_column_family(cf_id))
        , ranges(std::move(ranges_))
        , prs(dht::to_partition_ranges(ranges))
        , reader(cf.make_streaming_reader(cf.schema(), prs)) {
    }
    future<bool> has_relevant_range_on_this_shard() {
        return do_with(false, ranges.begin(), [this] (bool& found_relevant_range, dht::token_range_vector::iterator& ranges_it) {
            auto stop_cond = [this, &found_relevant_range, &ranges_it] { return ranges_it == ranges.end() || found_relevant_range; };
            return do_until(std::move(stop_cond), [this, &found_relevant_range, &ranges_it] {
                dht::token_range range = *ranges_it++;
                if (!found_relevant_range) {
                    auto sharder = dht::selective_token_range_sharder(cf.schema()->get_sharder(), std::move(range), this_shard_id());
                    auto range_shard = sharder.next();
                    if (range_shard) {
                        found_relevant_range = true;
                    }
                }
                return make_ready_future<>();
            }).then([&found_relevant_range] {
                return found_relevant_range;
            });
        });
    }
    future<size_t> estimate_partitions() {
        return do_with(cf.get_sstables(), size_t(0), [this] (auto& sstables, size_t& partition_count) {
            return do_for_each(*sstables, [this, &partition_count] (auto& sst) {
                return do_for_each(ranges, [this, &sst, &partition_count] (auto& range) {
                    partition_count += sst->estimated_keys_for_range(range);
                });
            }).then([&partition_count] {
                return partition_count;
            });
        });
    }
};

future<> send_mutation_fragments(lw_shared_ptr<send_info> si) {
 return si->reader.peek(db::no_timeout).then([si] (mutation_fragment* mfp) {
  if (!mfp) {
    // The reader contains no data
    sslog.info("[Stream #{}] Skip sending ks={}, cf={}, reader contains no data, with new rpc streaming",
        si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name());
    return make_ready_future<>();
  }
  return si->estimate_partitions().then([si] (size_t estimated_partitions) {
    sslog.info("[Stream #{}] Start sending ks={}, cf={}, estimated_partitions={}, with new rpc streaming", si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name(), estimated_partitions);
    return netw::get_local_messaging_service().make_sink_and_source_for_stream_mutation_fragments(si->reader.schema()->version(), si->plan_id, si->cf_id, estimated_partitions, si->reason, si->id).then_unpack([si] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd> sink, rpc::source<int32_t> source) mutable {
        auto got_error_from_peer = make_lw_shared<bool>(false);

        auto source_op = [source, got_error_from_peer, si] () mutable -> future<> {
            return repeat([source, got_error_from_peer, si] () mutable {
                return source().then([source, got_error_from_peer, si] (std::optional<std::tuple<int32_t>> status_opt) mutable {
                    if (status_opt) {
                        auto status = std::get<0>(*status_opt);
                        *got_error_from_peer = status == -1;
                        sslog.debug("Got status code from peer={}, plan_id={}, cf_id={}, status={}", si->id.addr, si->plan_id, si->cf_id, status);
                        // we've got an error from the other side, but we cannot just abandon rpc::source we
                        // need to continue reading until EOS since this will signal that no more work
                        // is left and rpc::source can be destroyed. The sender closes connection immediately
                        // after sending the status, so EOS should arrive shortly.
                        return stop_iteration::no;
                    } else {
                        return stop_iteration::yes;
                    }
                });
            });
        }();

        auto sink_op = [sink, si, got_error_from_peer] () mutable -> future<> {
            mutation_fragment_stream_validator validator(*(si->reader.schema()));
            return do_with(std::move(sink), std::move(validator), [si, got_error_from_peer] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd>& sink, mutation_fragment_stream_validator& validator) {
                return repeat([&sink, &validator, si, got_error_from_peer] () mutable {
                    return si->reader(db::no_timeout).then([&sink, &validator, si, s = si->reader.schema(), got_error_from_peer] (mutation_fragment_opt mf) mutable {
                        if (*got_error_from_peer) {
                            return make_exception_future<stop_iteration>(std::runtime_error("Got status error code from peer"));
                        }
                        if (mf) {
                            if (!validator(mf->mutation_fragment_kind())) {
                                return make_exception_future<stop_iteration>(std::runtime_error(format("Stream reader mutation_fragment validator failed, previous={}, current={}",
                                        validator.previous_mutation_fragment_kind(), mf->mutation_fragment_kind())));
                            }
                            frozen_mutation_fragment fmf = freeze(*s, *mf);
                            auto size = fmf.representation().size();
                            streaming::get_local_stream_manager().update_progress(si->plan_id, si->id.addr, streaming::progress_info::direction::OUT, size);
                            return sink(fmf, stream_mutation_fragments_cmd::mutation_fragment_data).then([] { return stop_iteration::no; });
                        } else {
                            if (!validator.on_end_of_stream()) {
                                return make_exception_future<stop_iteration>(std::runtime_error(format("Stream reader mutation_fragment validator failed on end_of_stream, previous={}, current=end_of_stream",
                                        validator.previous_mutation_fragment_kind())));
                            }
                            return make_ready_future<stop_iteration>(stop_iteration::yes);
                        }
                    });
                }).then([&sink] () mutable {
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::end_of_stream);
                }).handle_exception([&sink] (std::exception_ptr ep) mutable {
                    // Notify the receiver the sender has failed
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::error).then([ep = std::move(ep)] () mutable {
                        return make_exception_future<>(std::move(ep));
                    });
                }).finally([&sink] () mutable {
                    return sink.close();
                });
            });
        }();

        return when_all_succeed(std::move(source_op), std::move(sink_op)).then_unpack([got_error_from_peer, si] {
            if (*got_error_from_peer) {
                throw std::runtime_error(format("Peer failed to process mutation_fragment peer={}, plan_id={}, cf_id={}", si->id.addr, si->plan_id, si->cf_id));
            }
        });
    });
  });
 });
}

future<> stream_transfer_task::execute() {
    auto plan_id = session->plan_id();
    auto cf_id = this->cf_id;
    auto dst_cpu_id = session->dst_cpu_id;
    auto id = netw::messaging_service::msg_addr{session->peer, session->dst_cpu_id};
    sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}", plan_id, cf_id);
    sort_and_merge_ranges();
    auto reason = session->get_reason();
    return session->get_db().invoke_on_all([plan_id, cf_id, id, dst_cpu_id, ranges=this->_ranges, reason] (database& db) {
        auto si = make_lw_shared<send_info>(db, plan_id, cf_id, std::move(ranges), id, dst_cpu_id, reason);
        return si->has_relevant_range_on_this_shard().then([si, plan_id, cf_id] (bool has_relevant_range_on_this_shard) {
            if (!has_relevant_range_on_this_shard) {
                sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}: ignore ranges on shard={}",
                        plan_id, cf_id, this_shard_id());
                return make_ready_future<>();
            }
            if (si->db.features().cluster_supports_stream_with_rpc_stream()) {
                return send_mutation_fragments(std::move(si));
            } else {
                throw std::runtime_error("cluster does not support STREAM_WITH_RPC_STREAM feature");
            }
        });
    }).then([this, plan_id, cf_id, id] {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION_DONE to {}, cf_id={}", plan_id, id, cf_id);
        return session->ms().send_stream_mutation_done(id, plan_id, _ranges,
                cf_id, session->dst_cpu_id).handle_exception([plan_id, id, cf_id] (auto ep) {
            sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION_DONE to {}: {}", plan_id, id, ep);
            std::rethrow_exception(ep);
        });
    }).then([this, id, plan_id, cf_id] {
        sslog.debug("[Stream #{}] GOT STREAM_MUTATION_DONE Reply from {}", plan_id, id.addr);
    }).handle_exception([this, plan_id, id] (auto ep){
        sslog.warn("[Stream #{}] stream_transfer_task: Fail to send to {}: {}", plan_id, id, ep);
        std::rethrow_exception(ep);
    });
}

void stream_transfer_task::append_ranges(const dht::token_range_vector& ranges) {
    _ranges.insert(_ranges.end(), ranges.begin(), ranges.end());
}

void stream_transfer_task::sort_and_merge_ranges() {
    boost::icl::interval_set<dht::token> myset;
    dht::token_range_vector ranges;
    sslog.debug("cf_id = {}, before ranges = {}, size={}", cf_id, _ranges, _ranges.size());
    _ranges.swap(ranges);
    for (auto& range : ranges) {
        // TODO: We should convert range_to_interval and interval_to_range to
        // take nonwrapping_range ranges.
        myset += locator::token_metadata::range_to_interval(std::move(range));
    }
    ranges.clear();
    ranges.shrink_to_fit();
    for (auto& i : myset) {
        auto r = locator::token_metadata::interval_to_range(i);
        _ranges.push_back(dht::token_range(std::move(r)));
    }
    sslog.debug("cf_id = {}, after  ranges = {}, size={}", cf_id, _ranges, _ranges.size());
}

} // namespace streaming
<commit_msg>streaming: Keep messaging service on send_info<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Modified by ScyllaDB
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "log.hh"
#include "streaming/stream_detail.hh"
#include "streaming/stream_transfer_task.hh"
#include "streaming/stream_session.hh"
#include "streaming/stream_manager.hh"
#include "streaming/stream_reason.hh"
#include "streaming/stream_mutation_fragments_cmd.hh"
#include "mutation_reader.hh"
#include "flat_mutation_reader.hh"
#include "frozen_mutation.hh"
#include "mutation.hh"
#include "message/messaging_service.hh"
#include "range.hh"
#include "dht/i_partitioner.hh"
#include "dht/sharder.hh"
#include "service/priority_manager.hh"
#include <boost/range/irange.hpp>
#include <boost/icl/interval.hpp>
#include <boost/icl/interval_set.hpp>
#include "sstables/sstables.hh"
#include "database.hh"
#include "gms/feature_service.hh"

namespace streaming {

extern logging::logger sslog;

stream_transfer_task::stream_transfer_task(shared_ptr<stream_session> session, UUID cf_id, dht::token_range_vector ranges, long total_size)
    : stream_task(session, cf_id)
    , _ranges(std::move(ranges))
    , _total_size(total_size) {
}

stream_transfer_task::~stream_transfer_task() = default;

struct send_info {
    database& db;
    netw::messaging_service& ms;
    utils::UUID plan_id;
    utils::UUID cf_id;
    netw::messaging_service::msg_addr id;
    uint32_t dst_cpu_id;
    stream_reason reason;
    size_t mutations_nr{0};
    semaphore mutations_done{0};
    bool error_logged = false;
    column_family& cf;
    dht::token_range_vector ranges;
    dht::partition_range_vector prs;
    flat_mutation_reader reader;
    send_info(database& db_, netw::messaging_service& ms_, utils::UUID plan_id_, utils::UUID cf_id_,
              dht::token_range_vector ranges_, netw::messaging_service::msg_addr id_,
              uint32_t dst_cpu_id_, stream_reason reason_)
        : db(db_)
        , ms(ms_)
        , plan_id(plan_id_)
        , cf_id(cf_id_)
        , id(id_)
        , dst_cpu_id(dst_cpu_id_)
        , reason(reason_)
        , cf(db.find_column_family(cf_id))
        , ranges(std::move(ranges_))
        , prs(dht::to_partition_ranges(ranges))
        , reader(cf.make_streaming_reader(cf.schema(), prs)) {
    }
    future<bool> has_relevant_range_on_this_shard() {
        return do_with(false, ranges.begin(), [this] (bool& found_relevant_range, dht::token_range_vector::iterator& ranges_it) {
            auto stop_cond = [this, &found_relevant_range, &ranges_it] { return ranges_it == ranges.end() || found_relevant_range; };
            return do_until(std::move(stop_cond), [this, &found_relevant_range, &ranges_it] {
                dht::token_range range = *ranges_it++;
                if (!found_relevant_range) {
                    auto sharder = dht::selective_token_range_sharder(cf.schema()->get_sharder(), std::move(range), this_shard_id());
                    auto range_shard = sharder.next();
                    if (range_shard) {
                        found_relevant_range = true;
                    }
                }
                return make_ready_future<>();
            }).then([&found_relevant_range] {
                return found_relevant_range;
            });
        });
    }
    future<size_t> estimate_partitions() {
        return do_with(cf.get_sstables(), size_t(0), [this] (auto& sstables, size_t& partition_count) {
            return do_for_each(*sstables, [this, &partition_count] (auto& sst) {
                return do_for_each(ranges, [this, &sst, &partition_count] (auto& range) {
                    partition_count += sst->estimated_keys_for_range(range);
                });
            }).then([&partition_count] {
                return partition_count;
            });
        });
    }
};

future<> send_mutation_fragments(lw_shared_ptr<send_info> si) {
 return si->reader.peek(db::no_timeout).then([si] (mutation_fragment* mfp) {
  if (!mfp) {
    // The reader contains no data
    sslog.info("[Stream #{}] Skip sending ks={}, cf={}, reader contains no data, with new rpc streaming",
        si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name());
    return make_ready_future<>();
  }
  return si->estimate_partitions().then([si] (size_t estimated_partitions) {
    sslog.info("[Stream #{}] Start sending ks={}, cf={}, estimated_partitions={}, with new rpc streaming", si->plan_id, si->cf.schema()->ks_name(), si->cf.schema()->cf_name(), estimated_partitions);
    return si->ms.make_sink_and_source_for_stream_mutation_fragments(si->reader.schema()->version(), si->plan_id, si->cf_id, estimated_partitions, si->reason, si->id).then_unpack([si] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd> sink, rpc::source<int32_t> source) mutable {
        auto got_error_from_peer = make_lw_shared<bool>(false);

        auto source_op = [source, got_error_from_peer, si] () mutable -> future<> {
            return repeat([source, got_error_from_peer, si] () mutable {
                return source().then([source, got_error_from_peer, si] (std::optional<std::tuple<int32_t>> status_opt) mutable {
                    if (status_opt) {
                        auto status = std::get<0>(*status_opt);
                        *got_error_from_peer = status == -1;
                        sslog.debug("Got status code from peer={}, plan_id={}, cf_id={}, status={}", si->id.addr, si->plan_id, si->cf_id, status);
                        // we've got an error from the other side, but we cannot just abandon rpc::source we
                        // need to continue reading until EOS since this will signal that no more work
                        // is left and rpc::source can be destroyed. The sender closes connection immediately
                        // after sending the status, so EOS should arrive shortly.
                        return stop_iteration::no;
                    } else {
                        return stop_iteration::yes;
                    }
                });
            });
        }();

        auto sink_op = [sink, si, got_error_from_peer] () mutable -> future<> {
            mutation_fragment_stream_validator validator(*(si->reader.schema()));
            return do_with(std::move(sink), std::move(validator), [si, got_error_from_peer] (rpc::sink<frozen_mutation_fragment, stream_mutation_fragments_cmd>& sink, mutation_fragment_stream_validator& validator) {
                return repeat([&sink, &validator, si, got_error_from_peer] () mutable {
                    return si->reader(db::no_timeout).then([&sink, &validator, si, s = si->reader.schema(), got_error_from_peer] (mutation_fragment_opt mf) mutable {
                        if (*got_error_from_peer) {
                            return make_exception_future<stop_iteration>(std::runtime_error("Got status error code from peer"));
                        }
                        if (mf) {
                            if (!validator(mf->mutation_fragment_kind())) {
                                return make_exception_future<stop_iteration>(std::runtime_error(format("Stream reader mutation_fragment validator failed, previous={}, current={}",
                                        validator.previous_mutation_fragment_kind(), mf->mutation_fragment_kind())));
                            }
                            frozen_mutation_fragment fmf = freeze(*s, *mf);
                            auto size = fmf.representation().size();
                            streaming::get_local_stream_manager().update_progress(si->plan_id, si->id.addr, streaming::progress_info::direction::OUT, size);
                            return sink(fmf, stream_mutation_fragments_cmd::mutation_fragment_data).then([] { return stop_iteration::no; });
                        } else {
                            if (!validator.on_end_of_stream()) {
                                return make_exception_future<stop_iteration>(std::runtime_error(format("Stream reader mutation_fragment validator failed on end_of_stream, previous={}, current=end_of_stream",
                                        validator.previous_mutation_fragment_kind())));
                            }
                            return make_ready_future<stop_iteration>(stop_iteration::yes);
                        }
                    });
                }).then([&sink] () mutable {
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::end_of_stream);
                }).handle_exception([&sink] (std::exception_ptr ep) mutable {
                    // Notify the receiver the sender has failed
                    return sink(frozen_mutation_fragment(bytes_ostream()), stream_mutation_fragments_cmd::error).then([ep = std::move(ep)] () mutable {
                        return make_exception_future<>(std::move(ep));
                    });
                }).finally([&sink] () mutable {
                    return sink.close();
                });
            });
        }();

        return when_all_succeed(std::move(source_op), std::move(sink_op)).then_unpack([got_error_from_peer, si] {
            if (*got_error_from_peer) {
                throw std::runtime_error(format("Peer failed to process mutation_fragment peer={}, plan_id={}, cf_id={}", si->id.addr, si->plan_id, si->cf_id));
            }
        });
    });
  });
 });
}

future<> stream_transfer_task::execute() {
    auto plan_id = session->plan_id();
    auto cf_id = this->cf_id;
    auto dst_cpu_id = session->dst_cpu_id;
    auto id = netw::messaging_service::msg_addr{session->peer, session->dst_cpu_id};
    sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}", plan_id, cf_id);
    sort_and_merge_ranges();
    auto reason = session->get_reason();
    return session->get_db().invoke_on_all([plan_id, cf_id, id, dst_cpu_id, ranges=this->_ranges, reason] (database& db) {
        auto si = make_lw_shared<send_info>(db, stream_session::ms(), plan_id, cf_id, std::move(ranges), id, dst_cpu_id, reason);
        return si->has_relevant_range_on_this_shard().then([si, plan_id, cf_id] (bool has_relevant_range_on_this_shard) {
            if (!has_relevant_range_on_this_shard) {
                sslog.debug("[Stream #{}] stream_transfer_task: cf_id={}: ignore ranges on shard={}",
                        plan_id, cf_id, this_shard_id());
                return make_ready_future<>();
            }
            if (si->db.features().cluster_supports_stream_with_rpc_stream()) {
                return send_mutation_fragments(std::move(si));
            } else {
                throw std::runtime_error("cluster does not support STREAM_WITH_RPC_STREAM feature");
            }
        });
    }).then([this, plan_id, cf_id, id] {
        sslog.debug("[Stream #{}] SEND STREAM_MUTATION_DONE to {}, cf_id={}", plan_id, id, cf_id);
        return session->ms().send_stream_mutation_done(id, plan_id, _ranges,
                cf_id, session->dst_cpu_id).handle_exception([plan_id, id, cf_id] (auto ep) {
            sslog.warn("[Stream #{}] stream_transfer_task: Fail to send STREAM_MUTATION_DONE to {}: {}", plan_id, id, ep);
            std::rethrow_exception(ep);
        });
    }).then([this, id, plan_id, cf_id] {
        sslog.debug("[Stream #{}] GOT STREAM_MUTATION_DONE Reply from {}", plan_id, id.addr);
    }).handle_exception([this, plan_id, id] (auto ep){
        sslog.warn("[Stream #{}] stream_transfer_task: Fail to send to {}: {}", plan_id, id, ep);
        std::rethrow_exception(ep);
    });
}

void stream_transfer_task::append_ranges(const dht::token_range_vector& ranges) {
    _ranges.insert(_ranges.end(), ranges.begin(), ranges.end());
}

void stream_transfer_task::sort_and_merge_ranges() {
    boost::icl::interval_set<dht::token> myset;
    dht::token_range_vector ranges;
    sslog.debug("cf_id = {}, before ranges = {}, size={}", cf_id, _ranges, _ranges.size());
    _ranges.swap(ranges);
    for (auto& range : ranges) {
        // TODO: We should convert range_to_interval and interval_to_range to
        // take nonwrapping_range ranges.
        myset += locator::token_metadata::range_to_interval(std::move(range));
    }
    ranges.clear();
    ranges.shrink_to_fit();
    for (auto& i : myset) {
        auto r = locator::token_metadata::interval_to_range(i);
        _ranges.push_back(dht::token_range(std::move(r)));
    }
    sslog.debug("cf_id = {}, after  ranges = {}, size={}", cf_id, _ranges, _ranges.size());
}

} // namespace streaming
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "gdal_priv.h"
#include <iostream>
#include <complex>

// Do all comparison in double precision
const double Epsilon = 1.E-6;

template<class TPixel>
bool IsEqual(TPixel output, TPixel expected)
{
  // avoid division by zero
  return ( (expected == output) || (abs(expected - output) / abs(expected) < Epsilon) );
}

template<class FilePixelType>
int otbGDALReadPxlComplexGeneric(int argc, char * argv[])
{
  typedef std::complex<FilePixelType>              ComplexType;

  // Get Input parameters
  if (argc != 6)
    {
    std::cout << "Invalid Parameters: " << argv[0] << "<filenameIn> <posX> <posY> <sizeX> <sizeY><ouput filename>" << std::endl;
    return EXIT_FAILURE;
    }
  int posX = (int)atoi(argv[2]);
  int posY = (int)atoi(argv[3]);
  int sizeX = (int)atoi(argv[4]);
  int sizeY = (int)atoi(argv[5]);
  int nbPixelToRead = (sizeX)*(sizeY);

  // Open File
  GDALDataset  *poDataset;

  GDALAllRegister();

  poDataset = (GDALDataset *) GDALOpen( argv[1], GA_ReadOnly );
  if( poDataset == NULL )
     return EXIT_FAILURE;

  // Get some informations from file
  int nbBand = poDataset->GetRasterCount();

  GDALDataType pxlTypeInFile = poDataset->GetRasterBand(1)->GetRasterDataType();
  int bytePerPixel = sizeof(ComplexType);

  unsigned int sizeImgX = poDataset->GetRasterXSize();
  unsigned int sizeImgY = poDataset->GetRasterYSize();
  std::cout << "Dimensions of image are " << sizeImgX << ", "<< sizeImgY << ", "<< nbBand << std::endl;
  std::cout << "Type GDAL = " << GDALGetDataTypeName(poDataset->GetRasterBand(1)->GetRasterDataType()) << std::endl;

  //*******************
  // Method 1 : use RasterIO directly with the dataset
  std::cout << "**** METHOD 1 : Use RasterIO to read all selected data in file ****" << std::endl;
  std::streamoff nbBytes = nbBand * nbPixelToRead * bytePerPixel;
  std::cout << "nbBytes of the buffer = " << nbBytes << " = " \
      << nbBand << " x " << nbPixelToRead << " x " << bytePerPixel<< std::endl;

  char * loadBuffer = new char[nbBytes];

  int pixelOffset = bytePerPixel * nbBand;
  int lineOffset  = bytePerPixel * nbBand * sizeX;
  int bandOffset  = bytePerPixel;

  CPLErr lCrGdal = poDataset->RasterIO(GF_Read,
                                       posX, posY, sizeX, sizeY,
                                       loadBuffer, // pData
                                       sizeX, sizeY,
                                       pxlTypeInFile,
                                       nbBand, NULL,
                                       pixelOffset, lineOffset, bandOffset);
  // Check if gdal call succeed
  if (lCrGdal == CE_Failure)
    {
    std::cout << "Error while reading image buffer" << std::endl;
    }

  ComplexType *pxlValue = new ComplexType[nbPixelToRead * nbBand];
  unsigned int count = 0;
  for (unsigned int itPxl = 0; itPxl < (unsigned int) (nbPixelToRead * nbBand); itPxl++)
    {
    int indY = (int) ( (int)itPxl / ( sizeX * nbBand) );
    int indX = (int) ( (int)itPxl / nbBand ) - indY*sizeX;
    int indB = (int)itPxl - (int)((int) (itPxl) / nbBand) * nbBand;
    count = 2*( sizeImgX*sizeImgY*indB + sizeImgX*(posY + indY) + (posX + indX));
    pxlValue[itPxl] = *(static_cast<ComplexType*>( static_cast<void*>(loadBuffer)) + itPxl );
    std::cout << "loadBuffer["<< itPxl << "] or " \
              << "pxlValue[" << posX + indX << "," << posY + indY << "," << indB << "]" \
              << "= " << pxlValue[itPxl] \
              << " -> expectedValue = " << ComplexType(count, count+1) \
              << std::endl;

    if (!IsEqual(pxlValue[itPxl], ComplexType(count, count+1)))
      {
      std::cerr << "ERROR at position " << "[" << posX + indX << "," << posY + indY << "," << indB << "]" \
                << ". Got " << pxlValue[itPxl] << ", expected " << ComplexType(count, count+1) << std::endl;
      return EXIT_FAILURE;
      }
    }

  delete loadBuffer;
  delete pxlValue;

  //*******************
  // Method 2 : Use RasterIO to read each band included in the file
  std::cout << "**** METHOD 2 : Use RasterIO to read each band included in the file****" << std::endl;
  GDALRasterBand  *poBand;
  ComplexType pPixelValue[nbBand][nbPixelToRead];

  for (unsigned int itBand = 0; itBand < (unsigned int) nbBand; itBand++)
    {
    poBand = poDataset->GetRasterBand( itBand +1 );

    poBand->RasterIO( GF_Read, posX, posY, sizeX, sizeY,
                      &(pPixelValue[itBand]), sizeX, sizeY, poBand->GetRasterDataType(),
                      0, 0 );

    for (unsigned int itPxl = 0; itPxl < (unsigned int) nbPixelToRead; itPxl++)
      {
      int indY = (int) ((int)itPxl/sizeX);
      int indX = itPxl - indY * sizeX;
      count = 2*( sizeImgX*sizeImgY*itBand + sizeImgX*(posY + indY) + (posX + indX));
      std::cout << "pixelValue [" << posX + indX << "," << posY + indY << "," << itBand << "] = " \
                << pPixelValue[itBand][indX + indY * sizeX] \
                << " -> expectedValue = " << ComplexType(count, count+1)
                << std::endl;
      if (!IsEqual(pPixelValue[itBand][indX + indY * sizeX], ComplexType(count, count+1)))
        {
        std::cerr << "ERROR at position " << "[" << posX + indX << "," << posY + indY << "," << itBand << "]" \
                  << ". Got " << pPixelValue[itBand][indX + indY * sizeX] << ", expected " << ComplexType(count, count+1) << std::endl;
        return EXIT_FAILURE;
        }
      }
    }

  GDALClose(poDataset);

  return EXIT_SUCCESS;
}

int otbGDALReadPxlComplexFloat(int argc, char * argv[])
{
  return otbGDALReadPxlComplexGeneric<float>(argc, argv);
}

int otbGDALReadPxlComplexDouble(int argc, char * argv[])
{
  return otbGDALReadPxlComplexGeneric<double>(argc, argv);
}
<commit_msg>COMP: fix error with windows compilation<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "gdal_priv.h"
#include <iostream>
#include <complex>

// Do all comparison in double precision
const double Epsilon = 1.E-6;

template<class TPixel>
bool IsEqual(TPixel output, TPixel expected)
{
  // avoid division by zero
  return ( (expected == output) || (abs(expected - output) / abs(expected) < Epsilon) );
}

template<class FilePixelType>
int otbGDALReadPxlComplexGeneric(int argc, char * argv[])
{
  typedef std::complex<FilePixelType>              ComplexType;

  // Get Input parameters
  if (argc != 6)
    {
    std::cout << "Invalid Parameters: " << argv[0] << "<filenameIn> <posX> <posY> <sizeX> <sizeY><ouput filename>" << std::endl;
    return EXIT_FAILURE;
    }
  int posX = (int)atoi(argv[2]);
  int posY = (int)atoi(argv[3]);
  int sizeX = (int)atoi(argv[4]);
  int sizeY = (int)atoi(argv[5]);
  int nbPixelToRead = (sizeX)*(sizeY);

  // Open File
  GDALDataset  *poDataset;

  GDALAllRegister();

  poDataset = (GDALDataset *) GDALOpen( argv[1], GA_ReadOnly );
  if( poDataset == NULL )
     return EXIT_FAILURE;

  // Get some informations from file
  int nbBand = poDataset->GetRasterCount();

  GDALDataType pxlTypeInFile = poDataset->GetRasterBand(1)->GetRasterDataType();
  int bytePerPixel = sizeof(ComplexType);

  unsigned int sizeImgX = poDataset->GetRasterXSize();
  unsigned int sizeImgY = poDataset->GetRasterYSize();
  std::cout << "Dimensions of image are " << sizeImgX << ", "<< sizeImgY << ", "<< nbBand << std::endl;
  std::cout << "Type GDAL = " << GDALGetDataTypeName(poDataset->GetRasterBand(1)->GetRasterDataType()) << std::endl;

  //*******************
  // Method 1 : use RasterIO directly with the dataset
  std::cout << "**** METHOD 1 : Use RasterIO to read all selected data in file ****" << std::endl;
  std::streamoff nbBytes = static_cast<std::streamoff>(nbBand) * static_cast<std::streamoff>(nbPixelToRead) * static_cast<std::streamoff>(bytePerPixel);
  std::cout << "nbBytes of the buffer = " << nbBytes << " = " \
      << nbBand << " x " << nbPixelToRead << " x " << bytePerPixel<< std::endl;

  char * loadBuffer = new char[nbBytes];

  int pixelOffset = bytePerPixel * nbBand;
  int lineOffset  = bytePerPixel * nbBand * sizeX;
  int bandOffset  = bytePerPixel;

  CPLErr lCrGdal = poDataset->RasterIO(GF_Read,
                                       posX, posY, sizeX, sizeY,
                                       loadBuffer, // pData
                                       sizeX, sizeY,
                                       pxlTypeInFile,
                                       nbBand, NULL,
                                       pixelOffset, lineOffset, bandOffset);
  // Check if gdal call succeed
  if (lCrGdal == CE_Failure)
    {
    std::cout << "Error while reading image buffer" << std::endl;
    }

  ComplexType *pxlValue = new ComplexType[nbPixelToRead * nbBand];
  unsigned int count = 0;
  ComplexType expectedValue;
  for (unsigned int itPxl = 0; itPxl < (unsigned int) (nbPixelToRead * nbBand); itPxl++)
    {
    int indY = (int) ( (int)itPxl / ( sizeX * nbBand) );
    int indX = (int) ( (int)itPxl / nbBand ) - indY*sizeX;
    int indB = (int)itPxl - (int)((int) (itPxl) / nbBand) * nbBand;
    count = 2*( sizeImgX*sizeImgY*indB + sizeImgX*(posY + indY) + (posX + indX));
    pxlValue[itPxl] = *(static_cast<ComplexType*>( static_cast<void*>(loadBuffer)) + itPxl );
    expectedValue = ComplexType(count,count+1);
    std::cout << "loadBuffer["<< itPxl << "] or " \
              << "pxlValue[" << posX + indX << "," << posY + indY << "," << indB << "]" \
              << "= " << pxlValue[itPxl] \
              << " -> expectedValue = " << expectedValue \
              << std::endl;

    if (!IsEqual(pxlValue[itPxl], expectedValue))
      {
      std::cerr << "ERROR at position " << "[" << posX + indX << "," << posY + indY << "," << indB << "]" \
                << ". Got " << pxlValue[itPxl] << ", expected " << expectedValue << std::endl;
      return EXIT_FAILURE;
      }
    }

  delete loadBuffer;
  delete pxlValue;

  //*******************
  // Method 2 : Use RasterIO to read each band included in the file
  std::cout << "**** METHOD 2 : Use RasterIO to read each band included in the file****" << std::endl;
  GDALRasterBand  *poBand;
  //ComplexType pPixelValue[nbBand][nbPixelToRead];
  ComplexType* pPixelValue = new ComplexType[nbBand*nbPixelToRead];

  for (unsigned int itBand = 0; itBand < (unsigned int) nbBand; itBand++)
    {
    poBand = poDataset->GetRasterBand( itBand +1 );

    poBand->RasterIO( GF_Read, posX, posY, sizeX, sizeY,
                      &(pPixelValue[itBand*nbPixelToRead]), sizeX, sizeY, poBand->GetRasterDataType(),
                      0, 0 );

    for (unsigned int itPxl = 0; itPxl < (unsigned int) nbPixelToRead; itPxl++)
      {
      int indY = (int) ((int)itPxl/sizeX);
      int indX = itPxl - indY * sizeX;
      count = 2*( sizeImgX*sizeImgY*itBand + sizeImgX*(posY + indY) + (posX + indX));
      expectedValue = ComplexType(count,count+1);
      std::cout << "pixelValue [" << posX + indX << "," << posY + indY << "," << itBand << "] = " \
                << pPixelValue[itBand*nbPixelToRead + indX + indY * sizeX] \
                << " -> expectedValue = " << expectedValue \
                << std::endl;
      if (!IsEqual(pPixelValue[itBand*nbPixelToRead + indX + indY * sizeX], expectedValue))
        {
        std::cerr << "ERROR at position " << "[" << posX + indX << "," << posY + indY << "," << itBand << "]" \
                  << ". Got " << pPixelValue[itBand*nbPixelToRead + indX + indY * sizeX] << ", expected " << expectedValue << std::endl;
        return EXIT_FAILURE;
        }
      }
    }

  delete pPixelValue;
  GDALClose(poDataset);

  return EXIT_SUCCESS;
}

int otbGDALReadPxlComplexFloat(int argc, char * argv[])
{
  return otbGDALReadPxlComplexGeneric<float>(argc, argv);
}

int otbGDALReadPxlComplexDouble(int argc, char * argv[])
{
  return otbGDALReadPxlComplexGeneric<double>(argc, argv);
}
<|endoftext|>


<commit_before>// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.

#ifndef MFEM_MESH_OPERATORS
#define MFEM_MESH_OPERATORS

#include "../config/config.hpp"
#include "../general/array.hpp"
#include "mesh.hpp"
#include "../fem/estimators.hpp"

#include <limits>

namespace mfem
{

/** @brief The MeshOperator class serves as base for mesh manipulation classes.

    The purpose of the class is to provide a common abstraction for various
    AMR mesh control schemes. The typical use in an AMR loop is illustrated
    in examples 6/6p and 15/15p.

    A more general loop that also supports sequences of mesh operators with
    multiple updates looks like this:
    \code
       for (...)
       {
          // computations on the current mesh ...
          while (mesh_operator->Apply(mesh))
          {
             // update FiniteElementSpaces and interpolate GridFunctions ...
             if (mesh_operator->Continue()) { break; }
          }
          if (mesh_operator->Stop()) { break; }
       }
    \endcode
 */
class MeshOperator
{
private:
   int mod;

protected:
   friend class MeshOperatorSequence;

   /** @brief Implementation of the mesh operation. Invoked by the Apply()
              public method.
       @return Combination of ActionInfo constants. */
   virtual int ApplyImpl(Mesh &mesh) = 0;

   /// Constructor to be used by derived classes.
   MeshOperator() : mod(NONE) { }

public:
   /** @brief Action and information constants and masks.

       Combinations of constants are returned by the Apply() virtual method and
       can be accessed directly with GetActionInfo() or indirectly with methods
       like Stop(), Continue(), etc. The information bits (MASK_INFO) can be set
       only when the update bit is set (see MASK_UPDATE). */
   enum Action
   {
      NONE        = 0, /**< continue with computations without updating spaces
                            or grid-functions, i.e. the mesh was not modified */
      CONTINUE    = 1, /**< update spaces and grid-functions and continue
                            computations with the new mesh */
      STOP        = 2, ///< a stopping criterion was satisfied
      REPEAT      = 3, /**< update spaces and grid-functions and call the
                            operator Apply() method again */
      MASK_UPDATE = 1, ///< bit mask for the "update" bit
      MASK_ACTION = 3  ///< bit mask for all "action" bits
   };

   enum Info
   {
      REFINED     = 4*1, ///< the mesh was refined
      DEREFINED   = 4*2, ///< the mesh was de-refined
      REBALANCED  = 4*3, ///< the mesh was rebalanced
      MASK_INFO   = ~3   ///< bit mask for all "info" bits
   };

   /** @brief Perform the mesh operation.
       @return true if FiniteElementSpaces and GridFunctions need to be updated.
   */
   bool Apply(Mesh &mesh) { return ((mod = ApplyImpl(mesh)) & MASK_UPDATE); }

   /** @brief Check if STOP action is requested, e.g. stopping criterion is
       satisfied. */
   bool Stop() const { return ((mod & MASK_ACTION) == STOP); }
   /** @brief Check if REPEAT action is requested, i.e. FiniteElementSpaces and
       GridFunctions need to be updated, and Apply() must be called again. */
   bool Repeat() const { return ((mod & MASK_ACTION) == REPEAT); }
   /** @brief Check if CONTINUE action is requested, i.e. FiniteElementSpaces
       and GridFunctions need to be updated and computations should continue. */
   bool Continue() const { return ((mod & MASK_ACTION) == CONTINUE); }

   /// Check if the mesh was refined.
   bool Refined() const { return ((mod & MASK_INFO) == REFINED); }
   /// Check if the mesh was de-refined.
   bool Derefined() const { return ((mod & MASK_INFO) == DEREFINED); }
   /// Check if the mesh was rebalanced.
   bool Rebalanced() const { return ((mod & MASK_INFO) == REBALANCED); }

   /** @brief Get the full ActionInfo value generated by the last call to
       Apply(). */
   int GetActionInfo() const { return mod; }

   /// Reset the MeshOperator.
   virtual void Reset() = 0;

   /// The destructor is virtual.
   virtual ~MeshOperator() { }
};

/** Composition of MeshOperators into a sequence. Use the Append() method to
    create the sequence. */
class MeshOperatorSequence : public MeshOperator
{
protected:
   int step;
   Array<MeshOperator*> sequence; ///< MeshOperators sequence, owned by us.

   /// Do not allow copy construction, due to assumed ownership.
   MeshOperatorSequence(const MeshOperatorSequence &) { }

   /** @brief Apply the MeshOperatorSequence.
       @return ActionInfo value corresponding to the last applied operator from
       the sequence. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Constructor. Use the Append() method to create the sequence.
   MeshOperatorSequence() : step(-1) { }

   /// Delete all operators from the sequence.
   virtual ~MeshOperatorSequence();

   /** @brief Add an operator to the end of the sequence.
       The MeshOperatorSequence assumes ownership of the operator. */
   void Append(MeshOperator *mc) { sequence.Append(mc); }

   /// Access the underlying sequence.
   Array<MeshOperator*> &GetSequence() { return sequence; }

   /// Reset all MeshOperators in the sequence.
   virtual void Reset();
};

/** @brief Mesh refinement operator using an error threshold.

    This class uses the given ErrorEstimator to estimate local element errors
    and then marks for refinement all elements i such that loc_err_i > threshold.
    The threshold is computed as
    \code
       threshold = max(total_err * total_fraction * pow(num_elements,-1.0/p),
                       local_err_goal);
    \endcode
    where p (=total_norm_p), total_fraction, and local_err_goal are settable
    parameters, total_err = (sum_i local_err_i^p)^{1/p}, when p < inf,
    or total_err = max_i local_err_i, when p = inf.
*/
class ThresholdRefiner : public MeshOperator
{
protected:
   ErrorEstimator &estimator;
   AnisotropicErrorEstimator *aniso_estimator;

   double total_norm_p;
   double total_err_goal;
   double total_fraction;
   double local_err_goal;
   long   max_elements;

   double threshold;
   long num_marked_elements;

   Array<Refinement> marked_elements;
   long current_sequence;

   int non_conforming;
   int nc_limit;

   double GetNorm(const Vector &local_err, Mesh &mesh) const;

   /** @brief Apply the operator to the mesh.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Construct a ThresholdRefiner using the given ErrorEstimator.
   ThresholdRefiner(ErrorEstimator &est);

   // default destructor (virtual)

   /** @brief Set the exponent, p, of the discrete p-norm used to compute the
       total error from the local element errors. */
   void SetTotalErrorNormP(double norm_p = infinity())
   { total_norm_p = norm_p; }

   /** @brief Set the total error stopping criterion: stop when
       total_err <= total_err_goal. The default value is zero. */
   void SetTotalErrorGoal(double err_goal) { total_err_goal = err_goal; }

   /** @brief Set the total fraction used in the computation of the threshold.
       The default value is 1/2.
       @note If fraction == 0, total_err is essentially ignored in the threshold
       computation, i.e. threshold = local error goal. */
   void SetTotalErrorFraction(double fraction) { total_fraction = fraction; }

   /** @brief Set the local stopping criterion: stop when
       local_err_i <= local_err_goal. The default value is zero.
       @note If local_err_goal == 0, it is essentially ignored in the threshold
       computation. */
   void SetLocalErrorGoal(double err_goal) { local_err_goal = err_goal; }

   /** @brief Set the maximum number of elements stopping criterion: stop when
       the input mesh has num_elements >= max_elem. The default value is
       LONG_MAX. */
   void SetMaxElements(long max_elem) { max_elements = max_elem; }

   /// Use nonconforming refinement, if possible (triangles, quads, hexes).
   void PreferNonconformingRefinement() { non_conforming = 1; }

   /** @brief Use conforming refinement, if possible (triangles, tetrahedra)
       -- this is the default. */
   void PreferConformingRefinement() { non_conforming = -1; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). */
   void SetNCLimit(int nc_limit)
   {
      MFEM_ASSERT(nc_limit >= 0, "Invalid NC limit");
      this->nc_limit = nc_limit;
   }

   /// Get the number of marked elements in the last Apply() call.
   long GetNumMarkedElements() const { return num_marked_elements; }

   /// Get the threshold used in the last Apply() call.
   double GetThreshold() const { return threshold; }

   /// Reset the associated estimator.
   virtual void Reset();
};

// TODO: BulkRefiner to refine a portion of the global error

/** @brief De-refinement operator using an error threshold.

    This de-refinement operator marks elements in the hierarchy whose children
    are leaves and their combined error is below a given threshold. The
    errors of the children are combined by one of the following operations:
    - op = 0: minimum of the errors
    - op = 1: sum of the errors (default)
    - op = 2: maximum of the errors. */
class ThresholdDerefiner : public MeshOperator
{
protected:
   ErrorEstimator &estimator;

   double threshold;
   int nc_limit, op;

   /** @brief Apply the operator to the mesh.
       @return DEREFINED + CONTINUE if some elements were de-refined; NONE
       otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Construct a ThresholdDerefiner using the given ErrorEstimator.
   ThresholdDerefiner(ErrorEstimator &est)
      : estimator(est)
   {
      threshold = 0.0;
      nc_limit = 0;
      op = 1;
   }

   // default destructor (virtual)

   /// Set the de-refinement threshold. The default value is zero.
   void SetThreshold(double thresh) { threshold = thresh; }

   void SetOp(int op) { this->op = op; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). */
   void SetNCLimit(int nc_limit)
   {
      MFEM_ASSERT(nc_limit >= 0, "Invalid NC limit");
      this->nc_limit = nc_limit;
   }

   /// Reset the associated estimator.
   virtual void Reset() { estimator.Reset(); }
};

/** @brief Refinement operator to control data oscillation.

    This class uses the given computes osc_K(f) := || h ⋅ (I - Π) f ||_K  at
    each element K. Here, Π is the L2-projection and ||⋅||_K is the
    L2-norm, restricted to the element K. All elements satisfying the inequality
    \code
       osc_K(f) > threshold ⋅ ||f|| / sqrt(n_el),
    \endcode
    are refined. Here, threshold is a postive parameter, ||⋅|| is the
    L2-norm over the entire domain Ω, and n_el is the number of elements in the
    mesh.

    Note that if osc(f) = threshold ⋅ ||f|| / sqrt(n_el) for each K, then
    \code
       osc(f) = sqrt( sum_K osc_K^2(f)) = threshold ⋅ ||f||.
    \endcode
    This is the reason for the 1/sqrt(n_el) factor. */
class CoefficientRefiner : public MeshOperator
{
protected:
   int nc_limit = 1;
   int nonconforming = -1;
   int order;
   long max_elements = std::numeric_limits<long>::max();
   double threshold = 1.0e-2;
   double global_osc = 0.0;
   Array<int> mesh_refinements;
   Vector element_oscs;
   Coefficient *coeff = NULL;
   GridFunction *gf;
   const IntegrationRule *ir_default[Geometry::NumGeom];
   const IntegrationRule **irs = NULL;

   /** @brief Apply the operator to the mesh once.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Constructor
   CoefficientRefiner(int order_) : order(order_) { }

   /** @brief Apply the operator to the mesh max_it times or until tolerance
    *  achieved.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int PreprocessMesh(Mesh &mesh, int max_it);

   int PreprocessMesh(Mesh &mesh)
   {
      int max_it = 10;
      return PreprocessMesh(mesh, max_it);
   }

   /// Set the refinement threshold. The default value is 1.0e-3.
   void SetThreshold(double threshold_) { threshold = threshold_; }

   /** @brief Set the maximum number of elements stopping criterion: stop when
       the input mesh has num_elements >= max_elem. The default value is
       LONG_MAX. */
   void SetMaxElements(long max_elements_) { max_elements = max_elements_; }

   /// Set the function f
   void SetCoefficient(Coefficient &coeff_)
   {
      global_osc = 0.0;
      coeff = &coeff_;
   }

   /// Reset the oscillation order
   void SetOrder(double order_) { order = order_; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). The default value is 1, which helps ensure appropriate
       refinements in pathological situations where the default quadrature
       order is too low.  */
   void SetNCLimit(int nc_limit_)
   {
      MFEM_ASSERT(nc_limit_ >= 0, "Invalid NC limit");
      nc_limit = nc_limit_;
   }

   // Set a custom integration rule
   void SetIntRule(const IntegrationRule *irs_[]) { irs = irs_; }

   // Return the value of the global relative data oscillation
   double GetOsc() { return global_osc; }

   // Return the local relative data oscillation errors
   Vector GetLocalOscs()
   {
       MFEM_ASSERT(element_oscs.Size() > 0, "Local oscillations have not been computed yet")
       return element_oscs;
   }

   /// Reset
   virtual void Reset();
};

/** @brief ParMesh rebalancing operator.

    If the mesh is a parallel mesh, perform rebalancing; otherwise, do nothing.
*/
class Rebalancer : public MeshOperator
{
protected:
   /** @brief Rebalance a parallel mesh (only non-conforming parallel meshes are
       supported).
       @return CONTINUE + REBALANCE on success, NONE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Empty.
   virtual void Reset() { }
};

} // namespace mfem

#endif // MFEM_MESH_OPERATORS
<commit_msg>fixed style<commit_after>// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.

#ifndef MFEM_MESH_OPERATORS
#define MFEM_MESH_OPERATORS

#include "../config/config.hpp"
#include "../general/array.hpp"
#include "mesh.hpp"
#include "../fem/estimators.hpp"

#include <limits>

namespace mfem
{

/** @brief The MeshOperator class serves as base for mesh manipulation classes.

    The purpose of the class is to provide a common abstraction for various
    AMR mesh control schemes. The typical use in an AMR loop is illustrated
    in examples 6/6p and 15/15p.

    A more general loop that also supports sequences of mesh operators with
    multiple updates looks like this:
    \code
       for (...)
       {
          // computations on the current mesh ...
          while (mesh_operator->Apply(mesh))
          {
             // update FiniteElementSpaces and interpolate GridFunctions ...
             if (mesh_operator->Continue()) { break; }
          }
          if (mesh_operator->Stop()) { break; }
       }
    \endcode
 */
class MeshOperator
{
private:
   int mod;

protected:
   friend class MeshOperatorSequence;

   /** @brief Implementation of the mesh operation. Invoked by the Apply()
              public method.
       @return Combination of ActionInfo constants. */
   virtual int ApplyImpl(Mesh &mesh) = 0;

   /// Constructor to be used by derived classes.
   MeshOperator() : mod(NONE) { }

public:
   /** @brief Action and information constants and masks.

       Combinations of constants are returned by the Apply() virtual method and
       can be accessed directly with GetActionInfo() or indirectly with methods
       like Stop(), Continue(), etc. The information bits (MASK_INFO) can be set
       only when the update bit is set (see MASK_UPDATE). */
   enum Action
   {
      NONE        = 0, /**< continue with computations without updating spaces
                            or grid-functions, i.e. the mesh was not modified */
      CONTINUE    = 1, /**< update spaces and grid-functions and continue
                            computations with the new mesh */
      STOP        = 2, ///< a stopping criterion was satisfied
      REPEAT      = 3, /**< update spaces and grid-functions and call the
                            operator Apply() method again */
      MASK_UPDATE = 1, ///< bit mask for the "update" bit
      MASK_ACTION = 3  ///< bit mask for all "action" bits
   };

   enum Info
   {
      REFINED     = 4*1, ///< the mesh was refined
      DEREFINED   = 4*2, ///< the mesh was de-refined
      REBALANCED  = 4*3, ///< the mesh was rebalanced
      MASK_INFO   = ~3   ///< bit mask for all "info" bits
   };

   /** @brief Perform the mesh operation.
       @return true if FiniteElementSpaces and GridFunctions need to be updated.
   */
   bool Apply(Mesh &mesh) { return ((mod = ApplyImpl(mesh)) & MASK_UPDATE); }

   /** @brief Check if STOP action is requested, e.g. stopping criterion is
       satisfied. */
   bool Stop() const { return ((mod & MASK_ACTION) == STOP); }
   /** @brief Check if REPEAT action is requested, i.e. FiniteElementSpaces and
       GridFunctions need to be updated, and Apply() must be called again. */
   bool Repeat() const { return ((mod & MASK_ACTION) == REPEAT); }
   /** @brief Check if CONTINUE action is requested, i.e. FiniteElementSpaces
       and GridFunctions need to be updated and computations should continue. */
   bool Continue() const { return ((mod & MASK_ACTION) == CONTINUE); }

   /// Check if the mesh was refined.
   bool Refined() const { return ((mod & MASK_INFO) == REFINED); }
   /// Check if the mesh was de-refined.
   bool Derefined() const { return ((mod & MASK_INFO) == DEREFINED); }
   /// Check if the mesh was rebalanced.
   bool Rebalanced() const { return ((mod & MASK_INFO) == REBALANCED); }

   /** @brief Get the full ActionInfo value generated by the last call to
       Apply(). */
   int GetActionInfo() const { return mod; }

   /// Reset the MeshOperator.
   virtual void Reset() = 0;

   /// The destructor is virtual.
   virtual ~MeshOperator() { }
};

/** Composition of MeshOperators into a sequence. Use the Append() method to
    create the sequence. */
class MeshOperatorSequence : public MeshOperator
{
protected:
   int step;
   Array<MeshOperator*> sequence; ///< MeshOperators sequence, owned by us.

   /// Do not allow copy construction, due to assumed ownership.
   MeshOperatorSequence(const MeshOperatorSequence &) { }

   /** @brief Apply the MeshOperatorSequence.
       @return ActionInfo value corresponding to the last applied operator from
       the sequence. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Constructor. Use the Append() method to create the sequence.
   MeshOperatorSequence() : step(-1) { }

   /// Delete all operators from the sequence.
   virtual ~MeshOperatorSequence();

   /** @brief Add an operator to the end of the sequence.
       The MeshOperatorSequence assumes ownership of the operator. */
   void Append(MeshOperator *mc) { sequence.Append(mc); }

   /// Access the underlying sequence.
   Array<MeshOperator*> &GetSequence() { return sequence; }

   /// Reset all MeshOperators in the sequence.
   virtual void Reset();
};

/** @brief Mesh refinement operator using an error threshold.

    This class uses the given ErrorEstimator to estimate local element errors
    and then marks for refinement all elements i such that loc_err_i > threshold.
    The threshold is computed as
    \code
       threshold = max(total_err * total_fraction * pow(num_elements,-1.0/p),
                       local_err_goal);
    \endcode
    where p (=total_norm_p), total_fraction, and local_err_goal are settable
    parameters, total_err = (sum_i local_err_i^p)^{1/p}, when p < inf,
    or total_err = max_i local_err_i, when p = inf.
*/
class ThresholdRefiner : public MeshOperator
{
protected:
   ErrorEstimator &estimator;
   AnisotropicErrorEstimator *aniso_estimator;

   double total_norm_p;
   double total_err_goal;
   double total_fraction;
   double local_err_goal;
   long   max_elements;

   double threshold;
   long num_marked_elements;

   Array<Refinement> marked_elements;
   long current_sequence;

   int non_conforming;
   int nc_limit;

   double GetNorm(const Vector &local_err, Mesh &mesh) const;

   /** @brief Apply the operator to the mesh.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Construct a ThresholdRefiner using the given ErrorEstimator.
   ThresholdRefiner(ErrorEstimator &est);

   // default destructor (virtual)

   /** @brief Set the exponent, p, of the discrete p-norm used to compute the
       total error from the local element errors. */
   void SetTotalErrorNormP(double norm_p = infinity())
   { total_norm_p = norm_p; }

   /** @brief Set the total error stopping criterion: stop when
       total_err <= total_err_goal. The default value is zero. */
   void SetTotalErrorGoal(double err_goal) { total_err_goal = err_goal; }

   /** @brief Set the total fraction used in the computation of the threshold.
       The default value is 1/2.
       @note If fraction == 0, total_err is essentially ignored in the threshold
       computation, i.e. threshold = local error goal. */
   void SetTotalErrorFraction(double fraction) { total_fraction = fraction; }

   /** @brief Set the local stopping criterion: stop when
       local_err_i <= local_err_goal. The default value is zero.
       @note If local_err_goal == 0, it is essentially ignored in the threshold
       computation. */
   void SetLocalErrorGoal(double err_goal) { local_err_goal = err_goal; }

   /** @brief Set the maximum number of elements stopping criterion: stop when
       the input mesh has num_elements >= max_elem. The default value is
       LONG_MAX. */
   void SetMaxElements(long max_elem) { max_elements = max_elem; }

   /// Use nonconforming refinement, if possible (triangles, quads, hexes).
   void PreferNonconformingRefinement() { non_conforming = 1; }

   /** @brief Use conforming refinement, if possible (triangles, tetrahedra)
       -- this is the default. */
   void PreferConformingRefinement() { non_conforming = -1; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). */
   void SetNCLimit(int nc_limit)
   {
      MFEM_ASSERT(nc_limit >= 0, "Invalid NC limit");
      this->nc_limit = nc_limit;
   }

   /// Get the number of marked elements in the last Apply() call.
   long GetNumMarkedElements() const { return num_marked_elements; }

   /// Get the threshold used in the last Apply() call.
   double GetThreshold() const { return threshold; }

   /// Reset the associated estimator.
   virtual void Reset();
};

// TODO: BulkRefiner to refine a portion of the global error

/** @brief De-refinement operator using an error threshold.

    This de-refinement operator marks elements in the hierarchy whose children
    are leaves and their combined error is below a given threshold. The
    errors of the children are combined by one of the following operations:
    - op = 0: minimum of the errors
    - op = 1: sum of the errors (default)
    - op = 2: maximum of the errors. */
class ThresholdDerefiner : public MeshOperator
{
protected:
   ErrorEstimator &estimator;

   double threshold;
   int nc_limit, op;

   /** @brief Apply the operator to the mesh.
       @return DEREFINED + CONTINUE if some elements were de-refined; NONE
       otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Construct a ThresholdDerefiner using the given ErrorEstimator.
   ThresholdDerefiner(ErrorEstimator &est)
      : estimator(est)
   {
      threshold = 0.0;
      nc_limit = 0;
      op = 1;
   }

   // default destructor (virtual)

   /// Set the de-refinement threshold. The default value is zero.
   void SetThreshold(double thresh) { threshold = thresh; }

   void SetOp(int op) { this->op = op; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). */
   void SetNCLimit(int nc_limit)
   {
      MFEM_ASSERT(nc_limit >= 0, "Invalid NC limit");
      this->nc_limit = nc_limit;
   }

   /// Reset the associated estimator.
   virtual void Reset() { estimator.Reset(); }
};

/** @brief Refinement operator to control data oscillation.

    This class uses the given computes osc_K(f) := || h ⋅ (I - Π) f ||_K  at
    each element K. Here, Π is the L2-projection and ||⋅||_K is the
    L2-norm, restricted to the element K. All elements satisfying the inequality
    \code
       osc_K(f) > threshold ⋅ ||f|| / sqrt(n_el),
    \endcode
    are refined. Here, threshold is a postive parameter, ||⋅|| is the
    L2-norm over the entire domain Ω, and n_el is the number of elements in the
    mesh.

    Note that if osc(f) = threshold ⋅ ||f|| / sqrt(n_el) for each K, then
    \code
       osc(f) = sqrt( sum_K osc_K^2(f)) = threshold ⋅ ||f||.
    \endcode
    This is the reason for the 1/sqrt(n_el) factor. */
class CoefficientRefiner : public MeshOperator
{
protected:
   int nc_limit = 1;
   int nonconforming = -1;
   int order;
   long max_elements = std::numeric_limits<long>::max();
   double threshold = 1.0e-2;
   double global_osc = 0.0;
   Array<int> mesh_refinements;
   Vector element_oscs;
   Coefficient *coeff = NULL;
   GridFunction *gf;
   const IntegrationRule *ir_default[Geometry::NumGeom];
   const IntegrationRule **irs = NULL;

   /** @brief Apply the operator to the mesh once.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Constructor
   CoefficientRefiner(int order_) : order(order_) { }

   /** @brief Apply the operator to the mesh max_it times or until tolerance
    *  achieved.
       @return STOP if a stopping criterion is satisfied or no elements were
       marked for refinement; REFINED + CONTINUE otherwise. */
   virtual int PreprocessMesh(Mesh &mesh, int max_it);

   int PreprocessMesh(Mesh &mesh)
   {
      int max_it = 10;
      return PreprocessMesh(mesh, max_it);
   }

   /// Set the refinement threshold. The default value is 1.0e-3.
   void SetThreshold(double threshold_) { threshold = threshold_; }

   /** @brief Set the maximum number of elements stopping criterion: stop when
       the input mesh has num_elements >= max_elem. The default value is
       LONG_MAX. */
   void SetMaxElements(long max_elements_) { max_elements = max_elements_; }

   /// Set the function f
   void SetCoefficient(Coefficient &coeff_)
   {
      global_osc = 0.0;
      coeff = &coeff_;
   }

   /// Reset the oscillation order
   void SetOrder(double order_) { order = order_; }

   /** @brief Set the maximum ratio of refinement levels of adjacent elements
       (0 = unlimited). The default value is 1, which helps ensure appropriate
       refinements in pathological situations where the default quadrature
       order is too low.  */
   void SetNCLimit(int nc_limit_)
   {
      MFEM_ASSERT(nc_limit_ >= 0, "Invalid NC limit");
      nc_limit = nc_limit_;
   }

   // Set a custom integration rule
   void SetIntRule(const IntegrationRule *irs_[]) { irs = irs_; }

   // Return the value of the global relative data oscillation
   double GetOsc() { return global_osc; }

   // Return the local relative data oscillation errors
   Vector GetLocalOscs()
   {
      MFEM_ASSERT(element_oscs.Size() > 0,
                  "Local oscillations have not been computed yet")
      return element_oscs;
   }

   /// Reset
   virtual void Reset();
};

/** @brief ParMesh rebalancing operator.

    If the mesh is a parallel mesh, perform rebalancing; otherwise, do nothing.
*/
class Rebalancer : public MeshOperator
{
protected:
   /** @brief Rebalance a parallel mesh (only non-conforming parallel meshes are
       supported).
       @return CONTINUE + REBALANCE on success, NONE otherwise. */
   virtual int ApplyImpl(Mesh &mesh);

public:
   /// Empty.
   virtual void Reset() { }
};

} // namespace mfem

#endif // MFEM_MESH_OPERATORS
<|endoftext|>


<commit_before>#include "SimpleITKTestHarness.h"
#include "sitkCastImageFilter.h"
#include "sitkSimpleElastix.h"

namespace sitk = itk::simple;

class SimpleElastix4DTest : public ::testing::Test
{
protected:

    virtual void SetUp()
    {
        image4d = sitk::ReadImage( dataFinder.GetFile( "Input/4D.nii.gz" ) );
    }

    typedef sitk::SimpleElastix                      SimpleElastixType;
    typedef SimpleElastixType::ParameterMapType      ParameterMapType;
    typedef SimpleElastixType::ParameterValuesType   ParameterValuesType;

    sitk::Image image4d;

    bool IsEmpty( const sitk::Image image )
    {
      return ( image.GetWidth() == 0 && image.GetHeight() == 0 );
    }
};

TEST_F( SimpleElastix4DTest, GroupwiseRegistration )
{
    ParameterMapType parameterMap = sitk::GetDefaultParameterMap( "groupwise" );
    parameterMap["MaxiumNumberOfIterations"] = ParameterValuesType( 1, "4" ); // otherwise unreasonable testing time

    SimpleElastixType elastix;
    sitk::Image resultImage;

    EXPECT_NO_THROW( elastix.SetFixedImage( image4d ) );
    EXPECT_NO_THROW( elastix.SetMovingImage( image4d ) );
    EXPECT_NO_THROW( elastix.SetParameterMap( parameterMap ) );
    EXPECT_NO_THROW( elastix.LogToFolder( "." ) );
    EXPECT_NO_THROW( elastix.Execute() );
    EXPECT_NO_THROW( resultImage = elastix.GetResultImage() );
    EXPECT_FALSE( this->IsEmpty( resultImage ) );
}<commit_msg>ENH: Significantly speed up groupies registration test<commit_after>#include "SimpleITKTestHarness.h"
#include "sitkCastImageFilter.h"
#include "sitkSimpleElastix.h"

namespace sitk = itk::simple;

class SimpleElastix4DTest : public ::testing::Test
{
protected:

    virtual void SetUp()
    {
        image4d = sitk::ReadImage( dataFinder.GetFile( "Input/4D.nii.gz" ) );
    }

    typedef sitk::SimpleElastix                      SimpleElastixType;
    typedef SimpleElastixType::ParameterMapType      ParameterMapType;
    typedef SimpleElastixType::ParameterValuesType   ParameterValuesType;

    sitk::Image image4d;

    bool IsEmpty( const sitk::Image image )
    {
      return ( image.GetWidth() == 0 && image.GetHeight() == 0 );
    }
};

TEST_F( SimpleElastix4DTest, GroupwiseRegistration )
{
    ParameterMapType parameterMap = sitk::GetDefaultParameterMap( "groupwise", 1, 32 );

    // otherwise unreasonable testing time
    parameterMap["MaximumNumberOfIterations"] = ParameterValuesType( 1, "16" );

    SimpleElastixType elastix;
    sitk::Image resultImage;

    EXPECT_NO_THROW( elastix.SetFixedImage( image4d ) );
    EXPECT_NO_THROW( elastix.SetMovingImage( image4d ) );
    EXPECT_NO_THROW( elastix.SetParameterMap( parameterMap ) );
    EXPECT_NO_THROW( elastix.LogToFolder( "." ) );
    EXPECT_NO_THROW( elastix.Execute() );
    EXPECT_NO_THROW( resultImage = elastix.GetResultImage() );
    EXPECT_FALSE( this->IsEmpty( resultImage ) );
}<|endoftext|>


<commit_before>// Copyright (C) 2012-2014 Leap Motion, Inc. All rights reserved.
#include "stdafx.h"
#include "AutoPacketGraph.h"
#include "AutoPacketProfiler.h"
#include "CoreContext.h"
#include "demangle.h"
#include <fstream>
#include <string>

AutoPacketGraph::AutoPacketGraph()
{
}

void AutoPacketGraph::AddEdge(const std::type_info* ti, const AutoFilterDescriptor& descriptor, bool input) {
  DeliveryEdge edge { ti, descriptor, input };

  std::lock_guard<std::mutex> lk(m_lock);
  auto itr = m_deliveryGraph.find(edge);
  if (itr == m_deliveryGraph.end()) {
    m_deliveryGraph[edge] = 1;
  } else {
    itr->second++;
  }
}

void AutoPacketGraph::AutoFilter(AutoPacket& packet) {
  packet.AddTeardownListener([this, &packet] () {
    for (auto& decoration : packet.GetDispositions()) {
      if (decoration.m_publisher) {
        AddEdge(decoration.m_type, *decoration.m_publisher, false);
      }

      for (auto& subscriber : decoration.m_subscribers) {
        if (subscriber->called) {
          AddEdge(decoration.m_type, *subscriber, true);
        }
      }
    }
  });
}

bool AutoPacketGraph::WriteGV(const std::string& filename) const
{
  std::ofstream file(filename);
  if (!file && !file.good()) {
    return false;
  }

  file << "digraph \"" << autowiring::demangle(CoreContext::CurrentContext()->GetSigilType()) << " context\" {\n";

  std::lock_guard<std::mutex> lk(m_lock);

  // Containers for the unique types and descriptors
  std::unordered_set<std::string> typeNames;
  std::unordered_set<std::string> descriptorNames;

  for (auto& itr : m_deliveryGraph) {
    auto& edge = itr.first;
    auto type = edge.type_info;
    auto& descriptor = edge.descriptor;
    auto count = itr.second;

    // TODO: skip if type == AutoPacketGraph

    std::string typeName = autowiring::demangle(type);
    std::string descriptorName = autowiring::demangle(descriptor.GetType());

    // Get a unique set of types/descriptors
    if (typeNames.find(typeName) == typeNames.end())
      typeNames.insert(typeName);

    if (descriptorNames.find(descriptorName) == descriptorNames.end())
      descriptorNames.insert(descriptorName);

    // string format: "type" -> "AutoFilter" (or vice versa)
    std::stringstream ss;
    ss << "  \"";
    if (edge.input) {
      ss << typeName << "\" -> \"" << descriptorName;
    } else {
      ss << descriptorName << "\" -> \"" << typeName;
    }

    // TODO: count should probably be optional
    ss << "\" [label=\"" << count << "\"];" << std::endl;

    file << ss.str();
  }
  file << std::endl;

  // Setup the shapes for the types and descriptors
  for (auto& typeName : typeNames)
    file << "  \"" << typeName << "\" [shape=box];" << std::endl;

  for (auto& descriptorName : descriptorNames)
    file << "  \"" << descriptorName << "\" [shape=ellipse];" << std::endl;

  file << "}\n";

  return true;
}
<commit_msg>Fixed small bug where publisher needs to check if it has been called<commit_after>// Copyright (C) 2012-2014 Leap Motion, Inc. All rights reserved.
#include "stdafx.h"
#include "AutoPacketGraph.h"
#include "AutoPacketProfiler.h"
#include "CoreContext.h"
#include "demangle.h"
#include <fstream>
#include <string>

AutoPacketGraph::AutoPacketGraph()
{
}

void AutoPacketGraph::AddEdge(const std::type_info* ti, const AutoFilterDescriptor& descriptor, bool input) {
  DeliveryEdge edge { ti, descriptor, input };

  std::lock_guard<std::mutex> lk(m_lock);
  auto itr = m_deliveryGraph.find(edge);
  if (itr == m_deliveryGraph.end()) {
    m_deliveryGraph[edge] = 1;
  } else {
    itr->second++;
  }
}

void AutoPacketGraph::AutoFilter(AutoPacket& packet) {
  packet.AddTeardownListener([this, &packet] () {
    for (auto& decoration : packet.GetDispositions()) {
      auto publisher = decoration.m_publisher;
      auto type = decoration.m_type;

      if (publisher && publisher->called) {
        AddEdge(type, *publisher, false);
      }

      for (auto& subscriber : decoration.m_subscribers) {
        if (subscriber->called) {
          AddEdge(type, *subscriber, true);
        }
      }
    }
  });
}

bool AutoPacketGraph::WriteGV(const std::string& filename) const
{
  std::ofstream file(filename);
  if (!file && !file.good()) {
    return false;
  }

  file << "digraph \"" << autowiring::demangle(CoreContext::CurrentContext()->GetSigilType()) << " context\" {\n";

  std::lock_guard<std::mutex> lk(m_lock);

  // Containers for the unique types and descriptors
  std::unordered_set<std::string> typeNames;
  std::unordered_set<std::string> descriptorNames;

  for (auto& itr : m_deliveryGraph) {
    auto& edge = itr.first;
    auto type = edge.type_info;
    auto& descriptor = edge.descriptor;
    auto count = itr.second;

    // TODO: skip if type == AutoPacketGraph

    std::string typeName = autowiring::demangle(type);
    std::string descriptorName = autowiring::demangle(descriptor.GetType());

    // Get a unique set of types/descriptors
    if (typeNames.find(typeName) == typeNames.end())
      typeNames.insert(typeName);

    if (descriptorNames.find(descriptorName) == descriptorNames.end())
      descriptorNames.insert(descriptorName);

    // string format: "type" -> "AutoFilter" (or vice versa)
    std::stringstream ss;
    ss << "  \"";
    if (edge.input) {
      ss << typeName << "\" -> \"" << descriptorName;
    } else {
      ss << descriptorName << "\" -> \"" << typeName;
    }

    // TODO: count should probably be optional
    ss << "\" [label=\"" << count << "\"];" << std::endl;

    file << ss.str();
  }
  file << std::endl;

  // Setup the shapes for the types and descriptors
  for (auto& typeName : typeNames)
    file << "  \"" << typeName << "\" [shape=box];" << std::endl;

  for (auto& descriptorName : descriptorNames)
    file << "  \"" << descriptorName << "\" [shape=ellipse];" << std::endl;

  file << "}\n";

  return true;
}
<|endoftext|>


<commit_before>/** @file

  A brief file description

  @section license License

  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

#include "libts.h"

#include "ink_args.h"
#include "I_Version.h"
#include "Tokenizer.h"
#include "TextBuffer.h"
#include "mgmtapi.h"

static const char *programName;

static char ReadVar[1024];
static char SetVar[1024];
static char VarValue[1024];
static int ReRead;
static int Shutdown;
static int BounceCluster;
static int BounceLocal;
static int QueryDeadhosts;
static int Startup;
static int ShutdownMgmtCluster;
static int ShutdownMgmtLocal;
static int ClearCluster;
static int ClearNode;
static int VersionFlag;

static TSError
handleArgInvocation()
{
  if (ReRead == 1) {
    return TSReconfigure();
  } else if (ShutdownMgmtCluster == 1) {
    return TSRestart(true);
  } else if (ShutdownMgmtLocal == 1) {
    return TSRestart(false);
  } else if (Shutdown == 1) {
    return TSProxyStateSet(TS_PROXY_OFF, TS_CACHE_CLEAR_OFF);
  } else if (BounceCluster == 1) {
    return TSBounce(true);
  } else if (BounceLocal == 1) {
    return TSBounce(false);
  } else if (Startup == 1) {
    return TSProxyStateSet(TS_PROXY_ON, TS_CACHE_CLEAR_OFF);
  } else if (ClearCluster == 1) {
    return TSStatsReset(true);
  } else if (ClearNode == 1) {
    return TSStatsReset(false);
  } else if (QueryDeadhosts == 1) {
    fprintf(stderr, "Query Deadhosts is not implemented, it requires support for congestion control.\n");
    fprintf(stderr, "For more details, examine the old code in cli/CLI.cc: QueryDeadhosts()\n");
    return TS_ERR_FAIL;
  } else if (*ReadVar != '\0') {        // Handle a value read
    if (*SetVar != '\0' || *VarValue != '\0') {
      fprintf(stderr, "%s: Invalid Argument Combination: Can not read and set values at the same time\n", programName);
      return TS_ERR_FAIL;
    } else {
      TSError err;
      TSRecordEle *rec_ele = TSRecordEleCreate();

      if ((err = TSRecordGet(ReadVar, rec_ele)) != TS_ERR_OKAY) {
        fprintf(stderr, "%s: %s\n", programName, TSGetErrorMessage(err));
      } else {
        switch (rec_ele->rec_type) {
        case TS_REC_INT:
          printf("%" PRId64 "\n", rec_ele->int_val);
          break;
        case TS_REC_COUNTER:
          printf("%" PRId64 "\n", rec_ele->counter_val);
          break;
        case TS_REC_FLOAT:
          printf("%f\n", rec_ele->float_val);
          break;
        case TS_REC_STRING:
          printf("%s\n", rec_ele->string_val);
          break;
        default:
          fprintf(stderr, "%s: unknown record type (%d)\n", programName, rec_ele->rec_type);
          err = TS_ERR_FAIL;
          break;
        }
      }
      TSRecordEleDestroy(rec_ele);
      return err;
    }
  } else if (*SetVar != '\0') { // Setting a variable
    if (*VarValue == '\0') {
      fprintf(stderr, "%s: Set requires a -v argument\n", programName);
      return TS_ERR_FAIL;
    } else {
      TSError err;
      TSActionNeedT action;

      if ((err = TSRecordSet(SetVar, VarValue, &action)) != TS_ERR_OKAY)
        fprintf(stderr, "%s: Please correct your variable name and|or value\n", programName);
      return err;
    }
  } else if (*VarValue != '\0') {       // We have a value but no variable to set
    fprintf(stderr, "%s: Must specify variable to set with -s when using -v\n", programName);
    return TS_ERR_FAIL;
  }

  fprintf(stderr, "%s: No arguments specified\n", programName);
  return TS_ERR_FAIL;
}

int
main(int argc, char **argv)
{
  NOWARN_UNUSED(argc);
  AppVersionInfo appVersionInfo;
  TSError status;

  programName = argv[0];

  ReadVar[0] = '\0';
  SetVar[0] = '\0';
  VarValue[0] = '\0';
  ReRead = 0;
  Shutdown = 0;
  BounceCluster = 0;
  BounceLocal = 0;
  QueryDeadhosts = 0;
  Startup = 0;
  ShutdownMgmtCluster = 0;
  ShutdownMgmtLocal = 0;
  ClearCluster = 0;
  ClearNode = 0;
  VersionFlag = 0;

  // build the application information structure
  appVersionInfo.setup(PACKAGE_NAME,"traffic_line", PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, "");

/* Argument description table used to describe how to parse command line args, */
/* see 'ink_args.h' for meanings of the various fields */
  ArgumentDescription argument_descriptions[] = {
    {"query_deadhosts", 'q', "Query congested sites", "F", &QueryDeadhosts, NULL, NULL},
    {"read_var", 'r', "Read Variable", "S1024", &ReadVar, NULL, NULL},
    {"set_var", 's', "Set Variable (requires -v option)", "S1024", &SetVar, NULL, NULL},
    {"value", 'v', "Set Value (used with -s option)", "S1024", &VarValue, NULL, NULL},
    {"help", 'h', "Help", NULL, NULL, NULL, usage},
    {"reread_config", 'x', "Reread Config Files", "F", &ReRead, NULL, NULL},
    {"restart_cluster", 'M', "Restart traffic_manager (cluster wide)", "F", &ShutdownMgmtCluster, NULL, NULL},
    {"restart_local", 'L', "Restart traffic_manager (local node)", "F", &ShutdownMgmtLocal, NULL, NULL},
    {"shutdown", 'S', "Shutdown traffic_server (local node)", "F", &Shutdown, NULL, NULL},
    {"startup", 'U', "Start traffic_server (local node)", "F", &Startup, NULL, NULL},
    {"bounce_cluster", 'B', "Bounce traffic_server (cluster wide)", "F", &BounceCluster, NULL, NULL},
    {"bounce_local", 'b', "Bounce local traffic_server", "F", &BounceLocal, NULL, NULL},
    {"clear_cluster", 'C', "Clear Statistics (cluster wide)", "F", &ClearCluster, NULL, NULL},
    {"clear_node", 'c', "Clear Statistics (local node)", "F", &ClearNode, NULL, NULL},
    {"version", 'V', "Print Version Id", "T", &VersionFlag, NULL, NULL},
  };

  // Process command line arguments and dump into variables
  process_args(argument_descriptions, SIZE(argument_descriptions), argv);

  // check for the version number request
  if (VersionFlag) {
    ink_fputln(stderr, appVersionInfo.FullVersionInfoStr);
    exit(0);
  }

  // Connect to Local Manager and do it.
  TSInit(NULL, static_cast<TSInitOptionT>(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS));
  status = handleArgInvocation();

  // Done with the mgmt API.
  TSTerminate();

  if (TS_ERR_OKAY != status) {
    if (ReadVar[0] == '\0' && SetVar[0] == '\0')
      fprintf(stderr, "error: the requested command failed\n");
    exit(1);
  }

  exit(0);
}
<commit_msg>TS-1379 Better error message when mgmt socket is not available<commit_after>/** @file

  A brief file description

  @section license License

  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

#include "libts.h"

#include "ink_args.h"
#include "I_Version.h"
#include "Tokenizer.h"
#include "TextBuffer.h"
#include "mgmtapi.h"

static const char *programName;

static char ReadVar[1024];
static char SetVar[1024];
static char VarValue[1024];
static int ReRead;
static int Shutdown;
static int BounceCluster;
static int BounceLocal;
static int QueryDeadhosts;
static int Startup;
static int ShutdownMgmtCluster;
static int ShutdownMgmtLocal;
static int ClearCluster;
static int ClearNode;
static int VersionFlag;

static TSError
handleArgInvocation()
{
  if (ReRead == 1) {
    return TSReconfigure();
  } else if (ShutdownMgmtCluster == 1) {
    return TSRestart(true);
  } else if (ShutdownMgmtLocal == 1) {
    return TSRestart(false);
  } else if (Shutdown == 1) {
    return TSProxyStateSet(TS_PROXY_OFF, TS_CACHE_CLEAR_OFF);
  } else if (BounceCluster == 1) {
    return TSBounce(true);
  } else if (BounceLocal == 1) {
    return TSBounce(false);
  } else if (Startup == 1) {
    return TSProxyStateSet(TS_PROXY_ON, TS_CACHE_CLEAR_OFF);
  } else if (ClearCluster == 1) {
    return TSStatsReset(true);
  } else if (ClearNode == 1) {
    return TSStatsReset(false);
  } else if (QueryDeadhosts == 1) {
    fprintf(stderr, "Query Deadhosts is not implemented, it requires support for congestion control.\n");
    fprintf(stderr, "For more details, examine the old code in cli/CLI.cc: QueryDeadhosts()\n");
    return TS_ERR_FAIL;
  } else if (*ReadVar != '\0') {        // Handle a value read
    if (*SetVar != '\0' || *VarValue != '\0') {
      fprintf(stderr, "%s: Invalid Argument Combination: Can not read and set values at the same time\n", programName);
      return TS_ERR_FAIL;
    } else {
      TSError err;
      TSRecordEle *rec_ele = TSRecordEleCreate();

      if ((err = TSRecordGet(ReadVar, rec_ele)) != TS_ERR_OKAY) {
        fprintf(stderr, "%s: %s\n", programName, TSGetErrorMessage(err));
      } else {
        switch (rec_ele->rec_type) {
        case TS_REC_INT:
          printf("%" PRId64 "\n", rec_ele->int_val);
          break;
        case TS_REC_COUNTER:
          printf("%" PRId64 "\n", rec_ele->counter_val);
          break;
        case TS_REC_FLOAT:
          printf("%f\n", rec_ele->float_val);
          break;
        case TS_REC_STRING:
          printf("%s\n", rec_ele->string_val);
          break;
        default:
          fprintf(stderr, "%s: unknown record type (%d)\n", programName, rec_ele->rec_type);
          err = TS_ERR_FAIL;
          break;
        }
      }
      TSRecordEleDestroy(rec_ele);
      return err;
    }
  } else if (*SetVar != '\0') { // Setting a variable
    if (*VarValue == '\0') {
      fprintf(stderr, "%s: Set requires a -v argument\n", programName);
      return TS_ERR_FAIL;
    } else {
      TSError err;
      TSActionNeedT action;

      if ((err = TSRecordSet(SetVar, VarValue, &action)) != TS_ERR_OKAY)
        fprintf(stderr, "%s: Please correct your variable name and|or value\n", programName);
      return err;
    }
  } else if (*VarValue != '\0') {       // We have a value but no variable to set
    fprintf(stderr, "%s: Must specify variable to set with -s when using -v\n", programName);
    return TS_ERR_FAIL;
  }

  fprintf(stderr, "%s: No arguments specified\n", programName);
  return TS_ERR_FAIL;
}

int
main(int argc, char **argv)
{
  NOWARN_UNUSED(argc);
  AppVersionInfo appVersionInfo;
  TSError status;

  programName = argv[0];

  ReadVar[0] = '\0';
  SetVar[0] = '\0';
  VarValue[0] = '\0';
  ReRead = 0;
  Shutdown = 0;
  BounceCluster = 0;
  BounceLocal = 0;
  QueryDeadhosts = 0;
  Startup = 0;
  ShutdownMgmtCluster = 0;
  ShutdownMgmtLocal = 0;
  ClearCluster = 0;
  ClearNode = 0;
  VersionFlag = 0;

  // build the application information structure
  appVersionInfo.setup(PACKAGE_NAME,"traffic_line", PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, "");

/* Argument description table used to describe how to parse command line args, */
/* see 'ink_args.h' for meanings of the various fields */
  ArgumentDescription argument_descriptions[] = {
    {"query_deadhosts", 'q', "Query congested sites", "F", &QueryDeadhosts, NULL, NULL},
    {"read_var", 'r', "Read Variable", "S1024", &ReadVar, NULL, NULL},
    {"set_var", 's', "Set Variable (requires -v option)", "S1024", &SetVar, NULL, NULL},
    {"value", 'v', "Set Value (used with -s option)", "S1024", &VarValue, NULL, NULL},
    {"help", 'h', "Help", NULL, NULL, NULL, usage},
    {"reread_config", 'x', "Reread Config Files", "F", &ReRead, NULL, NULL},
    {"restart_cluster", 'M', "Restart traffic_manager (cluster wide)", "F", &ShutdownMgmtCluster, NULL, NULL},
    {"restart_local", 'L', "Restart traffic_manager (local node)", "F", &ShutdownMgmtLocal, NULL, NULL},
    {"shutdown", 'S', "Shutdown traffic_server (local node)", "F", &Shutdown, NULL, NULL},
    {"startup", 'U', "Start traffic_server (local node)", "F", &Startup, NULL, NULL},
    {"bounce_cluster", 'B', "Bounce traffic_server (cluster wide)", "F", &BounceCluster, NULL, NULL},
    {"bounce_local", 'b', "Bounce local traffic_server", "F", &BounceLocal, NULL, NULL},
    {"clear_cluster", 'C', "Clear Statistics (cluster wide)", "F", &ClearCluster, NULL, NULL},
    {"clear_node", 'c', "Clear Statistics (local node)", "F", &ClearNode, NULL, NULL},
    {"version", 'V', "Print Version Id", "T", &VersionFlag, NULL, NULL},
  };

  // Process command line arguments and dump into variables
  process_args(argument_descriptions, SIZE(argument_descriptions), argv);

  // check for the version number request
  if (VersionFlag) {
    ink_fputln(stderr, appVersionInfo.FullVersionInfoStr);
    exit(0);
  }

  // Connect to Local Manager and do it.
  if (TS_ERR_OKAY != TSInit(NULL, static_cast<TSInitOptionT>(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS))) {
    fprintf(stderr, "error: could not connect to management port, make sure traffic_manager is running\n");
    exit(1);
  }

  status = handleArgInvocation();

  // Done with the mgmt API.
  TSTerminate();

  if (TS_ERR_OKAY != status) {
    if (ReadVar[0] == '\0' && SetVar[0] == '\0')
      fprintf(stderr, "error: the requested command failed\n");
    exit(1);
  }

  exit(0);
}
<|endoftext|>