<commit_before>/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#pragma once

#include "core/reactor.hh"
#include "core/iostream.hh"
#include "core/distributed.hh"
#include "core/print.hh"
#include "core/sstring.hh"
#include "net/api.hh"
#include "util/serialization.hh"
#include "gms/inet_address.hh"
#include "rpc/rpc.hh"
#include <unordered_map>
#include "db/config.hh"
#include "frozen_mutation.hh"
#include "db/serializer.hh"

namespace net {

/* All verb handler identifiers */
enum class messaging_verb : int32_t {
    MUTATION,
    MUTATION_DONE,
    BINARY, // Deprecated
    READ_REPAIR,
    READ,
    READ_DATA,
    READ_DIGEST,
    REQUEST_RESPONSE, // client-initiated reads and writes
    STREAM_INITIATE, // Deprecated
    STREAM_INITIATE_DONE, // Deprecated
    STREAM_REPLY, // Deprecated
    STREAM_REQUEST, // Deprecated
    RANGE_SLICE,
    BOOTSTRAP_TOKEN, // Deprecated
    TREE_REQUEST, // Deprecated
    TREE_RESPONSE, // Deprecated
    JOIN, // Deprecated
    GOSSIP_DIGEST_SYN,
    GOSSIP_DIGEST_ACK,
    GOSSIP_DIGEST_ACK2,
    DEFINITIONS_ANNOUNCE, // Deprecated
    DEFINITIONS_UPDATE,
    TRUNCATE,
    SCHEMA_CHECK,
    INDEX_SCAN, // Deprecated
    REPLICATION_FINISHED,
    INTERNAL_RESPONSE, // responses to internal calls
    COUNTER_MUTATION,
    STREAMING_REPAIR_REQUEST, // Deprecated
    STREAMING_REPAIR_RESPONSE, // Deprecated
    SNAPSHOT, // Similar to nt snapshot
    MIGRATION_REQUEST,
    GOSSIP_SHUTDOWN,
    _TRACE,
    ECHO,
    REPAIR_MESSAGE,
    PAXOS_PREPARE,
    PAXOS_PROPOSE,
    PAXOS_COMMIT,
    PAGED_RANGE,
    UNUSED_1,
    UNUSED_2,
    UNUSED_3,
    // Used by streaming
    STREAM_INIT_MESSAGE,
    PREPARE_MESSAGE,
    STREAM_MUTATION,
    INCOMING_FILE_MESSAGE,
    OUTGOING_FILE_MESSAGE,
    RECEIVED_MESSAGE,
    RETRY_MESSAGE,
    COMPLETE_MESSAGE,
    SESSION_FAILED_MESSAGE,
    LAST,
};

} // namespace net

namespace std {
template <>
class hash<net::messaging_verb> {
public:
    size_t operator()(const net::messaging_verb& x) const {
        return hash<int32_t>()(int32_t(x));
    }
};
} // namespace std

namespace net {

future<> ser_messaging_verb(output_stream<char>& out, messaging_verb& v);
future<> des_messaging_verb(input_stream<char>& in, messaging_verb& v);
future<> ser_sstring(output_stream<char>& out, sstring& v);
future<> des_sstring(input_stream<char>& in, sstring& v);
future<> ser_frozen_mutation(output_stream<char>& out, const frozen_mutation& v);
future<> des_frozen_mutation(input_stream<char>& in, frozen_mutation& v);

// NOTE: operator(input_stream<char>&, T&) takes a reference to uninitialized
//       T object and should use placement new in case T is non POD
struct serializer {
    // For integer type
    template<typename T>
    inline auto operator()(output_stream<char>& out, T&& v, std::enable_if_t<std::is_integral<std::remove_reference_t<T>>::value, void*> = nullptr) {
        auto v_ = net::hton(v);
        return out.write(reinterpret_cast<const char*>(&v_), sizeof(T));
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, T& v, std::enable_if_t<std::is_integral<T>::value, void*> = nullptr) {
        return in.read_exactly(sizeof(v)).then([&v] (temporary_buffer<char> buf) mutable {
            if (buf.size() != sizeof(v)) {
                throw rpc::closed_error();
            }
            v = net::ntoh(*reinterpret_cast<const net::packed<T>*>(buf.get()));
        });
    }

    // For vectors
    template<typename T>
    inline auto operator()(output_stream<char>& out, std::vector<T>& v) {
        return operator()(out, v.size()).then([&out, &v, this] {
            return do_for_each(v.begin(), v.end(), [&out, this] (T& e) {
                return operator()(out, e);
            });
        });
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, std::vector<T>& v) {
        using size_type = typename  std::vector<T>::size_type;
        return in.read_exactly(sizeof(size_type)).then([&v, &in, this] (temporary_buffer<char> buf) {
            if (buf.size() != sizeof(size_type)) {
                throw rpc::closed_error();
            }
            size_type c = net::ntoh(*reinterpret_cast<const net::packed<size_type>*>(buf.get()));
            new (&v) std::vector<T>;
            v.reserve(c);
            union U {
                U(){}
                ~U(){}
                U(U&&) {}
                T v;
            };
            return do_with(U(), [c, &v, &in, this] (U& u) {
                return do_until([c = c] () mutable {return !c--;}, [&v, &in, &u, this] () mutable {
                    return operator()(in, u.v).then([&u, &v] {
                        v.emplace_back(std::move(u.v));
                    });
                });
            });
        });
    }

    // For messaging_verb
    inline auto operator()(output_stream<char>& out, messaging_verb& v) {
        return ser_messaging_verb(out, v);
    }
    inline auto operator()(input_stream<char>& in, messaging_verb& v) {
        return des_messaging_verb(in, v);
    }

    // For sstring
    inline auto operator()(output_stream<char>& out, sstring& v) {
        return ser_sstring(out, v);
    }
    inline auto operator()(input_stream<char>& in, sstring& v) {
        return des_sstring(in, v);
    }

    // For frozen_mutation
    inline auto operator()(output_stream<char>& out, const frozen_mutation& v) {
        return ser_frozen_mutation(out, v);
    }
    inline auto operator()(output_stream<char>& out, frozen_mutation& v) {
        return operator()(out, const_cast<const frozen_mutation&>(v));
    }
    inline auto operator()(input_stream<char>& in, frozen_mutation& v) {
        return des_frozen_mutation(in, v);
    }

    // For complex types which have serialize()/deserialize(),  e.g. gms::gossip_digest_syn, gms::gossip_digest_ack2
    template<typename T>
    inline auto operator()(output_stream<char>& out, T&& v, std::enable_if_t<!std::is_integral<std::remove_reference_t<T>>::value &&
                                                                             !std::is_enum<std::remove_reference_t<T>>::value, void*> = nullptr) {
        auto sz = serialize_int32_size + v.serialized_size();
        bytes b(bytes::initialized_later(), sz);
        auto _out = b.begin();
        serialize_int32(_out, int32_t(sz - serialize_int32_size));
        v.serialize(_out);
        return out.write(reinterpret_cast<const char*>(b.c_str()), sz);
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, T& v, std::enable_if_t<!std::is_integral<T>::value &&
                                                                          !std::is_enum<T>::value, void*> = nullptr) {
        return in.read_exactly(serialize_int32_size).then([&in, &v] (temporary_buffer<char> buf) mutable {
            if (buf.size() != serialize_int32_size) {
                throw rpc::closed_error();
            }
            size_t sz = net::ntoh(*reinterpret_cast<const net::packed<int32_t>*>(buf.get()));
            return in.read_exactly(sz).then([sz, &v] (temporary_buffer<char> buf) mutable {
                if (buf.size() != sz) {
                    throw rpc::closed_error();
                }
                bytes_view bv(reinterpret_cast<const int8_t*>(buf.get()), sz);
                new (&v) T(T::deserialize(bv));
                assert(bv.size() == 0);
                return make_ready_future<>();
            });
        });
    }
};

class messaging_service {
public:
    // FIXME: messaging service versioning
    static constexpr int32_t current_version = 0;

    struct shard_id {
        gms::inet_address addr;
        uint32_t cpu_id;
        friend inline bool operator==(const shard_id& x, const shard_id& y) {
            return x.addr == y.addr && x.cpu_id == y.cpu_id ;
        }
        friend inline bool operator<(const shard_id& x, const shard_id& y) {
            if (x.addr < y.addr) {
                return true;
            } else if (y.addr < x.addr) {
                return false;
            } else {
                return x.cpu_id < y.cpu_id;
            }
        }
        friend inline std::ostream& operator<<(std::ostream& os, const shard_id& x) {
            return os << x.addr << ":" << x.cpu_id;
        }
        struct hash {
            size_t operator()(const shard_id& id) const {
                return std::hash<uint32_t>()(id.cpu_id) + std::hash<uint32_t>()(id.addr.raw_addr());
            }
        };
    };
    struct shard_info {
        shard_info(std::unique_ptr<rpc::protocol<serializer, messaging_verb>::client>&& client)
            : rpc_client(std::move(client)) {
        }
        std::unique_ptr<rpc::protocol<serializer, messaging_verb>::client> rpc_client;
    };

    void foreach_client(std::function<void(const messaging_service::shard_id& id,
                    const messaging_service::shard_info& info)> f) const {
        for (auto i = _clients.cbegin(); i != _clients.cend(); i++) {
            f(i->first, i->second);
        }
    }

    void increment_dropped_messages(messaging_verb verb) {
        _dropped_messages[static_cast<int32_t>(verb)]++;
    }

    uint64_t get_dropped_messages(messaging_verb verb) const {
        return _dropped_messages[static_cast<int32_t>(verb)];
    }

    const uint64_t* get_dropped_messages() const {
        return _dropped_messages;
    }

    int32_t get_raw_version(const gms::inet_address& endpoint) const {
        // FIXME: messaging service versioning
        return current_version;
    }

    bool knows_version(const gms::inet_address& endpoint) const {
        // FIXME: messaging service versioning
        return true;
    }

private:
    static constexpr uint16_t _default_port = 7000;
    gms::inet_address _listen_address;
    uint16_t _port;
    rpc::protocol<serializer, messaging_verb> _rpc;
    rpc::protocol<serializer, messaging_verb>::server _server;
    std::unordered_map<shard_id, shard_info, shard_id::hash> _clients;
    uint64_t _dropped_messages[static_cast<int32_t>(messaging_verb::LAST)] = {};
public:
    messaging_service(gms::inet_address ip = gms::inet_address("0.0.0.0"))
        : _listen_address(ip)
        , _port(_default_port)
        , _rpc(serializer{})
        , _server(_rpc, ipv4_addr{_listen_address.raw_addr(), _port}) {
    }
public:
    uint16_t port() {
        return _port;
    }
    auto listen_address() {
        return _listen_address;
    }
    future<> stop() {
        return when_all(_server.stop(),
            parallel_for_each(_clients, [](std::pair<const shard_id, shard_info>& c) {
                return c.second.rpc_client->stop();
            })
        ).discard_result();
    }

    static auto no_wait() {
        return rpc::no_wait;
    }
public:
    // Register a handler (a callback lambda) for verb
    template <typename Func>
    void register_handler(messaging_verb verb, Func&& func) {
        _rpc.register_handler(verb, std::move(func));
    }

    // Send a message for verb
    template <typename MsgIn, typename... MsgOut>
    auto send_message(messaging_verb verb, shard_id id, MsgOut&&... msg) {
        auto& rpc_client = get_rpc_client(id);
        auto rpc_handler = _rpc.make_client<MsgIn(MsgOut...)>(verb);
        return rpc_handler(rpc_client, std::forward<MsgOut>(msg)...).then_wrapped([this, id, verb] (auto&& f) {
            try {
                if (f.failed()) {
                    this->increment_dropped_messages(verb);
                    f.get();
                    assert(false); // never reached
                }
                return std::move(f);
            } catch(...) {
                // FIXME: we need to distinguish between a transport error and
                // a server error.
                // remove_rpc_client(id);
                throw;
            }
        });
    }

    template <typename... MsgOut>
    auto send_message_oneway(messaging_verb verb, shard_id id, MsgOut&&... msg) {
        return send_message<rpc::no_wait_type>(std::move(verb), std::move(id), std::forward<MsgOut>(msg)...);
    }
private:
    // Return rpc::protocol::client for a shard which is a ip + cpuid pair.
    rpc::protocol<serializer, messaging_verb>::client& get_rpc_client(shard_id id) {
        auto it = _clients.find(id);
        if (it == _clients.end()) {
            auto remote_addr = ipv4_addr(id.addr.raw_addr(), _port);
            auto client = std::make_unique<rpc::protocol<serializer, messaging_verb>::client>(_rpc, remote_addr, ipv4_addr{_listen_address.raw_addr(), 0});
            it = _clients.emplace(id, shard_info(std::move(client))).first;
            return *it->second.rpc_client;
        } else {
            return *it->second.rpc_client;
        }
    }

    void remove_rpc_client(shard_id id) {
        _clients.erase(id);
    }
};

extern distributed<messaging_service> _the_messaging_service;

inline distributed<messaging_service>& get_messaging_service() {
    return _the_messaging_service;
}

inline messaging_service& get_local_messaging_service() {
    return _the_messaging_service.local();
}

future<> init_messaging_service(sstring listen_address, db::config::seed_provider_type seed_provider);
} // namespace net
<commit_msg>messaging_service: Extract integral reading logic<commit_after>/*
 * Copyright (C) 2015 Cloudius Systems, Ltd.
 */

#pragma once

#include "core/reactor.hh"
#include "core/iostream.hh"
#include "core/distributed.hh"
#include "core/print.hh"
#include "core/sstring.hh"
#include "net/api.hh"
#include "util/serialization.hh"
#include "gms/inet_address.hh"
#include "rpc/rpc.hh"
#include <unordered_map>
#include "db/config.hh"
#include "frozen_mutation.hh"
#include "db/serializer.hh"

namespace net {

/* All verb handler identifiers */
enum class messaging_verb : int32_t {
    MUTATION,
    MUTATION_DONE,
    BINARY, // Deprecated
    READ_REPAIR,
    READ,
    READ_DATA,
    READ_DIGEST,
    REQUEST_RESPONSE, // client-initiated reads and writes
    STREAM_INITIATE, // Deprecated
    STREAM_INITIATE_DONE, // Deprecated
    STREAM_REPLY, // Deprecated
    STREAM_REQUEST, // Deprecated
    RANGE_SLICE,
    BOOTSTRAP_TOKEN, // Deprecated
    TREE_REQUEST, // Deprecated
    TREE_RESPONSE, // Deprecated
    JOIN, // Deprecated
    GOSSIP_DIGEST_SYN,
    GOSSIP_DIGEST_ACK,
    GOSSIP_DIGEST_ACK2,
    DEFINITIONS_ANNOUNCE, // Deprecated
    DEFINITIONS_UPDATE,
    TRUNCATE,
    SCHEMA_CHECK,
    INDEX_SCAN, // Deprecated
    REPLICATION_FINISHED,
    INTERNAL_RESPONSE, // responses to internal calls
    COUNTER_MUTATION,
    STREAMING_REPAIR_REQUEST, // Deprecated
    STREAMING_REPAIR_RESPONSE, // Deprecated
    SNAPSHOT, // Similar to nt snapshot
    MIGRATION_REQUEST,
    GOSSIP_SHUTDOWN,
    _TRACE,
    ECHO,
    REPAIR_MESSAGE,
    PAXOS_PREPARE,
    PAXOS_PROPOSE,
    PAXOS_COMMIT,
    PAGED_RANGE,
    UNUSED_1,
    UNUSED_2,
    UNUSED_3,
    // Used by streaming
    STREAM_INIT_MESSAGE,
    PREPARE_MESSAGE,
    STREAM_MUTATION,
    INCOMING_FILE_MESSAGE,
    OUTGOING_FILE_MESSAGE,
    RECEIVED_MESSAGE,
    RETRY_MESSAGE,
    COMPLETE_MESSAGE,
    SESSION_FAILED_MESSAGE,
    LAST,
};

} // namespace net

namespace std {
template <>
class hash<net::messaging_verb> {
public:
    size_t operator()(const net::messaging_verb& x) const {
        return hash<int32_t>()(int32_t(x));
    }
};
} // namespace std

namespace net {

future<> ser_messaging_verb(output_stream<char>& out, messaging_verb& v);
future<> des_messaging_verb(input_stream<char>& in, messaging_verb& v);
future<> ser_sstring(output_stream<char>& out, sstring& v);
future<> des_sstring(input_stream<char>& in, sstring& v);
future<> ser_frozen_mutation(output_stream<char>& out, const frozen_mutation& v);
future<> des_frozen_mutation(input_stream<char>& in, frozen_mutation& v);

// NOTE: operator(input_stream<char>&, T&) takes a reference to uninitialized
//       T object and should use placement new in case T is non POD
struct serializer {
    template<typename T>
    inline future<T> read_integral(input_stream<char>& in) {
        static_assert(std::is_integral<T>::value, "T should be integral");

        return in.read_exactly(sizeof(T)).then([] (temporary_buffer<char> buf) {
            if (buf.size() != sizeof(T)) {
                throw rpc::closed_error();
            }
            return make_ready_future<T>(net::ntoh(*unaligned_cast<T*>(buf.get())));
        });
    }

    // For integer type
    template<typename T>
    inline auto operator()(output_stream<char>& out, T&& v, std::enable_if_t<std::is_integral<std::remove_reference_t<T>>::value, void*> = nullptr) {
        auto v_ = net::hton(v);
        return out.write(reinterpret_cast<const char*>(&v_), sizeof(T));
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, T& v, std::enable_if_t<std::is_integral<T>::value, void*> = nullptr) {
        return in.read_exactly(sizeof(v)).then([&v] (temporary_buffer<char> buf) mutable {
            if (buf.size() != sizeof(v)) {
                throw rpc::closed_error();
            }
            v = net::ntoh(*reinterpret_cast<const net::packed<T>*>(buf.get()));
        });
    }

    // For vectors
    template<typename T>
    inline auto operator()(output_stream<char>& out, std::vector<T>& v) {
        return operator()(out, v.size()).then([&out, &v, this] {
            return do_for_each(v.begin(), v.end(), [&out, this] (T& e) {
                return operator()(out, e);
            });
        });
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, std::vector<T>& v) {
        using size_type = typename  std::vector<T>::size_type;
        return read_integral<size_type>(in).then([&v, &in, this] (size_type c) {
            new (&v) std::vector<T>;
            v.reserve(c);
            union U {
                U(){}
                ~U(){}
                U(U&&) {}
                T v;
            };
            return do_with(U(), [c, &v, &in, this] (U& u) {
                return do_until([c = c] () mutable {return !c--;}, [&v, &in, &u, this] () mutable {
                    return operator()(in, u.v).then([&u, &v] {
                        v.emplace_back(std::move(u.v));
                    });
                });
            });
        });
    }

    // For messaging_verb
    inline auto operator()(output_stream<char>& out, messaging_verb& v) {
        return ser_messaging_verb(out, v);
    }
    inline auto operator()(input_stream<char>& in, messaging_verb& v) {
        return des_messaging_verb(in, v);
    }

    // For sstring
    inline auto operator()(output_stream<char>& out, sstring& v) {
        return ser_sstring(out, v);
    }
    inline auto operator()(input_stream<char>& in, sstring& v) {
        return des_sstring(in, v);
    }

    // For frozen_mutation
    inline auto operator()(output_stream<char>& out, const frozen_mutation& v) {
        return ser_frozen_mutation(out, v);
    }
    inline auto operator()(output_stream<char>& out, frozen_mutation& v) {
        return operator()(out, const_cast<const frozen_mutation&>(v));
    }
    inline auto operator()(input_stream<char>& in, frozen_mutation& v) {
        return des_frozen_mutation(in, v);
    }

    // For complex types which have serialize()/deserialize(),  e.g. gms::gossip_digest_syn, gms::gossip_digest_ack2
    template<typename T>
    inline auto operator()(output_stream<char>& out, T&& v, std::enable_if_t<!std::is_integral<std::remove_reference_t<T>>::value &&
                                                                             !std::is_enum<std::remove_reference_t<T>>::value, void*> = nullptr) {
        auto sz = serialize_int32_size + v.serialized_size();
        bytes b(bytes::initialized_later(), sz);
        auto _out = b.begin();
        serialize_int32(_out, int32_t(sz - serialize_int32_size));
        v.serialize(_out);
        return out.write(reinterpret_cast<const char*>(b.c_str()), sz);
    }
    template<typename T>
    inline auto operator()(input_stream<char>& in, T& v, std::enable_if_t<!std::is_integral<T>::value &&
                                                                          !std::is_enum<T>::value, void*> = nullptr) {
        return in.read_exactly(serialize_int32_size).then([&in, &v] (temporary_buffer<char> buf) mutable {
            if (buf.size() != serialize_int32_size) {
                throw rpc::closed_error();
            }
            size_t sz = net::ntoh(*reinterpret_cast<const net::packed<int32_t>*>(buf.get()));
            return in.read_exactly(sz).then([sz, &v] (temporary_buffer<char> buf) mutable {
                if (buf.size() != sz) {
                    throw rpc::closed_error();
                }
                bytes_view bv(reinterpret_cast<const int8_t*>(buf.get()), sz);
                new (&v) T(T::deserialize(bv));
                assert(bv.size() == 0);
                return make_ready_future<>();
            });
        });
    }
};

class messaging_service {
public:
    // FIXME: messaging service versioning
    static constexpr int32_t current_version = 0;

    struct shard_id {
        gms::inet_address addr;
        uint32_t cpu_id;
        friend inline bool operator==(const shard_id& x, const shard_id& y) {
            return x.addr == y.addr && x.cpu_id == y.cpu_id ;
        }
        friend inline bool operator<(const shard_id& x, const shard_id& y) {
            if (x.addr < y.addr) {
                return true;
            } else if (y.addr < x.addr) {
                return false;
            } else {
                return x.cpu_id < y.cpu_id;
            }
        }
        friend inline std::ostream& operator<<(std::ostream& os, const shard_id& x) {
            return os << x.addr << ":" << x.cpu_id;
        }
        struct hash {
            size_t operator()(const shard_id& id) const {
                return std::hash<uint32_t>()(id.cpu_id) + std::hash<uint32_t>()(id.addr.raw_addr());
            }
        };
    };
    struct shard_info {
        shard_info(std::unique_ptr<rpc::protocol<serializer, messaging_verb>::client>&& client)
            : rpc_client(std::move(client)) {
        }
        std::unique_ptr<rpc::protocol<serializer, messaging_verb>::client> rpc_client;
    };

    void foreach_client(std::function<void(const messaging_service::shard_id& id,
                    const messaging_service::shard_info& info)> f) const {
        for (auto i = _clients.cbegin(); i != _clients.cend(); i++) {
            f(i->first, i->second);
        }
    }

    void increment_dropped_messages(messaging_verb verb) {
        _dropped_messages[static_cast<int32_t>(verb)]++;
    }

    uint64_t get_dropped_messages(messaging_verb verb) const {
        return _dropped_messages[static_cast<int32_t>(verb)];
    }

    const uint64_t* get_dropped_messages() const {
        return _dropped_messages;
    }

    int32_t get_raw_version(const gms::inet_address& endpoint) const {
        // FIXME: messaging service versioning
        return current_version;
    }

    bool knows_version(const gms::inet_address& endpoint) const {
        // FIXME: messaging service versioning
        return true;
    }

private:
    static constexpr uint16_t _default_port = 7000;
    gms::inet_address _listen_address;
    uint16_t _port;
    rpc::protocol<serializer, messaging_verb> _rpc;
    rpc::protocol<serializer, messaging_verb>::server _server;
    std::unordered_map<shard_id, shard_info, shard_id::hash> _clients;
    uint64_t _dropped_messages[static_cast<int32_t>(messaging_verb::LAST)] = {};
public:
    messaging_service(gms::inet_address ip = gms::inet_address("0.0.0.0"))
        : _listen_address(ip)
        , _port(_default_port)
        , _rpc(serializer{})
        , _server(_rpc, ipv4_addr{_listen_address.raw_addr(), _port}) {
    }
public:
    uint16_t port() {
        return _port;
    }
    auto listen_address() {
        return _listen_address;
    }
    future<> stop() {
        return when_all(_server.stop(),
            parallel_for_each(_clients, [](std::pair<const shard_id, shard_info>& c) {
                return c.second.rpc_client->stop();
            })
        ).discard_result();
    }

    static auto no_wait() {
        return rpc::no_wait;
    }
public:
    // Register a handler (a callback lambda) for verb
    template <typename Func>
    void register_handler(messaging_verb verb, Func&& func) {
        _rpc.register_handler(verb, std::move(func));
    }

    // Send a message for verb
    template <typename MsgIn, typename... MsgOut>
    auto send_message(messaging_verb verb, shard_id id, MsgOut&&... msg) {
        auto& rpc_client = get_rpc_client(id);
        auto rpc_handler = _rpc.make_client<MsgIn(MsgOut...)>(verb);
        return rpc_handler(rpc_client, std::forward<MsgOut>(msg)...).then_wrapped([this, id, verb] (auto&& f) {
            try {
                if (f.failed()) {
                    this->increment_dropped_messages(verb);
                    f.get();
                    assert(false); // never reached
                }
                return std::move(f);
            } catch(...) {
                // FIXME: we need to distinguish between a transport error and
                // a server error.
                // remove_rpc_client(id);
                throw;
            }
        });
    }

    template <typename... MsgOut>
    auto send_message_oneway(messaging_verb verb, shard_id id, MsgOut&&... msg) {
        return send_message<rpc::no_wait_type>(std::move(verb), std::move(id), std::forward<MsgOut>(msg)...);
    }
private:
    // Return rpc::protocol::client for a shard which is a ip + cpuid pair.
    rpc::protocol<serializer, messaging_verb>::client& get_rpc_client(shard_id id) {
        auto it = _clients.find(id);
        if (it == _clients.end()) {
            auto remote_addr = ipv4_addr(id.addr.raw_addr(), _port);
            auto client = std::make_unique<rpc::protocol<serializer, messaging_verb>::client>(_rpc, remote_addr, ipv4_addr{_listen_address.raw_addr(), 0});
            it = _clients.emplace(id, shard_info(std::move(client))).first;
            return *it->second.rpc_client;
        } else {
            return *it->second.rpc_client;
        }
    }

    void remove_rpc_client(shard_id id) {
        _clients.erase(id);
    }
};

extern distributed<messaging_service> _the_messaging_service;

inline distributed<messaging_service>& get_messaging_service() {
    return _the_messaging_service;
}

inline messaging_service& get_local_messaging_service() {
    return _the_messaging_service.local();
}

future<> init_messaging_service(sstring listen_address, db::config::seed_provider_type seed_provider);
} // namespace net
<|endoftext|>


<commit_before>// Filename: config_text.cxx
// Created by:  drose (02Mar00)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license.  You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////

#include "config_text.h"
#include "staticTextFont.h"
#include "dynamicTextFont.h"
#include "dynamicTextPage.h"
#include "textFont.h"
#include "textNode.h"

#include <dconfig.h>

Configure(config_text);
NotifyCategoryDef(text, "");

ConfigureFn(config_text) {
  StaticTextFont::init_type();
  DynamicTextFont::init_type();
  DynamicTextPage::init_type();
  TextFont::init_type();
  TextNode::init_type();
}

const bool flatten_text = config_text.GetBool("flatten-text", true);
<commit_msg>fix for build without freetype<commit_after>// Filename: config_text.cxx
// Created by:  drose (02Mar00)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license.  You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////

#include "config_text.h"
#include "staticTextFont.h"
#include "textFont.h"
#include "textNode.h"
#include "dynamicTextFont.h"
#include "dynamicTextPage.h"

#include <dconfig.h>

Configure(config_text);
NotifyCategoryDef(text, "");

ConfigureFn(config_text) {
  StaticTextFont::init_type();
  TextFont::init_type();
  TextNode::init_type();

#ifdef HAVE_FREETYPE
  DynamicTextFont::init_type();
  DynamicTextPage::init_type();
#endif
}

const bool flatten_text = config_text.GetBool("flatten-text", true);
<|endoftext|>


<commit_before>#include "filehandler.h"

/**
 * @brief FileHandler::FileHandler
 */
FileHandler::FileHandler()
{
    this->m_pid = 0; // zero out counter ids
    this->m_fid = 0;
    this->m_did = 0;
    this->lastError = 0;

}
/**
 * @brief FileHandler::create_project
 * creates project and associated files.
 * @param std::string name
 * @return Project* created project
 */
Project* FileHandler::create_project(std::string projName){
    Project* proj =  new Project(this->m_pid, projName);
    this->m_projects.insert(std::make_pair(proj->m_id, proj));
    save_project(proj);
    this->m_pid++;
    return proj;
}
/**
 * @brief FileHandler::create_directory
 * @param dirpath
 * @return unique directory ID
 */
ID FileHandler::create_directory(std::string dirpath){
    this->lastError = make_dir(dirpath); //varying implementation, OS dependant
    ID id = this->add_dir(dirpath);
    return id;
}

/**
 * @brief FileHandler::delete_directory
 * @param id
 * @return errorcode, if deletion was done code is 0;
 * otherwise see OS relevant directoryfile.
 */
FH_ERROR FileHandler::delete_directory(ID id){
    FH_ERROR err = remove_dir(this->get_dir(id)); //varying implementation, OS dependant
    return err;
}

/**
 * @todo unfinished, needs full project structure
 * and program to file parser to finish
 * @brief FileHandler::save_project
 * @param Project* name
 * Creates project and associated files.
 * @return void
 */
void FileHandler::save_project(Project* proj){
    std::string projFile = proj->m_name + std::string(".txt"); //filename
    if(!proj->saved){
        ID dirID = create_directory(std::string(WORKSPACE) + "/"+ proj->m_name);//project directory

        proj->files->dir = dirID;

        proj->files->f_proj = create_file(projFile, dirID); //create project file

        std::string vidFile = proj->m_name + "_videos.txt";
        proj->files->f_videos = create_file(vidFile, dirID); //create video file

        std::string analysisFile = proj->m_name + "_analyses.txt";
        proj->files->f_analysis = create_file(analysisFile, dirID); //create analysis file

        std::string drawingFile = proj->m_name + "_drawings.txt";
        proj->files->f_drawings =create_file(drawingFile, dirID); //create drawings file
    }
    update_proj_files(proj);

}
/**
 * @todo unfinished, will be released with parser
 * however, is needed for creating
 * @brief FileHandler::save_project
 * @param Project* name
 * Creates project and associated files.
 * @return void
 */
void FileHandler::update_proj_files(Project* proj){
    ProjectStream ps;
    ps << *proj;
    write_file(proj->files->f_proj, ps.projFile.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_videos, ps.videos.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_analysis, ps.analyzes.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_drawings, ps.drawings.str(), WRITE_OPTION::OVERWRITE);
}
void FileHandler::load_proj_files(std::string str){
    ID id;
    std::string filepath;
    std::stringstream sstr;
    sstr << str;
    //read files until empty
    while(sstr >> id >> filepath){
        add_file(id, filepath);
    }
}

/**
 * @todo load analyses (in project <</>> operators)
 * @todo load drawings (in project <</>> operators)
 * @brief FileHandler::load_project
 * @param projname
 * @param dirpath
 * @return project
 */
Project* FileHandler::load_project(std::string projname, std::string dirpath){
    Project* proj = new Project();
    ProjectStream projStream;

    proj->saved = true;
//    Read project file
    std::string projFilePath = dirpath + "/" + projname + ".txt";
    proj->files->f_proj = load_project_file(projFilePath, projStream.projFile);

//    Read video file
    std::string videoFilePath = dirpath + "/" + projname + "_videos.txt";
    proj->files->f_videos = load_project_file(videoFilePath, projStream.videos);
//    Read Analyzes
    std::string analysesFilePath = dirpath + "/" + projname + "_analyses.txt";
    proj->files->f_analysis = load_project_file(analysesFilePath, projStream.videos);
//    Read Drawings
    std::string drawingsFilePath = dirpath + "/" + projname + "_drawings.txt";
    proj->files->f_drawings = load_project_file(drawingsFilePath, projStream.drawings);
//    Read project from projstream
    projStream >> *proj;
    return proj;
}
ID FileHandler::load_project_file(std::string filePath, std::stringstream& projFileStream){
    std::string buf;
    ID projFileID = add_file(filePath);
    read_file(projFileID, buf);
    projFileStream << buf; // Read project name
    return projFileID;
}

/**
 * @brief FileHandler::delete_project
 * Deletes project, its associated files and contents.
 * OBS! This operation is as of now irreversible
 * @param Project*
 * @return FH_ERROR errorcode
 */
FH_ERROR FileHandler::delete_project(Project* proj){
    ProjFiles* pf = proj->files;
    delete_file(pf->f_proj);
    delete_file(pf->f_videos);
    delete_file(pf->f_analysis);
    delete_file(pf->f_drawings);
    return delete_directory(proj->files->dir);

}
/**
 * @todo make threadsafe
 * @brief FileHandler::add_video
 * @param Project*,string filepath
 *
 * string
 * Add a video filepath to a given project.
 * Creates Video object which is accessed further by returned id.
 */
void FileHandler::add_video(Project* proj, std::string filePath){
    Video* v = new Video(filePath);
    proj->add_video(v);
    this->add_file(filePath);
}

 /**
  * @brief FileHandler::create_file
  * create a file by given name in already excisting
  * application tracked directory
  * @param std::string file name, ID directory id
  */

ID FileHandler::create_file(std::string filename, ID dirID){
    std::ofstream f;
    std::string filePath = this->get_dir(dirID)+"/"+filename;
    f.open(filePath.c_str());
    return this->add_file(filePath);
  }
/**
 * @todo make threadsafe
 * @brief FileHandler::delete_file
 * delete application tracked file
 * @param ID file id
 */
 FH_ERROR FileHandler::delete_file(ID id){
    std::string file = this->get_file(id);
    return std::remove(file.c_str());
 }
 /**
  * @todo make threadsafe
  * @brief FileHandler::write_file
  *  Write given text to an application tracked file
  * @param ID file id, std::string text
  * @return void
  */
 void FileHandler::write_file(ID id, std::string text, WRITE_OPTION opt){
    std::string fileName = this->get_file(id);
    std::ofstream f;
    switch(opt){
    case WRITE_OPTION::OVERWRITE:
        f.open(fileName.c_str(), std::ios::in | std::ios::out);
        break;
    case WRITE_OPTION::APPEND:
        f.open(fileName.c_str(), std::ios::in | std::ios::out | std::ios::ate);
        break;
    default:
        return; // no open file
        break;
    }
    if(f.is_open()) f << text.c_str() << std::endl;
 }

 /**
  * @brief FileHandler::read_file
  *  Read given lenght of lines to buffer from application
  *  tracked file. OBS! If number of lines exceeds =>
  *  reads to end of file (EOF)
  *  @param ID file id, std::string text
  *  @return voi
  */
 void FileHandler::read_file(ID id,  std::string& buf, int linesToRead){
     std::ifstream f(this->get_file(id));
     std::string temp;
     if(f.is_open()){
         while(linesToRead-- && std::getline(f, temp)){
            buf += temp;
         }
     }
 }
 /**
  * @brief FileHandler::get_project
  * Getter
  * @param ID project id
  * @return Project*
  */
 Project* FileHandler::get_project(ID pid){
    this->projMapLock.lock();
    Project* p = this->m_projects.at(pid);
    this->projMapLock.unlock();
    return p;
 }
 /**
  * @brief FileHandler::get_file
  * Getter
  * @param ID project file id
  * @return std::string filepath
  */
 std::string FileHandler::get_file(ID id){
    this->fileMapLock.lock();
    std::string file = this->m_fileMap.at(id);
    this->fileMapLock.unlock();
    return file;
 }
 /**
  * @brief FileHandler::get_dir
  * @param ID directory id
  * @return directory path
  */
 std::string FileHandler::get_dir(ID id){
    this->dirMapLock.lock();
    std::string dir = this->m_dirMap.at(id);
    this->dirMapLock.unlock();
    return dir;
 }

 /**
  * @brief FileHandler::add_projectr
  * @param std::pari<<ID, Project*> pair
  * @return void
  */
 void FileHandler::add_project(std::pair<ID,Project*> pair){
    this->projMapLock.lock();
    this->m_projects.insert(pair);
    this->projMapLock.unlock();

 }
 /**
  * @brief FileHandler::add_file
  * @param std::string filepath
  * @return unique file identifier
  */
ID FileHandler::add_file(std::string filepath){
    add_file(this->m_fid, filepath);
    return this->m_fid++;
 }
/**
 * @brief FileHandler::add_file
 * @param id
 * @param filepath
 */
void FileHandler::add_file(ID id ,std::string filepath){
    std::pair<ID,std::string> pair = std::make_pair(id, filepath);
    this->fileMapLock.lock();
    this->m_fileMap.insert(pair);
    this->fileMapLock.unlock();
}
 /**
  * @brief FileHandler::add_dir
  * @param std::string dirpath
  * @return unique directory identifier
  */
ID FileHandler::add_dir(std::string dirpath){
    std::pair<ID,std::string> pair = std::make_pair(this->m_did, dirpath);
    this->dirMapLock.lock();
    this->m_dirMap.insert(pair);
    this->dirMapLock.unlock();
    return this->m_did++;
 }

<commit_msg>properly set project id on load<commit_after>#include "filehandler.h"

/**
 * @brief FileHandler::FileHandler
 */
FileHandler::FileHandler()
{
    this->m_pid = 0; // zero out counter ids
    this->m_fid = 0;
    this->m_did = 0;
    this->lastError = 0;

}
/**
 * @brief FileHandler::create_project
 * creates project and associated files.
 * @param std::string name
 * @return Project* created project
 */
Project* FileHandler::create_project(std::string projName){
    Project* proj =  new Project(this->m_pid, projName);
    this->m_projects.insert(std::make_pair(proj->m_id, proj));
    save_project(proj);
    this->m_pid++;
    return proj;
}
/**
 * @brief FileHandler::create_directory
 * @param dirpath
 * @return unique directory ID
 */
ID FileHandler::create_directory(std::string dirpath){
    this->lastError = make_dir(dirpath); //varying implementation, OS dependant
    ID id = this->add_dir(dirpath);
    return id;
}

/**
 * @brief FileHandler::delete_directory
 * @param id
 * @return errorcode, if deletion was done code is 0;
 * otherwise see OS relevant directoryfile.
 */
FH_ERROR FileHandler::delete_directory(ID id){
    FH_ERROR err = remove_dir(this->get_dir(id)); //varying implementation, OS dependant
    return err;
}

/**
 * @todo unfinished, needs full project structure
 * and program to file parser to finish
 * @brief FileHandler::save_project
 * @param Project* name
 * Creates project and associated files.
 * @return void
 */
void FileHandler::save_project(Project* proj){
    std::string projFile = proj->m_name + std::string(".txt"); //filename
    if(!proj->saved){
        ID dirID = create_directory(std::string(WORKSPACE) + "/"+ proj->m_name);//project directory

        proj->files->dir = dirID;

        proj->files->f_proj = create_file(projFile, dirID); //create project file

        std::string vidFile = proj->m_name + "_videos.txt";
        proj->files->f_videos = create_file(vidFile, dirID); //create video file

        std::string analysisFile = proj->m_name + "_analyses.txt";
        proj->files->f_analysis = create_file(analysisFile, dirID); //create analysis file

        std::string drawingFile = proj->m_name + "_drawings.txt";
        proj->files->f_drawings =create_file(drawingFile, dirID); //create drawings file
    }
    update_proj_files(proj);

}
/**
 * @todo unfinished, will be released with parser
 * however, is needed for creating
 * @brief FileHandler::save_project
 * @param Project* name
 * Creates project and associated files.
 * @return void
 */
void FileHandler::update_proj_files(Project* proj){
    ProjectStream ps;
    ps << *proj;
    write_file(proj->files->f_proj, ps.projFile.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_videos, ps.videos.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_analysis, ps.analyzes.str(), WRITE_OPTION::OVERWRITE);
    write_file(proj->files->f_drawings, ps.drawings.str(), WRITE_OPTION::OVERWRITE);
}
void FileHandler::load_proj_files(std::string str){
    ID id;
    std::string filepath;
    std::stringstream sstr;
    sstr << str;
    //read files until empty
    while(sstr >> id >> filepath){
        add_file(id, filepath);
    }
}

/**
 * @todo load analyses (in project <</>> operators)
 * @todo load drawings (in project <</>> operators)
 * @brief FileHandler::load_project
 * @param projname
 * @param dirpath
 * @return project
 */
Project* FileHandler::load_project(std::string projname, std::string dirpath){
    Project* proj = new Project();
    add_project(std::make_pair(this->m_pid++, proj));
    ProjectStream projStream;

    proj->saved = true;
//    Read project file
    std::string projFilePath = dirpath + "/" + projname + ".txt";
    proj->files->f_proj = load_project_file(projFilePath, projStream.projFile);

//    Read video file
    std::string videoFilePath = dirpath + "/" + projname + "_videos.txt";
    proj->files->f_videos = load_project_file(videoFilePath, projStream.videos);
//    Read Analyzes
    std::string analysesFilePath = dirpath + "/" + projname + "_analyses.txt";
    proj->files->f_analysis = load_project_file(analysesFilePath, projStream.videos);
//    Read Drawings
    std::string drawingsFilePath = dirpath + "/" + projname + "_drawings.txt";
    proj->files->f_drawings = load_project_file(drawingsFilePath, projStream.drawings);
//    Read project from projstream
    projStream >> *proj;
    return proj;
}
ID FileHandler::load_project_file(std::string filePath, std::stringstream& projFileStream){
    std::string buf;
    ID projFileID = add_file(filePath);
    read_file(projFileID, buf);
    projFileStream << buf; // Read project name
    return projFileID;
}

/**
 * @brief FileHandler::delete_project
 * Deletes project, its associated files and contents.
 * OBS! This operation is as of now irreversible
 * @param Project*
 * @return FH_ERROR errorcode
 */
FH_ERROR FileHandler::delete_project(Project* proj){
    ProjFiles* pf = proj->files;
    delete_file(pf->f_proj);
    delete_file(pf->f_videos);
    delete_file(pf->f_analysis);
    delete_file(pf->f_drawings);
    return delete_directory(proj->files->dir);

}
/**
 * @todo make threadsafe
 * @brief FileHandler::add_video
 * @param Project*,string filepath
 *
 * string
 * Add a video filepath to a given project.
 * Creates Video object which is accessed further by returned id.
 */
void FileHandler::add_video(Project* proj, std::string filePath){
    Video* v = new Video(filePath);
    proj->add_video(v);
    this->add_file(filePath);
}

 /**
  * @brief FileHandler::create_file
  * create a file by given name in already excisting
  * application tracked directory
  * @param std::string file name, ID directory id
  */

ID FileHandler::create_file(std::string filename, ID dirID){
    std::ofstream f;
    std::string filePath = this->get_dir(dirID)+"/"+filename;
    f.open(filePath.c_str());
    return this->add_file(filePath);
  }
/**
 * @todo make threadsafe
 * @brief FileHandler::delete_file
 * delete application tracked file
 * @param ID file id
 */
 FH_ERROR FileHandler::delete_file(ID id){
    std::string file = this->get_file(id);
    return std::remove(file.c_str());
 }
 /**
  * @todo make threadsafe
  * @brief FileHandler::write_file
  *  Write given text to an application tracked file
  * @param ID file id, std::string text
  * @return void
  */
 void FileHandler::write_file(ID id, std::string text, WRITE_OPTION opt){
    std::string fileName = this->get_file(id);
    std::ofstream f;
    switch(opt){
    case WRITE_OPTION::OVERWRITE:
        f.open(fileName.c_str(), std::ios::in | std::ios::out);
        break;
    case WRITE_OPTION::APPEND:
        f.open(fileName.c_str(), std::ios::in | std::ios::out | std::ios::ate);
        break;
    default:
        return; // no open file
        break;
    }
    if(f.is_open()) f << text.c_str() << std::endl;
 }

 /**
  * @brief FileHandler::read_file
  *  Read given lenght of lines to buffer from application
  *  tracked file. OBS! If number of lines exceeds =>
  *  reads to end of file (EOF)
  *  @param ID file id, std::string text
  *  @return voi
  */
 void FileHandler::read_file(ID id,  std::string& buf, int linesToRead){
     std::ifstream f(this->get_file(id));
     std::string temp;
     if(f.is_open()){
         while(linesToRead-- && std::getline(f, temp)){
            buf += temp;
         }
     }
 }
 /**
  * @brief FileHandler::get_project
  * Getter
  * @param ID project id
  * @return Project*
  */
 Project* FileHandler::get_project(ID pid){
    this->projMapLock.lock();
    Project* p = this->m_projects.at(pid);
    this->projMapLock.unlock();
    return p;
 }
 /**
  * @brief FileHandler::get_file
  * Getter
  * @param ID project file id
  * @return std::string filepath
  */
 std::string FileHandler::get_file(ID id){
    this->fileMapLock.lock();
    std::string file = this->m_fileMap.at(id);
    this->fileMapLock.unlock();
    return file;
 }
 /**
  * @brief FileHandler::get_dir
  * @param ID directory id
  * @return directory path
  */
 std::string FileHandler::get_dir(ID id){
    this->dirMapLock.lock();
    std::string dir = this->m_dirMap.at(id);
    this->dirMapLock.unlock();
    return dir;
 }

 /**
  * @brief FileHandler::add_projectr
  * @param std::pari<<ID, Project*> pair
  * @return void
  */
 void FileHandler::add_project(std::pair<ID,Project*> pair){
    this->projMapLock.lock();
    this->m_projects.insert(pair);
    this->projMapLock.unlock();

 }
 /**
  * @brief FileHandler::add_file
  * @param std::string filepath
  * @return unique file identifier
  */
ID FileHandler::add_file(std::string filepath){
    add_file(this->m_fid, filepath);
    return this->m_fid++;
 }
/**
 * @brief FileHandler::add_file
 * @param id
 * @param filepath
 */
void FileHandler::add_file(ID id ,std::string filepath){
    std::pair<ID,std::string> pair = std::make_pair(id, filepath);
    this->fileMapLock.lock();
    this->m_fileMap.insert(pair);
    this->fileMapLock.unlock();
}
 /**
  * @brief FileHandler::add_dir
  * @param std::string dirpath
  * @return unique directory identifier
  */
ID FileHandler::add_dir(std::string dirpath){
    std::pair<ID,std::string> pair = std::make_pair(this->m_did, dirpath);
    this->dirMapLock.lock();
    this->m_dirMap.insert(pair);
    this->dirMapLock.unlock();
    return this->m_did++;
 }

<|endoftext|>


<commit_before>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// version: $Id$
// author:  Vassil Vassilev <vvasilev@cern.ch>
//------------------------------------------------------------------------------

#include "cling/Interpreter/InterpreterCallbacks.h"

#include "cling/Interpreter/Interpreter.h"

#include "clang/Frontend/CompilerInstance.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Sema.h"
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ASTDeserializationListener.h"

using namespace clang;

namespace cling {

  ///\brief Translates 'interesting' for the interpreter
  /// ASTDeserializationListener events into interpreter callback.
  ///
  class InterpreterPPCallbacks : public PPCallbacks {
  private:
    cling::InterpreterCallbacks* m_Callbacks;
  public:
    InterpreterPPCallbacks(InterpreterCallbacks* C) : m_Callbacks(C) { }
    ~InterpreterPPCallbacks() { }

    virtual bool FileNotFound(llvm::StringRef FileName,
                              llvm::SmallVectorImpl<char>& RecoveryPath) {
      if (m_Callbacks)
        return m_Callbacks->FileNotFound(FileName, RecoveryPath);
      // Returning true would mean that the preprocessor should try to recover.
      return false;
    }
  };

  ///\brief Translates 'interesting' for the interpreter
  /// ASTDeserializationListener events into interpreter callback.
  ///
  class InterpreterDeserializationListener : public ASTDeserializationListener {
  private:
    cling::InterpreterCallbacks* m_Callbacks;
  public:
    InterpreterDeserializationListener(InterpreterCallbacks* C)
      : m_Callbacks(C) {}

    virtual void DeclRead(serialization::DeclID, const Decl *D) {
      if (m_Callbacks)
        m_Callbacks->DeclDeserialized(D);
    }
    virtual void TypeRead(serialization::TypeIdx, QualType T) {
      if (m_Callbacks)
        m_Callbacks->TypeDeserialized(T.getTypePtr());
    }
  };

  ///\brief Translates 'interesting' for the interpreter ExternalSemaSource
  /// events into interpreter callbacks.
  ///
  class InterpreterExternalSemaSource : public clang::ExternalSemaSource {
  protected:
    ///\brief The interpreter callback which are subscribed for the events.
    ///
    /// Usually the callbacks is the owner of the class and the interpreter owns
    /// the callbacks so they can't be out of sync. Eg we notifying the wrong
    /// callback class.
    ///
    InterpreterCallbacks* m_Callbacks; // we don't own it.

  public:
    InterpreterExternalSemaSource(InterpreterCallbacks* C) : m_Callbacks(C){}

    ~InterpreterExternalSemaSource() {}

    InterpreterCallbacks* getCallbacks() const { return m_Callbacks; }

    /// \brief Provides last resort lookup for failed unqualified lookups.
    ///
    /// This gets translated into InterpreterCallback's call.
    ///
    ///\param[out] R The recovered symbol.
    ///\param[in] S The scope in which the lookup failed.
    ///
    ///\returns true if a suitable declaration is found.
    ///
    virtual bool LookupUnqualified(clang::LookupResult& R, clang::Scope* S) {
      if (m_Callbacks)
        return m_Callbacks->LookupObject(R, S);

      return false;
    }

    virtual bool FindExternalVisibleDeclsByName(const clang::DeclContext* DC,
                                                clang::DeclarationName Name) {
      if (m_Callbacks)
        return m_Callbacks->LookupObject(DC, Name);

      return false;
    }
    // Silence warning virtual function was hidden.
    using ExternalASTSource::CompleteType(clang::ObjCInterfaceDecl*);
    virtual void CompleteType(TagDecl* Tag) {
      if (m_Callbacks)
        m_Callbacks->LookupObject(Tag);
    }

    void UpdateWithNewDeclsFwd(const DeclContext *DC, DeclarationName Name,
                               llvm::ArrayRef<NamedDecl*> Decls) {
      SetExternalVisibleDeclsForName(DC, Name, Decls);
    }
  };

  InterpreterCallbacks::InterpreterCallbacks(Interpreter* interp,
                                            InterpreterExternalSemaSource* IESS,
                                        InterpreterDeserializationListener* IDL,
                                             InterpreterPPCallbacks* IPPC)
    : m_Interpreter(interp),  m_ExternalSemaSource(IESS),
      m_DeserializationListener(IDL), m_IsRuntime(false) {
    if (IESS)
      m_Interpreter->getSema().addExternalSource(m_ExternalSemaSource.get());
    ASTReader* Reader = m_Interpreter->getCI()->getModuleManager();
    if (IDL && Reader)
      Reader->setDeserializationListener(IDL);
    if (IPPC)
      m_Interpreter->getCI()->getPreprocessor().addPPCallbacks(IPPC);
  }

  InterpreterCallbacks::InterpreterCallbacks(Interpreter* interp,
                             bool enableExternalSemaSourceCallbacks/* = false*/,
                        bool enableDeserializationListenerCallbacks/* = false*/,
                                             bool enablePPCallbacks/* = false*/)
    : m_Interpreter(interp), m_IsRuntime(false) {

    if (enableExternalSemaSourceCallbacks) {
      m_ExternalSemaSource.reset(new InterpreterExternalSemaSource(this));
      m_Interpreter->getSema().addExternalSource(m_ExternalSemaSource.get());
    }

    ASTReader* Reader = m_Interpreter->getCI()->getModuleManager();
    if (enableDeserializationListenerCallbacks && Reader) {
      // FIXME: need to create a multiplexer if a DeserializationListener is
      // alreday present.
      m_DeserializationListener.
        reset(new InterpreterDeserializationListener(this));
      Reader->setDeserializationListener(m_DeserializationListener.get());
    }

    if (enablePPCallbacks) {
      m_PPCallbacks.reset(new InterpreterPPCallbacks(this));
      Preprocessor& PP = m_Interpreter->getCI()->getPreprocessor();
      PP.addPPCallbacks(m_PPCallbacks.get());
    }
  }

  // pin the vtable here
  InterpreterCallbacks::~InterpreterCallbacks() {
    // FIXME: we have to remove the external source at destruction time. Needs
    // further tweaks of the patch in clang. This will be done later once the
    // patch is in clang's mainline.
  }

  ExternalSemaSource*
  InterpreterCallbacks::getInterpreterExternalSemaSource() const {
    return m_ExternalSemaSource.get();
  }

  ASTDeserializationListener*
  InterpreterCallbacks::getInterpreterDeserializationListener() const {
    return m_DeserializationListener.get();
  }

  bool InterpreterCallbacks::FileNotFound(llvm::StringRef FileName,
                                    llvm::SmallVectorImpl<char>& RecoveryPath) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(LookupResult&, Scope*) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(const DeclContext*, DeclarationName) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(TagDecl*) {
    // Default implementation is no op.
    return false;
  }

  void InterpreterCallbacks::UpdateWithNewDecls(const DeclContext *DC,
                                                DeclarationName Name,
                                             llvm::ArrayRef<NamedDecl*> Decls) {
    if (m_ExternalSemaSource)
      m_ExternalSemaSource->UpdateWithNewDeclsFwd(DC, Name, Decls);
  }
} // end namespace cling

// TODO: Make the build system in the testsuite aware how to build that class
// and extract it out there again.
#include "DynamicLookup.h"
#include "cling/Utils/AST.h"

#include "clang/AST/ASTContext.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Lookup.h"
#include "clang/Sema/Scope.h"
namespace cling {
namespace test {
  TestProxy* Tester = 0;

  extern "C" int printf(const char* fmt, ...);
  TestProxy::TestProxy(){}
  int TestProxy::Draw(){ return 12; }
  const char* TestProxy::getVersion(){ return "Interpreter.cpp"; }

  int TestProxy::Add10(int num) { return num + 10;}

  int TestProxy::Add(int a, int b) {
    return a + b;
  }

  void TestProxy::PrintString(std::string s) { printf("%s\n", s.c_str()); }

  bool TestProxy::PrintArray(int a[], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      printf("%i", a[i]);

    printf("%s", "\n");

    return true;
  }

  void TestProxy::PrintArray(float a[][5], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      for (unsigned j = 0; j < 5; ++j)
        printf("%i", (int)a[i][j]);

    printf("%s", "\n");
  }

  void TestProxy::PrintArray(int a[][4][5], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      for (unsigned j = 0; j < 4; ++j)
        for (unsigned k = 0; k < 5; ++k)
          printf("%i", a[i][j][k]);

    printf("%s", "\n");
  }

  SymbolResolverCallback::SymbolResolverCallback(Interpreter* interp)
    : InterpreterCallbacks(interp, /*enableExternalSemaSourceCallbacks*/true),
      m_TesterDecl(0) {
    m_Interpreter->process("cling::test::Tester = new cling::test::TestProxy();");
  }

  SymbolResolverCallback::~SymbolResolverCallback() { }

  bool SymbolResolverCallback::LookupObject(LookupResult& R, Scope* S) {
    if (m_IsRuntime) {
      // Only for demo resolve all unknown objects to cling::test::Tester
      if (!m_TesterDecl) {
        clang::Sema& SemaR = m_Interpreter->getSema();
        clang::NamespaceDecl* NSD = utils::Lookup::Namespace(&SemaR, "cling");
        NSD = utils::Lookup::Namespace(&SemaR, "test", NSD);
        m_TesterDecl = utils::Lookup::Named(&SemaR, "Tester", NSD);
      }
      assert (m_TesterDecl && "Tester not found!");
      R.addDecl(m_TesterDecl);
      return true; // Tell clang to continue.
    }

    if (ShouldResolveAtRuntime(R, S)) {
      ASTContext& C = R.getSema().getASTContext();
      DeclContext* DC = 0;
      // For DeclContext-less scopes like if (dyn_expr) {}
      while (!DC) {
        DC = static_cast<DeclContext*>(S->getEntity());
        S = S->getParent();
      }
      DeclarationName Name = R.getLookupName();
      IdentifierInfo* II = Name.getAsIdentifierInfo();
      SourceLocation Loc = R.getNameLoc();
      VarDecl* Res = VarDecl::Create(C, DC, Loc, Loc, II, C.DependentTy,
                                     /*TypeSourceInfo*/0, SC_None);

      // Annotate the decl to give a hint in cling. FIXME: Current implementation
      // is a gross hack, because TClingCallbacks shouldn't know about
      // EvaluateTSynthesizer at all!
      SourceRange invalidRange;
      Res->addAttr(new (C) AnnotateAttr(invalidRange, C, "__ResolveAtRuntime"));
      R.addDecl(Res);
      DC->addDecl(Res);
      // Say that we can handle the situation. Clang should try to recover
      return true;
    }

    return false;
  }

  bool SymbolResolverCallback::ShouldResolveAtRuntime(LookupResult& R,
                                                      Scope* S) {

    if (R.getLookupKind() != Sema::LookupOrdinaryName)
      return false;

    if (R.isForRedeclaration())
      return false;

    if (!R.empty())
      return false;

    // FIXME: Figure out better way to handle:
    // C++ [basic.lookup.classref]p1:
    //   In a class member access expression (5.2.5), if the . or -> token is
    //   immediately followed by an identifier followed by a <, the
    //   identifier must be looked up to determine whether the < is the
    //   beginning of a template argument list (14.2) or a less-than operator.
    //   The identifier is first looked up in the class of the object
    //   expression. If the identifier is not found, it is then looked up in
    //   the context of the entire postfix-expression and shall name a class
    //   or function template.
    //
    // We want to ignore object(.|->)member<template>
    if (R.getSema().PP.LookAhead(0).getKind() == tok::less)
      // TODO: check for . or -> in the cached token stream
      return false;

    for (Scope* DepScope = S; DepScope; DepScope = DepScope->getParent()) {
      if (DeclContext* Ctx = static_cast<DeclContext*>(DepScope->getEntity())) {
        if (!Ctx->isDependentContext())
          // For now we support only the prompt.
          if (isa<FunctionDecl>(Ctx))
            return true;
      }
    }

    return false;
  }

} // end test
} // end cling
<commit_msg>Spell it syntactically correct.<commit_after>//------------------------------------------------------------------------------
// CLING - the C++ LLVM-based InterpreterG :)
// version: $Id$
// author:  Vassil Vassilev <vvasilev@cern.ch>
//------------------------------------------------------------------------------

#include "cling/Interpreter/InterpreterCallbacks.h"

#include "cling/Interpreter/Interpreter.h"

#include "clang/Frontend/CompilerInstance.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Sema.h"
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ASTDeserializationListener.h"

using namespace clang;

namespace cling {

  ///\brief Translates 'interesting' for the interpreter
  /// ASTDeserializationListener events into interpreter callback.
  ///
  class InterpreterPPCallbacks : public PPCallbacks {
  private:
    cling::InterpreterCallbacks* m_Callbacks;
  public:
    InterpreterPPCallbacks(InterpreterCallbacks* C) : m_Callbacks(C) { }
    ~InterpreterPPCallbacks() { }

    virtual bool FileNotFound(llvm::StringRef FileName,
                              llvm::SmallVectorImpl<char>& RecoveryPath) {
      if (m_Callbacks)
        return m_Callbacks->FileNotFound(FileName, RecoveryPath);
      // Returning true would mean that the preprocessor should try to recover.
      return false;
    }
  };

  ///\brief Translates 'interesting' for the interpreter
  /// ASTDeserializationListener events into interpreter callback.
  ///
  class InterpreterDeserializationListener : public ASTDeserializationListener {
  private:
    cling::InterpreterCallbacks* m_Callbacks;
  public:
    InterpreterDeserializationListener(InterpreterCallbacks* C)
      : m_Callbacks(C) {}

    virtual void DeclRead(serialization::DeclID, const Decl *D) {
      if (m_Callbacks)
        m_Callbacks->DeclDeserialized(D);
    }
    virtual void TypeRead(serialization::TypeIdx, QualType T) {
      if (m_Callbacks)
        m_Callbacks->TypeDeserialized(T.getTypePtr());
    }
  };

  ///\brief Translates 'interesting' for the interpreter ExternalSemaSource
  /// events into interpreter callbacks.
  ///
  class InterpreterExternalSemaSource : public clang::ExternalSemaSource {
  protected:
    ///\brief The interpreter callback which are subscribed for the events.
    ///
    /// Usually the callbacks is the owner of the class and the interpreter owns
    /// the callbacks so they can't be out of sync. Eg we notifying the wrong
    /// callback class.
    ///
    InterpreterCallbacks* m_Callbacks; // we don't own it.

  public:
    InterpreterExternalSemaSource(InterpreterCallbacks* C) : m_Callbacks(C){}

    ~InterpreterExternalSemaSource() {}

    InterpreterCallbacks* getCallbacks() const { return m_Callbacks; }

    /// \brief Provides last resort lookup for failed unqualified lookups.
    ///
    /// This gets translated into InterpreterCallback's call.
    ///
    ///\param[out] R The recovered symbol.
    ///\param[in] S The scope in which the lookup failed.
    ///
    ///\returns true if a suitable declaration is found.
    ///
    virtual bool LookupUnqualified(clang::LookupResult& R, clang::Scope* S) {
      if (m_Callbacks)
        return m_Callbacks->LookupObject(R, S);

      return false;
    }

    virtual bool FindExternalVisibleDeclsByName(const clang::DeclContext* DC,
                                                clang::DeclarationName Name) {
      if (m_Callbacks)
        return m_Callbacks->LookupObject(DC, Name);

      return false;
    }
    // Silence warning virtual function was hidden.
    using ExternalASTSource::CompleteType;
    virtual void CompleteType(TagDecl* Tag) {
      if (m_Callbacks)
        m_Callbacks->LookupObject(Tag);
    }

    void UpdateWithNewDeclsFwd(const DeclContext *DC, DeclarationName Name,
                               llvm::ArrayRef<NamedDecl*> Decls) {
      SetExternalVisibleDeclsForName(DC, Name, Decls);
    }
  };

  InterpreterCallbacks::InterpreterCallbacks(Interpreter* interp,
                                            InterpreterExternalSemaSource* IESS,
                                        InterpreterDeserializationListener* IDL,
                                             InterpreterPPCallbacks* IPPC)
    : m_Interpreter(interp),  m_ExternalSemaSource(IESS),
      m_DeserializationListener(IDL), m_IsRuntime(false) {
    if (IESS)
      m_Interpreter->getSema().addExternalSource(m_ExternalSemaSource.get());
    ASTReader* Reader = m_Interpreter->getCI()->getModuleManager();
    if (IDL && Reader)
      Reader->setDeserializationListener(IDL);
    if (IPPC)
      m_Interpreter->getCI()->getPreprocessor().addPPCallbacks(IPPC);
  }

  InterpreterCallbacks::InterpreterCallbacks(Interpreter* interp,
                             bool enableExternalSemaSourceCallbacks/* = false*/,
                        bool enableDeserializationListenerCallbacks/* = false*/,
                                             bool enablePPCallbacks/* = false*/)
    : m_Interpreter(interp), m_IsRuntime(false) {

    if (enableExternalSemaSourceCallbacks) {
      m_ExternalSemaSource.reset(new InterpreterExternalSemaSource(this));
      m_Interpreter->getSema().addExternalSource(m_ExternalSemaSource.get());
    }

    ASTReader* Reader = m_Interpreter->getCI()->getModuleManager();
    if (enableDeserializationListenerCallbacks && Reader) {
      // FIXME: need to create a multiplexer if a DeserializationListener is
      // alreday present.
      m_DeserializationListener.
        reset(new InterpreterDeserializationListener(this));
      Reader->setDeserializationListener(m_DeserializationListener.get());
    }

    if (enablePPCallbacks) {
      m_PPCallbacks.reset(new InterpreterPPCallbacks(this));
      Preprocessor& PP = m_Interpreter->getCI()->getPreprocessor();
      PP.addPPCallbacks(m_PPCallbacks.get());
    }
  }

  // pin the vtable here
  InterpreterCallbacks::~InterpreterCallbacks() {
    // FIXME: we have to remove the external source at destruction time. Needs
    // further tweaks of the patch in clang. This will be done later once the
    // patch is in clang's mainline.
  }

  ExternalSemaSource*
  InterpreterCallbacks::getInterpreterExternalSemaSource() const {
    return m_ExternalSemaSource.get();
  }

  ASTDeserializationListener*
  InterpreterCallbacks::getInterpreterDeserializationListener() const {
    return m_DeserializationListener.get();
  }

  bool InterpreterCallbacks::FileNotFound(llvm::StringRef FileName,
                                    llvm::SmallVectorImpl<char>& RecoveryPath) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(LookupResult&, Scope*) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(const DeclContext*, DeclarationName) {
    // Default implementation is no op.
    return false;
  }

  bool InterpreterCallbacks::LookupObject(TagDecl*) {
    // Default implementation is no op.
    return false;
  }

  void InterpreterCallbacks::UpdateWithNewDecls(const DeclContext *DC,
                                                DeclarationName Name,
                                             llvm::ArrayRef<NamedDecl*> Decls) {
    if (m_ExternalSemaSource)
      m_ExternalSemaSource->UpdateWithNewDeclsFwd(DC, Name, Decls);
  }
} // end namespace cling

// TODO: Make the build system in the testsuite aware how to build that class
// and extract it out there again.
#include "DynamicLookup.h"
#include "cling/Utils/AST.h"

#include "clang/AST/ASTContext.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Lookup.h"
#include "clang/Sema/Scope.h"
namespace cling {
namespace test {
  TestProxy* Tester = 0;

  extern "C" int printf(const char* fmt, ...);
  TestProxy::TestProxy(){}
  int TestProxy::Draw(){ return 12; }
  const char* TestProxy::getVersion(){ return "Interpreter.cpp"; }

  int TestProxy::Add10(int num) { return num + 10;}

  int TestProxy::Add(int a, int b) {
    return a + b;
  }

  void TestProxy::PrintString(std::string s) { printf("%s\n", s.c_str()); }

  bool TestProxy::PrintArray(int a[], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      printf("%i", a[i]);

    printf("%s", "\n");

    return true;
  }

  void TestProxy::PrintArray(float a[][5], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      for (unsigned j = 0; j < 5; ++j)
        printf("%i", (int)a[i][j]);

    printf("%s", "\n");
  }

  void TestProxy::PrintArray(int a[][4][5], size_t size) {
    for (unsigned i = 0; i < size; ++i)
      for (unsigned j = 0; j < 4; ++j)
        for (unsigned k = 0; k < 5; ++k)
          printf("%i", a[i][j][k]);

    printf("%s", "\n");
  }

  SymbolResolverCallback::SymbolResolverCallback(Interpreter* interp)
    : InterpreterCallbacks(interp, /*enableExternalSemaSourceCallbacks*/true),
      m_TesterDecl(0) {
    m_Interpreter->process("cling::test::Tester = new cling::test::TestProxy();");
  }

  SymbolResolverCallback::~SymbolResolverCallback() { }

  bool SymbolResolverCallback::LookupObject(LookupResult& R, Scope* S) {
    if (m_IsRuntime) {
      // Only for demo resolve all unknown objects to cling::test::Tester
      if (!m_TesterDecl) {
        clang::Sema& SemaR = m_Interpreter->getSema();
        clang::NamespaceDecl* NSD = utils::Lookup::Namespace(&SemaR, "cling");
        NSD = utils::Lookup::Namespace(&SemaR, "test", NSD);
        m_TesterDecl = utils::Lookup::Named(&SemaR, "Tester", NSD);
      }
      assert (m_TesterDecl && "Tester not found!");
      R.addDecl(m_TesterDecl);
      return true; // Tell clang to continue.
    }

    if (ShouldResolveAtRuntime(R, S)) {
      ASTContext& C = R.getSema().getASTContext();
      DeclContext* DC = 0;
      // For DeclContext-less scopes like if (dyn_expr) {}
      while (!DC) {
        DC = static_cast<DeclContext*>(S->getEntity());
        S = S->getParent();
      }
      DeclarationName Name = R.getLookupName();
      IdentifierInfo* II = Name.getAsIdentifierInfo();
      SourceLocation Loc = R.getNameLoc();
      VarDecl* Res = VarDecl::Create(C, DC, Loc, Loc, II, C.DependentTy,
                                     /*TypeSourceInfo*/0, SC_None);

      // Annotate the decl to give a hint in cling. FIXME: Current implementation
      // is a gross hack, because TClingCallbacks shouldn't know about
      // EvaluateTSynthesizer at all!
      SourceRange invalidRange;
      Res->addAttr(new (C) AnnotateAttr(invalidRange, C, "__ResolveAtRuntime"));
      R.addDecl(Res);
      DC->addDecl(Res);
      // Say that we can handle the situation. Clang should try to recover
      return true;
    }

    return false;
  }

  bool SymbolResolverCallback::ShouldResolveAtRuntime(LookupResult& R,
                                                      Scope* S) {

    if (R.getLookupKind() != Sema::LookupOrdinaryName)
      return false;

    if (R.isForRedeclaration())
      return false;

    if (!R.empty())
      return false;

    // FIXME: Figure out better way to handle:
    // C++ [basic.lookup.classref]p1:
    //   In a class member access expression (5.2.5), if the . or -> token is
    //   immediately followed by an identifier followed by a <, the
    //   identifier must be looked up to determine whether the < is the
    //   beginning of a template argument list (14.2) or a less-than operator.
    //   The identifier is first looked up in the class of the object
    //   expression. If the identifier is not found, it is then looked up in
    //   the context of the entire postfix-expression and shall name a class
    //   or function template.
    //
    // We want to ignore object(.|->)member<template>
    if (R.getSema().PP.LookAhead(0).getKind() == tok::less)
      // TODO: check for . or -> in the cached token stream
      return false;

    for (Scope* DepScope = S; DepScope; DepScope = DepScope->getParent()) {
      if (DeclContext* Ctx = static_cast<DeclContext*>(DepScope->getEntity())) {
        if (!Ctx->isDependentContext())
          // For now we support only the prompt.
          if (isa<FunctionDecl>(Ctx))
            return true;
      }
    }

    return false;
  }

} // end test
} // end cling
<|endoftext|>


<commit_before>// Copyright 2021 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUUtils.h"
#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/LoweringConfig.h"
#include "iree/compiler/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree/compiler/Dialect/LinalgExt/Transforms/Transforms.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
#include "mlir/Dialect/GPU/Passes.h"
#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Support/MathExtras.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/Passes.h"

#define DEBUG_TYPE "iree-llvmgpu-tile-and-distribute"

namespace mlir {
namespace iree_compiler {

/// Patterns for workgroup level tiling. Workgroup tiling is done at the flow
/// level but we may have extra tiling for the reduction dimension. Therefore we
/// tile again without distributing.
static void populateTilingReductionPatterns(
    MLIRContext *context, OwningRewritePatternList &patterns) {
  auto tileSizesFn = [&](OpBuilder &builder,
                         Operation *op) -> SmallVector<Value, 4> {
    SmallVector<unsigned> partitionedLoops = getPartitionedLoops(op);
    SmallVector<int64_t, 4> tileSizes = getTileSizes(op, 0);
    Location loc = op->getLoc();
    auto tileSizesVal =
        llvm::to_vector<4>(llvm::map_range(tileSizes, [&](int64_t v) -> Value {
          return builder.create<ConstantIndexOp>(loc, v);
        }));
    auto zero = builder.create<ConstantIndexOp>(loc, 0);
    for (unsigned depth : partitionedLoops) {
      if (depth < tileSizesVal.size()) {
        tileSizesVal[depth] = zero;
      }
    }
    return tileSizesVal;
  };

  auto tilingOptions = linalg::LinalgTilingOptions()
                           .setLoopType(linalg::LinalgTilingLoopType::Loops)
                           .setTileSizeComputationFunction(tileSizesFn);

  patterns.insert<linalg::LinalgTilingPattern<linalg::MatmulOp>,
                  linalg::LinalgTilingPattern<linalg::BatchMatmulOp>,
                  linalg::LinalgTilingPattern<linalg::GenericOp>>(
      context, tilingOptions,
      linalg::LinalgTransformationFilter(
          {Identifier::get(getWorkgroupMarker(), context)},
          Identifier::get(getWorkgroupKTiledMarker(), context)));
}

/// Patterns for thread level tiling.
static void populateTilingToInvocationPatterns(
    MLIRContext *context, OwningRewritePatternList &patterns,
    ArrayRef<int64_t> workgroupSize) {
  linalg::TileSizeComputationFunction getInnerTileSizeFn =
      [](OpBuilder &builder, Operation *operation) {
        SmallVector<Value, 4> tileSizesVal;
        SmallVector<int64_t, 4> tileSizes = getTileSizes(operation, 2);
        if (tileSizes.empty()) return SmallVector<Value, 4>();
        SmallVector<unsigned> partitionedLoops = getPartitionedLoops(operation);
        llvm::DenseSet<unsigned> partitionedLoopsSet(partitionedLoops.begin(),
                                                     partitionedLoops.end());
        tileSizesVal.reserve(tileSizes.size());
        for (auto val : llvm::enumerate(tileSizes)) {
          int64_t useTileSize =
              partitionedLoopsSet.count(val.index()) ? val.value() : 0;
          tileSizesVal.push_back(builder.create<ConstantIndexOp>(
              operation->getLoc(), useTileSize));
        }
        return tileSizesVal;
      };

  auto getThreadProcInfoFn = [workgroupSize](
                                 OpBuilder &builder, Location loc,
                                 ArrayRef<Range> parallelLoopRanges) {
    return getGPUThreadIdsAndCounts(builder, loc, parallelLoopRanges.size(),
                                    workgroupSize);
  };
  linalg::LinalgLoopDistributionOptions invocationDistributionOptions;
  invocationDistributionOptions.procInfo = getThreadProcInfoFn;
  invocationDistributionOptions.distributionMethod = {
      {linalg::DistributionMethod::Cyclic, linalg::DistributionMethod::Cyclic,
       linalg::DistributionMethod::Cyclic}};

  auto tilingOptions =
      linalg::LinalgTilingOptions()
          .setLoopType(linalg::LinalgTilingLoopType::Loops)
          .setTileSizeComputationFunction(getInnerTileSizeFn)
          .setDistributionOptions(invocationDistributionOptions);

  patterns
      .insert<linalg::LinalgTilingPattern<linalg::MatmulOp>,
              linalg::LinalgTilingPattern<linalg::FillOp>,
              linalg::LinalgTilingPattern<linalg::CopyOp>,
              linalg::LinalgTilingPattern<linalg::BatchMatmulOp>,
              linalg::LinalgTilingPattern<linalg::GenericOp>,
              linalg::LinalgTilingPattern<linalg::Conv2DNhwcHwcfOp>,
              linalg::LinalgTilingPattern<linalg::DepthwiseConv2DNhwcOp>,
              linalg::LinalgTilingPattern<linalg::Conv2DNhwcHwcfOp>,
              linalg::LinalgTilingPattern<linalg::DepthwiseConv2DNhwcOp>,
              linalg::LinalgTilingPattern<linalg::DepthwiseConv2DNhwcOp>,
              linalg_ext::TiledOpInterfaceTilingPattern<linalg_ext::ScatterOp>>(
          context, tilingOptions,
          linalg::LinalgTransformationFilter(
              {Identifier::get(getWorkgroupMarker(), context),
               Identifier::get(getWorkgroupKTiledMarker(), context),
               Identifier::get(getWorkgroupMemoryMarker(), context)},
              Identifier::get(getVectorizeMarker(), context)));
}

static LogicalResult copyToWorkgroupMemory(OpBuilder &b, Value src, Value dst) {
  auto copyOp = b.create<linalg::CopyOp>(src.getLoc(), src, dst);
  setMarker(copyOp, getCopyToWorkgroupMemoryMarker());
  return success();
}

static Optional<Value> allocateWorkgroupMemory(
    OpBuilder &b, memref::SubViewOp subview,
    ArrayRef<Value> boundingSubViewSize, DataLayout &layout) {
  // In CUDA workgroup memory is represented by a global variable. Create a
  // global variable and a memref.GetGlobalOp at the beginning of the funtion to
  // get the memref.
  OpBuilder::InsertionGuard guard(b);
  FuncOp funcOp = subview->getParentOfType<FuncOp>();
  if (!funcOp) {
    subview.emitError("expected op to be within std.func");
    return llvm::None;
  }
  ModuleOp moduleOp = funcOp->getParentOfType<ModuleOp>();
  SymbolTable symbolTable(moduleOp);

  // The bounding subview size is expected to be constant. This specified the
  // shape of the allocation.
  SmallVector<int64_t, 2> shape = llvm::to_vector<2>(
      llvm::map_range(boundingSubViewSize, [](Value v) -> int64_t {
        APInt value;
        if (matchPattern(v, m_ConstantInt(&value))) return value.getSExtValue();
        return -1;
      }));
  if (llvm::any_of(shape, [](int64_t v) { return v == -1; })) return {};
  Type allocType =
      MemRefType::get(shape, subview.getType().getElementType(), {},
                      gpu::GPUDialect::getWorkgroupAddressSpace());
  b.setInsertionPoint(&moduleOp.front());
  auto global =
      b.create<memref::GlobalOp>(funcOp.getLoc(), "__shared_memory__",
                                 /*sym_visibility=*/b.getStringAttr("private"),
                                 /*type=*/allocType,
                                 /*initial_value=*/ElementsAttr(),
                                 /*constant=*/false);
  symbolTable.insert(global);

  b.setInsertionPointToStart(&(*funcOp.getBody().begin()));
  Value buffer = b.create<memref::GetGlobalOp>(funcOp.getLoc(), global.type(),
                                               global.getName());
  return buffer;
}

static LogicalResult deallocateWorkgroupMemory(OpBuilder &b, Value buffer) {
  // Nothing to do.
  return success();
}

static void populatePromotionPatterns(MLIRContext *context,
                                      OwningRewritePatternList &patterns) {
  patterns.insert<linalg::LinalgPromotionPattern<linalg::MatmulOp>,
                  linalg::LinalgPromotionPattern<linalg::BatchMatmulOp>>(
      context,
      linalg::LinalgPromotionOptions()
          .setAllocationDeallocationFns(allocateWorkgroupMemory,
                                        deallocateWorkgroupMemory)
          .setCopyInOutFns(copyToWorkgroupMemory, copyToWorkgroupMemory)
          .setOperandsToPromote({0, 1})
          .setUseFullTileBuffers({false, false}),
      linalg::LinalgTransformationFilter(
          {Identifier::get(getWorkgroupKTiledMarker(), context)},
          Identifier::get(getWorkgroupMemoryMarker(), context)));
}

namespace {
struct LLVMGPUTileAndDistributePass
    : public LLVMGPUTileAndDistributeBase<LLVMGPUTileAndDistributePass> {
  void getDependentDialects(DialectRegistry &registry) const override {
    registry.insert<AffineDialect, gpu::GPUDialect>();
  }
  void runOnOperation() override {
    MLIRContext *context = &getContext();
    auto funcOp = getOperation();
    if (!isEntryPoint(funcOp)) return;

    {
      // Tile again at the workgroup level since redution dimension were
      // ignored. Dimensions already tiled will be ignore since we tile to the
      // same size.
      OwningRewritePatternList wgTilingPatterns(context);
      populateTilingReductionPatterns(context, wgTilingPatterns);
      (void)applyPatternsAndFoldGreedily(funcOp, std::move(wgTilingPatterns));
    }

    {
      RewritePatternSet wgTilingCanonicalizationPatterns =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      populateAffineMinSCFCanonicalizationPattern(
          wgTilingCanonicalizationPatterns);
      (void)applyPatternsAndFoldGreedily(
          funcOp, std::move(wgTilingCanonicalizationPatterns));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After tile reductions:";
      funcOp.dump();
    });

    auto workgroupSize = llvm::to_vector<4>(llvm::map_range(
        getEntryPoint(funcOp).workgroup_size().getValue(),
        [&](Attribute attr) { return attr.cast<IntegerAttr>().getInt(); }));
    int64_t flatWorkgroupSize =
        workgroupSize[0] * workgroupSize[1] * workgroupSize[2];
    // Only promote to workgroup size if there are multiple warps.
    if (flatWorkgroupSize > 32) {
      OwningRewritePatternList promotionPatterns(&getContext());
      populatePromotionPatterns(context, promotionPatterns);
      (void)applyPatternsAndFoldGreedily(funcOp, std::move(promotionPatterns));
      // Insert barriers before and after copies to workgroup memory and skip
      // insert barriers between back to back copy to workgroup memory.
      OpBuilder builder(&getContext());
      funcOp.walk([&builder](linalg::CopyOp copyOp) {
        if (hasMarker(copyOp, getCopyToWorkgroupMemoryMarker())) {
          Operation *prevOp = copyOp->getPrevNode();
          if (!prevOp || !hasMarker(prevOp, getCopyToWorkgroupMemoryMarker())) {
            builder.setInsertionPoint(copyOp);
            builder.create<gpu::BarrierOp>(copyOp.getLoc());
          }
          Operation *nextOp = copyOp->getNextNode();
          if (!nextOp || !hasMarker(nextOp, getCopyToWorkgroupMemoryMarker())) {
            builder.setInsertionPointAfter(copyOp);
            builder.create<gpu::BarrierOp>(copyOp.getLoc());
          }
        }
      });
    }

    {
      RewritePatternSet promotionCanonicalization =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      (void)applyPatternsAndFoldGreedily(funcOp,
                                         std::move(promotionCanonicalization));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After promotion:";
      funcOp.dump();
    });

    {
      // Apply last level of tiling and distribute to threads.
      OwningRewritePatternList threadLevelTilingPatterns(context);
      populateTilingToInvocationPatterns(context, threadLevelTilingPatterns,
                                         workgroupSize);
      (void)applyPatternsAndFoldGreedily(funcOp,
                                         std::move(threadLevelTilingPatterns));
    }
    {
      // Apply canonicalization patterns.
      RewritePatternSet threadTilingCanonicalizationPatterns =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      populateAffineMinSCFCanonicalizationPattern(
          threadTilingCanonicalizationPatterns);
      (void)applyPatternsAndFoldGreedily(
          funcOp, std::move(threadTilingCanonicalizationPatterns));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After tile and distribute to threads:";
      funcOp.dump();
    });
  }
};
}  // namespace

std::unique_ptr<OperationPass<FuncOp>>
createLLVMGPUTileAndDistributeToThreads() {
  return std::make_unique<LLVMGPUTileAndDistributePass>();
}

}  // namespace iree_compiler
}  // namespace mlir
<commit_msg>Fixes for 4beb756a748e710203bd309d57dc449ee9c8e3f0<commit_after>// Copyright 2021 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUUtils.h"
#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/LoweringConfig.h"
#include "iree/compiler/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree/compiler/Dialect/LinalgExt/Transforms/Transforms.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
#include "mlir/Dialect/GPU/Passes.h"
#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Support/MathExtras.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "mlir/Transforms/Passes.h"

#define DEBUG_TYPE "iree-llvmgpu-tile-and-distribute"

namespace mlir {
namespace iree_compiler {

/// Patterns for workgroup level tiling. Workgroup tiling is done at the flow
/// level but we may have extra tiling for the reduction dimension. Therefore we
/// tile again without distributing.
static void populateTilingReductionPatterns(
    MLIRContext *context, OwningRewritePatternList &patterns) {
  auto tileSizesFn = [&](OpBuilder &builder,
                         Operation *op) -> SmallVector<Value, 4> {
    SmallVector<unsigned> partitionedLoops = getPartitionedLoops(op);
    SmallVector<int64_t, 4> tileSizes = getTileSizes(op, 0);
    Location loc = op->getLoc();
    auto tileSizesVal =
        llvm::to_vector<4>(llvm::map_range(tileSizes, [&](int64_t v) -> Value {
          return builder.create<ConstantIndexOp>(loc, v);
        }));
    auto zero = builder.create<ConstantIndexOp>(loc, 0);
    for (unsigned depth : partitionedLoops) {
      if (depth < tileSizesVal.size()) {
        tileSizesVal[depth] = zero;
      }
    }
    return tileSizesVal;
  };

  auto tilingOptions = linalg::LinalgTilingOptions()
                           .setLoopType(linalg::LinalgTilingLoopType::Loops)
                           .setTileSizeComputationFunction(tileSizesFn);

  patterns.insert<linalg::LinalgTilingPattern<linalg::MatmulOp>,
                  linalg::LinalgTilingPattern<linalg::BatchMatmulOp>,
                  linalg::LinalgTilingPattern<linalg::GenericOp>>(
      context, tilingOptions,
      linalg::LinalgTransformationFilter(
          {Identifier::get(getWorkgroupMarker(), context)},
          Identifier::get(getWorkgroupKTiledMarker(), context)));
}

/// Patterns for thread level tiling.
static void populateTilingToInvocationPatterns(
    MLIRContext *context, OwningRewritePatternList &patterns,
    ArrayRef<int64_t> workgroupSize) {
  linalg::TileSizeComputationFunction getInnerTileSizeFn =
      [](OpBuilder &builder, Operation *operation) {
        SmallVector<Value, 4> tileSizesVal;
        SmallVector<int64_t, 4> tileSizes = getTileSizes(operation, 2);
        if (tileSizes.empty()) return SmallVector<Value, 4>();
        SmallVector<unsigned> partitionedLoops = getPartitionedLoops(operation);
        llvm::DenseSet<unsigned> partitionedLoopsSet(partitionedLoops.begin(),
                                                     partitionedLoops.end());
        tileSizesVal.reserve(tileSizes.size());
        for (auto val : llvm::enumerate(tileSizes)) {
          int64_t useTileSize =
              partitionedLoopsSet.count(val.index()) ? val.value() : 0;
          tileSizesVal.push_back(builder.create<ConstantIndexOp>(
              operation->getLoc(), useTileSize));
        }
        return tileSizesVal;
      };

  auto getThreadProcInfoFn = [workgroupSize](
                                 OpBuilder &builder, Location loc,
                                 ArrayRef<Range> parallelLoopRanges) {
    return getGPUThreadIdsAndCounts(builder, loc, parallelLoopRanges.size(),
                                    workgroupSize);
  };
  linalg::LinalgLoopDistributionOptions invocationDistributionOptions;
  invocationDistributionOptions.procInfo = getThreadProcInfoFn;
  invocationDistributionOptions.distributionMethod = {
      {linalg::DistributionMethod::Cyclic, linalg::DistributionMethod::Cyclic,
       linalg::DistributionMethod::Cyclic}};

  auto tilingOptions =
      linalg::LinalgTilingOptions()
          .setLoopType(linalg::LinalgTilingLoopType::Loops)
          .setTileSizeComputationFunction(getInnerTileSizeFn)
          .setDistributionOptions(invocationDistributionOptions);

  patterns
      .insert<linalg::LinalgTilingPattern<linalg::MatmulOp>,
              linalg::LinalgTilingPattern<linalg::FillOp>,
              linalg::LinalgTilingPattern<linalg::CopyOp>,
              linalg::LinalgTilingPattern<linalg::BatchMatmulOp>,
              linalg::LinalgTilingPattern<linalg::GenericOp>,
              linalg::LinalgTilingPattern<linalg::Conv2DNhwcHwcfOp>,
              linalg::LinalgTilingPattern<linalg::DepthwiseConv2DNhwOp>,
              linalg::LinalgTilingPattern<linalg::DepthwiseConv2DNhwcOp>,
              linalg_ext::TiledOpInterfaceTilingPattern<linalg_ext::ScatterOp>>(
          context, tilingOptions,
          linalg::LinalgTransformationFilter(
              {Identifier::get(getWorkgroupMarker(), context),
               Identifier::get(getWorkgroupKTiledMarker(), context),
               Identifier::get(getWorkgroupMemoryMarker(), context)},
              Identifier::get(getVectorizeMarker(), context)));
}

static LogicalResult copyToWorkgroupMemory(OpBuilder &b, Value src, Value dst) {
  auto copyOp = b.create<linalg::CopyOp>(src.getLoc(), src, dst);
  setMarker(copyOp, getCopyToWorkgroupMemoryMarker());
  return success();
}

static Optional<Value> allocateWorkgroupMemory(
    OpBuilder &b, memref::SubViewOp subview,
    ArrayRef<Value> boundingSubViewSize, DataLayout &layout) {
  // In CUDA workgroup memory is represented by a global variable. Create a
  // global variable and a memref.GetGlobalOp at the beginning of the funtion to
  // get the memref.
  OpBuilder::InsertionGuard guard(b);
  FuncOp funcOp = subview->getParentOfType<FuncOp>();
  if (!funcOp) {
    subview.emitError("expected op to be within std.func");
    return llvm::None;
  }
  ModuleOp moduleOp = funcOp->getParentOfType<ModuleOp>();
  SymbolTable symbolTable(moduleOp);

  // The bounding subview size is expected to be constant. This specified the
  // shape of the allocation.
  SmallVector<int64_t, 2> shape = llvm::to_vector<2>(
      llvm::map_range(boundingSubViewSize, [](Value v) -> int64_t {
        APInt value;
        if (matchPattern(v, m_ConstantInt(&value))) return value.getSExtValue();
        return -1;
      }));
  if (llvm::any_of(shape, [](int64_t v) { return v == -1; })) return {};
  Type allocType =
      MemRefType::get(shape, subview.getType().getElementType(), {},
                      gpu::GPUDialect::getWorkgroupAddressSpace());
  b.setInsertionPoint(&moduleOp.front());
  auto global =
      b.create<memref::GlobalOp>(funcOp.getLoc(), "__shared_memory__",
                                 /*sym_visibility=*/b.getStringAttr("private"),
                                 /*type=*/allocType,
                                 /*initial_value=*/ElementsAttr(),
                                 /*constant=*/false);
  symbolTable.insert(global);

  b.setInsertionPointToStart(&(*funcOp.getBody().begin()));
  Value buffer = b.create<memref::GetGlobalOp>(funcOp.getLoc(), global.type(),
                                               global.getName());
  return buffer;
}

static LogicalResult deallocateWorkgroupMemory(OpBuilder &b, Value buffer) {
  // Nothing to do.
  return success();
}

static void populatePromotionPatterns(MLIRContext *context,
                                      OwningRewritePatternList &patterns) {
  patterns.insert<linalg::LinalgPromotionPattern<linalg::MatmulOp>,
                  linalg::LinalgPromotionPattern<linalg::BatchMatmulOp>>(
      context,
      linalg::LinalgPromotionOptions()
          .setAllocationDeallocationFns(allocateWorkgroupMemory,
                                        deallocateWorkgroupMemory)
          .setCopyInOutFns(copyToWorkgroupMemory, copyToWorkgroupMemory)
          .setOperandsToPromote({0, 1})
          .setUseFullTileBuffers({false, false}),
      linalg::LinalgTransformationFilter(
          {Identifier::get(getWorkgroupKTiledMarker(), context)},
          Identifier::get(getWorkgroupMemoryMarker(), context)));
}

namespace {
struct LLVMGPUTileAndDistributePass
    : public LLVMGPUTileAndDistributeBase<LLVMGPUTileAndDistributePass> {
  void getDependentDialects(DialectRegistry &registry) const override {
    registry.insert<AffineDialect, gpu::GPUDialect>();
  }
  void runOnOperation() override {
    MLIRContext *context = &getContext();
    auto funcOp = getOperation();
    if (!isEntryPoint(funcOp)) return;

    {
      // Tile again at the workgroup level since redution dimension were
      // ignored. Dimensions already tiled will be ignore since we tile to the
      // same size.
      OwningRewritePatternList wgTilingPatterns(context);
      populateTilingReductionPatterns(context, wgTilingPatterns);
      (void)applyPatternsAndFoldGreedily(funcOp, std::move(wgTilingPatterns));
    }

    {
      RewritePatternSet wgTilingCanonicalizationPatterns =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      populateAffineMinSCFCanonicalizationPattern(
          wgTilingCanonicalizationPatterns);
      (void)applyPatternsAndFoldGreedily(
          funcOp, std::move(wgTilingCanonicalizationPatterns));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After tile reductions:";
      funcOp.dump();
    });

    auto workgroupSize = llvm::to_vector<4>(llvm::map_range(
        getEntryPoint(funcOp).workgroup_size().getValue(),
        [&](Attribute attr) { return attr.cast<IntegerAttr>().getInt(); }));
    int64_t flatWorkgroupSize =
        workgroupSize[0] * workgroupSize[1] * workgroupSize[2];
    // Only promote to workgroup size if there are multiple warps.
    if (flatWorkgroupSize > 32) {
      OwningRewritePatternList promotionPatterns(&getContext());
      populatePromotionPatterns(context, promotionPatterns);
      (void)applyPatternsAndFoldGreedily(funcOp, std::move(promotionPatterns));
      // Insert barriers before and after copies to workgroup memory and skip
      // insert barriers between back to back copy to workgroup memory.
      OpBuilder builder(&getContext());
      funcOp.walk([&builder](linalg::CopyOp copyOp) {
        if (hasMarker(copyOp, getCopyToWorkgroupMemoryMarker())) {
          Operation *prevOp = copyOp->getPrevNode();
          if (!prevOp || !hasMarker(prevOp, getCopyToWorkgroupMemoryMarker())) {
            builder.setInsertionPoint(copyOp);
            builder.create<gpu::BarrierOp>(copyOp.getLoc());
          }
          Operation *nextOp = copyOp->getNextNode();
          if (!nextOp || !hasMarker(nextOp, getCopyToWorkgroupMemoryMarker())) {
            builder.setInsertionPointAfter(copyOp);
            builder.create<gpu::BarrierOp>(copyOp.getLoc());
          }
        }
      });
    }

    {
      RewritePatternSet promotionCanonicalization =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      (void)applyPatternsAndFoldGreedily(funcOp,
                                         std::move(promotionCanonicalization));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After promotion:";
      funcOp.dump();
    });

    {
      // Apply last level of tiling and distribute to threads.
      OwningRewritePatternList threadLevelTilingPatterns(context);
      populateTilingToInvocationPatterns(context, threadLevelTilingPatterns,
                                         workgroupSize);
      (void)applyPatternsAndFoldGreedily(funcOp,
                                         std::move(threadLevelTilingPatterns));
    }
    {
      // Apply canonicalization patterns.
      RewritePatternSet threadTilingCanonicalizationPatterns =
          linalg::getLinalgTilingCanonicalizationPatterns(context);
      populateAffineMinSCFCanonicalizationPattern(
          threadTilingCanonicalizationPatterns);
      (void)applyPatternsAndFoldGreedily(
          funcOp, std::move(threadTilingCanonicalizationPatterns));
    }

    LLVM_DEBUG({
      llvm::dbgs() << "After tile and distribute to threads:";
      funcOp.dump();
    });
  }
};
}  // namespace

std::unique_ptr<OperationPass<FuncOp>>
createLLVMGPUTileAndDistributeToThreads() {
  return std::make_unique<LLVMGPUTileAndDistributePass>();
}

}  // namespace iree_compiler
}  // namespace mlir
<|endoftext|>