<commit_before>#include <cstdint>
#include <vector>

#include <scoped_allocator>
#include <rte_allocator.hpp>

#include <rte_common.h>
#include <rte_config.h>
#include <rte_lcore.h>
#include <rte_hash.h>
#include <rte_jhash.h>
#include <rte_debug.h>
#include <rte_errno.h>
#include <rte_malloc.h>

template<typename T> using alloc = std::scoped_allocator_adaptor<rte_allocator<T>>;

namespace flowtracker {
    constexpr uint8_t IPV6_ADDR_LEN = 16;

    struct ipv4_5tuple {
        uint32_t ip_dst;
        uint32_t ip_src;
        uint16_t port_dst;
        uint16_t port_src;
        uint8_t  proto;
    } __attribute__((__packed__));

    struct ipv6_5tuple {
        uint8_t  ip_dst[IPV6_ADDR_LEN];
        uint8_t  ip_src[IPV6_ADDR_LEN];
        uint16_t port_dst;
        uint16_t port_src;
        uint8_t  proto;
    } __attribute__((__packed__));

    template<typename T>
    struct flowtracker {
        explicit flowtracker(const std::uint32_t max_flows = 1024) :
                ipv4_flowdata(std::vector<T>(max_flows)),
                //ipv4_flowdata(std::vector<T, alloc<T>>(max_flows)),
                ipv6_flowdata(std::vector<T>(max_flows)) {
            rte_hash_parameters params = {};
            params.entries = max_flows;
            params.key_len = sizeof(ipv4_5tuple);
            params.hash_func = rte_jhash;
            params.hash_func_init_val = 0;
            params.socket_id = rte_socket_id();
            params.name = "ipv4_flow_map";
            ipv4_map = rte_hash_create(&params);
            if(ipv4_map == NULL)
                rte_panic("Could not create IPv4 flow hash map, errno = %d\n", rte_errno);
            params.key_len = sizeof(ipv6_5tuple);
            params.name = "ipv6_flow_map";
            ipv6_map = rte_hash_create(&params);
            if(ipv6_map == NULL)
                rte_panic("Could not create IPv6 flow hash map, errno = %d\n", rte_errno);
        }

        ~flowtracker() {
            rte_hash_free(const_cast<rte_hash*>(ipv4_map));
            rte_hash_free(const_cast<rte_hash*>(ipv6_map));
        }

        std::int32_t add_flow(const ipv4_5tuple& flow_tuple, const T* flow_data) {
            std::int32_t ret = rte_hash_add_key(this->ipv4_map, &flow_tuple);
            if (ret < 0)
                return ret;
            this->ipv4_flowdata[ret] = *flow_data;
            return ret;
        }

        std::int32_t add_flow(const ipv6_5tuple& flow_tuple, const T* flow_data) {
            std::int32_t ret = rte_hash_add_key(this->ipv6_map, &flow_tuple);
            if (ret < 0)
                return ret;
            this->ipv6_flowdata[ret] = *flow_data;
            return ret;
        }

        template<typename F>
        T* get_flow_data(const F& flow_tuple);

        T* get_flow_data(const ipv4_5tuple& flow_tuple) {
            std::int32_t ret = rte_hash_lookup(this->ipv4_map, &flow_tuple);
            if (ret < 0)
                return NULL;
            return &this->ipv4_flowdata[ret];
        }

        T* get_flow_data(const ipv6_5tuple& flow_tuple) {
            std::int32_t ret = rte_hash_lookup(this->ipv6_flow_map, flow_tuple);
            if (ret < 0)
                return NULL;
            return this->ipv6_flowdata[ret];
        }

        template<typename F>
        std::int32_t remove_flow(const F& flow_tuple);

        std::int32_t remove_flow(const ipv4_5tuple& flow_tuple) {
            return rte_hash_del_key(this->ipv4_flow_map, flow_tuple);
        }

        std::int32_t remove_flow(const ipv6_5tuple& flow_tuple) {
            return rte_hash_del_key(this->ipv6_flow_map, flow_tuple);
        }

    private:
        const struct rte_hash* ipv4_map;
        const struct rte_hash* ipv6_map;
        std::vector<T> ipv4_flowdata;
        //std::vector<T, alloc<T>> ipv4_flowdata;
        std::vector<T> ipv6_flowdata;
    };
}

extern "C" {
    // Copied from QQ.hpp
    struct packet_header {
        uint64_t timestamp:48;  //!< Stores a timestamp. Unit is microseconds.
        uint64_t vlan:12;       //!< Field to store the VLAN tag. Prevents messy Ethernet header.
        uint16_t len;           //!< Holds the length of the data array.
        uint8_t data[];         //!< Flexible array member. Valid since C99, not really in C++.
    } __attribute__((__packed__));

    struct foo_flow_data {
        uint64_t start_ts;
        uint64_t end_ts;
        uint8_t observed_ttl;
    } __attribute__((__packed__));

    using D = foo_flow_data;
    using tracker = flowtracker::flowtracker<D>;
    using v4tpl = flowtracker::ipv4_5tuple;
    using v6tpl = flowtracker::ipv6_5tuple;

    tracker* flowtracker_create(uint32_t max_flows) {
        void* temp = rte_malloc(NULL, sizeof(tracker), 0);
        if (temp == NULL)
            rte_panic("Unable to allocate memory for flowtracker\n");
#if 0
        auto* tr = new(temp) tracker(max_flows);
        printf("new()\n");
        return tr;
#endif
        return new(temp) tracker(max_flows);
    }

    void flowtracker_delete(tracker* tr) {
        rte_free(tr);
    }

    /*
    int32_t flowtracker_add_flow_v4(tracker* tr, const v4tpl* const tpl, const D* flow_data) {
        return tr->add_flow(tpl, flow_data);
    }
    */

    int32_t flowtracker_add_flow_v4(tracker* tr, uint32_t ip_src, uint16_t port_src,
                                    uint32_t ip_dst, uint16_t port_dst, uint8_t proto,
                                    const D* flow_data) {
        flowtracker::ipv4_5tuple tpl {
            ip_dst, ip_src, port_dst, port_src, proto
        };
#ifdef DNDEBUG
        return tr->add_flow(tpl, flow_data);
#else
        int ret = tr->add_flow(tpl, flow_data);
        if (ret < 0)
            rte_exit(EXIT_FAILURE, "Unable to add key to ipv4_map: %d = %s\n", ret, rte_strerror(-ret));
        return ret;
#endif
    }

    int32_t flowtracker_add_flow_v6(tracker* tr, const v6tpl* const tpl, const D* flow_data) {
        return tr->add_flow(*tpl, flow_data);
    }

    int32_t flowtracker_remove_flow_v4(tracker* tr, const flowtracker::ipv4_5tuple* const tpl) {
        return tr->remove_flow(tpl);
    }

    int32_t flowtracker_remove_flow_v6(tracker* tr, const flowtracker::ipv6_5tuple* const tpl) {
        return tr->remove_flow(tpl);
    }

    /*
    foo_flow_data* flowtracker_get_flow_data_v4(tracker* tr, const flowtracker::ipv4_5tuple* const tpl) {
        return tr->get_flow_data(tpl);
    }
    */

    foo_flow_data* flowtracker_get_flow_data_v6(tracker* tr, const flowtracker::ipv6_5tuple* const tpl) {
        return tr->get_flow_data(tpl);
    }

    foo_flow_data* flowtracker_get_flow_data_v4(tracker* tr, uint32_t ip_src, uint16_t port_src,
                                                uint32_t ip_dst, uint16_t port_dst, uint8_t proto) {
        flowtracker::ipv4_5tuple tpl {
            ip_dst, ip_src, port_dst, port_src, proto
        };
        return tr->get_flow_data(tpl);
    }

    void analyze(tracker* tr, const packet_header* const pkt_hdr);

    void analyze_v4(tracker* tr, uint64_t ts, const flowtracker::ipv4_5tuple* const tpl, const uint8_t ttl);

    void analyze_v6(tracker* tr, uint64_t ts, const flowtracker::ipv6_5tuple* const tpl, const uint8_t ttl);

    /* rte_hash wrapper */

    struct rte_hash* rte_hash_create_v4(uint32_t max_flows, const char* name) {
        rte_hash_parameters params = {};
        params.entries = max_flows;
        params.key_len = sizeof(flowtracker::ipv4_5tuple);
        params.hash_func = rte_jhash;
        params.hash_func_init_val = 0;
        params.socket_id = rte_socket_id();
        params.name = name;
        params.extra_flag |= (RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD);
        auto ipv4_map = rte_hash_create(&params);
        if(ipv4_map == NULL)
            rte_panic("Could not create IPv4 flow hash map %s, %d = %s\n", name, rte_errno, rte_strerror(rte_errno));

        return ipv4_map;
    }

    int32_t rte_hash_lookup_v4(const struct rte_hash *h, const void *key) {
        return rte_hash_lookup(h, key);
    }

    void rte_hash_free_v4(struct rte_hash *h) {
        rte_hash_free(h);
    }

    int32_t rte_hash_add_key_v4(const struct rte_hash *h, const void *key) {
        return rte_hash_add_key(h, key);
    }

    int rte_hash_lookup_bulk_v4(const struct rte_hash *h, const void **keys, uint32_t num_keys, int32_t *positions) {
        return rte_hash_lookup_bulk(h, keys, num_keys, positions);
    }

}
<commit_msg>Add bulk lookup to flowtracker<commit_after>#include <cstdint>
#include <vector>

#include <scoped_allocator>
#include <rte_allocator.hpp>

#include <rte_common.h>
#include <rte_config.h>
#include <rte_lcore.h>
#include <rte_hash.h>
#include <rte_jhash.h>
#include <rte_debug.h>
#include <rte_errno.h>
#include <rte_malloc.h>

template<typename T> using alloc = std::scoped_allocator_adaptor<rte_allocator<T>>;

namespace flowtracker {
    constexpr std::uint8_t IPV6_ADDR_LEN = 16;
    constexpr std::size_t batch_size = 64;

    struct ipv4_5tuple {
        std::uint32_t ip_dst;
        std::uint32_t ip_src;
        std::uint16_t port_dst;
        std::uint16_t port_src;
        std::uint8_t  proto;
    } __attribute__((__packed__));

    struct ipv6_5tuple {
        std::uint8_t  ip_dst[IPV6_ADDR_LEN];
        std::uint8_t  ip_src[IPV6_ADDR_LEN];
        std::uint16_t port_dst;
        std::uint16_t port_src;
        std::uint8_t  proto;
    } __attribute__((__packed__));

    template<typename T>
    struct flowtracker {
        explicit flowtracker(const std::uint32_t max_flows = 1024) :
                ipv4_flowdata(std::vector<T>(max_flows)),
                //ipv4_flowdata(std::vector<T, alloc<T>>(max_flows)),
                ipv6_flowdata(std::vector<T>(max_flows)) {
            rte_hash_parameters params = {};
            params.entries = max_flows;
            params.key_len = sizeof(ipv4_5tuple);
            params.hash_func = rte_jhash;
            params.hash_func_init_val = 0;
            params.socket_id = rte_socket_id();
            params.name = "ipv4_flow_map";
            params.extra_flag |= (RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD);
            ipv4_map = rte_hash_create(&params);
            if(ipv4_map == NULL)
                rte_panic("Could not create IPv4 flow hash map, errno = %d (%s)\n", rte_errno, rte_strerror(rte_errno));
            params.key_len = sizeof(ipv6_5tuple);
            params.name = "ipv6_flow_map";
            ipv6_map = rte_hash_create(&params);
            if(ipv6_map == NULL)
                rte_panic("Could not create IPv6 flow hash map, errno = %d (%s)\n", rte_errno, rte_strerror(rte_errno));
        }

        ~flowtracker() {
            rte_hash_free(const_cast<rte_hash*>(ipv4_map));
            rte_hash_free(const_cast<rte_hash*>(ipv6_map));
        }

        std::int32_t add_flow(const ipv4_5tuple& flow_tuple, const T* flow_data) {
            std::int32_t ret = rte_hash_add_key(this->ipv4_map, &flow_tuple);
            if (ret < 0)
                return ret;
            this->ipv4_flowdata[ret] = *flow_data;
            return ret;
        }

        std::int32_t add_flow(const ipv6_5tuple& flow_tuple, const T* flow_data) {
            std::int32_t ret = rte_hash_add_key(this->ipv6_map, &flow_tuple);
            if (ret < 0)
                return ret;
            this->ipv6_flowdata[ret] = *flow_data;
            return ret;
        }

        T* get_flow_data(const ipv4_5tuple& flow_tuple) {
            std::int32_t ret = rte_hash_lookup(this->ipv4_map, &flow_tuple);
            if (ret < 0)
                return NULL;
            return &this->ipv4_flowdata[ret];
        }

        T* get_flow_data(const ipv6_5tuple& flow_tuple) {
            std::int32_t ret = rte_hash_lookup(this->ipv6_map, &flow_tuple);
            if (ret < 0)
                return NULL;
            return &this->ipv6_flowdata[ret];
        }

        // FIXME: return flowdata, not lookup results
        int get_flow_data_bulk(const ipv4_5tuple** keys, std::uint32_t num_keys, std::int32_t* positions) {
            return rte_hash_lookup_bulk(this->ipv4_map, reinterpret_cast<const void**>(keys), num_keys, positions);
        }

        // FIXME: return flowdata, not lookup results
        int get_flow_data_bulk(const ipv6_5tuple** keys, std::uint32_t num_keys, std::int32_t* positions) {
            return rte_hash_lookup_bulk(this->ipv4_map, reinterpret_cast<const void**>(keys), num_keys, positions);
        }

        std::int32_t remove_flow(const ipv4_5tuple& flow_tuple) {
            return rte_hash_del_key(this->ipv4_map, &flow_tuple);
        }

        std::int32_t remove_flow(const ipv6_5tuple& flow_tuple) {
            return rte_hash_del_key(this->ipv6_map, &flow_tuple);
        }

    private:
        const struct rte_hash* ipv4_map;
        const struct rte_hash* ipv6_map;
        std::vector<T> ipv4_flowdata;
        //std::vector<T, alloc<T>> ipv4_flowdata;
        std::vector<T> ipv6_flowdata;
    };
}

extern "C" {
    // Copied from QQ.hpp
    struct packet_header {
        std::uint64_t timestamp:48;  //!< Stores a timestamp. Unit is microseconds.
        std::uint64_t vlan:12;       //!< Field to store the VLAN tag. Prevents messy Ethernet header.
        std::uint16_t len;           //!< Holds the length of the data array.
        std::uint8_t data[];         //!< Flexible array member. Valid since C99, not really in C++.
    } __attribute__((__packed__));

    struct foo_flow_data {
        std::uint64_t start_ts;
        std::uint64_t end_ts;
        std::uint8_t observed_ttl;
    } __attribute__((__packed__));

    using D = foo_flow_data;
    using tracker = flowtracker::flowtracker<D>;
    using v4tpl = flowtracker::ipv4_5tuple;
    using v6tpl = flowtracker::ipv6_5tuple;

    tracker* flowtracker_create(std::uint32_t max_flows) {
        void* temp = rte_malloc(NULL, sizeof(tracker), 0);
        if (temp == NULL)
            rte_panic("Unable to allocate memory for flowtracker\n");
#if 0
        auto* tr = new(temp) tracker(max_flows);
        printf("new()\n");
        return tr;
#endif
        return new(temp) tracker(max_flows);
    }

    void flowtracker_delete(tracker* tr) {
        rte_free(tr);
    }

    /*
    int32_t flowtracker_add_flow_v4(tracker* tr, uint32_t ip_src, uint16_t port_src,
                                    uint32_t ip_dst, uint16_t port_dst, uint8_t proto,
                                    const D* flow_data) {
        flowtracker::ipv4_5tuple tpl {
            ip_dst, ip_src, port_dst, port_src, proto
        };
#ifdef DNDEBUG
        return tr->add_flow(tpl, flow_data);
#else
        int ret = tr->add_flow(tpl, flow_data);
        if (ret < 0)
            rte_exit(EXIT_FAILURE, "Unable to add key to ipv4_map: %d = %s\n", ret, rte_strerror(-ret));
        return ret;
#endif
    }
    */

    std::int32_t flowtracker_add_flow_v4(tracker* tr, const v4tpl* const tpl, const D* flow_data) {
        return tr->add_flow(*tpl, flow_data);
    }

    std::int32_t flowtracker_add_flow_v6(tracker* tr, const v6tpl* const tpl, const D* flow_data) {
        return tr->add_flow(*tpl, flow_data);
    }

    std::int32_t flowtracker_remove_flow_v4(tracker* tr, const v4tpl* const tpl) {
        return tr->remove_flow(*tpl);
    }

    std::int32_t flowtracker_remove_flow_v6(tracker* tr, const v6tpl* const tpl) {
        return tr->remove_flow(*tpl);
    }

    foo_flow_data* flowtracker_get_flow_data_v4(tracker* tr, const v4tpl* const tpl) {
        return tr->get_flow_data(*tpl);
    }

    foo_flow_data* flowtracker_get_flow_data_v6(tracker* tr, const v6tpl* const tpl) {
        return tr->get_flow_data(*tpl);
    }

    int flowtracker_get_flow_data_bulk_v4(tracker* tr, const v4tpl** const keys, std::uint32_t num_keys, std::int32_t* positions) {
        return tr->get_flow_data_bulk(keys, num_keys, positions);
    }

    void analyze(tracker* tr, const packet_header* const pkt_hdr);

    void analyze_v4(tracker* tr, std::uint64_t ts, const flowtracker::ipv4_5tuple* const tpl, const std::uint8_t ttl);

    void analyze_v6(tracker* tr, std::uint64_t ts, const flowtracker::ipv6_5tuple* const tpl, const std::uint8_t ttl);

    /* rte_hash wrapper */

    struct rte_hash* rte_hash_create_v4(std::uint32_t max_flows, const char* name) {
        rte_hash_parameters params = {};
        params.entries = max_flows;
        params.key_len = sizeof(flowtracker::ipv4_5tuple);
        params.hash_func = rte_jhash;
        params.hash_func_init_val = 0;
        params.socket_id = rte_socket_id();
        params.name = name;
        params.extra_flag |= (RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD);
        auto ipv4_map = rte_hash_create(&params);
        if(ipv4_map == NULL)
            rte_panic("Could not create IPv4 flow hash map %s, %d = %s\n", name, rte_errno, rte_strerror(rte_errno));

        return ipv4_map;
    }

    std::int32_t rte_hash_lookup_v4(const struct rte_hash *h, const void *key) {
        return rte_hash_lookup(h, key);
    }

    void rte_hash_free_v4(struct rte_hash *h) {
        rte_hash_free(h);
    }

    std::int32_t rte_hash_add_key_v4(const struct rte_hash *h, const void *key) {
        return rte_hash_add_key(h, key);
    }

    int rte_hash_lookup_bulk_v4(const struct rte_hash *h, const void **keys, std::uint32_t num_keys, std::int32_t *positions) {
        return rte_hash_lookup_bulk(h, keys, num_keys, positions);
    }

}
<|endoftext|>


<commit_before><commit_msg>Removed empty file<commit_after><|endoftext|>


<commit_before>/*
**  Copyright (C) 2012 Aldebaran Robotics
**  See COPYING for the license
*/

/*
 * this allow removing spurious warning when a deprecated declaration use another deprecated declaration.
 *
 * QI_API_DEPRECATED class Foo;
 *
 * #include <qi/detail/diagnostic_push_ignore_deprecated.hpp>
 * //Would emit a spurious deprecated warning about Foo otherwise
 * QI_API_DEPRECATED void bar(Foo foo);
 * #include <qi/detail/diagnostic_pop.hpp>
 */

// clang and GCC
#if defined(__GNUC__)
# if defined(__clang__) || (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
#  pragma GCC diagnostic push
# endif
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#elif defined(_MSC_VER)
# pragma warning(push)
# pragma warning(disable : 4996)
#endif
<commit_msg>Fix push/pop deprecated warning exemple<commit_after>/*
**  Copyright (C) 2012 Aldebaran Robotics
**  See COPYING for the license
*/

/*
 * this allow removing spurious warning when a deprecated declaration use another deprecated declaration.
 *
 * QI_API_DEPRECATED class Foo;
 *
 * #include <qi/detail/warn_push_ignore_deprecated.hpp>
 * //Would emit a spurious deprecated warning about Foo otherwise
 * QI_API_DEPRECATED void bar(Foo foo);
 * #include <qi/detail/warn_pop_ignore_deprecated.hpp>
 */

// clang and GCC
#if defined(__GNUC__)
# if defined(__clang__) || (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
#  pragma GCC diagnostic push
# endif
# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#elif defined(_MSC_VER)
# pragma warning(push)
# pragma warning(disable : 4996)
#endif
<|endoftext|>


<commit_before>/*!
    \file async_processor.cpp
    \brief Asynchronous logging processor definition
    \author Ivan Shynkarenka
    \date 01.08.2016
    \copyright MIT License
*/

#include "logging/processors/async_processor.h"

#include "errors/exceptions.h"
#include "errors/fatal.h"

namespace CppLogging {

AsyncProcessor::AsyncProcessor(bool discard_on_overflow, size_t capacity, size_t concurrency)
    : _discard_on_overflow(discard_on_overflow),
      _buffer(capacity, concurrency)
{
    // Start processing thread
    _thread = std::thread([this]() { ProcessBufferedRecords(); });
}

AsyncProcessor::~AsyncProcessor()
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1);

    // Calculate logging record size
    uint32_t size = sizeof(uint8_t);

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Stop operation record
    const uint8_t type = 2;

    // Serialize stop operation record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);

    // Enqueue buffered flush operation record
    EnqueueRecord(false, local.data(), local.size());

    // Wait for processing thread
    _thread.join();
}

bool AsyncProcessor::ProcessRecord(Record& record)
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1024);

    // Calculate logging record size
    size_t size = sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(Level) + sizeof(uint8_t) + record.logger.second + sizeof(uint16_t) + record.message.second + sizeof(uint32_t) + record.buffer.second + sizeof(uint32_t) + record.raw.second;

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Logging record
    const uint8_t type = 0;

    // Serialize logging record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);
    std::memcpy(buffer, &record.timestamp, sizeof(uint64_t));
    buffer += sizeof(uint64_t);
    std::memcpy(buffer, &record.thread, sizeof(uint64_t));
    buffer += sizeof(uint64_t);
    std::memcpy(buffer, &record.level, sizeof(Level));
    buffer += sizeof(Level);
    std::memcpy(buffer, &record.logger.second, sizeof(uint8_t));
    buffer += sizeof(uint8_t);
    std::memcpy(buffer, record.logger.first, record.logger.second);
    buffer += record.logger.second;
    std::memcpy(buffer, &record.message.second, sizeof(uint16_t));
    buffer += sizeof(uint16_t);
    std::memcpy(buffer, record.message.first, record.message.second);
    buffer += record.message.second;
    std::memcpy(buffer, &record.buffer.second, sizeof(uint32_t));
    buffer += sizeof(uint32_t);
    std::memcpy(buffer, record.buffer.first, record.buffer.second);
    buffer += record.buffer.second;
    std::memcpy(buffer, &record.raw.second, sizeof(uint32_t));
    buffer += sizeof(uint32_t);
    std::memcpy(buffer, record.raw.first, record.raw.second);
    buffer += record.buffer.second;

    // Enqueue buffered logger record
    return EnqueueRecord(_discard_on_overflow, local.data(), local.size());
}

bool AsyncProcessor::EnqueueRecord(bool discard_on_overflow, const void* chunk, size_t size)
{
    // Try to enqueue the given chunk of memory
    if (!_buffer.Enqueue(chunk, size))
    {
        // If the overflow policy is discard logging record, return immediately
        if (discard_on_overflow)
            return false;

        // If the overflow policy is blocking then spin, yield and sleep until the buffer is free
        int spins = 1000;
        int yields = 100;
        int sleep = 1;
        const int sleeps = 256;
        do
        {
            if (spins-- > 0)
            {
                // Spin...
            }
            else if (yields-- > 0)
            {
                // Yield...
                CppCommon::Thread::Yield();
            }
            else if (sleep < sleeps)
            {
                // Sleep...
                CppCommon::Thread::Sleep(sleep);
                sleep <<= 1;
            }
        } while (!_buffer.Enqueue(chunk, size));
    }

    return true;
}

void AsyncProcessor::ProcessBufferedRecords()
{
    // Call initialize thread handler
    OnThreadInitialize();

    try
    {
        // Local buffer to store the logging record
		static thread_local std::vector<uint8_t> local(_buffer.capacity());

        // Waiting strategy parameters
        int spins = 1000;
        int yields = 100;
        int sleep = 1;
        const int sleeps = 256;

        do
        {
            // Read local buffer
            size_t local_size = _buffer.capacity();
            if (_buffer.Dequeue(local.data(), local_size))
            {
                // Get the local buffer start and end positions
                uint8_t* buffer = local.data();
                uint8_t* end = local.data() + local_size;

                // Process all buffered logging records
                while (buffer < end)
                {
                    // Read the record type
                    uint8_t type;
                    std::memcpy(&type, buffer, sizeof(uint8_t));
                    buffer += sizeof(uint8_t);
                    switch (type)
                    {
                        case 0:
                        {
                            // Deserialize logging record
                            Record record;
                            std::memcpy(&record.timestamp, buffer, sizeof(uint64_t));
                            buffer += sizeof(uint64_t);
                            std::memcpy(&record.thread, buffer, sizeof(uint64_t));
                            buffer += sizeof(uint64_t);
                            std::memcpy(&record.level, buffer, sizeof(Level));
                            buffer += sizeof(Level);
                            std::memcpy(&record.logger.second, buffer, sizeof(uint8_t));
                            buffer += sizeof(uint8_t);
                            record.logger.first = (char*)buffer;
                            buffer += record.logger.second;
                            std::memcpy(&record.message.second, buffer, sizeof(uint16_t));
                            buffer += sizeof(uint16_t);
                            record.message.first = (char*)buffer;
                            buffer += record.message.second;
                            std::memcpy(&record.buffer.second, buffer, sizeof(uint32_t));
                            buffer += sizeof(uint32_t);
                            record.buffer.first = buffer;
                            buffer += record.buffer.second;
                            std::memcpy(&record.raw.second, buffer, sizeof(uint32_t));
                            buffer += sizeof(uint32_t);
                            record.raw.first = buffer;
                            buffer += record.raw.second;

                            // Process logging record
                            Processor::ProcessRecord(record);
                            break;
                        }
                        case 1:
                        {
                            // Flush the logging processor
                            Processor::Flush();
                            break;
                        }
                        case 2:
                            return;
                        default:
                            throwex CppCommon::RuntimeException("Invalid logging record detected - unknown operation record!");
                    }
                }

                // Reset waiting strategy parameters
                spins = 1000;
                yields = 100;
                sleep = 1;
            }
            else
            {
                // Wait for the next data to process
                if (spins-- > 0)
                {
                    // Spin...
                }
                else if (yields-- > 0)
                {
                    // Yield...
                    CppCommon::Thread::Yield();
                }
                else if (sleep < sleeps)
                {
                    // Sleep...
                    CppCommon::Thread::Sleep(sleep);
                    sleep <<= 1;
                }
            }
        } while (true);
    }
    catch (const CppCommon::RuntimeException& ex)
    {
        fatality("Asynchronous logging processor terminated: " + ex.to_string());
    }
    catch (...)
    {
        fatality("Asynchronous logging processor terminated!");
    }

    // Call cleanup thread handler
    OnThreadCleanup();
}

void AsyncProcessor::Flush()
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1);

    // Calculate logging record size
    uint32_t size = sizeof(uint8_t);

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Flush operation record
    const uint8_t type = 1;

    // Serialize flush operation record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);

    // Enqueue buffered flush operation record
    EnqueueRecord(false, local.data(), local.size());
}

} // namespace CppLogging
<commit_msg>Bugfixing<commit_after>/*!
    \file async_processor.cpp
    \brief Asynchronous logging processor definition
    \author Ivan Shynkarenka
    \date 01.08.2016
    \copyright MIT License
*/

#include "logging/processors/async_processor.h"

#include "errors/exceptions.h"
#include "errors/fatal.h"

namespace CppLogging {

AsyncProcessor::AsyncProcessor(bool discard_on_overflow, size_t capacity, size_t concurrency)
    : _discard_on_overflow(discard_on_overflow),
      _buffer(capacity, concurrency)
{
    // Start processing thread
    _thread = std::thread([this]() { ProcessBufferedRecords(); });
}

AsyncProcessor::~AsyncProcessor()
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1);

    // Calculate logging record size
    uint32_t size = sizeof(uint8_t);

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Stop operation record
    const uint8_t type = 2;

    // Serialize stop operation record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);

    // Enqueue buffered flush operation record
    EnqueueRecord(false, local.data(), local.size());

    // Wait for processing thread
    _thread.join();
}

bool AsyncProcessor::ProcessRecord(Record& record)
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1024);

    // Calculate logging record size
    size_t size = sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(Level) + sizeof(uint8_t) + record.logger.second + sizeof(uint16_t) + record.message.second + sizeof(uint32_t) + record.buffer.second + sizeof(uint32_t) + record.raw.second;

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Logging record
    const uint8_t type = 0;

    // Serialize logging record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);
    std::memcpy(buffer, &record.timestamp, sizeof(uint64_t));
    buffer += sizeof(uint64_t);
    std::memcpy(buffer, &record.thread, sizeof(uint64_t));
    buffer += sizeof(uint64_t);
    std::memcpy(buffer, &record.level, sizeof(Level));
    buffer += sizeof(Level);
    std::memcpy(buffer, &record.logger.second, sizeof(uint8_t));
    buffer += sizeof(uint8_t);
    std::memcpy(buffer, record.logger.first, record.logger.second);
    buffer += record.logger.second;
    std::memcpy(buffer, &record.message.second, sizeof(uint16_t));
    buffer += sizeof(uint16_t);
    std::memcpy(buffer, record.message.first, record.message.second);
    buffer += record.message.second;
    std::memcpy(buffer, &record.buffer.second, sizeof(uint32_t));
    buffer += sizeof(uint32_t);
    std::memcpy(buffer, record.buffer.first, record.buffer.second);
    buffer += record.buffer.second;
    std::memcpy(buffer, &record.raw.second, sizeof(uint32_t));
    buffer += sizeof(uint32_t);
    std::memcpy(buffer, record.raw.first, record.raw.second);
    buffer += record.buffer.second;

    // Enqueue buffered logger record
    return EnqueueRecord(_discard_on_overflow, local.data(), local.size());
}

bool AsyncProcessor::EnqueueRecord(bool discard_on_overflow, const void* chunk, size_t size)
{
    // Try to enqueue the given chunk of memory
    if (!_buffer.Enqueue(chunk, size))
    {
        // If the overflow policy is discard logging record, return immediately
        if (discard_on_overflow)
            return false;

        // If the overflow policy is blocking then spin, yield and sleep until the buffer is free
        int spins = 1000;
        int yields = 100;
        int sleep = 1;
        const int sleeps = 256;
        do
        {
            if (spins-- > 0)
            {
                // Spin...
            }
            else if (yields-- > 0)
            {
                // Yield...
                CppCommon::Thread::Yield();
            }
            else if (sleep < sleeps)
            {
                // Sleep...
                CppCommon::Thread::Sleep(sleep);
                sleep <<= 1;
            }
        } while (!_buffer.Enqueue(chunk, size));
    }

    return true;
}

void AsyncProcessor::ProcessBufferedRecords()
{
    // Call initialize thread handler
    OnThreadInitialize();

    try
    {
        // Local buffer to store the logging record
		static thread_local std::vector<uint8_t> local(_buffer.capacity());

        // Waiting strategy parameters
        int spins = 1000;
        int yields = 100;
        int sleep = 1;
        const int sleeps = 256;

        do
        {
            // Read local buffer
            size_t local_size = local.size();
            if (_buffer.Dequeue(local.data(), local_size))
            {
                // Get the local buffer start and end positions
                uint8_t* buffer = local.data();
                uint8_t* end = local.data() + local_size;

                // Process all buffered logging records
                while (buffer < end)
                {
                    // Read the record type
                    uint8_t type;
                    std::memcpy(&type, buffer, sizeof(uint8_t));
                    buffer += sizeof(uint8_t);
                    switch (type)
                    {
                        case 0:
                        {
                            // Deserialize logging record
                            Record record;
                            std::memcpy(&record.timestamp, buffer, sizeof(uint64_t));
                            buffer += sizeof(uint64_t);
                            std::memcpy(&record.thread, buffer, sizeof(uint64_t));
                            buffer += sizeof(uint64_t);
                            std::memcpy(&record.level, buffer, sizeof(Level));
                            buffer += sizeof(Level);
                            std::memcpy(&record.logger.second, buffer, sizeof(uint8_t));
                            buffer += sizeof(uint8_t);
                            record.logger.first = (char*)buffer;
                            buffer += record.logger.second;
                            std::memcpy(&record.message.second, buffer, sizeof(uint16_t));
                            buffer += sizeof(uint16_t);
                            record.message.first = (char*)buffer;
                            buffer += record.message.second;
                            std::memcpy(&record.buffer.second, buffer, sizeof(uint32_t));
                            buffer += sizeof(uint32_t);
                            record.buffer.first = buffer;
                            buffer += record.buffer.second;
                            std::memcpy(&record.raw.second, buffer, sizeof(uint32_t));
                            buffer += sizeof(uint32_t);
                            record.raw.first = buffer;
                            buffer += record.raw.second;

                            // Process logging record
                            Processor::ProcessRecord(record);
                            break;
                        }
                        case 1:
                        {
                            // Flush the logging processor
                            Processor::Flush();
                            break;
                        }
                        case 2:
                            return;
                        default:
                            throwex CppCommon::RuntimeException("Invalid logging record detected - unknown operation record!");
                    }
                }

                // Reset waiting strategy parameters
                spins = 1000;
                yields = 100;
                sleep = 1;
            }
            else
            {
                // Wait for the next data to process
                if (spins-- > 0)
                {
                    // Spin...
                }
                else if (yields-- > 0)
                {
                    // Yield...
                    CppCommon::Thread::Yield();
                }
                else if (sleep < sleeps)
                {
                    // Sleep...
                    CppCommon::Thread::Sleep(sleep);
                    sleep <<= 1;
                }
            }
        } while (true);
    }
    catch (const CppCommon::RuntimeException& ex)
    {
        fatality("Asynchronous logging processor terminated: " + ex.to_string());
    }
    catch (...)
    {
        fatality("Asynchronous logging processor terminated!");
    }

    // Call cleanup thread handler
    OnThreadCleanup();
}

void AsyncProcessor::Flush()
{
    // Local buffer to store the logging record
    static thread_local std::vector<uint8_t> local(1);

    // Calculate logging record size
    uint32_t size = sizeof(uint8_t);

    // Resize the local buffer to required size
    local.resize(size);

    // Get the buffer start position
    uint8_t* buffer = local.data();

    // Flush operation record
    const uint8_t type = 1;

    // Serialize flush operation record
    std::memcpy(buffer, &type, sizeof(uint8_t));
    buffer += sizeof(uint8_t);

    // Enqueue buffered flush operation record
    EnqueueRecord(false, local.data(), local.size());
}

} // namespace CppLogging
<|endoftext|>


<commit_before>#pragma once
#include "detail/typelist.hpp"
#include "support/traits.hpp"

#include <algorithm>

namespace tbm { namespace num {

/// Array scalar type
enum class Tag {f32, cf32, f64, cf64, b, i8, i16, i32, i64, u8, u16, u32, u64};

namespace detail {
    template<class scalar_t> constexpr Tag get_tag();
    template<> constexpr Tag get_tag<float>()                { return Tag::f32;  }
    template<> constexpr Tag get_tag<std::complex<float>>()  { return Tag::cf32; }
    template<> constexpr Tag get_tag<double>()               { return Tag::f64;  }
    template<> constexpr Tag get_tag<std::complex<double>>() { return Tag::cf64; }
    template<> constexpr Tag get_tag<bool>()                 { return Tag::b;    }
    template<> constexpr Tag get_tag<std::int8_t>()          { return Tag::i8;   }
    template<> constexpr Tag get_tag<std::int16_t>()         { return Tag::i16;  }
    template<> constexpr Tag get_tag<std::int32_t>()         { return Tag::i32;  }
    template<> constexpr Tag get_tag<std::int64_t>()         { return Tag::i64;  }
    template<> constexpr Tag get_tag<std::uint8_t>()         { return Tag::u8;   }
    template<> constexpr Tag get_tag<std::uint16_t>()        { return Tag::u16;  }
    template<> constexpr Tag get_tag<std::uint32_t>()        { return Tag::u32;  }
    template<> constexpr Tag get_tag<std::uint64_t>()        { return Tag::u64;  }
} // namespace detail

/// Reference to any 1D or 2D array with any scalar type supported by Tag
struct ArrayRef {
    Tag const tag;
    bool const is_row_major;
    void const* const data;
    int const rows;
    int const cols;
};

/// Reference to an array which is limited to a set of scalar types
template<class Scalar, class... Scalars>
struct VariantArrayRef : ArrayRef {
    using First = Scalar;
    using Types = TypeList<Scalar, Scalars...>;

    VariantArrayRef(ArrayRef const& other) : ArrayRef(other) {
        auto const possible_tags = {detail::get_tag<Scalar>(), detail::get_tag<Scalars>()...};
        auto const is_invalid = std::none_of(possible_tags.begin(), possible_tags.end(),
                                             [&](Tag tag) { return other.tag == tag; });
        if (is_invalid) {
            throw std::runtime_error{"Invalid VariantArrayRef assignment"};
        }
    }
};

// Common typedefs
using RealArrayRef = VariantArrayRef<float, double>;
using ComplexArrayRef = VariantArrayRef<float, double, std::complex<float>, std::complex<double>>;

/**
 Creates an actual container from an ArrayRef

 To be specialized by concrete containers. The container can adopt the reference,
 as is the case with Eigen::Map, or it can create a copy of the ArrayRef's data.
 */
template<class Container>
struct MakeContainer {
    // Intentionally unimplemented, specializations should do it
     static Container make(ArrayRef const&);
};

namespace detail {
    template<class Function, template<class> class Container, class Variant>
    using MatchResult = typename std::result_of<
        Function(Container<typename Variant::First>)
    >::type;

    template<class Result, template<class> class Container, class Function>
    Result try_match(ArrayRef const&, Function, TypeList<>) {
        throw std::runtime_error{"A match was not found"};
    };

    template<class Result, template<class> class Container, class Function,
             class Scalar, class... Tail>
    Result try_match(ArrayRef const& ref, Function lambda, TypeList<Scalar, Tail...>) {
        if (ref.tag == detail::get_tag<Scalar>()) {
            return lambda(MakeContainer<Container<Scalar>>::make(ref));
        } else {
            return try_match<Result, Container>(ref, lambda, TypeList<Tail...>{});
        }
    };

    template<class Function, template<class> class Container1, template<class> class Container2,
             class Variant1, class Variant2>
    using Match2Result = typename std::result_of<
        Function(Container1<typename Variant1::First>, Container2<typename Variant2::First>)
    >::type;

    template<class Result, template<class> class Container1, template<class> class Container2,
             class Function>
    Result try_match2(ArrayRef const&, ArrayRef const&, Function, TypeList<>) {
        throw std::runtime_error{"A match was not found"};
    };

    template<class Result, template<class> class Container1, template<class> class Container2,
             class Function, class Scalar1, class Scalar2, class... Tail>
    Result try_match2(ArrayRef const& ref1, ArrayRef const& ref2, Function lambda,
                      TypeList<TypeList<Scalar1, Scalar2>, Tail...>) {
        if (ref1.tag == detail::get_tag<Scalar1>() && ref2.tag == detail::get_tag<Scalar2>()) {
            return lambda(MakeContainer<Container1<Scalar1>>::make(ref1),
                          MakeContainer<Container2<Scalar2>>::make(ref2));
        } else {
            return try_match2<Result, Container1, Container2>(ref1, ref2, lambda,
                                                              TypeList<Tail...>{});
        }
    };

    template<class List>
    struct IsSamePrecision;

    template<class T1, class T2>
    struct IsSamePrecision<TypeList<T1, T2>> {
        static constexpr auto value = std::is_same<
            num::get_real_t<T1>, num::get_real_t<T2>
        >::value;
    };
} // namespace detail

/// Match an VariantArrayRef to a Container and pass it to Function
template<template<class> class Container, class Variant, class Function,
         class Result = detail::MatchResult<Function, Container, Variant>>
Result match(Variant ref, Function lambda) {
    return detail::try_match<Result, Container>(ref, lambda, typename Variant::Types{});
}

/// Match two ArrayRefs to Containers (considering all combinations) and pass them to Function
template<template<class> class Container1, template<class> class Container2,
         class Variant1, class Variant2, class Function,
         class Result = detail::Match2Result<Function, Container1, Container2, Variant1, Variant2>>
Result match2(Variant1 ref1, Variant2 ref2, Function lambda) {
    using List = tl::Combinations<typename Variant1::Types, typename Variant2::Types>;
    return detail::try_match2<Result, Container1, Container2>(ref1, ref2, lambda, List{});
}

/// Same as match2, but only considers matches where both scalar types have the same precision
template<template<class> class Container1, template<class> class Container2,
         class Variant1, class Variant2, class Function,
         class Result = detail::Match2Result<Function, Container1, Container2, Variant1, Variant2>>
Result match2sp(Variant1 ref1, Variant2 ref2, Function lambda) {
    using List = tl::Combinations<typename Variant1::Types, typename Variant2::Types>;
    using FilteredList = tl::Filter<List, detail::IsSamePrecision>;
    return detail::try_match2<Result, Container1, Container2>(ref1, ref2, lambda, FilteredList{});
}

}} // namespace tbm::num
<commit_msg>Fix missing include<commit_after>#pragma once
#include "detail/typelist.hpp"
#include "support/traits.hpp"

#include <algorithm>
#include <stdexcept>

namespace tbm { namespace num {

/// Array scalar type
enum class Tag {f32, cf32, f64, cf64, b, i8, i16, i32, i64, u8, u16, u32, u64};

namespace detail {
    template<class scalar_t> constexpr Tag get_tag();
    template<> constexpr Tag get_tag<float>()                { return Tag::f32;  }
    template<> constexpr Tag get_tag<std::complex<float>>()  { return Tag::cf32; }
    template<> constexpr Tag get_tag<double>()               { return Tag::f64;  }
    template<> constexpr Tag get_tag<std::complex<double>>() { return Tag::cf64; }
    template<> constexpr Tag get_tag<bool>()                 { return Tag::b;    }
    template<> constexpr Tag get_tag<std::int8_t>()          { return Tag::i8;   }
    template<> constexpr Tag get_tag<std::int16_t>()         { return Tag::i16;  }
    template<> constexpr Tag get_tag<std::int32_t>()         { return Tag::i32;  }
    template<> constexpr Tag get_tag<std::int64_t>()         { return Tag::i64;  }
    template<> constexpr Tag get_tag<std::uint8_t>()         { return Tag::u8;   }
    template<> constexpr Tag get_tag<std::uint16_t>()        { return Tag::u16;  }
    template<> constexpr Tag get_tag<std::uint32_t>()        { return Tag::u32;  }
    template<> constexpr Tag get_tag<std::uint64_t>()        { return Tag::u64;  }
} // namespace detail

/// Reference to any 1D or 2D array with any scalar type supported by Tag
struct ArrayRef {
    Tag const tag;
    bool const is_row_major;
    void const* const data;
    int const rows;
    int const cols;
};

/// Reference to an array which is limited to a set of scalar types
template<class Scalar, class... Scalars>
struct VariantArrayRef : ArrayRef {
    using First = Scalar;
    using Types = TypeList<Scalar, Scalars...>;

    VariantArrayRef(ArrayRef const& other) : ArrayRef(other) {
        auto const possible_tags = {detail::get_tag<Scalar>(), detail::get_tag<Scalars>()...};
        auto const is_invalid = std::none_of(possible_tags.begin(), possible_tags.end(),
                                             [&](Tag tag) { return other.tag == tag; });
        if (is_invalid) {
            throw std::runtime_error{"Invalid VariantArrayRef assignment"};
        }
    }
};

// Common typedefs
using RealArrayRef = VariantArrayRef<float, double>;
using ComplexArrayRef = VariantArrayRef<float, double, std::complex<float>, std::complex<double>>;

/**
 Creates an actual container from an ArrayRef

 To be specialized by concrete containers. The container can adopt the reference,
 as is the case with Eigen::Map, or it can create a copy of the ArrayRef's data.
 */
template<class Container>
struct MakeContainer {
    // Intentionally unimplemented, specializations should do it
     static Container make(ArrayRef const&);
};

namespace detail {
    template<class Function, template<class> class Container, class Variant>
    using MatchResult = typename std::result_of<
        Function(Container<typename Variant::First>)
    >::type;

    template<class Result, template<class> class Container, class Function>
    Result try_match(ArrayRef const&, Function, TypeList<>) {
        throw std::runtime_error{"A match was not found"};
    };

    template<class Result, template<class> class Container, class Function,
             class Scalar, class... Tail>
    Result try_match(ArrayRef const& ref, Function lambda, TypeList<Scalar, Tail...>) {
        if (ref.tag == detail::get_tag<Scalar>()) {
            return lambda(MakeContainer<Container<Scalar>>::make(ref));
        } else {
            return try_match<Result, Container>(ref, lambda, TypeList<Tail...>{});
        }
    };

    template<class Function, template<class> class Container1, template<class> class Container2,
             class Variant1, class Variant2>
    using Match2Result = typename std::result_of<
        Function(Container1<typename Variant1::First>, Container2<typename Variant2::First>)
    >::type;

    template<class Result, template<class> class Container1, template<class> class Container2,
             class Function>
    Result try_match2(ArrayRef const&, ArrayRef const&, Function, TypeList<>) {
        throw std::runtime_error{"A match was not found"};
    };

    template<class Result, template<class> class Container1, template<class> class Container2,
             class Function, class Scalar1, class Scalar2, class... Tail>
    Result try_match2(ArrayRef const& ref1, ArrayRef const& ref2, Function lambda,
                      TypeList<TypeList<Scalar1, Scalar2>, Tail...>) {
        if (ref1.tag == detail::get_tag<Scalar1>() && ref2.tag == detail::get_tag<Scalar2>()) {
            return lambda(MakeContainer<Container1<Scalar1>>::make(ref1),
                          MakeContainer<Container2<Scalar2>>::make(ref2));
        } else {
            return try_match2<Result, Container1, Container2>(ref1, ref2, lambda,
                                                              TypeList<Tail...>{});
        }
    };

    template<class List>
    struct IsSamePrecision;

    template<class T1, class T2>
    struct IsSamePrecision<TypeList<T1, T2>> {
        static constexpr auto value = std::is_same<
            num::get_real_t<T1>, num::get_real_t<T2>
        >::value;
    };
} // namespace detail

/// Match an VariantArrayRef to a Container and pass it to Function
template<template<class> class Container, class Variant, class Function,
         class Result = detail::MatchResult<Function, Container, Variant>>
Result match(Variant ref, Function lambda) {
    return detail::try_match<Result, Container>(ref, lambda, typename Variant::Types{});
}

/// Match two ArrayRefs to Containers (considering all combinations) and pass them to Function
template<template<class> class Container1, template<class> class Container2,
         class Variant1, class Variant2, class Function,
         class Result = detail::Match2Result<Function, Container1, Container2, Variant1, Variant2>>
Result match2(Variant1 ref1, Variant2 ref2, Function lambda) {
    using List = tl::Combinations<typename Variant1::Types, typename Variant2::Types>;
    return detail::try_match2<Result, Container1, Container2>(ref1, ref2, lambda, List{});
}

/// Same as match2, but only considers matches where both scalar types have the same precision
template<template<class> class Container1, template<class> class Container2,
         class Variant1, class Variant2, class Function,
         class Result = detail::Match2Result<Function, Container1, Container2, Variant1, Variant2>>
Result match2sp(Variant1 ref1, Variant2 ref2, Function lambda) {
    using List = tl::Combinations<typename Variant1::Types, typename Variant2::Types>;
    using FilteredList = tl::Filter<List, detail::IsSamePrecision>;
    return detail::try_match2<Result, Container1, Container2>(ref1, ref2, lambda, FilteredList{});
}

}} // namespace tbm::num
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "browse.h"

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#include "../build/browse_py.h"

void RunBrowsePython(State* /* state */, const char* ninja_command,
                     const char* initial_target) {
  // Fork off a Python process and have it run our code via its stdin.
  // (Actually the Python process becomes the parent.)
  int pipefd[2];
  if (pipe(pipefd) < 0) {
    perror("ninja: pipe");
    return;
  }

  pid_t pid = fork();
  if (pid < 0) {
    perror("ninja: fork");
    return;
  }

  if (pid > 0) {  // Parent.
    close(pipefd[1]);
    do {
      if (dup2(pipefd[0], 0) < 0) {
        perror("ninja: dup2");
        break;
      }

      // exec Python, telling it to run the program from stdin.
      const char* command[] = {
        "python", "-", ninja_command, initial_target, NULL
      };
      execvp(command[0], (char**)command);
      perror("ninja: execvp");
    } while (false);
    _exit(1);
  } else {  // Child.
    close(pipefd[0]);

    // Write the script file into the stdin of the Python process.
    ssize_t len = write(pipefd[1], kBrowsePy, sizeof(kBrowsePy));
    if (len < (ssize_t)sizeof(kBrowsePy))
      perror("ninja: write");
    close(pipefd[1]);
    exit(0);
  }
}
<commit_msg>Renamed `python` to `python2`.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "browse.h"

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#include "../build/browse_py.h"

void RunBrowsePython(State* /* state */, const char* ninja_command,
                     const char* initial_target) {
  // Fork off a Python process and have it run our code via its stdin.
  // (Actually the Python process becomes the parent.)
  int pipefd[2];
  if (pipe(pipefd) < 0) {
    perror("ninja: pipe");
    return;
  }

  pid_t pid = fork();
  if (pid < 0) {
    perror("ninja: fork");
    return;
  }

  if (pid > 0) {  // Parent.
    close(pipefd[1]);
    do {
      if (dup2(pipefd[0], 0) < 0) {
        perror("ninja: dup2");
        break;
      }

      // exec Python, telling it to run the program from stdin.
      const char* command[] = {
        "python2", "-", ninja_command, initial_target, NULL
      };
      execvp(command[0], (char**)command);
      perror("ninja: execvp");
    } while (false);
    _exit(1);
  } else {  // Child.
    close(pipefd[0]);

    // Write the script file into the stdin of the Python process.
    ssize_t len = write(pipefd[1], kBrowsePy, sizeof(kBrowsePy));
    if (len < (ssize_t)sizeof(kBrowsePy))
      perror("ninja: write");
    close(pipefd[1]);
    exit(0);
  }
}
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

  Copyright (c) GET / ENST Bretagne. All rights reserved.
  See GETCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

//  Software Guide : BeginCommandLineArgs
//    INPUTS: {ROI_QB_MUL_4.tif}
//    OUTPUTS: {SEMClassif.png}
//    4 40 5
//  Software Guide : EndCommandLineArgs
//
//  Software Guide : BeginLatex
//
// In this example, we present OTB's implementation of SEM, through the class
// \doxygen{otb}{SEMClassifier}. This class performs a stochastic version
// of the EM algorithm, but instead of inheriting from
// \doxygen{itk}{ExpectationMaximizationMixtureModelEstimator}, we choosed to
// inherit from \subdoxygen{itk}{Statistics}{ListSample< TSample >},
// in the same way as \doxygen{otb}{SVMClassifier}.
//
// The program begins with \doxygen{otb}{VectorImage} and outputs
// \doxygen{itb}{Image}. Then appropriate header files have to be included:
//
// Software Guide : EndLatex

#include <iostream>

#include <itkListSample.h>
#include <itkVector.h>
#include <itkVariableLengthVector.h>

//  Software Guide : BeginCodeSnippet
#include <otbImage.h>
#include <otbVectorImage.h>
#include <otbImageFileReader.h>
#include <otbImageFileWriter.h>
//  Software Guide : EndCodeSnippet

// Software Guide : BeginLatex
//
// \doxygen{otb}{SEMClassifier} performs estimation of mixture to fit the
// initial histogram. Actually, mixture of Gaussian pdf can be performed.
// Those generic pdf are treated in
// \subdoxygen{otb}{Statistics}{ModelComponentBase}. The Gaussian model
// is taken in charge with the class
// \subdoxygen{otb}{Statistics}{GaussianModelComponent}.
//
// Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
#include "otbGaussianModelComponent.h"
#include "otbSEMClassifier.h"
//  Software Guide : EndCodeSnippet

int main ( int argc, char * argv[] )
{
  try {

    if ( argc != 6 )
    {
      std::cerr << "Unsupervised Image Segmentation with SEM approach\n";
      std::cerr << argv[0] << " imageIn imgClassif num_of_class ";
      std::cerr << "nbIteration size_of_the_neighborhood\n";
      return EXIT_FAILURE ;
    }

// Software Guide : BeginLatex
//
// Input/Ouput images type are define in a classical way.
// In fact, a \doxygen{itk}{VariableLengthVector} is to be
// considered for the templated \code{MeasurementVectorType}, which
// will be used in the \code{ListSample} interface.
//
// Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    typedef double PixelType;

    typedef otb::VectorImage< PixelType, 2 > ImageType;
    typedef otb::ImageFileReader< ImageType > ReaderType;

    typedef itk::Image< unsigned char, 2 > OutputImageType;
    typedef otb::ImageFileWriter< OutputImageType > WriterType;
//  Software Guide : EndCodeSnippet

    char * fileNameIn = argv[1];
    char * fileNameImgInit = NULL;
    char * fileNameOut = argv[2];
    int numberOfClasses = atoi( argv[3] );
    int numberOfIteration = atoi( argv[4] );
    int neighborhood = atoi( argv[5] );
    double terminationThreshold = 1e-5;

    ReaderType::Pointer reader = ReaderType::New();
    reader->SetFileName( fileNameIn );
    reader->Update();

//  Software Guide : BeginLatex
//
// Once the input image is opened, the classifier may be initialised by
// \code{SmartPointer}.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    typedef otb::SEMClassifier< ImageType, OutputImageType > ClassifType;
    ClassifType::Pointer classifier = ClassifType::New();
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  Then, it follows, classical initialisations of the pipeline.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    classifier->SetNumberOfClasses( numberOfClasses ) ;
    classifier->SetMaximumIteration( numberOfIteration );
    classifier->SetNeighborhood( neighborhood );
    classifier->SetTerminationThreshold( terminationThreshold );
    classifier->SetSample( reader->GetOutput() );
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
// When an initial segmentation is available, the classifier may use it
// as image (of type \code{OutputImageType}) or as a
// \doxygen{itk}{SampleClassifier} result (of type
// \subdoxygen{itk}{Statistics}{MembershipSample< SampleType >}).
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    if ( fileNameImgInit != NULL )
    {
      typedef otb::ImageFileReader< OutputImageType > ImgInitReaderType;
      ImgInitReaderType::Pointer segReader = ImgInitReaderType::New();
      segReader->SetFileName( fileNameImgInit );
      segReader->Update();
      classifier->SetClassLabels( segReader->GetOutput() );
    }
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  By default, \doxygen{otb}{SEMClassifier} performs initialisation of
// \code{ModelComponentBase} by as many instanciation of
// \subdoxygen{otb}{Statistics}{GaussianModelComponent} as the number of
// classes to estimate in the mixture. Nevertheless, the user may add specific
// distribution into the mixture estimation. It is permited by the use of
// \code{AddComponent} for the given class number and the specific distribution.
//  Software Guide : EndLatex

    std::cerr << "Explicit component initialization\n";

//  Software Guide : BeginCodeSnippet
    typedef ClassifType::ClassSampleType ClassSampleType;
    typedef otb::Statistics::GaussianModelComponent< ClassSampleType >
        GaussianType;

    for ( int i = 0; i < numberOfClasses; i++ )
      classifier->AddComponent( i, GaussianType::New() );
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  Once the pipeline is instanciated. The segmentation by itself may be
// launched by using the \code{Update} function.
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    try {
      classifier->Update() ;
    }
//  Software Guide : EndCodeSnippet

    catch ( itk::ExceptionObject & err )
    {
      std::cerr << "ExceptionObject caught in " << argv[0] << "!\n";
      std::cerr << err << std::endl;
      return -1;
    }

//  Software Guide : BeginLatex
//
//  The segmentation may outputs a result of type
// \subdoxygen{itk}{Statistics}{MembershipSample< SampleType >} as it is the
// case for the \doxygen{otb}{SVMClassifier}. But when using
// \code{GetOutputImage} the output is directly an Image.
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    WriterType::Pointer writer = WriterType::New();
    writer->SetFileName( fileNameOut );
    writer->SetInput( classifier->GetOutputImage() );
    writer->Update();
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
// Figure \ref{fig::RESSEMCLASSIF} show the result of the SEM segmentation
// with 4 different classes and a contextual neighborhood of 3 pixels.
// \begin{figure}
//  \center
//  \includegraphics[width=0.35\textwidth]{SEMClassif.eps}
//  \itkcaption[SEM Classification results]{SEM Classification results.}
//  \label{fig:RESSEMCLASSIF}
// \end{figure}
//
// As soon as the segmentation is performed by an iterative stochastic
// process, it is worth verifying the output status: does the segmentation
// ends when it has converged or just at the limit of the iteration numbers.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    std::cerr << "Program terminated with a ";
    if ( classifier->GetTerminationCode() == ClassifType::CONVERGED )
      std::cerr << "converged ";
    else
      std::cerr << "not-converged ";
    std::cerr << "code...\n";
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  The text output gives for each class the parameters of the pdf (e.g. mean
// of each component of the class and there covariance matrix, in the case of a
// Gaussian mixture model).
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    classifier->Print( std::cerr );
//  Software Guide : EndCodeSnippet
  }
  catch( itk::ExceptionObject & err )
  {
    std::cerr << "Exception itk::ExceptionObject thrown !\n";
    std::cerr << err << std::endl;
    return EXIT_FAILURE;
  }
  catch( ... )
  {
    std::cerr << "Unknown exception thrown !\n";
    return EXIT_FAILURE;
  }
  return EXIT_SUCCESS;
}

<commit_msg>Rescale<commit_after>/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.

  Copyright (c) GET / ENST Bretagne. All rights reserved.
  See GETCopyright.txt for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

//  Software Guide : BeginCommandLineArgs
//    INPUTS: {ROI_QB_MUL_4.tif}
//    OUTPUTS: {SEMClassif.png}
//    4 40 5
//  Software Guide : EndCommandLineArgs
//
//  Software Guide : BeginLatex
//
// In this example, we present OTB's implementation of SEM, through the class
// \doxygen{otb}{SEMClassifier}. This class performs a stochastic version
// of the EM algorithm, but instead of inheriting from
// \doxygen{itk}{ExpectationMaximizationMixtureModelEstimator}, we choosed to
// inherit from \subdoxygen{itk}{Statistics}{ListSample< TSample >},
// in the same way as \doxygen{otb}{SVMClassifier}.
//
// The program begins with \doxygen{otb}{VectorImage} and outputs
// \doxygen{itb}{Image}. Then appropriate header files have to be included:
//
// Software Guide : EndLatex

#include <iostream>

#include "itkListSample.h"
#include "itkVector.h"
#include "itkVariableLengthVector.h"
#include "itkRescaleIntensityImageFilter.h"

//  Software Guide : BeginCodeSnippet
#include "otbImage.h"
#include "otbVectorImage.h"
#include "otbImageFileReader.h"
#include "otbImageFileWriter.h"
//  Software Guide : EndCodeSnippet

// Software Guide : BeginLatex
//
// \doxygen{otb}{SEMClassifier} performs estimation of mixture to fit the
// initial histogram. Actually, mixture of Gaussian pdf can be performed.
// Those generic pdf are treated in
// \subdoxygen{otb}{Statistics}{ModelComponentBase}. The Gaussian model
// is taken in charge with the class
// \subdoxygen{otb}{Statistics}{GaussianModelComponent}.
//
// Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
#include "otbGaussianModelComponent.h"
#include "otbSEMClassifier.h"
//  Software Guide : EndCodeSnippet

int main ( int argc, char * argv[] )
{
  try {

    if ( argc != 6 )
    {
      std::cerr << "Unsupervised Image Segmentation with SEM approach\n";
      std::cerr << argv[0] << " imageIn imgClassif num_of_class ";
      std::cerr << "nbIteration size_of_the_neighborhood\n";
      return EXIT_FAILURE ;
    }

// Software Guide : BeginLatex
//
// Input/Ouput images type are define in a classical way.
// In fact, a \doxygen{itk}{VariableLengthVector} is to be
// considered for the templated \code{MeasurementVectorType}, which
// will be used in the \code{ListSample} interface.
//
// Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    typedef double PixelType;

    typedef otb::VectorImage< PixelType, 2 > ImageType;
    typedef otb::ImageFileReader< ImageType > ReaderType;

    typedef itk::Image< unsigned char, 2 > OutputImageType;
    typedef otb::ImageFileWriter< OutputImageType > WriterType;
//  Software Guide : EndCodeSnippet

    char * fileNameIn = argv[1];
    char * fileNameImgInit = NULL;
    char * fileNameOut = argv[2];
    int numberOfClasses = atoi( argv[3] );
    int numberOfIteration = atoi( argv[4] );
    int neighborhood = atoi( argv[5] );
    double terminationThreshold = 1e-5;

    ReaderType::Pointer reader = ReaderType::New();
    reader->SetFileName( fileNameIn );
    reader->Update();

//  Software Guide : BeginLatex
//
// Once the input image is opened, the classifier may be initialised by
// \code{SmartPointer}.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    typedef otb::SEMClassifier< ImageType, OutputImageType > ClassifType;
    ClassifType::Pointer classifier = ClassifType::New();
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  Then, it follows, classical initialisations of the pipeline.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    classifier->SetNumberOfClasses( numberOfClasses ) ;
    classifier->SetMaximumIteration( numberOfIteration );
    classifier->SetNeighborhood( neighborhood );
    classifier->SetTerminationThreshold( terminationThreshold );
    classifier->SetSample( reader->GetOutput() );
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
// When an initial segmentation is available, the classifier may use it
// as image (of type \code{OutputImageType}) or as a
// \doxygen{itk}{SampleClassifier} result (of type
// \subdoxygen{itk}{Statistics}{MembershipSample< SampleType >}).
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    if ( fileNameImgInit != NULL )
    {
      typedef otb::ImageFileReader< OutputImageType > ImgInitReaderType;
      ImgInitReaderType::Pointer segReader = ImgInitReaderType::New();
      segReader->SetFileName( fileNameImgInit );
      segReader->Update();
      classifier->SetClassLabels( segReader->GetOutput() );
    }
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  By default, \doxygen{otb}{SEMClassifier} performs initialisation of
// \code{ModelComponentBase} by as many instanciation of
// \subdoxygen{otb}{Statistics}{GaussianModelComponent} as the number of
// classes to estimate in the mixture. Nevertheless, the user may add specific
// distribution into the mixture estimation. It is permited by the use of
// \code{AddComponent} for the given class number and the specific distribution.
//  Software Guide : EndLatex

    std::cerr << "Explicit component initialization\n";

//  Software Guide : BeginCodeSnippet
    typedef ClassifType::ClassSampleType ClassSampleType;
    typedef otb::Statistics::GaussianModelComponent< ClassSampleType >
        GaussianType;

    for ( int i = 0; i < numberOfClasses; i++ )
      classifier->AddComponent( i, GaussianType::New() );
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  Once the pipeline is instanciated. The segmentation by itself may be
// launched by using the \code{Update} function.
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    try {
      classifier->Update() ;
    }
//  Software Guide : EndCodeSnippet

    catch ( itk::ExceptionObject & err )
    {
      std::cerr << "ExceptionObject caught in " << argv[0] << "!\n";
      std::cerr << err << std::endl;
      return -1;
    }

//  Software Guide : BeginLatex
//
//  The segmentation may outputs a result of type
// \subdoxygen{itk}{Statistics}{MembershipSample< SampleType >} as it is the
// case for the \doxygen{otb}{SVMClassifier}. But when using
// \code{GetOutputImage} the output is directly an Image.
//
// Only for visualization purposes, we choose to rescale the image of
// classes before sving it to a file. We will use the
// \doxygen{itk}{RescaleIntensityImageFilter} for this purpose.
//
// Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    typedef itk::RescaleIntensityImageFilter< OutputImageType,
      OutputImageType > RescalerType;

    RescalerType::Pointer rescaler = RescalerType::New();

    rescaler->SetOutputMinimum( itk::NumericTraits< unsigned char >::min());
    rescaler->SetOutputMaximum( itk::NumericTraits< unsigned char >::max());

    rescaler->SetInput( classifier->GetOutputImage() );

    WriterType::Pointer writer = WriterType::New();
    writer->SetFileName( fileNameOut );
    writer->SetInput( rescaler->GetOutput() );
    writer->Update();
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
// Figure \ref{fig::RESSEMCLASSIF} show the result of the SEM segmentation
// with 4 different classes and a contextual neighborhood of 3 pixels.
// \begin{figure}
//  \center
//  \includegraphics[width=0.35\textwidth]{SEMClassif.eps}
//  \itkcaption[SEM Classification results]{SEM Classification results.}
//  \label{fig:RESSEMCLASSIF}
// \end{figure}
//
// As soon as the segmentation is performed by an iterative stochastic
// process, it is worth verifying the output status: does the segmentation
// ends when it has converged or just at the limit of the iteration numbers.
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    std::cerr << "Program terminated with a ";
    if ( classifier->GetTerminationCode() == ClassifType::CONVERGED )
      std::cerr << "converged ";
    else
      std::cerr << "not-converged ";
    std::cerr << "code...\n";
//  Software Guide : EndCodeSnippet

//  Software Guide : BeginLatex
//
//  The text output gives for each class the parameters of the pdf (e.g. mean
// of each component of the class and there covariance matrix, in the case of a
// Gaussian mixture model).
//
//  Software Guide : EndLatex

//  Software Guide : BeginCodeSnippet
    classifier->Print( std::cerr );
//  Software Guide : EndCodeSnippet
  }
  catch( itk::ExceptionObject & err )
  {
    std::cerr << "Exception itk::ExceptionObject thrown !\n";
    std::cerr << err << std::endl;
    return EXIT_FAILURE;
  }
  catch( ... )
  {
    std::cerr << "Unknown exception thrown !\n";
    return EXIT_FAILURE;
  }
  return EXIT_SUCCESS;
}

<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/audio_processing/transient/transient_detector.h"

#include <sstream>
#include <string>

#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/modules/audio_processing/transient/common.h"
#include "webrtc/modules/audio_processing/transient/file_utils.h"
#include "webrtc/system_wrappers/interface/file_wrapper.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/typedefs.h"

namespace webrtc {

static const int kSampleRatesHz[] = {ts::kSampleRate8kHz,
                                     ts::kSampleRate16kHz,
                                     ts::kSampleRate32kHz,
                                     ts::kSampleRate48kHz};
static const size_t kNumberOfSampleRates =
    sizeof(kSampleRatesHz) / sizeof(*kSampleRatesHz);

// Fails Android ARM64. https://code.google.com/p/webrtc/issues/detail?id=4200
#if defined(WEBRTC_ANDROID) && defined(__aarch64__)
#define MAYBE_CorrectnessBasedOnFiles DISABLED_CorrectnessBasedOnFiles
#else
#define MAYBE_CorrectnessBasedOnFiles CorrectnessBasedOnFiles
#endif
// This test is for the correctness of the transient detector.
// Checks the results comparing them with the ones stored in the detect files in
// the directory: resources/audio_processing/transient/
// The files contain all the results in double precision (Little endian).
// The audio files used with different sample rates are stored in the same
// directory.
TEST(TransientDetectorTest, MAYBE_CorrectnessBasedOnFiles) {
  for (size_t i = 0; i < kNumberOfSampleRates; ++i) {
    int sample_rate_hz = kSampleRatesHz[i];

    // Prepare detect file.
    std::stringstream detect_file_name;
    detect_file_name << "audio_processing/transient/detect"
                     << (sample_rate_hz / 1000) << "kHz";

    scoped_ptr<FileWrapper> detect_file(FileWrapper::Create());

    detect_file->OpenFile(
        test::ResourcePath(detect_file_name.str(), "dat").c_str(),
        true,    // Read only.
        false,   // No loop.
        false);  // No text.

    bool file_opened = detect_file->Open();
    ASSERT_TRUE(file_opened) << "File could not be opened.\n"
          << detect_file_name.str().c_str();

    // Prepare audio file.
    std::stringstream audio_file_name;
    audio_file_name << "audio_processing/transient/audio"
                    << (sample_rate_hz / 1000) << "kHz";

    scoped_ptr<FileWrapper> audio_file(FileWrapper::Create());

    audio_file->OpenFile(
        test::ResourcePath(audio_file_name.str(), "pcm").c_str(),
        true,    // Read only.
        false,   // No loop.
        false);  // No text.

    // Create detector.
    TransientDetector detector(sample_rate_hz);

    const size_t buffer_length = sample_rate_hz * ts::kChunkSizeMs / 1000;
    scoped_ptr<float[]> buffer(new float[buffer_length]);

    const float kTolerance = 0.01f;

    size_t frames_read = 0;

    while (ReadInt16FromFileToFloatBuffer(audio_file.get(),
                                          buffer_length,
                                          buffer.get()) == buffer_length) {
      ++frames_read;

      float detector_value =
          detector.Detect(buffer.get(), buffer_length, NULL, 0);
      double file_value;
      ASSERT_EQ(1u, ReadDoubleBufferFromFile(detect_file.get(), 1, &file_value))
          << "Detect test file is malformed.\n";

      // Compare results with data from the matlab test file.
      EXPECT_NEAR(file_value, detector_value, kTolerance) << "Frame: "
          << frames_read;
    }

    detect_file->CloseFile();
    audio_file->CloseFile();
  }
}

}  // namespace webrtc
<commit_msg>Fix TransientDetectorTest in modules_unittests on Android ARM64<commit_after>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/audio_processing/transient/transient_detector.h"

#include <sstream>
#include <string>

#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/modules/audio_processing/transient/common.h"
#include "webrtc/modules/audio_processing/transient/file_utils.h"
#include "webrtc/system_wrappers/interface/file_wrapper.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/typedefs.h"

namespace webrtc {

static const int kSampleRatesHz[] = {ts::kSampleRate8kHz,
                                     ts::kSampleRate16kHz,
                                     ts::kSampleRate32kHz,
                                     ts::kSampleRate48kHz};
static const size_t kNumberOfSampleRates =
    sizeof(kSampleRatesHz) / sizeof(*kSampleRatesHz);

// This test is for the correctness of the transient detector.
// Checks the results comparing them with the ones stored in the detect files in
// the directory: resources/audio_processing/transient/
// The files contain all the results in double precision (Little endian).
// The audio files used with different sample rates are stored in the same
// directory.
TEST(TransientDetectorTest, CorrectnessBasedOnFiles) {
  for (size_t i = 0; i < kNumberOfSampleRates; ++i) {
    int sample_rate_hz = kSampleRatesHz[i];

    // Prepare detect file.
    std::stringstream detect_file_name;
    detect_file_name << "audio_processing/transient/detect"
                     << (sample_rate_hz / 1000) << "kHz";

    scoped_ptr<FileWrapper> detect_file(FileWrapper::Create());

    detect_file->OpenFile(
        test::ResourcePath(detect_file_name.str(), "dat").c_str(),
        true,    // Read only.
        false,   // No loop.
        false);  // No text.

    bool file_opened = detect_file->Open();
    ASSERT_TRUE(file_opened) << "File could not be opened.\n"
          << detect_file_name.str().c_str();

    // Prepare audio file.
    std::stringstream audio_file_name;
    audio_file_name << "audio_processing/transient/audio"
                    << (sample_rate_hz / 1000) << "kHz";

    scoped_ptr<FileWrapper> audio_file(FileWrapper::Create());

    audio_file->OpenFile(
        test::ResourcePath(audio_file_name.str(), "pcm").c_str(),
        true,    // Read only.
        false,   // No loop.
        false);  // No text.

    // Create detector.
    TransientDetector detector(sample_rate_hz);

    const size_t buffer_length = sample_rate_hz * ts::kChunkSizeMs / 1000;
    scoped_ptr<float[]> buffer(new float[buffer_length]);

    const float kTolerance = 0.02f;

    size_t frames_read = 0;

    while (ReadInt16FromFileToFloatBuffer(audio_file.get(),
                                          buffer_length,
                                          buffer.get()) == buffer_length) {
      ++frames_read;

      float detector_value =
          detector.Detect(buffer.get(), buffer_length, NULL, 0);
      double file_value;
      ASSERT_EQ(1u, ReadDoubleBufferFromFile(detect_file.get(), 1, &file_value))
          << "Detect test file is malformed.\n";

      // Compare results with data from the matlab test file.
      EXPECT_NEAR(file_value, detector_value, kTolerance) << "Frame: "
          << frames_read;
    }

    detect_file->CloseFile();
    audio_file->CloseFile();
  }
}

}  // namespace webrtc
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include "src/Options.hpp"
#include "src/Cluster.hpp"
#include <map>

using namespace std;
int main(int argc, char *argv[]){

    map<string,string > options2;

    options2["eps_iter"]                        = "1e-4";
    /* Dirichlet_explicit, Dirichlet_implicit, lumped */
    options2["preconditioner"]                  = "lumped";
    //options2["preconditioner"]                  = "Dirichlet_explicit";
    options2["path2data"]                       =  "data/";
    options2["young_modulus"]                   =  "1000";
    options2["poissons_ratio"]                  =  "0.3";

    /* dissection | pardiso */
    options2["linear_solver"]                   =  "dissection";

    /* {0, 1, 2, 3 }                                            */
    options2["print_matrices"]                  =  "0";

    /* ker, cor, all                                            */
    options2["typeBc"]                          =  "cor";

    /* true,  false                                             */
    options2["Ac_extended_by_kerGc"]            =  "false";

    /* true,  false                                             */
    options2["GcTGc_assembl_block_by_block"]    =  "true";

    /* true, false                                              */
    options2["Bc_fullRank"]                     =  "true";

    /* true, false                                              */
    options2["create_analytic_ker_K"]           =  "false";

    options2["Nx"]                              =  "2";
    options2["Ny"]                              =  "2";
    options2["Nz"]                              =  "2";
    options2["nx"]                              =  "5";
    options2["ny"]                              =  "5";
    options2["nz"]                              =  "5";

    printf("+++++++++++++++++++++++++++++++++++      %s\n", options2["path2data"].c_str());

    if (options2["linear_solver"] ==  "pardiso" &&
            options2["create_analytic_ker_K"] == "false"){
        cout <<"###########################################################" << endl;
        cout << "if pardiso, kernel must be provided by analytical formula" << endl;
        cout <<"###########################################################" << endl;
        options2["create_analytic_ker_K"] = "true";
    }

    if (argc > 1) {
       options2["Nx"]   =  (argv[1]);
    }
    if (argc > 2) {
       options2["Ny"]   =  (argv[2]);
    }
    if (argc > 3) {
       options2["Nz"]   =  (argv[3]);
    }
    if (argc > 4) {
       options2["nx"]   =  (argv[4]);
    }
    if (argc > 5) {
       options2["ny"]   =  (argv[5]);
    }
    if (argc > 6) {
       options2["nz"]   =  (argv[6]);
    }

    cout << argv[0] << endl;
    Options options;
    Cluster cluster(options,options2);
    cout << "----------------- done -----------------\n" ;

// HOW TO PRINT OPTIONS2

//    cout << "\t\t\toptions2.size() "  << options2.size() << endl;
//    for (map<string,string>::const_iterator it=options2.begin(); it!=options2.end(); ++it)
//        cout << "\t\t\t" << it->first << " => " << it->second << '\n';

    return 0;
}

<commit_msg>Bc from kernels set up<commit_after>#include <iostream>
#include <string>
#include "src/Options.hpp"
#include "src/Cluster.hpp"
#include <map>

using namespace std;
int main(int argc, char *argv[]){

    map<string,string > options2;

    options2["eps_iter"]                        = "1e-4";
    /* Dirichlet_explicit, Dirichlet_implicit, lumped */
    options2["preconditioner"]                  = "lumped";
    //options2["preconditioner"]                  = "Dirichlet_explicit";
    options2["path2data"]                       =  "data/";
    options2["young_modulus"]                   =  "1000";
    options2["poissons_ratio"]                  =  "0.3";

    /* dissection | pardiso */
    options2["linear_solver"]                   =  "dissection";

    /* {0, 1, 2, 3 }                                            */
    options2["print_matrices"]                  =  "0";

    /* ker, cor, all                                            */
    options2["typeBc"]                          =  "ker";

    /* true,  false                                             */
    options2["Ac_extended_by_kerGc"]            =  "false";

    /* true,  false                                             */
    options2["GcTGc_assembl_block_by_block"]    =  "true";

    /* true, false                                              */
    options2["Bc_fullRank"]                     =  "true";

    /* true, false                                              */
    options2["create_analytic_ker_K"]           =  "false";

    options2["Nx"]                              =  "2";
    options2["Ny"]                              =  "2";
    options2["Nz"]                              =  "2";
    options2["nx"]                              =  "5";
    options2["ny"]                              =  "5";
    options2["nz"]                              =  "5";

    printf("+++++++++++++++++++++++++++++++++++      %s\n", options2["path2data"].c_str());

    if (options2["linear_solver"] ==  "pardiso" &&
            options2["create_analytic_ker_K"] == "false"){
        cout <<"###########################################################" << endl;
        cout << "if pardiso, kernel must be provided by analytical formula" << endl;
        cout <<"###########################################################" << endl;
        options2["create_analytic_ker_K"] = "true";
    }

    if (argc > 1) {
       options2["Nx"]   =  (argv[1]);
    }
    if (argc > 2) {
       options2["Ny"]   =  (argv[2]);
    }
    if (argc > 3) {
       options2["Nz"]   =  (argv[3]);
    }
    if (argc > 4) {
       options2["nx"]   =  (argv[4]);
    }
    if (argc > 5) {
       options2["ny"]   =  (argv[5]);
    }
    if (argc > 6) {
       options2["nz"]   =  (argv[6]);
    }

    cout << argv[0] << endl;
    Options options;
    Cluster cluster(options,options2);
    cout << "----------------- done -----------------\n" ;

// HOW TO PRINT OPTIONS2

//    cout << "\t\t\toptions2.size() "  << options2.size() << endl;
//    for (map<string,string>::const_iterator it=options2.begin(); it!=options2.end(); ++it)
//        cout << "\t\t\t" << it->first << " => " << it->second << '\n';

    return 0;
}

<|endoftext|>


<commit_before>/*
 * Steve Vaught
 * This program uses the Unlicense
 */

// == Include statements ==

#include <iostream>
#include <cmath>
#include <string>
#include <fstream>
#include <iterator>
#include <vector>
#include <random>
using namespace std;

// == Function prototypes ==

string buildRandomString(int length);
void mutate(string& scrambled, const string target);

template<typename Iter, typename RandomGenerator>
Iter select_randomly(Iter start, Iter end, RandomGenerator& g) {
  std::uniform_int_distribution<> dis(0,std::distance(start, end) -1);
  std::advance(start,dis(g));
  return start;
}

template<typename Iter>
Iter select_randomly(Iter start, Iter end) {
  static std::random_device rd;
  static std::mt19937 gen(rd());
  return select_randomly(start,end,gen);
}

// == Main ==
int main(){

  string target("Methinks it is like a weasel");

  string scrambled = buildRandomString( target.length() );

  while( target.compare(scrambled) != 0 ) {
    mutate(scrambled, target);
  }

  return 0;
}

string buildRandomString(int length){

  vector<char> alphanumerics
  {
    'A','B','C','D','E','F',
    'G','H','I','J','K',
    'L','M','N','O','P',
    'Q','R','S','T','U',
    'V','W','X','Y','Z',
    'a','b','c','d','e','f',
    'g','h','i','j','k',
    'l','m','n','o','p',
    'q','r','s','t','u',
    'v','w','x','y','z',
    ' '
  }; //include space in the alphabet

  string randstring = "";
  for (auto i = 0; i < length; i++){
    randstring.push_back( *select_randomly(alphanumerics.begin(), alphanumerics.end()) );
  }

  return randstring;

} //end buildRandomString

void mutate(string& scrambled, const string target){

}
<commit_msg>more work on score, mutate<commit_after>/*
 * Steve Vaught
 * This program uses the Unlicense
 */

// == Include statements ==

#include <iostream>
#include <cmath>
#include <string>
#include <fstream>
#include <iterator>
#include <vector>
#include <random>
using namespace std;

// == Function prototypes ==

string buildRandomString(int length);
void mutate(string& scrambled, const string target);
const int score(const string scrambled, const string target);

template<typename Iter, typename RandomGenerator>
Iter select_randomly(Iter start, Iter end, RandomGenerator& g) {
  std::uniform_int_distribution<> dis(0,std::distance(start, end) -1);
  std::advance(start,dis(g));
  return start;
}

template<typename Iter>
Iter select_randomly(Iter start, Iter end) {
  static std::random_device rd;
  static std::mt19937 gen(rd());
  return select_randomly(start,end,gen);
}

// == Main ==
int main(){

  string target("Methinks it is like a weasel");

  string scrambled = buildRandomString( target.length() );

  int attempts = 0;
  while( attempts < 3 && target.compare(scrambled) != 0 ) {
    cout << ++attempts << ": " << scrambled << " does not match! Score: " << score(scrambled,target) << endl;
    mutate(scrambled, target);
  }

  cout << scrambled << " matches!" << endl;

  return 0;
}

vector<char> getAlphanumerics(){
  return vector<char> {
    'A','B','C','D','E','F',
    'G','H','I','J','K',
    'L','M','N','O','P',
    'Q','R','S','T','U',
    'V','W','X','Y','Z',
    'a','b','c','d','e','f',
    'g','h','i','j','k',
    'l','m','n','o','p',
    'q','r','s','t','u',
    'v','w','x','y','z',
    ' '
  }; //include space in the alphabet
}

string buildRandomString(int length){

  vector<char> alphanumerics = getAlphanumerics();

  string randstring = "";
  for (auto i = 0; i < length; i++){
    randstring.push_back( *select_randomly(alphanumerics.begin(), alphanumerics.end()) );
  }

  return randstring;

} //end buildRandomString

void mutate(string& scrambled, const string target){
  scrambled = buildRandomString( target.length() );
  return;
}

const int score(const string scrambled, const string target){

  string::const_iterator it = scrambled.begin();
  string::const_iterator targetit = target.begin();
  //int score = 0;

  for (; it < scrambled.end(); it++, targetit++){
    cout << "Scrambled: " << *it << " | Target: " << *targetit << endl;

  }
  cout << endl;
  return 0;
}

<|endoftext|>