<commit_before>/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Copyright (C) 2017 ScyllaDB
 */

#pragma once

#include <map>
#include <seastar/core/future.hh>
#include "reader_permit.hh"

using namespace seastar;

class flat_mutation_reader;

/// Specific semaphore for controlling reader concurrency
///
/// Use `make_permit()` to create a permit to track the resource consumption
/// of a specific read. The permit should be created before the read is even
/// started so it is available to track resource consumption from the start.
/// Reader concurrency is dual limited by count and memory.
/// The semaphore can be configured with the desired limits on
/// construction. New readers will only be admitted when there is both
/// enough count and memory units available. Readers are admitted in
/// FIFO order.
/// Semaphore's `name` must be provided in ctor and its only purpose is
/// to increase readability of exceptions: both timeout exceptions and
/// queue overflow exceptions (read below) include this `name` in messages.
/// It's also possible to specify the maximum allowed number of waiting
/// readers by the `max_queue_length` constructor parameter. When the
/// number of waiting readers becomes equal or greater than
/// `max_queue_length` (upon calling `wait_admission()`) an exception of
/// type `std::runtime_error` is thrown. Optionally, some additional
/// code can be executed just before throwing (`prethrow_action`
/// constructor parameter).
class reader_concurrency_semaphore {
public:
    using resources = reader_resources;

    friend class reader_permit;

    enum class evict_reason {
        permit, // evicted due to permit shortage
        time, // evicted due to expiring ttl
        manual, // evicted manually via `try_evict_one_inactive_read()`
    };

    using eviction_notify_handler = noncopyable_function<void(evict_reason)>;

    class inactive_read_handle {
        reader_concurrency_semaphore* _sem = nullptr;
        uint64_t _id = 0;

        friend class reader_concurrency_semaphore;

        explicit inactive_read_handle(reader_concurrency_semaphore& sem, uint64_t id)
            : _sem(&sem), _id(id) {
        }
    public:
        inactive_read_handle() = default;
        inactive_read_handle(inactive_read_handle&& o) : _sem(std::exchange(o._sem, nullptr)), _id(std::exchange(o._id, 0)) {
        }
        inactive_read_handle& operator=(inactive_read_handle&& o) {
            _sem = std::exchange(o._sem, nullptr);
            _id = std::exchange(o._id, 0);
            return *this;
        }
        explicit operator bool() const {
            return bool(_id);
        }
    };

    struct stats {
        // The number of inactive reads evicted to free up permits.
        uint64_t permit_based_evictions = 0;
        // The number of inactive reads evicted due to expiring.
        uint64_t time_based_evictions = 0;
        // The number of inactive reads currently registered.
        uint64_t inactive_reads = 0;
        // Total number of successful reads executed through this semaphore.
        uint64_t total_successful_reads = 0;
        // Total number of failed reads executed through this semaphore.
        uint64_t total_failed_reads = 0;
        // Total number of reads rejected because the admission queue reached its max capacity
        uint64_t total_reads_shed_due_to_overload = 0;
    };

    struct permit_list;

private:
    struct entry {
        promise<reader_permit::resource_units> pr;
        reader_permit permit;
        resources res;
        entry(promise<reader_permit::resource_units>&& pr, reader_permit permit, resources r)
            : pr(std::move(pr)), permit(std::move(permit)), res(r) {}
    };

    class expiry_handler {
        reader_concurrency_semaphore& _semaphore;
    public:
        explicit expiry_handler(reader_concurrency_semaphore& semaphore)
            : _semaphore(semaphore) {}
        void operator()(entry& e) noexcept;
    };

    struct inactive_read {
        std::unique_ptr<flat_mutation_reader> reader;
        eviction_notify_handler notify_handler;
        std::optional<timer<lowres_clock>> ttl_timer;

        explicit inactive_read(flat_mutation_reader);
        inactive_read(inactive_read&&) = default;
        ~inactive_read();
    };

    using inactive_reads_type = std::map<uint64_t, inactive_read>;

private:
    const resources _initial_resources;
    resources _resources;

    expiring_fifo<entry, expiry_handler, db::timeout_clock> _wait_list;

    sstring _name;
    size_t _max_queue_length = std::numeric_limits<size_t>::max();
    std::function<void()> _prethrow_action;
    uint64_t _next_id = 1;
    inactive_reads_type _inactive_reads;
    stats _stats;
    std::unique_ptr<permit_list> _permit_list;

private:
    inactive_reads_type::iterator evict(inactive_reads_type::iterator it, evict_reason reason);

    bool has_available_units(const resources& r) const;

    bool may_proceed(const resources& r) const;

    future<reader_permit::resource_units> do_wait_admission(reader_permit permit, size_t memory, db::timeout_clock::time_point timeout);

public:
    struct no_limits { };

    reader_concurrency_semaphore(int count,
            ssize_t memory,
            sstring name,
            size_t max_queue_length = std::numeric_limits<size_t>::max(),
            std::function<void()> prethrow_action = nullptr);

    /// Create a semaphore with practically unlimited count and memory.
    ///
    /// And conversely, no queue limit either.
    explicit reader_concurrency_semaphore(no_limits, sstring name = "unlimited reader_concurrency_semaphore");

    ~reader_concurrency_semaphore();

    reader_concurrency_semaphore(const reader_concurrency_semaphore&) = delete;
    reader_concurrency_semaphore& operator=(const reader_concurrency_semaphore&) = delete;

    reader_concurrency_semaphore(reader_concurrency_semaphore&&) = delete;
    reader_concurrency_semaphore& operator=(reader_concurrency_semaphore&&) = delete;

    /// Returns the name of the semaphore
    ///
    /// If the semaphore has no name, "unnamed reader concurrency semaphore" is returned.
    std::string_view name() const {
        return _name.empty() ? "unnamed reader concurrency semaphore" : std::string_view(_name);
    }

    /// Register an inactive read.
    ///
    /// The semaphore will evict this read when there is a shortage of
    /// permits. This might be immediate, during this register call.
    /// Clients can use the returned handle to unregister the read, when it
    /// stops being inactive and hence evictable.
    ///
    /// An inactive read is an object implementing the `inactive_read`
    /// interface.
    /// The semaphore takes ownership of the created object and destroys it if
    /// it is evicted.
    inactive_read_handle register_inactive_read(flat_mutation_reader ir, eviction_notify_handler handler = {});
    inactive_read_handle register_inactive_read(flat_mutation_reader ir, std::chrono::seconds ttl, eviction_notify_handler handler = {});

    /// Unregister the previously registered inactive read.
    ///
    /// If the read was not evicted, the inactive read object, passed in to the
    /// register call, will be returned. Otherwise a nullptr is returned.
    std::unique_ptr<flat_mutation_reader> unregister_inactive_read(inactive_read_handle irh);

    /// Try to evict an inactive read.
    ///
    /// Return true if an inactive read was evicted and false otherwise
    /// (if there was no reader to evict).
    bool try_evict_one_inactive_read();

    void clear_inactive_reads() {
        _inactive_reads.clear();
    }

    const stats& get_stats() const {
        return _stats;
    }

    stats& get_stats() {
        return _stats;
    }

    /// Make a permit
    ///
    /// The permit is associated with a schema, which is the schema of the table
    /// the read is executed against, and the operation name, which should be a
    /// name such that we can identify the operation which created this permit.
    /// Ideally this should be a unique enough name that we not only can identify
    /// the kind of read, but the exact code-path that was taken.
    ///
    /// Some permits cannot be associated with any table, so passing nullptr as
    /// the schema parameter is allowed.
    reader_permit make_permit(const schema* const schema, const char* const op_name);
    reader_permit make_permit(const schema* const schema, sstring&& op_name);

    const resources initial_resources() const {
        return _initial_resources;
    }

    bool is_unlimited() const {
        return _initial_resources == reader_resources{std::numeric_limits<int>::max(), std::numeric_limits<ssize_t>::max()};
    }

    const resources available_resources() const {
        return _resources;
    }

    void consume(resources r) {
        _resources -= r;
    }

    void signal(const resources& r) noexcept;

    size_t waiters() const {
        return _wait_list.size();
    }

    void broken(std::exception_ptr ex);
};
<commit_msg>reader_concurrency_semaphore: inactive_read_handle: mark methods noexcept<commit_after>/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Copyright (C) 2017 ScyllaDB
 */

#pragma once

#include <map>
#include <seastar/core/future.hh>
#include "reader_permit.hh"

using namespace seastar;

class flat_mutation_reader;

/// Specific semaphore for controlling reader concurrency
///
/// Use `make_permit()` to create a permit to track the resource consumption
/// of a specific read. The permit should be created before the read is even
/// started so it is available to track resource consumption from the start.
/// Reader concurrency is dual limited by count and memory.
/// The semaphore can be configured with the desired limits on
/// construction. New readers will only be admitted when there is both
/// enough count and memory units available. Readers are admitted in
/// FIFO order.
/// Semaphore's `name` must be provided in ctor and its only purpose is
/// to increase readability of exceptions: both timeout exceptions and
/// queue overflow exceptions (read below) include this `name` in messages.
/// It's also possible to specify the maximum allowed number of waiting
/// readers by the `max_queue_length` constructor parameter. When the
/// number of waiting readers becomes equal or greater than
/// `max_queue_length` (upon calling `wait_admission()`) an exception of
/// type `std::runtime_error` is thrown. Optionally, some additional
/// code can be executed just before throwing (`prethrow_action`
/// constructor parameter).
class reader_concurrency_semaphore {
public:
    using resources = reader_resources;

    friend class reader_permit;

    enum class evict_reason {
        permit, // evicted due to permit shortage
        time, // evicted due to expiring ttl
        manual, // evicted manually via `try_evict_one_inactive_read()`
    };

    using eviction_notify_handler = noncopyable_function<void(evict_reason)>;

    class inactive_read_handle {
        reader_concurrency_semaphore* _sem = nullptr;
        uint64_t _id = 0;

        friend class reader_concurrency_semaphore;

        explicit inactive_read_handle(reader_concurrency_semaphore& sem, uint64_t id) noexcept
            : _sem(&sem), _id(id) {
        }
    public:
        inactive_read_handle() = default;
        inactive_read_handle(inactive_read_handle&& o) noexcept
            : _sem(std::exchange(o._sem, nullptr)), _id(std::exchange(o._id, 0)) {
        }
        inactive_read_handle& operator=(inactive_read_handle&& o) noexcept {
            _sem = std::exchange(o._sem, nullptr);
            _id = std::exchange(o._id, 0);
            return *this;
        }
        explicit operator bool() const noexcept {
            return bool(_id);
        }
    };

    struct stats {
        // The number of inactive reads evicted to free up permits.
        uint64_t permit_based_evictions = 0;
        // The number of inactive reads evicted due to expiring.
        uint64_t time_based_evictions = 0;
        // The number of inactive reads currently registered.
        uint64_t inactive_reads = 0;
        // Total number of successful reads executed through this semaphore.
        uint64_t total_successful_reads = 0;
        // Total number of failed reads executed through this semaphore.
        uint64_t total_failed_reads = 0;
        // Total number of reads rejected because the admission queue reached its max capacity
        uint64_t total_reads_shed_due_to_overload = 0;
    };

    struct permit_list;

private:
    struct entry {
        promise<reader_permit::resource_units> pr;
        reader_permit permit;
        resources res;
        entry(promise<reader_permit::resource_units>&& pr, reader_permit permit, resources r)
            : pr(std::move(pr)), permit(std::move(permit)), res(r) {}
    };

    class expiry_handler {
        reader_concurrency_semaphore& _semaphore;
    public:
        explicit expiry_handler(reader_concurrency_semaphore& semaphore)
            : _semaphore(semaphore) {}
        void operator()(entry& e) noexcept;
    };

    struct inactive_read {
        std::unique_ptr<flat_mutation_reader> reader;
        eviction_notify_handler notify_handler;
        std::optional<timer<lowres_clock>> ttl_timer;

        explicit inactive_read(flat_mutation_reader);
        inactive_read(inactive_read&&) = default;
        ~inactive_read();
    };

    using inactive_reads_type = std::map<uint64_t, inactive_read>;

private:
    const resources _initial_resources;
    resources _resources;

    expiring_fifo<entry, expiry_handler, db::timeout_clock> _wait_list;

    sstring _name;
    size_t _max_queue_length = std::numeric_limits<size_t>::max();
    std::function<void()> _prethrow_action;
    uint64_t _next_id = 1;
    inactive_reads_type _inactive_reads;
    stats _stats;
    std::unique_ptr<permit_list> _permit_list;

private:
    inactive_reads_type::iterator evict(inactive_reads_type::iterator it, evict_reason reason);

    bool has_available_units(const resources& r) const;

    bool may_proceed(const resources& r) const;

    future<reader_permit::resource_units> do_wait_admission(reader_permit permit, size_t memory, db::timeout_clock::time_point timeout);

public:
    struct no_limits { };

    reader_concurrency_semaphore(int count,
            ssize_t memory,
            sstring name,
            size_t max_queue_length = std::numeric_limits<size_t>::max(),
            std::function<void()> prethrow_action = nullptr);

    /// Create a semaphore with practically unlimited count and memory.
    ///
    /// And conversely, no queue limit either.
    explicit reader_concurrency_semaphore(no_limits, sstring name = "unlimited reader_concurrency_semaphore");

    ~reader_concurrency_semaphore();

    reader_concurrency_semaphore(const reader_concurrency_semaphore&) = delete;
    reader_concurrency_semaphore& operator=(const reader_concurrency_semaphore&) = delete;

    reader_concurrency_semaphore(reader_concurrency_semaphore&&) = delete;
    reader_concurrency_semaphore& operator=(reader_concurrency_semaphore&&) = delete;

    /// Returns the name of the semaphore
    ///
    /// If the semaphore has no name, "unnamed reader concurrency semaphore" is returned.
    std::string_view name() const {
        return _name.empty() ? "unnamed reader concurrency semaphore" : std::string_view(_name);
    }

    /// Register an inactive read.
    ///
    /// The semaphore will evict this read when there is a shortage of
    /// permits. This might be immediate, during this register call.
    /// Clients can use the returned handle to unregister the read, when it
    /// stops being inactive and hence evictable.
    ///
    /// An inactive read is an object implementing the `inactive_read`
    /// interface.
    /// The semaphore takes ownership of the created object and destroys it if
    /// it is evicted.
    inactive_read_handle register_inactive_read(flat_mutation_reader ir, eviction_notify_handler handler = {});
    inactive_read_handle register_inactive_read(flat_mutation_reader ir, std::chrono::seconds ttl, eviction_notify_handler handler = {});

    /// Unregister the previously registered inactive read.
    ///
    /// If the read was not evicted, the inactive read object, passed in to the
    /// register call, will be returned. Otherwise a nullptr is returned.
    std::unique_ptr<flat_mutation_reader> unregister_inactive_read(inactive_read_handle irh);

    /// Try to evict an inactive read.
    ///
    /// Return true if an inactive read was evicted and false otherwise
    /// (if there was no reader to evict).
    bool try_evict_one_inactive_read();

    void clear_inactive_reads() {
        _inactive_reads.clear();
    }

    const stats& get_stats() const {
        return _stats;
    }

    stats& get_stats() {
        return _stats;
    }

    /// Make a permit
    ///
    /// The permit is associated with a schema, which is the schema of the table
    /// the read is executed against, and the operation name, which should be a
    /// name such that we can identify the operation which created this permit.
    /// Ideally this should be a unique enough name that we not only can identify
    /// the kind of read, but the exact code-path that was taken.
    ///
    /// Some permits cannot be associated with any table, so passing nullptr as
    /// the schema parameter is allowed.
    reader_permit make_permit(const schema* const schema, const char* const op_name);
    reader_permit make_permit(const schema* const schema, sstring&& op_name);

    const resources initial_resources() const {
        return _initial_resources;
    }

    bool is_unlimited() const {
        return _initial_resources == reader_resources{std::numeric_limits<int>::max(), std::numeric_limits<ssize_t>::max()};
    }

    const resources available_resources() const {
        return _resources;
    }

    void consume(resources r) {
        _resources -= r;
    }

    void signal(const resources& r) noexcept;

    size_t waiters() const {
        return _wait_list.size();
    }

    void broken(std::exception_ptr ex);
};
<|endoftext|>


<commit_before>/* QSyncable Project
   Author: Ben Lau
   License: Apache-2.0
   Web: https://github.com/benlau/qsyncable
*/
#include <QHash>
#include <QDebug>
#include "qsdiffrunner.h"

#define MISSING_KEY_WARNING "QSDiffRunner.compare() - Duplicated or missing key."

static QList<QSPatch> merge(const QList<QSPatch> list) {

    if (list.size() <= 1) {
        return list;
    }

    QList<QSPatch> res;

    QSPatch prev;

    for (int i = 0 ; i < list.size() ; i++) {
        QSPatch current = list.at(i);

        if (prev.isNull()) {
            prev = current;
            continue;
        }

        if (prev.canMerge(current)) {
            prev = prev.merge(current);
        } else {
            res << prev;
            prev = current;
        }
    }

    if (!prev.isNull()) {
        res << prev;
    }

    return res;
}

static QVariantMap compareMap(const QVariantMap& prev, const QVariantMap &current) {
    // To make this function faster, it won't track removed fields from prev.
    // Clear a field to null value should set it explicitly.

    QVariantMap diff;

    QMap<QString, QVariant>::const_iterator iter = current.begin();

    while (iter != current.end()) {
        QString key = iter.key();
        if (!prev.contains(key) ||
             prev[key] != iter.value()) {
            diff[key] = iter.value();
        }
        iter++;
    }

    return diff;
}

static QList<QSPatch> compareWithoutKey(const QVariantList& from, const QVariantList& to) {

    QList<QSPatch> patches;

    int max = qMax(from.size(), to.size());

    for (int i = 0 ; i < max ; i++) {
        if (i >= from.size()) {
            patches << QSPatch(QSPatch::Insert, i, i, 1, to[i].toMap());
        } else if (i >= to.size() ) {
            patches << QSPatch(QSPatch::Remove, i, i, 1);
        } else {
            QVariantMap diff = compareMap(from[i].toMap(), to[i].toMap());
            if (diff.size()) {
                patches << QSPatch(QSPatch::Update, i, i, 1, diff);
            }
        }
    }

    return merge(patches);
}

QSDiffRunner::QSDiffRunner()
{

}

QString QSDiffRunner::keyField() const
{
    return m_keyField;
}

void QSDiffRunner::setKeyField(const QString &keyField)
{
    m_keyField = keyField;
}

/*! \fn QList<QSChange> QSDiffRunner::compare(const QVariantList &from, const QVariantList &to)

    Call this function to compare two list, then return a
    list of patches required to transform from a list to other with
    the minimum number of steps. It uses an algorithm with O(n) runtime.
 */

QList<QSPatch> QSDiffRunner::compare(const QVariantList &from, const QVariantList &to)
{
    patches.clear();
    updatePatches.clear();

    if (m_keyField.isEmpty()) {
        return compareWithoutKey(from, to);
    }

    int start = preprocess(from, to);

    if (start >= from.size() &&
        start >= to.size()) {
        // Nothing moved
        return combine();
    }

    QVariantList fromList;

    QHash<QString, int> toHashTable;
    QHash<QString, int> fromHashTable;
    QVariantMap item;
    int offset = 0;

    toHashTable.reserve(to.size() - start + 10);
    fromHashTable.reserve(from.size() - start + 10);

    /* Step 1 - Check Removal */

    for (int i = start ; i < to.size() ; i++) {
        // Build toHashTable
        item = to.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (toHashTable.contains(key)) {
            qWarning() << MISSING_KEY_WARNING;
            return compareWithoutKey(from, to);
        }

        toHashTable[key] = i;
    }

    if (start != 0) {
        //@TODO - remove
        fromList = from.mid(0, start);
    }

    fromList.reserve(from.size());

    // Find removed item and build index table.
    for (int i = start ; i < from.size() ; i++) {
        item = from.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (!toHashTable.contains(key)) {
            patches << QSPatch(QSPatch::Remove,
                           i, i, 1);
        } else {
            fromList << item;

            if (fromHashTable.contains(key)) {
                qWarning() << MISSING_KEY_WARNING;
                return compareWithoutKey(from, to);
            }

            fromHashTable[key] = fromList.count() - 1;
        }
    }

    /* Step 2 - Compare to find move and update */

    if (fromList.size() == 0 && to.size() > 0) {
        // Original items are all removed.
        // A special case. Insert all
        patches <<  QSPatch(QSPatch::Insert, 0, to.size() - 1,to.size(), to);
        return combine();
    }

    for (int i = start ; i < to.size() ; i++) {
        item = to.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (!fromHashTable.contains(key)) {
            offset++;
            patches << QSPatch(QSPatch::Insert, i, i, 1, item);
        } else {
            int prevPos = fromHashTable[key];
            int expectedPos = prevPos + offset;

            if (expectedPos != i) {
                QSPatch change(QSPatch::Move, prevPos, i, 1);
                patches << change;

                offset++;
            }

            QVariantMap before = fromList.at(prevPos).toMap();
            QVariantMap after = item;
            QVariantMap diff = compareMap(before, after);
            if (diff.size() > 0) {
                updatePatches << QSPatch(QSPatch::Update, i, i, 1, diff);
            }
        }
    }

    return combine();
}

bool QSDiffRunner::patch(QSPatchable *patchable, const QList<QSPatch>& patches) const
{
    QVariantMap diff;
    foreach (QSPatch patch, patches) {
        switch (patch.type()) {
        case QSPatch::Remove:
            patchable->remove(patch.from(), patch.count());
            break;
        case QSPatch::Insert:
            patchable->insert(patch.from(), patch.data());
            break;
        case QSPatch::Move:
            patchable->move(patch.from(), patch.to(), patch.count());
            break;
        case QSPatch::Update:
            if (patch.data().size() > 0) {
                diff = patch.data().at(0).toMap();
            }
            patchable->setProperties(patch.from(), diff);
            break;
        default:
            break;
        }

    }

    return true;
}

QList<QSPatch> QSDiffRunner::combine()
{
    QList<QSPatch> tmp = merge(patches);

    if (updatePatches.size() > 0) {
        tmp.append(updatePatches);
    }

    patches.clear();
    updatePatches.clear();

    return tmp;
}

int QSDiffRunner::preprocess(const QVariantList &from, const QVariantList &to)
{
    int index = 0;
    int min = qMin(from.size(), to.size());

    for (index = 0 ; index < min ;index++) {

        QVariantMap f = from[index].toMap();
        QVariantMap t = to[index].toMap();

        if (f[m_keyField] != t[m_keyField]) {
            break;
        }

        QVariantMap diff = compareMap(f,t);
        if (diff.size()) {
            updatePatches << QSPatch::createUpdate(index, diff);
        }
    }

    return index;
}

<commit_msg>QSDiffRunner::compare() - Prevent to add insert patch one by one<commit_after>/* QSyncable Project
   Author: Ben Lau
   License: Apache-2.0
   Web: https://github.com/benlau/qsyncable
*/
#include <QHash>
#include <QDebug>
#include "qsdiffrunner.h"

#define MISSING_KEY_WARNING "QSDiffRunner.compare() - Duplicated or missing key."

static QSPatch createInsertPath(int from, int to, const QVariantList& source ) {
    int count = to - from + 1;
    qDebug() << count;

    return QSPatch(QSPatch::Insert, from, to, count, source.mid(from, count));
}

static QList<QSPatch> merge(const QList<QSPatch> list) {

    if (list.size() <= 1) {
        return list;
    }

    QList<QSPatch> res;

    QSPatch prev;

    for (int i = 0 ; i < list.size() ; i++) {
        QSPatch current = list.at(i);

        if (prev.isNull()) {
            prev = current;
            continue;
        }

        if (prev.canMerge(current)) {
            prev = prev.merge(current);
        } else {
            res << prev;
            prev = current;
        }
    }

    if (!prev.isNull()) {
        res << prev;
    }

    return res;
}

static QVariantMap compareMap(const QVariantMap& prev, const QVariantMap &current) {
    // To make this function faster, it won't track removed fields from prev.
    // Clear a field to null value should set it explicitly.

    QVariantMap diff;

    QMap<QString, QVariant>::const_iterator iter = current.begin();

    while (iter != current.end()) {
        QString key = iter.key();
        if (!prev.contains(key) ||
             prev[key] != iter.value()) {
            diff[key] = iter.value();
        }
        iter++;
    }

    return diff;
}

static QList<QSPatch> compareWithoutKey(const QVariantList& from, const QVariantList& to) {

    QList<QSPatch> patches;

    int max = qMax(from.size(), to.size());

    for (int i = 0 ; i < max ; i++) {
        if (i >= from.size()) {
            patches << QSPatch(QSPatch::Insert, i, i, 1, to[i].toMap());
        } else if (i >= to.size() ) {
            patches << QSPatch(QSPatch::Remove, i, i, 1);
        } else {
            QVariantMap diff = compareMap(from[i].toMap(), to[i].toMap());
            if (diff.size()) {
                patches << QSPatch(QSPatch::Update, i, i, 1, diff);
            }
        }
    }

    return merge(patches);
}

QSDiffRunner::QSDiffRunner()
{

}

QString QSDiffRunner::keyField() const
{
    return m_keyField;
}

void QSDiffRunner::setKeyField(const QString &keyField)
{
    m_keyField = keyField;
}

/*! \fn QList<QSChange> QSDiffRunner::compare(const QVariantList &from, const QVariantList &to)

    Call this function to compare two list, then return a
    list of patches required to transform from a list to other with
    the minimum number of steps. It uses an algorithm with O(n) runtime.
 */

QList<QSPatch> QSDiffRunner::compare(const QVariantList &from, const QVariantList &to)
{
    patches.clear();
    updatePatches.clear();

    if (m_keyField.isEmpty()) {
        return compareWithoutKey(from, to);
    }

    // Compare the list, until it found moved component.
    int start = preprocess(from, to);

    if (start >= from.size() &&
        start >= to.size()) {
        // Nothing moved
        return combine();
    }

    QVariantList fromList;

    QHash<QString, int> toHashTable;
    QHash<QString, int> fromHashTable;
    QVariantMap item;
    int shift = 0;

    toHashTable.reserve(to.size() - start + 10);
    fromHashTable.reserve(from.size() - start + 10);

    /* Step 1 - Check Removal */

    for (int i = start ; i < to.size() ; i++) {
        // Build toHashTable
        item = to.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (toHashTable.contains(key)) {
            qWarning() << MISSING_KEY_WARNING;
            return compareWithoutKey(from, to);
        }

        toHashTable[key] = i;
    }

    fromList.reserve(from.size() - start);

    for (int i = start ; i < from.size() ; i++) {
        // Find removed item and build index table.

        item = from.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (!toHashTable.contains(key)) {
            patches << QSPatch(QSPatch::Remove,
                           i, i, 1);
        } else {
            fromList << item;

            if (fromHashTable.contains(key)) {
                qWarning() << MISSING_KEY_WARNING;
                return compareWithoutKey(from, to);
            }

            fromHashTable[key] = fromList.count() - 1;
        }
    }

    /* Step 2 - Compare to find move and update */

    int insertStart = -1;

    for (int i = start ; i < to.size() ; i++) {
        item = to.at(i).toMap();
        QString key = item[m_keyField].toString();

        if (!fromHashTable.contains(key)) {
            shift++;

            if (insertStart < 0) {
                insertStart = i;
            }

        } else {

            if (insertStart >=0 ) {
                patches << createInsertPath(insertStart, i - 1, to);
                insertStart = -1;
            }

            int realPos = fromHashTable[key]; // Real position in fromList
            int expectedPos = realPos + start; // Expected position in from
            int shiftedPos = realPos + shift + start; // Expected position + shift;

            if (shiftedPos != i) {
                QSPatch change(QSPatch::Move, expectedPos, i, 1);
                patches << change;
                shift++;
            }

            QVariantMap before = fromList.at(realPos).toMap();
            QVariantMap after = item;
            QVariantMap diff = compareMap(before, after);
            if (diff.size() > 0) {
                updatePatches << QSPatch(QSPatch::Update, i, i, 1, diff);
            }
        }
    }

    if (insertStart >=0) {
        patches << createInsertPath(insertStart, to.size()  - 1, to);
    }

    return combine();
}

bool QSDiffRunner::patch(QSPatchable *patchable, const QList<QSPatch>& patches) const
{
    QVariantMap diff;
    foreach (QSPatch patch, patches) {
        switch (patch.type()) {
        case QSPatch::Remove:
            patchable->remove(patch.from(), patch.count());
            break;
        case QSPatch::Insert:
            patchable->insert(patch.from(), patch.data());
            break;
        case QSPatch::Move:
            patchable->move(patch.from(), patch.to(), patch.count());
            break;
        case QSPatch::Update:
            if (patch.data().size() > 0) {
                diff = patch.data().at(0).toMap();
            }
            patchable->setProperties(patch.from(), diff);
            break;
        default:
            break;
        }

    }

    return true;
}

QList<QSPatch> QSDiffRunner::combine()
{
    QList<QSPatch> tmp = merge(patches);

    if (updatePatches.size() > 0) {
        tmp.append(updatePatches);
    }

    patches.clear();
    updatePatches.clear();

    return tmp;
}

int QSDiffRunner::preprocess(const QVariantList &from, const QVariantList &to)
{
    int index = 0;
    int min = qMin(from.size(), to.size());

    for (index = 0 ; index < min ;index++) {

        QVariantMap f = from[index].toMap();
        QVariantMap t = to[index].toMap();

        if (f[m_keyField] != t[m_keyField]) {
            break;
        }

        QVariantMap diff = compareMap(f,t);
        if (diff.size()) {
            updatePatches << QSPatch::createUpdate(index, diff);
        }
    }

    return index;
}

<|endoftext|>


<commit_before>// g++ -O3 -DNDEBUG -I.. -L /usr/lib64/atlas/ benchBlasGemm.cpp -o benchBlasGemm -lrt -lcblas
// possible options:
//    -DEIGEN_DONT_VECTORIZE
//    -msse2

// #define EIGEN_DEFAULT_TO_ROW_MAJOR
#define _FLOAT

#include <iostream>

#include <Eigen/Core>
#include "BenchTimer.h"

// include the BLAS headers
extern "C" {
#include <cblas.h>
}
#include <string>

#ifdef _FLOAT
typedef float Scalar;
#define CBLAS_GEMM cblas_sgemm
#else
typedef double Scalar;
#define CBLAS_GEMM cblas_dgemm
#endif

typedef Eigen::Matrix<Scalar,Eigen::Dynamic,Eigen::Dynamic> MyMatrix;
void bench_eigengemm(MyMatrix& mc, const MyMatrix& ma, const MyMatrix& mb, int nbloops);
void check_product(int M, int N, int K);
void check_product(void);

int main(int argc, char *argv[])
{
  // disable SSE exceptions
  #ifdef __GNUC__
  {
    int aux;
    asm(
    "stmxcsr   %[aux]           \n\t"
    "orl       $32832, %[aux]   \n\t"
    "ldmxcsr   %[aux]           \n\t"
    : : [aux] "m" (aux));
  }
  #endif

  Eigen::setCpuCacheSizes(262144, 6000000);

  int nbtries=1, nbloops=1, M, N, K;

  if (argc==2)
  {
    if (std::string(argv[1])=="check")
      check_product();
    else
      M = N = K = atoi(argv[1]);
  }
  else if ((argc==3) && (std::string(argv[1])=="auto"))
  {
    M = N = K = atoi(argv[2]);
    nbloops = 1000000000/(M*M*M);
    if (nbloops<1)
      nbloops = 1;
    nbtries = 6;
  }
  else if (argc==4)
  {
    M = N = K = atoi(argv[1]);
    nbloops = atoi(argv[2]);
    nbtries = atoi(argv[3]);
  }
  else if (argc==6)
  {
    M = atoi(argv[1]);
    N = atoi(argv[2]);
    K = atoi(argv[3]);
    nbloops = atoi(argv[4]);
    nbtries = atoi(argv[5]);
  }
  else
  {
    std::cout << "Usage: " << argv[0] << " size  \n";
    std::cout << "Usage: " << argv[0] << " auto size\n";
    std::cout << "Usage: " << argv[0] << " size nbloops nbtries\n";
    std::cout << "Usage: " << argv[0] << " M N K nbloops nbtries\n";
    std::cout << "Usage: " << argv[0] << " check\n";
    std::cout << "Options:\n";
    std::cout << "    size       unique size of the 2 matrices (integer)\n";
    std::cout << "    auto       automatically set the number of repetitions and tries\n";
    std::cout << "    nbloops    number of times the GEMM routines is executed\n";
    std::cout << "    nbtries    number of times the loop is benched (return the best try)\n";
    std::cout << "    M N K      sizes of the matrices: MxN  =  MxK * KxN (integers)\n";
    std::cout << "    check      check eigen product using cblas as a reference\n";
    exit(1);
  }

  double nbmad = double(M) * double(N) * double(K) * double(nbloops);

  if (!(std::string(argv[1])=="auto"))
    std::cout << M << " x " << N << " x " << K << "\n";

  Scalar alpha, beta;
  MyMatrix ma(M,K), mb(K,N), mc(M,N);
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  mc = MyMatrix::Random(M,N);

  Eigen::BenchTimer timer;

  // we simply compute c += a*b, so:
  alpha = 1;
  beta = 1;

  // bench cblas
  // ROWS_A, COLS_B, COLS_A, 1.0,  A, COLS_A, B, COLS_B, 0.0, C, COLS_B);
  if (!(std::string(argv[1])=="auto"))
  {
    timer.reset();
    for (uint k=0 ; k<nbtries ; ++k)
    {
        timer.start();
        for (uint j=0 ; j<nbloops ; ++j)
              #ifdef EIGEN_DEFAULT_TO_ROW_MAJOR
              CBLAS_GEMM(CblasRowMajor, CblasNoTrans, CblasNoTrans, M, N, K, alpha, ma.data(), K, mb.data(), N, beta, mc.data(), N);
              #else
              CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasNoTrans, M, N, K, alpha, ma.data(), M, mb.data(), K, beta, mc.data(), M);
              #endif
        timer.stop();
    }
    if (!(std::string(argv[1])=="auto"))
      std::cout << "cblas: " << timer.value() << " (" << 1e-3*floor(1e-6*nbmad/timer.value()) << " GFlops/s)\n";
    else
        std::cout << M << " : " << timer.value() << " ; " << 1e-3*floor(1e-6*nbmad/timer.value()) << "\n";
  }

  // clear
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  mc = MyMatrix::Random(M,N);

  // eigen
//   if (!(std::string(argv[1])=="auto"))
  {
      timer.reset();
      for (uint k=0 ; k<nbtries ; ++k)
      {
          timer.start();
          bench_eigengemm(mc, ma, mb, nbloops);
          timer.stop();
      }
      if (!(std::string(argv[1])=="auto"))
        std::cout << "eigen : " << timer.value() << " (" << 1e-3*floor(1e-6*nbmad/timer.value()) << " GFlops/s)\n";
      else
        std::cout << M << " : " << timer.value() << " ; " << 1e-3*floor(1e-6*nbmad/timer.value()) << "\n";
  }

  std::cout << "l1: " << Eigen::l1CacheSize() << std::endl;
  std::cout << "l2: " << Eigen::l2CacheSize() << std::endl;

  return 0;
}

using namespace Eigen;

void bench_eigengemm(MyMatrix& mc, const MyMatrix& ma, const MyMatrix& mb, int nbloops)
{
  for (uint j=0 ; j<nbloops ; ++j)
      mc.noalias() += ma * mb;
}

#define MYVERIFY(A,M) if (!(A)) { \
    std::cout << "FAIL: " << M << "\n"; \
  }
void check_product(int M, int N, int K)
{
  MyMatrix ma(M,K), mb(K,N), mc(M,N), maT(K,M), mbT(N,K), meigen(M,N), mref(M,N);
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  maT = ma.transpose();
  mbT = mb.transpose();
  mc = MyMatrix::Random(M,N);

  MyMatrix::Scalar eps = 1e-4;

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasNoTrans, M, N, K, 1, ma.data(), M, mb.data(), K, 1, mref.data(), M);
  meigen += ma * mb;
  MYVERIFY(meigen.isApprox(mref, eps),". * .");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasTrans, CblasNoTrans, M, N, K, 1, maT.data(), K, mb.data(), K, 1, mref.data(), M);
  meigen += maT.transpose() * mb;
  MYVERIFY(meigen.isApprox(mref, eps),"T * .");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasTrans, CblasTrans, M, N, K, 1, maT.data(), K, mbT.data(), N, 1, mref.data(), M);
  meigen += (maT.transpose()) * (mbT.transpose());
  MYVERIFY(meigen.isApprox(mref, eps),"T * T");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasTrans, M, N, K, 1, ma.data(), M, mbT.data(), N, 1, mref.data(), M);
  meigen += ma * mbT.transpose();
  MYVERIFY(meigen.isApprox(mref, eps),". * T");
}

void check_product(void)
{
  int M, N, K;
  for (uint i=0; i<1000; ++i)
  {
    M = ei_random<int>(1,64);
    N = ei_random<int>(1,768);
    K = ei_random<int>(1,768);
    M = (0 + M) * 1;
    std::cout << M << " x " << N << " x " << K << "\n";
    check_product(M, N, K);
  }
}

<commit_msg>didn't want to commit that bench change.<commit_after>// g++ -O3 -DNDEBUG -I.. -L /usr/lib64/atlas/ benchBlasGemm.cpp -o benchBlasGemm -lrt -lcblas
// possible options:
//    -DEIGEN_DONT_VECTORIZE
//    -msse2

// #define EIGEN_DEFAULT_TO_ROW_MAJOR
#define _FLOAT

#include <iostream>

#include <Eigen/Core>
#include "BenchTimer.h"

// include the BLAS headers
extern "C" {
#include <cblas.h>
}
#include <string>

#ifdef _FLOAT
typedef float Scalar;
#define CBLAS_GEMM cblas_sgemm
#else
typedef double Scalar;
#define CBLAS_GEMM cblas_dgemm
#endif

typedef Eigen::Matrix<Scalar,Eigen::Dynamic,Eigen::Dynamic> MyMatrix;
void bench_eigengemm(MyMatrix& mc, const MyMatrix& ma, const MyMatrix& mb, int nbloops);
void check_product(int M, int N, int K);
void check_product(void);

int main(int argc, char *argv[])
{
  // disable SSE exceptions
  #ifdef __GNUC__
  {
    int aux;
    asm(
    "stmxcsr   %[aux]           \n\t"
    "orl       $32832, %[aux]   \n\t"
    "ldmxcsr   %[aux]           \n\t"
    : : [aux] "m" (aux));
  }
  #endif

  int nbtries=1, nbloops=1, M, N, K;

  if (argc==2)
  {
    if (std::string(argv[1])=="check")
      check_product();
    else
      M = N = K = atoi(argv[1]);
  }
  else if ((argc==3) && (std::string(argv[1])=="auto"))
  {
    M = N = K = atoi(argv[2]);
    nbloops = 1000000000/(M*M*M);
    if (nbloops<1)
      nbloops = 1;
    nbtries = 6;
  }
  else if (argc==4)
  {
    M = N = K = atoi(argv[1]);
    nbloops = atoi(argv[2]);
    nbtries = atoi(argv[3]);
  }
  else if (argc==6)
  {
    M = atoi(argv[1]);
    N = atoi(argv[2]);
    K = atoi(argv[3]);
    nbloops = atoi(argv[4]);
    nbtries = atoi(argv[5]);
  }
  else
  {
    std::cout << "Usage: " << argv[0] << " size  \n";
    std::cout << "Usage: " << argv[0] << " auto size\n";
    std::cout << "Usage: " << argv[0] << " size nbloops nbtries\n";
    std::cout << "Usage: " << argv[0] << " M N K nbloops nbtries\n";
    std::cout << "Usage: " << argv[0] << " check\n";
    std::cout << "Options:\n";
    std::cout << "    size       unique size of the 2 matrices (integer)\n";
    std::cout << "    auto       automatically set the number of repetitions and tries\n";
    std::cout << "    nbloops    number of times the GEMM routines is executed\n";
    std::cout << "    nbtries    number of times the loop is benched (return the best try)\n";
    std::cout << "    M N K      sizes of the matrices: MxN  =  MxK * KxN (integers)\n";
    std::cout << "    check      check eigen product using cblas as a reference\n";
    exit(1);
  }

  double nbmad = double(M) * double(N) * double(K) * double(nbloops);

  if (!(std::string(argv[1])=="auto"))
    std::cout << M << " x " << N << " x " << K << "\n";

  Scalar alpha, beta;
  MyMatrix ma(M,K), mb(K,N), mc(M,N);
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  mc = MyMatrix::Random(M,N);

  Eigen::BenchTimer timer;

  // we simply compute c += a*b, so:
  alpha = 1;
  beta = 1;

  // bench cblas
  // ROWS_A, COLS_B, COLS_A, 1.0,  A, COLS_A, B, COLS_B, 0.0, C, COLS_B);
  if (!(std::string(argv[1])=="auto"))
  {
    timer.reset();
    for (uint k=0 ; k<nbtries ; ++k)
    {
        timer.start();
        for (uint j=0 ; j<nbloops ; ++j)
              #ifdef EIGEN_DEFAULT_TO_ROW_MAJOR
              CBLAS_GEMM(CblasRowMajor, CblasNoTrans, CblasNoTrans, M, N, K, alpha, ma.data(), K, mb.data(), N, beta, mc.data(), N);
              #else
              CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasNoTrans, M, N, K, alpha, ma.data(), M, mb.data(), K, beta, mc.data(), M);
              #endif
        timer.stop();
    }
    if (!(std::string(argv[1])=="auto"))
      std::cout << "cblas: " << timer.value() << " (" << 1e-3*floor(1e-6*nbmad/timer.value()) << " GFlops/s)\n";
    else
        std::cout << M << " : " << timer.value() << " ; " << 1e-3*floor(1e-6*nbmad/timer.value()) << "\n";
  }

  // clear
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  mc = MyMatrix::Random(M,N);

  // eigen
//   if (!(std::string(argv[1])=="auto"))
  {
      timer.reset();
      for (uint k=0 ; k<nbtries ; ++k)
      {
          timer.start();
          bench_eigengemm(mc, ma, mb, nbloops);
          timer.stop();
      }
      if (!(std::string(argv[1])=="auto"))
        std::cout << "eigen : " << timer.value() << " (" << 1e-3*floor(1e-6*nbmad/timer.value()) << " GFlops/s)\n";
      else
        std::cout << M << " : " << timer.value() << " ; " << 1e-3*floor(1e-6*nbmad/timer.value()) << "\n";
  }

  std::cout << "l1: " << Eigen::l1CacheSize() << std::endl;
  std::cout << "l2: " << Eigen::l2CacheSize() << std::endl;

  return 0;
}

using namespace Eigen;

void bench_eigengemm(MyMatrix& mc, const MyMatrix& ma, const MyMatrix& mb, int nbloops)
{
  for (uint j=0 ; j<nbloops ; ++j)
      mc.noalias() += ma * mb;
}

#define MYVERIFY(A,M) if (!(A)) { \
    std::cout << "FAIL: " << M << "\n"; \
  }
void check_product(int M, int N, int K)
{
  MyMatrix ma(M,K), mb(K,N), mc(M,N), maT(K,M), mbT(N,K), meigen(M,N), mref(M,N);
  ma = MyMatrix::Random(M,K);
  mb = MyMatrix::Random(K,N);
  maT = ma.transpose();
  mbT = mb.transpose();
  mc = MyMatrix::Random(M,N);

  MyMatrix::Scalar eps = 1e-4;

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasNoTrans, M, N, K, 1, ma.data(), M, mb.data(), K, 1, mref.data(), M);
  meigen += ma * mb;
  MYVERIFY(meigen.isApprox(mref, eps),". * .");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasTrans, CblasNoTrans, M, N, K, 1, maT.data(), K, mb.data(), K, 1, mref.data(), M);
  meigen += maT.transpose() * mb;
  MYVERIFY(meigen.isApprox(mref, eps),"T * .");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasTrans, CblasTrans, M, N, K, 1, maT.data(), K, mbT.data(), N, 1, mref.data(), M);
  meigen += (maT.transpose()) * (mbT.transpose());
  MYVERIFY(meigen.isApprox(mref, eps),"T * T");

  meigen = mref = mc;
  CBLAS_GEMM(CblasColMajor, CblasNoTrans, CblasTrans, M, N, K, 1, ma.data(), M, mbT.data(), N, 1, mref.data(), M);
  meigen += ma * mbT.transpose();
  MYVERIFY(meigen.isApprox(mref, eps),". * T");
}

void check_product(void)
{
  int M, N, K;
  for (uint i=0; i<1000; ++i)
  {
    M = ei_random<int>(1,64);
    N = ei_random<int>(1,768);
    K = ei_random<int>(1,768);
    M = (0 + M) * 1;
    std::cout << M << " x " << N << " x " << K << "\n";
    check_product(M, N, K);
  }
}

<|endoftext|>


<commit_before>#include <cassert>
#include <iostream>
#include <chrono>
#include "gemm-bitserial.h"

using namespace std;

void benchmark_unrolledpopcount(size_t numBits, float secs) {
  string bench_name = "popcount";
  double opcount = numBits;
  uint8_t * rnd_vec = new uint8_t[numBits/8];
  uint64_t * ppcnt = (uint64_t *) rnd_vec;
  uint64_t res_pcnt = 0;
  cout << "======================================================================" << endl;
  cout << numBits << "-bit popcount using __builtin_popcntll, for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res_pcnt = 0;
    for(unsigned int r = 0; r < numBits/64; r+=4) {
      res_pcnt += __builtin_popcountll(ppcnt[r]);
      res_pcnt += __builtin_popcountll(ppcnt[r+1]);
      res_pcnt += __builtin_popcountll(ppcnt[r+2]);
      res_pcnt += __builtin_popcountll(ppcnt[r+3]);
    }
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Returned result: " << res_pcnt << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
  cout << (numBits/8) / (nscount) << " GB/s" << endl;
  delete [] rnd_vec;
}

void benchmark_andcardinality(size_t numBits, float secs) {
  string bench_name = "and_cardinality";
  double opcount = 2*numBits;
  MyBitVector a(numBits), b(numBits);
  cout << "======================================================================" << endl;
  cout << numBits << "-bit " << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  uint64_t res = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = a.and_cardinality(b);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Returned result: " << res << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
}

void benchmark_matvec(size_t rows, size_t cols, size_t mbits, size_t vbits, float secs) {
  string bench_name = std::to_string(rows)+"x"+std::to_string(cols)+" "+std::to_string(mbits)+"-bit matrix "+std::to_string(vbits) +"-bits vector";
  double opcount = 2*rows*cols;
  uint8_t * rnd_mat = new uint8_t[rows*cols];
  uint8_t * rnd_vec = new uint8_t[cols];
  BitSerialVector bsv = toBitSerialVector(rnd_vec, cols, vbits);
  BitSerialMatrix bsm = toBitSerialMatrix(rnd_mat, rows, cols, mbits);
  AccumulateVector res;

  cout << "======================================================================" << endl;
  cout << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = bitSerialMatrixVector(bsm, bsv);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Size of returned result: " << res.size() << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
  delete [] rnd_mat;
  delete [] rnd_vec;
}

void benchmark_matmat(size_t rowsA, size_t colsA, size_t colsB, size_t abits, size_t bbits, float secs) {
  string bench_name = std::to_string(rowsA)+"x"+std::to_string(colsA)+" "+std::to_string(abits)+"-bit matrix times ";
  bench_name += std::to_string(colsA)+"x"+std::to_string(colsB)+" "+std::to_string(bbits)+"-bit matrix ";
  double opcount = 2*rowsA*colsA*colsB;
  uint8_t * rnd_matA = new uint8_t[rowsA*colsA];
  uint8_t * rnd_matB = new uint8_t[colsA*colsB];
  BitSerialMatrix bsmA = toBitSerialMatrix(rnd_matA, rowsA, colsA, abits);
  // bitSerialMatrixMatrix wants matrix B to be transposed, so rows and cols switched
  BitSerialMatrix bsmB = toBitSerialMatrix(rnd_matB, colsB, colsA, bbits);
  AccumulateMatrix res;

  cout << "======================================================================" << endl;
  cout << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = bitSerialMatrixMatrix(bsmA, bsmB);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Size of returned result: " << res.size() << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
}

int main(int argc, char const *argv[]) {
  vector<size_t> bits {1, 2, 3, 4};
  vector<size_t> mdims {256, 1024, 8192};
  vector<size_t> dims {256, 1024, 2048, 4096, 8192, 16384, 32768, 65536};

  for(auto &d: dims) {
    benchmark_andcardinality(d, 5);
    benchmark_unrolledpopcount(d, 5);
  }
  for(auto &bm: bits) {
    for(auto &bv: bits) {
      for(auto &r: mdims) {
        for(auto &c: mdims) {
          benchmark_matvec(r, c, bm, bv, 5);
        }
      }
    }
  }
  for(auto &ba: bits) {
    for(auto &bb: bits) {
      for(auto &ra: mdims) {
        for(auto &ca: mdims) {
          for(auto &cb: mdims) {
            benchmark_matmat(ra, ca, cb, ba, bb, 5);
          }
        }
      }
    }
  }
  return 0;
}
<commit_msg>add CaffeNet GEMMs to benchmark<commit_after>#include <cassert>
#include <iostream>
#include <chrono>
#include "gemm-bitserial.h"

using namespace std;

void benchmark_unrolledpopcount(size_t numBits, float secs) {
  string bench_name = "popcount";
  double opcount = numBits;
  uint8_t * rnd_vec = new uint8_t[numBits/8];
  uint64_t * ppcnt = (uint64_t *) rnd_vec;
  uint64_t res_pcnt = 0;
  cout << "======================================================================" << endl;
  cout << numBits << "-bit popcount using __builtin_popcntll, for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res_pcnt = 0;
    for(unsigned int r = 0; r < numBits/64; r+=4) {
      res_pcnt += __builtin_popcountll(ppcnt[r]);
      res_pcnt += __builtin_popcountll(ppcnt[r+1]);
      res_pcnt += __builtin_popcountll(ppcnt[r+2]);
      res_pcnt += __builtin_popcountll(ppcnt[r+3]);
    }
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Returned result: " << res_pcnt << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
  cout << (numBits/8) / (nscount) << " GB/s" << endl;
  delete [] rnd_vec;
}

void benchmark_andcardinality(size_t numBits, float secs) {
  string bench_name = "and_cardinality";
  double opcount = 2*numBits;
  MyBitVector a(numBits), b(numBits);
  cout << "======================================================================" << endl;
  cout << numBits << "-bit " << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  uint64_t res = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = a.and_cardinality(b);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Returned result: " << res << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
}

void benchmark_matvec(size_t rows, size_t cols, size_t mbits, size_t vbits, float secs) {
  string bench_name = std::to_string(rows)+"x"+std::to_string(cols)+" "+std::to_string(mbits)+"-bit matrix "+std::to_string(vbits) +"-bits vector";
  double opcount = 2*rows*cols;
  uint8_t * rnd_mat = new uint8_t[rows*cols];
  uint8_t * rnd_vec = new uint8_t[cols];
  BitSerialVector bsv = toBitSerialVector(rnd_vec, cols, vbits);
  BitSerialMatrix bsm = toBitSerialMatrix(rnd_mat, rows, cols, mbits);
  AccumulateVector res;

  cout << "======================================================================" << endl;
  cout << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = bitSerialMatrixVector(bsm, bsv);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Size of returned result: " << res.size() << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
  delete [] rnd_mat;
  delete [] rnd_vec;
}

void benchmark_matmat(size_t rowsA, size_t colsA, size_t colsB, size_t abits, size_t bbits, float secs) {
  string bench_name = std::to_string(rowsA)+"x"+std::to_string(colsA)+" "+std::to_string(abits)+"-bit matrix times ";
  bench_name += std::to_string(colsA)+"x"+std::to_string(colsB)+" "+std::to_string(bbits)+"-bit matrix ";
  double opcount = 2*rowsA*colsA*colsB;
  uint8_t * rnd_matA = new uint8_t[rowsA*colsA];
  uint8_t * rnd_matB = new uint8_t[colsA*colsB];
  BitSerialMatrix bsmA = toBitSerialMatrix(rnd_matA, rowsA, colsA, abits);
  // bitSerialMatrixMatrix wants matrix B to be transposed, so rows and cols switched
  BitSerialMatrix bsmB = toBitSerialMatrix(rnd_matB, colsB, colsA, bbits);
  AccumulateMatrix res;

  cout << "======================================================================" << endl;
  cout << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    res = bitSerialMatrixMatrix(bsmA, bsmB);
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
  }
  cout << "Size of returned result: " << res.size() << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
}

void benchmark_caffenet(float secs) {
  string bench_name = "CaffeNet matrices";
  const int caffenet_gemm_sizes[] = {
      96, 363, 3025,
      256, 2400, 729,
      384, 2304, 169,
      384, 3456, 169,
      256, 3456, 169,
      4096, 9216, 1,
      4096, 4096, 1,
      1000, 4096, 1
  };
  double opcount = 0;
  size_t wbits = 2;
  size_t abits = 2;
  const std::size_t num_caffenet_gemms =
      sizeof(caffenet_gemm_sizes) / (3 * sizeof(caffenet_gemm_sizes[0]));
  // prepare workload
  std::vector<BitSerialMatrix> caffenet_gemms_A, caffenet_gemms_B;
  for (std::size_t i = 0; i < num_caffenet_gemms; i++) {
    size_t rows = caffenet_gemm_sizes[3 * i + 0];
    size_t depth = caffenet_gemm_sizes[3 * i + 1];
    size_t cols = caffenet_gemm_sizes[3 * i + 2];
    opcount += 2*rows*depth*cols;
    uint8_t * rnd_matA = new uint8_t[rows*depth];
    uint8_t * rnd_matB = new uint8_t[depth*cols];
    generateRandomVector(wbits, rows*depth, rnd_matA);
    generateRandomVector(abits, depth*cols, rnd_matB);
    caffenet_gemms_A.push_back(toBitSerialMatrix(rnd_matA, rows, depth, wbits));
    caffenet_gemms_B.push_back(toBitSerialMatrix(rnd_matB, cols, depth, abits));
    delete [] rnd_matA;
    delete [] rnd_matB;
  }
  AccumulateMatrix res;

  cout << "======================================================================" << endl;
  cout << bench_name << " for " << secs << " seconds..." << endl;
  unsigned int reps = 0;
  auto start = chrono::high_resolution_clock::now();
  auto end = chrono::high_resolution_clock::now();
  while (chrono::duration_cast<std::chrono::seconds>(end-start).count() < secs) {
    // =============== start of benchmark kernel =============
    for(size_t i = 0; i < num_caffenet_gemms; i++) {
      res = bitSerialMatrixMatrix(caffenet_gemms_A[i], caffenet_gemms_B[i]);
    }
    // =============== end of benchmark kernel ================
    reps += 1;
    end = chrono::high_resolution_clock::now();
    // ignore the first iteration, it's just for warmup
    if(reps == 1) {
      start = end;
    }
  }
  cout << "Completed " << reps << " iterations" << endl;
  cout << "Size of returned result: " << res.size() << endl;
  double nscount = chrono::duration_cast<std::chrono::nanoseconds>(end-start).count() / (double)reps;
  double perf = opcount / (nscount); // billion bit operations per second
  cout << "Time for a single " << bench_name << ": " << nscount << " nanoseconds" << endl;
  cout << "Performance for " << bench_name << ": " << perf << " GOPS per second" << endl;
}

int main(int argc, char const *argv[]) {
  vector<size_t> bits {1, 2, 3, 4};
  vector<size_t> mdims {256, 1024, 8192};
  vector<size_t> dims {256, 1024, 2048, 4096, 8192, 16384, 32768, 65536};

  benchmark_caffenet(20);

  for(auto &d: dims) {
    benchmark_andcardinality(d, 5);
    benchmark_unrolledpopcount(d, 5);
  }

  for(auto &bm: bits) {
    for(auto &bv: bits) {
      for(auto &r: mdims) {
        for(auto &c: mdims) {
          benchmark_matvec(r, c, bm, bv, 5);
        }
      }
    }
  }
  for(auto &ba: bits) {
    for(auto &bb: bits) {
      for(auto &ra: mdims) {
        for(auto &ca: mdims) {
          for(auto &cb: mdims) {
            benchmark_matmat(ra, ca, cb, ba, bb, 5);
          }
        }
      }
    }
  }
  return 0;
}
<|endoftext|>


<commit_before>/*
 * main.cpp
 *  sample codes for getopt.hpp
 *
 *  compile option
 *      - debug
 *          - g++ -Wall --pedantic main.cpp
 *          - cl /EHsc /W4 main.cpp
 *      - optimized
 *          - g++ -Wall --pedantic -O2 -DNDEBUG main.cpp
 *          - cl /EHsc /W4 /O2 /DNDEBUG main.cpp
 *
 *  written by janus_wel<janus.wel.3@gmail.com>
 *  This source code is in public domain, and has NO WARRANTY.
 * */

#include "../../header/dlogger.hpp"
#include "../../header/event.hpp"
#include "../../header/getopt.hpp"
#include "../../header/tconv.hpp"

#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

// event definition
// event kinds
enum event_kind_t {
    VERSION,
    HELP,
    SIZE,
    OUTPUT
};
// a template structure of event
template<typename valueT>
struct EventOpt {
    typedef valueT  value_t;
    event_kind_t type;
    value_t val;
};
// partial specialization where valueT = void
template<>
struct EventOpt<void> {
    typedef void  value_t;
    event_kind_t type;
};

// instantiations of event types
typedef EventOpt<void>                              EventVoid;
typedef pattern::event::event_listener<EventVoid>   VoidListener;
typedef pattern::event::event_source<EventVoid>     VoidSource;

typedef EventOpt<unsigned int>                      EventUint;
typedef pattern::event::event_listener<EventUint>   UintListener;
typedef pattern::event::event_source<EventUint>     UintSource;

typedef EventOpt<std::string>                       EventString;
typedef pattern::event::event_listener<EventString> StringListener;
typedef pattern::event::event_source<EventString>   StringSource;

// my getopt definition
class MyGetOpt : public util::getopt::getopt, VoidListener, UintListener, StringListener {
    private:
        // variables to contain option values
        bool mv_version;
        bool mv_help;
        unsigned int mv_size;
        std::string mv_input;
        std::string mv_output;

    public:
        // getter
        bool version(void) { return mv_version; }
        bool help(void) { return mv_help; }
        unsigned int size(void) { return mv_size; }
        const std::string& input(void) { return mv_input; }
        const std::string& output(void) { return mv_output; }

    public:
        // event handlers
        void handle_event(const EventVoid& event) {
            DBGLOG("handle_event(const EventVoid&)");
            switch (event.type) {
                case VERSION:   mv_version = true;
                                break;
                case HELP:      mv_help = true;
                                break;
                default:        throw std::logic_error("unknown event");
            }
        }

        void handle_event(const EventUint& event) {
            DBGLOG("handle_event(const EventUint&)");
            mv_size = event.val;
        }

        void handle_event(const EventString& event) {
            DBGLOG("handle_event(const EventString&)");
            mv_output = event.val;
        }

    private:
        // option definition
        class OptVersion : public option_t, public VoidSource {
            public:
                OptVersion(void) { DBGLOG("OptVersion"); }
                ~OptVersion(void) { DBGLOG("~OptVersion"); }

            protected:
                const char_t* shortname(void) const { return "v"; }
                const char_t* longname(void) const { return "version"; }

                unsigned int handle_params(const parameters_t&) {
                    DBGLOG("OptVersion::handle_params");
                    EventVoid event = { VERSION };
                    dispatch_event(event);
                    return 1;
                }
        } opt_version;

        class OptHelp : public option_t, public VoidSource {
            public:
                OptHelp(void) { DBGLOG("OptHelp"); }
                ~OptHelp(void) { DBGLOG("~OptHelp"); }

            protected:
                const char_t* shortname(void) const { return "h"; }
                const char_t* longname(void) const { return "help"; }

                unsigned int handle_params(const parameters_t&) {
                    DBGLOG("OptHelp::handle_params");
                    EventVoid event = { HELP };
                    dispatch_event(event);
                    return 1;
                }
        } opt_help;

        class OptSize : public option_t, public UintSource {
            public:
                OptSize(void) { DBGLOG("OptSize"); }
                ~OptSize(void) { DBGLOG("~OptSize"); }

            protected:
                const char_t* shortname(void) const { return "s"; }
                const char_t* longname(void) const { return "size"; }

                unsigned int handle_params(const parameters_t& params) {
                    DBGLOG("OptSize::handle_params");
                    parameters_t::const_iterator next = params.current() + 1;
                    if (next != params.end()) {
                        util::string::tconv tconv;
                        EventUint event = { SIZE, tconv.strto<unsigned int>(*next) };
                        dispatch_event(event);
                        return 2;
                    }
                    else {
                        throw std::runtime_error("specify size: " + *(params.current()));
                    }
                }
        } opt_size;

        class OptOutput : public option_t, public StringSource {
            public:
                OptOutput(void) { DBGLOG("OptOutput"); }
                ~OptOutput(void) { DBGLOG("~OptOutput"); }

            protected:
                const char_t* shortname(void) const { return "o"; }
                const char_t* longname(void) const { return "output"; }

                unsigned int handle_params(const parameters_t& params) {
                    DBGLOG("OptOutput::handle_params");
                    parameters_t::const_iterator next = params.current() + 1;
                    if (next != params.end()) {
                        EventString event = { OUTPUT, *next };
                        dispatch_event(event);
                        return 2;
                    }
                    else throw std::runtime_error("specify output: " + *(params.current()));
                }
        } opt_output;

    protected:
        // non-option handler
        unsigned int handle_nonopt(const parameters_t& params) {
            DBGLOG("handle_nonopt");

            const string_t& current = *(params.current());

            if (option_t::has_opt_prefix(current)) {
                throw std::runtime_error("unknown option: " + current);
            }

            if (params.current() + 1 == params.end()) {
                mv_input = current;
                return 1;
            }
            else {
                throw std::runtime_error("too many parameters.");
            }
        }

    public:
        // constructor
        MyGetOpt(void)
            : mv_version(false), mv_help(false), mv_size(4096), mv_input("") {
                DBGLOG("MyGetOpt::MyGetOpt");

                // register event listners
                opt_version.add_event_listener(this);
                opt_help.add_event_listener(this);
                opt_size.add_event_listener(this);
                opt_output.add_event_listener(this);

                // register options
                register_option(opt_version);
                register_option(opt_help);
                register_option(opt_size);
                register_option(opt_output);
            }

        // destructor
        ~MyGetOpt(void) {
            DBGLOG("MyGetOpt::~MyGetOpt");
        }
};

int main(const int argc, const char* const argv[]) {
    try {
        MyGetOpt opt;
        opt.analyze(argc, argv);

        std::cout << std::left << std::boolalpha
            << std::setw(10) << "version"    << opt.version() << "\n"
            << std::setw(10) << "help"       << opt.help() << "\n"
            << std::setw(10) << "size"       << opt.size() << "\n"
            << std::setw(10) << "input"      << opt.input() << "\n"
            << std::setw(10) << "output"     << opt.output() << "\n"
            << std::endl;
    }
    catch (const std::exception& ex) {
        std::cerr << ex.what() << std::endl;
    }

    return 0;
}

<commit_msg>Change to use the class pattern::event::basic_event<commit_after>/*
 * main.cpp
 *  sample codes for getopt.hpp
 *
 *  compile option
 *      - debug
 *          - g++ -Wall --pedantic main.cpp
 *          - cl /EHsc /W4 main.cpp
 *      - optimized
 *          - g++ -Wall --pedantic -O2 -DNDEBUG main.cpp
 *          - cl /EHsc /W4 /O2 /DNDEBUG main.cpp
 *
 *  written by janus_wel<janus.wel.3@gmail.com>
 *  This source code is in public domain, and has NO WARRANTY.
 * */

#include "../../header/dlogger.hpp"
#include "../../header/event.hpp"
#include "../../header/getopt.hpp"
#include "../../header/tconv.hpp"

#include <iomanip>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

// event definition
// event kinds
enum event_kind_t {
    VERSION,
    HELP,
    SIZE,
    OUTPUT
};

// instantiations of event types
typedef pattern::event::basic_event<event_kind_t, void> EventVoid;
typedef pattern::event::event_listener<EventVoid>       EventListenerVoid;
typedef pattern::event::event_source<EventVoid>         EventSourceVoid;

typedef pattern::event::basic_event<event_kind_t, unsigned int> EventUint;
typedef pattern::event::event_listener<EventUint>               EventListenerUint;
typedef pattern::event::event_source<EventUint>                 EventSourceUint;

typedef pattern::event::basic_event<event_kind_t, std::string>  EventStr;
typedef pattern::event::event_listener<EventStr>                EventListenerStr;
typedef pattern::event::event_source<EventStr>                  EventSourceStr;

// my getopt definition
class MyGetOpt : public util::getopt::getopt, EventListenerVoid, EventListenerUint, EventListenerStr {
    private:
        // variables to contain option values
        bool mv_version;
        bool mv_help;
        unsigned int mv_size;
        std::string mv_input;
        std::string mv_output;

    public:
        // getter
        bool version(void) { return mv_version; }
        bool help(void) { return mv_help; }
        unsigned int size(void) { return mv_size; }
        const std::string& input(void) { return mv_input; }
        const std::string& output(void) { return mv_output; }

    public:
        // event handlers
        void handle_event(const EventVoid& event) {
            DBGLOG("handle_event(const EventVoid&)");
            switch (event.kind) {
                case VERSION:   mv_version = true;
                                break;
                case HELP:      mv_help = true;
                                break;
                default:        throw std::logic_error("unknown event");
            }
        }

        void handle_event(const EventUint& event) {
            DBGLOG("handle_event(const EventUint&)");
            mv_size = event.data;
        }

        void handle_event(const EventStr& event) {
            DBGLOG("handle_event(const EventStr&)");
            mv_output = event.data;
        }

    private:
        // option definition
        class OptVersion : public option_t, public EventSourceVoid {
            public:
                OptVersion(void) { DBGLOG("OptVersion"); }
                ~OptVersion(void) { DBGLOG("~OptVersion"); }

            protected:
                const char_t* shortname(void) const { return "v"; }
                const char_t* longname(void) const { return "version"; }

                unsigned int handle_params(const parameters_t&) {
                    DBGLOG("OptVersion::handle_params");
                    EventVoid event = { VERSION };
                    dispatch_event(event);
                    return 1;
                }
        } opt_version;

        class OptHelp : public option_t, public EventSourceVoid {
            public:
                OptHelp(void) { DBGLOG("OptHelp"); }
                ~OptHelp(void) { DBGLOG("~OptHelp"); }

            protected:
                const char_t* shortname(void) const { return "h"; }
                const char_t* longname(void) const { return "help"; }

                unsigned int handle_params(const parameters_t&) {
                    DBGLOG("OptHelp::handle_params");
                    EventVoid event = { HELP };
                    dispatch_event(event);
                    return 1;
                }
        } opt_help;

        class OptSize : public option_t, public EventSourceUint {
            public:
                OptSize(void) { DBGLOG("OptSize"); }
                ~OptSize(void) { DBGLOG("~OptSize"); }

            protected:
                const char_t* shortname(void) const { return "s"; }
                const char_t* longname(void) const { return "size"; }

                unsigned int handle_params(const parameters_t& params) {
                    DBGLOG("OptSize::handle_params");
                    parameters_t::const_iterator next = params.current() + 1;
                    if (next != params.end()) {
                        util::string::tconv tconv;
                        EventUint event = { SIZE, tconv.strto<unsigned int>(*next) };
                        dispatch_event(event);
                        return 2;
                    }
                    else {
                        throw std::runtime_error("specify size: " + *(params.current()));
                    }
                }
        } opt_size;

        class OptOutput : public option_t, public EventSourceStr {
            public:
                OptOutput(void) { DBGLOG("OptOutput"); }
                ~OptOutput(void) { DBGLOG("~OptOutput"); }

            protected:
                const char_t* shortname(void) const { return "o"; }
                const char_t* longname(void) const { return "output"; }

                unsigned int handle_params(const parameters_t& params) {
                    DBGLOG("OptOutput::handle_params");
                    parameters_t::const_iterator next = params.current() + 1;
                    if (next != params.end()) {
                        EventStr event = { OUTPUT, *next };
                        dispatch_event(event);
                        return 2;
                    }
                    else throw std::runtime_error("specify output: " + *(params.current()));
                }
        } opt_output;

    protected:
        // non-option handler
        unsigned int handle_nonopt(const parameters_t& params) {
            DBGLOG("handle_nonopt");

            const string_t& current = *(params.current());

            if (option_t::has_opt_prefix(current)) {
                throw std::runtime_error("unknown option: " + current);
            }

            if (params.current() + 1 == params.end()) {
                mv_input = current;
                return 1;
            }
            else {
                throw std::runtime_error("too many parameters.");
            }
        }

    public:
        // constructor
        MyGetOpt(void)
            : mv_version(false), mv_help(false), mv_size(4096), mv_input("") {
                DBGLOG("MyGetOpt::MyGetOpt");

                // register event listners
                opt_version.add_event_listener(this);
                opt_help.add_event_listener(this);
                opt_size.add_event_listener(this);
                opt_output.add_event_listener(this);

                // register options
                register_option(opt_version);
                register_option(opt_help);
                register_option(opt_size);
                register_option(opt_output);
            }

        // destructor
        ~MyGetOpt(void) {
            DBGLOG("MyGetOpt::~MyGetOpt");
        }
};

int main(const int argc, const char* const argv[]) {
    try {
        MyGetOpt opt;
        opt.analyze(argc, argv);

        std::cout << std::left << std::boolalpha
            << std::setw(10) << "version"    << opt.version() << "\n"
            << std::setw(10) << "help"       << opt.help() << "\n"
            << std::setw(10) << "size"       << opt.size() << "\n"
            << std::setw(10) << "input"      << opt.input() << "\n"
            << std::setw(10) << "output"     << opt.output() << "\n"
            << std::endl;
    }
    catch (const std::exception& ex) {
        std::cerr << ex.what() << std::endl;
    }

    return 0;
}

<|endoftext|>